Init

scripts/.gitignore

@@ -0,0 +1 @@
+/bot.js

scripts/core/csv.js

@@ -0,0 +1,71 @@
+const csv2json = require('csvtojson')
+const fs = require('mz/fs')
+const {
+	Parser,
+	transforms: { flatten },
+	formatters: { stringQuoteOnlyIfNecessary }
+} = require('json2csv')
+
+const csv2jsonOptions = {
+	checkColumn: true,
+	trim: true,
+	colParser: {
+		countries: listParser,
+		languages: listParser,
+		categories: listParser,
+		broadcast_area: listParser,
+		is_nsfw: boolParser,
+		logo: nullable,
+		subdivision: nullable,
+		city: nullable,
+		network: nullable
+	}
+}
+
+const json2csv = new Parser({
+	transforms: [flattenArray],
+	formatters: {
+		string: stringQuoteOnlyIfNecessary()
+	}
+})
+
+const csv = {}
+
+csv.load = async function (filepath) {
+	return csv2json(csv2jsonOptions).fromFile(filepath)
+}
+
+csv.save = async function (filepath, data) {
+	const string = json2csv.parse(data)
+
+	return fs.writeFile(filepath, string)
+}
+
+csv.saveSync = function (filepath, data) {
+	const string = json2csv.parse(data)
+
+	return fs.writeFileSync(filepath, string)
+}
+
+module.exports = csv
+
+function flattenArray(item) {
+	for (let prop in item) {
+		const value = item[prop]
+		item[prop] = Array.isArray(value) ? value.join(';') : value
+	}
+
+	return item
+}
+
+function listParser(value) {
+	return value.split(';').filter(i => i)
+}
+
+function boolParser(value) {
+	return value === 'true'
+}
+
+function nullable(value) {
+	return value === '' ? null : value
+}

scripts/core/file.js

@@ -0,0 +1,68 @@
+const path = require('path')
+const glob = require('glob')
+const fs = require('mz/fs')
+
+const file = {}
+
+file.list = function (pattern) {
+  return new Promise(resolve => {
+    glob(pattern, function (err, files) {
+      resolve(files)
+    })
+  })
+}
+
+file.getFilename = function (filepath) {
+  return path.parse(filepath).name
+}
+
+file.createDir = async function (dir) {
+  if (await file.exists(dir)) return
+
+  return fs.mkdir(dir, { recursive: true }).catch(console.error)
+}
+
+file.exists = function (filepath) {
+  return fs.exists(path.resolve(filepath))
+}
+
+file.read = function (filepath) {
+  return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
+}
+
+file.append = function (filepath, data) {
+  return fs.appendFile(path.resolve(filepath), data).catch(console.error)
+}
+
+file.create = function (filepath, data = '') {
+  filepath = path.resolve(filepath)
+  const dir = path.dirname(filepath)
+
+  return file
+    .createDir(dir)
+    .then(() => file.write(filepath, data))
+    .catch(console.error)
+}
+
+file.write = function (filepath, data = '') {
+  return fs.writeFile(path.resolve(filepath), data, { encoding: 'utf8' }).catch(console.error)
+}
+
+file.clear = async function (filepath) {
+  if (await file.exists(filepath)) return file.write(filepath, '')
+  return true
+}
+
+file.resolve = function (filepath) {
+  return path.resolve(filepath)
+}
+
+file.dirname = function (filepath) {
+  return path.dirname(filepath)
+}
+
+file.basename = function (filepath) {
+  return path.basename(filepath)
+}
+
+module.exports = file

scripts/core/index.js

@@ -0,0 +1,3 @@
+exports.csv = require('./csv')
+exports.file = require('./file')
+exports.logger = require('./logger')

scripts/core/logger.js

@@ -0,0 +1,13 @@
+const { Signale } = require('signale')
+
+const options = {}
+
+const logger = new Signale(options)
+
+logger.config({
+  displayLabel: false,
+  displayScope: false,
+  displayBadge: false
+})
+
+module.exports = logger

scripts/db/export.js

@@ -0,0 +1,23 @@
+const { csv } = require('../core')
+const path = require('path')
+const glob = require('glob')
+const fs = require('fs')
+
+const DATA_DIR = process.env.DATA_DIR || './data'
+const OUTPUT_DIR = process.env.OUTPUT_DIR || './.gh-pages'
+
+fs.exists(OUTPUT_DIR, function (exists) {
+	if (!exists) {
+		fs.mkdirSync(OUTPUT_DIR)
+	}
+})
+
+glob(`${DATA_DIR}/*.csv`, async function (err, files) {
+	for (const inputFile of files) {
+		const inputFilename = path.parse(inputFile).name
+		const outputFile = `${OUTPUT_DIR}/${inputFilename}.json`
+
+		const json = await csv.load(inputFile)
+		fs.writeFileSync(path.resolve(outputFile), JSON.stringify(json))
+	}
+})

scripts/db/schemes/categories.js

@@ -0,0 +1,10 @@
+const Joi = require('joi')
+
+module.exports = {
+	id: Joi.string()
+		.regex(/^[a-z]+$/)
+		.required(),
+	name: Joi.string()
+		.regex(/^[A-Z]+$/i)
+		.required()
+}

scripts/db/schemes/channels.js

@@ -0,0 +1,27 @@
+const Joi = require('joi')
+
+module.exports = {
+	id: Joi.string()
+		.regex(/^[A-Za-z0-9]+\.[a-z]{2}$/)
+		.required(),
+	name: Joi.string()
+		.regex(/^[\sa-zA-Z\u00C0-\u00FF0-9-!:&.+'/»#%°$@?()]+$/)
+		.required(),
+	network: Joi.string().allow(null),
+	country: Joi.string()
+		.regex(/^[A-Z]{2}$/)
+		.required(),
+	subdivision: Joi.string()
+		.regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)
+		.allow(null),
+	city: Joi.string().allow(null),
+	broadcast_area: Joi.array().items(
+		Joi.string().regex(/^(s\/[A-Z]{2}-[A-Z0-9]{1,3}|c\/[A-Z]{2}|r\/[A-Z0-9]{3,7})$/)
+	),
+	languages: Joi.array()
+		.items(Joi.string().regex(/^[a-z]{3}$/))
+		.allow(''),
+	categories: Joi.array().items(Joi.string().regex(/^[a-z]+$/)),
+	is_nsfw: Joi.boolean().required(),
+	logo: Joi.string().uri().allow(null)
+}

scripts/db/schemes/countries.js

@@ -0,0 +1,16 @@
+const Joi = require('joi')
+
+module.exports = {
+	name: Joi.string()
+		.regex(/^[\sA-Z\u00C0-\u00FF().-]+$/i)
+		.required(),
+	code: Joi.string()
+		.regex(/^[A-Z]{2}$/)
+		.required(),
+	lang: Joi.string()
+		.regex(/^[a-z]{3}$/)
+		.required(),
+	flag: Joi.string()
+		.regex(/^[\uD83C][\uDDE6-\uDDFF][\uD83C][\uDDE6-\uDDFF]$/)
+		.required()
+}

scripts/db/schemes/index.js

@@ -0,0 +1,6 @@
+exports.channels = require('./channels')
+exports.categories = require('./categories')
+exports.countries = require('./countries')
+exports.languages = require('./languages')
+exports.regions = require('./regions')
+exports.subdivisions = require('./subdivisions')

scripts/db/schemes/languages.js

@@ -0,0 +1,8 @@
+const Joi = require('joi')
+
+module.exports = {
+	code: Joi.string()
+		.regex(/^[a-z]{3}$/)
+		.required(),
+	name: Joi.string().required()
+}

scripts/db/schemes/regions.js

@@ -0,0 +1,15 @@
+const Joi = require('joi')
+
+module.exports = {
+	name: Joi.string()
+		.regex(/^[\sA-Z\u00C0-\u00FF().,-]+$/i)
+		.required(),
+	code: Joi.string()
+		.regex(/^[A-Z]{3,7}$/)
+		.required(),
+	countries: Joi.array().items(
+		Joi.string()
+			.regex(/^[A-Z]{2}$/)
+			.allow('')
+	)
+}

scripts/db/schemes/subdivisions.js

@@ -0,0 +1,11 @@
+const Joi = require('joi')
+
+module.exports = {
+	country: Joi.string()
+		.regex(/^[A-Z]{2}$/)
+		.required(),
+	name: Joi.string().required(),
+	code: Joi.string()
+		.regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)
+		.required()
+}

scripts/db/validate.js

@@ -0,0 +1,84 @@
+const { logger, file, csv } = require('../core')
+const { program } = require('commander')
+const schemes = require('./schemes')
+const chalk = require('chalk')
+const Joi = require('joi')
+
+program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
+
+async function main() {
+	let errors = []
+	const files = program.args.length
+		? program.args
+		: [
+				'data/categories.csv',
+				'data/channels.csv',
+				'data/countries.csv',
+				'data/languages.csv',
+				'data/regions.csv',
+				'data/subdivisions.csv'
+		  ]
+	for (const filepath of files) {
+		if (!filepath.endsWith('.csv')) continue
+		const data = await csv.load(filepath)
+
+		const filename = file.getFilename(filepath)
+
+		if (!schemes[filename]) {
+			logger.error(chalk.red(`\nERR: "${filename}" scheme is missing`))
+			process.exit(1)
+		}
+
+		let fileErrors = []
+		if (filename === 'channels') {
+			fileErrors = fileErrors.concat(findDuplicatesById(data))
+		}
+
+		const schema = Joi.object(schemes[filename])
+		data.forEach((row, i) => {
+			const { error } = schema.validate(row, { abortEarly: false })
+			if (error) {
+				error.details.forEach(detail => {
+					fileErrors.push({ line: i + 2, message: detail.message })
+				})
+			}
+		})
+
+		if (fileErrors.length) {
+			logger.info(`\n${chalk.underline(filepath)}`)
+			fileErrors.forEach(err => {
+				const position = err.line.toString().padEnd(6, ' ')
+				logger.error(` ${chalk.gray(position)} ${err.message}`)
+			})
+			errors = errors.concat(fileErrors)
+		}
+	}
+
+	if (errors.length) {
+		logger.error(chalk.red(`\n${errors.length} error(s)`))
+		process.exit(1)
+	}
+}
+
+main()
+
+function findDuplicatesById(data) {
+	data = data.map(i => {
+		i.id = i.id.toLowerCase()
+		return i
+	})
+
+	const errors = []
+	const schema = Joi.array().unique((a, b) => a.id === b.id)
+	const { error } = schema.validate(data, { abortEarly: false })
+	if (error) {
+		error.details.forEach(detail => {
+			errors.push({
+				line: detail.context.pos + 2,
+				message: `Entry with the id "${detail.context.value.id}" already exists`
+			})
+		})
+	}
+
+	return errors
+}