diff --git a/scripts/core/checker.js b/scripts/core/checker.js new file mode 100644 index 000000000..552110072 --- /dev/null +++ b/scripts/core/checker.js @@ -0,0 +1,19 @@ +const IPTVChecker = require('iptv-checker') + +const checker = {} + +checker.check = async function (item, config) { + const ic = new IPTVChecker(config) + const result = await ic.checkStream({ url: item.url, http: item.http }) + + return { + _id: item._id, + url: item.url, + http: item.http, + error: !result.status.ok ? result.status.reason : null, + streams: result.status.ok ? result.status.metadata.streams : [], + requests: result.status.ok ? result.status.metadata.requests : [] + } +} + +module.exports = checker diff --git a/scripts/core/db.js b/scripts/core/db.js new file mode 100644 index 000000000..27360a839 --- /dev/null +++ b/scripts/core/db.js @@ -0,0 +1,61 @@ +const Database = require('nedb-promises') +const file = require('./file') + +const DB_FILEPATH = process.env.DB_FILEPATH || './scripts/channels.db' + +const nedb = Database.create({ + filename: file.resolve(DB_FILEPATH), + autoload: true, + onload(err) { + if (err) console.error(err) + }, + compareStrings: (a, b) => { + a = a.replace(/\s/g, '_') + b = b.replace(/\s/g, '_') + + return a.localeCompare(b, undefined, { + sensitivity: 'accent', + numeric: true + }) + } +}) + +const db = {} + +db.removeIndex = function (field) { + return nedb.removeIndex(field) +} + +db.addIndex = function (options) { + return nedb.ensureIndex(options) +} + +db.compact = function () { + return nedb.persistence.compactDatafile() +} + +db.reset = function () { + return file.clear(DB_FILEPATH) +} + +db.count = function (query) { + return nedb.count(query) +} + +db.insert = function (doc) { + return nedb.insert(doc) +} + +db.update = function (query, update) { + return nedb.update(query, update) +} + +db.find = function (query) { + return nedb.find(query) +} + +db.remove = function (query, options) { + return nedb.remove(query, options) +} + +module.exports = db diff --git a/scripts/core/file.js b/scripts/core/file.js new file mode 100644 index 000000000..56da4db96 --- /dev/null +++ b/scripts/core/file.js @@ -0,0 +1,67 @@ +const path = require('path') +const glob = require('glob') +const fs = require('mz/fs') + +const file = {} + +file.list = function (pattern) { + return new Promise(resolve => { + glob(pattern, function (err, files) { + resolve(files) + }) + }) +} + +file.getFilename = function (filepath) { + return path.parse(filepath).name +} + +file.createDir = async function (dir) { + if (await file.exists(dir)) return + + return fs.mkdir(dir, { recursive: true }).catch(console.error) +} + +file.exists = function (filepath) { + return fs.exists(path.resolve(filepath)) +} + +file.read = function (filepath) { + return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error) +} + +file.append = function (filepath, data) { + return fs.appendFile(path.resolve(filepath), data).catch(console.error) +} + +file.create = function (filepath, data = '') { + filepath = path.resolve(filepath) + const dir = path.dirname(filepath) + + return file + .createDir(dir) + .then(() => fs.writeFile(filepath, data, { encoding: 'utf8', flag: 'w' })) + .catch(console.error) +} + +file.write = function (filepath, data = '') { + return fs.writeFile(path.resolve(filepath), data).catch(console.error) +} + +file.clear = function (filepath) { + return file.write(filepath, '') +} + +file.resolve = function (filepath) { + return path.resolve(filepath) +} + +file.dirname = function (filepath) { + return path.dirname(filepath) +} + +file.basename = function (filepath) { + return path.basename(filepath) +} + +module.exports = file diff --git a/scripts/core/generator.js b/scripts/core/generator.js new file mode 100644 index 000000000..046ac15fd --- /dev/null +++ b/scripts/core/generator.js @@ -0,0 +1,114 @@ +const { create: createPlaylist } = require('./playlist') +const store = require('./store') +const file = require('./file') +const logger = require('./logger') +const db = require('./db') +const _ = require('lodash') + +const generator = {} + +generator.generate = async function (filepath, query = {}, options = {}) { + options = { + ...{ + format: 'm3u', + saveEmpty: false, + includeNSFW: false, + includeGuides: true, + includeBroken: false, + onLoad: r => r, + uniqBy: item => item.id || _.uniqueId(), + sortBy: null + }, + ...options + } + + query['is_nsfw'] = options.includeNSFW ? { $in: [true, false] } : false + query['is_broken'] = options.includeBroken ? { $in: [true, false] } : false + + let items = await db + .find(query) + .sort({ name: 1, 'status.level': 1, 'resolution.height': -1, url: 1 }) + + items = _.uniqBy(items, 'url') + if (!options.saveEmpty && !items.length) return { filepath, query, options, count: 0 } + if (options.uniqBy) items = _.uniqBy(items, options.uniqBy) + + items = options.onLoad(items) + + if (options.sortBy) items = _.sortBy(items, options.sortBy) + + switch (options.format) { + case 'json': + await saveAsJSON(filepath, items, options) + break + case 'm3u': + default: + await saveAsM3U(filepath, items, options) + break + } + + return { filepath, query, options, count: items.length } +} + +async function saveAsM3U(filepath, items, options) { + const playlist = await createPlaylist(filepath) + + const header = {} + if (options.includeGuides) { + let guides = items.map(item => item.guides) + guides = _.uniq(_.flatten(guides)).sort().join(',') + + header['x-tvg-url'] = guides + } + + await playlist.header(header) + for (const item of items) { + const stream = store.create(item) + await playlist.link( + stream.get('url'), + stream.get('title'), + { + 'tvg-id': stream.get('tvg_id'), + 'tvg-country': stream.get('tvg_country'), + 'tvg-language': stream.get('tvg_language'), + 'tvg-logo': stream.get('tvg_logo'), + // 'tvg-url': stream.get('tvg_url') || undefined, + 'user-agent': stream.get('http.user-agent') || undefined, + 'group-title': stream.get('group_title') + }, + { + 'http-referrer': stream.get('http.referrer') || undefined, + 'http-user-agent': stream.get('http.user-agent') || undefined + } + ) + } +} + +async function saveAsJSON(filepath, items, options) { + const output = items.map(item => { + const stream = store.create(item) + const categories = stream.get('categories').map(c => ({ name: c.name, slug: c.slug })) + const countries = stream.get('countries').map(c => ({ name: c.name, code: c.code })) + + return { + name: stream.get('name'), + logo: stream.get('logo'), + url: stream.get('url'), + categories, + countries, + languages: stream.get('languages'), + tvg: { + id: stream.get('tvg_id'), + name: stream.get('name'), + url: stream.get('tvg_url') + } + } + }) + + await file.create(filepath, JSON.stringify(output)) +} + +generator.saveAsM3U = saveAsM3U +generator.saveAsJSON = saveAsJSON + +module.exports = generator diff --git a/scripts/core/index.js b/scripts/core/index.js new file mode 100644 index 000000000..948ff6c6d --- /dev/null +++ b/scripts/core/index.js @@ -0,0 +1,10 @@ +exports.db = require('./db') +exports.logger = require('./logger') +exports.file = require('./file') +exports.timer = require('./timer') +exports.parser = require('./parser') +exports.checker = require('./checker') +exports.generator = require('./generator') +exports.playlist = require('./playlist') +exports.store = require('./store') +exports.markdown = require('./markdown') diff --git a/scripts/core/logger.js b/scripts/core/logger.js new file mode 100644 index 000000000..a109a050b --- /dev/null +++ b/scripts/core/logger.js @@ -0,0 +1,42 @@ +const { createLogger, format, transports, addColors } = require('winston') +const { combine, timestamp, printf } = format + +const consoleFormat = ({ level, message, timestamp }) => { + if (typeof message === 'object') return JSON.stringify(message) + return message +} + +const config = { + levels: { + error: 0, + warn: 1, + info: 2, + failed: 3, + success: 4, + http: 5, + verbose: 6, + debug: 7, + silly: 8 + }, + colors: { + info: 'white', + success: 'green', + failed: 'red' + } +} + +const t = [ + new transports.Console({ + format: format.combine(format.printf(consoleFormat)) + }) +] + +const logger = createLogger({ + transports: t, + levels: config.levels, + level: 'verbose' +}) + +addColors(config.colors) + +module.exports = logger diff --git a/scripts/core/markdown.js b/scripts/core/markdown.js new file mode 100644 index 000000000..32dc1110e --- /dev/null +++ b/scripts/core/markdown.js @@ -0,0 +1,39 @@ +const markdownInclude = require('markdown-include') +const file = require('./file') + +const markdown = {} + +markdown.createTable = function (data, cols) { + let output = '
${column.name} | ` + } + output += '
---|
${item[prop]} | ` + i++ + } + output += '