diff --git a/scripts/commands/epg/grab.js b/scripts/commands/epg/grab.js new file mode 100644 index 00000000..d94e2ed4 --- /dev/null +++ b/scripts/commands/epg/grab.js @@ -0,0 +1,184 @@ +const { program } = require('commander') +const _ = require('lodash') +const { EPGGrabber, generateXMLTV, Channel, Program } = require('epg-grabber') +const { db, logger, date, timer, file, parser, api, zip } = require('../../core') +const path = require('path') +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') + +dayjs.extend(utc) + +program + .requiredOption('-s, --site ', 'Name of the site to parse') + .option('-l, --lang ', 'Filter channels by language (ISO 639-2 code)') + .option('-o, --output ', 'Path to output file') + .option('--gzip', 'Create a compressed version of the guide as well', false) + .parse(process.argv) + +const options = program.opts() +const BASE_DIR = process.env.BASE_DIR || '.' +const CURR_DATE = process.env.CURR_DATE || new Date() + +options.config = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}.config.js`) +options.channels = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}*.channels.xml`) + +let channels = [] +let programs = [] + +async function main() { + logger.info('starting...') + timer.start() + + let config = await loadConfig(options.config) + let queue = await createQueue(options.channels, config) + + await grab(queue, config) + + let outputPath = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`) + + await save(outputPath, channels, programs) + + logger.info(`done in ${timer.format('HH[h] mm[m] ss[s]')}`) +} + +async function loadConfig(configPath) { + logger.info(`loading "${configPath}"...`) + + let config = require(file.resolve(configPath)) + config = _.merge(config, { + debug: options.debug, + delay: options.delay, + request: { + timeout: options.timeout + } + }) + + return config +} + +async function grab(queue, config) { + const grabber = new EPGGrabber(config) + const total = queue.length + + let i = 1 + for (const item of queue) { + let channel = item.channel + let date = item.date + channels.push(item.channel) + await grabber + .grab(channel, date, (data, err) => { + logger.info( + `[${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs + .utc(data.date) + .format('MMM D, YYYY')} (${data.programs.length} programs)` + ) + if (i < total) i++ + + if (err) { + logger.error(` Error: ${err.message}`) + } + }) + .then(results => { + programs = programs.concat(results) + }) + } +} + +async function createQueue(channelsPath, config) { + let queue = {} + + await api.channels.load().catch(console.error) + const files = await file.list(channelsPath).catch(console.error) + const utcDate = date.getUTC(CURR_DATE) + for (const filepath of files) { + logger.info(`loading "${filepath}"...`) + try { + const dir = file.dirname(filepath) + const { channels } = await parser.parseChannels(filepath) + const filename = file.basename(filepath) + const days = config.days || 1 + const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd')) + for (const channel of channels) { + if (!channel.site || !channel.xmltv_id) continue + if (options.lang && channel.lang !== options.lang) continue + const found = api.channels.find({ id: channel.xmltv_id }) + if (found) { + channel.logo = found.logo + } + for (const d of dates) { + const dateString = d.toJSON() + const key = `${channel.site}:${channel.lang}:${channel.xmltv_id}:${dateString}` + if (!queue[key]) { + queue[key] = { + channel, + date: dateString, + config, + error: null + } + } + } + } + } catch (err) { + console.error(err) + continue + } + } + + queue = Object.values(queue) + + return queue +} + +async function save(template, parsedChannels, programs = []) { + const variables = file.templateVariables(template) + + const groups = _.groupBy(parsedChannels, channel => { + let groupId = '' + for (let key in channel) { + if (variables.includes(key)) { + groupId += channel[key] + } + } + + return groupId + }) + + for (let groupId in groups) { + const channels = groups[groupId] + + let output = { + channels, + programs: [], + date: CURR_DATE + } + + for (let program of programs) { + let programLang = program.titles[0].lang + let channel = channels.find(c => c.xmltv_id === program.channel && c.lang === programLang) + if (!channel) continue + + output.programs.push(new Program(program, channel)) + } + + output.channels = _.sortBy(output.channels, 'id') + output.channels = _.uniqBy(output.channels, 'id') + + output.programs = _.sortBy(output.programs, ['channel', 'start']) + output.programs = _.uniqBy(output.programs, p => p.channel + p.start) + + const outputPath = file.templateFormat(template, output.channels[0]) + const xmlFilepath = outputPath + const xmltv = generateXMLTV(output) + await file.create(xmlFilepath, xmltv) + logger.info(`file "${xmlFilepath}" saved`) + + if (options.gzip) { + const gzFilepath = `${outputPath}.gz` + const compressed = await zip.compress(xmltv) + await file.create(gzFilepath, compressed) + logger.info(`file "${gzFilepath}" saved`) + } + } +} + +main()