From 3a4f59721a274c1188038ea17e5824fe73842ee4 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 19 Jul 2023 22:31:48 +0300 Subject: [PATCH] Update grab.js --- scripts/commands/epg/grab.js | 98 ++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/scripts/commands/epg/grab.js b/scripts/commands/epg/grab.js index d94e2ed4..b52f35b9 100644 --- a/scripts/commands/epg/grab.js +++ b/scripts/commands/epg/grab.js @@ -5,57 +5,90 @@ const { db, logger, date, timer, file, parser, api, zip } = require('../../core' const path = require('path') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') +const CronJob = require('cron').CronJob dayjs.extend(utc) +const BASE_DIR = process.env.BASE_DIR || '.' +const CURR_DATE = process.env.CURR_DATE || new Date() + program .requiredOption('-s, --site ', 'Name of the site to parse') .option('-l, --lang ', 'Filter channels by language (ISO 639-2 code)') .option('-o, --output ', 'Path to output file') + .option('--cron ', 'Schedule a script run') .option('--gzip', 'Create a compressed version of the guide as well', false) .parse(process.argv) const options = program.opts() -const BASE_DIR = process.env.BASE_DIR || '.' -const CURR_DATE = process.env.CURR_DATE || new Date() +options.output = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`) options.config = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}.config.js`) options.channels = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}*.channels.xml`) let channels = [] let programs = [] +let runIndex = 0 async function main() { - logger.info('starting...') - timer.start() + logger.start('staring...') - let config = await loadConfig(options.config) - let queue = await createQueue(options.channels, config) + logger.info('settings:') + for (let prop in options) { + logger.info(` ${prop}: ${options[prop]}`) + } - await grab(queue, config) + const config = await loadConfig(options.config) + const queue = await createQueue(options.channels, config) + const outputPath = options.output - let outputPath = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`) - - await save(outputPath, channels, programs) - - logger.info(`done in ${timer.format('HH[h] mm[m] ss[s]')}`) + if (options.cron) { + const job = new CronJob(options.cron, function () { + runJob(config, queue, outputPath) + }) + job.start() + } else { + await runJob(config, queue, outputPath) + } } async function loadConfig(configPath) { - logger.info(`loading "${configPath}"...`) - let config = require(file.resolve(configPath)) - config = _.merge(config, { - debug: options.debug, - delay: options.delay, - request: { - timeout: options.timeout - } - }) + config = _.merge(config, {}) + config.days = config.days || 1 + + logger.info('config:') + logConfig(config) return config } +function logConfig(config, level = 1) { + let padLeft = ' '.repeat(level) + for (let prop in config) { + if (typeof config[prop] === 'string' || typeof config[prop] === 'number') { + logger.info(`${padLeft}${prop}: ${config[prop]}`) + } else if (typeof config[prop] === 'object') { + level++ + logger.info(`${padLeft}${prop}:`) + logConfig(config[prop], level) + } + } +} + +async function runJob(config, queue, outputPath) { + runIndex++ + logger.info(`run #${runIndex}:`) + + timer.start() + + await grab(queue, config) + + await save(outputPath, channels, programs) + + logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`) +} + async function grab(queue, config) { const grabber = new EPGGrabber(config) const total = queue.length @@ -68,14 +101,14 @@ async function grab(queue, config) { await grabber .grab(channel, date, (data, err) => { logger.info( - `[${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs + ` [${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs .utc(data.date) .format('MMM D, YYYY')} (${data.programs.length} programs)` ) if (i < total) i++ if (err) { - logger.error(` Error: ${err.message}`) + logger.info(` ERR: ${err.message}`) } }) .then(results => { @@ -85,19 +118,18 @@ async function grab(queue, config) { } async function createQueue(channelsPath, config) { + logger.info('creating queue...') let queue = {} - - await api.channels.load().catch(console.error) - const files = await file.list(channelsPath).catch(console.error) + await api.channels.load().catch(logger.error) + const files = await file.list(channelsPath).catch(logger.error) const utcDate = date.getUTC(CURR_DATE) for (const filepath of files) { - logger.info(`loading "${filepath}"...`) + logger.info(` loading "${filepath}"...`) try { const dir = file.dirname(filepath) const { channels } = await parser.parseChannels(filepath) const filename = file.basename(filepath) - const days = config.days || 1 - const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd')) + const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd')) for (const channel of channels) { if (!channel.site || !channel.xmltv_id) continue if (options.lang && channel.lang !== options.lang) continue @@ -119,13 +151,15 @@ async function createQueue(channelsPath, config) { } } } catch (err) { - console.error(err) + logger.error(err) continue } } queue = Object.values(queue) + logger.info(` added ${queue.length} items`) + return queue } @@ -169,14 +203,14 @@ async function save(template, parsedChannels, programs = []) { const outputPath = file.templateFormat(template, output.channels[0]) const xmlFilepath = outputPath const xmltv = generateXMLTV(output) + logger.info(` saving to "${xmlFilepath}"...`) await file.create(xmlFilepath, xmltv) - logger.info(`file "${xmlFilepath}" saved`) if (options.gzip) { const gzFilepath = `${outputPath}.gz` const compressed = await zip.compress(xmltv) + logger.info(` saving to "${gzFilepath}"...`) await file.create(gzFilepath, compressed) - logger.info(`file "${gzFilepath}" saved`) } } }