Update grab.js

scripts/commands/epg/grab.js

@@ -5,57 +5,90 @@ const { db, logger, date, timer, file, parser, api, zip } = require('../../core'
 const path = require('path')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const CronJob = require('cron').CronJob
 
 dayjs.extend(utc)
 
+const BASE_DIR = process.env.BASE_DIR || '.'
+const CURR_DATE = process.env.CURR_DATE || new Date()
+
 program
   .requiredOption('-s, --site <name>', 'Name of the site to parse')
   .option('-l, --lang <code>', 'Filter channels by language (ISO 639-2 code)')
   .option('-o, --output <path>', 'Path to output file')
+  .option('--cron <expression>', 'Schedule a script run')
   .option('--gzip', 'Create a compressed version of the guide as well', false)
   .parse(process.argv)
 
 const options = program.opts()
-const BASE_DIR = process.env.BASE_DIR || '.'
-const CURR_DATE = process.env.CURR_DATE || new Date()
 
+options.output = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`)
 options.config = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}.config.js`)
 options.channels = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}*.channels.xml`)
 
 let channels = []
 let programs = []
+let runIndex = 0
 
 async function main() {
-  logger.info('starting...')
-  timer.start()
+  logger.start('staring...')
 
-  let config = await loadConfig(options.config)
-  let queue = await createQueue(options.channels, config)
+  logger.info('settings:')
+  for (let prop in options) {
+    logger.info(`  ${prop}: ${options[prop]}`)
+  }
 
-  await grab(queue, config)
+  const config = await loadConfig(options.config)
+  const queue = await createQueue(options.channels, config)
+  const outputPath = options.output
 
-  let outputPath = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`)
-
-  await save(outputPath, channels, programs)
-
-  logger.info(`done in ${timer.format('HH[h] mm[m] ss[s]')}`)
+  if (options.cron) {
+    const job = new CronJob(options.cron, function () {
+      runJob(config, queue, outputPath)
+    })
+    job.start()
+  } else {
+    await runJob(config, queue, outputPath)
+  }
 }
 
 async function loadConfig(configPath) {
-  logger.info(`loading "${configPath}"...`)
-
   let config = require(file.resolve(configPath))
-  config = _.merge(config, {
-    debug: options.debug,
-    delay: options.delay,
-    request: {
-      timeout: options.timeout
-    }
-  })
+  config = _.merge(config, {})
+  config.days = config.days || 1
+
+  logger.info('config:')
+  logConfig(config)
 
   return config
 }
 
+function logConfig(config, level = 1) {
+  let padLeft = '  '.repeat(level)
+  for (let prop in config) {
+    if (typeof config[prop] === 'string' || typeof config[prop] === 'number') {
+      logger.info(`${padLeft}${prop}: ${config[prop]}`)
+    } else if (typeof config[prop] === 'object') {
+      level++
+      logger.info(`${padLeft}${prop}:`)
+      logConfig(config[prop], level)
+    }
+  }
+}
+
+async function runJob(config, queue, outputPath) {
+  runIndex++
+  logger.info(`run #${runIndex}:`)
+
+  timer.start()
+
+  await grab(queue, config)
+
+  await save(outputPath, channels, programs)
+
+  logger.success(`  done in ${timer.format('HH[h] mm[m] ss[s]')}`)
+}
+
 async function grab(queue, config) {
   const grabber = new EPGGrabber(config)
   const total = queue.length
@@ -68,14 +101,14 @@ async function grab(queue, config) {
     await grabber
       .grab(channel, date, (data, err) => {
         logger.info(
-          `[${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs
+          `  [${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs
             .utc(data.date)
             .format('MMM D, YYYY')} (${data.programs.length} programs)`
         )
         if (i < total) i++
 
         if (err) {
-          logger.error(`  Error: ${err.message}`)
+          logger.info(`    ERR: ${err.message}`)
         }
       })
       .then(results => {
@@ -85,19 +118,18 @@ async function grab(queue, config) {
 }
 
 async function createQueue(channelsPath, config) {
+  logger.info('creating queue...')
   let queue = {}
-
-  await api.channels.load().catch(console.error)
-  const files = await file.list(channelsPath).catch(console.error)
+  await api.channels.load().catch(logger.error)
+  const files = await file.list(channelsPath).catch(logger.error)
   const utcDate = date.getUTC(CURR_DATE)
   for (const filepath of files) {
-    logger.info(`loading "${filepath}"...`)
+    logger.info(`  loading "${filepath}"...`)
     try {
       const dir = file.dirname(filepath)
       const { channels } = await parser.parseChannels(filepath)
       const filename = file.basename(filepath)
-      const days = config.days || 1
-      const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
+      const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
       for (const channel of channels) {
         if (!channel.site || !channel.xmltv_id) continue
         if (options.lang && channel.lang !== options.lang) continue
@@ -119,13 +151,15 @@ async function createQueue(channelsPath, config) {
         }
       }
     } catch (err) {
-      console.error(err)
+      logger.error(err)
       continue
     }
   }
 
   queue = Object.values(queue)
 
+  logger.info(`  added ${queue.length} items`)
+
   return queue
 }
 
@@ -169,14 +203,14 @@ async function save(template, parsedChannels, programs = []) {
     const outputPath = file.templateFormat(template, output.channels[0])
     const xmlFilepath = outputPath
     const xmltv = generateXMLTV(output)
+    logger.info(`  saving to "${xmlFilepath}"...`)
     await file.create(xmlFilepath, xmltv)
-    logger.info(`file "${xmlFilepath}" saved`)
 
     if (options.gzip) {
       const gzFilepath = `${outputPath}.gz`
       const compressed = await zip.compress(xmltv)
+      logger.info(`  saving to "${gzFilepath}"...`)
       await file.create(gzFilepath, compressed)
-      logger.info(`file "${gzFilepath}" saved`)
     }
   }
 }