Update load-cluster.js

scripts/commands/load-cluster.js

@@ -5,7 +5,6 @@ const { db, logger, timer, file, parser } = require('../core')
 
 const options = program
   .requiredOption('-c, --cluster-id <cluster-id>', 'The ID of cluster to load', parser.parseNumber)
-  .option('--days <days>', 'Number of days for which to grab the program', parser.parseNumber, 1)
   .option('--delay <delay>', 'Delay between requests (in mileseconds)', parser.parseNumber)
   .option(
     '-t, --timeout <timeout>',
@@ -17,19 +16,18 @@ const options = program
   .opts()
 
 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
+const CLUSTER_PATH = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
 
 async function main() {
   logger.info('Starting...')
   timer.start()
 
-  const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
   logger.info(`Loading cluster: ${options.clusterId}`)
-  logger.info(`Creating '${clusterLog}'...`)
+  logger.info(`Creating '${CLUSTER_PATH}'...`)
-  await file.create(clusterLog)
+  await file.create(CLUSTER_PATH)
   await db.queue.load()
   const items = await db.queue.find({ cluster_id: options.clusterId })
-  const total = options.days * items.length
+  const total = items.length
-  logger.info(`Total ${total} requests`)
 
   logger.info('Loading...')
   const results = {}
@@ -38,7 +36,6 @@ async function main() {
     let config = require(file.resolve(item.configPath))
 
     config = _.merge(config, {
-      days: options.days,
       debug: options.debug,
       delay: options.delay,
       request: {
@@ -46,7 +43,7 @@ async function main() {
       }
     })
 
-    await grabber.grab(item, config, async (data, err) => {
+    await grabber.grab(item, item.date, config, async (data, err) => {
       logger.info(
         `[${i}/${total}] ${item.site} - ${item.xmltv_id} - ${data.date.format('MMM D, YYYY')} (${
           data.programs.length
@@ -56,13 +53,12 @@ async function main() {
       if (err) logger.error(err.message)
 
       const result = {
-        channel: data.channel,
+        _qid: item._id,
         programs: data.programs,
-        date: data.date.format(),
         error: err ? err.message : null
       }
 
-      await file.append(clusterLog, JSON.stringify(result) + '\n')
+      await file.append(CLUSTER_PATH, JSON.stringify(result) + '\n')
 
       if (i < total) i++
     })

tests/__data__/expected/logs/load-cluster/cluster_1.log

@@ -1,2 +1,2 @@
-{"channel":{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr","bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"_id":"0Wefq0oMR3feCcuY"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null}
+{"_qid":"0Wefq0oMR3feCcuY","programs":[],"error":null}
-{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"_id":"1XzrxNkSF2AQNBrT"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null}
+{"_qid":"1XzrxNkSF2AQNBrT","programs":[],"error":null}

tests/__data__/input/database/queue.db

@@ -1,4 +1,4 @@
-{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"programCount":0,"date":"2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"}
+{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"}
-{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"date":"2022-01-21T00:00:00Z","error":null,"_id":"0Wefq0oMR3feCcuY"}
+{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"}
-{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1XzrxNkSF2AQNBrT"}
+{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"}
-{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1lnhXpN7g0ER5XwN"}
+{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5XwN"}

tests/commands/load-cluster.test.js

@@ -23,15 +23,6 @@ it('can load cluster', () => {
   let output = content('tests/__data__/output/logs/load-cluster/cluster_1.log')
   let expected = content('tests/__data__/expected/logs/load-cluster/cluster_1.log')
 
-  output = output.map(i => {
-    i.date = null
-    return i
-  })
-  expected = expected.map(i => {
-    i.date = null
-    return i
-  })
-
   expect(output).toEqual(expected)
 })