Update load-cluster.js

scripts/commands/load-cluster.js

@@ -1,10 +1,18 @@
+const _ = require('lodash')
 const grabber = require('epg-grabber')
 const { program } = require('commander')
 const { db, logger, timer, file, parser } = require('../core')
 
 const options = program
   .requiredOption('-c, --cluster-id <cluster-id>', 'The ID of cluster to load', parser.parseNumber)
-  .option('-d, --days <days>', 'Number of days for which to grab the program', parser.parseNumber)
+  .option('--days <days>', 'Number of days for which to grab the program', parser.parseNumber, 1)
+  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parser.parseNumber)
+  .option(
+    '-t, --timeout <timeout>',
+    'Set a timeout for each request (in mileseconds)',
+    parser.parseNumber
+  )
+  .option('--debug', 'Enable debug mode', false)
   .parse(process.argv)
   .opts()
 
@@ -19,16 +27,24 @@ async function main() {
   logger.info(`Creating '${clusterLog}'...`)
   await file.create(clusterLog)
   const channels = await db.channels.find({ cluster_id: options.clusterId })
-  const days = options.days || 1
+  const total = options.days * channels.length
-  const total = days * channels.length
   logger.info(`Total ${total} requests`)
 
   logger.info('Loading...')
   const results = {}
   let i = 1
   for (const channel of channels) {
-    const config = require(file.resolve(channel.configPath))
+    let config = require(file.resolve(channel.configPath))
-    config.days = config.days || days
+
+    config = _.merge(config, {
+      days: options.days,
+      debug: options.debug,
+      delay: options.delay,
+      request: {
+        timeout: options.timeout
+      }
+    })
+
     const programs = await grabber.grab(channel, config, (data, err) => {
       logger.info(
         `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format(
@@ -40,10 +56,16 @@ async function main() {
 
       if (i < total) i++
     })
+
     await file.append(
       clusterLog,
-      JSON.stringify({ _id: channel._id, site: config.site, country: channel.country, programs }) +
+      JSON.stringify({
-        '\n'
+        _id: channel._id,
+        site: config.site,
+        country: channel.country,
+        gid: channel.gid,
+        programs
+      }) + '\n'
     )
   }
 

tests/__data__/input/database/channels.db

@@ -1,4 +1,4 @@
-{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"}
+{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","gid":"us","cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"}
-{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://proxymedia.woopic.com/api/v1/images/553%2Flogos%2Fv2%2Flogos%2Flivetv_cnn%2F20151026_092415%2FwebTVLogo%2Flogo_180x96.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr.config.js","cluster_id":1,"_id":"0Wefq0oMR3feCcuY"}
+{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://proxymedia.woopic.com/api/v1/images/553%2Flogos%2Fv2%2Flogos%2Flivetv_cnn%2F20151026_092415%2FwebTVLogo%2Flogo_180x96.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr.config.js","gid":"fr","cluster_id":1,"_id":"0Wefq0oMR3feCcuY"}
-{"lang":"ru","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"140","logo":"https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg","name":"CNN Int","site":"magticom.ge","channelsPath":"sites/magticom.ge/magticom.ge_ge.channels.xml","configPath":"sites/magticom.ge/magticom.ge.config.js","cluster_id":1,"_id":"1XzrxNkSF2AQNBrT"}
+{"lang":"ru","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"140","logo":"https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg","name":"CNN Int","site":"magticom.ge","channelsPath":"sites/magticom.ge/magticom.ge_ge.channels.xml","configPath":"sites/magticom.ge/magticom.ge.config.js","gid":"ge","cluster_id":1,"_id":"1XzrxNkSF2AQNBrT"}
-{"lang":"en","country":"ZA","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","logo":"https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png","name":"M-Net Movies 2","site":"dstv.com","channelsPath":"sites/dstv.com/dstv.com_zw.channels.xml","configPath":"sites/dstv.com/dstv.com.config.js","cluster_id":120,"_id":"1lnhXpN7g0ER5XwN"}
+{"lang":"en","country":"ZA","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","logo":"https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png","name":"M-Net Movies 2","site":"dstv.com","channelsPath":"sites/dstv.com/dstv.com_zw.channels.xml","configPath":"sites/dstv.com/dstv.com.config.js","gid":"zw","cluster_id":120,"_id":"1lnhXpN7g0ER5XwN"}

tests/commands/load-cluster.test.js

@@ -12,7 +12,7 @@ beforeEach(() => {
   )
 
   execSync(
-    'DB_DIR=tests/__data__/temp/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1',
+    'DB_DIR=tests/__data__/temp/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1 --timeout=1',
     { encoding: 'utf8' }
   )
 })
@@ -22,14 +22,32 @@ afterEach(() => {
 })
 
 it('can load cluster', () => {
-  const output = fs.readFileSync(
+  const output = content('tests/__data__/output/logs/load-cluster/cluster_1.log')
-    path.resolve('tests/__data__/output/logs/load-cluster/cluster_1.log'),
-    {
-      encoding: 'utf8'
-    }
-  )
-  const lines = output.split('\n')
-  const parsed = JSON.parse(lines[0])
 
-  expect(parsed._id).toBe('0Wefq0oMR3feCcuY')
+  expect(output[0]).toMatchObject({
+    _id: '0Wefq0oMR3feCcuY',
+    site: 'chaines-tv.orange.fr',
+    country: 'US',
+    gid: 'fr'
+  })
+
+  expect(output[1]).toMatchObject({
+    _id: '1XzrxNkSF2AQNBrT',
+    site: 'magticom.ge',
+    country: 'US',
+    gid: 'ge'
+  })
 })
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
+    encoding: 'utf8'
+  })
+
+  return data
+    .split('\n')
+    .filter(l => l)
+    .map(l => {
+      return JSON.parse(l)
+    })
+}