From 7e5e1f34d35a6244fb0b7de586efe93e1a517c6a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Jan 2022 23:11:56 +0300 Subject: [PATCH] Update load-cluster.js --- scripts/commands/load-cluster.js | 18 +++++++----------- .../expected/logs/load-cluster/cluster_1.log | 4 ++-- tests/__data__/input/database/queue.db | 8 ++++---- tests/commands/load-cluster.test.js | 9 --------- 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/scripts/commands/load-cluster.js b/scripts/commands/load-cluster.js index 1fb95e05..f4192e3b 100644 --- a/scripts/commands/load-cluster.js +++ b/scripts/commands/load-cluster.js @@ -5,7 +5,6 @@ const { db, logger, timer, file, parser } = require('../core') const options = program .requiredOption('-c, --cluster-id ', 'The ID of cluster to load', parser.parseNumber) - .option('--days ', 'Number of days for which to grab the program', parser.parseNumber, 1) .option('--delay ', 'Delay between requests (in mileseconds)', parser.parseNumber) .option( '-t, --timeout ', @@ -17,19 +16,18 @@ const options = program .opts() const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' +const CLUSTER_PATH = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log` async function main() { logger.info('Starting...') timer.start() - const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log` logger.info(`Loading cluster: ${options.clusterId}`) - logger.info(`Creating '${clusterLog}'...`) - await file.create(clusterLog) + logger.info(`Creating '${CLUSTER_PATH}'...`) + await file.create(CLUSTER_PATH) await db.queue.load() const items = await db.queue.find({ cluster_id: options.clusterId }) - const total = options.days * items.length - logger.info(`Total ${total} requests`) + const total = items.length logger.info('Loading...') const results = {} @@ -38,7 +36,6 @@ async function main() { let config = require(file.resolve(item.configPath)) config = _.merge(config, { - days: options.days, debug: options.debug, delay: options.delay, request: { @@ -46,7 +43,7 @@ async function main() { } }) - await grabber.grab(item, config, async (data, err) => { + await grabber.grab(item, item.date, config, async (data, err) => { logger.info( `[${i}/${total}] ${item.site} - ${item.xmltv_id} - ${data.date.format('MMM D, YYYY')} (${ data.programs.length @@ -56,13 +53,12 @@ async function main() { if (err) logger.error(err.message) const result = { - channel: data.channel, + _qid: item._id, programs: data.programs, - date: data.date.format(), error: err ? err.message : null } - await file.append(clusterLog, JSON.stringify(result) + '\n') + await file.append(CLUSTER_PATH, JSON.stringify(result) + '\n') if (i < total) i++ }) diff --git a/tests/__data__/expected/logs/load-cluster/cluster_1.log b/tests/__data__/expected/logs/load-cluster/cluster_1.log index 5153c726..feb81b42 100644 --- a/tests/__data__/expected/logs/load-cluster/cluster_1.log +++ b/tests/__data__/expected/logs/load-cluster/cluster_1.log @@ -1,2 +1,2 @@ -{"channel":{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr","bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"_id":"0Wefq0oMR3feCcuY"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null} -{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"_id":"1XzrxNkSF2AQNBrT"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null} +{"_qid":"0Wefq0oMR3feCcuY","programs":[],"error":null} +{"_qid":"1XzrxNkSF2AQNBrT","programs":[],"error":null} diff --git a/tests/__data__/input/database/queue.db b/tests/__data__/input/database/queue.db index 75532315..b9006972 100644 --- a/tests/__data__/input/database/queue.db +++ b/tests/__data__/input/database/queue.db @@ -1,4 +1,4 @@ -{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"programCount":0,"date":"2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"} -{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"date":"2022-01-21T00:00:00Z","error":null,"_id":"0Wefq0oMR3feCcuY"} -{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1XzrxNkSF2AQNBrT"} -{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1lnhXpN7g0ER5XwN"} +{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"} +{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"} +{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"} +{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5XwN"} diff --git a/tests/commands/load-cluster.test.js b/tests/commands/load-cluster.test.js index 9fe95422..41347eb2 100644 --- a/tests/commands/load-cluster.test.js +++ b/tests/commands/load-cluster.test.js @@ -23,15 +23,6 @@ it('can load cluster', () => { let output = content('tests/__data__/output/logs/load-cluster/cluster_1.log') let expected = content('tests/__data__/expected/logs/load-cluster/cluster_1.log') - output = output.map(i => { - i.date = null - return i - }) - expected = expected.map(i => { - i.date = null - return i - }) - expect(output).toEqual(expected) })