From b3413b6c99a431620498f19ed487c9ece495f860 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Fri, 14 Jan 2022 17:52:44 +0300 Subject: [PATCH] Update load-cluster.js --- scripts/commands/load-cluster.js | 36 ++++++++++++++++----- tests/__data__/input/database/channels.db | 8 ++--- tests/commands/load-cluster.test.js | 38 +++++++++++++++++------ 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/scripts/commands/load-cluster.js b/scripts/commands/load-cluster.js index e438d771..bf148ae6 100644 --- a/scripts/commands/load-cluster.js +++ b/scripts/commands/load-cluster.js @@ -1,10 +1,18 @@ +const _ = require('lodash') const grabber = require('epg-grabber') const { program } = require('commander') const { db, logger, timer, file, parser } = require('../core') const options = program .requiredOption('-c, --cluster-id ', 'The ID of cluster to load', parser.parseNumber) - .option('-d, --days ', 'Number of days for which to grab the program', parser.parseNumber) + .option('--days ', 'Number of days for which to grab the program', parser.parseNumber, 1) + .option('--delay ', 'Delay between requests (in mileseconds)', parser.parseNumber) + .option( + '-t, --timeout ', + 'Set a timeout for each request (in mileseconds)', + parser.parseNumber + ) + .option('--debug', 'Enable debug mode', false) .parse(process.argv) .opts() @@ -19,16 +27,24 @@ async function main() { logger.info(`Creating '${clusterLog}'...`) await file.create(clusterLog) const channels = await db.channels.find({ cluster_id: options.clusterId }) - const days = options.days || 1 - const total = days * channels.length + const total = options.days * channels.length logger.info(`Total ${total} requests`) logger.info('Loading...') const results = {} let i = 1 for (const channel of channels) { - const config = require(file.resolve(channel.configPath)) - config.days = config.days || days + let config = require(file.resolve(channel.configPath)) + + config = _.merge(config, { + days: options.days, + debug: options.debug, + delay: options.delay, + request: { + timeout: options.timeout + } + }) + const programs = await grabber.grab(channel, config, (data, err) => { logger.info( `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format( @@ -40,10 +56,16 @@ async function main() { if (i < total) i++ }) + await file.append( clusterLog, - JSON.stringify({ _id: channel._id, site: config.site, country: channel.country, programs }) + - '\n' + JSON.stringify({ + _id: channel._id, + site: config.site, + country: channel.country, + gid: channel.gid, + programs + }) + '\n' ) } diff --git a/tests/__data__/input/database/channels.db b/tests/__data__/input/database/channels.db index 48880755..1e2198ee 100644 --- a/tests/__data__/input/database/channels.db +++ b/tests/__data__/input/database/channels.db @@ -1,4 +1,4 @@ -{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"} -{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://proxymedia.woopic.com/api/v1/images/553%2Flogos%2Fv2%2Flogos%2Flivetv_cnn%2F20151026_092415%2FwebTVLogo%2Flogo_180x96.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr.config.js","cluster_id":1,"_id":"0Wefq0oMR3feCcuY"} -{"lang":"ru","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"140","logo":"https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg","name":"CNN Int","site":"magticom.ge","channelsPath":"sites/magticom.ge/magticom.ge_ge.channels.xml","configPath":"sites/magticom.ge/magticom.ge.config.js","cluster_id":1,"_id":"1XzrxNkSF2AQNBrT"} -{"lang":"en","country":"ZA","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","logo":"https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png","name":"M-Net Movies 2","site":"dstv.com","channelsPath":"sites/dstv.com/dstv.com_zw.channels.xml","configPath":"sites/dstv.com/dstv.com.config.js","cluster_id":120,"_id":"1lnhXpN7g0ER5XwN"} +{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","gid":"us","cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"} +{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://proxymedia.woopic.com/api/v1/images/553%2Flogos%2Fv2%2Flogos%2Flivetv_cnn%2F20151026_092415%2FwebTVLogo%2Flogo_180x96.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr.config.js","gid":"fr","cluster_id":1,"_id":"0Wefq0oMR3feCcuY"} +{"lang":"ru","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"140","logo":"https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg","name":"CNN Int","site":"magticom.ge","channelsPath":"sites/magticom.ge/magticom.ge_ge.channels.xml","configPath":"sites/magticom.ge/magticom.ge.config.js","gid":"ge","cluster_id":1,"_id":"1XzrxNkSF2AQNBrT"} +{"lang":"en","country":"ZA","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","logo":"https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png","name":"M-Net Movies 2","site":"dstv.com","channelsPath":"sites/dstv.com/dstv.com_zw.channels.xml","configPath":"sites/dstv.com/dstv.com.config.js","gid":"zw","cluster_id":120,"_id":"1lnhXpN7g0ER5XwN"} diff --git a/tests/commands/load-cluster.test.js b/tests/commands/load-cluster.test.js index 00b1cb32..f7152cc9 100644 --- a/tests/commands/load-cluster.test.js +++ b/tests/commands/load-cluster.test.js @@ -12,7 +12,7 @@ beforeEach(() => { ) execSync( - 'DB_DIR=tests/__data__/temp/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1', + 'DB_DIR=tests/__data__/temp/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1 --timeout=1', { encoding: 'utf8' } ) }) @@ -22,14 +22,32 @@ afterEach(() => { }) it('can load cluster', () => { - const output = fs.readFileSync( - path.resolve('tests/__data__/output/logs/load-cluster/cluster_1.log'), - { - encoding: 'utf8' - } - ) - const lines = output.split('\n') - const parsed = JSON.parse(lines[0]) + const output = content('tests/__data__/output/logs/load-cluster/cluster_1.log') - expect(parsed._id).toBe('0Wefq0oMR3feCcuY') + expect(output[0]).toMatchObject({ + _id: '0Wefq0oMR3feCcuY', + site: 'chaines-tv.orange.fr', + country: 'US', + gid: 'fr' + }) + + expect(output[1]).toMatchObject({ + _id: '1XzrxNkSF2AQNBrT', + site: 'magticom.ge', + country: 'US', + gid: 'ge' + }) }) + +function content(filepath) { + const data = fs.readFileSync(path.resolve(filepath), { + encoding: 'utf8' + }) + + return data + .split('\n') + .filter(l => l) + .map(l => { + return JSON.parse(l) + }) +}