iptv-org-mirror/epg
1
0
Fork 0
mirror of https://github.com/iptv-org/epg.git synced 2025-05-09 16:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update load-cluster.js

Browse source
This commit is contained in:
Aleksandr Statciuk 2022-01-30 23:11:56 +03:00
parent ca3913f357
commit 7e5e1f34d3
4 changed files with 13 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

18
scripts/commands/load-cluster.js
View file

@ -5,7 +5,6 @@ const { db, logger, timer, file, parser } = require('../core')
const options = program
.requiredOption('-c, --cluster-id <cluster-id>', 'The ID of cluster to load', parser.parseNumber)
.option('--days <days>', 'Number of days for which to grab the program', parser.parseNumber, 1)
.option('--delay <delay>', 'Delay between requests (in mileseconds)', parser.parseNumber)
.option(
'-t, --timeout <timeout>',
@ -17,19 +16,18 @@ const options = program
.opts()
const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
const CLUSTER_PATH = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
async function main() {
logger.info('Starting...')
timer.start()
const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
logger.info(`Loading cluster: ${options.clusterId}`)
logger.info(`Creating '${clusterLog}'...`)
await file.create(clusterLog)
logger.info(`Creating '${CLUSTER_PATH}'...`)
await file.create(CLUSTER_PATH)
await db.queue.load()
const items = await db.queue.find({ cluster_id: options.clusterId })
const total = options.days * items.length
logger.info(`Total ${total} requests`)
const total = items.length
logger.info('Loading...')
const results = {}
@ -38,7 +36,6 @@ async function main() {
let config = require(file.resolve(item.configPath))
config = _.merge(config, {
days: options.days,
debug: options.debug,
delay: options.delay,
request: {
@ -46,7 +43,7 @@ async function main() {
}
})
await grabber.grab(item, config, async (data, err) => {
await grabber.grab(item, item.date, config, async (data, err) => {
logger.info(
`[${i}/${total}] ${item.site} - ${item.xmltv_id} - ${data.date.format('MMM D, YYYY')} (${
data.programs.length
@ -56,13 +53,12 @@ async function main() {
if (err) logger.error(err.message)
const result = {
channel: data.channel,
_qid: item._id,
programs: data.programs,
date: data.date.format(),
error: err ? err.message : null
}
await file.append(clusterLog, JSON.stringify(result) + '\n')
await file.append(CLUSTER_PATH, JSON.stringify(result) + '\n')
if (i < total) i++
})

4
tests/__data__/expected/logs/load-cluster/cluster_1.log
View file

@ -1,2 +1,2 @@
{"channel":{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr","bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"_id":"0Wefq0oMR3feCcuY"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null}
{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"_id":"1XzrxNkSF2AQNBrT"},"programs":[],"date":"2022-01-29T00:00:00Z","error":null}
{"_qid":"0Wefq0oMR3feCcuY","programs":[],"error":null}
{"_qid":"1XzrxNkSF2AQNBrT","programs":[],"error":null}

8
tests/__data__/input/database/queue.db
View file

@ -1,4 +1,4 @@
{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"programCount":0,"date":"2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"}
{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"date":"2022-01-21T00:00:00Z","error":null,"_id":"0Wefq0oMR3feCcuY"}
{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1XzrxNkSF2AQNBrT"}
{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1lnhXpN7g0ER5XwN"}
{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"}
{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"}
{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"}
{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5XwN"}

9
tests/commands/load-cluster.test.js
View file

@ -23,15 +23,6 @@ it('can load cluster', () => {
let output = content('tests/__data__/output/logs/load-cluster/cluster_1.log')
let expected = content('tests/__data__/expected/logs/load-cluster/cluster_1.log')
output = output.map(i => {
i.date = null
return i
})
expected = expected.map(i => {
i.date = null
return i
})
expect(output).toEqual(expected)
})