From 54d482269f933765f6f7a7a3d98ccca8b4d6ab4f Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 19 Sep 2023 06:14:59 +0300 Subject: [PATCH 1/4] Move `api:load` to `postinstall` --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6ae78ad5..4c3aca8c 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,9 @@ "test:commands": "npx jest --runInBand -- commands", "test:sites": "TZ=Pacific/Nauru npx jest --runInBand -- sites", "check": "npm run api:load && npm run channels:lint sites/**/*.js && npm run channels:validate sites/**/*.xml", - "grab": "npm run api:load && node scripts/commands/epg/grab.js", - "serve": "npx serve" + "grab": "node scripts/commands/epg/grab.js", + "serve": "npx serve", + "postinstall": "npm run api:load" }, "private": true, "author": "Arhey", From 4b9d0bd6782a9b93edfadb1dbb9f9ab9f757fc58 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 19 Sep 2023 06:15:48 +0300 Subject: [PATCH 2/4] Update tests data --- tests/__data__/expected/guides/en/example.com.xml | 1 + .../__data__/input/sites/epg-grab/epg-grab.channels.xml | 1 + tests/__data__/input/sites/epg-grab/epg-grab.config.js | 9 +++++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/__data__/expected/guides/en/example.com.xml b/tests/__data__/expected/guides/en/example.com.xml index 0f94474c..e99bf680 100644 --- a/tests/__data__/expected/guides/en/example.com.xml +++ b/tests/__data__/expected/guides/en/example.com.xml @@ -1,4 +1,5 @@ Channel 1https://example.com +Channel 2https://example.com Program1 \ No newline at end of file diff --git a/tests/__data__/input/sites/epg-grab/epg-grab.channels.xml b/tests/__data__/input/sites/epg-grab/epg-grab.channels.xml index 561ce412..75fa7c3c 100644 --- a/tests/__data__/input/sites/epg-grab/epg-grab.channels.xml +++ b/tests/__data__/input/sites/epg-grab/epg-grab.channels.xml @@ -2,6 +2,7 @@ Channel 1 + Channel 2 Channel 1 \ No newline at end of file diff --git a/tests/__data__/input/sites/epg-grab/epg-grab.config.js b/tests/__data__/input/sites/epg-grab/epg-grab.config.js index 493907db..dca7aeea 100644 --- a/tests/__data__/input/sites/epg-grab/epg-grab.config.js +++ b/tests/__data__/input/sites/epg-grab/epg-grab.config.js @@ -1,10 +1,15 @@ module.exports = { site: 'example.com', - days: 2, + days: 1, + request: { + timeout: 1000 + }, url() { return `https://example.com` }, - parser() { + parser({ channel }) { + if (channel.xmltv_id === 'Channel2.us') return [] + return [ { title: 'Program1', From a8a0af2254391b22947e79cec1993c8caa26a25c Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 19 Sep 2023 06:16:28 +0300 Subject: [PATCH 3/4] Update grab.js Fixes #2161 --- scripts/commands/epg/grab.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/commands/epg/grab.js b/scripts/commands/epg/grab.js index b52f35b9..25d365ed 100644 --- a/scripts/commands/epg/grab.js +++ b/scripts/commands/epg/grab.js @@ -16,6 +16,7 @@ program .requiredOption('-s, --site ', 'Name of the site to parse') .option('-l, --lang ', 'Filter channels by language (ISO 639-2 code)') .option('-o, --output ', 'Path to output file') + .option('--days ', 'Override the number of days for which the program will be loaded') .option('--cron ', 'Schedule a script run') .option('--gzip', 'Create a compressed version of the guide as well', false) .parse(process.argv) @@ -123,13 +124,14 @@ async function createQueue(channelsPath, config) { await api.channels.load().catch(logger.error) const files = await file.list(channelsPath).catch(logger.error) const utcDate = date.getUTC(CURR_DATE) + const days = options.days ? parseInt(options.days) : config.days for (const filepath of files) { logger.info(` loading "${filepath}"...`) try { const dir = file.dirname(filepath) const { channels } = await parser.parseChannels(filepath) const filename = file.basename(filepath) - const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd')) + const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd')) for (const channel of channels) { if (!channel.site || !channel.xmltv_id) continue if (options.lang && channel.lang !== options.lang) continue @@ -194,8 +196,8 @@ async function save(template, parsedChannels, programs = []) { output.programs.push(new Program(program, channel)) } - output.channels = _.sortBy(output.channels, 'id') - output.channels = _.uniqBy(output.channels, 'id') + output.channels = _.sortBy(output.channels, 'xmltv_id') + output.channels = _.uniqBy(output.channels, 'xmltv_id') output.programs = _.sortBy(output.programs, ['channel', 'start']) output.programs = _.uniqBy(output.programs, p => p.channel + p.start) From db4c2c8a492aa17e355d93d7a375d2beccf1097d Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Tue, 19 Sep 2023 06:16:33 +0300 Subject: [PATCH 4/4] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 23e6dd89..554bb486 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,12 @@ To download a guide in a specific language pass its [ISO 639-1](https://en.wikip npm run grab -- --site=example.com --lang=fr ``` +To override the number of days for which the program will be loaded use the `--days` argument (the default is the value specified in the site config): + +```sh +npm run grab -- --site=example.com --days=3 +``` + To also create a compressed version of the guide, add the `--gzip` flag: ```sh