From ef4c11a1444acef1224ed6981974b5a0140163b3 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 15:58:37 +0300 Subject: [PATCH 1/6] Update load.js --- scripts/commands/cluster/load.js | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/commands/cluster/load.js b/scripts/commands/cluster/load.js index 20863cf0..9539a6a0 100644 --- a/scripts/commands/cluster/load.js +++ b/scripts/commands/cluster/load.js @@ -1,5 +1,5 @@ const _ = require('lodash') -const grabber = require('epg-grabber') +const EPGGrabber = require('epg-grabber') const { program } = require('commander') const { db, logger, timer, file, parser } = require('../../core') @@ -11,6 +11,11 @@ const options = program 'Set a timeout for each request (in mileseconds)', parser.parseNumber ) + .option( + '--cache-max-age', + 'Maximum time for storing each request (in milliseconds)', + parser.parseNumber + ) .option('--debug', 'Enable debug mode', false) .parse(process.argv) .opts() @@ -33,18 +38,20 @@ async function main() { logger.info('Loading...') const results = {} let i = 1 - for (const item of items) { - let config = require(file.resolve(item.configPath)) - - config = _.merge(config, { - debug: options.debug, - delay: options.delay, - request: { - timeout: options.timeout + let config = require(file.resolve(items[0].configPath)) + config = _.merge(config, { + debug: options.debug, + delay: options.delay, + request: { + timeout: options.timeout, + cache: { + maxAge: options.cacheMaxAge } - }) - - await grabber.grab(item.channel, item.date, config, async (data, err) => { + } + }) + const grabber = new EPGGrabber(config) + for (const item of items) { + await grabber.grab(item.channel, item.date, async (data, err) => { logger.info( `[${i}/${total}] ${item.channel.site} (${item.channel.lang}) - ${ item.channel.xmltv_id From 41517452c18852de505aca072ad606ae3a78274e Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 17:56:12 +0300 Subject: [PATCH 2/6] Upgrade epg-grabber --- package-lock.json | 105 ++++++++++++++++++++++++++++++++++++++++++---- package.json | 2 +- 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 946423fe..c194f3fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "commander": "^8.2.0", "csv-parser": "^3.0.0", "dayjs": "^1.10.8", - "epg-grabber": "^0.24.0", + "epg-grabber": "^0.25.1", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "fs-extra": "^10.0.1", @@ -1270,6 +1270,18 @@ "follow-redirects": "^1.14.0" } }, + "node_modules/axios-cache-adapter": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz", + "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==", + "dependencies": { + "cache-control-esm": "1.0.0", + "md5": "^2.2.1" + }, + "peerDependencies": { + "axios": "~0.21.1" + } + }, "node_modules/axios-cookiejar-support": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz", @@ -1529,6 +1541,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/cache-control-esm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz", + "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g==" + }, "node_modules/cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -1621,6 +1638,14 @@ "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", + "engines": { + "node": "*" + } + }, "node_modules/cheerio": { "version": "1.0.0-rc.10", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz", @@ -1875,6 +1900,14 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", + "engines": { + "node": "*" + } + }, "node_modules/css-select": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz", @@ -2213,11 +2246,12 @@ } }, "node_modules/epg-grabber": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz", - "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==", + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz", + "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==", "dependencies": { "axios": "^0.21.1", + "axios-cache-adapter": "^2.7.3", "axios-cookiejar-support": "^1.0.1", "axios-mock-adapter": "^1.20.0", "commander": "^7.1.0", @@ -4072,6 +4106,21 @@ "markdown-include": "bin/cli.js" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, + "node_modules/md5/node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -6890,6 +6939,15 @@ "follow-redirects": "^1.14.0" } }, + "axios-cache-adapter": { + "version": "2.7.3", + "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz", + "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==", + "requires": { + "cache-control-esm": "1.0.0", + "md5": "^2.2.1" + } + }, "axios-cookiejar-support": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz", @@ -7075,6 +7133,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "cache-control-esm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz", + "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g==" + }, "cacheable-lookup": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", @@ -7138,6 +7201,11 @@ "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, + "charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" + }, "cheerio": { "version": "1.0.0-rc.10", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz", @@ -7348,6 +7416,11 @@ "which": "^2.0.1" } }, + "crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" + }, "css-select": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz", @@ -7591,11 +7664,12 @@ "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" }, "epg-grabber": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz", - "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==", + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz", + "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==", "requires": { "axios": "^0.21.1", + "axios-cache-adapter": "^2.7.3", "axios-cookiejar-support": "^1.0.1", "axios-mock-adapter": "^1.20.0", "commander": "^7.1.0", @@ -8976,6 +9050,23 @@ "q": "^1.2.0" } }, + "md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "requires": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + }, + "dependencies": { + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + } + } + }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", diff --git a/package.json b/package.json index f05cc50a..70397e52 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "commander": "^8.2.0", "csv-parser": "^3.0.0", "dayjs": "^1.10.8", - "epg-grabber": "^0.24.0", + "epg-grabber": "^0.25.1", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "fs-extra": "^10.0.1", From 78f65d544b2157eb47637f391e1d394cb33099e5 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 17:57:48 +0300 Subject: [PATCH 3/6] Update _load.yml --- .github/workflows/_load.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml index 689d1da7..3fe7b1d1 100644 --- a/.github/workflows/_load.yml +++ b/.github/workflows/_load.yml @@ -32,7 +32,7 @@ jobs: cache: 'npm' - run: npm install - run: CHANNELS_PATH=sites/${{inputs.site}}/*.channels.xml npm run queue:create -- --max-clusters=1 --days=2 - - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cluster-id=1 + - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1 - run: npm run programs:save - run: npm run guides:update - uses: tibdex/github-app-token@v1 From 6374c5a36ebf9b4e4ad47898deefa109a7625bcc Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 18:03:16 +0300 Subject: [PATCH 4/6] Update _load.yml --- .github/workflows/_load.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml index 3fe7b1d1..4e2d3f7e 100644 --- a/.github/workflows/_load.yml +++ b/.github/workflows/_load.yml @@ -32,7 +32,7 @@ jobs: cache: 'npm' - run: npm install - run: CHANNELS_PATH=sites/${{inputs.site}}/*.channels.xml npm run queue:create -- --max-clusters=1 --days=2 - - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1 + - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1 --debug - run: npm run programs:save - run: npm run guides:update - uses: tibdex/github-app-token@v1 From 102864965ebe45da1214534fb102483a6e0e7b30 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 18:08:32 +0300 Subject: [PATCH 5/6] Update load.js --- scripts/commands/cluster/load.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/cluster/load.js b/scripts/commands/cluster/load.js index 9539a6a0..96eb27b6 100644 --- a/scripts/commands/cluster/load.js +++ b/scripts/commands/cluster/load.js @@ -12,7 +12,7 @@ const options = program parser.parseNumber ) .option( - '--cache-max-age', + '--cache-max-age ', 'Maximum time for storing each request (in milliseconds)', parser.parseNumber ) From 9e506438aa9963a3375d12c80827a20fe7216240 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 29 Mar 2022 18:13:48 +0300 Subject: [PATCH 6/6] Update _load.yml --- .github/workflows/_load.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml index 4e2d3f7e..3fe7b1d1 100644 --- a/.github/workflows/_load.yml +++ b/.github/workflows/_load.yml @@ -32,7 +32,7 @@ jobs: cache: 'npm' - run: npm install - run: CHANNELS_PATH=sites/${{inputs.site}}/*.channels.xml npm run queue:create -- --max-clusters=1 --days=2 - - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1 --debug + - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1 - run: npm run programs:save - run: npm run guides:update - uses: tibdex/github-app-token@v1