diff --git a/.husky/check_channels.sh b/.husky/channels_lint.sh similarity index 84% rename from .husky/check_channels.sh rename to .husky/channels_lint.sh index f934e7a4..45a15a83 100755 --- a/.husky/check_channels.sh +++ b/.husky/channels_lint.sh @@ -3,6 +3,5 @@ channels_changed="$(git diff --staged --name-only --diff-filter=ACMR -- 'sites/**/*.channels.xml' | sed 's| |\\ |g')" if [ ! -z "$channels_changed" ]; then - echo "npx eslint $channels_changed" npm run channels:lint -- $channels_changed fi \ No newline at end of file diff --git a/.husky/channels_validate.sh b/.husky/channels_validate.sh new file mode 100755 index 00000000..e0fe710e --- /dev/null +++ b/.husky/channels_validate.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +channels_changed="$(git diff --staged --name-only --diff-filter=ACMR -- 'sites/**/*.channels.xml' | sed 's| |\\ |g')" + +if [ ! -z "$channels_changed" ]; then + npm run channels:validate -- $channels_changed +fi \ No newline at end of file diff --git a/.husky/pre-commit b/.husky/pre-commit index ac5a8377..ec89363b 100644 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,2 +1,3 @@ -.husky/check_scripts.sh -.husky/check_channels.sh \ No newline at end of file +.husky/scripts_lint.sh +.husky/channels_lint.sh +.husky/channels_validate.sh \ No newline at end of file diff --git a/.husky/check_scripts.sh b/.husky/scripts_lint.sh similarity index 88% rename from .husky/check_scripts.sh rename to .husky/scripts_lint.sh index a29f7607..b6298c27 100755 --- a/.husky/check_scripts.sh +++ b/.husky/scripts_lint.sh @@ -3,6 +3,5 @@ scripts_changed="$(git diff --staged --name-only --diff-filter=ACMR -- 'tests/**/*.ts' 'tests/**/*.js' 'scripts/**/*.ts' 'scripts/**/*.mts' 'scripts/**/*.js' 'sites/**/*.js' 'sites/**/*.ts' | sed 's| |\\ |g')" if [ ! -z "$scripts_changed" ]; then - echo "npx eslint $scripts_changed" npx eslint $scripts_changed fi \ No newline at end of file diff --git a/README.md b/README.md index eb966322..e1840292 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Options: -l, --lang Filter channels by language (ISO 639-2 code) -t, --timeout Override the default timeout for each request -d, --delay Override the default delay between request + -x, --proxy Use the specified proxy (example: "socks5://username:password@127.0.0.1:1234") --days Override the number of days for which the program will be loaded (defaults to the value from the site config) --maxConnections Limit on the number of concurrent requests (default: 1) diff --git a/package-lock.json b/package-lock.json index 31a33148..7c8e0bb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,8 @@ "@alex_neo/jest-expect-message": "^1.0.5", "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.17.0", - "@freearhey/core": "^0.5.0", - "@ntlab/sfetch": "^1.0.0", + "@freearhey/core": "^0.5.1", + "@ntlab/sfetch": "^1.2.0", "@octokit/core": "^6.1.3", "@octokit/plugin-paginate-rest": "^11.3.6", "@octokit/plugin-rest-endpoint-methods": "^13.2.6", @@ -38,7 +38,7 @@ "csv-parser": "^3.0.0", "cwait": "^1.1.2", "dayjs": "^1.11.10", - "epg-grabber": "^0.37.4", + "epg-grabber": "^0.38.0", "epg-parser": "^0.3.1", "eslint": "^9.17.0", "eslint-config-prettier": "^9.0.0", @@ -69,6 +69,7 @@ "serve": "^14.2.4", "signale": "^1.4.0", "skip-postinstall": "^1.0.0", + "socks-proxy-agent": "^8.0.5", "srcset": "^4.0.0", "table2array": "^0.0.2", "tabletojson": "^2.0.7", @@ -1203,9 +1204,9 @@ } }, "node_modules/@freearhey/core": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.5.0.tgz", - "integrity": "sha512-FcA5Pv9RvFvLYAwNmD/2vlSR49Rx+kihJ+xbIUgIACHY6lBUptfbNznm00DQoUyWRJG/cfT3dkYCwIxSUsdP+w==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.5.1.tgz", + "integrity": "sha512-UDKIOyrtcUXaiAeIvjNFTI6DlempiOQaRB83CqHNF1VPRHNBiNhGhERWyInHE2cjLp/cc0CA/IykOYS39kBK7Q==", "dependencies": { "@types/fs-extra": "^11.0.2", "@types/lodash": "^4.14.198", @@ -1863,9 +1864,9 @@ } }, "node_modules/@ntlab/sfetch": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz", - "integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.2.0.tgz", + "integrity": "sha512-9SE4NnqWo8l6mG0rnAkgng6ozSamIpF3EC+GOTQGGa6eAC0tNJvzrylMz6YRjjEGH6mOfn7ZBAuKj5WIZUul6A==", "dependencies": { "axios": "^1.7.9" } @@ -4192,9 +4193,9 @@ } }, "node_modules/epg-grabber": { - "version": "0.37.4", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.37.4.tgz", - "integrity": "sha512-PS104bH9tHRa9kivSwx47AKMkfHwKy51XQTx+GO6sIXvIp2Z4LBpwMEXGcfPoAsdIGxgs2Wrl0dZ/QGL+7x6YQ==", + "version": "0.38.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.38.0.tgz", + "integrity": "sha512-jbwTgi6G7e+zrb2oNC0C7mcQYoRkFnvhXCurexeICaEy4avRB6WS5rD/yfqYoiqaXOM3x1BNBpCKFYoS7Ob5YA==", "dependencies": { "axios": "^1.6.1", "axios-cache-interceptor": "^0.10.3", @@ -4209,6 +4210,7 @@ "http-cookie-agent": "^6.0.8", "lodash": "^4.17.21", "node-gzip": "^1.1.2", + "socks-proxy-agent": "^8.0.5", "tough-cookie": "^5.0.0", "winston": "^3.3.3", "xml-js": "^1.6.11" @@ -5247,6 +5249,23 @@ "node": ">=8" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -6141,6 +6160,11 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "node_modules/jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -7641,6 +7665,41 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -9172,9 +9231,9 @@ } }, "@freearhey/core": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.5.0.tgz", - "integrity": "sha512-FcA5Pv9RvFvLYAwNmD/2vlSR49Rx+kihJ+xbIUgIACHY6lBUptfbNznm00DQoUyWRJG/cfT3dkYCwIxSUsdP+w==", + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/@freearhey/core/-/core-0.5.1.tgz", + "integrity": "sha512-UDKIOyrtcUXaiAeIvjNFTI6DlempiOQaRB83CqHNF1VPRHNBiNhGhERWyInHE2cjLp/cc0CA/IykOYS39kBK7Q==", "requires": { "@types/fs-extra": "^11.0.2", "@types/lodash": "^4.14.198", @@ -9661,9 +9720,9 @@ } }, "@ntlab/sfetch": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz", - "integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.2.0.tgz", + "integrity": "sha512-9SE4NnqWo8l6mG0rnAkgng6ozSamIpF3EC+GOTQGGa6eAC0tNJvzrylMz6YRjjEGH6mOfn7ZBAuKj5WIZUul6A==", "requires": { "axios": "^1.7.9" } @@ -11280,9 +11339,9 @@ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" }, "epg-grabber": { - "version": "0.37.4", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.37.4.tgz", - "integrity": "sha512-PS104bH9tHRa9kivSwx47AKMkfHwKy51XQTx+GO6sIXvIp2Z4LBpwMEXGcfPoAsdIGxgs2Wrl0dZ/QGL+7x6YQ==", + "version": "0.38.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.38.0.tgz", + "integrity": "sha512-jbwTgi6G7e+zrb2oNC0C7mcQYoRkFnvhXCurexeICaEy4avRB6WS5rD/yfqYoiqaXOM3x1BNBpCKFYoS7Ob5YA==", "requires": { "axios": "^1.6.1", "axios-cache-interceptor": "^0.10.3", @@ -11297,6 +11356,7 @@ "http-cookie-agent": "^6.0.8", "lodash": "^4.17.21", "node-gzip": "^1.1.2", + "socks-proxy-agent": "^8.0.5", "tough-cookie": "^5.0.0", "winston": "^3.3.3", "xml-js": "^1.6.11" @@ -12006,6 +12066,22 @@ } } }, + "ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "requires": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "dependencies": { + "sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + } + } + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -12650,6 +12726,11 @@ "esprima": "^4.0.0" } }, + "jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -13776,6 +13857,30 @@ "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==" }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, + "socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "requires": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + } + }, + "socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "requires": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index 46dcb653..95072df8 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,8 @@ "@alex_neo/jest-expect-message": "^1.0.5", "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.17.0", - "@freearhey/core": "^0.5.0", - "@ntlab/sfetch": "^1.0.0", + "@freearhey/core": "^0.5.1", + "@ntlab/sfetch": "^1.2.0", "@octokit/core": "^6.1.3", "@octokit/plugin-paginate-rest": "^11.3.6", "@octokit/plugin-rest-endpoint-methods": "^13.2.6", @@ -67,7 +67,7 @@ "csv-parser": "^3.0.0", "cwait": "^1.1.2", "dayjs": "^1.11.10", - "epg-grabber": "^0.37.4", + "epg-grabber": "^0.38.0", "epg-parser": "^0.3.1", "eslint": "^9.17.0", "eslint-config-prettier": "^9.0.0", @@ -98,6 +98,7 @@ "serve": "^14.2.4", "signale": "^1.4.0", "skip-postinstall": "^1.0.0", + "socks-proxy-agent": "^8.0.5", "srcset": "^4.0.0", "table2array": "^0.0.2", "tabletojson": "^2.0.7", diff --git a/scripts/commands/channels/parse.ts b/scripts/commands/channels/parse.ts index 96c57a52..572b5ed6 100644 --- a/scripts/commands/channels/parse.ts +++ b/scripts/commands/channels/parse.ts @@ -26,7 +26,7 @@ async function main() { const logger = new Logger() const file = new File(options.config) const dir = file.dirname() - const config = (await import(pathToFileURL(options.config))).default + const config = (await import(pathToFileURL(options.config).toString())).default const outputFilepath = options.output || `${dir}/${config.site}.channels.xml` let channels = new Collection() diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index a0d63037..c38b22b9 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -16,6 +16,7 @@ program .option('-l, --lang ', 'Filter channels by language (ISO 639-2 code)') .option('-t, --timeout ', 'Override the default timeout for each request') .option('-d, --delay ', 'Override the default delay between request') + .option('-x, --proxy ', 'Use the specified proxy') .option( '--days ', 'Override the number of days for which the program will be loaded (defaults to the value from the site config)', @@ -42,6 +43,7 @@ export type GrabOptions = { lang?: string days?: number cron?: string + proxy?: string } const options: GrabOptions = program.opts() diff --git a/scripts/commands/sites/update.ts b/scripts/commands/sites/update.ts index 4b587df4..9ffc0b4b 100644 --- a/scripts/commands/sites/update.ts +++ b/scripts/commands/sites/update.ts @@ -40,19 +40,20 @@ async function main() { const data = new Collection() sites.forEach((site: Site) => { data.add([ - `${site.domain}`, - `${site.totalChannels} / ${site.markedChannels}`, - site.getStatus().emoji, - site.getIssues().all().join(', ') + { value: `${site.domain}` }, + { value: site.totalChannels, align: 'right' }, + { value: site.markedChannels, align: 'right' }, + { value: site.getStatus().emoji, align: 'center' }, + { value: site.getIssues().all().join(', ') } ]) }) logger.info('updating sites.md...') const table = new HTMLTable(data.all(), [ - { name: 'Site' }, - { name: 'Channels *', align: 'center' }, - { name: 'Status' }, - { name: 'Notes' } + { name: 'Site', align: 'left' }, + { name: 'Channels
(total / with xmltv-id)', colspan: 2, align: 'left' }, + { name: 'Status', align: 'left' }, + { name: 'Notes', align: 'left' } ]) const rootStorage = new Storage(ROOT_DIR) const sitesTemplate = await new Storage().load('scripts/templates/_sites.md') diff --git a/scripts/core/configLoader.ts b/scripts/core/configLoader.ts index 36c93d20..1beb3703 100644 --- a/scripts/core/configLoader.ts +++ b/scripts/core/configLoader.ts @@ -7,7 +7,7 @@ export class ConfigLoader { const fileUrl = pathToFileURL(filepath).toString() const config = (await import(fileUrl)).default const defaultConfig = { - days: 2, + days: 1, delay: 0, output: 'guide.xml', request: { diff --git a/scripts/core/grabber.ts b/scripts/core/grabber.ts index 4bed77a3..2248cbc5 100644 --- a/scripts/core/grabber.ts +++ b/scripts/core/grabber.ts @@ -1,8 +1,9 @@ import { EPGGrabber, GrabCallbackData, EPGGrabberMock, SiteConfig, Channel } from 'epg-grabber' import { Logger, Collection } from '@freearhey/core' -import { Queue } from './' +import { Queue, ProxyParser } from './' import { GrabOptions } from '../commands/epg/grab' import { TaskQueue, PromisyClass } from 'cwait' +import { SocksProxyAgent } from 'socks-proxy-agent' type GrabberProps = { logger: Logger @@ -14,6 +15,7 @@ export class Grabber { logger: Logger queue: Queue options: GrabOptions + grabber: EPGGrabber | EPGGrabberMock constructor({ logger, queue, options }: GrabberProps) { this.logger = logger @@ -23,6 +25,7 @@ export class Grabber { } async grab(): Promise<{ channels: Collection; programs: Collection }> { + const proxyParser = new ProxyParser() const taskQueue = new TaskQueue(Promise as PromisyClass, this.options.maxConnections) const total = this.queue.size() @@ -49,6 +52,24 @@ export class Grabber { config.delay = delay } + if (this.options.proxy !== undefined) { + const proxy = proxyParser.parse(this.options.proxy) + + if ( + proxy.protocol && + ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) + ) { + const socksProxyAgent = new SocksProxyAgent(this.options.proxy) + + config.request = { + ...config.request, + ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } + } + } else { + config.request = { ...config.request, ...{ proxy } } + } + } + const _programs = await this.grabber.grab( channel, date, diff --git a/scripts/core/htmlTable.ts b/scripts/core/htmlTable.ts index 72d6bd8d..144ba01b 100644 --- a/scripts/core/htmlTable.ts +++ b/scripts/core/htmlTable.ts @@ -2,9 +2,15 @@ type Column = { name: string nowrap?: boolean align?: string + colspan?: number } -type DataItem = string[] +type DataItem = { + value: string + nowrap?: boolean + align?: string + colspan?: number +}[] export class HTMLTable { data: DataItem[] @@ -20,20 +26,23 @@ export class HTMLTable { output += ' \r\n ' for (const column of this.columns) { - output += `${column.name}` + const nowrap = column.nowrap ? ' nowrap' : '' + const align = column.align ? ` align="${column.align}"` : '' + const colspan = column.colspan ? ` colspan="${column.colspan}"` : '' + + output += `${column.name}` } output += '\r\n \r\n' output += ' \r\n' - for (const item of this.data) { + for (const row of this.data) { output += ' ' - let i = 0 - for (const prop in item) { - const column = this.columns[i] - const nowrap = column.nowrap ? ' nowrap' : '' - const align = column.align ? ` align="${column.align}"` : '' - output += `${item[prop]}` - i++ + for (const item of row) { + const nowrap = item.nowrap ? ' nowrap' : '' + const align = item.align ? ` align="${item.align}"` : '' + const colspan = item.colspan ? ` colspan="${item.colspan}"` : '' + + output += `${item.value}` } output += '\r\n' } diff --git a/scripts/core/index.ts b/scripts/core/index.ts index 2c7309ed..db3e75a5 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -13,3 +13,4 @@ export * from './queueCreator' export * from './issueLoader' export * from './issueParser' export * from './htmlTable' +export * from './proxyParser' diff --git a/scripts/core/proxyParser.ts b/scripts/core/proxyParser.ts new file mode 100644 index 00000000..244290d5 --- /dev/null +++ b/scripts/core/proxyParser.ts @@ -0,0 +1,27 @@ +import { URL } from 'node:url' + +type ProxyParserResult = { + protocol: string | null + auth: { + username: string | null + password: string | null + } + host: string + port: number | null +} + +export class ProxyParser { + parse(_url: string): ProxyParserResult { + const parsed = new URL(_url) + + return { + protocol: parsed.protocol.replace(':', '') || null, + auth: { + username: parsed.username || null, + password: parsed.password || null + }, + host: parsed.hostname, + port: parsed.port ? parseInt(parsed.port) : null + } + } +} diff --git a/scripts/templates/_sites.md b/scripts/templates/_sites.md index a4c4b906..46191a4e 100644 --- a/scripts/templates/_sites.md +++ b/scripts/templates/_sites.md @@ -1,5 +1,3 @@ # Sites -\* Total number of channels / with a valid `xmltv-id` - _TABLE_ diff --git a/scripts/templates/_test.js b/scripts/templates/_test.js index 2bb4cebb..6375d7e7 100644 --- a/scripts/templates/_test.js +++ b/scripts/templates/_test.js @@ -6,7 +6,7 @@ dayjs.extend(customParseFormat) dayjs.extend(utc) const date = dayjs.utc('2025-01-12', 'YYYY-MM-DD').startOf('d') -const channel = { site_id: 'bbc1', xmltv_id: 'BBCOne.uk' } +const channel = { site_id: 'bbc1' } it('can generate valid url', () => { expect(url({ channel, date })).toBe('https://example.com/api/bbc1/2025-01-12') @@ -32,11 +32,7 @@ it('can parse response', () => { }) it('can handle empty guide', () => { - const result = parser({ - date, - channel, - content: '' - }) + const results = parser({ content: '' }) - expect(result).toMatchObject([]) + expect(results).toMatchObject([]) }) diff --git a/tests/__data__/expected/SITES.md b/tests/__data__/expected/SITES.md index 5f8597e3..4b99e838 100644 --- a/tests/__data__/expected/SITES.md +++ b/tests/__data__/expected/SITES.md @@ -1,14 +1,12 @@ # Sites -\* Total number of channels / with a valid `xmltv-id` - - + - - - + + +
SiteChannels *StatusNotes
SiteChannels
(total / with xmltv-id)		StatusNotes
iltalehti.fi142 / 44🟡https://github.com/iptv-org/epg/issues/2396
indihometv.com130 / 124🟢
kan.org.il6 / 6🔴https://github.com/iptv-org/epg/issues/2273
iltalehti.fi14244🟡https://github.com/iptv-org/epg/issues/2396
indihometv.com130124🟢
kan.org.il66🔴https://github.com/iptv-org/epg/issues/2273
diff --git a/tests/__data__/expected/custom_guide.xml b/tests/__data__/expected/custom_guide.xml index 41efdac8..a57b3322 100644 --- a/tests/__data__/expected/custom_guide.xml +++ b/tests/__data__/expected/custom_guide.xml @@ -9,9 +9,6 @@ Programme1 (example2.com) Program1 (example.com) Programme1 (example.com) -Programme1 (example2.com) Program1 (example2.com) -Program1 (example2.com) Program1 (example2.com) -Program1 (example2.com) \ No newline at end of file diff --git a/tests/__data__/expected/guide.xml b/tests/__data__/expected/guide.xml index 3de58428..66da48ee 100644 --- a/tests/__data__/expected/guide.xml +++ b/tests/__data__/expected/guide.xml @@ -9,9 +9,6 @@ Programme1 (example2.com) Program1 (example.com) Programme1 (example.com) -Programme1 (example2.com) Program1 (example2.com) -Program1 (example2.com) Program1 (example2.com) -Program1 (example2.com) \ No newline at end of file diff --git a/tests/__data__/expected/guide.xml.gz b/tests/__data__/expected/guide.xml.gz index e634dcdf..fb9654e9 100644 Binary files a/tests/__data__/expected/guide.xml.gz and b/tests/__data__/expected/guide.xml.gz differ diff --git a/tests/__data__/expected/sites/sites-init/example.com.test.js b/tests/__data__/expected/sites/sites-init/example.com.test.js index 37d2d7b3..f462b6ef 100644 --- a/tests/__data__/expected/sites/sites-init/example.com.test.js +++ b/tests/__data__/expected/sites/sites-init/example.com.test.js @@ -6,7 +6,7 @@ dayjs.extend(customParseFormat) dayjs.extend(utc) const date = dayjs.utc('2025-01-12', 'YYYY-MM-DD').startOf('d') -const channel = { site_id: 'bbc1', xmltv_id: 'BBCOne.uk' } +const channel = { site_id: 'bbc1' } it('can generate valid url', () => { expect(url({ channel, date })).toBe('https://example.com/api/bbc1/2025-01-12') @@ -32,11 +32,7 @@ it('can parse response', () => { }) it('can handle empty guide', () => { - const result = parser({ - date, - channel, - content: '' - }) + const results = parser({ content: '' }) - expect(result).toMatchObject([]) + expect(results).toMatchObject([]) }) diff --git a/tests/commands/epg/grab.test.ts b/tests/commands/epg/grab.test.ts index 8d40f16f..648d5057 100644 --- a/tests/commands/epg/grab.test.ts +++ b/tests/commands/epg/grab.test.ts @@ -97,6 +97,30 @@ describe('epg:grab', () => { expect(stdout).toContain('ERR: Connection timeout') }) + + it('can grab epg via https proxy', () => { + const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=https://bob:123456@proxy.com:1234 --output="${path.resolve( + 'tests/__data__/output/guide.xml' + )}"` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guide.xml')).toEqual( + content('tests/__data__/expected/guide2.xml') + ) + }) + + it('can grab epg via socks5 proxy', () => { + const cmd = `${ENV_VAR} npm run grab --- --site=example.com --proxy=socks5://bob:123456@proxy.com:1234 --output="${path.resolve( + 'tests/__data__/output/guide.xml' + )}"` + const stdout = execSync(cmd, { encoding: 'utf8' }) + if (process.env.DEBUG === 'true') console.log(cmd, stdout) + + expect(content('tests/__data__/output/guide.xml')).toEqual( + content('tests/__data__/expected/guide2.xml') + ) + }) }) function content(filepath: string) { diff --git a/yarn.lock b/yarn.lock index 1ba3d3bd..15726e5c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -396,10 +396,10 @@ dependencies: levn "^0.4.1" -"@freearhey/core@^0.5.0": - version "0.5.0" - resolved "https://registry.npmjs.org/@freearhey/core/-/core-0.5.0.tgz" - integrity sha512-FcA5Pv9RvFvLYAwNmD/2vlSR49Rx+kihJ+xbIUgIACHY6lBUptfbNznm00DQoUyWRJG/cfT3dkYCwIxSUsdP+w== +"@freearhey/core@^0.5.1": + version "0.5.1" + resolved "https://registry.npmjs.org/@freearhey/core/-/core-0.5.1.tgz" + integrity sha512-UDKIOyrtcUXaiAeIvjNFTI6DlempiOQaRB83CqHNF1VPRHNBiNhGhERWyInHE2cjLp/cc0CA/IykOYS39kBK7Q== dependencies: "@types/fs-extra" "^11.0.2" "@types/lodash" "^4.14.198" @@ -741,10 +741,10 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@ntlab/sfetch@^1.0.0": - version "1.0.0" - resolved "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz" - integrity sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ== +"@ntlab/sfetch@^1.2.0": + version "1.2.0" + resolved "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.2.0.tgz" + integrity sha512-9SE4NnqWo8l6mG0rnAkgng6ozSamIpF3EC+GOTQGGa6eAC0tNJvzrylMz6YRjjEGH6mOfn7ZBAuKj5WIZUul6A== dependencies: axios "^1.7.9" @@ -1271,7 +1271,7 @@ acorn-walk@^8.1.1: resolved "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz" integrity sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA== -agent-base@^7.1.3: +agent-base@^7.1.2, agent-base@^7.1.3: version "7.1.3" resolved "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz" integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw== @@ -2158,10 +2158,10 @@ entities@^4.2.0, entities@^4.3.0, entities@^4.4.0: resolved "https://registry.npmjs.org/entities/-/entities-4.4.0.tgz" integrity sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA== -epg-grabber@^0.37.4: - version "0.37.4" - resolved "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.37.4.tgz" - integrity sha512-PS104bH9tHRa9kivSwx47AKMkfHwKy51XQTx+GO6sIXvIp2Z4LBpwMEXGcfPoAsdIGxgs2Wrl0dZ/QGL+7x6YQ== +epg-grabber@^0.38.0: + version "0.38.0" + resolved "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.38.0.tgz" + integrity sha512-jbwTgi6G7e+zrb2oNC0C7mcQYoRkFnvhXCurexeICaEy4avRB6WS5rD/yfqYoiqaXOM3x1BNBpCKFYoS7Ob5YA== dependencies: axios "^1.6.1" axios-cache-interceptor "^0.10.3" @@ -2176,6 +2176,7 @@ epg-grabber@^0.37.4: http-cookie-agent "^6.0.8" lodash "^4.17.21" node-gzip "^1.1.2" + socks-proxy-agent "^8.0.5" tough-cookie "^5.0.0" winston "^3.3.3" xml-js "^1.6.11" @@ -2835,6 +2836,14 @@ inquirer@^8.2.6: through "^2.3.6" wrap-ansi "^6.0.1" +ip-address@^9.0.5: + version "9.0.5" + resolved "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz" + integrity sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g== + dependencies: + jsbn "1.1.0" + sprintf-js "^1.1.3" + is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz" @@ -3373,6 +3382,11 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +jsbn@1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz" + integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A== + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz" @@ -4393,6 +4407,28 @@ slash@^3.0.0: resolved "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== +smart-buffer@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz" + integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== + +socks-proxy-agent@^8.0.5: + version "8.0.5" + resolved "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz" + integrity sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw== + dependencies: + agent-base "^7.1.2" + debug "^4.3.4" + socks "^2.8.3" + +socks@^2.8.3: + version "2.8.3" + resolved "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz" + integrity sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw== + dependencies: + ip-address "^9.0.5" + smart-buffer "^4.2.0" + source-map-support@0.5.13: version "0.5.13" resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz" @@ -4406,6 +4442,11 @@ source-map@^0.6.0, source-map@^0.6.1: resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +sprintf-js@^1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz" + integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz"