diff --git a/package.json b/package.json index e388af72..f54438de 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "programs:load": "node scripts/commands/programs/load.js", "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", - "guides:update_legacy": "node scripts/commands/guides/update_legacy.js", "api:load": "./scripts/commands/api/load.sh", "api:update": "node scripts/commands/api/update.js", "readme:update": "node scripts/commands/readme/update.js", diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index 00b0a62a..9a7e6621 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -1,7 +1,7 @@ #!/bin/bash -mkdir -p scripts/data -curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json -curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json -curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json +mkdir -p scripts/data +curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json +curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json +curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file diff --git a/scripts/commands/guides/update.js b/scripts/commands/guides/update.js index e0e09b17..daecafd9 100644 --- a/scripts/commands/guides/update.js +++ b/scripts/commands/guides/update.js @@ -9,13 +9,19 @@ const CURR_DATE = process.env.CURR_DATE || new Date() const logPath = `${LOGS_DIR}/guides/update.log` +let guides = [] +let db_programs = [] + async function main() { logger.info(`starting...`) - logger.info('loading API data...') + logger.info('loading data/countries.json...') await api.countries.load() + logger.info('loading data/channels.json...') await api.channels.load() + logger.info('loading data/regions.json...') await api.regions.load() + logger.info('loading data/subdivisions.json...') await api.subdivisions.load() let countries = await api.countries.all() @@ -31,7 +37,7 @@ async function main() { logger.info('loading database/programs.db...') await db.programs.load() - let db_programs = await db.programs.find({}) + db_programs = await db.programs.find({}) db_programs = db_programs .map(p => { if (p.titles.length) { @@ -45,9 +51,6 @@ async function main() { .filter(Boolean) logger.info(`found ${db_programs.length} programs`) - logger.info(`creating ${logPath}...`) - await file.create(logPath) - for (let country of countries) { let countryBroadcastCode = `c/${country.code}` let countryRegions = api_regions @@ -122,17 +125,20 @@ async function main() { await file.create(jsonFilepath, JSON.stringify({ channels, programs })) for (let channel of channels) { - let result = { + guides.push({ country: country.code, lang: channel.lang, site: channel.site, channel: channel.id, filename - } - - await file.append(logPath, JSON.stringify(result) + '\r\n') + }) } } + + logger.info(`creating ${logPath}...`) + await file.create(logPath, guides.map(g => JSON.stringify(g)).join('\r\n')) + + await makeReport() } main() @@ -176,3 +182,20 @@ function calcScore(program) { return score } + +async function makeReport() { + const errors = [] + + let programs = _.uniqBy(db_programs, p => p.site + p.channel) + for (let program of programs) { + if (!guides.find(g => g.channel === program.channel)) { + const channel = await api.channels.find({ id: program.channel }) + errors.push({ type: 'no_guide', ...program, ...channel }) + } + } + + console.log() + logger.info(`report:`) + console.table(errors, ['type', 'site', 'lang', 'channel', 'broadcast_area', 'languages']) + logger.error(`found ${errors.length} error(s)`) +} diff --git a/tests/__data__/expected/logs/guides/update.log b/tests/__data__/expected/logs/guides/update.log index fdba1db3..701a3e2a 100644 --- a/tests/__data__/expected/logs/guides/update.log +++ b/tests/__data__/expected/logs/guides/update.log @@ -1,2 +1,2 @@ {"country":"DK","lang":"da","site":"allente.se","channel":"6eren.dk","filename":"dk"} -{"country":"UK","lang":"en","site":"virginmedia.com","channel":"BBCNews.uk","filename":"uk"} +{"country":"UK","lang":"en","site":"virginmedia.com","channel":"BBCNews.uk","filename":"uk"} \ No newline at end of file diff --git a/tests/__data__/input/data/channels.json b/tests/__data__/input/data/channels.json index 46da2402..28255c4a 100644 --- a/tests/__data__/input/data/channels.json +++ b/tests/__data__/input/data/channels.json @@ -53,5 +53,22 @@ "logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png" }, {"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"}, - {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"} + {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"}, + { + "id": "CNN.us", + "name": "CNN", + "network": null, + "country": "US", + "subdivision": null, + "city": null, + "broadcast_area": [ + "c/US" + ], + "languages": [ + "eng" + ], + "categories": [], + "is_nsfw": false, + "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png" + } ] \ No newline at end of file diff --git a/tests/__data__/input/database/update-guides/programs.db b/tests/__data__/input/database/update-guides/programs.db index 5e78bd6b..e72a3252 100644 --- a/tests/__data__/input/database/update-guides/programs.db +++ b/tests/__data__/input/database/update-guides/programs.db @@ -1,4 +1,5 @@ {"site":"allente.se","channel":"6eren.dk","titles":[{"value":"Diners, Drive-Ins and Dives","lang":"da"}],"sub_titles":[],"descriptions":[{"value":"Underholdning","lang":"da"}],"icon":{"src":"https://viasatps.api.comspace.se/PS/channeldate/image/viasat.ps/487/2022-10-24/se.cs.6eren.event.B_0254194276971024040000.jpg?size=2560x1440"},"episodeNumbers":[{"system":"xmltv_ns","value":"23.5.0/1"},{"system":"onscreen","value":"S24E06"}],"date":null,"start":1666584000000,"stop":1666585500000,"urls":[],"ratings":[],"categories":[{"value":"series","lang":"da"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"f6cxSM73LfZ8TdYz","_id":"HxsrTRTFj1z05TAK"} {"site":"allente.se","channel":"6eren.dk","titles":[{"value":"Diners, Drive-Ins and Dives","lang":"da"}],"sub_titles":[],"descriptions":[{"value":"Underholdning","lang":"da"}],"icon":{"src":"https://viasatps.api.comspace.se/PS/channeldate/image/viasat.ps/487/2022-10-24/se.cs.6eren.event.B_0254194276971024040000.jpg?size=2560x1440"},"episodeNumbers":[{"system":"xmltv_ns","value":"23.5.0/1"},{"system":"onscreen","value":"S24E06"}],"date":null,"start":1666584000000,"stop":1666585500000,"urls":[],"ratings":[],"categories":[{"value":"series","lang":"da"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"f6cxTM73LfZ8TdYz","_id":"HxsrBRTFj1z05TAK"} {"site":"virginmedia.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news, followed by weather.","lang":"en"}],"icon":{"src":""},"episodeNumbers":[{"system":"xmltv_ns","value":"96839999.145799123.0/1"},{"system":"onscreen","value":"S96840000E145799124"}],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[{"value":"News","lang":"en"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"lNXh3lBnb4n1DBzs","_id":"quKCInjZV98xFUAf"} -{"site":"sky.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news from the BBC. [S,SL]","lang":"en"}],"icon":{"src":"http://epgstatic.sky.com/epgdata/1.0/paimage/46/1/lisa/5.2.2/linear/channel/ca247bc8-6be0-48f9-88d1-865f87f7680e/2011"},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"36duI92slofEXlSa","_id":"ryzed0Bqda1QtE7i"} \ No newline at end of file +{"site":"sky.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news from the BBC. [S,SL]","lang":"en"}],"icon":{"src":"http://epgstatic.sky.com/epgdata/1.0/paimage/46/1/lisa/5.2.2/linear/channel/ca247bc8-6be0-48f9-88d1-865f87f7680e/2011"},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"36duI92slofEXlSa","_id":"ryzed0Bqda1QtE7i"} +{"site":"sky.com","channel":"CNN.us","titles":[{"value":"French title","lang":"fr"}],"sub_titles":[],"descriptions":[],"icon":{},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"37duI92slofEXlSa","_id":"rxzed0Bqda1QtE7i"} \ No newline at end of file diff --git a/tests/commands/guides/update.test.js b/tests/commands/guides/update.test.js index fe47a2c9..eac2e69c 100644 --- a/tests/commands/guides/update.test.js +++ b/tests/commands/guides/update.test.js @@ -9,14 +9,44 @@ beforeEach(() => { 'tests/__data__/input/database/update-guides/programs.db', 'tests/__data__/output/programs.db' ) +}) +it('can generate /guides', () => { const stdout = execSync( 'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output CURR_DATE=2022-10-20 npm run guides:update', { encoding: 'utf8' } ) -}) -it('can generate /guides', () => { + expect(stdout).toBe( + ` +> guides:update +> NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js + +starting... +loading data/countries.json... +loading data/channels.json... +loading data/regions.json... +loading data/subdivisions.json... +loading database/programs.db... +found 5 programs +creating tests/__data__/output/guides/dk.xml... +creating tests/__data__/output/guides/dk.xml.gz... +creating tests/__data__/output/guides/dk.json... +creating tests/__data__/output/guides/uk.xml... +creating tests/__data__/output/guides/uk.xml.gz... +creating tests/__data__/output/guides/uk.json... +creating tests/__data__/output/logs/guides/update.log... + +report: +┌─────────┬────────────┬───────────┬──────┬──────────┬────────────────┬───────────┐ +│ (index) │ type │ site │ lang │ channel │ broadcast_area │ languages │ +├─────────┼────────────┼───────────┼──────┼──────────┼────────────────┼───────────┤ +│ 0 │ 'no_guide' │ 'sky.com' │ 'fr' │ 'CNN.us' │ [ 'c/US' ] │ [ 'eng' ] │ +└─────────┴────────────┴───────────┴──────┴──────────┴────────────────┴───────────┘ +found 1 error(s) +` + ) + const uncompressed = glob .sync('tests/__data__/expected/guides/*.xml') .map(f => f.replace('tests/__data__/expected/', ''))