From f21ae7f6462e1b93d8af1a4a5166b7d1585dbc6e Mon Sep 17 00:00:00 2001 From: RevGear <95308545+RevGear@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:08:48 +0000 Subject: [PATCH 01/16] Validate Guides --- package.json | 1 + scripts/commands/api/load.sh | 11 ++++---- scripts/commands/guides/validate.js | 41 +++++++++++++++++++++++++++++ scripts/core/api.js | 1 + 4 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 scripts/commands/guides/validate.js diff --git a/package.json b/package.json index e388af72..0a270759 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", "guides:update_legacy": "node scripts/commands/guides/update_legacy.js", + "guides:validate": "node scripts/commands/guides/validate.js", "api:load": "./scripts/commands/api/load.sh", "api:update": "node scripts/commands/api/update.js", "readme:update": "node scripts/commands/readme/update.js", diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index 00b0a62a..6915c456 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -1,7 +1,8 @@ #!/bin/bash -mkdir -p scripts/data -curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json -curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json -curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json -curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file +mkdir -p scripts/data +curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json +curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json +curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json +curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json +curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json \ No newline at end of file diff --git a/scripts/commands/guides/validate.js b/scripts/commands/guides/validate.js new file mode 100644 index 00000000..bfd045eb --- /dev/null +++ b/scripts/commands/guides/validate.js @@ -0,0 +1,41 @@ +const { db, logger, api } = require('../../core') +const chalk = require('chalk') + +async function main() { + await api.channels.load() + await api.guides.load() + logger.info('loading database/programs.db...') + await db.programs.load() + let db_programs = await db.programs.find({}) + logger.info(`found ${db_programs.length} programs`) + + const stats = { + files: 0, + errors: 0 + } + const errors = [] + let api_channels = await api.channels.all() + + api_channels.forEach(channel => { + let programs = db_programs.filter(p => p.channel == channel.id) + if (programs.length > 0) { + if (!api.guides.find({ channel: channel.id })) { + errors.push({ type: 'no_guide', xmltv_id: channel.id, ...channel }) + stats.errors++ + } + } + }) + + if (errors.length) { + console.table(errors, ['type', 'xmltv_id', 'name', 'country']) + console.log() + stats.files++ + } + + if (stats.errors > 0) { + logger.error(chalk.red(`${stats.errors} error(s)`)) + process.exit(1) + } +} + +main() diff --git a/scripts/core/api.js b/scripts/core/api.js index e18e4da3..9f96a2cd 100644 --- a/scripts/core/api.js +++ b/scripts/core/api.js @@ -28,5 +28,6 @@ api.channels = new API(`${DATA_DIR}/channels.json`) api.regions = new API(`${DATA_DIR}/regions.json`) api.countries = new API(`${DATA_DIR}/countries.json`) api.subdivisions = new API(`${DATA_DIR}/subdivisions.json`) +api.guides = new API(`${DATA_DIR}/guides.json`) module.exports = api From 1dd2a55cfd51b604f571492734b40fcec833b602 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 09:59:21 +0300 Subject: [PATCH 02/16] Update validate.js --- scripts/commands/guides/validate.js | 64 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/scripts/commands/guides/validate.js b/scripts/commands/guides/validate.js index bfd045eb..bad9d188 100644 --- a/scripts/commands/guides/validate.js +++ b/scripts/commands/guides/validate.js @@ -1,41 +1,43 @@ -const { db, logger, api } = require('../../core') +const { db, logger, api, parser } = require('../../core') const chalk = require('chalk') +const _ = require('lodash') + +const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' async function main() { - await api.channels.load() - await api.guides.load() - logger.info('loading database/programs.db...') - await db.programs.load() - let db_programs = await db.programs.find({}) - logger.info(`found ${db_programs.length} programs`) + logger.info('loading data/channels.json...') + await api.channels.load() - const stats = { - files: 0, - errors: 0 + const logPath = `${LOGS_DIR}/guides/update.log` + logger.info(`loading ${logPath}...`) + const guides = await parser.parseLogs(logPath) + + logger.info('loading database/programs.db...') + await db.programs.load() + let db_programs = await db.programs.find({}) + logger.info(`found ${db_programs.length} programs`) + + const errors = [] + + let programs = db_programs.map(p => ({ + site: p.site, + xmltv_id: p.channel, + lang: p.titles[0].lang + })) + programs = _.uniqBy(programs, p => p.site + p.xmltv_id) + for (let program of programs) { + if (!guides.find(g => g.channel === program.xmltv_id)) { + const channel = await api.channels.find({ id: program.xmltv_id }) + errors.push({ type: 'no_guide', ...program, ...channel }) } - const errors = [] - let api_channels = await api.channels.all() + } - api_channels.forEach(channel => { - let programs = db_programs.filter(p => p.channel == channel.id) - if (programs.length > 0) { - if (!api.guides.find({ channel: channel.id })) { - errors.push({ type: 'no_guide', xmltv_id: channel.id, ...channel }) - stats.errors++ - } - } - }) + if (errors.length) { + console.table(errors, ['type', 'site', 'lang', 'xmltv_id', 'broadcast_area', 'languages']) + console.log() - if (errors.length) { - console.table(errors, ['type', 'xmltv_id', 'name', 'country']) - console.log() - stats.files++ - } - - if (stats.errors > 0) { - logger.error(chalk.red(`${stats.errors} error(s)`)) - process.exit(1) - } + logger.error(chalk.red(`${errors.length} error(s)`)) + } } main() From f43944793fe96085dbc8d88640a52027a15669c4 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 09:59:29 +0300 Subject: [PATCH 03/16] Create validate.test.js --- tests/commands/guides/validate.test.js | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/commands/guides/validate.test.js diff --git a/tests/commands/guides/validate.test.js b/tests/commands/guides/validate.test.js new file mode 100644 index 00000000..370a0a08 --- /dev/null +++ b/tests/commands/guides/validate.test.js @@ -0,0 +1,37 @@ +const { execSync } = require('child_process') +const fs = require('fs-extra') + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + + fs.copyFileSync( + 'tests/__data__/input/database/update-guides/programs.db', + 'tests/__data__/output/programs.db' + ) +}) + +it('will show a message if the channel is not in the guide', () => { + const stdout = execSync( + 'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/input/logs DATA_DIR=tests/__data__/input/data npm run guides:validate', + { + encoding: 'utf8' + } + ) + expect(stdout).toBe( + `\n> guides:validate\n> node scripts/commands/guides/validate.js + +loading data/channels.json... +loading tests/__data__/input/logs/guides/update.log... +loading database/programs.db... +found 4 programs +┌─────────┬────────────┬───────────────────┬──────┬──────────────┬────────────────┬───────────┐ +│ (index) │ type │ site │ lang │ xmltv_id │ broadcast_area │ languages │ +├─────────┼────────────┼───────────────────┼──────┼──────────────┼────────────────┼───────────┤ +│ 0 │ 'no_guide' │ 'virginmedia.com' │ 'en' │ 'BBCNews.uk' │ [ 'c/UK' ] │ [ 'eng' ] │ +│ 1 │ 'no_guide' │ 'sky.com' │ 'en' │ 'BBCNews.uk' │ [ 'c/UK' ] │ [ 'eng' ] │ +└─────────┴────────────┴───────────────────┴──────┴──────────────┴────────────────┴───────────┘ + +2 error(s) +` + ) +}) From 25d6af13a26dac384d5e21dcf4aa5cd81bfcd9f6 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 09:59:56 +0300 Subject: [PATCH 04/16] Update _update.yml --- .github/workflows/_update.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_update.yml b/.github/workflows/_update.yml index 8ae5b884..7e74fd85 100644 --- a/.github/workflows/_update.yml +++ b/.github/workflows/_update.yml @@ -29,6 +29,7 @@ jobs: - if: ${{ !env.ACT }} run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run programs:load - run: npm run guides:update + - run: npm run guides:validate - run: npm run api:update - run: npm run readme:update - run: npm run status:update From e0937d4acd40ee2ae253bdc695c943629cac5850 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 09:59:59 +0300 Subject: [PATCH 05/16] Update load.sh --- scripts/commands/api/load.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index 6915c456..9a7e6621 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -4,5 +4,4 @@ mkdir -p scripts/data curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json -curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json -curl -L -o scripts/data/guides.json https://iptv-org.github.io/api/guides.json \ No newline at end of file +curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file From 0750fa820d45c79745410306a41a523e5f7bde04 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 10:00:02 +0300 Subject: [PATCH 06/16] Update api.js --- scripts/core/api.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/core/api.js b/scripts/core/api.js index 9f96a2cd..e18e4da3 100644 --- a/scripts/core/api.js +++ b/scripts/core/api.js @@ -28,6 +28,5 @@ api.channels = new API(`${DATA_DIR}/channels.json`) api.regions = new API(`${DATA_DIR}/regions.json`) api.countries = new API(`${DATA_DIR}/countries.json`) api.subdivisions = new API(`${DATA_DIR}/subdivisions.json`) -api.guides = new API(`${DATA_DIR}/guides.json`) module.exports = api From fb682ccdc027282d795fddaa24ea9f153f69eaa9 Mon Sep 17 00:00:00 2001 From: "iptv-bot[bot]" <84861620+iptv-bot[bot]@users.noreply.github.com> Date: Sun, 8 Jan 2023 12:26:06 +0300 Subject: [PATCH 07/16] [Bot] Daily update (#1626) * [Bot] Update README.md * [Bot] Update STATUS.md Co-authored-by: iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com> --- README.md | 192 +++++++++++++++++++++++++++--------------------------- 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/README.md b/README.md index b979e2ea..6d5cf1c4 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇦🇫 Afghanistan72https://iptv-org.github.io/epg/guides/af.xml 🇦🇱 Albania101https://iptv-org.github.io/epg/guides/al.xml - 🇩🇿 Algeria123https://iptv-org.github.io/epg/guides/dz.xml + 🇩🇿 Algeria122https://iptv-org.github.io/epg/guides/dz.xml 🇦🇸 American Samoa67https://iptv-org.github.io/epg/guides/as.xml 🇦🇩 Andorra91https://iptv-org.github.io/epg/guides/ad.xml - 🇦🇴 Angola120https://iptv-org.github.io/epg/guides/ao.xml + 🇦🇴 Angola119https://iptv-org.github.io/epg/guides/ao.xml 🇦🇮 Anguilla68https://iptv-org.github.io/epg/guides/ai.xml 🇦🇶 Antarctica56https://iptv-org.github.io/epg/guides/aq.xml 🇦🇬 Antigua and Barbuda68https://iptv-org.github.io/epg/guides/ag.xml @@ -30,73 +30,73 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇦🇹 Austria155https://iptv-org.github.io/epg/guides/at.xml 🇦🇿 Azerbaijan103https://iptv-org.github.io/epg/guides/az.xml 🇧🇸 Bahamas76https://iptv-org.github.io/epg/guides/bs.xml - 🇧🇭 Bahrain91https://iptv-org.github.io/epg/guides/bh.xml + 🇧🇭 Bahrain89https://iptv-org.github.io/epg/guides/bh.xml 🇧🇩 Bangladesh76https://iptv-org.github.io/epg/guides/bd.xml 🇧🇧 Barbados74https://iptv-org.github.io/epg/guides/bb.xml 🇧🇾 Belarus117https://iptv-org.github.io/epg/guides/by.xml 🇧🇪 Belgium198https://iptv-org.github.io/epg/guides/be.xml 🇧🇿 Belize92https://iptv-org.github.io/epg/guides/bz.xml - 🇧🇯 Benin118https://iptv-org.github.io/epg/guides/bj.xml + 🇧🇯 Benin117https://iptv-org.github.io/epg/guides/bj.xml 🇧🇲 Bermuda62https://iptv-org.github.io/epg/guides/bm.xml 🇧🇹 Bhutan73https://iptv-org.github.io/epg/guides/bt.xml 🇧🇴 Bolivia211https://iptv-org.github.io/epg/guides/bo.xml 🇧🇶 Bonaire60https://iptv-org.github.io/epg/guides/bq.xml 🇧🇦 Bosnia and Herzegovina124https://iptv-org.github.io/epg/guides/ba.xml - 🇧🇼 Botswana106https://iptv-org.github.io/epg/guides/bw.xml + 🇧🇼 Botswana105https://iptv-org.github.io/epg/guides/bw.xml 🇧🇻 Bouvet Island58https://iptv-org.github.io/epg/guides/bv.xml - 🇧🇷 Brazil308https://iptv-org.github.io/epg/guides/br.xml + 🇧🇷 Brazil309https://iptv-org.github.io/epg/guides/br.xml 🇮🇴 British Indian Ocean Territory56https://iptv-org.github.io/epg/guides/io.xml 🇻🇬 British Virgin Islands68https://iptv-org.github.io/epg/guides/vg.xml - 🇧🇳 Brunei112https://iptv-org.github.io/epg/guides/bn.xml + 🇧🇳 Brunei114https://iptv-org.github.io/epg/guides/bn.xml 🇧🇬 Bulgaria159https://iptv-org.github.io/epg/guides/bg.xml - 🇧🇫 Burkina Faso123https://iptv-org.github.io/epg/guides/bf.xml - 🇧🇮 Burundi114https://iptv-org.github.io/epg/guides/bi.xml - 🇰🇭 Cambodia109https://iptv-org.github.io/epg/guides/kh.xml - 🇨🇲 Cameroon124https://iptv-org.github.io/epg/guides/cm.xml - 🇨🇦 Canada284https://iptv-org.github.io/epg/guides/ca.xml - 🇨🇻 Cape Verde117https://iptv-org.github.io/epg/guides/cv.xml + 🇧🇫 Burkina Faso122https://iptv-org.github.io/epg/guides/bf.xml + 🇧🇮 Burundi113https://iptv-org.github.io/epg/guides/bi.xml + 🇰🇭 Cambodia111https://iptv-org.github.io/epg/guides/kh.xml + 🇨🇲 Cameroon123https://iptv-org.github.io/epg/guides/cm.xml + 🇨🇦 Canada285https://iptv-org.github.io/epg/guides/ca.xml + 🇨🇻 Cape Verde116https://iptv-org.github.io/epg/guides/cv.xml 🇰🇾 Cayman Islands68https://iptv-org.github.io/epg/guides/ky.xml - 🇨🇫 Central African Republic113https://iptv-org.github.io/epg/guides/cf.xml - 🇹🇩 Chad116https://iptv-org.github.io/epg/guides/td.xml + 🇨🇫 Central African Republic112https://iptv-org.github.io/epg/guides/cf.xml + 🇹🇩 Chad115https://iptv-org.github.io/epg/guides/td.xml 🇨🇱 Chile240https://iptv-org.github.io/epg/guides/cl.xml - 🇨🇳 China188https://iptv-org.github.io/epg/guides/cn.xml + 🇨🇳 China187https://iptv-org.github.io/epg/guides/cn.xml 🇨🇽 Christmas Island56https://iptv-org.github.io/epg/guides/cx.xml 🇨🇨 Cocos (Keeling) Islands56https://iptv-org.github.io/epg/guides/cc.xml 🇨🇴 Colombia231https://iptv-org.github.io/epg/guides/co.xml - 🇰🇲 Comoros130https://iptv-org.github.io/epg/guides/km.xml + 🇰🇲 Comoros129https://iptv-org.github.io/epg/guides/km.xml 🇨🇰 Cook Islands67https://iptv-org.github.io/epg/guides/ck.xml 🇨🇷 Costa Rica204https://iptv-org.github.io/epg/guides/cr.xml - 🇭🇷 Croatia146https://iptv-org.github.io/epg/guides/hr.xml + 🇭🇷 Croatia147https://iptv-org.github.io/epg/guides/hr.xml 🇨🇺 Cuba203https://iptv-org.github.io/epg/guides/cu.xml 🇨🇼 Curacao72https://iptv-org.github.io/epg/guides/cw.xml - 🇨🇾 Cyprus122https://iptv-org.github.io/epg/guides/cy.xml + 🇨🇾 Cyprus120https://iptv-org.github.io/epg/guides/cy.xml 🇨🇿 Czech Republic207https://iptv-org.github.io/epg/guides/cz.xml - 🇨🇩 Democratic Republic of the Congo125https://iptv-org.github.io/epg/guides/cd.xml + 🇨🇩 Democratic Republic of the Congo124https://iptv-org.github.io/epg/guides/cd.xml 🇩🇰 Denmark120https://iptv-org.github.io/epg/guides/dk.xml - 🇩🇯 Djibouti134https://iptv-org.github.io/epg/guides/dj.xml + 🇩🇯 Djibouti133https://iptv-org.github.io/epg/guides/dj.xml 🇩🇲 Dominica68https://iptv-org.github.io/epg/guides/dm.xml 🇩🇴 Dominican Republic208https://iptv-org.github.io/epg/guides/do.xml - 🇹🇱 East Timor114https://iptv-org.github.io/epg/guides/tl.xml + 🇹🇱 East Timor116https://iptv-org.github.io/epg/guides/tl.xml 🇪🇨 Ecuador206https://iptv-org.github.io/epg/guides/ec.xml - 🇪🇬 Egypt158https://iptv-org.github.io/epg/guides/eg.xml + 🇪🇬 Egypt155https://iptv-org.github.io/epg/guides/eg.xml 🇸🇻 El Salvador207https://iptv-org.github.io/epg/guides/sv.xml - 🇬🇶 Equatorial Guinea128https://iptv-org.github.io/epg/guides/gq.xml - 🇪🇷 Eritrea106https://iptv-org.github.io/epg/guides/er.xml + 🇬🇶 Equatorial Guinea127https://iptv-org.github.io/epg/guides/gq.xml + 🇪🇷 Eritrea105https://iptv-org.github.io/epg/guides/er.xml 🇪🇪 Estonia120https://iptv-org.github.io/epg/guides/ee.xml - 🇪🇹 Ethiopia106https://iptv-org.github.io/epg/guides/et.xml + 🇪🇹 Ethiopia105https://iptv-org.github.io/epg/guides/et.xml 🇫🇰 Falkland Islands58https://iptv-org.github.io/epg/guides/fk.xml 🇫🇴 Faroe Islands61https://iptv-org.github.io/epg/guides/fo.xml 🇫🇯 Fiji67https://iptv-org.github.io/epg/guides/fj.xml - 🇫🇮 Finland131https://iptv-org.github.io/epg/guides/fi.xml - 🇫🇷 France384https://iptv-org.github.io/epg/guides/fr.xml + 🇫🇮 Finland130https://iptv-org.github.io/epg/guides/fi.xml + 🇫🇷 France374https://iptv-org.github.io/epg/guides/fr.xml 🇬🇫 French Guiana75https://iptv-org.github.io/epg/guides/gf.xml 🇵🇫 French Polynesia71https://iptv-org.github.io/epg/guides/pf.xml - 🇹🇫 French Southern Territories105https://iptv-org.github.io/epg/guides/tf.xml - 🇬🇦 Gabon117https://iptv-org.github.io/epg/guides/ga.xml - 🇬🇲 Gambia106https://iptv-org.github.io/epg/guides/gm.xml + 🇹🇫 French Southern Territories104https://iptv-org.github.io/epg/guides/tf.xml + 🇬🇦 Gabon116https://iptv-org.github.io/epg/guides/ga.xml + 🇬🇲 Gambia105https://iptv-org.github.io/epg/guides/gm.xml 🇬🇪 Georgia125https://iptv-org.github.io/epg/guides/ge.xml 🇩🇪 Germany280https://iptv-org.github.io/epg/guides/de.xml - 🇬🇭 Ghana109https://iptv-org.github.io/epg/guides/gh.xml + 🇬🇭 Ghana108https://iptv-org.github.io/epg/guides/gh.xml 🇬🇮 Gibraltar56https://iptv-org.github.io/epg/guides/gi.xml 🇬🇷 Greece166https://iptv-org.github.io/epg/guides/gr.xml 🇬🇱 Greenland65https://iptv-org.github.io/epg/guides/gl.xml @@ -105,55 +105,55 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇬🇺 Guam73https://iptv-org.github.io/epg/guides/gu.xml 🇬🇹 Guatemala206https://iptv-org.github.io/epg/guides/gt.xml 🇬🇬 Guernsey59https://iptv-org.github.io/epg/guides/gg.xml - 🇬🇳 Guinea119https://iptv-org.github.io/epg/guides/gn.xml - 🇬🇼 Guinea-Bissau113https://iptv-org.github.io/epg/guides/gw.xml + 🇬🇳 Guinea118https://iptv-org.github.io/epg/guides/gn.xml + 🇬🇼 Guinea-Bissau112https://iptv-org.github.io/epg/guides/gw.xml 🇬🇾 Guyana58https://iptv-org.github.io/epg/guides/gy.xml 🇭🇹 Haiti96https://iptv-org.github.io/epg/guides/ht.xml 🇭🇲 Heard Island and McDonald Islands56https://iptv-org.github.io/epg/guides/hm.xml 🇭🇳 Honduras209https://iptv-org.github.io/epg/guides/hn.xml 🇭🇰 Hong Kong155https://iptv-org.github.io/epg/guides/hk.xml 🇭🇺 Hungary212https://iptv-org.github.io/epg/guides/hu.xml - 🇮🇸 Iceland104https://iptv-org.github.io/epg/guides/is.xml + 🇮🇸 Iceland103https://iptv-org.github.io/epg/guides/is.xml 🇮🇳 India420https://iptv-org.github.io/epg/guides/in.xml - 🇮🇩 Indonesia211https://iptv-org.github.io/epg/guides/id.xml - 🇮🇷 Iran85https://iptv-org.github.io/epg/guides/ir.xml - 🇮🇶 Iraq94https://iptv-org.github.io/epg/guides/iq.xml + 🇮🇩 Indonesia213https://iptv-org.github.io/epg/guides/id.xml + 🇮🇷 Iran75https://iptv-org.github.io/epg/guides/ir.xml + 🇮🇶 Iraq92https://iptv-org.github.io/epg/guides/iq.xml 🇮🇪 Ireland158https://iptv-org.github.io/epg/guides/ie.xml 🇮🇲 Isle of Man56https://iptv-org.github.io/epg/guides/im.xml - 🇮🇱 Israel76https://iptv-org.github.io/epg/guides/il.xml + 🇮🇱 Israel74https://iptv-org.github.io/epg/guides/il.xml 🇮🇹 Italy251https://iptv-org.github.io/epg/guides/it.xml - 🇨🇮 Ivory Coast125https://iptv-org.github.io/epg/guides/ci.xml + 🇨🇮 Ivory Coast124https://iptv-org.github.io/epg/guides/ci.xml 🇯🇲 Jamaica70https://iptv-org.github.io/epg/guides/jm.xml 🇯🇵 Japan191https://iptv-org.github.io/epg/guides/jp.xml 🇯🇪 Jersey58https://iptv-org.github.io/epg/guides/je.xml - 🇯🇴 Jordan92https://iptv-org.github.io/epg/guides/jo.xml + 🇯🇴 Jordan90https://iptv-org.github.io/epg/guides/jo.xml 🇰🇿 Kazakhstan102https://iptv-org.github.io/epg/guides/kz.xml - 🇰🇪 Kenya107https://iptv-org.github.io/epg/guides/ke.xml + 🇰🇪 Kenya106https://iptv-org.github.io/epg/guides/ke.xml 🇰🇮 Kiribati67https://iptv-org.github.io/epg/guides/ki.xml 🇽🇰 Kosovo81https://iptv-org.github.io/epg/guides/xk.xml - 🇰🇼 Kuwait95https://iptv-org.github.io/epg/guides/kw.xml + 🇰🇼 Kuwait93https://iptv-org.github.io/epg/guides/kw.xml 🇰🇬 Kyrgyzstan65https://iptv-org.github.io/epg/guides/kg.xml - 🇱🇦 Laos109https://iptv-org.github.io/epg/guides/la.xml + 🇱🇦 Laos111https://iptv-org.github.io/epg/guides/la.xml 🇱🇻 Latvia104https://iptv-org.github.io/epg/guides/lv.xml - 🇱🇧 Lebanon98https://iptv-org.github.io/epg/guides/lb.xml - 🇱🇸 Lesotho107https://iptv-org.github.io/epg/guides/ls.xml - 🇱🇷 Liberia106https://iptv-org.github.io/epg/guides/lr.xml - 🇱🇾 Libya122https://iptv-org.github.io/epg/guides/ly.xml + 🇱🇧 Lebanon96https://iptv-org.github.io/epg/guides/lb.xml + 🇱🇸 Lesotho106https://iptv-org.github.io/epg/guides/ls.xml + 🇱🇷 Liberia105https://iptv-org.github.io/epg/guides/lr.xml + 🇱🇾 Libya121https://iptv-org.github.io/epg/guides/ly.xml 🇱🇮 Liechtenstein99https://iptv-org.github.io/epg/guides/li.xml 🇱🇹 Lithuania117https://iptv-org.github.io/epg/guides/lt.xml 🇱🇺 Luxembourg116https://iptv-org.github.io/epg/guides/lu.xml 🇲🇴 Macao62https://iptv-org.github.io/epg/guides/mo.xml - 🇲🇬 Madagascar122https://iptv-org.github.io/epg/guides/mg.xml - 🇲🇼 Malawi106https://iptv-org.github.io/epg/guides/mw.xml - 🇲🇾 Malaysia175https://iptv-org.github.io/epg/guides/my.xml + 🇲🇬 Madagascar121https://iptv-org.github.io/epg/guides/mg.xml + 🇲🇼 Malawi105https://iptv-org.github.io/epg/guides/mw.xml + 🇲🇾 Malaysia177https://iptv-org.github.io/epg/guides/my.xml 🇲🇻 Maldives73https://iptv-org.github.io/epg/guides/mv.xml - 🇲🇱 Mali122https://iptv-org.github.io/epg/guides/ml.xml + 🇲🇱 Mali121https://iptv-org.github.io/epg/guides/ml.xml 🇲🇹 Malta117https://iptv-org.github.io/epg/guides/mt.xml 🇲🇭 Marshall Islands67https://iptv-org.github.io/epg/guides/mh.xml 🇲🇶 Martinique89https://iptv-org.github.io/epg/guides/mq.xml - 🇲🇷 Mauritania122https://iptv-org.github.io/epg/guides/mr.xml - 🇲🇺 Mauritius115https://iptv-org.github.io/epg/guides/mu.xml - 🇾🇹 Mayotte107https://iptv-org.github.io/epg/guides/yt.xml + 🇲🇷 Mauritania121https://iptv-org.github.io/epg/guides/mr.xml + 🇲🇺 Mauritius114https://iptv-org.github.io/epg/guides/mu.xml + 🇾🇹 Mayotte106https://iptv-org.github.io/epg/guides/yt.xml 🇲🇽 Mexico259https://iptv-org.github.io/epg/guides/mx.xml 🇫🇲 Micronesia67https://iptv-org.github.io/epg/guides/fm.xml 🇲🇩 Moldova99https://iptv-org.github.io/epg/guides/md.xml @@ -161,45 +161,45 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇲🇳 Mongolia65https://iptv-org.github.io/epg/guides/mn.xml 🇲🇪 Montenegro102https://iptv-org.github.io/epg/guides/me.xml 🇲🇸 Montserrat68https://iptv-org.github.io/epg/guides/ms.xml - 🇲🇦 Morocco124https://iptv-org.github.io/epg/guides/ma.xml - 🇲🇿 Mozambique122https://iptv-org.github.io/epg/guides/mz.xml - 🇲🇲 Myanmar (Burma)109https://iptv-org.github.io/epg/guides/mm.xml - 🇳🇦 Namibia107https://iptv-org.github.io/epg/guides/na.xml + 🇲🇦 Morocco123https://iptv-org.github.io/epg/guides/ma.xml + 🇲🇿 Mozambique121https://iptv-org.github.io/epg/guides/mz.xml + 🇲🇲 Myanmar (Burma)111https://iptv-org.github.io/epg/guides/mm.xml + 🇳🇦 Namibia106https://iptv-org.github.io/epg/guides/na.xml 🇳🇷 Nauru67https://iptv-org.github.io/epg/guides/nr.xml 🇳🇵 Nepal76https://iptv-org.github.io/epg/guides/np.xml 🇳🇱 Netherlands265https://iptv-org.github.io/epg/guides/nl.xml 🇳🇨 New Caledonia70https://iptv-org.github.io/epg/guides/nc.xml 🇳🇿 New Zealand150https://iptv-org.github.io/epg/guides/nz.xml 🇳🇮 Nicaragua199https://iptv-org.github.io/epg/guides/ni.xml - 🇳🇪 Niger116https://iptv-org.github.io/epg/guides/ne.xml - 🇳🇬 Nigeria142https://iptv-org.github.io/epg/guides/ng.xml + 🇳🇪 Niger115https://iptv-org.github.io/epg/guides/ne.xml + 🇳🇬 Nigeria140https://iptv-org.github.io/epg/guides/ng.xml 🇳🇺 Niue67https://iptv-org.github.io/epg/guides/nu.xml 🇳🇫 Norfolk Island67https://iptv-org.github.io/epg/guides/nf.xml 🇰🇵 North Korea65https://iptv-org.github.io/epg/guides/kp.xml 🇲🇰 North Macedonia112https://iptv-org.github.io/epg/guides/mk.xml 🇲🇵 Northern Mariana Islands67https://iptv-org.github.io/epg/guides/mp.xml 🇳🇴 Norway122https://iptv-org.github.io/epg/guides/no.xml - 🇴🇲 Oman91https://iptv-org.github.io/epg/guides/om.xml + 🇴🇲 Oman89https://iptv-org.github.io/epg/guides/om.xml 🇵🇰 Pakistan76https://iptv-org.github.io/epg/guides/pk.xml 🇵🇼 Palau67https://iptv-org.github.io/epg/guides/pw.xml - 🇵🇸 Palestine92https://iptv-org.github.io/epg/guides/ps.xml + 🇵🇸 Palestine90https://iptv-org.github.io/epg/guides/ps.xml 🇵🇦 Panama202https://iptv-org.github.io/epg/guides/pa.xml 🇵🇬 Papua New Guinea67https://iptv-org.github.io/epg/guides/pg.xml - 🇵🇾 Paraguay217https://iptv-org.github.io/epg/guides/py.xml + 🇵🇾 Paraguay218https://iptv-org.github.io/epg/guides/py.xml 🇵🇪 Peru233https://iptv-org.github.io/epg/guides/pe.xml - 🇵🇭 Philippines161https://iptv-org.github.io/epg/guides/ph.xml + 🇵🇭 Philippines147https://iptv-org.github.io/epg/guides/ph.xml 🇵🇳 Pitcairn Islands67https://iptv-org.github.io/epg/guides/pn.xml 🇵🇱 Poland265https://iptv-org.github.io/epg/guides/pl.xml - 🇵🇹 Portugal181https://iptv-org.github.io/epg/guides/pt.xml + 🇵🇹 Portugal182https://iptv-org.github.io/epg/guides/pt.xml 🇵🇷 Puerto Rico196https://iptv-org.github.io/epg/guides/pr.xml - 🇶🇦 Qatar136https://iptv-org.github.io/epg/guides/qa.xml - 🇨🇬 Republic of the Congo120https://iptv-org.github.io/epg/guides/cg.xml + 🇶🇦 Qatar134https://iptv-org.github.io/epg/guides/qa.xml + 🇨🇬 Republic of the Congo119https://iptv-org.github.io/epg/guides/cg.xml 🇷🇴 Romania201https://iptv-org.github.io/epg/guides/ro.xml - 🇷🇺 Russia327https://iptv-org.github.io/epg/guides/ru.xml - 🇷🇼 Rwanda123https://iptv-org.github.io/epg/guides/rw.xml - 🇷🇪 Réunion108https://iptv-org.github.io/epg/guides/re.xml + 🇷🇺 Russia325https://iptv-org.github.io/epg/guides/ru.xml + 🇷🇼 Rwanda122https://iptv-org.github.io/epg/guides/rw.xml + 🇷🇪 Réunion107https://iptv-org.github.io/epg/guides/re.xml 🇧🇱 Saint Barthélemy83https://iptv-org.github.io/epg/guides/bl.xml - 🇸🇭 Saint Helena100https://iptv-org.github.io/epg/guides/sh.xml + 🇸🇭 Saint Helena99https://iptv-org.github.io/epg/guides/sh.xml 🇰🇳 Saint Kitts and Nevis68https://iptv-org.github.io/epg/guides/kn.xml 🇱🇨 Saint Lucia68https://iptv-org.github.io/epg/guides/lc.xml 🇲🇫 Saint Martin83https://iptv-org.github.io/epg/guides/mf.xml @@ -207,49 +207,49 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇻🇨 Saint Vincent and the Grenadines68https://iptv-org.github.io/epg/guides/vc.xml 🇼🇸 Samoa67https://iptv-org.github.io/epg/guides/ws.xml 🇸🇲 San Marino92https://iptv-org.github.io/epg/guides/sm.xml - 🇸🇦 Saudi Arabia109https://iptv-org.github.io/epg/guides/sa.xml - 🇸🇳 Senegal126https://iptv-org.github.io/epg/guides/sn.xml + 🇸🇦 Saudi Arabia107https://iptv-org.github.io/epg/guides/sa.xml + 🇸🇳 Senegal125https://iptv-org.github.io/epg/guides/sn.xml 🇷🇸 Serbia310https://iptv-org.github.io/epg/guides/rs.xml - 🇸🇨 Seychelles113https://iptv-org.github.io/epg/guides/sc.xml - 🇸🇱 Sierra Leone106https://iptv-org.github.io/epg/guides/sl.xml + 🇸🇨 Seychelles112https://iptv-org.github.io/epg/guides/sc.xml + 🇸🇱 Sierra Leone105https://iptv-org.github.io/epg/guides/sl.xml 🇸🇬 Singapore154https://iptv-org.github.io/epg/guides/sg.xml 🇸🇽 Sint Maarten77https://iptv-org.github.io/epg/guides/sx.xml 🇸🇰 Slovakia180https://iptv-org.github.io/epg/guides/sk.xml 🇸🇮 Slovenia156https://iptv-org.github.io/epg/guides/si.xml 🇸🇧 Solomon Islands67https://iptv-org.github.io/epg/guides/sb.xml - 🇸🇴 Somalia122https://iptv-org.github.io/epg/guides/so.xml + 🇸🇴 Somalia121https://iptv-org.github.io/epg/guides/so.xml 🇿🇦 South Africa185https://iptv-org.github.io/epg/guides/za.xml 🇬🇸 South Georgia and the South Sandwich Islands58https://iptv-org.github.io/epg/guides/gs.xml 🇰🇷 South Korea143https://iptv-org.github.io/epg/guides/kr.xml - 🇸🇸 South Sudan106https://iptv-org.github.io/epg/guides/ss.xml + 🇸🇸 South Sudan105https://iptv-org.github.io/epg/guides/ss.xml 🇪🇸 Spain253https://iptv-org.github.io/epg/guides/es.xml 🇱🇰 Sri Lanka73https://iptv-org.github.io/epg/guides/lk.xml - 🇸🇩 Sudan127https://iptv-org.github.io/epg/guides/sd.xml + 🇸🇩 Sudan126https://iptv-org.github.io/epg/guides/sd.xml 🇸🇷 Suriname62https://iptv-org.github.io/epg/guides/sr.xml 🇸🇯 Svalbard and Jan Mayen56https://iptv-org.github.io/epg/guides/sj.xml - 🇸🇿 Swaziland106https://iptv-org.github.io/epg/guides/sz.xml - 🇸🇪 Sweden184https://iptv-org.github.io/epg/guides/se.xml - 🇨🇭 Switzerland240https://iptv-org.github.io/epg/guides/ch.xml - 🇸🇾 Syria93https://iptv-org.github.io/epg/guides/sy.xml - 🇸🇹 São Tomé and Príncipe113https://iptv-org.github.io/epg/guides/st.xml + 🇸🇿 Swaziland105https://iptv-org.github.io/epg/guides/sz.xml + 🇸🇪 Sweden183https://iptv-org.github.io/epg/guides/se.xml + 🇨🇭 Switzerland234https://iptv-org.github.io/epg/guides/ch.xml + 🇸🇾 Syria91https://iptv-org.github.io/epg/guides/sy.xml + 🇸🇹 São Tomé and Príncipe112https://iptv-org.github.io/epg/guides/st.xml 🇹🇼 Taiwan72https://iptv-org.github.io/epg/guides/tw.xml 🇹🇯 Tajikistan64https://iptv-org.github.io/epg/guides/tj.xml - 🇹🇿 Tanzania110https://iptv-org.github.io/epg/guides/tz.xml - 🇹🇭 Thailand164https://iptv-org.github.io/epg/guides/th.xml - 🇹🇬 Togo117https://iptv-org.github.io/epg/guides/tg.xml + 🇹🇿 Tanzania109https://iptv-org.github.io/epg/guides/tz.xml + 🇹🇭 Thailand167https://iptv-org.github.io/epg/guides/th.xml + 🇹🇬 Togo116https://iptv-org.github.io/epg/guides/tg.xml 🇹🇰 Tokelau67https://iptv-org.github.io/epg/guides/tk.xml 🇹🇴 Tonga67https://iptv-org.github.io/epg/guides/to.xml 🇹🇹 Trinidad and Tobago68https://iptv-org.github.io/epg/guides/tt.xml - 🇹🇳 Tunisia125https://iptv-org.github.io/epg/guides/tn.xml - 🇹🇷 Turkey250https://iptv-org.github.io/epg/guides/tr.xml + 🇹🇳 Tunisia124https://iptv-org.github.io/epg/guides/tn.xml + 🇹🇷 Turkey247https://iptv-org.github.io/epg/guides/tr.xml 🇹🇲 Turkmenistan60https://iptv-org.github.io/epg/guides/tm.xml 🇹🇨 Turks and Caicos Islands68https://iptv-org.github.io/epg/guides/tc.xml 🇹🇻 Tuvalu67https://iptv-org.github.io/epg/guides/tv.xml 🇺🇲 U.S. Minor Outlying Islands56https://iptv-org.github.io/epg/guides/um.xml 🇻🇮 U.S. Virgin Islands68https://iptv-org.github.io/epg/guides/vi.xml - 🇺🇬 Uganda107https://iptv-org.github.io/epg/guides/ug.xml + 🇺🇬 Uganda106https://iptv-org.github.io/epg/guides/ug.xml 🇺🇦 Ukraine91https://iptv-org.github.io/epg/guides/ua.xml - 🇦🇪 United Arab Emirates110https://iptv-org.github.io/epg/guides/ae.xml + 🇦🇪 United Arab Emirates105https://iptv-org.github.io/epg/guides/ae.xml 🇬🇧 United Kingdom423https://iptv-org.github.io/epg/guides/uk.xml 🇺🇸 United States1400https://iptv-org.github.io/epg/guides/us.xml 🇺🇾 Uruguay225https://iptv-org.github.io/epg/guides/uy.xml @@ -257,12 +257,12 @@ To load a program guide, all you need to do is copy the link to one or more of t 🇻🇺 Vanuatu69https://iptv-org.github.io/epg/guides/vu.xml 🇻🇦 Vatican City92https://iptv-org.github.io/epg/guides/va.xml 🇻🇪 Venezuela214https://iptv-org.github.io/epg/guides/ve.xml - 🇻🇳 Vietnam111https://iptv-org.github.io/epg/guides/vn.xml + 🇻🇳 Vietnam113https://iptv-org.github.io/epg/guides/vn.xml 🇼🇫 Wallis and Futuna69https://iptv-org.github.io/epg/guides/wf.xml - 🇪🇭 Western Sahara111https://iptv-org.github.io/epg/guides/eh.xml - 🇾🇪 Yemen91https://iptv-org.github.io/epg/guides/ye.xml - 🇿🇲 Zambia110https://iptv-org.github.io/epg/guides/zm.xml - 🇿🇼 Zimbabwe106https://iptv-org.github.io/epg/guides/zw.xml + 🇪🇭 Western Sahara110https://iptv-org.github.io/epg/guides/eh.xml + 🇾🇪 Yemen89https://iptv-org.github.io/epg/guides/ye.xml + 🇿🇲 Zambia109https://iptv-org.github.io/epg/guides/zm.xml + 🇿🇼 Zimbabwe105https://iptv-org.github.io/epg/guides/zw.xml 🇦🇽 Åland63https://iptv-org.github.io/epg/guides/ax.xml From 3dad85dbad6c7c3cc5d89d359822cdc32d97178a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 12:43:23 +0300 Subject: [PATCH 08/16] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0a270759..09c161a7 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", "guides:update_legacy": "node scripts/commands/guides/update_legacy.js", - "guides:validate": "node scripts/commands/guides/validate.js", + "guides:validate": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/validate.js", "api:load": "./scripts/commands/api/load.sh", "api:update": "node scripts/commands/api/update.js", "readme:update": "node scripts/commands/readme/update.js", From c7d9cf8356c8f90de80cda85897ad7c08bf8c392 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 12:43:58 +0300 Subject: [PATCH 09/16] Update validate.js --- scripts/commands/guides/validate.js | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/commands/guides/validate.js b/scripts/commands/guides/validate.js index bad9d188..79ce13b5 100644 --- a/scripts/commands/guides/validate.js +++ b/scripts/commands/guides/validate.js @@ -19,11 +19,19 @@ async function main() { const errors = [] - let programs = db_programs.map(p => ({ - site: p.site, - xmltv_id: p.channel, - lang: p.titles[0].lang - })) + let programs = db_programs + .map(p => { + if (p.titles.length) { + return { + site: p.site, + xmltv_id: p.channel, + lang: p.titles[0].lang + } + } + + return null + }) + .filter(Boolean) programs = _.uniqBy(programs, p => p.site + p.xmltv_id) for (let program of programs) { if (!guides.find(g => g.channel === program.xmltv_id)) { From ba88448119be00c940451c9f1e6ab9f4d99db1c4 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:17:40 +0300 Subject: [PATCH 10/16] Update tests/__data__ --- .../__data__/expected/logs/guides/update.log | 2 +- tests/__data__/input/data/channels.json | 19 ++++++++++++++++++- .../input/database/update-guides/programs.db | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/__data__/expected/logs/guides/update.log b/tests/__data__/expected/logs/guides/update.log index fdba1db3..701a3e2a 100644 --- a/tests/__data__/expected/logs/guides/update.log +++ b/tests/__data__/expected/logs/guides/update.log @@ -1,2 +1,2 @@ {"country":"DK","lang":"da","site":"allente.se","channel":"6eren.dk","filename":"dk"} -{"country":"UK","lang":"en","site":"virginmedia.com","channel":"BBCNews.uk","filename":"uk"} +{"country":"UK","lang":"en","site":"virginmedia.com","channel":"BBCNews.uk","filename":"uk"} \ No newline at end of file diff --git a/tests/__data__/input/data/channels.json b/tests/__data__/input/data/channels.json index 46da2402..28255c4a 100644 --- a/tests/__data__/input/data/channels.json +++ b/tests/__data__/input/data/channels.json @@ -53,5 +53,22 @@ "logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png" }, {"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"}, - {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"} + {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"}, + { + "id": "CNN.us", + "name": "CNN", + "network": null, + "country": "US", + "subdivision": null, + "city": null, + "broadcast_area": [ + "c/US" + ], + "languages": [ + "eng" + ], + "categories": [], + "is_nsfw": false, + "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png" + } ] \ No newline at end of file diff --git a/tests/__data__/input/database/update-guides/programs.db b/tests/__data__/input/database/update-guides/programs.db index 5e78bd6b..e72a3252 100644 --- a/tests/__data__/input/database/update-guides/programs.db +++ b/tests/__data__/input/database/update-guides/programs.db @@ -1,4 +1,5 @@ {"site":"allente.se","channel":"6eren.dk","titles":[{"value":"Diners, Drive-Ins and Dives","lang":"da"}],"sub_titles":[],"descriptions":[{"value":"Underholdning","lang":"da"}],"icon":{"src":"https://viasatps.api.comspace.se/PS/channeldate/image/viasat.ps/487/2022-10-24/se.cs.6eren.event.B_0254194276971024040000.jpg?size=2560x1440"},"episodeNumbers":[{"system":"xmltv_ns","value":"23.5.0/1"},{"system":"onscreen","value":"S24E06"}],"date":null,"start":1666584000000,"stop":1666585500000,"urls":[],"ratings":[],"categories":[{"value":"series","lang":"da"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"f6cxSM73LfZ8TdYz","_id":"HxsrTRTFj1z05TAK"} {"site":"allente.se","channel":"6eren.dk","titles":[{"value":"Diners, Drive-Ins and Dives","lang":"da"}],"sub_titles":[],"descriptions":[{"value":"Underholdning","lang":"da"}],"icon":{"src":"https://viasatps.api.comspace.se/PS/channeldate/image/viasat.ps/487/2022-10-24/se.cs.6eren.event.B_0254194276971024040000.jpg?size=2560x1440"},"episodeNumbers":[{"system":"xmltv_ns","value":"23.5.0/1"},{"system":"onscreen","value":"S24E06"}],"date":null,"start":1666584000000,"stop":1666585500000,"urls":[],"ratings":[],"categories":[{"value":"series","lang":"da"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"f6cxTM73LfZ8TdYz","_id":"HxsrBRTFj1z05TAK"} {"site":"virginmedia.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news, followed by weather.","lang":"en"}],"icon":{"src":""},"episodeNumbers":[{"system":"xmltv_ns","value":"96839999.145799123.0/1"},{"system":"onscreen","value":"S96840000E145799124"}],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[{"value":"News","lang":"en"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"lNXh3lBnb4n1DBzs","_id":"quKCInjZV98xFUAf"} -{"site":"sky.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news from the BBC. [S,SL]","lang":"en"}],"icon":{"src":"http://epgstatic.sky.com/epgdata/1.0/paimage/46/1/lisa/5.2.2/linear/channel/ca247bc8-6be0-48f9-88d1-865f87f7680e/2011"},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"36duI92slofEXlSa","_id":"ryzed0Bqda1QtE7i"} \ No newline at end of file +{"site":"sky.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news from the BBC. [S,SL]","lang":"en"}],"icon":{"src":"http://epgstatic.sky.com/epgdata/1.0/paimage/46/1/lisa/5.2.2/linear/channel/ca247bc8-6be0-48f9-88d1-865f87f7680e/2011"},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"36duI92slofEXlSa","_id":"ryzed0Bqda1QtE7i"} +{"site":"sky.com","channel":"CNN.us","titles":[{"value":"French title","lang":"fr"}],"sub_titles":[],"descriptions":[],"icon":{},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"37duI92slofEXlSa","_id":"rxzed0Bqda1QtE7i"} \ No newline at end of file From 1a8e256aab40f0ab56edb3eff9ae20bb1cecd689 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:17:49 +0300 Subject: [PATCH 11/16] Update update.test.js --- tests/commands/guides/update.test.js | 34 ++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/commands/guides/update.test.js b/tests/commands/guides/update.test.js index fe47a2c9..eac2e69c 100644 --- a/tests/commands/guides/update.test.js +++ b/tests/commands/guides/update.test.js @@ -9,14 +9,44 @@ beforeEach(() => { 'tests/__data__/input/database/update-guides/programs.db', 'tests/__data__/output/programs.db' ) +}) +it('can generate /guides', () => { const stdout = execSync( 'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output CURR_DATE=2022-10-20 npm run guides:update', { encoding: 'utf8' } ) -}) -it('can generate /guides', () => { + expect(stdout).toBe( + ` +> guides:update +> NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js + +starting... +loading data/countries.json... +loading data/channels.json... +loading data/regions.json... +loading data/subdivisions.json... +loading database/programs.db... +found 5 programs +creating tests/__data__/output/guides/dk.xml... +creating tests/__data__/output/guides/dk.xml.gz... +creating tests/__data__/output/guides/dk.json... +creating tests/__data__/output/guides/uk.xml... +creating tests/__data__/output/guides/uk.xml.gz... +creating tests/__data__/output/guides/uk.json... +creating tests/__data__/output/logs/guides/update.log... + +report: +┌─────────┬────────────┬───────────┬──────┬──────────┬────────────────┬───────────┐ +│ (index) │ type │ site │ lang │ channel │ broadcast_area │ languages │ +├─────────┼────────────┼───────────┼──────┼──────────┼────────────────┼───────────┤ +│ 0 │ 'no_guide' │ 'sky.com' │ 'fr' │ 'CNN.us' │ [ 'c/US' ] │ [ 'eng' ] │ +└─────────┴────────────┴───────────┴──────┴──────────┴────────────────┴───────────┘ +found 1 error(s) +` + ) + const uncompressed = glob .sync('tests/__data__/expected/guides/*.xml') .map(f => f.replace('tests/__data__/expected/', '')) From 12ad4d2a58953081fa57ae83c0c0bae4d407d456 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:17:56 +0300 Subject: [PATCH 12/16] Update update.js --- scripts/commands/guides/update.js | 41 ++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/scripts/commands/guides/update.js b/scripts/commands/guides/update.js index e0e09b17..daecafd9 100644 --- a/scripts/commands/guides/update.js +++ b/scripts/commands/guides/update.js @@ -9,13 +9,19 @@ const CURR_DATE = process.env.CURR_DATE || new Date() const logPath = `${LOGS_DIR}/guides/update.log` +let guides = [] +let db_programs = [] + async function main() { logger.info(`starting...`) - logger.info('loading API data...') + logger.info('loading data/countries.json...') await api.countries.load() + logger.info('loading data/channels.json...') await api.channels.load() + logger.info('loading data/regions.json...') await api.regions.load() + logger.info('loading data/subdivisions.json...') await api.subdivisions.load() let countries = await api.countries.all() @@ -31,7 +37,7 @@ async function main() { logger.info('loading database/programs.db...') await db.programs.load() - let db_programs = await db.programs.find({}) + db_programs = await db.programs.find({}) db_programs = db_programs .map(p => { if (p.titles.length) { @@ -45,9 +51,6 @@ async function main() { .filter(Boolean) logger.info(`found ${db_programs.length} programs`) - logger.info(`creating ${logPath}...`) - await file.create(logPath) - for (let country of countries) { let countryBroadcastCode = `c/${country.code}` let countryRegions = api_regions @@ -122,17 +125,20 @@ async function main() { await file.create(jsonFilepath, JSON.stringify({ channels, programs })) for (let channel of channels) { - let result = { + guides.push({ country: country.code, lang: channel.lang, site: channel.site, channel: channel.id, filename - } - - await file.append(logPath, JSON.stringify(result) + '\r\n') + }) } } + + logger.info(`creating ${logPath}...`) + await file.create(logPath, guides.map(g => JSON.stringify(g)).join('\r\n')) + + await makeReport() } main() @@ -176,3 +182,20 @@ function calcScore(program) { return score } + +async function makeReport() { + const errors = [] + + let programs = _.uniqBy(db_programs, p => p.site + p.channel) + for (let program of programs) { + if (!guides.find(g => g.channel === program.channel)) { + const channel = await api.channels.find({ id: program.channel }) + errors.push({ type: 'no_guide', ...program, ...channel }) + } + } + + console.log() + logger.info(`report:`) + console.table(errors, ['type', 'site', 'lang', 'channel', 'broadcast_area', 'languages']) + logger.error(`found ${errors.length} error(s)`) +} From 8b01015283addf35b52d230998ab6c3aacdfd37b Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:17:58 +0300 Subject: [PATCH 13/16] Delete validate.test.js --- tests/commands/guides/validate.test.js | 37 -------------------------- 1 file changed, 37 deletions(-) delete mode 100644 tests/commands/guides/validate.test.js diff --git a/tests/commands/guides/validate.test.js b/tests/commands/guides/validate.test.js deleted file mode 100644 index 370a0a08..00000000 --- a/tests/commands/guides/validate.test.js +++ /dev/null @@ -1,37 +0,0 @@ -const { execSync } = require('child_process') -const fs = require('fs-extra') - -beforeEach(() => { - fs.emptyDirSync('tests/__data__/output') - - fs.copyFileSync( - 'tests/__data__/input/database/update-guides/programs.db', - 'tests/__data__/output/programs.db' - ) -}) - -it('will show a message if the channel is not in the guide', () => { - const stdout = execSync( - 'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/input/logs DATA_DIR=tests/__data__/input/data npm run guides:validate', - { - encoding: 'utf8' - } - ) - expect(stdout).toBe( - `\n> guides:validate\n> node scripts/commands/guides/validate.js - -loading data/channels.json... -loading tests/__data__/input/logs/guides/update.log... -loading database/programs.db... -found 4 programs -┌─────────┬────────────┬───────────────────┬──────┬──────────────┬────────────────┬───────────┐ -│ (index) │ type │ site │ lang │ xmltv_id │ broadcast_area │ languages │ -├─────────┼────────────┼───────────────────┼──────┼──────────────┼────────────────┼───────────┤ -│ 0 │ 'no_guide' │ 'virginmedia.com' │ 'en' │ 'BBCNews.uk' │ [ 'c/UK' ] │ [ 'eng' ] │ -│ 1 │ 'no_guide' │ 'sky.com' │ 'en' │ 'BBCNews.uk' │ [ 'c/UK' ] │ [ 'eng' ] │ -└─────────┴────────────┴───────────────────┴──────┴──────────────┴────────────────┴───────────┘ - -2 error(s) -` - ) -}) From b78fde8df417e0612973133e018ebe928526f1bf Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:18:00 +0300 Subject: [PATCH 14/16] Delete validate.js --- scripts/commands/guides/validate.js | 51 ----------------------------- 1 file changed, 51 deletions(-) delete mode 100644 scripts/commands/guides/validate.js diff --git a/scripts/commands/guides/validate.js b/scripts/commands/guides/validate.js deleted file mode 100644 index 79ce13b5..00000000 --- a/scripts/commands/guides/validate.js +++ /dev/null @@ -1,51 +0,0 @@ -const { db, logger, api, parser } = require('../../core') -const chalk = require('chalk') -const _ = require('lodash') - -const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' - -async function main() { - logger.info('loading data/channels.json...') - await api.channels.load() - - const logPath = `${LOGS_DIR}/guides/update.log` - logger.info(`loading ${logPath}...`) - const guides = await parser.parseLogs(logPath) - - logger.info('loading database/programs.db...') - await db.programs.load() - let db_programs = await db.programs.find({}) - logger.info(`found ${db_programs.length} programs`) - - const errors = [] - - let programs = db_programs - .map(p => { - if (p.titles.length) { - return { - site: p.site, - xmltv_id: p.channel, - lang: p.titles[0].lang - } - } - - return null - }) - .filter(Boolean) - programs = _.uniqBy(programs, p => p.site + p.xmltv_id) - for (let program of programs) { - if (!guides.find(g => g.channel === program.xmltv_id)) { - const channel = await api.channels.find({ id: program.xmltv_id }) - errors.push({ type: 'no_guide', ...program, ...channel }) - } - } - - if (errors.length) { - console.table(errors, ['type', 'site', 'lang', 'xmltv_id', 'broadcast_area', 'languages']) - console.log() - - logger.error(chalk.red(`${errors.length} error(s)`)) - } -} - -main() From 06fb8516c52754688182dfa4cdfc875c17e50894 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 13:28:51 +0300 Subject: [PATCH 15/16] Update _update.yml --- .github/workflows/_update.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_update.yml b/.github/workflows/_update.yml index 039342b1..190b532b 100644 --- a/.github/workflows/_update.yml +++ b/.github/workflows/_update.yml @@ -37,7 +37,6 @@ jobs: with: name: logs path: scripts/logs - - run: npm run guides:validate - run: npm run api:update - run: npm run readme:update - run: npm run status:update From d1e538bd9487b55c635b4180a859b9f628a9f6db Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 8 Jan 2023 14:43:02 +0300 Subject: [PATCH 16/16] Update package.json --- package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.json b/package.json index 09c161a7..f54438de 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,6 @@ "programs:load": "node scripts/commands/programs/load.js", "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", - "guides:update_legacy": "node scripts/commands/guides/update_legacy.js", - "guides:validate": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/validate.js", "api:load": "./scripts/commands/api/load.sh", "api:update": "node scripts/commands/api/update.js", "readme:update": "node scripts/commands/readme/update.js",