Merge pull request #1623 from RevGear/ValidateGuides

Validate Guides

package.json

@@ -10,7 +10,6 @@
     "programs:load": "node scripts/commands/programs/load.js",
     "programs:save": "node scripts/commands/programs/save.js",
     "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js",
-    "guides:update_legacy": "node scripts/commands/guides/update_legacy.js",
     "api:load": "./scripts/commands/api/load.sh",
     "api:update": "node scripts/commands/api/update.js",
     "readme:update": "node scripts/commands/readme/update.js",

scripts/commands/guides/update.js

@@ -9,13 +9,19 @@ const CURR_DATE = process.env.CURR_DATE || new Date()
 
 const logPath = `${LOGS_DIR}/guides/update.log`
 
+let guides = []
+let db_programs = []
+
 async function main() {
   logger.info(`starting...`)
 
-  logger.info('loading API data...')
+  logger.info('loading data/countries.json...')
   await api.countries.load()
+  logger.info('loading data/channels.json...')
   await api.channels.load()
+  logger.info('loading data/regions.json...')
   await api.regions.load()
+  logger.info('loading data/subdivisions.json...')
   await api.subdivisions.load()
 
   let countries = await api.countries.all()
@@ -31,7 +37,7 @@ async function main() {
 
   logger.info('loading database/programs.db...')
   await db.programs.load()
-  let db_programs = await db.programs.find({})
+  db_programs = await db.programs.find({})
   db_programs = db_programs
     .map(p => {
       if (p.titles.length) {
@@ -45,9 +51,6 @@ async function main() {
     .filter(Boolean)
   logger.info(`found ${db_programs.length} programs`)
 
-  logger.info(`creating ${logPath}...`)
-  await file.create(logPath)
-
   for (let country of countries) {
     let countryBroadcastCode = `c/${country.code}`
     let countryRegions = api_regions
@@ -122,17 +125,20 @@ async function main() {
     await file.create(jsonFilepath, JSON.stringify({ channels, programs }))
 
     for (let channel of channels) {
-      let result = {
+      guides.push({
         country: country.code,
         lang: channel.lang,
         site: channel.site,
         channel: channel.id,
         filename
+      })
+    }
   }
 
-      await file.append(logPath, JSON.stringify(result) + '\r\n')
+  logger.info(`creating ${logPath}...`)
-    }
+  await file.create(logPath, guides.map(g => JSON.stringify(g)).join('\r\n'))
-  }
+
+  await makeReport()
 }
 
 main()
@@ -176,3 +182,20 @@ function calcScore(program) {
 
   return score
 }
+
+async function makeReport() {
+  const errors = []
+
+  let programs = _.uniqBy(db_programs, p => p.site + p.channel)
+  for (let program of programs) {
+    if (!guides.find(g => g.channel === program.channel)) {
+      const channel = await api.channels.find({ id: program.channel })
+      errors.push({ type: 'no_guide', ...program, ...channel })
+    }
+  }
+
+  console.log()
+  logger.info(`report:`)
+  console.table(errors, ['type', 'site', 'lang', 'channel', 'broadcast_area', 'languages'])
+  logger.error(`found ${errors.length} error(s)`)
+}

tests/__data__/input/data/channels.json

@@ -53,5 +53,22 @@
     "logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png"
   },
   {"id":"6eren.dk","name":"6'eren","alt_names":[],"network":null,"owners":["Warner Bros. Discovery EMEA"],"country":"DK","subdivision":null,"city":null,"broadcast_area":["c/DK"],"languages":["dan"],"categories":[],"is_nsfw":false,"launched":"2009-01-01","closed":null,"replaced_by":null,"website":"http://www.6-eren.dk/","logo":"https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"},
-  {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"}
+  {"id":"BBCNews.uk","name":"BBC News","alt_names":[],"network":null,"owners":[],"country":"UK","subdivision":null,"city":null,"broadcast_area":["c/UK"],"languages":["eng"],"categories":["news"],"is_nsfw":false,"launched":null,"closed":null,"replaced_by":null,"website":"http://news.bbc.co.uk/","logo":"https://i.imgur.com/rPzH88J.png"},
+  {
+    "id": "CNN.us",
+    "name": "CNN",
+    "network": null,
+    "country": "US",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "c/US"
+    ],
+    "languages": [
+      "eng"
+    ],
+    "categories": [],
+    "is_nsfw": false,
+    "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
+  }
 ]

tests/__data__/input/database/update-guides/programs.db

@@ -2,3 +2,4 @@
 {"site":"allente.se","channel":"6eren.dk","titles":[{"value":"Diners, Drive-Ins and Dives","lang":"da"}],"sub_titles":[],"descriptions":[{"value":"Underholdning","lang":"da"}],"icon":{"src":"https://viasatps.api.comspace.se/PS/channeldate/image/viasat.ps/487/2022-10-24/se.cs.6eren.event.B_0254194276971024040000.jpg?size=2560x1440"},"episodeNumbers":[{"system":"xmltv_ns","value":"23.5.0/1"},{"system":"onscreen","value":"S24E06"}],"date":null,"start":1666584000000,"stop":1666585500000,"urls":[],"ratings":[],"categories":[{"value":"series","lang":"da"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"f6cxTM73LfZ8TdYz","_id":"HxsrBRTFj1z05TAK"}
 {"site":"virginmedia.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news, followed by weather.","lang":"en"}],"icon":{"src":""},"episodeNumbers":[{"system":"xmltv_ns","value":"96839999.145799123.0/1"},{"system":"onscreen","value":"S96840000E145799124"}],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[{"value":"News","lang":"en"}],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"lNXh3lBnb4n1DBzs","_id":"quKCInjZV98xFUAf"}
 {"site":"sky.com","channel":"BBCNews.uk","titles":[{"value":"BBC News at One","lang":"en"}],"sub_titles":[],"descriptions":[{"value":"The latest national and international news from the BBC. [S,SL]","lang":"en"}],"icon":{"src":"http://epgstatic.sky.com/epgdata/1.0/paimage/46/1/lisa/5.2.2/linear/channel/ca247bc8-6be0-48f9-88d1-865f87f7680e/2011"},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"36duI92slofEXlSa","_id":"ryzed0Bqda1QtE7i"}
+{"site":"sky.com","channel":"CNN.us","titles":[{"value":"French title","lang":"fr"}],"sub_titles":[],"descriptions":[],"icon":{},"episodeNumbers":[],"date":null,"start":1666872000000,"stop":1666873800000,"urls":[],"ratings":[],"categories":[],"directors":[],"actors":[],"writers":[],"adapters":[],"producers":[],"composers":[],"editors":[],"presenters":[],"commentators":[],"guests":[],"_qid":"37duI92slofEXlSa","_id":"rxzed0Bqda1QtE7i"}

tests/commands/guides/update.test.js

@@ -9,14 +9,44 @@ beforeEach(() => {
     'tests/__data__/input/database/update-guides/programs.db',
     'tests/__data__/output/programs.db'
   )
+})
 
+it('can generate /guides', () => {
   const stdout = execSync(
     'DB_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output CURR_DATE=2022-10-20 npm run guides:update',
     { encoding: 'utf8' }
   )
-})
 
-it('can generate /guides', () => {
+  expect(stdout).toBe(
+    `
+> guides:update
+> NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js
+
+starting...
+loading data/countries.json...
+loading data/channels.json...
+loading data/regions.json...
+loading data/subdivisions.json...
+loading database/programs.db...
+found 5 programs
+creating tests/__data__/output/guides/dk.xml...
+creating tests/__data__/output/guides/dk.xml.gz...
+creating tests/__data__/output/guides/dk.json...
+creating tests/__data__/output/guides/uk.xml...
+creating tests/__data__/output/guides/uk.xml.gz...
+creating tests/__data__/output/guides/uk.json...
+creating tests/__data__/output/logs/guides/update.log...
+
+report:
+┌─────────┬────────────┬───────────┬──────┬──────────┬────────────────┬───────────┐
+│ (index) │    type    │   site    │ lang │ channel  │ broadcast_area │ languages │
+├─────────┼────────────┼───────────┼──────┼──────────┼────────────────┼───────────┤
+│    0    │ 'no_guide' │ 'sky.com' │ 'fr' │ 'CNN.us' │   [ 'c/US' ]   │ [ 'eng' ] │
+└─────────┴────────────┴───────────┴──────┴──────────┴────────────────┴───────────┘
+found 1 error(s)
+`
+  )
+
   const uncompressed = glob
     .sync('tests/__data__/expected/guides/*.xml')
     .map(f => f.replace('tests/__data__/expected/', ''))