Update save-results.js

scripts/commands/save-results.js

@@ -4,8 +4,9 @@ const _ = require('lodash')
 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
 
 async function main() {
+  const errorsLog = `${LOGS_DIR}/errors.log`
+  await file.create(errorsLog)
   await db.channels.load()
-
   await db.programs.load()
   await db.programs.reset()
   const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`)
@@ -13,8 +14,6 @@ async function main() {
     logger.info(`Parsing "${filepath}"...`)
     const results = await parser.parseLogs(filepath)
     for (const result of results) {
-      await db.channels.update({ _id: result._id }, { $set: { logo: result.logo } })
-
       const programs = result.programs.map(program => {
         return {
           title: program.title,
@@ -27,13 +26,24 @@ async function main() {
           lang: program.lang,
           start: program.start,
           stop: program.stop,
-          site: result.site,
+          _cid: result.channel._id
-          country: result.country,
-          gid: result.gid
         }
       })
-
       await db.programs.insert(programs)
+
+      if (result.channel.logo) {
+        await db.channels.update(
+          { _id: result.channel._id },
+          { $set: { logo: result.channel.logo } }
+        )
+      }
+
+      if (result.error) {
+        await file.append(
+          errorsLog,
+          JSON.stringify({ ...result.channel, date: result.date, error: result.error }) + '\n'
+        )
+      }
     }
   }
 

tests/__data__/input/logs/errors.log

@@ -0,0 +1 @@
+{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}

tests/__data__/input/logs/load-cluster/cluster_1.log

@@ -1 +1,2 @@
-{"_id":"0Wefq0oMR3feCcuY","site":"andorradifusio.ad","gid":"ad","country":"AD","logo":"https://example.com/logo.png","programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}]}
+{"channel":{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://example.com/logo.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr"],"cluster_id":1,"_id":"0Wefq0oMR3feCcuY"},"programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}],"date":"2022-01-21T00:00:00Z","error":null}
+{"channel":{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"},"programs":[],"date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}

tests/commands/save-results.test.js

@@ -12,38 +12,50 @@ beforeEach(() => {
     'tests/__data__/output/database/channels.db'
   )
 
-  execSync(
+  const stdout = execSync(
     'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs node scripts/commands/save-results.js',
     { encoding: 'utf8' }
   )
+  console.log(stdout)
 })
 
-it('can save results to database', () => {
+it('can save results', () => {
-  const output = content('tests/__data__/output/database/programs.db')
+  const programs = content('tests/__data__/output/database/programs.db')
 
-  expect(Object.keys(output[0]).sort()).toEqual([
+  expect(Object.keys(programs[0]).sort()).toEqual([
+    '_cid',
     '_id',
     'category',
     'channel',
-    'country',
     'description',
     'episode',
-    'gid',
     'icon',
     'lang',
     'season',
-    'site',
     'start',
     'stop',
     'title'
   ])
 
-  const database = content('tests/__data__/output/database/channels.db')
+  expect(programs[0]).toMatchObject({
+    _cid: '0Wefq0oMR3feCcuY'
+  })
 
-  expect(database[1]).toMatchObject({
+  const channels = content('tests/__data__/output/database/channels.db')
+
+  expect(channels[1]).toMatchObject({
     _id: '0Wefq0oMR3feCcuY',
     logo: 'https://example.com/logo.png'
   })
+
+  const errors = content('tests/__data__/input/logs/errors.log')
+
+  expect(errors[0]).toMatchObject({
+    _id: '00AluKCrCnfgrl8W',
+    site: 'directv.com',
+    xmltv_id: 'BravoEast.us',
+    error: 'Invalid header value char'
+  })
 })
 
 function content(filepath) {