Update update-guides.js

scripts/commands/update-guides.js

@@ -4,7 +4,8 @@ const _ = require('lodash')
 
 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
 const PUBLIC_DIR = process.env.PUBLIC_DIR || '.gh-pages'
-const LOG_PATH = `${LOGS_DIR}/update-guides.log`
+const GUIDES_PATH = `${LOGS_DIR}/guides.log`
+const ERRORS_PATH = `${LOGS_DIR}/errors.log`
 
 async function main() {
   await setUp()
@@ -24,41 +25,59 @@ async function generateGuides() {
 
   for (const key in grouped) {
     const filepath = `${PUBLIC_DIR}/guides/${key}.epg.xml`
-    const items = grouped[key]
+    let items = grouped[key]
-    const channels = items
+    items = items
       .map(i => {
         const channel = api.channels.find({ id: i.xmltv_id })
         i.name = channel.name
         i.logo = channel.logo
+
         return i
       })
       .filter(i => i)
 
-    const programs = await loadProgramsForChannels(channels)
+    const errors = []
-    const output = grabber.convertToXMLTV({ channels, programs })
+    for (const item of items) {
+      if (item.error) {
+        const error = {
+          xmltv_id: item.xmltv_id,
+          site: item.site,
+          site_id: item.site_id,
+          lang: item.lang,
+          date: item.date,
+          error: item.error
+        }
+        errors.push(error)
+        await logError(error)
+      }
+    }
+
+    const programs = await loadProgramsForChannels(items)
 
     logger.info(`Creating "${filepath}"...`)
+    const output = grabber.convertToXMLTV({ channels: items, programs })
     await file.create(filepath, output)
 
-    await log({
+    await logGuide({
       group: key,
-      count: channels.length
+      count: items.length,
+      status: errors.length > 0 ? 1 : 0
     })
   }
 
   logger.info(`Done`)
 }
 
-function groupByGroup(channels = []) {
+function groupByGroup(items = []) {
   const groups = {}
 
-  channels.forEach(channel => {
+  items.forEach(item => {
-    channel.groups.forEach(key => {
+    item.groups.forEach(key => {
       if (!groups[key]) {
         groups[key] = []
       }
 
-      groups[key].push(channel)
+      groups[key].push(item)
     })
   })
 
@@ -70,7 +89,7 @@ async function loadQueue() {
 
   await db.queue.load()
 
-  return await db.queue.find({ programCount: { $gt: 0 } }).sort({ xmltv_id: 1 })
+  return await db.queue.find({}).sort({ xmltv_id: 1 })
 }
 
 async function loadProgramsForChannels(channels = []) {
@@ -80,10 +99,15 @@ async function loadProgramsForChannels(channels = []) {
 }
 
 async function setUp() {
-  logger.info(`Creating '${LOG_PATH}'...`)
+  logger.info(`Creating '${GUIDES_PATH}'...`)
-  await file.create(LOG_PATH)
+  await file.create(GUIDES_PATH)
+  await file.create(ERRORS_PATH)
 }
 
-async function log(data) {
+async function logGuide(data) {
-  await file.append(LOG_PATH, JSON.stringify(data) + '\n')
+  await file.append(GUIDES_PATH, JSON.stringify(data) + '\r\n')
+}
+
+async function logError(data) {
+  await file.append(ERRORS_PATH, JSON.stringify(data) + '\r\n')
 }

tests/__data__/expected/logs/errors.log

@@ -0,0 +1 @@
+{"xmltv_id":"BravoEast.us","site":"directv.com","site_id":"237","lang":"en","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}

tests/__data__/expected/logs/guides.log

@@ -0,0 +1,5 @@
+{"group":"us/directv.com","count":1,"status":1}
+{"group":"fr/chaines-tv.orange.fr","count":1,"status":0}
+{"group":"bh/chaines-tv.orange.fr","count":1,"status":0}
+{"group":"ge/magticom.ge","count":1,"status":0}
+{"group":"zw/dstv.com","count":1,"status":0}

tests/__data__/expected/logs/update-guides.log

@@ -1,3 +0,0 @@
-{"group":"fr/chaines-tv.orange.fr","count":1}
-{"group":"bh/chaines-tv.orange.fr","count":1}
-{"group":"zw/dstv.com","count":1}

tests/__data__/input/database/queue.db

@@ -1,4 +1,4 @@
-{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"programCount":0,"error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"}
+{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"programCount":0,"date":"2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"}
-{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"error":null,"_id":"0Wefq0oMR3feCcuY"}
+{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"date":"2022-01-21T00:00:00Z","error":null,"_id":"0Wefq0oMR3feCcuY"}
-{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"error":null,"_id":"1XzrxNkSF2AQNBrT"}
+{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1XzrxNkSF2AQNBrT"}
-{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"error":null,"_id":"1lnhXpN7g0ER5XwN"}
+{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"date":"2022-01-21T00:00:00Z","error":null,"_id":"1lnhXpN7g0ER5XwN"}

tests/commands/update-guides.test.js

@@ -16,6 +16,8 @@ beforeEach(() => {
     'DB_DIR=tests/__data__/temp/database DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs node scripts/commands/update-guides.js',
     { encoding: 'utf8' }
   )
+
+  console.log(stdout)
 })
 
 afterEach(() => {
@@ -32,11 +34,20 @@ it('can generate /guides', () => {
   const expected2 = content('tests/__data__/expected/guides/zw/dstv.com.epg.xml')
 
   expect(output2).toBe(expected2)
+})
 
-  const output3 = content('tests/__data__/output/logs/update-guides.log')
+it('can create guides.log', () => {
-  const expected3 = content('tests/__data__/expected/logs/update-guides.log')
+  const output = content('tests/__data__/output/logs/guides.log')
+  const expected = content('tests/__data__/expected/logs/guides.log')
 
-  expect(output3).toBe(expected3)
+  expect(output).toBe(expected)
+})
+
+it('can create errors.log', () => {
+  const output = content('tests/__data__/output/logs/errors.log')
+  const expected = content('tests/__data__/expected/logs/errors.log')
+
+  expect(output).toBe(expected)
 })
 
 function content(filepath) {