diff --git a/scripts/commands/save-results.js b/scripts/commands/save-results.js index b5e03450..667692db 100644 --- a/scripts/commands/save-results.js +++ b/scripts/commands/save-results.js @@ -4,8 +4,9 @@ const _ = require('lodash') const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' async function main() { + const errorsLog = `${LOGS_DIR}/errors.log` + await file.create(errorsLog) await db.channels.load() - await db.programs.load() await db.programs.reset() const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`) @@ -13,8 +14,6 @@ async function main() { logger.info(`Parsing "${filepath}"...`) const results = await parser.parseLogs(filepath) for (const result of results) { - await db.channels.update({ _id: result._id }, { $set: { logo: result.logo } }) - const programs = result.programs.map(program => { return { title: program.title, @@ -27,13 +26,24 @@ async function main() { lang: program.lang, start: program.start, stop: program.stop, - site: result.site, - country: result.country, - gid: result.gid + _cid: result.channel._id } }) - await db.programs.insert(programs) + + if (result.channel.logo) { + await db.channels.update( + { _id: result.channel._id }, + { $set: { logo: result.channel.logo } } + ) + } + + if (result.error) { + await file.append( + errorsLog, + JSON.stringify({ ...result.channel, date: result.date, error: result.error }) + '\n' + ) + } } } diff --git a/tests/__data__/input/logs/errors.log b/tests/__data__/input/logs/errors.log new file mode 100644 index 00000000..d16cba84 --- /dev/null +++ b/tests/__data__/input/logs/errors.log @@ -0,0 +1 @@ +{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"} diff --git a/tests/__data__/input/logs/load-cluster/cluster_1.log b/tests/__data__/input/logs/load-cluster/cluster_1.log index c182f4a2..bad7dbaf 100644 --- a/tests/__data__/input/logs/load-cluster/cluster_1.log +++ b/tests/__data__/input/logs/load-cluster/cluster_1.log @@ -1 +1,2 @@ -{"_id":"0Wefq0oMR3feCcuY","site":"andorradifusio.ad","gid":"ad","country":"AD","logo":"https://example.com/logo.png","programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}]} +{"channel":{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://example.com/logo.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr"],"cluster_id":1,"_id":"0Wefq0oMR3feCcuY"},"programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}],"date":"2022-01-21T00:00:00Z","error":null} +{"channel":{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"},"programs":[],"date":"2022-01-21T00:00:00Z","error":"Invalid header value char"} diff --git a/tests/commands/save-results.test.js b/tests/commands/save-results.test.js index 41407dd7..ba664823 100644 --- a/tests/commands/save-results.test.js +++ b/tests/commands/save-results.test.js @@ -12,38 +12,50 @@ beforeEach(() => { 'tests/__data__/output/database/channels.db' ) - execSync( + const stdout = execSync( 'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs node scripts/commands/save-results.js', { encoding: 'utf8' } ) + console.log(stdout) }) -it('can save results to database', () => { - const output = content('tests/__data__/output/database/programs.db') +it('can save results', () => { + const programs = content('tests/__data__/output/database/programs.db') - expect(Object.keys(output[0]).sort()).toEqual([ + expect(Object.keys(programs[0]).sort()).toEqual([ + '_cid', '_id', 'category', 'channel', - 'country', 'description', 'episode', - 'gid', 'icon', 'lang', 'season', - 'site', 'start', 'stop', 'title' ]) - const database = content('tests/__data__/output/database/channels.db') + expect(programs[0]).toMatchObject({ + _cid: '0Wefq0oMR3feCcuY' + }) - expect(database[1]).toMatchObject({ + const channels = content('tests/__data__/output/database/channels.db') + + expect(channels[1]).toMatchObject({ _id: '0Wefq0oMR3feCcuY', logo: 'https://example.com/logo.png' }) + + const errors = content('tests/__data__/input/logs/errors.log') + + expect(errors[0]).toMatchObject({ + _id: '00AluKCrCnfgrl8W', + site: 'directv.com', + xmltv_id: 'BravoEast.us', + error: 'Invalid header value char' + }) }) function content(filepath) {