diff --git a/scripts/commands/load-cluster.js b/scripts/commands/load-cluster.js index 4592c441..c55e8441 100644 --- a/scripts/commands/load-cluster.js +++ b/scripts/commands/load-cluster.js @@ -46,30 +46,29 @@ async function main() { } }) - const programs = await grabber.grab(channel, config, async (data, err) => { - await db.channels.update({ _id: channel._id }, { $set: { logo: data.channel.logo } }) + await grabber.grab(channel, config, async (data, err) => { + await file.append( + clusterLog, + JSON.stringify({ + _id: channel._id, + site: channel.site, + country: channel.country, + logo: data.channel.logo, + gid: channel.gid, + programs: data.programs + }) + '\n' + ) logger.info( - `[${i}/${total}] ${channel.site} - ${data.id} - ${data.date.format('MMM D, YYYY')} (${ - data.programs.length - } programs)` + `[${i}/${total}] ${channel.site} - ${channel.xmltv_id} - ${data.date.format( + 'MMM D, YYYY' + )} (${data.programs.length} programs)` ) if (err) logger.error(err.message) if (i < total) i++ }) - - await file.append( - clusterLog, - JSON.stringify({ - _id: channel._id, - site: channel.site, - country: channel.country, - gid: channel.gid, - programs - }) + '\n' - ) } db.channels.compact() diff --git a/scripts/commands/save-results.js b/scripts/commands/save-results.js index 8fba8448..5b3cb3ad 100644 --- a/scripts/commands/save-results.js +++ b/scripts/commands/save-results.js @@ -4,6 +4,8 @@ const _ = require('lodash') const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' async function main() { + await db.channels.load() + await db.programs.load() await db.programs.reset() const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`) @@ -11,6 +13,8 @@ async function main() { logger.info(`Parsing "${filepath}"...`) const results = await parser.parseLogs(filepath) for (const result of results) { + await db.channels.update({ _id: result._id }, { $set: { logo: result.logo } }) + const programs = result.programs.map(program => { program.site = result.site program.country = result.country @@ -22,6 +26,8 @@ async function main() { await db.programs.insert(programs) } } + + await db.channels.compact() } main() diff --git a/tests/commands/load-cluster.test.js b/tests/commands/load-cluster.test.js index 6f4db1d7..e603ed0c 100644 --- a/tests/commands/load-cluster.test.js +++ b/tests/commands/load-cluster.test.js @@ -24,6 +24,7 @@ it('can load cluster', () => { expect(output[0]).toMatchObject({ _id: '0Wefq0oMR3feCcuY', site: 'chaines-tv.orange.fr', + logo: 'https://example.com/logo.png', country: 'US', gid: 'fr' }) @@ -31,21 +32,10 @@ it('can load cluster', () => { expect(output[1]).toMatchObject({ _id: '1XzrxNkSF2AQNBrT', site: 'magticom.ge', + logo: 'https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg', country: 'US', gid: 'ge' }) - - const database = content('tests/__data__/temp/database/channels.db') - - expect(database[1]).toMatchObject({ - _id: '0Wefq0oMR3feCcuY', - logo: 'https://example.com/logo.png' - }) - - expect(database[2]).toMatchObject({ - _id: '1XzrxNkSF2AQNBrT', - logo: 'https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg' - }) }) function content(filepath) { diff --git a/tests/commands/save-results.test.js b/tests/commands/save-results.test.js index 2cb0e2df..bf639022 100644 --- a/tests/commands/save-results.test.js +++ b/tests/commands/save-results.test.js @@ -5,6 +5,12 @@ const { execSync } = require('child_process') beforeEach(() => { fs.rmdirSync('tests/__data__/output', { recursive: true }) fs.mkdirSync('tests/__data__/output') + fs.mkdirSync('tests/__data__/output/database', { recursive: true }) + + fs.copyFileSync( + 'tests/__data__/input/database/channels.db', + 'tests/__data__/output/database/channels.db' + ) execSync( 'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/input/logs node scripts/commands/save-results.js', @@ -29,6 +35,13 @@ it('can save results to database', () => { 'stop', 'title' ]) + + const database = content('tests/__data__/output/database/channels.db') + + expect(database[1]).toMatchObject({ + _id: '0Wefq0oMR3feCcuY', + logo: 'https://example.com/logo.png' + }) }) function content(filepath) {