diff --git a/.github/workflows/_update.yml b/.github/workflows/_update.yml index 7a7856ce..1ab62d8c 100644 --- a/.github/workflows/_update.yml +++ b/.github/workflows/_update.yml @@ -27,7 +27,7 @@ jobs: - run: npm install - run: npm run api:load - if: ${{ !env.ACT }} - run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run programs:load + run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run db:load - uses: actions/upload-artifact@v3 with: name: database diff --git a/package.json b/package.json index c5883adb..d396c8ae 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "channels:editor": "node scripts/commands/channels/editor.js", "queue:create": "node scripts/commands/queue/create.js", "cluster:load": "node scripts/commands/cluster/load.js", - "programs:load": "node scripts/commands/programs/load.js", + "db:load": "node scripts/commands/database/load.js", "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", "api:load": "./scripts/commands/api/load.sh", diff --git a/scripts/commands/programs/load.js b/scripts/commands/database/load.js similarity index 83% rename from scripts/commands/programs/load.js rename to scripts/commands/database/load.js index a0e900a7..147f16f5 100644 --- a/scripts/commands/programs/load.js +++ b/scripts/commands/database/load.js @@ -9,7 +9,8 @@ dayjs.extend(isToday) dayjs.extend(utc) const DB_DIR = process.env.DB_DIR || './scripts/database' -const dbPath = `${DB_DIR}/programs.db` +const programsPath = `${DB_DIR}/programs.db` +const queuePath = `${DB_DIR}/queue.db` const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN @@ -20,7 +21,8 @@ async function main() { let workflows = await getWorkflows() logger.info(`found ${workflows.length} workflows\r\n`) - await file.create(dbPath) + await file.create(programsPath) + await file.create(queuePath) const total = workflows.length for (let [i, workflow] of workflows.entries()) { logger.info(`[${i + 1}/${total}] ${workflow.name}`) @@ -29,8 +31,11 @@ async function main() { let artifact = await getRunArtifacts(run) - const buffer = await downloadArtifact(artifact) - await file.append(dbPath, buffer) + const programsBuffer = await downloadArtifact(artifact, 'programs.db') + await file.append(programsPath, programsBuffer) + + const queueBuffer = await downloadArtifact(artifact, 'queue.db') + await file.append(queuePath, queueBuffer) } } catch (err) { console.log(err.message) @@ -39,7 +44,7 @@ async function main() { main() -async function downloadArtifact(artifact) { +async function downloadArtifact(artifact, filename) { let results = await octokit.request( 'GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}', { @@ -52,7 +57,7 @@ async function downloadArtifact(artifact) { const { entries } = await unzipit.unzip(results.data) - const arrayBuffer = await entries['programs.db'].arrayBuffer() + const arrayBuffer = await entries[filename].arrayBuffer() return toString(arrayBuffer) } diff --git a/scripts/commands/guides/update.js b/scripts/commands/guides/update.js index 0ed5f146..f4ee10f9 100644 --- a/scripts/commands/guides/update.js +++ b/scripts/commands/guides/update.js @@ -9,6 +9,7 @@ const CURR_DATE = process.env.CURR_DATE || new Date() const logPath = `${LOGS_DIR}/guides/update.log` let api_channels = {} +let db_queue = [] let db_programs = [] let guides = [] @@ -22,6 +23,11 @@ async function main() { api_channels[channel.id] = channel }) + logger.info('loading database/queue.db...') + await db.queue.load() + db_queue = await db.queue.find({}) + logger.info(`found ${db_queue.length} items`) + logger.info('loading database/programs.db...') await db.programs.load() db_programs = await db.programs.find({}) @@ -38,14 +44,28 @@ async function main() { main() async function generate() { + let queue = _.uniqBy(db_queue, i => i.channel.lang + i.channel.id + i.channel.site) + queue = _.groupBy(queue, i => (i.channel ? `${i.channel.lang}/${i.channel.site}` : `_`)) + delete queue['_'] + let programs = _.groupBy(db_programs, p => p.titles.length ? `${p.titles[0].lang}/${p.site}` : `_` ) delete programs['_'] - for (let filename in programs) { - let { channels } = await save(filename, programs[filename]) + for (let filename in queue) { + if (!queue[filename]) continue + const channels = queue[filename].map(i => { + const channelData = api_channels[i.channel.id] + channelData.site = i.channel.site + channelData.site_id = i.channel.site_id + channelData.lang = i.channel.lang + + return new Channel(channelData) + }) + + await save(filename, channels, programs[filename]) for (let channel of channels) { const configPath = `sites/${channel.site}/${channel.site}.config.js` @@ -62,24 +82,19 @@ async function generate() { } } -async function save(filepath, programs) { +async function save(filepath, channels, programs = []) { let output = { - channels: [], + channels, programs: [], date: CURR_DATE } for (let programData of programs) { - let channelData = api_channels[programData.channel] - if (!channelData) continue + let channel = channels.find(c => c.id === programData.channel) + if (!channel) continue - channelData.site = programData.site - channelData.lang = programData.titles[0].lang - - let channel = new Channel(channelData) let program = new Program(programData, channel) - output.channels.push(channel) output.programs.push(program) } @@ -100,17 +115,3 @@ async function save(filepath, programs) { return output } - -// function merge(p1, p2) { -// for (let prop in p1) { -// if (Array.isArray(p1[prop])) { -// p1[prop] = _.orderBy( -// _.uniqWith(p1[prop].concat(p2[prop]), _.isEqual), -// v => (v.lang === 'en' ? Infinity : 1), -// 'desc' -// ) -// } -// } - -// return p1 -// } diff --git a/tests/__data__/expected/guides/en/directv.com.xml b/tests/__data__/expected/guides/en/directv.com.xml new file mode 100644 index 00000000..2321284a --- /dev/null +++ b/tests/__data__/expected/guides/en/directv.com.xml @@ -0,0 +1,3 @@ + +Bravo Easthttps://directv.com + \ No newline at end of file diff --git a/tests/__data__/expected/guides/en/directv.com.xml.gz b/tests/__data__/expected/guides/en/directv.com.xml.gz new file mode 100644 index 00000000..8e41c067 Binary files /dev/null and b/tests/__data__/expected/guides/en/directv.com.xml.gz differ diff --git a/tests/__data__/expected/logs/guides/update.log b/tests/__data__/expected/logs/guides/update.log index bf20652e..ed53a3be 100644 --- a/tests/__data__/expected/logs/guides/update.log +++ b/tests/__data__/expected/logs/guides/update.log @@ -1,5 +1,6 @@ {"site":"allente.se","lang":"da","days":2,"channel":"6eren.dk","filename":"da/allente.se"} +{"site":"directv.com","lang":"en","days":2,"channel":"BravoEast.us","filename":"en/directv.com"} +{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"} {"site":"virginmedia.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/virginmedia.com"} {"site":"sky.com","lang":"fr","days":2,"channel":"BBCNews.uk","filename":"fr/sky.com"} -{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"} -{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"} \ No newline at end of file +{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"} \ No newline at end of file diff --git a/tests/__data__/input/database/update-guides/queue.db b/tests/__data__/input/database/update-guides/queue.db new file mode 100644 index 00000000..62ab23c3 --- /dev/null +++ b/tests/__data__/input/database/update-guides/queue.db @@ -0,0 +1,8 @@ +{"channel":{"lang":"da","id":"6eren.dk","name":"6eren","site_id":"237","logo":"","site":"allente.se","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"} +{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"virginmedia.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"} +{"channel":{"lang":"fr","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Zefq0oMR3feCcuY"} +{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Qefq0oMR3feCcuY"} +{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"} +{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-22T00:00:00Z","_id":"2XzrxNkSF2AQNBrT"} +{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"01AluKCrCnfgrl8W"} +{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-22T00:00:00Z","_id":"02AluKCrCnfgrl8W"} \ No newline at end of file diff --git a/tests/commands/guides/update.test.js b/tests/commands/guides/update.test.js index 61aba01c..304706eb 100644 --- a/tests/commands/guides/update.test.js +++ b/tests/commands/guides/update.test.js @@ -9,6 +9,10 @@ beforeEach(() => { 'tests/__data__/input/database/update-guides/programs.db', 'tests/__data__/output/programs.db' ) + fs.copyFileSync( + 'tests/__data__/input/database/update-guides/queue.db', + 'tests/__data__/output/queue.db' + ) }) it('can generate /guides', () => {