From 7a8bfc69078f0d1a74182bfaa9d7c4fcb68ee8bb Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:03:41 +0300 Subject: [PATCH 1/9] Update update.test.js --- tests/commands/guides/update.test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/commands/guides/update.test.js b/tests/commands/guides/update.test.js index 61aba01c..304706eb 100644 --- a/tests/commands/guides/update.test.js +++ b/tests/commands/guides/update.test.js @@ -9,6 +9,10 @@ beforeEach(() => { 'tests/__data__/input/database/update-guides/programs.db', 'tests/__data__/output/programs.db' ) + fs.copyFileSync( + 'tests/__data__/input/database/update-guides/queue.db', + 'tests/__data__/output/queue.db' + ) }) it('can generate /guides', () => { From 43e4aedc2f7edc85fae02ace7ec8d9a3f510e5a4 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:03:59 +0300 Subject: [PATCH 2/9] Update tests/__data__ --- tests/__data__/expected/guides/en/directv.com.xml | 3 +++ .../expected/guides/en/directv.com.xml.gz | Bin 0 -> 200 bytes tests/__data__/expected/logs/guides/update.log | 5 +++-- .../input/database/update-guides/queue.db | 6 ++++++ 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 tests/__data__/expected/guides/en/directv.com.xml create mode 100644 tests/__data__/expected/guides/en/directv.com.xml.gz create mode 100644 tests/__data__/input/database/update-guides/queue.db diff --git a/tests/__data__/expected/guides/en/directv.com.xml b/tests/__data__/expected/guides/en/directv.com.xml new file mode 100644 index 00000000..2321284a --- /dev/null +++ b/tests/__data__/expected/guides/en/directv.com.xml @@ -0,0 +1,3 @@ + +Bravo Easthttps://directv.com + \ No newline at end of file diff --git a/tests/__data__/expected/guides/en/directv.com.xml.gz b/tests/__data__/expected/guides/en/directv.com.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..8e41c0678ead3a0f57d6afcd6bd8a524ea67c787 GIT binary patch literal 200 zcmV;(05|_1iwFP!000006IISjN(31E`awK9uJk49wF!AE$W&dwWCf z%%y~^k~i?&c+?k`t}g1f7EM|RI! literal 0 HcmV?d00001 diff --git a/tests/__data__/expected/logs/guides/update.log b/tests/__data__/expected/logs/guides/update.log index bf20652e..ed53a3be 100644 --- a/tests/__data__/expected/logs/guides/update.log +++ b/tests/__data__/expected/logs/guides/update.log @@ -1,5 +1,6 @@ {"site":"allente.se","lang":"da","days":2,"channel":"6eren.dk","filename":"da/allente.se"} +{"site":"directv.com","lang":"en","days":2,"channel":"BravoEast.us","filename":"en/directv.com"} +{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"} {"site":"virginmedia.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/virginmedia.com"} {"site":"sky.com","lang":"fr","days":2,"channel":"BBCNews.uk","filename":"fr/sky.com"} -{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"} -{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"} \ No newline at end of file +{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"} \ No newline at end of file diff --git a/tests/__data__/input/database/update-guides/queue.db b/tests/__data__/input/database/update-guides/queue.db new file mode 100644 index 00000000..125d68f5 --- /dev/null +++ b/tests/__data__/input/database/update-guides/queue.db @@ -0,0 +1,6 @@ +{"channel":{"lang":"da","id":"6eren.dk","name":"6eren","site_id":"237","logo":"","site":"allente.se","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"} +{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"virginmedia.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"} +{"channel":{"lang":"fr","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Zefq0oMR3feCcuY"} +{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Qefq0oMR3feCcuY"} +{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"} +{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"01AluKCrCnfgrl8W"} \ No newline at end of file From 500746eaa54e6aa66d7e5dcfcaef3e9a784db696 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:04:04 +0300 Subject: [PATCH 3/9] Update update.js --- scripts/commands/guides/update.js | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/commands/guides/update.js b/scripts/commands/guides/update.js index 0ed5f146..f31e7cf8 100644 --- a/scripts/commands/guides/update.js +++ b/scripts/commands/guides/update.js @@ -9,6 +9,7 @@ const CURR_DATE = process.env.CURR_DATE || new Date() const logPath = `${LOGS_DIR}/guides/update.log` let api_channels = {} +let db_queue = [] let db_programs = [] let guides = [] @@ -22,6 +23,11 @@ async function main() { api_channels[channel.id] = channel }) + logger.info('loading database/queue.db...') + await db.queue.load() + db_queue = await db.queue.find({}) + logger.info(`found ${db_queue.length} channels`) + logger.info('loading database/programs.db...') await db.programs.load() db_programs = await db.programs.find({}) @@ -38,14 +44,27 @@ async function main() { main() async function generate() { + let queue = _.groupBy(db_queue, i => (i.channel ? `${i.channel.lang}/${i.channel.site}` : `_`)) + delete queue['_'] + let programs = _.groupBy(db_programs, p => p.titles.length ? `${p.titles[0].lang}/${p.site}` : `_` ) delete programs['_'] - for (let filename in programs) { - let { channels } = await save(filename, programs[filename]) + for (let filename in queue) { + if (!queue[filename]) continue + const channels = queue[filename].map(i => { + const channelData = api_channels[i.channel.id] + channelData.site = i.channel.site + channelData.site_id = i.channel.site_id + channelData.lang = i.channel.lang + + return new Channel(channelData) + }) + + await save(filename, channels, programs[filename]) for (let channel of channels) { const configPath = `sites/${channel.site}/${channel.site}.config.js` @@ -62,24 +81,19 @@ async function generate() { } } -async function save(filepath, programs) { +async function save(filepath, channels, programs = []) { let output = { - channels: [], + channels, programs: [], date: CURR_DATE } for (let programData of programs) { - let channelData = api_channels[programData.channel] - if (!channelData) continue + let channel = channels.find(c => c.id === programData.channel) + if (!channel) continue - channelData.site = programData.site - channelData.lang = programData.titles[0].lang - - let channel = new Channel(channelData) let program = new Program(programData, channel) - output.channels.push(channel) output.programs.push(program) } @@ -100,17 +114,3 @@ async function save(filepath, programs) { return output } - -// function merge(p1, p2) { -// for (let prop in p1) { -// if (Array.isArray(p1[prop])) { -// p1[prop] = _.orderBy( -// _.uniqWith(p1[prop].concat(p2[prop]), _.isEqual), -// v => (v.lang === 'en' ? Infinity : 1), -// 'desc' -// ) -// } -// } - -// return p1 -// } From c7a7d856b56de32d2206f42da1ddc98a6e1859a9 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:33:48 +0300 Subject: [PATCH 4/9] Delete load.js --- scripts/commands/programs/load.js | 110 ------------------------------ 1 file changed, 110 deletions(-) delete mode 100644 scripts/commands/programs/load.js diff --git a/scripts/commands/programs/load.js b/scripts/commands/programs/load.js deleted file mode 100644 index a0e900a7..00000000 --- a/scripts/commands/programs/load.js +++ /dev/null @@ -1,110 +0,0 @@ -const { Octokit } = require('@octokit/core') -const dayjs = require('dayjs') -const isToday = require('dayjs/plugin/isToday') -const utc = require('dayjs/plugin/utc') -const unzipit = require('unzipit') -const { file, logger } = require('../../core') - -dayjs.extend(isToday) -dayjs.extend(utc) - -const DB_DIR = process.env.DB_DIR || './scripts/database' -const dbPath = `${DB_DIR}/programs.db` - -const octokit = new Octokit({ - auth: process.env.GITHUB_TOKEN -}) - -async function main() { - try { - let workflows = await getWorkflows() - logger.info(`found ${workflows.length} workflows\r\n`) - - await file.create(dbPath) - const total = workflows.length - for (let [i, workflow] of workflows.entries()) { - logger.info(`[${i + 1}/${total}] ${workflow.name}`) - const run = await getWorkflowRun(workflow) - if (!run) continue - - let artifact = await getRunArtifacts(run) - - const buffer = await downloadArtifact(artifact) - await file.append(dbPath, buffer) - } - } catch (err) { - console.log(err.message) - } -} - -main() - -async function downloadArtifact(artifact) { - let results = await octokit.request( - 'GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}', - { - owner: 'iptv-org', - repo: 'epg', - artifact_id: artifact.id, - archive_format: 'zip' - } - ) - - const { entries } = await unzipit.unzip(results.data) - - const arrayBuffer = await entries['programs.db'].arrayBuffer() - - return toString(arrayBuffer) -} - -async function getRunArtifacts(run) { - let results = await octokit.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', { - owner: 'iptv-org', - repo: 'epg', - run_id: run.id - }) - - return results.data.artifacts.find(a => a.name === 'database') -} - -async function getWorkflowRun(workflow) { - let today = dayjs.utc().subtract(1, 'd').format('YYYY-MM-DD') - let results = await octokit.request( - 'GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', - { - owner: 'iptv-org', - repo: 'epg', - workflow_id: workflow.id, - status: 'success', - created: `>=${today}` - } - ) - - return results.data.workflow_runs.find( - r => r.event === 'schedule' || r.event === 'workflow_dispatch' - ) -} - -async function getWorkflows() { - let workflows = [] - for (let page of [1, 2, 3]) { - try { - let results = await octokit.request('GET /repos/{owner}/{repo}/actions/workflows', { - owner: 'iptv-org', - repo: 'epg', - per_page: 100, - page - }) - - workflows = workflows.concat(results.data.workflows) - } catch (err) { - console.log(err.message) - } - } - - return workflows.filter(w => !/^_/.test(w.name) && w.name !== 'pages-build-deployment') -} - -function toString(arrayBuffer) { - return new TextDecoder().decode(arrayBuffer) -} From 66303f4a604f6a07280c7bb4dbd3cd876f8f822f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:33:58 +0300 Subject: [PATCH 5/9] Create load.js --- scripts/commands/database/load.js | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 scripts/commands/database/load.js diff --git a/scripts/commands/database/load.js b/scripts/commands/database/load.js new file mode 100644 index 00000000..147f16f5 --- /dev/null +++ b/scripts/commands/database/load.js @@ -0,0 +1,115 @@ +const { Octokit } = require('@octokit/core') +const dayjs = require('dayjs') +const isToday = require('dayjs/plugin/isToday') +const utc = require('dayjs/plugin/utc') +const unzipit = require('unzipit') +const { file, logger } = require('../../core') + +dayjs.extend(isToday) +dayjs.extend(utc) + +const DB_DIR = process.env.DB_DIR || './scripts/database' +const programsPath = `${DB_DIR}/programs.db` +const queuePath = `${DB_DIR}/queue.db` + +const octokit = new Octokit({ + auth: process.env.GITHUB_TOKEN +}) + +async function main() { + try { + let workflows = await getWorkflows() + logger.info(`found ${workflows.length} workflows\r\n`) + + await file.create(programsPath) + await file.create(queuePath) + const total = workflows.length + for (let [i, workflow] of workflows.entries()) { + logger.info(`[${i + 1}/${total}] ${workflow.name}`) + const run = await getWorkflowRun(workflow) + if (!run) continue + + let artifact = await getRunArtifacts(run) + + const programsBuffer = await downloadArtifact(artifact, 'programs.db') + await file.append(programsPath, programsBuffer) + + const queueBuffer = await downloadArtifact(artifact, 'queue.db') + await file.append(queuePath, queueBuffer) + } + } catch (err) { + console.log(err.message) + } +} + +main() + +async function downloadArtifact(artifact, filename) { + let results = await octokit.request( + 'GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}', + { + owner: 'iptv-org', + repo: 'epg', + artifact_id: artifact.id, + archive_format: 'zip' + } + ) + + const { entries } = await unzipit.unzip(results.data) + + const arrayBuffer = await entries[filename].arrayBuffer() + + return toString(arrayBuffer) +} + +async function getRunArtifacts(run) { + let results = await octokit.request('GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts', { + owner: 'iptv-org', + repo: 'epg', + run_id: run.id + }) + + return results.data.artifacts.find(a => a.name === 'database') +} + +async function getWorkflowRun(workflow) { + let today = dayjs.utc().subtract(1, 'd').format('YYYY-MM-DD') + let results = await octokit.request( + 'GET /repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs', + { + owner: 'iptv-org', + repo: 'epg', + workflow_id: workflow.id, + status: 'success', + created: `>=${today}` + } + ) + + return results.data.workflow_runs.find( + r => r.event === 'schedule' || r.event === 'workflow_dispatch' + ) +} + +async function getWorkflows() { + let workflows = [] + for (let page of [1, 2, 3]) { + try { + let results = await octokit.request('GET /repos/{owner}/{repo}/actions/workflows', { + owner: 'iptv-org', + repo: 'epg', + per_page: 100, + page + }) + + workflows = workflows.concat(results.data.workflows) + } catch (err) { + console.log(err.message) + } + } + + return workflows.filter(w => !/^_/.test(w.name) && w.name !== 'pages-build-deployment') +} + +function toString(arrayBuffer) { + return new TextDecoder().decode(arrayBuffer) +} From f24992f9eaea2e5bd92765a9502a4ddfd99571e1 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:34:03 +0300 Subject: [PATCH 6/9] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c5883adb..d396c8ae 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "channels:editor": "node scripts/commands/channels/editor.js", "queue:create": "node scripts/commands/queue/create.js", "cluster:load": "node scripts/commands/cluster/load.js", - "programs:load": "node scripts/commands/programs/load.js", + "db:load": "node scripts/commands/database/load.js", "programs:save": "node scripts/commands/programs/save.js", "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js", "api:load": "./scripts/commands/api/load.sh", From 1afa72e87fce2e7ff25acdeb4d1a3f86e6c2d0d7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:34:06 +0300 Subject: [PATCH 7/9] Update _update.yml --- .github/workflows/_update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_update.yml b/.github/workflows/_update.yml index 7a7856ce..1ab62d8c 100644 --- a/.github/workflows/_update.yml +++ b/.github/workflows/_update.yml @@ -27,7 +27,7 @@ jobs: - run: npm install - run: npm run api:load - if: ${{ !env.ACT }} - run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run programs:load + run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run db:load - uses: actions/upload-artifact@v3 with: name: database From 51165a1e920dd757031c9bd3804343427643ff14 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:40:22 +0300 Subject: [PATCH 8/9] Update queue.db --- tests/__data__/input/database/update-guides/queue.db | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/__data__/input/database/update-guides/queue.db b/tests/__data__/input/database/update-guides/queue.db index 125d68f5..62ab23c3 100644 --- a/tests/__data__/input/database/update-guides/queue.db +++ b/tests/__data__/input/database/update-guides/queue.db @@ -3,4 +3,6 @@ {"channel":{"lang":"fr","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Zefq0oMR3feCcuY"} {"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Qefq0oMR3feCcuY"} {"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"} -{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"01AluKCrCnfgrl8W"} \ No newline at end of file +{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-22T00:00:00Z","_id":"2XzrxNkSF2AQNBrT"} +{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"01AluKCrCnfgrl8W"} +{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-22T00:00:00Z","_id":"02AluKCrCnfgrl8W"} \ No newline at end of file From f581df2b6d5d29dcaf4e392c504edcec42a51746 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 9 Feb 2023 13:40:24 +0300 Subject: [PATCH 9/9] Update update.js --- scripts/commands/guides/update.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/commands/guides/update.js b/scripts/commands/guides/update.js index f31e7cf8..f4ee10f9 100644 --- a/scripts/commands/guides/update.js +++ b/scripts/commands/guides/update.js @@ -26,7 +26,7 @@ async function main() { logger.info('loading database/queue.db...') await db.queue.load() db_queue = await db.queue.find({}) - logger.info(`found ${db_queue.length} channels`) + logger.info(`found ${db_queue.length} items`) logger.info('loading database/programs.db...') await db.programs.load() @@ -44,7 +44,8 @@ async function main() { main() async function generate() { - let queue = _.groupBy(db_queue, i => (i.channel ? `${i.channel.lang}/${i.channel.site}` : `_`)) + let queue = _.uniqBy(db_queue, i => i.channel.lang + i.channel.id + i.channel.site) + queue = _.groupBy(queue, i => (i.channel ? `${i.channel.lang}/${i.channel.site}` : `_`)) delete queue['_'] let programs = _.groupBy(db_programs, p =>