Merge pull request #1995 from iptv-org/patch-2023.02.1

Patch 2023.02.1

.github/workflows/_update.yml

@@ -27,7 +27,7 @@ jobs:
       - run: npm install
       - run: npm run api:load
       - if: ${{ !env.ACT }}
-        run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run programs:load
+        run: GITHUB_TOKEN=${{ steps.create-app-token.outputs.token }} npm run db:load
       - uses: actions/upload-artifact@v3
         with:
           name: database

package.json

@@ -7,7 +7,7 @@
     "channels:editor": "node scripts/commands/channels/editor.js",
     "queue:create": "node scripts/commands/queue/create.js",
     "cluster:load": "node scripts/commands/cluster/load.js",
-    "programs:load": "node scripts/commands/programs/load.js",
+    "db:load": "node scripts/commands/database/load.js",
     "programs:save": "node scripts/commands/programs/save.js",
     "guides:update": "NODE_OPTIONS=--max-old-space-size=5120 node scripts/commands/guides/update.js",
     "api:load": "./scripts/commands/api/load.sh",

scripts/commands/database/load.js

@@ -9,7 +9,8 @@ dayjs.extend(isToday)
 dayjs.extend(utc)
 
 const DB_DIR = process.env.DB_DIR || './scripts/database'
-const dbPath = `${DB_DIR}/programs.db`
+const programsPath = `${DB_DIR}/programs.db`
+const queuePath = `${DB_DIR}/queue.db`
 
 const octokit = new Octokit({
   auth: process.env.GITHUB_TOKEN
@@ -20,7 +21,8 @@ async function main() {
     let workflows = await getWorkflows()
     logger.info(`found ${workflows.length} workflows\r\n`)
 
-    await file.create(dbPath)
+    await file.create(programsPath)
+    await file.create(queuePath)
     const total = workflows.length
     for (let [i, workflow] of workflows.entries()) {
       logger.info(`[${i + 1}/${total}] ${workflow.name}`)
@@ -29,8 +31,11 @@ async function main() {
 
       let artifact = await getRunArtifacts(run)
 
-      const buffer = await downloadArtifact(artifact)
-      await file.append(dbPath, buffer)
+      const programsBuffer = await downloadArtifact(artifact, 'programs.db')
+      await file.append(programsPath, programsBuffer)
+
+      const queueBuffer = await downloadArtifact(artifact, 'queue.db')
+      await file.append(queuePath, queueBuffer)
     }
   } catch (err) {
     console.log(err.message)
@@ -39,7 +44,7 @@ async function main() {
 
 main()
 
-async function downloadArtifact(artifact) {
+async function downloadArtifact(artifact, filename) {
   let results = await octokit.request(
     'GET /repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}',
     {
@@ -52,7 +57,7 @@ async function downloadArtifact(artifact) {
 
   const { entries } = await unzipit.unzip(results.data)
 
-  const arrayBuffer = await entries['programs.db'].arrayBuffer()
+  const arrayBuffer = await entries[filename].arrayBuffer()
 
   return toString(arrayBuffer)
 }

scripts/commands/guides/update.js

@@ -9,6 +9,7 @@ const CURR_DATE = process.env.CURR_DATE || new Date()
 const logPath = `${LOGS_DIR}/guides/update.log`
 
 let api_channels = {}
+let db_queue = []
 let db_programs = []
 let guides = []
 
@@ -22,6 +23,11 @@ async function main() {
     api_channels[channel.id] = channel
   })
 
+  logger.info('loading database/queue.db...')
+  await db.queue.load()
+  db_queue = await db.queue.find({})
+  logger.info(`found ${db_queue.length} items`)
+
   logger.info('loading database/programs.db...')
   await db.programs.load()
   db_programs = await db.programs.find({})
@@ -38,14 +44,28 @@ async function main() {
 main()
 
 async function generate() {
+  let queue = _.uniqBy(db_queue, i => i.channel.lang + i.channel.id + i.channel.site)
+  queue = _.groupBy(queue, i => (i.channel ? `${i.channel.lang}/${i.channel.site}` : `_`))
+  delete queue['_']
+
   let programs = _.groupBy(db_programs, p =>
     p.titles.length ? `${p.titles[0].lang}/${p.site}` : `_`
   )
 
   delete programs['_']
 
-  for (let filename in programs) {
-    let { channels } = await save(filename, programs[filename])
+  for (let filename in queue) {
+    if (!queue[filename]) continue
+    const channels = queue[filename].map(i => {
+      const channelData = api_channels[i.channel.id]
+      channelData.site = i.channel.site
+      channelData.site_id = i.channel.site_id
+      channelData.lang = i.channel.lang
+
+      return new Channel(channelData)
+    })
+
+    await save(filename, channels, programs[filename])
 
     for (let channel of channels) {
       const configPath = `sites/${channel.site}/${channel.site}.config.js`
@@ -62,24 +82,19 @@ async function generate() {
   }
 }
 
-async function save(filepath, programs) {
+async function save(filepath, channels, programs = []) {
   let output = {
-    channels: [],
+    channels,
     programs: [],
     date: CURR_DATE
   }
 
   for (let programData of programs) {
-    let channelData = api_channels[programData.channel]
-    if (!channelData) continue
+    let channel = channels.find(c => c.id === programData.channel)
+    if (!channel) continue
 
-    channelData.site = programData.site
-    channelData.lang = programData.titles[0].lang
-
-    let channel = new Channel(channelData)
     let program = new Program(programData, channel)
 
-    output.channels.push(channel)
     output.programs.push(program)
   }
 
@@ -100,17 +115,3 @@ async function save(filepath, programs) {
 
   return output
 }
-
-// function merge(p1, p2) {
-//   for (let prop in p1) {
-//     if (Array.isArray(p1[prop])) {
-//       p1[prop] = _.orderBy(
-//         _.uniqWith(p1[prop].concat(p2[prop]), _.isEqual),
-//         v => (v.lang === 'en' ? Infinity : 1),
-//         'desc'
-//       )
-//     }
-//   }
-
-//   return p1
-// }

tests/__data__/expected/guides/en/directv.com.xml

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="BravoEast.us"><display-name>Bravo East</display-name><icon src="https://www.directv.com/images/logos/channels/dark/large/579.png"/><url>https://directv.com</url></channel>
+</tv>

tests/__data__/expected/logs/guides/update.log

@@ -1,5 +1,6 @@
 {"site":"allente.se","lang":"da","days":2,"channel":"6eren.dk","filename":"da/allente.se"}
+{"site":"directv.com","lang":"en","days":2,"channel":"BravoEast.us","filename":"en/directv.com"}
+{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"}
 {"site":"virginmedia.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/virginmedia.com"}
 {"site":"sky.com","lang":"fr","days":2,"channel":"BBCNews.uk","filename":"fr/sky.com"}
-{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"}
-{"site":"sky.com","lang":"en","days":2,"channel":"BBCNews.uk","filename":"en/sky.com"}
+{"site":"sky.com","lang":"fr","days":2,"channel":"CNN.us","filename":"fr/sky.com"}

tests/__data__/input/database/update-guides/queue.db

@@ -0,0 +1,8 @@
+{"channel":{"lang":"da","id":"6eren.dk","name":"6eren","site_id":"237","logo":"","site":"allente.se","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"}
+{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"virginmedia.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"}
+{"channel":{"lang":"fr","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Zefq0oMR3feCcuY"}
+{"channel":{"lang":"en","id":"BBCNews.uk","name":"BBC News","site_id":"53","logo":"","site":"sky.com","url":"https://chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Qefq0oMR3feCcuY"}
+{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"}
+{"channel":{"lang":"fr","id":"CNN.us","name":"CNN","site_id":"140","logo":"","site":"sky.com","url":"https://magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","error":null,"cluster_id":1,"date":"2022-01-22T00:00:00Z","_id":"2XzrxNkSF2AQNBrT"}
+{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"01AluKCrCnfgrl8W"}
+{"channel":{"lang":"en","id":"BravoEast.us","name":"Bravo East","site_id":"237","logo":"","site":"directv.com","url":"https://directv.com"},"configPath":"sites/directv.com/directv.com.config.js","error":"Invalid header value char","cluster_id":84,"date":"2022-01-22T00:00:00Z","_id":"02AluKCrCnfgrl8W"}

tests/commands/guides/update.test.js

@@ -9,6 +9,10 @@ beforeEach(() => {
     'tests/__data__/input/database/update-guides/programs.db',
     'tests/__data__/output/programs.db'
   )
+  fs.copyFileSync(
+    'tests/__data__/input/database/update-guides/queue.db',
+    'tests/__data__/output/queue.db'
+  )
 })
 
 it('can generate /guides', () => {