Create commands/load-cluster.js

scripts/commands/create-database.js

@@ -32,9 +32,11 @@ async function loadChannels() {
 
   const files = await file.list(options.channels)
   for (const filepath of files) {
+    const dir = file.dirname(filepath)
     const items = await parser.parseChannels(filepath)
     for (const item of items) {
-      item.filepath = filepath
+      item.channelsPath = filepath
+      item.configPath = `${dir}/${item.site}.config.js`
       channels.push(item)
     }
   }

scripts/commands/load-cluster.js

@@ -0,0 +1,46 @@
+const grabber = require('epg-grabber')
+const { program } = require('commander')
+const { db, logger, timer, file, parser } = require('../core')
+
+const options = program
+  .requiredOption('-c, --cluster-id <cluster-id>', 'The ID of cluster to load', parser.parseNumber)
+  .parse(process.argv)
+  .opts()
+
+const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs'
+
+async function main() {
+  logger.info('Starting...')
+  timer.start()
+
+  const clusterLog = `${LOGS_PATH}/load-cluster/cluster_${options.clusterId}.log`
+  logger.info(`Loading cluster: ${options.clusterId}`)
+  logger.info(`Creating '${clusterLog}'...`)
+  await file.create(clusterLog)
+  const items = await db.find({ cluster_id: options.clusterId })
+  const total = items.length
+  logger.info(`Found ${total} links`)
+
+  logger.info('Loading...')
+  const results = {}
+  let i = 1
+  for (const item of items) {
+    const config = require(file.resolve(item.configPath))
+    const programs = await grabber.grab(item, config, (data, err) => {
+      logger.info(
+        `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format(
+          'MMM D, YYYY'
+        )} (${data.programs.length} programs)`
+      )
+
+      if (err) logger.error(err.message)
+
+      if (i < total) i++
+    })
+    await file.append(clusterLog, JSON.stringify({ [item._id]: programs }) + '\n')
+  }
+
+  logger.info(`Done in ${timer.format('HH[h] mm[m] ss[s]')}`)
+}
+
+main()

scripts/core/index.js

@@ -2,3 +2,4 @@ exports.db = require('./db')
 exports.logger = require('./logger')
 exports.file = require('./file')
 exports.parser = require('./parser')
+exports.timer = require('./timer')

scripts/core/timer.js

@@ -0,0 +1,29 @@
+const { performance } = require('perf_hooks')
+const dayjs = require('dayjs')
+const duration = require('dayjs/plugin/duration')
+const relativeTime = require('dayjs/plugin/relativeTime')
+
+dayjs.extend(relativeTime)
+dayjs.extend(duration)
+
+const timer = {}
+
+let t0 = 0
+
+timer.start = function () {
+  t0 = performance.now()
+}
+
+timer.format = function (f) {
+  let t1 = performance.now()
+
+  return dayjs.duration(t1 - t0).format(f)
+}
+
+timer.humanize = function (suffix = true) {
+  let t1 = performance.now()
+
+  return dayjs.duration(t1 - t0).humanize(suffix)
+}
+
+module.exports = timer

tests/__data__/input/site.config.js

@@ -0,0 +1,65 @@
+const cheerio = require('cheerio')
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+const timezone = require('dayjs/plugin/timezone')
+const customParseFormat = require('dayjs/plugin/customParseFormat')
+require('dayjs/locale/ca')
+
+dayjs.extend(utc)
+dayjs.extend(timezone)
+dayjs.extend(customParseFormat)
+
+module.exports = {
+  site: 'andorradifusio.ad',
+  url({ channel }) {
+    return `https://www.andorradifusio.ad/programacio/${channel.site_id}`
+  },
+  parser({ content, date }) {
+    const programs = []
+    const items = parseItems(content, date)
+    items.forEach(item => {
+      const prev = programs[programs.length - 1]
+      let start = parseStart(item, date)
+      if (prev) {
+        if (start.isBefore(prev.start)) {
+          start = start.add(1, 'd')
+          date = date.add(1, 'd')
+        }
+        prev.stop = start
+      }
+      const stop = start.add(1, 'h')
+      programs.push({
+        title: item.title,
+        start,
+        stop
+      })
+    })
+
+    return programs
+  }
+}
+
+function parseStart(item, date) {
+  const dateString = `${date.format('MM/DD/YYYY')} ${item.time}`
+
+  return dayjs.tz(dateString, 'MM/DD/YYYY HH:mm', 'Europe/Madrid')
+}
+
+function parseItems(content, date) {
+  const $ = cheerio.load(content)
+  const dayOfWeek = dayjs(date).locale('ca').format('dddd').toLowerCase()
+  const column = $('.programacio-dia > h3')
+    .filter((i, el) => $(el).text().startsWith(dayOfWeek))
+    .first()
+    .parent()
+  const items = []
+  const titles = column.find(`p`).toArray()
+  column.find(`h4`).each((i, time) => {
+    items.push({
+      time: $(time).text(),
+      title: $(titles[i]).text()
+    })
+  })
+
+  return items
+}

tests/__data__/input/test.db

@@ -0,0 +1 @@
+{"lang":"ca","xmltv_id":"AndorraTV.ad","site_id":"atv","name":"Andorra TV","site":"andorradifusio.ad","channelsPath":"tests/__data__/input/site.channels.xml","configPath":"tests/__data__/input/site.config.js","cluster_id":1,"_id":"K1kaxwsWVjsRIZL6"}

tests/__data__/output/test.db

@@ -1 +0,0 @@
-{"lang":"ca","xmltv_id":"AndorraTV.ad","site_id":"atv","name":"Andorra TV","site":"andorradifusio.ad","filepath":"tests/__data__/input/site.channels.xml","cluster_id":1,"_id":"2lf1xEfDjyxPsXhO"}

tests/commands/create-database.test.js

@@ -23,7 +23,8 @@ it('can create database', () => {
     xmltv_id: 'AndorraTV.ad',
     site_id: 'atv',
     site: 'andorradifusio.ad',
-    filepath: 'tests/__data__/input/site.channels.xml',
+    channelsPath: 'tests/__data__/input/site.channels.xml',
+    configPath: 'tests/__data__/input/andorradifusio.ad.config.js',
     cluster_id: 1
   })
 })

tests/commands/load-cluster.test.js

@@ -0,0 +1,36 @@
+const fs = require('fs')
+const path = require('path')
+const { execSync } = require('child_process')
+
+beforeEach(() => {
+  fs.rmdirSync('tests/__data__/output', { recursive: true })
+  fs.mkdirSync('tests/__data__/output')
+  fs.copyFileSync('tests/__data__/input/test.db', 'tests/__data__/temp/test.db')
+})
+
+afterEach(() => {
+  fs.rmdirSync('tests/__data__/temp', { recursive: true })
+  fs.mkdirSync('tests/__data__/temp')
+})
+
+it('can load cluster', () => {
+  const result = execSync(
+    'DB_FILEPATH=tests/__data__/temp/test.db LOGS_PATH=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1',
+    { encoding: 'utf8' }
+  )
+  const logs = fs.readFileSync(
+    path.resolve('tests/__data__/output/logs/load-cluster/cluster_1.log'),
+    {
+      encoding: 'utf8'
+    }
+  )
+  const lines = logs.split('\n')
+  const parsed = JSON.parse(lines[0])
+  expect(parsed['K1kaxwsWVjsRIZL6'][0]).toMatchObject({
+    title: 'InfoNeu ',
+    start: '2022-01-06T07:00:00.000Z',
+    stop: '2022-01-06T08:00:00.000Z',
+    channel: 'AndorraTV.ad',
+    lang: 'ca'
+  })
+})