Merge pull request #471 from iptv-org/patch-02.2022

Patch 02.2022
2025-05-11 01:20:08 -04:00 · 2022-01-31 21:58:01 +03:00 · 2022-01-31 21:58:01 +03:00 · b4f69fced5
commit b4f69fced5
parent 19e8f0d705 dfe65a9f3f
53 changed files with 2518 additions and 1913 deletions
--- a/.github/workflows/auto-update.yml
+++ b/.github/workflows/auto-update.yml
@ -8,13 +8,26 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
+      - name: Delete .artifacts/ folder
+        if: ${{ env.ACT }}
+        run: rm -rf .artifacts/
+      - name: Download data from API
+        run: |
+          mkdir -p scripts/data
+          curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json
+          curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json
+          curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json
+      - uses: actions/upload-artifact@v2
+        with:
+          name: data
+          path: scripts/data
      - uses: actions/setup-node@v2
        if: ${{ !env.ACT }}
        with:
          node-version: '14'
          cache: 'npm'
      - run: npm install
-      - run: node scripts/commands/create-database.js
+      - run: node scripts/commands/create-queue.js --max-clusters=256 --days=2
      - run: node scripts/commands/create-matrix.js
        id: create-matrix
      - uses: actions/upload-artifact@v2
@ -44,7 +57,7 @@ jobs:
        with:
          node-version: '14'
      - run: npm install
-      - run: NODE_OPTIONS=--insecure-http-parser node scripts/commands/load-cluster.js --days=2 --timeout=30000 --cluster-id=${{ matrix.cluster_id }}
+      - run: NODE_OPTIONS=--insecure-http-parser node scripts/commands/load-cluster.js --timeout=30000 --cluster-id=${{ matrix.cluster_id }}
      - uses: actions/upload-artifact@v2
        with:
          name: logs
@ -59,11 +72,14 @@ jobs:
      - run: git config user.name 'iptv-bot[bot]'
      - run: git config user.email '84861620+iptv-bot[bot]@users.noreply.github.com'
      - run: git checkout -b ${{ steps.create-branch-name.outputs.branch_name }}
-      - run: curl -L -o scripts/data/codes.json https://iptv-org.github.io/epg/codes.json
      - uses: actions/download-artifact@v2
        with:
          name: database
          path: scripts/database
+      - uses: actions/download-artifact@v2
+        with:
+          name: data
+          path: scripts/data
      - uses: actions/download-artifact@v2
        with:
          name: logs
@ -74,36 +90,31 @@ jobs:
          node-version: '14'
      - run: npm install
      - run: node scripts/commands/save-results.js
+      - run: NODE_OPTIONS="--max-old-space-size=4096" node scripts/commands/update-guides.js
+      - run: NODE_OPTIONS="--max-old-space-size=4096" node scripts/commands/update-api.js
      - uses: actions/upload-artifact@v2
        with:
          name: database
          path: scripts/database
-      - run: NODE_OPTIONS="--max-old-space-size=4096" node scripts/commands/update-api.js
-      - run: NODE_OPTIONS="--max-old-space-size=4096" node scripts/commands/update-guides.js
      - uses: actions/upload-artifact@v2
        with:
          name: logs
          path: scripts/logs
      - uses: actions/upload-artifact@v2
        with:
-          name: errors.log
-          path: scripts/logs/errors.log
-      - uses: actions/upload-artifact@v2
-        with:
-          name: guides.json
-          path: .gh-pages/api/guides.json
+          name: errors
+          path: scripts/logs/errors
      - uses: actions/upload-artifact@v2
        with:
          name: programs.json
          path: .gh-pages/api/programs.json
-      - uses: actions/upload-artifact@v2
-        with:
-          name: guides
-          path: .gh-pages/guides
      - run: node scripts/commands/update-readme.js
-      - run: git add README.md
-      - run: git commit -m "[Bot] Update README.md"
-      - run: git push -u origin ${{ steps.create-branch-name.outputs.branch_name }}
+      - name: Commit changes in README.md
+        if: ${{ !env.ACT }}
+        run: |
+          git add README.md
+          git commit -m "[Bot] Update README.md"
+          git push -u origin ${{ steps.create-branch-name.outputs.branch_name }}
      - uses: tibdex/github-app-token@v1
        if: ${{ !env.ACT }}
        id: create-app-token
--- a/package-lock.json
+++ b/package-lock.json
@ -12,7 +12,7 @@
        "commander": "^8.2.0",
        "csv-parser": "^3.0.0",
        "dayjs": "^1.10.4",
-        "epg-grabber": "^0.19.0",
+        "epg-grabber": "^0.20.0",
        "epg-parser": "^0.1.6",
        "form-data": "^4.0.0",
        "glob": "^7.2.0",
@ -2007,9 +2007,9 @@
      }
    },
    "node_modules/epg-grabber": {
-      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.19.0.tgz",
-      "integrity": "sha512-qXqvhlfT9dqw7L/yo/VaSi4LETCrnQst21YPAqNTmbSsjCeABxF/MfvY5dtJVKf31TfMt0/aK5hPLGmYOsb9mA==",
+      "version": "0.20.0",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.20.0.tgz",
+      "integrity": "sha512-6xmz1AfYKDduEjZqQlYyacR41fb8ITDymORyOcKmTdCs9XXrofYL84lULkEmhsNfksAuPf5hMef/uFptbDh/GA==",
      "dependencies": {
        "axios": "^0.21.1",
        "axios-cookiejar-support": "^1.0.1",
@ -6426,9 +6426,9 @@
      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
    },
    "epg-grabber": {
-      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.19.0.tgz",
-      "integrity": "sha512-qXqvhlfT9dqw7L/yo/VaSi4LETCrnQst21YPAqNTmbSsjCeABxF/MfvY5dtJVKf31TfMt0/aK5hPLGmYOsb9mA==",
+      "version": "0.20.0",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.20.0.tgz",
+      "integrity": "sha512-6xmz1AfYKDduEjZqQlYyacR41fb8ITDymORyOcKmTdCs9XXrofYL84lULkEmhsNfksAuPf5hMef/uFptbDh/GA==",
      "requires": {
        "axios": "^0.21.1",
        "axios-cookiejar-support": "^1.0.1",
--- a/package.json
+++ b/package.json
@ -19,7 +19,7 @@
    "commander": "^8.2.0",
    "csv-parser": "^3.0.0",
    "dayjs": "^1.10.4",
-    "epg-grabber": "^0.19.0",
+    "epg-grabber": "^0.20.0",
    "epg-parser": "^0.1.6",
    "form-data": "^4.0.0",
    "glob": "^7.2.0",
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@ -1,2 +1,3 @@
 database/
-logs/
+logs/
+data/
--- a/scripts/commands/create-database.js
+++ b/scripts/commands/create-database.js
@ -1,88 +0,0 @@
-const { db, file, parser, logger } = require('../core')
-const { program } = require('commander')
-const { shuffle } = require('lodash')
-
-const options = program
-  .option(
-    '--max-clusters <max-clusters>',
-    'Set maximum number of clusters',
-    parser.parseNumber,
-    256
-  )
-  .option('--channels <channels>', 'Set path to channels.xml file', 'sites/**/*.channels.xml')
-  .parse(process.argv)
-  .opts()
-
-async function main() {
-  logger.info('Starting...')
-  logger.info(`Number of clusters: ${options.maxClusters}`)
-
-  await saveToDatabase(await getChannels())
-
-  logger.info('Done')
-}
-
-main()
-
-async function getChannels() {
-  logger.info(`Loading channels...`)
-
-  let channels = {}
-
-  const files = await file.list(options.channels)
-  for (const filepath of files) {
-    const dir = file.dirname(filepath)
-    const { site, channels: items } = await parser.parseChannels(filepath)
-    if (!site) continue
-    const configPath = `${dir}/${site}.config.js`
-    const config = require(file.resolve(configPath))
-    if (config.ignore) continue
-    const filename = file.basename(filepath)
-    const [__, region] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
-    const groupId = `${region}/${site}`
-    for (const item of items) {
-      if (!item.site || !item.site_id || !item.xmltv_id || !item.name) continue
-      const key = `${item.site}:${item.site_id}`
-      if (!channels[key]) {
-        const countryCode = item.xmltv_id.split('.')[1]
-        item.country = countryCode ? countryCode.toUpperCase() : null
-        item.channelsPath = filepath
-        item.configPath = configPath
-        item.groups = []
-
-        channels[key] = item
-      }
-
-      if (!channels[key].groups.includes(groupId)) {
-        channels[key].groups.push(groupId)
-      }
-    }
-  }
-
-  channels = Object.values(channels)
-
-  logger.info(`Found ${channels.length} channels`)
-
-  return channels
-}
-
-async function saveToDatabase(channels = []) {
-  logger.info('Saving to the database...')
-  await db.channels.load()
-  await db.channels.reset()
-  const chunks = split(shuffle(channels), options.maxClusters)
-  for (const [i, chunk] of chunks.entries()) {
-    for (const item of chunk) {
-      item.cluster_id = i + 1
-      await db.channels.insert(item)
-    }
-  }
-}
-
-function split(arr, n) {
-  let result = []
-  for (let i = n; i > 0; i--) {
-    result.push(arr.splice(0, Math.ceil(arr.length / i)))
-  }
-  return result
-}
--- a/scripts/commands/create-matrix.js
+++ b/scripts/commands/create-matrix.js
@ -1,8 +1,8 @@
 const { logger, db } = require('../core')

 async function main() {
-  await db.channels.load()
-  const docs = await db.channels.find({}).sort({ cluster_id: 1 })
+  await db.queue.load()
+  const docs = await db.queue.find({}).sort({ cluster_id: 1 })
  const cluster_id = docs.reduce((acc, curr) => {
    if (!acc.includes(curr.cluster_id)) acc.push(curr.cluster_id)
    return acc
--- a/scripts/commands/create-queue.js
+++ b/scripts/commands/create-queue.js
@ -0,0 +1,129 @@
+const { db, file, parser, logger, date, api } = require('../core')
+const { program } = require('commander')
+const _ = require('lodash')
+
+const options = program
+  .option(
+    '--max-clusters <max-clusters>',
+    'Set maximum number of clusters',
+    parser.parseNumber,
+    256
+  )
+  .option('--days <days>', 'Number of days for which to grab the program', parser.parseNumber, 1)
+  .parse(process.argv)
+  .opts()
+
+const CHANNELS_PATH = process.env.CHANNELS_PATH || 'sites/**/*.channels.xml'
+const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
+
+async function main() {
+  logger.info('Starting...')
+  logger.info(`Number of clusters: ${options.maxClusters}`)
+
+  await saveToDatabase(await createQueue())
+
+  logger.info('Done')
+}
+
+main()
+
+async function createQueue() {
+  logger.info(`Create queue...`)
+
+  let queue = {}
+
+  await api.channels.load()
+  const files = await file.list(CHANNELS_PATH)
+  const utcDate = date.getUTC()
+  const dates = Array.from({ length: options.days }, (_, i) => utcDate.add(i, 'd'))
+  for (const filepath of files) {
+    const dir = file.dirname(filepath)
+    const { site, channels: items } = await parser.parseChannels(filepath)
+    if (!site) continue
+    const configPath = `${dir}/${site}.config.js`
+    const config = require(file.resolve(configPath))
+    if (config.ignore) continue
+    const filename = file.basename(filepath)
+    const [__, region] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
+    const groupId = `${region}/${site}`
+    for (const item of items) {
+      if (!item.site || !item.site_id || !item.xmltv_id) continue
+      const channel = api.channels.find({ id: item.xmltv_id })
+      if (!channel) {
+        await logError(groupId, {
+          xmltv_id: item.xmltv_id,
+          site: item.site,
+          site_id: item.site_id,
+          lang: item.lang,
+          date: undefined,
+          error: 'The channel has the wrong xmltv_id'
+        })
+        continue
+      }
+
+      for (const d of dates) {
+        const dString = d.toJSON()
+        const key = `${item.site}:${item.site_id}:${item.lang}:${dString}`
+        if (!queue[key]) {
+          queue[key] = {
+            channel: {
+              lang: item.lang,
+              xmltv_id: item.xmltv_id,
+              site_id: item.site_id,
+              site: item.site
+            },
+            date: dString,
+            configPath,
+            groups: [],
+            error: null
+          }
+        }
+
+        if (!queue[key].groups.includes(groupId)) {
+          queue[key].groups.push(groupId)
+        }
+      }
+    }
+  }
+
+  queue = Object.values(queue)
+
+  logger.info(`Added ${queue.length} items`)
+
+  return queue
+}
+
+async function saveToDatabase(items = []) {
+  logger.info('Saving to the database...')
+  await db.queue.load()
+  await db.queue.reset()
+  let queue = []
+  const chunks = split(_.shuffle(items), options.maxClusters)
+  for (const [i, chunk] of chunks.entries()) {
+    for (const item of chunk) {
+      item.cluster_id = i + 1
+      queue.push(item)
+    }
+  }
+
+  queue = _.sortBy(queue, ['channel.xmltv_id', 'date'])
+
+  await db.queue.insert(queue)
+}
+
+function split(arr, n) {
+  let result = []
+  for (let i = n; i > 0; i--) {
+    result.push(arr.splice(0, Math.ceil(arr.length / i)))
+  }
+  return result
+}
+
+async function logError(key, data) {
+  const filepath = `${LOGS_DIR}/errors/${key}.log`
+  if (!(await file.exists(filepath))) {
+    await file.create(filepath)
+  }
+
+  await file.append(filepath, JSON.stringify(data) + '\r\n')
+}
--- a/scripts/commands/load-cluster.js
+++ b/scripts/commands/load-cluster.js
@ -5,7 +5,6 @@ const { db, logger, timer, file, parser } = require('../core')

 const options = program
  .requiredOption('-c, --cluster-id <cluster-id>', 'The ID of cluster to load', parser.parseNumber)
-  .option('--days <days>', 'Number of days for which to grab the program', parser.parseNumber, 1)
  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parser.parseNumber)
  .option(
    '-t, --timeout <timeout>',
@ -17,28 +16,26 @@ const options = program
  .opts()

 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
+const CLUSTER_PATH = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`

 async function main() {
  logger.info('Starting...')
  timer.start()

-  const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
  logger.info(`Loading cluster: ${options.clusterId}`)
-  logger.info(`Creating '${clusterLog}'...`)
-  await file.create(clusterLog)
-  await db.channels.load()
-  const channels = await db.channels.find({ cluster_id: options.clusterId })
-  const total = options.days * channels.length
-  logger.info(`Total ${total} requests`)
+  logger.info(`Creating '${CLUSTER_PATH}'...`)
+  await file.create(CLUSTER_PATH)
+  await db.queue.load()
+  const items = await db.queue.find({ cluster_id: options.clusterId })
+  const total = items.length

  logger.info('Loading...')
  const results = {}
  let i = 1
-  for (const channel of channels) {
-    let config = require(file.resolve(channel.configPath))
+  for (const item of items) {
+    let config = require(file.resolve(item.configPath))

    config = _.merge(config, {
-      days: options.days,
      debug: options.debug,
      delay: options.delay,
      request: {
@ -46,9 +43,9 @@ async function main() {
      }
    })

-    await grabber.grab(channel, config, async (data, err) => {
+    await grabber.grab(item.channel, item.date, config, async (data, err) => {
      logger.info(
-        `[${i}/${total}] ${channel.site} - ${channel.xmltv_id} - ${data.date.format(
+        `[${i}/${total}] ${item.channel.site} - ${item.channel.xmltv_id} - ${data.date.format(
          'MMM D, YYYY'
        )} (${data.programs.length} programs)`
      )
@ -56,19 +53,18 @@ async function main() {
      if (err) logger.error(err.message)

      const result = {
-        channel: data.channel,
+        _qid: item._id,
        programs: data.programs,
-        date: data.date.format(),
        error: err ? err.message : null
      }

-      await file.append(clusterLog, JSON.stringify(result) + '\n')
+      await file.append(CLUSTER_PATH, JSON.stringify(result) + '\n')

      if (i < total) i++
    })
  }

-  db.channels.compact()
+  db.queue.compact()

  logger.info(`Done in ${timer.format('HH[h] mm[m] ss[s]')}`)
 }
--- a/scripts/commands/parse-channels.js
+++ b/scripts/commands/parse-channels.js
@ -1,5 +1,5 @@
 const { Command } = require('commander')
-const { db } = require('../core')
+const { db, logger } = require('../core')
 const path = require('path')
 const _ = require('lodash')
 const fs = require('fs')
@ -46,7 +46,7 @@ async function main() {

  fs.writeFileSync(path.resolve(output), xml)

-  console.log(`File '${output}' successfully saved`)
+  logger.info(`File '${output}' successfully saved`)
 }

 main()
--- a/scripts/commands/save-results.js
+++ b/scripts/commands/save-results.js
@ -4,9 +4,7 @@ const _ = require('lodash')
 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'

 async function main() {
-  const errorsLog = `${LOGS_DIR}/errors.log`
-  await file.create(errorsLog)
-  await db.channels.load()
+  await db.queue.load()
  await db.programs.load()
  await db.programs.reset()
  const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`)
@ -14,6 +12,9 @@ async function main() {
    logger.info(`Parsing "${filepath}"...`)
    const results = await parser.parseLogs(filepath)
    for (const result of results) {
+      const queue = await db.queue.find({ _id: result._qid }).limit(1)
+      if (!queue.length) continue
+      const item = queue[0]
      const programs = result.programs.map(program => {
        return {
          title: program.title,
@ -26,29 +27,18 @@ async function main() {
          lang: program.lang,
          start: program.start,
          stop: program.stop,
-          site: result.channel.site,
-          _cid: result.channel._id
+          stop: program.stop,
+          site: item.channel.site,
+          _qid: result._qid
        }
      })
      await db.programs.insert(programs)

-      if (result.channel.logo) {
-        await db.channels.update(
-          { _id: result.channel._id },
-          { $set: { logo: result.channel.logo, programCount: result.programs.length } }
-        )
-      }
-
-      if (result.error) {
-        await file.append(
-          errorsLog,
-          JSON.stringify({ ...result.channel, date: result.date, error: result.error }) + '\n'
-        )
-      }
+      await db.queue.update({ _id: result._qid }, { $set: { error: result.error } })
    }
  }

-  await db.channels.compact()
+  await db.queue.compact()
 }

 main()
--- a/scripts/commands/update-api.js
+++ b/scripts/commands/update-api.js
@ -5,28 +5,37 @@ const DB_DIR = process.env.DB_DIR || 'scripts/database'
 const API_DIR = process.env.API_DIR || '.gh-pages/api'

 async function main() {
-  await saveToGuidesJson(await loadGuides())
-  await saveToProgramsJson(await loadPrograms())
+  await loadQueue()
+
+  const programs = await loadPrograms()
+  const guides = await getGuides(programs)
+
+  await saveToGuidesJson(guides)
+  await saveToProgramsJson(programs)
 }

 main()

-async function loadGuides() {
-  logger.info('Loading guides from database...')
+async function loadQueue() {
+  logger.info('Loading queue...')

-  await db.channels.load()
+  await db.queue.load()
+}

-  const channels = await db.channels.find({}).sort({ xmltv_id: 1 })
+async function getGuides(programs = []) {
+  programs = _.groupBy(programs, i => i._qid)
+
+  const queue = await db.queue.find({}).sort({ xmltv_id: 1 })

  const output = []
-  for (const channel of channels) {
-    channel.groups.forEach(group => {
-      if (channel.programCount) {
+  for (const item of queue) {
+    item.groups.forEach(group => {
+      const channelPrograms = programs[item._id]
+      if (!item.error && channelPrograms) {
        output.push({
-          channel: channel.xmltv_id,
-          display_name: channel.name,
-          site: channel.site,
-          lang: channel.lang,
+          channel: item.channel.xmltv_id,
+          site: item.channel.site,
+          lang: item.channel.lang,
          url: `https://iptv-org.github.io/epg/guides/${group}.epg.xml`
        })
      }
@ -37,11 +46,20 @@ async function loadGuides() {
 }

 async function loadPrograms() {
-  logger.info('Loading programs from database...')
-
+  logger.info('Loading programs...')
  await db.programs.load()
+  return await db.programs.find({})
+}

-  let programs = await db.programs.find({})
+async function saveToGuidesJson(guides = []) {
+  const guidesPath = `${API_DIR}/guides.json`
+  logger.info(`Saving to "${guidesPath}"...`)
+  await file.create(guidesPath, JSON.stringify(guides))
+}
+
+async function saveToProgramsJson(programs = []) {
+  const programsPath = `${API_DIR}/programs.json`
+  logger.info(`Saving to "${programsPath}"...`)

  programs = programs.map(item => {
    const categories = Array.isArray(item.category) ? item.category : [item.category]
@ -63,17 +81,5 @@ async function loadPrograms() {

  programs = _.sortBy(programs, ['channel', 'site', 'start'])

-  return programs
-}
-
-async function saveToGuidesJson(guides = []) {
-  const channelsPath = `${API_DIR}/guides.json`
-  logger.info(`Saving to "${channelsPath}"...`)
-  await file.create(channelsPath, JSON.stringify(guides))
-}
-
-async function saveToProgramsJson(programs = []) {
-  const programsPath = `${API_DIR}/programs.json`
-  logger.info(`Saving to "${programsPath}"...`)
  await file.create(programsPath, JSON.stringify(programs))
 }
--- a/scripts/commands/update-guides.js
+++ b/scripts/commands/update-guides.js
@ -1,11 +1,10 @@
-const { db, logger, file } = require('../core')
+const { db, logger, file, api } = require('../core')
 const grabber = require('epg-grabber')
 const _ = require('lodash')

-const DB_DIR = process.env.DB_DIR || 'scripts/database'
 const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
 const PUBLIC_DIR = process.env.PUBLIC_DIR || '.gh-pages'
-const LOG_PATH = `${LOGS_DIR}/update-guides.log`
+const GUIDES_PATH = `${LOGS_DIR}/guides.log`

 async function main() {
  await setUp()
@ -17,64 +16,132 @@ main()
 async function generateGuides() {
  logger.info(`Generating guides/...`)

-  const grouped = groupByGroup(await loadChannels())
-
  logger.info('Loading "database/programs.db"...')
  await db.programs.load()
+  await api.channels.load()

+  const grouped = groupByGroup(await loadQueue())
  for (const key in grouped) {
    const filepath = `${PUBLIC_DIR}/guides/${key}.epg.xml`
-    const channels = grouped[key]
-    const programs = await loadProgramsForChannels(channels)
-    const output = grabber.convertToXMLTV({ channels, programs })
+    const criticalErrors = []
+    let channels = {}
+    let programs = []
+    for (const item of grouped[key]) {
+      if (channels[item.channel.xmltv_id]) continue
+
+      if (item.error) {
+        const error = {
+          xmltv_id: item.channel.xmltv_id,
+          site: item.channel.site,
+          site_id: item.channel.site_id,
+          lang: item.channel.lang,
+          date: item.date,
+          error: item.error
+        }
+        criticalErrors.push(error)
+        await logError(key, error)
+      } else {
+        const itemPrograms = await loadProgramsForItem(item)
+        if (!itemPrograms.length) {
+          await logError(key, {
+            xmltv_id: item.channel.xmltv_id,
+            site: item.channel.site,
+            site_id: item.channel.site_id,
+            lang: item.channel.lang,
+            date: item.date,
+            error: 'Programs not found'
+          })
+          continue
+        }
+
+        const channel = api.channels.find({ id: item.channel.xmltv_id })
+        if (!channel) {
+          await logError(key, {
+            xmltv_id: item.channel.xmltv_id,
+            site: item.channel.site,
+            site_id: item.channel.site_id,
+            lang: item.channel.lang,
+            date: item.date,
+            error: 'The channel has the wrong xmltv_id'
+          })
+          continue
+        }
+
+        channels[channel.id] = {
+          xmltv_id: channel.id,
+          name: channel.name,
+          logo: channel.logo,
+          site: item.channel.site
+        }
+
+        programs = programs.concat(itemPrograms)
+      }
+    }
+
+    channels = Object.values(channels)

    logger.info(`Creating "${filepath}"...`)
+    const output = grabber.convertToXMLTV({ channels, programs })
    await file.create(filepath, output)

-    await log({
+    let status = 0
+    if (criticalErrors.length > 0 || !channels.length) {
+      status = 1
+    }
+
+    await logGuide({
      group: key,
-      count: channels.length
+      count: channels.length,
+      status
    })
  }

  logger.info(`Done`)
 }

-function groupByGroup(channels = []) {
+function groupByGroup(items = []) {
  const groups = {}

-  channels.forEach(channel => {
-    channel.groups.forEach(key => {
+  items.forEach(item => {
+    item.groups.forEach(key => {
      if (!groups[key]) {
        groups[key] = []
      }

-      groups[key].push(channel)
+      groups[key].push(item)
    })
  })

  return groups
 }

-async function loadChannels() {
-  logger.info('Loading channels...')
+async function loadQueue() {
+  logger.info('Loading queue...')

-  await db.channels.load()
+  await db.queue.load()

-  return await db.channels.find({ programCount: { $gt: 0 } }).sort({ xmltv_id: 1 })
+  return await db.queue.find({}).sort({ xmltv_id: 1 })
 }

-async function loadProgramsForChannels(channels = []) {
-  const cids = channels.map(c => c._id)
-
-  return await db.programs.find({ _cid: { $in: cids } }).sort({ channel: 1, start: 1 })
+async function loadProgramsForItem(item) {
+  return await db.programs.find({ _qid: item._id }).sort({ channel: 1, start: 1 })
 }

 async function setUp() {
-  logger.info(`Creating '${LOG_PATH}'...`)
-  await file.create(LOG_PATH)
+  logger.info(`Creating '${GUIDES_PATH}'...`)
+  await file.create(GUIDES_PATH)
+  await file.createDir(`${LOGS_DIR}/errors`)
 }

-async function log(data) {
-  await file.append(LOG_PATH, JSON.stringify(data) + '\n')
+async function logGuide(data) {
+  await file.append(GUIDES_PATH, JSON.stringify(data) + '\r\n')
+}
+
+async function logError(key, data) {
+  const filepath = `${LOGS_DIR}/errors/${key}.log`
+  if (!(await file.exists(filepath))) {
+    await file.create(filepath)
+  }
+
+  await file.append(filepath, JSON.stringify(data) + '\r\n')
 }
--- a/scripts/commands/update-readme.js
+++ b/scripts/commands/update-readme.js
@ -1,7 +1,4 @@
-const { file, markdown, parser, logger } = require('../core')
-const provinces = require('../data/ca-provinces.json')
-const countries = require('../data/countries.json')
-const states = require('../data/us-states.json')
+const { file, markdown, parser, logger, api } = require('../core')
 const { program } = require('commander')
 const _ = require('lodash')

@ -12,7 +9,14 @@ const options = program
  .parse(process.argv)
  .opts()

+const statuses = {
+  0: '🟢',
+  1: '🔴'
+}
+
 async function main() {
+  await api.countries.load()
+  await api.subdivisions.load()
  const records = await getLogRecords()
  await generateCountriesTable(records)
  await generateUSStatesTable(records)
@ -27,21 +31,22 @@ async function generateCountriesTable(items = []) {

  let rows = []
  for (const item of items) {
-    const country = countries[item.code]
+    const country = api.countries.find({ code: item.code })
    if (!country) continue

    rows.push({
      flag: country.flag,
      name: country.name,
      channels: item.count,
-      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`
+      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`,
+      status: statuses[item.status]
    })
  }

  rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc'])
  rows = _.groupBy(rows, 'name')

-  const table = markdown.createTable(rows, ['Country', 'Channels', 'EPG'])
+  const table = markdown.createTable(rows, ['Country', 'Channels', 'EPG', 'Status'])

  await file.create('./.readme/_countries.md', table)
 }
@ -51,20 +56,22 @@ async function generateUSStatesTable(items = []) {

  let rows = []
  for (const item of items) {
-    const state = states[item.code]
+    if (!item.code.startsWith('US-')) continue
+    const state = api.subdivisions.find({ code: item.code })
    if (!state) continue

    rows.push({
      name: state.name,
      channels: item.count,
-      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`
+      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`,
+      status: statuses[item.status]
    })
  }

  rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc'])
  rows = _.groupBy(rows, 'name')

-  const table = markdown.createTable(rows, ['State', 'Channels', 'EPG'])
+  const table = markdown.createTable(rows, ['State', 'Channels', 'EPG', 'Status'])

  await file.create('./.readme/_us-states.md', table)
 }
@ -74,20 +81,22 @@ async function generateCanadaProvincesTable(items = []) {

  let rows = []
  for (const item of items) {
-    const province = provinces[item.code]
+    if (!item.code.startsWith('CA-')) continue
+    const province = api.subdivisions.find({ code: item.code })
    if (!province) continue

    rows.push({
      name: province.name,
      channels: item.count,
-      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`
+      epg: `<code>https://iptv-org.github.io/epg/guides/${item.group}.epg.xml</code>`,
+      status: statuses[item.status]
    })
  }

  rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc'])
  rows = _.groupBy(rows, 'name')

-  const table = markdown.createTable(rows, ['Province', 'Channels', 'EPG'])
+  const table = markdown.createTable(rows, ['Province', 'Channels', 'EPG', 'Status'])

  await file.create('./.readme/_ca-provinces.md', table)
 }
@ -101,14 +110,9 @@ async function updateReadme() {
 }

 async function getLogRecords() {
-  const logPath = `${LOGS_DIR}/update-guides.log`
+  const logPath = `${LOGS_DIR}/guides.log`
  const records = await parser.parseLogs(logPath)

-  if (!records.length) {
-    logger.error(`File "${logPath}" is empty`)
-    process.exit(1)
-  }
-
  return records.map(item => {
    const code = item.group.split('/')[0] || ''
    item.code = code.toUpperCase()
--- a/scripts/core/api.js
+++ b/scripts/core/api.js
@ -0,0 +1,27 @@
+const _ = require('lodash')
+const file = require('./file')
+
+const DATA_DIR = process.env.DATA_DIR || './scripts/data'
+
+class API {
+  constructor(filepath) {
+    this.filepath = file.resolve(filepath)
+  }
+
+  async load() {
+    const data = await file.read(this.filepath)
+    this.collection = JSON.parse(data)
+  }
+
+  find(query) {
+    return _.find(this.collection, query)
+  }
+}
+
+const api = {}
+
+api.channels = new API(`${DATA_DIR}/channels.json`)
+api.countries = new API(`${DATA_DIR}/countries.json`)
+api.subdivisions = new API(`${DATA_DIR}/subdivisions.json`)
+
+module.exports = api
--- a/scripts/core/date.js
+++ b/scripts/core/date.js
@ -0,0 +1,13 @@
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+dayjs.extend(utc)
+
+const date = {}
+
+date.getUTC = function (d = null) {
+	if (typeof d === 'string') return dayjs.utc(d).startOf('d')
+
+	return dayjs.utc().startOf('d')
+}
+
+module.exports = date
--- a/scripts/core/db.js
+++ b/scripts/core/db.js
@ -70,7 +70,7 @@ class Database {

 const db = {}

-db.channels = new Database(`${DB_DIR}/channels.db`)
+db.queue = new Database(`${DB_DIR}/queue.db`)
 db.programs = new Database(`${DB_DIR}/programs.db`)

 module.exports = db
--- a/scripts/core/index.js
+++ b/scripts/core/index.js
@ -4,3 +4,5 @@ exports.file = require('./file')
 exports.parser = require('./parser')
 exports.timer = require('./timer')
 exports.markdown = require('./markdown')
+exports.api = require('./api')
+exports.date = require('./date')
--- a/scripts/core/markdown.js
+++ b/scripts/core/markdown.js
@ -24,6 +24,7 @@ markdown.createTable = function (data, cols) {
      }
      output += `<td align="right">${item.channels}</td>`
      output += `<td align="left" nowrap>${item.epg}</td>`
+      output += `<td align="center" nowrap>${item.status}</td>`
      output += '</tr>\n'
    }
  }
--- a/scripts/data/ca-provinces.json
+++ b/scripts/data/ca-provinces.json
@ -1,67 +0,0 @@
-{
-  "CA-AB":
-  {
-    "name": "Alberta",
-    "code": "CA-AB"
-  },
-  "CA-BC":
-  {
-    "name": "British Columbia",
-    "code": "CA-BC"
-  },
-  "CA-MB":
-  {
-    "name": "Manitoba",
-    "code": "CA-MB"
-  },
-  "CA-NB":
-  {
-    "name": "New Brunswick",
-    "code": "CA-NB"
-  },
-  "CA-NL":
-  {
-    "name": "Newfoundland and Labrador",
-    "code": "CA-NL"
-  },
-  "CA-NT":
-  {
-    "name": "Northwest Territories",
-    "code": "CA-NT"
-  },
-  "CA-NS":
-  {
-    "name": "Nova Scotia",
-    "code": "CA-NS"
-  },
-  "CA-NU":
-  {
-    "name": "Nunavut",
-    "code": "CA-NU"
-  },
-  "CA-ON":
-  {
-    "name": "Ontario",
-    "code": "CA-ON"
-  },
-  "CA-PE":
-  {
-    "name": "Prince Edward Island",
-    "code": "CA-PE"
-  },
-  "CA-QC":
-  {
-    "name": "Quebec",
-    "code": "CA-QC"
-  },
-  "CA-SK":
-  {
-    "name": "Saskatchewan",
-    "code": "CA-SK"
-  },
-  "CA-YT":
-  {
-    "name": "Yukon Territory",
-    "code": "CA-YT"
-  }
-}
--- a/scripts/data/countries.json
+++ b/scripts/data/countries.json
--- a/tests/data/expected/api/guides.json
+++ b/tests/data/expected/api/guides.json
@ -1 +1 @@
-[{"channel":"CNNInternationalEurope.us","display_name":"CNN International Europe","site":"chaines-tv.orange.fr","lang":"fr","url":"https://iptv-org.github.io/epg/guides/fr/chaines-tv.orange.fr.epg.xml"},{"channel":"CNNInternationalEurope.us","display_name":"CNN International Europe","site":"chaines-tv.orange.fr","lang":"fr","url":"https://iptv-org.github.io/epg/guides/bh/chaines-tv.orange.fr.epg.xml"},{"channel":"MNetMovies2.za","display_name":"M-Net Movies 2","site":"dstv.com","lang":"en","url":"https://iptv-org.github.io/epg/guides/zw/dstv.com.epg.xml"}]
+[{"channel":"CNNInternationalEurope.us","site":"chaines-tv.orange.fr","lang":"fr","url":"https://iptv-org.github.io/epg/guides/fr/chaines-tv.orange.fr.epg.xml"},{"channel":"CNNInternationalEurope.us","site":"chaines-tv.orange.fr","lang":"fr","url":"https://iptv-org.github.io/epg/guides/bh/chaines-tv.orange.fr.epg.xml"},{"channel":"MNetMovies2.za","site":"dstv.com","lang":"en","url":"https://iptv-org.github.io/epg/guides/zw/dstv.com.epg.xml"}]
--- a/tests/data/expected/database/programs.db
+++ b/tests/data/expected/database/programs.db
@ -0,0 +1,23 @@
+{"title":"InfoNeu ","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"6AzYe7lqcSN05ZUq"}
+{"title":"Club Piolet","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641715200,"stop":1641718800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"5vmbOpGwkj1Dc8FJ"}
+{"title":"InfoNeu ","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"ThxnjcPz8zOuRZuF"}
+{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641729600,"stop":1641730800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"p8kQkIzlX2ebpIfN"}
+{"title":"El Trànsit","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641730800,"stop":1641732000,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"DYZHd71eCvOl49jT"}
+{"title":"El Trànsit","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732000,"stop":1641732300,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"EkIqGqryukUIkwLg"}
+{"title":"Informatiu migdia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732300,"stop":1641733800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"ILpRjp36kwPoEG03"}
+{"title":"El Trànsit","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641733800,"stop":1641736200,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"dqKxildlF1bMGLrU"}
+{"title":"La Terre vue du Sport","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736200,"stop":1641736800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"gGgrefSDo9Gqlfy2"}
+{"title":"Informatiu migdia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736800,"stop":1641738300,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"LSDJTeJ2L9PUcgEM"}
+{"title":"Club Piolet","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641738300,"stop":1641741900,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"c5jmzbB2jGzY3aY3"}
+{"title":"Informatiu migdia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641741900,"stop":1641743400,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"j9pYfk5wvqBTSFUf"}
+{"title":"El Trànsit","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641743400,"stop":1641750900,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"eb91kjF1CeJh52Oy"}
+{"title":"La rotonda","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641750900,"stop":1641753600,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"NTNNjmk2r6uTVIBz"}
+{"title":"Club Piolet","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641753600,"stop":1641757200,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"5ulSbWY45V6krNjX"}
+{"title":"El Trànsit","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757200,"stop":1641757500,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"Eh3UMow3zKnqz8mq"}
+{"title":"Informatiu vespre","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757500,"stop":1641759000,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"P3Fce8tELLKRN4Wu"}
+{"title":"Recull setmanal","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641759000,"stop":1641761100,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"BmcYIFHG15JCrdKs"}
+{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641761100,"stop":1641763800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"1ZbtgoH47f297xyb"}
+{"title":"El cafè dels matins","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641763800,"stop":1641766800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"Zrrg7bLwvjMe0jqb"}
+{"title":"La Terre vue du Sport","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641766800,"stop":1641767400,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"Q5qBBURrSryIIS2V"}
+{"title":"Informatiu vespre","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641767400,"stop":1641772800,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"yEMHeWXc4Si9sGb1"}
+{"title":"Àrea Andorra Difusió","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641772800,"stop":1641776400,"site": "chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"vPe2XXH6Knru6zzL"}
--- a/tests/data/expected/database/queue-with-errors.db
+++ b/tests/data/expected/database/queue-with-errors.db
@ -0,0 +1,5 @@
+{"channel":{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com"},"configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"date": "2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"00AluKCrCnfgrl8W"}
+{"channel":{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr","bh/chaines-tv.orange.fr"],"cluster_id":1,"date": "2022-01-21T00:00:00Z","error":null,"_id":"0Wefq0oMR3feCcuY"}
+{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"date": "2022-01-21T00:00:00Z","error":"Invalid header value char","_id":"1XzrxNkSF2AQNBrT"}
+{"channel":{"lang":"ru","xmltv_id":"Perviykanal.ru","site_id":"1","site":"yandex.ru"},"configPath":"sites/yandex.ru/yandex.ru.config.js","groups":["ru/yandex.ru"],"error":"Some error","cluster_id":4,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5Xw5"}
+{"channel":{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com"},"configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"date": "2022-01-21T00:00:00Z","error":null,"_id":"1lnhXpN7g0ER5XwN"}
--- a/tests/data/expected/database/queue.db
+++ b/tests/data/expected/database/queue.db
@ -0,0 +1,2 @@
+{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"configPath":"tests/__data__/input/sites/example.com.config.js","date":"2022-01-30T00:00:00.000Z","groups":["ca-nl/example.com"],"error":null,"cluster_id":1,"_id":"TYDwYLsrkmPtTLT2"}
+{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"configPath":"tests/__data__/input/sites/example.com.config.js","date":"2022-01-31T00:00:00.000Z","groups":["ca-nl/example.com"],"error":null,"cluster_id":1,"_id":"98cKRthEhMmKEnwx"}
--- a/tests/data/expected/guides/bh/chaines-tv.orange.fr.epg.xml
+++ b/tests/data/expected/guides/bh/chaines-tv.orange.fr.epg.xml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv>
-<channel id="CNNInternationalEurope.us"><display-name>CNN International Europe</display-name><url>https://chaines-tv.orange.fr</url></channel>
+<channel id="CNNInternationalEurope.us"><display-name>CNN International Europe</display-name><icon src="https://i.imgur.com/2BXCg0x.jpg"/><url>https://chaines-tv.orange.fr</url></channel>
 <programme start="20220110000000 +0000" stop="20220110010000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">CNN Newsroom Sunday</title><desc lang="ru">Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.</desc><category lang="ru">Category1</category><category lang="ru">Category2</category></programme>
 <programme start="20220110010000 +0000" stop="20220110020000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">Fareed Zakaria GPS</title><desc lang="ru">Интервью с главными игроками мировой политики.</desc><category lang="ru">Category1</category></programme>
 <programme start="20220110020000 +0000" stop="20220110023000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">African Voices Changemakers. 114-я серия</title><desc lang="ru">114-я серия. Африка сегодня - люди, новости, события.</desc></programme>
--- a/tests/data/expected/guides/fr/chaines-tv.orange.fr.epg.xml
+++ b/tests/data/expected/guides/fr/chaines-tv.orange.fr.epg.xml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?><tv>
-<channel id="CNNInternationalEurope.us"><display-name>CNN International Europe</display-name><url>https://chaines-tv.orange.fr</url></channel>
+<channel id="CNNInternationalEurope.us"><display-name>CNN International Europe</display-name><icon src="https://i.imgur.com/2BXCg0x.jpg"/><url>https://chaines-tv.orange.fr</url></channel>
 <programme start="20220110000000 +0000" stop="20220110010000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">CNN Newsroom Sunday</title><desc lang="ru">Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.</desc><category lang="ru">Category1</category><category lang="ru">Category2</category></programme>
 <programme start="20220110010000 +0000" stop="20220110020000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">Fareed Zakaria GPS</title><desc lang="ru">Интервью с главными игроками мировой политики.</desc><category lang="ru">Category1</category></programme>
 <programme start="20220110020000 +0000" stop="20220110023000 +0000" channel="CNNInternationalEurope.us"><title lang="ru">African Voices Changemakers. 114-я серия</title><desc lang="ru">114-я серия. Африка сегодня - люди, новости, события.</desc></programme>
--- a/tests/data/expected/logs/errors/ca-nl/example.com.log
+++ b/tests/data/expected/logs/errors/ca-nl/example.com.log
@ -0,0 +1 @@
+{"xmltv_id":"CNNInternationalEurope2.us","site":"example.com","site_id":"141","lang":"en","error":"The channel has the wrong xmltv_id"}
--- a/tests/data/expected/logs/errors/ge/magticom.ge.log
+++ b/tests/data/expected/logs/errors/ge/magticom.ge.log
@ -0,0 +1 @@
+{"xmltv_id":"CNNInternationalEurope.us","site":"magticom.ge","site_id":"140","lang":"ru","date":"2022-01-21T00:00:00Z","error":"Programs not found"}
--- a/tests/data/expected/logs/errors/ru/yandex.ru.log
+++ b/tests/data/expected/logs/errors/ru/yandex.ru.log
@ -0,0 +1 @@
+{"xmltv_id":"Perviykanal.ru","site":"yandex.ru","site_id":"1","lang":"ru","date":"2022-01-21T00:00:00Z","error":"Some error"}
--- a/tests/data/expected/logs/errors/us/directv.com.log
+++ b/tests/data/expected/logs/errors/us/directv.com.log
@ -0,0 +1 @@
+{"xmltv_id":"BravoEast.us","site":"directv.com","site_id":"237","lang":"en","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}
--- a/tests/data/expected/logs/guides.log
+++ b/tests/data/expected/logs/guides.log
@ -0,0 +1,6 @@
+{"group":"us/directv.com","count":0,"status":1}
+{"group":"fr/chaines-tv.orange.fr","count":1,"status":0}
+{"group":"bh/chaines-tv.orange.fr","count":1,"status":0}
+{"group":"ge/magticom.ge","count":0,"status":1}
+{"group":"ru/yandex.ru","count":0,"status":1}
+{"group":"zw/dstv.com","count":1,"status":0}
--- a/tests/data/expected/logs/load-cluster/cluster_1.log
+++ b/tests/data/expected/logs/load-cluster/cluster_1.log
@ -0,0 +1,2 @@
+{"_qid":"0Wefq0oMR3feCcuY","programs":[],"error":null}
+{"_qid":"1XzrxNkSF2AQNBrT","programs":[],"error":null}
--- a/tests/data/expected/logs/update-guides.log
+++ b/tests/data/expected/logs/update-guides.log
@ -1,3 +0,0 @@
-{"group":"fr/chaines-tv.orange.fr","count":1}
-{"group":"bh/chaines-tv.orange.fr","count":1}
-{"group":"zw/dstv.com","count":1}
--- a/tests/data/expected/readme.md
+++ b/tests/data/expected/readme.md
@ -11,12 +11,12 @@ To load a program guide, all you need to do is copy the link to one or more of t
 <!-- prettier-ignore -->
 <table>
  <thead>
-    <tr><th align="left">Country</th><th align="left">Channels</th><th align="left">EPG</th></tr>
+    <tr><th align="left">Country</th><th align="left">Channels</th><th align="left">EPG</th><th align="left">Status</th></tr>
  </thead>
  <tbody>
-    <tr><td align="left" valign="top" nowrap>🇿🇦 South Africa</td><td align="right">1</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/za/dstv.com.epg.xml</code></td></tr>
-    <tr><td align="left" valign="top" nowrap rowspan="2">🇺🇸 United States</td><td align="right">372</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us/tvtv.us.epg.xml</code></td></tr>
-    <tr><td align="right">74</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us/magticom.ge.epg.xml</code></td></tr>
+    <tr><td align="left" valign="top" nowrap>🇿🇦 South Africa</td><td align="right">1</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/za/dstv.com.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
+    <tr><td align="left" valign="top" nowrap rowspan="2">🇺🇸 United States</td><td align="right">372</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us/tvtv.us.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
+    <tr><td align="right">74</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us/magticom.ge.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
  </tbody>
 </table>

@ -25,12 +25,12 @@ To load a program guide, all you need to do is copy the link to one or more of t
 <!-- prettier-ignore -->
 <table>
  <thead>
-    <tr><th align="left">State</th><th align="left">Channels</th><th align="left">EPG</th></tr>
+    <tr><th align="left">State</th><th align="left">Channels</th><th align="left">EPG</th><th align="left">Status</th></tr>
  </thead>
  <tbody>
-    <tr><td align="left" valign="top" nowrap rowspan="3">Puerto Rico</td><td align="right">14</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/tvtv.us.epg.xml</code></td></tr>
-    <tr><td align="right">7</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/gatotv.com.epg.xml</code></td></tr>
-    <tr><td align="right">1</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/directv.com.epg.xml</code></td></tr>
+    <tr><td align="left" valign="top" nowrap rowspan="3">Puerto Rico</td><td align="right">14</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/tvtv.us.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
+    <tr><td align="right">7</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/gatotv.com.epg.xml</code></td><td align="center" nowrap>🔴</td></tr>
+    <tr><td align="right">0</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/us-pr/directv.com.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
  </tbody>
 </table>

@ -39,10 +39,10 @@ To load a program guide, all you need to do is copy the link to one or more of t
 <!-- prettier-ignore -->
 <table>
  <thead>
-    <tr><th align="left">Province</th><th align="left">Channels</th><th align="left">EPG</th></tr>
+    <tr><th align="left">Province</th><th align="left">Channels</th><th align="left">EPG</th><th align="left">Status</th></tr>
  </thead>
  <tbody>
-    <tr><td align="left" valign="top" nowrap>Newfoundland and Labrador</td><td align="right">1</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/ca-nl/tvtv.us.epg.xml</code></td></tr>
+    <tr><td align="left" valign="top" nowrap>Newfoundland and Labrador</td><td align="right">1</td><td align="left" nowrap><code>https://iptv-org.github.io/epg/guides/ca-nl/tvtv.us.epg.xml</code></td><td align="center" nowrap>🟢</td></tr>
  </tbody>
 </table>

--- a/tests/data/input/data/channels.json
+++ b/tests/data/input/data/channels.json
@ -0,0 +1,55 @@
+[
+  {
+    "id": "BravoEast.us",
+    "name": "Bravo East",
+    "network": null,
+    "country": "US",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "c/US"
+    ],
+    "languages": [
+      "eng"
+    ],
+    "categories": [],
+    "is_nsfw": false,
+    "logo": "https://www.directv.com/images/logos/channels/dark/large/579.png"
+  },
+  {
+    "id": "CNNInternationalEurope.us",
+    "name": "CNN International Europe",
+    "network": null,
+    "country": "US",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "r/EUR"
+    ],
+    "languages": [
+      "eng"
+    ],
+    "categories": [
+      "news"
+    ],
+    "is_nsfw": false,
+    "logo": "https://i.imgur.com/2BXCg0x.jpg"
+  },
+  {
+    "id": "MNetMovies2.za",
+    "name": "M-Net Movies 2",
+    "network": null,
+    "country": "ZA",
+    "subdivision": null,
+    "city": null,
+    "broadcast_area": [
+      "c/ZA"
+    ],
+    "languages": [
+      "afr"
+    ],
+    "categories": [],
+    "is_nsfw": false,
+    "logo": "https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png"
+  }
+]
--- a/tests/data/input/data/countries.json
+++ b/tests/data/input/data/countries.json
--- a/tests/data/input/data/subdivisions.json
+++ b/tests/data/input/data/subdivisions.json
@ -1,297 +1,352 @@
-{
-  "US-AL":
-  {
-    "name": "Alabama",
-    "code": "US-AL"
-  },
-  "US-AK":
-  {
-    "name": "Alaska",
-    "code": "US-AK"
-  },
-  "US-AS":
-  {
-    "name": "American Samoa",
-    "code": "US-AS"
-  },
-  "US-AZ":
-  {
-    "name": "Arizona",
-    "code": "US-AZ"
-  },
-  "US-AR":
-  {
-    "name": "Arkansas",
-    "code": "US-AR"
-  },
-  "US-CA":
-  {
-    "name": "California",
-    "code": "US-CA"
-  },
-  "US-CO":
-  {
-    "name": "Colorado",
-    "code": "US-CO"
-  },
-  "US-CT":
-  {
-    "name": "Connecticut",
-    "code": "US-CT"
-  },
-  "US-DE":
-  {
-    "name": "Delaware",
-    "code": "US-DE"
-  },
-  "US-DC":
-  {
-    "name": "District Of Columbia",
-    "code": "US-DC"
-  },
-  "US-FM":
-  {
-    "name": "Federated States Of Micronesia",
-    "code": "US-FM"
-  },
-  "US-FL":
-  {
-    "name": "Florida",
-    "code": "US-FL"
-  },
-  "US-GA":
-  {
-    "name": "Georgia",
-    "code": "US-GA"
-  },
-  "US-GU":
-  {
-    "name": "Guam",
-    "code": "US-GU"
-  },
-  "US-HI":
-  {
-    "name": "Hawaii",
-    "code": "US-HI"
-  },
-  "US-ID":
-  {
-    "name": "Idaho",
-    "code": "US-ID"
-  },
-  "US-IL":
-  {
-    "name": "Illinois",
-    "code": "US-IL"
-  },
-  "US-IN":
-  {
-    "name": "Indiana",
-    "code": "US-IN"
-  },
-  "US-IA":
-  {
-    "name": "Iowa",
-    "code": "US-IA"
-  },
-  "US-KS":
-  {
-    "name": "Kansas",
-    "code": "US-KS"
-  },
-  "US-KY":
-  {
-    "name": "Kentucky",
-    "code": "US-KY"
-  },
-  "US-LA":
-  {
-    "name": "Louisiana",
-    "code": "US-LA"
-  },
-  "US-ME":
-  {
-    "name": "Maine",
-    "code": "US-ME"
-  },
-  "US-MH":
-  {
-    "name": "Marshall Islands",
-    "code": "US-MH"
-  },
-  "US-MD":
-  {
-    "name": "Maryland",
-    "code": "US-MD"
-  },
-  "US-MA":
-  {
-    "name": "Massachusetts",
-    "code": "US-MA"
-  },
-  "US-MI":
-  {
-    "name": "Michigan",
-    "code": "US-MI"
-  },
-  "US-MN":
-  {
-    "name": "Minnesota",
-    "code": "US-MN"
-  },
-  "US-MS":
-  {
-    "name": "Mississippi",
-    "code": "US-MS"
-  },
-  "US-MO":
-  {
-    "name": "Missouri",
-    "code": "US-MO"
-  },
-  "US-MT":
-  {
-    "name": "Montana",
-    "code": "US-MT"
-  },
-  "US-NE":
-  {
-    "name": "Nebraska",
-    "code": "US-NE"
-  },
-  "US-NV":
-  {
-    "name": "Nevada",
-    "code": "US-NV"
-  },
-  "US-NH":
-  {
-    "name": "New Hampshire",
-    "code": "US-NH"
-  },
-  "US-NJ":
-  {
-    "name": "New Jersey",
-    "code": "US-NJ"
-  },
-  "US-NM":
-  {
-    "name": "New Mexico",
-    "code": "US-NM"
-  },
-  "US-NY":
-  {
-    "name": "New York",
-    "code": "US-NY"
-  },
-  "US-NC":
-  {
-    "name": "North Carolina",
-    "code": "US-NC"
-  },
-  "US-ND":
-  {
-    "name": "North Dakota",
-    "code": "US-ND"
-  },
-  "US-MP":
-  {
-    "name": "Northern Mariana Islands",
-    "code": "US-MP"
-  },
-  "US-OH":
-  {
-    "name": "Ohio",
-    "code": "US-OH"
-  },
-  "US-OK":
-  {
-    "name": "Oklahoma",
-    "code": "US-OK"
-  },
-  "US-OR":
-  {
-    "name": "Oregon",
-    "code": "US-OR"
-  },
-  "US-PW":
-  {
-    "name": "Palau",
-    "code": "US-PW"
-  },
-  "US-PA":
-  {
-    "name": "Pennsylvania",
-    "code": "US-PA"
-  },
-  "US-PR":
-  {
-    "name": "Puerto Rico",
-    "code": "US-PR"
-  },
-  "US-RI":
-  {
-    "name": "Rhode Island",
-    "code": "US-RI"
-  },
-  "US-SC":
-  {
-    "name": "South Carolina",
-    "code": "US-SC"
-  },
-  "US-SD":
-  {
-    "name": "South Dakota",
-    "code": "US-SD"
-  },
-  "US-TN":
-  {
-    "name": "Tennessee",
-    "code": "US-TN"
-  },
-  "US-TX":
-  {
-    "name": "Texas",
-    "code": "US-TX"
-  },
-  "US-UT":
-  {
-    "name": "Utah",
-    "code": "US-UT"
-  },
-  "US-VT":
-  {
-    "name": "Vermont",
-    "code": "US-VT"
-  },
-  "US-VI":
-  {
-    "name": "Virgin Islands",
-    "code": "US-VI"
-  },
-  "US-VA":
-  {
-    "name": "Virginia",
-    "code": "US-VA"
-  },
-  "US-WA":
-  {
-    "name": "Washington",
-    "code": "US-WA"
-  },
-  "US-WV":
-  {
-    "name": "West Virginia",
-    "code": "US-WV"
-  },
-  "US-WI":
-  {
-    "name": "Wisconsin",
-    "code": "US-WI"
-  },
-  "US-WY":
-  {
-    "name": "Wyoming",
-    "code": "US-WY"
-  }
-}
+[
+  {
+    "country": "CA",
+    "name": "Alberta",
+    "code": "CA-AB"
+  },
+  {
+    "country": "CA",
+    "name": "British Columbia",
+    "code": "CA-BC"
+  },
+  {
+    "country": "CA",
+    "name": "Manitoba",
+    "code": "CA-MB"
+  },
+  {
+    "country": "CA",
+    "name": "New Brunswick",
+    "code": "CA-NB"
+  },
+  {
+    "country": "CA",
+    "name": "Newfoundland and Labrador",
+    "code": "CA-NL"
+  },
+  {
+    "country": "CA",
+    "name": "Northwest Territories",
+    "code": "CA-NT"
+  },
+  {
+    "country": "CA",
+    "name": "Nova Scotia",
+    "code": "CA-NS"
+  },
+  {
+    "country": "CA",
+    "name": "Nunavut",
+    "code": "CA-NU"
+  },
+  {
+    "country": "CA",
+    "name": "Ontario",
+    "code": "CA-ON"
+  },
+  {
+    "country": "CA",
+    "name": "Prince Edward Island",
+    "code": "CA-PE"
+  },
+  {
+    "country": "CA",
+    "name": "Quebec",
+    "code": "CA-QC"
+  },
+  {
+    "country": "CA",
+    "name": "Saskatchewan",
+    "code": "CA-SK"
+  },
+  {
+    "country": "CA",
+    "name": "Yukon",
+    "code": "CA-YT"
+  },
+  {
+    "country": "US",
+    "name": "Alabama",
+    "code": "US-AL"
+  },
+  {
+    "country": "US",
+    "name": "Alaska",
+    "code": "US-AK"
+  },
+  {
+    "country": "US",
+    "name": "American Samoa",
+    "code": "US-AS"
+  },
+  {
+    "country": "US",
+    "name": "Arizona",
+    "code": "US-AZ"
+  },
+  {
+    "country": "US",
+    "name": "Arkansas",
+    "code": "US-AR"
+  },
+  {
+    "country": "US",
+    "name": "California",
+    "code": "US-CA"
+  },
+  {
+    "country": "US",
+    "name": "Colorado",
+    "code": "US-CO"
+  },
+  {
+    "country": "US",
+    "name": "Connecticut",
+    "code": "US-CT"
+  },
+  {
+    "country": "US",
+    "name": "Delaware",
+    "code": "US-DE"
+  },
+  {
+    "country": "US",
+    "name": "District of Columbia",
+    "code": "US-DC"
+  },
+  {
+    "country": "US",
+    "name": "Florida",
+    "code": "US-FL"
+  },
+  {
+    "country": "US",
+    "name": "Georgia",
+    "code": "US-GA"
+  },
+  {
+    "country": "US",
+    "name": "Guam",
+    "code": "US-GU"
+  },
+  {
+    "country": "US",
+    "name": "Hawaii",
+    "code": "US-HI"
+  },
+  {
+    "country": "US",
+    "name": "Idaho",
+    "code": "US-ID"
+  },
+  {
+    "country": "US",
+    "name": "Illinois",
+    "code": "US-IL"
+  },
+  {
+    "country": "US",
+    "name": "Indiana",
+    "code": "US-IN"
+  },
+  {
+    "country": "US",
+    "name": "Iowa",
+    "code": "US-IA"
+  },
+  {
+    "country": "US",
+    "name": "Kansas",
+    "code": "US-KS"
+  },
+  {
+    "country": "US",
+    "name": "Kentucky",
+    "code": "US-KY"
+  },
+  {
+    "country": "US",
+    "name": "Louisiana",
+    "code": "US-LA"
+  },
+  {
+    "country": "US",
+    "name": "Maine",
+    "code": "US-ME"
+  },
+  {
+    "country": "US",
+    "name": "Maryland",
+    "code": "US-MD"
+  },
+  {
+    "country": "US",
+    "name": "Massachusetts",
+    "code": "US-MA"
+  },
+  {
+    "country": "US",
+    "name": "Michigan",
+    "code": "US-MI"
+  },
+  {
+    "country": "US",
+    "name": "Minnesota",
+    "code": "US-MN"
+  },
+  {
+    "country": "US",
+    "name": "Mississippi",
+    "code": "US-MS"
+  },
+  {
+    "country": "US",
+    "name": "Missouri",
+    "code": "US-MO"
+  },
+  {
+    "country": "US",
+    "name": "Montana",
+    "code": "US-MT"
+  },
+  {
+    "country": "US",
+    "name": "Nebraska",
+    "code": "US-NE"
+  },
+  {
+    "country": "US",
+    "name": "Nevada",
+    "code": "US-NV"
+  },
+  {
+    "country": "US",
+    "name": "New Hampshire",
+    "code": "US-NH"
+  },
+  {
+    "country": "US",
+    "name": "New Jersey",
+    "code": "US-NJ"
+  },
+  {
+    "country": "US",
+    "name": "New Mexico",
+    "code": "US-NM"
+  },
+  {
+    "country": "US",
+    "name": "New York",
+    "code": "US-NY"
+  },
+  {
+    "country": "US",
+    "name": "North Carolina",
+    "code": "US-NC"
+  },
+  {
+    "country": "US",
+    "name": "North Dakota",
+    "code": "US-ND"
+  },
+  {
+    "country": "US",
+    "name": "Northern Mariana Islands",
+    "code": "US-MP"
+  },
+  {
+    "country": "US",
+    "name": "Ohio",
+    "code": "US-OH"
+  },
+  {
+    "country": "US",
+    "name": "Oklahoma",
+    "code": "US-OK"
+  },
+  {
+    "country": "US",
+    "name": "Oregon",
+    "code": "US-OR"
+  },
+  {
+    "country": "US",
+    "name": "Pennsylvania",
+    "code": "US-PA"
+  },
+  {
+    "country": "US",
+    "name": "Puerto Rico",
+    "code": "US-PR"
+  },
+  {
+    "country": "US",
+    "name": "Rhode Island",
+    "code": "US-RI"
+  },
+  {
+    "country": "US",
+    "name": "South Carolina",
+    "code": "US-SC"
+  },
+  {
+    "country": "US",
+    "name": "South Dakota",
+    "code": "US-SD"
+  },
+  {
+    "country": "US",
+    "name": "Tennessee",
+    "code": "US-TN"
+  },
+  {
+    "country": "US",
+    "name": "Texas",
+    "code": "US-TX"
+  },
+  {
+    "country": "US",
+    "name": "U.S. Virgin Islands",
+    "code": "US-VI"
+  },
+  {
+    "country": "US",
+    "name": "United States Minor Outlying Islands",
+    "code": "US-UM"
+  },
+  {
+    "country": "US",
+    "name": "Utah",
+    "code": "US-UT"
+  },
+  {
+    "country": "US",
+    "name": "Vermont",
+    "code": "US-VT"
+  },
+  {
+    "country": "US",
+    "name": "Virginia",
+    "code": "US-VA"
+  },
+  {
+    "country": "US",
+    "name": "Washington",
+    "code": "US-WA"
+  },
+  {
+    "country": "US",
+    "name": "West Virginia",
+    "code": "US-WV"
+  },
+  {
+    "country": "US",
+    "name": "Wisconsin",
+    "code": "US-WI"
+  },
+  {
+    "country": "US",
+    "name": "Wyoming",
+    "code": "US-WY"
+  }
+]
--- a/tests/data/input/database/channels.db
+++ b/tests/data/input/database/channels.db
@ -1,4 +0,0 @@
-{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"country":"US","programCount":0,"_id":"00AluKCrCnfgrl8W"}
-{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":null,"name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"cluster_id":1,"programCount":32,"_id":"0Wefq0oMR3feCcuY"}
-{"lang":"ru","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"140","logo":"https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg","name":"CNN Int","site":"magticom.ge","channelsPath":"sites/magticom.ge/magticom.ge_ge.channels.xml","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"cluster_id":1,"programCount":0,"_id":"1XzrxNkSF2AQNBrT"}
-{"lang":"en","country":"ZA","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","logo":"https://rndcdn.dstv.com/dstvcms/2020/08/31/M-Net_Movies_2_Logo_4-3_lightbackground_xlrg.png","name":"M-Net Movies 2","site":"dstv.com","channelsPath":"sites/dstv.com/dstv.com_zw.channels.xml","configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"cluster_id":120,"programCount":14,"_id":"1lnhXpN7g0ER5XwN"}
--- a/tests/data/input/database/programs.db
+++ b/tests/data/input/database/programs.db
@ -1,46 +1,46 @@
-{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641825900,"stop":1641826800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"12AJc0GeEJE9p4c3"}
-{"title":"Connecting Africa. 114-я серия","description":"114-я серия. Проект, рассказывающий о людях и компаниях, которые совершают революцию в африканском бизнесе, и о тех, кто объединяет континент, выступая за свободную торговлю в Африке.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641843900,"stop":1641844800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"1dxcT34nyxzOlxBL"}
-{"title":"Connect the World","description":"Актуальная мировая информация с разных континентов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641826800,"stop":1641830400,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"2uJe4w2lgvjNOXo0"}
-{"title":"The Lead with Jake Tapper","description":"Оперативная сводка новостей страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641844800,"stop":1641848400,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"6As6GzEVhb3OWM0M"}
-{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641817800,"stop":1641819600,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"6DXKlITWehX1Jx4F"}
-{"title":"CNN Newsroom with Michael Holmes","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641790800,"stop":1641794400,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"AadPdMZ3s72y8NMk"}
-{"title":"The Situation Room with Wolf Blitzer","description":"Командный центр новостей, политики и неординарных репортажей со всего мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641852000,"stop":1641855600,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"Az3ABKy3HnE7sJZk"}
-{"title":"One World with Zain Asher","description":"Освещаются важные новости с каждого континента, от политики и текущих дел до социальных вопросов и многого другого.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641830400,"stop":1641833100,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"DMurxgt5OD0E9OIE"}
-{"title":"TBD","description":"Информационно-познавательный проект CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641783600,"stop":1641785400,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"HQJqM2kIa77llWbC"}
-{"title":"Marketplace Africa. 548-я серия","description":"548-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641782700,"stop":1641783600,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"Jn3khh5n9Brkxq4U"}
-{"title":"CNN Newsroom with Michael Holmes","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641787200,"stop":1641789900,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"KcrIoQTXtUdw74sO"}
-{"title":"The Global Brief with Bianca Nobilo","description":"Global Brief с Бьянкой Нобило проницательно исследует меняющийся мир для меняющейся аудитории, обеспечивая непревзойденную глубину и качество для занятых зрителей в быстро меняющемся мире.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641848400,"stop":1641850200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"LGD7WmQogDRxZn01"}
-{"title":"CNN Newsroom with Rosemary Church","description":"Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641798000,"stop":1641805200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"LyCBivUTdZFW9X53"}
-{"title":"Marketplace Africa. 549-я серия","description":"549-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641833100,"stop":1641834000,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"PbrZinuZKgBHqDVj"}
-{"title":"African Voices Changemakers. 114-я серия","description":"114-я серия. Африка сегодня - люди, новости, события.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641780000,"stop":1641781800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"SvrCK31v78V5y7EA"}
-{"title":"Anderson Cooper 360","description":"Уникальный взгляд Андерсона Купера на главные события мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641859200,"stop":1641862800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"TFGrOFJGkaOs9pU7"}
-{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641850200,"stop":1641852000,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"UynlLeT41MsjFElg"}
-{"title":"New Day","description":"Свежий обзор событий в стране и мире.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641808800,"stop":1641817800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"UyvhQ4wRNq5d5XRd"}
-{"title":"Amanpour","description":"Сводка новостей от знаменитой ведущей канала CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641834000,"stop":1641837600,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"WbsOCkmPH5gjmo4M"}
-{"title":"Early Start","description":"Новости дня с Кристиной Романс и Дейвом Бриггсом.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641805200,"stop":1641808800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"YB96P2mMO4TA0pID"}
-{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641789900,"stop":1641790800,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"aDdCAlgqLG2yxM1m"}
-{"title":"CNN Newsroom Sunday","description":"Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.","category":["Category1","Category2"],"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641772800,"stop":1641776400,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"aYCk87dUOAkCJE9x"}
-{"title":"Fareed Zakaria GPS","description":"Интервью с главными игроками мировой политики.","category":"Category1","season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641776400,"stop":1641780000,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"c1nCoWVetBZ3mn5q"}
-{"title":"Inside Africa. 586-я серия","description":"586-я серия. Своеобразное \"путешествие\" по Африке - почувствуйте все разнообразие культур различных стран и регионов континента.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641785400,"stop":1641787200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"goaDr7BsGGm3LCfz"}
-{"title":"CNN Newsroom with Robyn Curnow","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641794400,"stop":1641797100,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"nixd3gRF1S1K0ZOs"}
-{"title":"Marketplace Africa. 549-я серия","description":"549-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641781800,"stop":1641782700,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"r1b8EvZc0tYs88ga"}
-{"title":"Erin Burnett OutFront","description":"Обсуждение самых важных мировых тем в эфире канала CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641855600,"stop":1641859200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"sIQtUtowtATc7dLj"}
-{"title":"Connect the World","description":"Актуальная мировая информация с разных континентов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641823200,"stop":1641825900,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"tXBIZ2BZBIkhnoTZ"}
-{"title":"Quest Means Business","description":"Ричард Квест возглавляет группу экспертов и корреспондентов, чтобы предоставить актуальные факты, цифры и анализ из делового мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641841200,"stop":1641843900,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"xlE5epkjzdfUQpXO"}
-{"title":"First Move with Julia Chatterley","description":"Несколько больших историй, связанных с открытием рынков в США.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641819600,"stop":1641823200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"yEVXucyUomVmktMF"}
-{"title":"Hala Gorani Tonight","description":"Используя свой 25-летний журналистский опыт, Хала Горани будет освещать ключевые события в картине дня посредством диалога с гостями и экспертами-аналитиками.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641837600,"stop":1641841200,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"yPgmYrWwfxHW3WUA"}
-{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641797100,"stop":1641798000,"site":"chaines-tv.orange.fr","_cid":"0Wefq0oMR3feCcuY","_id":"zX70wOz5drExRTJX"}
-{"title":"Robin Hood","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641822300,"stop":1641829200,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"1AoKArQw6MxP6pVU"}
-{"title":"The Water Diviner","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641776100,"stop":1641782700,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"6v7w0SB4IlnfEEu3"}
-{"title":"Bad Boys For Life","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641847200,"stop":1641850800,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"83VRYvggmyfCzkOm"}
-{"title":"12 Strong","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641814500,"stop":1641822300,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"DbjwscjIuVDY8TPx"}
-{"title":"Backdraft","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641801300,"stop":1641809400,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"IwuwkjCKqWvio7ba"}
-{"title":"Force Of Nature","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641841200,"stop":1641847200,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"LP56HczEup0ed3Xx"}
-{"title":"Mafia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641809400,"stop":1641814500,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"MM9DPxERAgGGak39"}
-{"title":"The Last Witch Hunter","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641834780,"stop":1641841200,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"MciJOpN3YCodj6Na"}
-{"title":"Beyond The Line","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641796500,"stop":1641801300,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"ZKA2s6QrM0xRrfGz"}
-{"title":"Paranoia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641790200,"stop":1641796500,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"ZpdIZeSRhPycDX9D"}
-{"title":"The Scorpion King","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641829200,"stop":1641834780,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"doO4Lh1pAt6L6wHa"}
-{"title":"Fatman","description":null,"category":null,"season":9,"episode":257,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641761700,"stop":1641767700,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"fHahGuzHnU7xVEJX"}
-{"title":"Outbreak","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641782700,"stop":1641790200,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"mkvcMP4FMwL2a5ax"}
-{"title":"Motherless Brooklyn","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641767700,"stop":1641776100,"site":"dstv.com","_cid":"1lnhXpN7g0ER5XwN","_id":"nxTIAJsBwyXztRun"}
+{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641825900,"stop":1641826800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"12AJc0GeEJE9p4c3"}
+{"title":"Connecting Africa. 114-я серия","description":"114-я серия. Проект, рассказывающий о людях и компаниях, которые совершают революцию в африканском бизнесе, и о тех, кто объединяет континент, выступая за свободную торговлю в Африке.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641843900,"stop":1641844800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"1dxcT34nyxzOlxBL"}
+{"title":"Connect the World","description":"Актуальная мировая информация с разных континентов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641826800,"stop":1641830400,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"2uJe4w2lgvjNOXo0"}
+{"title":"The Lead with Jake Tapper","description":"Оперативная сводка новостей страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641844800,"stop":1641848400,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"6As6GzEVhb3OWM0M"}
+{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641817800,"stop":1641819600,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"6DXKlITWehX1Jx4F"}
+{"title":"CNN Newsroom with Michael Holmes","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641790800,"stop":1641794400,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"AadPdMZ3s72y8NMk"}
+{"title":"The Situation Room with Wolf Blitzer","description":"Командный центр новостей, политики и неординарных репортажей со всего мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641852000,"stop":1641855600,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"Az3ABKy3HnE7sJZk"}
+{"title":"One World with Zain Asher","description":"Освещаются важные новости с каждого континента, от политики и текущих дел до социальных вопросов и многого другого.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641830400,"stop":1641833100,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"DMurxgt5OD0E9OIE"}
+{"title":"TBD","description":"Информационно-познавательный проект CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641783600,"stop":1641785400,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"HQJqM2kIa77llWbC"}
+{"title":"Marketplace Africa. 548-я серия","description":"548-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641782700,"stop":1641783600,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"Jn3khh5n9Brkxq4U"}
+{"title":"CNN Newsroom with Michael Holmes","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641787200,"stop":1641789900,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"KcrIoQTXtUdw74sO"}
+{"title":"The Global Brief with Bianca Nobilo","description":"Global Brief с Бьянкой Нобило проницательно исследует меняющийся мир для меняющейся аудитории, обеспечивая непревзойденную глубину и качество для занятых зрителей в быстро меняющемся мире.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641848400,"stop":1641850200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"LGD7WmQogDRxZn01"}
+{"title":"CNN Newsroom with Rosemary Church","description":"Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641798000,"stop":1641805200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"LyCBivUTdZFW9X53"}
+{"title":"Marketplace Africa. 549-я серия","description":"549-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641833100,"stop":1641834000,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"PbrZinuZKgBHqDVj"}
+{"title":"African Voices Changemakers. 114-я серия","description":"114-я серия. Африка сегодня - люди, новости, события.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641780000,"stop":1641781800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"SvrCK31v78V5y7EA"}
+{"title":"Anderson Cooper 360","description":"Уникальный взгляд Андерсона Купера на главные события мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641859200,"stop":1641862800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"TFGrOFJGkaOs9pU7"}
+{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641850200,"stop":1641852000,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"UynlLeT41MsjFElg"}
+{"title":"New Day","description":"Свежий обзор событий в стране и мире.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641808800,"stop":1641817800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"UyvhQ4wRNq5d5XRd"}
+{"title":"Amanpour","description":"Сводка новостей от знаменитой ведущей канала CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641834000,"stop":1641837600,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"WbsOCkmPH5gjmo4M"}
+{"title":"Early Start","description":"Новости дня с Кристиной Романс и Дейвом Бриггсом.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641805200,"stop":1641808800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"YB96P2mMO4TA0pID"}
+{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641789900,"stop":1641790800,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"aDdCAlgqLG2yxM1m"}
+{"title":"CNN Newsroom Sunday","description":"Свежая мировая информационная сводка от CNN. О политике, экономике, общественной жизни, культуре, спорте.","category":["Category1","Category2"],"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641772800,"stop":1641776400,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"aYCk87dUOAkCJE9x"}
+{"title":"Fareed Zakaria GPS","description":"Интервью с главными игроками мировой политики.","category":"Category1","season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641776400,"stop":1641780000,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"c1nCoWVetBZ3mn5q"}
+{"title":"Inside Africa. 586-я серия","description":"586-я серия. Своеобразное \"путешествие\" по Африке - почувствуйте все разнообразие культур различных стран и регионов континента.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641785400,"stop":1641787200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"goaDr7BsGGm3LCfz"}
+{"title":"CNN Newsroom with Robyn Curnow","description":"Обзор самых важных и актуальных новостей и событий из жизни страны и мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641794400,"stop":1641797100,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"nixd3gRF1S1K0ZOs"}
+{"title":"Marketplace Africa. 549-я серия","description":"549-я серия. Информационная передача об экономических событиях африканского региона. Анализируются проблемы, даются экономические прогнозы.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641781800,"stop":1641782700,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"r1b8EvZc0tYs88ga"}
+{"title":"Erin Burnett OutFront","description":"Обсуждение самых важных мировых тем в эфире канала CNN.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641855600,"stop":1641859200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"sIQtUtowtATc7dLj"}
+{"title":"Connect the World","description":"Актуальная мировая информация с разных континентов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641823200,"stop":1641825900,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"tXBIZ2BZBIkhnoTZ"}
+{"title":"Quest Means Business","description":"Ричард Квест возглавляет группу экспертов и корреспондентов, чтобы предоставить актуальные факты, цифры и анализ из делового мира.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641841200,"stop":1641843900,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"xlE5epkjzdfUQpXO"}
+{"title":"First Move with Julia Chatterley","description":"Несколько больших историй, связанных с открытием рынков в США.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641819600,"stop":1641823200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"yEVXucyUomVmktMF"}
+{"title":"Hala Gorani Tonight","description":"Используя свой 25-летний журналистский опыт, Хала Горани будет освещать ключевые события в картине дня посредством диалога с гостями и экспертами-аналитиками.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641837600,"stop":1641841200,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"yPgmYrWwfxHW3WUA"}
+{"title":"World Sport","description":"Все о главных спортивных событиях мира. Обзоры самых важных спортивных событий, аналитика, мнения экспертов.","category":null,"season":null,"episode":null,"icon":null,"channel":"CNNInternationalEurope.us","lang":"ru","start":1641797100,"stop":1641798000,"site":"chaines-tv.orange.fr","_qid":"0Wefq0oMR3feCcuY","_id":"zX70wOz5drExRTJX"}
+{"title":"Robin Hood","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641822300,"stop":1641829200,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"1AoKArQw6MxP6pVU"}
+{"title":"The Water Diviner","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641776100,"stop":1641782700,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"6v7w0SB4IlnfEEu3"}
+{"title":"Bad Boys For Life","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641847200,"stop":1641850800,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"83VRYvggmyfCzkOm"}
+{"title":"12 Strong","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641814500,"stop":1641822300,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"DbjwscjIuVDY8TPx"}
+{"title":"Backdraft","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641801300,"stop":1641809400,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"IwuwkjCKqWvio7ba"}
+{"title":"Force Of Nature","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641841200,"stop":1641847200,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"LP56HczEup0ed3Xx"}
+{"title":"Mafia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641809400,"stop":1641814500,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"MM9DPxERAgGGak39"}
+{"title":"The Last Witch Hunter","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641834780,"stop":1641841200,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"MciJOpN3YCodj6Na"}
+{"title":"Beyond The Line","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641796500,"stop":1641801300,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"ZKA2s6QrM0xRrfGz"}
+{"title":"Paranoia","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641790200,"stop":1641796500,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"ZpdIZeSRhPycDX9D"}
+{"title":"The Scorpion King","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641829200,"stop":1641834780,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"doO4Lh1pAt6L6wHa"}
+{"title":"Fatman","description":null,"category":null,"season":9,"episode":257,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641761700,"stop":1641767700,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"fHahGuzHnU7xVEJX"}
+{"title":"Outbreak","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641782700,"stop":1641790200,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"mkvcMP4FMwL2a5ax"}
+{"title":"Motherless Brooklyn","description":null,"category":null,"season":null,"episode":null,"icon":null,"channel":"MNetMovies2.za","lang":"en","start":1641767700,"stop":1641776100,"site":"dstv.com","_qid":"1lnhXpN7g0ER5XwN","_id":"nxTIAJsBwyXztRun"}
--- a/tests/data/input/database/queue.db
+++ b/tests/data/input/database/queue.db
@ -0,0 +1,5 @@
+{"channel":{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com"},"configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"error":"Invalid header value char","cluster_id":84,"date":"2022-01-21T00:00:00Z","_id":"00AluKCrCnfgrl8W"}
+{"channel":{"lang":"fr","xmltv_id":"CNNInternationalEurope.us","site_id":"53","site":"chaines-tv.orange.fr"},"configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr/chaines-tv.orange.fr", "bh/chaines-tv.orange.fr"],"error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"0Wefq0oMR3feCcuY"}
+{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"magticom.ge"},"configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ge/magticom.ge"],"error":null,"cluster_id":1,"date":"2022-01-21T00:00:00Z","_id":"1XzrxNkSF2AQNBrT"}
+{"channel":{"lang":"en","xmltv_id":"MNetMovies2.za","site_id":"404a052b-3dea-4cac-a19c-de9a7d6f191d#MAP","site":"dstv.com"},"configPath":"sites/dstv.com/dstv.com.config.js","groups":["zw/dstv.com"],"error":null,"cluster_id":120,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5XwN"}
+{"channel":{"lang":"ru","xmltv_id":"Perviykanal.ru","site_id":"1","site":"yandex.ru"},"configPath":"sites/yandex.ru/yandex.ru.config.js","groups":["ru/yandex.ru"],"error":"Some error","cluster_id":4,"date":"2022-01-21T00:00:00Z","_id":"1lnhXpN7g0ER5Xw5"}
--- a/tests/data/input/logs/errors.log
+++ b/tests/data/input/logs/errors.log
@ -1 +1 @@
-{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}
+{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","site":"directv.com","configPath":"sites/directv.com/directv.com.config.js","groups":["us/directv.com"],"cluster_id":84,"_id":"00AluKCrCnfgrl8W","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}
--- a/tests/data/input/logs/guides.log
+++ b/tests/data/input/logs/guides.log
@ -0,0 +1,7 @@
+{"group":"us/magticom.ge","count":74,"status":0}
+{"group":"za/dstv.com","count":1,"status":0}
+{"group":"us-pr/tvtv.us","count":14,"status":0}
+{"group":"us-pr/gatotv.com","count":7,"status":1}
+{"group":"us-pr/directv.com","count":0,"status":0}
+{"group":"ca-nl/tvtv.us","count":1,"status":0}
+{"group":"us/tvtv.us","count":372,"status":0}
--- a/tests/data/input/logs/load-cluster/cluster_1.log
+++ b/tests/data/input/logs/load-cluster/cluster_1.log
@ -1,2 +1,2 @@
-{"channel":{"lang":"fr","country":"US","xmltv_id":"CNNInternationalEurope.us","site_id":"53","logo":"https://example.com/logo.png","name":"CNN International Europe","site":"chaines-tv.orange.fr","channelsPath":"sites/chaines-tv.orange.fr/chaines-tv.orange.fr_fr.channels.xml","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["fr"],"cluster_id":1,"_id":"0Wefq0oMR3feCcuY"},"programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}],"date":"2022-01-21T00:00:00Z","error":null}
-{"channel":{"lang":"en","xmltv_id":"BravoEast.us","site_id":"237","logo":"https://www.directv.com/images/logos/channels/dark/large/579.png","name":"Bravo East","site":"directv.com","channelsPath":"sites/directv.com/directv.com_us.channels.xml","configPath":"sites/directv.com/directv.com.config.js","groups":["us"],"cluster_id":84,"country":"US","_id":"00AluKCrCnfgrl8W"},"programs":[],"date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}
+{"_qid":"0Wefq0oMR3feCcuY","programs":[{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641715200,"stop":1641718800},{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600},{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641729600,"stop":1641730800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641730800,"stop":1641732000},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732000,"stop":1641732300},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641732300,"stop":1641733800},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641733800,"stop":1641736200},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736200,"stop":1641736800},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641736800,"stop":1641738300},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641738300,"stop":1641741900},{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641741900,"stop":1641743400},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641743400,"stop":1641750900},{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641750900,"stop":1641753600},{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641753600,"stop":1641757200},{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757200,"stop":1641757500},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641757500,"stop":1641759000},{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641759000,"stop":1641761100},{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641761100,"stop":1641763800},{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641763800,"stop":1641766800},{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641766800,"stop":1641767400},{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641767400,"stop":1641772800},{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","season":null,"episode":null,"start":1641772800,"stop":1641776400}],"error":null}
+{"_qid":"1XzrxNkSF2AQNBrT","programs":[],"error":"Invalid header value char"}
--- a/tests/data/input/logs/update-guides.log
+++ b/tests/data/input/logs/update-guides.log
@ -1,7 +0,0 @@
-{"group":"us/magticom.ge","count":74}
-{"group":"za/dstv.com","count":1}
-{"group":"us-pr/tvtv.us","count":14}
-{"group":"us-pr/gatotv.com","count":7}
-{"group":"us-pr/directv.com","count":1}
-{"group":"ca-nl/tvtv.us","count":1}
-{"group":"us/tvtv.us","count":372}
--- a/tests/data/input/sites/example.com.config.js
+++ b/tests/data/input/sites/example.com.config.js
@ -3,9 +3,6 @@ module.exports = {
  url() {
    return `https://example.com`
  },
-  logo() {
-    return 'https://example.com/logo.png'
-  },
  parser() {
    return []
  }
--- a/tests/commands/create-database.test.js
+++ b/tests/commands/create-database.test.js
@ -1,59 +0,0 @@
-const fs = require('fs')
-const path = require('path')
-const { execSync } = require('child_process')
-
-beforeEach(() => {
-  fs.rmdirSync('tests/__data__/output', { recursive: true })
-  fs.mkdirSync('tests/__data__/output')
-
-  const stdout = execSync(
-    'DB_DIR=tests/__data__/output/database node scripts/commands/create-database.js --channels=tests/__data__/input/sites/*.channels.xml --max-clusters=1',
-    { encoding: 'utf8' }
-  )
-})
-
-it('can create channels database', () => {
-  const output = content('tests/__data__/output/database/channels.db')
-
-  expect(output).toEqual(
-    expect.arrayContaining([
-      expect.objectContaining({
-        lang: 'ru',
-        country: 'US',
-        xmltv_id: 'CNNInternationalEurope.us',
-        site_id: '140',
-        name: 'CNN International Europe',
-        site: 'example.com',
-        channelsPath: 'tests/__data__/input/sites/example.com_ca-nl.channels.xml',
-        configPath: 'tests/__data__/input/sites/example.com.config.js',
-        groups: ['ca-nl/example.com'],
-        cluster_id: 1
-      }),
-      expect.objectContaining({
-        lang: 'en',
-        xmltv_id: 'CNNInternationalEurope2.us',
-        site_id: '141',
-        name: 'CNN International Europe 2',
-        site: 'example.com',
-        country: 'US',
-        channelsPath: 'tests/__data__/input/sites/example.com_ca-nl.channels.xml',
-        configPath: 'tests/__data__/input/sites/example.com.config.js',
-        groups: ['ca-nl/example.com'],
-        cluster_id: 1
-      })
-    ])
-  )
-})
-
-function content(filepath) {
-  const data = fs.readFileSync(path.resolve(filepath), {
-    encoding: 'utf8'
-  })
-
-  return data
-    .split('\n')
-    .filter(l => l)
-    .map(l => {
-      return JSON.parse(l)
-    })
-}
--- a/tests/commands/create-matrix.test.js
+++ b/tests/commands/create-matrix.test.js
@ -6,10 +6,7 @@ beforeEach(() => {
  fs.rmdirSync('tests/__data__/output', { recursive: true })
  fs.mkdirSync('tests/__data__/output')
  fs.mkdirSync('tests/__data__/temp/database', { recursive: true })
-  fs.copyFileSync(
-    'tests/__data__/input/database/channels.db',
-    'tests/__data__/temp/database/channels.db'
-  )
+  fs.copyFileSync('tests/__data__/input/database/queue.db', 'tests/__data__/temp/database/queue.db')
 })

 afterEach(() => {
@ -24,5 +21,5 @@ it('can create valid matrix', () => {
    }
  )

-  expect(result).toBe('::set-output name=matrix::{"cluster_id":[1,84,120]}\n')
+  expect(result).toBe('::set-output name=matrix::{"cluster_id":[1,4,84,120]}\n')
 })
--- a/tests/commands/create-queue.test.js
+++ b/tests/commands/create-queue.test.js
@ -0,0 +1,56 @@
+const fs = require('fs')
+const path = require('path')
+const { execSync } = require('child_process')
+
+beforeEach(() => {
+  fs.rmdirSync('tests/__data__/output', { recursive: true })
+  fs.mkdirSync('tests/__data__/output')
+
+  const stdout = execSync(
+    'DB_DIR=tests/__data__/output/database LOGS_DIR=tests/__data__/output/logs CHANNELS_PATH=tests/__data__/input/sites/*.channels.xml node scripts/commands/create-queue.js --max-clusters=1 --days=2',
+    { encoding: 'utf8' }
+  )
+})
+
+it('can create queue', () => {
+  let output = content('tests/__data__/output/database/queue.db')
+  let expected = content('tests/__data__/expected/database/queue.db')
+
+  output = output.map(i => {
+    i._id = null
+    i.date = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    i.date = null
+    return i
+  })
+
+  expect(output).toEqual(
+    expect.arrayContaining([
+      expect.objectContaining(expected[0]),
+      expect.objectContaining(expected[1])
+    ])
+  )
+})
+
+it('can log errors', () => {
+  let output = content('tests/__data__/output/logs/errors/ca-nl/example.com.log')
+  let expected = content('tests/__data__/expected/logs/errors/ca-nl/example.com.log')
+
+  expect(output).toEqual(expected)
+})
+
+function content(filepath) {
+  const data = fs.readFileSync(path.resolve(filepath), {
+    encoding: 'utf8'
+  })
+
+  return data
+    .split('\n')
+    .filter(l => l)
+    .map(l => {
+      return JSON.parse(l)
+    })
+}
--- a/tests/commands/load-cluster.test.js
+++ b/tests/commands/load-cluster.test.js
@ -11,10 +11,7 @@ beforeEach(() => {
  fs.rmdirSync('tests/__data__/output', { recursive: true })
  fs.mkdirSync('tests/__data__/output')
  fs.mkdirSync('tests/__data__/temp/database', { recursive: true })
-  fs.copyFileSync(
-    'tests/__data__/input/database/channels.db',
-    'tests/__data__/temp/database/channels.db'
-  )
+  fs.copyFileSync('tests/__data__/input/database/queue.db', 'tests/__data__/temp/database/queue.db')

  execSync(
    'DB_DIR=tests/__data__/temp/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1 --timeout=10000',
@ -23,25 +20,10 @@ beforeEach(() => {
 })

 it('can load cluster', () => {
-  const output = content('tests/__data__/output/logs/load-cluster/cluster_1.log')
+  let output = content('tests/__data__/output/logs/load-cluster/cluster_1.log')
+  let expected = content('tests/__data__/expected/logs/load-cluster/cluster_1.log')

-  expect(Object.keys(output[0]).sort()).toEqual(['channel', 'date', 'error', 'programs'])
-
-  expect(output[0]).toMatchObject({
-    channel: {
-      _id: '0Wefq0oMR3feCcuY',
-      logo: 'https://example.com/logo.png'
-    },
-    date: dayjs.utc().startOf('d').format(),
-    error: null
-  })
-
-  expect(output[1]).toMatchObject({
-    channel: {
-      _id: '1XzrxNkSF2AQNBrT',
-      logo: 'https://www.magticom.ge/images/channels/MjAxOC8wOS8xMC9lZmJhNWU5Yy0yMmNiLTRkMTAtOWY5Ny01ODM0MzY0ZTg0MmEuanBn.jpg'
-    }
-  })
+  expect(output).toEqual(expected)
 })

 function content(filepath) {
--- a/tests/commands/save-results.test.js
+++ b/tests/commands/save-results.test.js
@ -8,8 +8,8 @@ beforeEach(() => {
  fs.mkdirSync('tests/__data__/output/database', { recursive: true })

  fs.copyFileSync(
-    'tests/__data__/input/database/channels.db',
-    'tests/__data__/output/database/channels.db'
+    'tests/__data__/input/database/queue.db',
+    'tests/__data__/output/database/queue.db'
  )

  const stdout = execSync(
@ -18,60 +18,27 @@ beforeEach(() => {
  )
 })

-it('can save results', () => {
-  const programs = content('tests/__data__/output/database/programs.db')
+it('can save programs to database', () => {
+  let output = content('tests/__data__/output/database/programs.db')
+  let expected = content('tests/__data__/expected/database/programs.db')

-  expect(Object.keys(programs[0]).sort()).toEqual([
-    '_cid',
-    '_id',
-    'category',
-    'channel',
-    'description',
-    'episode',
-    'icon',
-    'lang',
-    'season',
-    'site',
-    'start',
-    'stop',
-    'title'
-  ])
-
-  expect(programs[0]).toMatchObject({
-    _cid: '0Wefq0oMR3feCcuY'
+  output = output.map(i => {
+    i._id = null
+    return i
+  })
+  expected = expected.map(i => {
+    i._id = null
+    return i
  })

-  const channels = content('tests/__data__/output/database/channels.db')
+  expect(output).toEqual(expected)
+})

-  expect(Object.keys(channels[0]).sort()).toEqual([
-    '_id',
-    'channelsPath',
-    'cluster_id',
-    'configPath',
-    'country',
-    'groups',
-    'lang',
-    'logo',
-    'name',
-    'programCount',
-    'site',
-    'site_id',
-    'xmltv_id'
-  ])
+it('can update queue', () => {
+  const output = content('tests/__data__/output/database/queue.db')
+  const expected = content('tests/__data__/expected/database/queue-with-errors.db')

-  expect(channels[1]).toMatchObject({
-    _id: '0Wefq0oMR3feCcuY',
-    logo: 'https://example.com/logo.png'
-  })
-
-  const errors = content('tests/__data__/input/logs/errors.log')
-
-  expect(errors[0]).toMatchObject({
-    _id: '00AluKCrCnfgrl8W',
-    site: 'directv.com',
-    xmltv_id: 'BravoEast.us',
-    error: 'Invalid header value char'
-  })
+  expect(output).toEqual(expected)
 })

 function content(filepath) {
--- a/tests/commands/update-api.test.js
+++ b/tests/commands/update-api.test.js
@ -6,10 +6,7 @@ beforeEach(() => {
  fs.rmdirSync('tests/__data__/output', { recursive: true })
  fs.mkdirSync('tests/__data__/output')
  fs.mkdirSync('tests/__data__/temp/database', { recursive: true })
-  fs.copyFileSync(
-    'tests/__data__/input/database/channels.db',
-    'tests/__data__/temp/database/channels.db'
-  )
+  fs.copyFileSync('tests/__data__/input/database/queue.db', 'tests/__data__/temp/database/queue.db')
  fs.copyFileSync(
    'tests/__data__/input/database/programs.db',
    'tests/__data__/temp/database/programs.db'
--- a/tests/commands/update-guides.test.js
+++ b/tests/commands/update-guides.test.js
@ -6,17 +6,14 @@ beforeEach(() => {
  fs.rmdirSync('tests/__data__/output', { recursive: true })
  fs.mkdirSync('tests/__data__/output')
  fs.mkdirSync('tests/__data__/temp/database', { recursive: true })
-  fs.copyFileSync(
-    'tests/__data__/input/database/channels.db',
-    'tests/__data__/temp/database/channels.db'
-  )
+  fs.copyFileSync('tests/__data__/input/database/queue.db', 'tests/__data__/temp/database/queue.db')
  fs.copyFileSync(
    'tests/__data__/input/database/programs.db',
    'tests/__data__/temp/database/programs.db'
  )

  const stdout = execSync(
-    'DB_DIR=tests/__data__/temp/database PUBLIC_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs node scripts/commands/update-guides.js',
+    'DB_DIR=tests/__data__/temp/database DATA_DIR=tests/__data__/input/data PUBLIC_DIR=tests/__data__/output LOGS_DIR=tests/__data__/output/logs node scripts/commands/update-guides.js',
    { encoding: 'utf8' }
  )
 })
@ -35,9 +32,28 @@ it('can generate /guides', () => {
  const expected2 = content('tests/__data__/expected/guides/zw/dstv.com.epg.xml')

  expect(output2).toBe(expected2)
+})

-  const output3 = content('tests/__data__/output/logs/update-guides.log')
-  const expected3 = content('tests/__data__/expected/logs/update-guides.log')
+it('can create guides.log', () => {
+  const output = content('tests/__data__/output/logs/guides.log')
+  const expected = content('tests/__data__/expected/logs/guides.log')
+
+  expect(output).toBe(expected)
+})
+
+it('can log errors', () => {
+  const output1 = content('tests/__data__/output/logs/errors/ru/yandex.ru.log')
+  const expected1 = content('tests/__data__/expected/logs/errors/ru/yandex.ru.log')
+
+  expect(output1).toBe(expected1)
+
+  const output2 = content('tests/__data__/output/logs/errors/us/directv.com.log')
+  const expected2 = content('tests/__data__/expected/logs/errors/us/directv.com.log')
+
+  expect(output2).toBe(expected2)
+
+  const output3 = content('tests/__data__/output/logs/errors/ge/magticom.ge.log')
+  const expected3 = content('tests/__data__/expected/logs/errors/ge/magticom.ge.log')

  expect(output3).toBe(expected3)
 })
--- a/tests/commands/update-readme.test.js
+++ b/tests/commands/update-readme.test.js
@ -7,7 +7,7 @@ beforeEach(() => {
  fs.mkdirSync('tests/__data__/output')

  const stdout = execSync(
-    'LOGS_DIR=tests/__data__/input/logs node scripts/commands/update-readme.js --config=tests/__data__/input/_readme.json',
+    'LOGS_DIR=tests/__data__/input/logs DATA_DIR=tests/__data__/input/data node scripts/commands/update-readme.js --config=tests/__data__/input/_readme.json',
    { encoding: 'utf8' }
  )
 })
				`@ -0,0 +1 @@`
				`{"xmltv_id":"CNNInternationalEurope2.us","site":"example.com","site_id":"141","lang":"en","error":"The channel has the wrong xmltv_id"}`
				`@ -0,0 +1 @@`
				`{"xmltv_id":"CNNInternationalEurope.us","site":"magticom.ge","site_id":"140","lang":"ru","date":"2022-01-21T00:00:00Z","error":"Programs not found"}`
				`@ -0,0 +1 @@`
				`{"xmltv_id":"Perviykanal.ru","site":"yandex.ru","site_id":"1","lang":"ru","date":"2022-01-21T00:00:00Z","error":"Some error"}`
				`@ -0,0 +1 @@`
				`{"xmltv_id":"BravoEast.us","site":"directv.com","site_id":"237","lang":"en","date":"2022-01-21T00:00:00Z","error":"Invalid header value char"}`