From af69100bc51cb840e8df3e450adb8f71844cb1ed Mon Sep 17 00:00:00 2001
From: Aleksandr Statciuk <github@oiya.xyz>
Date: Fri, 21 Jan 2022 17:30:14 +0300
Subject: [PATCH] Update create-databasee.js

---
 scripts/commands/create-database.js           | 32 +++++++++++++------
 .../sites/example.com_ca-nl.channels.xml      |  1 +
 tests/commands/create-database.test.js        |  2 +-
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/scripts/commands/create-database.js b/scripts/commands/create-database.js
index 5697dcd3..973e4059 100644
--- a/scripts/commands/create-database.js
+++ b/scripts/commands/create-database.js
@@ -1,6 +1,6 @@
 const { db, file, parser, logger } = require('../core')
 const { program } = require('commander')
-const _ = require('lodash')
+const { shuffle } = require('lodash')
 
 const options = program
   .option(
@@ -27,7 +27,8 @@ main()
 async function getChannels() {
   logger.info(`Loading channels...`)
 
-  const channels = []
+  let channels = {}
+
   const files = await file.list(options.channels)
   for (const filepath of files) {
     const dir = file.dirname(filepath)
@@ -37,17 +38,28 @@ async function getChannels() {
     const configPath = `${dir}/${site}.config.js`
     const config = require(file.resolve(configPath))
     if (config.ignore) continue
-    const [__, gid] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
+    const [__, groupId] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
     const items = await parser.parseChannels(filepath)
     for (const item of items) {
-      const countryCode = item.xmltv_id.split('.')[1]
-      item.country = countryCode ? countryCode.toUpperCase() : null
-      item.channelsPath = filepath
-      item.configPath = configPath
-      item.gid = gid
-      channels.push(item)
+      const key = `${item.site}:${item.site_id}`
+      if (!channels[key]) {
+        const countryCode = item.xmltv_id.split('.')[1]
+        item.country = countryCode ? countryCode.toUpperCase() : null
+        item.channelsPath = filepath
+        item.configPath = configPath
+        item.groups = []
+
+        channels[key] = item
+      }
+
+      if (!channels[key].groups.includes(groupId)) {
+        channels[key].groups.push(groupId)
+      }
     }
   }
+
+  channels = Object.values(channels)
+
   logger.info(`Found ${channels.length} channels`)
 
   return channels
@@ -57,7 +69,7 @@ async function saveToDatabase(channels = []) {
   logger.info('Saving to the database...')
   await db.channels.load()
   await db.channels.reset()
-  const chunks = split(_.shuffle(channels), options.maxClusters)
+  const chunks = split(shuffle(channels), options.maxClusters)
   for (const [i, chunk] of chunks.entries()) {
     for (const item of chunk) {
       item.cluster_id = i + 1
diff --git a/tests/__data__/input/sites/example.com_ca-nl.channels.xml b/tests/__data__/input/sites/example.com_ca-nl.channels.xml
index d067c030..aa4b1ac9 100644
--- a/tests/__data__/input/sites/example.com_ca-nl.channels.xml
+++ b/tests/__data__/input/sites/example.com_ca-nl.channels.xml
@@ -2,5 +2,6 @@
 <site site="example.com">
   <channels>
     <channel lang="ru" xmltv_id="CNNInternationalEurope.us" site_id="140">CNN International Europe</channel>
+    <channel lang="en" xmltv_id="CNNInternationalEurope2.us" site_id="140">CNN International Europe 2</channel>
   </channels>
 </site>
\ No newline at end of file
diff --git a/tests/commands/create-database.test.js b/tests/commands/create-database.test.js
index d84c9146..2e3c3402 100644
--- a/tests/commands/create-database.test.js
+++ b/tests/commands/create-database.test.js
@@ -25,7 +25,7 @@ it('can create channels database', () => {
       site: 'example.com',
       channelsPath: 'tests/__data__/input/sites/example.com_ca-nl.channels.xml',
       configPath: 'tests/__data__/input/sites/example.com.config.js',
-      gid: 'ca-nl',
+      groups: ['ca-nl'],
       cluster_id: 1
     }
   ])