Update create-databasee.js

scripts/commands/create-database.js

@@ -1,6 +1,6 @@
 const { db, file, parser, logger } = require('../core')
 const { program } = require('commander')
-const _ = require('lodash')
+const { shuffle } = require('lodash')
 
 const options = program
   .option(
@@ -27,7 +27,8 @@ main()
 async function getChannels() {
   logger.info(`Loading channels...`)
 
-  const channels = []
+  let channels = {}
+
   const files = await file.list(options.channels)
   for (const filepath of files) {
     const dir = file.dirname(filepath)
@@ -37,17 +38,28 @@ async function getChannels() {
     const configPath = `${dir}/${site}.config.js`
     const config = require(file.resolve(configPath))
     if (config.ignore) continue
-    const [__, gid] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
+    const [__, groupId] = filename.match(/_([a-z-]+)\.channels\.xml/i) || [null, null]
     const items = await parser.parseChannels(filepath)
     for (const item of items) {
+      const key = `${item.site}:${item.site_id}`
+      if (!channels[key]) {
         const countryCode = item.xmltv_id.split('.')[1]
         item.country = countryCode ? countryCode.toUpperCase() : null
         item.channelsPath = filepath
         item.configPath = configPath
-      item.gid = gid
-      channels.push(item)
+        item.groups = []
+
+        channels[key] = item
+      }
+
+      if (!channels[key].groups.includes(groupId)) {
+        channels[key].groups.push(groupId)
       }
     }
+  }
+
+  channels = Object.values(channels)
+
   logger.info(`Found ${channels.length} channels`)
 
   return channels
@@ -57,7 +69,7 @@ async function saveToDatabase(channels = []) {
   logger.info('Saving to the database...')
   await db.channels.load()
   await db.channels.reset()
-  const chunks = split(_.shuffle(channels), options.maxClusters)
+  const chunks = split(shuffle(channels), options.maxClusters)
   for (const [i, chunk] of chunks.entries()) {
     for (const item of chunk) {
       item.cluster_id = i + 1

tests/__data__/input/sites/example.com_ca-nl.channels.xml

@@ -2,5 +2,6 @@
 <site site="example.com">
   <channels>
     <channel lang="ru" xmltv_id="CNNInternationalEurope.us" site_id="140">CNN International Europe</channel>
+    <channel lang="en" xmltv_id="CNNInternationalEurope2.us" site_id="140">CNN International Europe 2</channel>
   </channels>
 </site>

tests/commands/create-database.test.js

@@ -25,7 +25,7 @@ it('can create channels database', () => {
       site: 'example.com',
       channelsPath: 'tests/__data__/input/sites/example.com_ca-nl.channels.xml',
       configPath: 'tests/__data__/input/sites/example.com.config.js',
-      gid: 'ca-nl',
+      groups: ['ca-nl'],
       cluster_id: 1
     }
   ])