Update scripts

scripts/commands/api/generate.ts

@@ -2,10 +2,11 @@ import { Logger, Storage, Collection } from '@freearhey/core'
 import { ChannelsParser } from '../../core'
 import path from 'path'
 import { SITES_DIR, API_DIR } from '../../constants'
-import { Channel } from 'epg-grabber'
+import epgGrabber from 'epg-grabber'
 
 type OutputItem = {
   channel: string | null
+  feed: string | null
   site: string
   site_id: string
   site_name: string
@@ -31,9 +32,13 @@ async function main() {
 
   logger.info(`  found ${parsedChannels.count()} channel(s)`)
 
-  const output = parsedChannels.map((channel: Channel): OutputItem => {
+  const output = parsedChannels.map((channel: epgGrabber.Channel): OutputItem => {
+    const xmltv_id = channel.xmltv_id || ''
+    const [channelId, feedId] = xmltv_id.split('@')
+
     return {
-      channel: channel.xmltv_id || null,
+      channel: channelId || null,
+      feed: feedId || null,
       site: channel.site || '',
       site_id: channel.site_id || '',
       site_name: channel.name,

scripts/commands/api/load.ts

@@ -7,6 +7,7 @@ async function main() {
 
   const requests = [
     client.download('channels.json'),
+    client.download('feeds.json'),
     client.download('countries.json'),
     client.download('regions.json'),
     client.download('subdivisions.json')

scripts/commands/channels/.gitignore

@@ -1 +0,0 @@
-/replace.ts

scripts/commands/channels/edit.ts

@@ -1,12 +1,16 @@
+import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
+import { select, input } from '@inquirer/prompts'
+import { ChannelsParser, XML } from '../../core'
+import { Channel, Feed } from '../../models'
 import { DATA_DIR } from '../../constants'
-import { Storage, Collection, Logger } from '@freearhey/core'
-import { ChannelsParser, XML, ApiChannel } from '../../core'
-import { Channel } from 'epg-grabber'
 import nodeCleanup from 'node-cleanup'
-import { program } from 'commander'
-import inquirer, { QuestionCollection } from 'inquirer'
-import Fuse from 'fuse.js'
+import epgGrabber from 'epg-grabber'
+import { Command } from 'commander'
 import readline from 'readline'
+import Fuse from 'fuse.js'
+
+type ChoiceValue = { type: string; value?: Feed | Channel }
+type Choice = { name: string; short?: string; value: ChoiceValue }
 
 if (process.platform === 'win32') {
   readline
@@ -19,105 +23,159 @@ if (process.platform === 'win32') {
     })
 }
 
+const program = new Command()
+
 program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
 
 const filepath = program.args[0]
-
 const logger = new Logger()
 const storage = new Storage()
-let channels = new Collection()
+let parsedChannels = new Collection()
 
-async function main() {
+main(filepath)
+nodeCleanup(() => {
+  save(filepath)
+})
+
+export default async function main(filepath: string) {
   if (!(await storage.exists(filepath))) {
     throw new Error(`File "${filepath}" does not exists`)
   }
 
   const parser = new ChannelsParser({ storage })
-  channels = await parser.parse(filepath)
+  parsedChannels = await parser.parse(filepath)
 
   const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await dataStorage.json('channels.json')
-  const searchIndex = new Fuse(channelsContent, { keys: ['name', 'alt_names'], threshold: 0.4 })
+  const channelsData = await dataStorage.json('channels.json')
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data => new Feed(data))
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
 
-  for (const channel of channels.all()) {
+  const searchIndex: Fuse<Channel> = new Fuse(channels.all(), {
+    keys: ['name', 'alt_names'],
+    threshold: 0.4
+  })
+
+  for (const channel of parsedChannels.all()) {
     if (channel.xmltv_id) continue
-    const question: QuestionCollection = {
-      name: 'option',
-      message: `Select xmltv_id for "${channel.name}" (${channel.site_id}):`,
-      type: 'list',
-      choices: getOptions(searchIndex, channel),
-      pageSize: 10
+    try {
+      channel.xmltv_id = await selectChannel(channel, searchIndex, feedsGroupedByChannelId)
+    } catch {
+      break
     }
-
-    await inquirer.prompt(question).then(async selected => {
-      switch (selected.option) {
-        case 'Type...':
-          const input = await getInput(channel)
-          channel.xmltv_id = input.xmltv_id
-          break
-        case 'Skip':
-          channel.xmltv_id = '-'
-          break
-        default:
-          const [, xmltv_id] = selected.option
-            .replace(/ \[.*\]/, '')
-            .split('|')
-            .map((i: string) => i.trim())
-          channel.xmltv_id = xmltv_id
-          break
-      }
-    })
   }
 
-  channels.forEach((channel: Channel) => {
+  parsedChannels.forEach((channel: epgGrabber.Channel) => {
     if (channel.xmltv_id === '-') {
       channel.xmltv_id = ''
     }
   })
 }
 
-main()
+async function selectChannel(
+  channel: epgGrabber.Channel,
+  searchIndex: Fuse<Channel>,
+  feedsGroupedByChannelId: Dictionary
+): Promise<string> {
+  const similarChannels = searchIndex
+    .search(channel.name)
+    .map((result: { item: Channel }) => result.item)
 
-function save() {
+  const selected: ChoiceValue = await select({
+    message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
+    choices: getChannelChoises(new Collection(similarChannels)),
+    pageSize: 10
+  })
+
+  switch (selected.type) {
+    case 'skip':
+      return '-'
+    case 'type': {
+      const typedChannelId = await input({ message: '  Channel ID:' })
+      const typedFeedId = await input({ message: '  Feed ID:', default: 'SD' })
+      return [typedChannelId, typedFeedId].join('@')
+    }
+    case 'channel': {
+      const selectedChannel = selected.value
+      if (!selectedChannel) return ''
+      const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
+      return [selectedChannel.id, selectedFeedId].join('@')
+    }
+  }
+
+  return ''
+}
+
+async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
+  const channelFeeds = feedsGroupedByChannelId.get(channelId) || []
+  if (channelFeeds.length <= 1) return ''
+
+  const selected: ChoiceValue = await select({
+    message: `Select feed ID for "${channelId}":`,
+    choices: getFeedChoises(channelFeeds),
+    pageSize: 10
+  })
+
+  switch (selected.type) {
+    case 'type':
+      return await input({ message: '  Feed ID:' })
+    case 'feed':
+      const selectedFeed = selected.value
+      if (!selectedFeed) return ''
+      return selectedFeed.id
+  }
+
+  return ''
+}
+
+function getChannelChoises(channels: Collection): Choice[] {
+  const choises: Choice[] = []
+
+  channels.forEach((channel: Channel) => {
+    const names = [channel.name, ...channel.altNames.all()].join(', ')
+
+    choises.push({
+      value: {
+        type: 'channel',
+        value: channel
+      },
+      name: `${channel.id} (${names})`,
+      short: `${channel.id}`
+    })
+  })
+
+  choises.push({ name: 'Type...', value: { type: 'type' } })
+  choises.push({ name: 'Skip', value: { type: 'skip' } })
+
+  return choises
+}
+
+function getFeedChoises(feeds: Collection): Choice[] {
+  const choises: Choice[] = []
+
+  feeds.forEach((feed: Feed) => {
+    let name = `${feed.id} (${feed.name})`
+    if (feed.isMain) name += ' [main]'
+
+    choises.push({
+      value: {
+        type: 'feed',
+        value: feed
+      },
+      name,
+      short: feed.id
+    })
+  })
+
+  choises.push({ name: 'Type...', value: { type: 'type' } })
+
+  return choises
+}
+
+function save(filepath: string) {
   if (!storage.existsSync(filepath)) return
-
-  const xml = new XML(channels)
-
+  const xml = new XML(parsedChannels)
   storage.saveSync(filepath, xml.toString())
-
   logger.info(`\nFile '${filepath}' successfully saved`)
 }
-
-nodeCleanup(() => {
-  save()
-})
-
-async function getInput(channel: Channel) {
-  const name = channel.name.trim()
-  const input = await inquirer.prompt([
-    {
-      name: 'xmltv_id',
-      message: '  xmltv_id:',
-      type: 'input'
-    }
-  ])
-
-  return { name, xmltv_id: input['xmltv_id'] }
-}
-
-function getOptions(index, channel: Channel) {
-  const similar = index.search(channel.name).map(result => new ApiChannel(result.item))
-
-  const variants = new Collection()
-  similar.forEach((_channel: ApiChannel) => {
-    const altNames = _channel.altNames.notEmpty() ? ` (${_channel.altNames.join(',')})` : ''
-    const closed = _channel.closed ? ` [closed:${_channel.closed}]` : ''
-    const replacedBy = _channel.replacedBy ? `[replaced_by:${_channel.replacedBy}]` : ''
-
-    variants.add(`${_channel.name}${altNames} | ${_channel.id}${closed}${replacedBy}`)
-  })
-  variants.add('Type...')
-  variants.add('Skip')
-
-  return variants.all()
-}

scripts/commands/channels/validate.ts

@@ -1,10 +1,11 @@
 import { Storage, Collection, Dictionary, File } from '@freearhey/core'
-import { ChannelsParser, ApiChannel } from '../../core'
+import { ChannelsParser } from '../../core'
+import { Channel } from '../../models'
 import { program } from 'commander'
 import chalk from 'chalk'
 import langs from 'langs'
 import { DATA_DIR } from '../../constants'
-import { Channel } from 'epg-grabber'
+import epgGrabber from 'epg-grabber'
 
 program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv)
 
@@ -21,8 +22,9 @@ async function main() {
   const parser = new ChannelsParser({ storage: new Storage() })
 
   const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await dataStorage.json('channels.json')
-  const channels = new Collection(channelsContent).map(data => new ApiChannel(data))
+  const channelsData = await dataStorage.json('channels.json')
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const channelsGroupedById = channels.groupBy((channel: Channel) => channel.id)
 
   let totalFiles = 0
   let totalErrors = 0
@@ -37,7 +39,7 @@ async function main() {
 
     const bufferBySiteId = new Dictionary()
     const errors: ValidationError[] = []
-    parsedChannels.forEach((channel: Channel) => {
+    parsedChannels.forEach((channel: epgGrabber.Channel) => {
       const bufferId: string = channel.site_id
       if (bufferBySiteId.missing(bufferId)) {
         bufferBySiteId.set(bufferId, true)
@@ -52,10 +54,8 @@ async function main() {
       }
 
       if (!channel.xmltv_id) return
-
-      const foundChannel = channels.first(
-        (_channel: ApiChannel) => _channel.id === channel.xmltv_id
-      )
+      const [channelId] = channel.xmltv_id.split('@')
+      const foundChannel = channelsGroupedById.get(channelId)
       if (!foundChannel) {
         errors.push({ type: 'wrong_xmltv_id', ...channel })
         totalErrors++

scripts/commands/sites/init.ts

@@ -1,8 +1,8 @@
 import { Logger, Storage } from '@freearhey/core'
-import { program } from 'commander'
 import { SITES_DIR } from '../../constants'
-import fs from 'fs-extra'
 import { pathToFileURL } from 'node:url'
+import { program } from 'commander'
+import fs from 'fs-extra'
 
 program.argument('<site>', 'Domain name of the site').parse(process.argv)
 

scripts/commands/sites/update.ts

@@ -1,8 +1,8 @@
-import { Channel } from 'epg-grabber'
-import { Logger, Storage, Collection } from '@freearhey/core'
 import { IssueLoader, HTMLTable, ChannelsParser } from '../../core'
-import { Issue, Site } from '../../models'
+import { Logger, Storage, Collection } from '@freearhey/core'
 import { SITES_DIR, ROOT_DIR } from '../../constants'
+import { Issue, Site } from '../../models'
+import { Channel } from 'epg-grabber'
 
 async function main() {
   const logger = new Logger({ disabled: true })
@@ -15,11 +15,17 @@ async function main() {
   const folders = await sitesStorage.list('*/')
 
   logger.info('loading issues...')
-  const issues = await loadIssues(loader)
+  const issues = await loader.load()
 
   logger.info('putting the data together...')
+  const brokenGuideReports = issues.filter(issue =>
+    issue.labels.find((label: string) => label === 'broken guide')
+  )
   for (const domain of folders) {
-    const filteredIssues = issues.filter((issue: Issue) => domain === issue.data.get('site'))
+    const filteredIssues = brokenGuideReports.filter(
+      (issue: Issue) => domain === issue.data.get('site')
+    )
+
     const site = new Site({
       domain,
       issues: filteredIssues
@@ -62,10 +68,3 @@ async function main() {
 }
 
 main()
-
-async function loadIssues(loader: IssueLoader) {
-  const issuesWithStatusWarning = await loader.load({ labels: ['broken guide', 'status:warning'] })
-  const issuesWithStatusDown = await loader.load({ labels: ['broken guide', 'status:down'] })
-
-  return issuesWithStatusWarning.concat(issuesWithStatusDown)
-}