Merge pull request #2752 from iptv-org/patch-2025.04.3

Patch 2025.04.3
2025-05-09 00:20:08 -04:00 · 2025-04-22 06:04:26 +03:00 · 2025-04-22 06:04:26 +03:00 · db56a4d6c0
commit db56a4d6c0
parent 113395c6b5 11cab21cfc
35 changed files with 672 additions and 166 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -391,9 +391,9 @@ npm run channels:edit path/to/channels.xml
 This way, you can map channels by simply selecting the proper ID from the list:

 ```sh
-? Select xmltv_id for "BBC One" (bbc1): (Use arrow keys)
-❯ BBC One (BBC1, BBC Television, BBC Television Service) | BBCOne.uk
-  BBC One HD | BBCOneHD.uk
+? Select channel ID for "BBC One" (bbc1): (Use arrow keys)
+❯ BBCOne.uk (BBC One, BBC1, BBC Television, BBC Television Service)
+  BBCOneHD.uk (BBC One HD)
  Type...
  Skip
 ```
--- a/README.md
+++ b/README.md
@ -57,7 +57,7 @@ Options:
  -c, --channels <path>         Path to *.channels.xml file (required if the "--site" attribute is
                                not specified)
  -o, --output <path>           Path to output file (default: "guide.xml")
-  -l, --lang <code>             Allows to limit the download to channels in the specified language only (ISO 639-1 code)
+  -l, --lang <codes>            Allows you to restrict downloading to channels in specified languages only (example: "en,id")
  -t, --timeout <milliseconds>  Timeout for each request in milliseconds (default: 0)
  -d, --delay <milliseconds>    Delay between request in milliseconds (default: 0)
  -x, --proxy <url>             Use the specified proxy (example: "socks5://username:password@127.0.0.1:1234")
--- a/package-lock.json
+++ b/package-lock.json
@ -12,6 +12,7 @@
        "@eslint/eslintrc": "^3.2.0",
        "@eslint/js": "^9.17.0",
        "@freearhey/core": "^0.7.0",
+        "@freearhey/search-js": "^0.1.2",
        "@ntlab/sfetch": "^1.2.0",
        "@octokit/core": "^6.1.3",
        "@octokit/plugin-paginate-rest": "^11.3.6",
@ -43,7 +44,6 @@
        "eslint-config-prettier": "^9.0.0",
        "form-data": "^4.0.0",
        "fs-extra": "^10.0.1",
-        "fuse.js": "^7.0.0",
        "glob": "^7.2.0",
        "globals": "^15.14.0",
        "husky": "^9.1.7",
@ -1179,6 +1179,17 @@
        "url": "https://github.com/sponsors/isaacs"
      }
    },
+    "node_modules/@freearhey/search-js": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@freearhey/search-js/-/search-js-0.1.2.tgz",
+      "integrity": "sha512-F2o+xpGCXOK4OsZfKEHfXNNkAZmny2eBnPOp+P0iyV20ja7gJGfTFaEc6okcuEo6OB6P7LnSxTvISkoArFtlfg==",
+      "dependencies": {
+        "lodash": "^4.17.21"
+      },
+      "engines": {
+        "node": ">=16.0.0"
+      }
+    },
    "node_modules/@humanfs/core": {
      "version": "0.19.1",
      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
@ -5359,14 +5370,6 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
-    "node_modules/fuse.js": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.0.0.tgz",
-      "integrity": "sha512-14F4hBIxqKvD4Zz/XjDc3y94mNZN6pRv3U13Udo0lNLCWRBUsrMv2xwcF/y/Z5sV6+FQW+/ow68cHpm4sunt8Q==",
-      "engines": {
-        "node": ">=10"
-      }
-    },
    "node_modules/gensync": {
      "version": "1.0.0-beta.2",
      "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@ -10189,6 +10192,14 @@
        }
      }
    },
+    "@freearhey/search-js": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/@freearhey/search-js/-/search-js-0.1.2.tgz",
+      "integrity": "sha512-F2o+xpGCXOK4OsZfKEHfXNNkAZmny2eBnPOp+P0iyV20ja7gJGfTFaEc6okcuEo6OB6P7LnSxTvISkoArFtlfg==",
+      "requires": {
+        "lodash": "^4.17.21"
+      }
+    },
    "@humanfs/core": {
      "version": "0.19.1",
      "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz",
@ -13127,11 +13138,6 @@
      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="
    },
-    "fuse.js": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/fuse.js/-/fuse.js-7.0.0.tgz",
-      "integrity": "sha512-14F4hBIxqKvD4Zz/XjDc3y94mNZN6pRv3U13Udo0lNLCWRBUsrMv2xwcF/y/Z5sV6+FQW+/ow68cHpm4sunt8Q=="
-    },
    "gensync": {
      "version": "1.0.0-beta.2",
      "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
--- a/package.json
+++ b/package.json
@ -43,6 +43,7 @@
    "@eslint/eslintrc": "^3.2.0",
    "@eslint/js": "^9.17.0",
    "@freearhey/core": "^0.7.0",
+    "@freearhey/search-js": "^0.1.2",
    "@ntlab/sfetch": "^1.2.0",
    "@octokit/core": "^6.1.3",
    "@octokit/plugin-paginate-rest": "^11.3.6",
@ -74,7 +75,6 @@
    "eslint-config-prettier": "^9.0.0",
    "form-data": "^4.0.0",
    "fs-extra": "^10.0.1",
-    "fuse.js": "^7.0.0",
    "glob": "^7.2.0",
    "globals": "^15.14.0",
    "husky": "^9.1.7",
--- a/scripts/commands/api/load.ts
+++ b/scripts/commands/api/load.ts
@ -1,19 +1,24 @@
-import { Logger } from '@freearhey/core'
-import { ApiClient } from '../../core'
+import { DATA_DIR } from '../../constants'
+import { Storage } from '@freearhey/core'
+import { DataLoader } from '../../core'

 async function main() {
-  const logger = new Logger()
-  const client = new ApiClient({ logger })
+  const storage = new Storage(DATA_DIR)
+  const loader = new DataLoader({ storage })

-  const requests = [
-    client.download('channels.json'),
-    client.download('feeds.json'),
-    client.download('countries.json'),
-    client.download('regions.json'),
-    client.download('subdivisions.json')
-  ]
-
-  await Promise.all(requests)
+  await Promise.all([
+    loader.download('blocklist.json'),
+    loader.download('categories.json'),
+    loader.download('channels.json'),
+    loader.download('countries.json'),
+    loader.download('languages.json'),
+    loader.download('regions.json'),
+    loader.download('subdivisions.json'),
+    loader.download('feeds.json'),
+    loader.download('timezones.json'),
+    loader.download('guides.json'),
+    loader.download('streams.json')
+  ])
 }

 main()
--- a/scripts/commands/channels/edit.ts
+++ b/scripts/commands/channels/edit.ts
@ -4,13 +4,17 @@ import { ChannelsParser, XML } from '../../core'
 import { Channel, Feed } from '../../models'
 import { DATA_DIR } from '../../constants'
 import nodeCleanup from 'node-cleanup'
-import epgGrabber from 'epg-grabber'
 import { Command } from 'commander'
 import readline from 'readline'
-import Fuse from 'fuse.js'
+import sjs from '@freearhey/search-js'
+import { DataProcessor, DataLoader } from '../../core'
+import type { DataLoaderData } from '../../types/dataLoader'
+import type { DataProcessorData } from '../../types/dataProcessor'
+import epgGrabber from 'epg-grabber'
+import { ChannelSearchableData } from '../../types/channel'

 type ChoiceValue = { type: string; value?: Feed | Channel }
-type Choice = { name: string; short?: string; value: ChoiceValue }
+type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }

 if (process.platform === 'win32') {
  readline
@ -42,31 +46,48 @@ export default async function main(filepath: string) {
    throw new Error(`File "${filepath}" does not exists`)
  }

+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
+  const dataStorage = new Storage(DATA_DIR)
+  const loader = new DataLoader({ storage: dataStorage })
+  const data: DataLoaderData = await loader.load()
+  const { feedsGroupedByChannelId, channels, channelsKeyById }: DataProcessorData =
+    processor.process(data)
+
+  logger.info('loading channels...')
  const parser = new ChannelsParser({ storage })
  parsedChannels = await parser.parse(filepath)
+  const parsedChannelsWithoutId = parsedChannels.filter(
+    (channel: epgGrabber.Channel) => !channel.xmltv_id
+  )

-  const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
-  const channels = new Collection(channelsData).map(data => new Channel(data))
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data => new Feed(data))
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
+  logger.info(
+    `found ${parsedChannels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
+  )

-  const searchIndex: Fuse<Channel> = new Fuse(channels.all(), {
-    keys: ['name', 'alt_names'],
-    threshold: 0.4
+  logger.info('creating search index...')
+  const items = channels.map((channel: Channel) => channel.getSearchable()).all()
+  const searchIndex = sjs.createIndex(items, {
+    searchable: ['name', 'altNames', 'guideNames', 'streamNames', 'feedFullNames']
  })

-  for (const channel of parsedChannels.all()) {
-    if (channel.xmltv_id) continue
+  logger.info('starting...\n')
+
+  for (const parsedChannel of parsedChannelsWithoutId.all()) {
    try {
-      channel.xmltv_id = await selectChannel(channel, searchIndex, feedsGroupedByChannelId)
-    } catch {
+      parsedChannel.xmltv_id = await selectChannel(
+        parsedChannel,
+        searchIndex,
+        feedsGroupedByChannelId,
+        channelsKeyById
+      )
+    } catch (err) {
+      logger.info(err.message)
      break
    }
  }

-  parsedChannels.forEach((channel: epgGrabber.Channel) => {
+  parsedChannelsWithoutId.forEach((channel: epgGrabber.Channel) => {
    if (channel.xmltv_id === '-') {
      channel.xmltv_id = ''
    }
@ -75,12 +96,14 @@ export default async function main(filepath: string) {

 async function selectChannel(
  channel: epgGrabber.Channel,
-  searchIndex: Fuse<Channel>,
-  feedsGroupedByChannelId: Dictionary
+  searchIndex,
+  feedsGroupedByChannelId: Dictionary,
+  channelsKeyById: Dictionary
 ): Promise<string> {
+  const query = escapeRegex(channel.name)
  const similarChannels = searchIndex
-    .search(channel.name)
-    .map((result: { item: Channel }) => result.item)
+    .search(query)
+    .map((item: ChannelSearchableData) => channelsKeyById.get(item.id))

  const selected: ChoiceValue = await select({
    message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
@ -93,13 +116,16 @@ async function selectChannel(
      return '-'
    case 'type': {
      const typedChannelId = await input({ message: '  Channel ID:' })
-      const typedFeedId = await input({ message: '  Feed ID:', default: 'SD' })
-      return [typedChannelId, typedFeedId].join('@')
+      if (!typedChannelId) return ''
+      const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return typedChannelId
+      return [typedChannelId, selectedFeedId].join('@')
    }
    case 'channel': {
      const selectedChannel = selected.value
      if (!selectedChannel) return ''
      const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return selectedChannel.id
      return [selectedChannel.id, selectedFeedId].join('@')
    }
  }
@ -108,18 +134,22 @@ async function selectChannel(
 }

 async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
-  const channelFeeds = feedsGroupedByChannelId.get(channelId) || []
-  if (channelFeeds.length <= 1) return ''
+  const channelFeeds = feedsGroupedByChannelId.has(channelId)
+    ? new Collection(feedsGroupedByChannelId.get(channelId))
+    : new Collection()
+  const choices = getFeedChoises(channelFeeds)

  const selected: ChoiceValue = await select({
    message: `Select feed ID for "${channelId}":`,
-    choices: getFeedChoises(channelFeeds),
+    choices,
    pageSize: 10
  })

  switch (selected.type) {
+    case 'skip':
+      return '-'
    case 'type':
-      return await input({ message: '  Feed ID:' })
+      return await input({ message: '  Feed ID:', default: 'SD' })
    case 'feed':
      const selectedFeed = selected.value
      if (!selectedFeed) return ''
@ -133,7 +163,7 @@ function getChannelChoises(channels: Collection): Choice[] {
  const choises: Choice[] = []

  channels.forEach((channel: Channel) => {
-    const names = [channel.name, ...channel.altNames.all()].join(', ')
+    const names = new Collection([channel.name, ...channel.getAltNames().all()]).uniq().join(', ')

    choises.push({
      value: {
@ -163,12 +193,14 @@ function getFeedChoises(feeds: Collection): Choice[] {
        type: 'feed',
        value: feed
      },
+      default: feed.isMain,
      name,
      short: feed.id
    })
  })

  choises.push({ name: 'Type...', value: { type: 'type' } })
+  choises.push({ name: 'Skip', value: { type: 'skip' } })

  return choises
 }
@ -179,3 +211,7 @@ function save(filepath: string) {
  storage.saveSync(filepath, xml.toString())
  logger.info(`\nFile '${filepath}' successfully saved`)
 }
+
+function escapeRegex(string: string) {
+  return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
+}
--- a/scripts/commands/epg/grab.ts
+++ b/scripts/commands/epg/grab.ts
@ -14,7 +14,7 @@ program
    )
  )
  .addOption(new Option('-o, --output <path>', 'Path to output file').default('guide.xml'))
-  .addOption(new Option('-l, --lang <code>', 'Filter channels by language (ISO 639-1 code)'))
+  .addOption(new Option('-l, --lang <codes>', 'Filter channels by languages (ISO 639-1 codes)'))
  .addOption(
    new Option('-t, --timeout <milliseconds>', 'Override the default timeout for each request').env(
      'TIMEOUT'
@ -90,7 +90,11 @@ async function main() {
    parsedChannels = parsedChannels.concat(await parser.parse(filepath))
  }
  if (options.lang) {
-    parsedChannels = parsedChannels.filter((channel: Channel) => channel.lang === options.lang)
+    parsedChannels = parsedChannels.filter((channel: Channel) => {
+      if (!options.lang || !channel.lang) return true
+
+      return options.lang.includes(channel.lang)
+    })
  }
  logger.info(`  found ${parsedChannels.count()} channel(s)`)

--- a/scripts/core/apiClient.ts
+++ b/scripts/core/apiClient.ts
@ -1,59 +1,16 @@
-import { Logger, Storage } from '@freearhey/core'
-import axios, { AxiosInstance, AxiosResponse, AxiosProgressEvent } from 'axios'
-import cliProgress, { MultiBar } from 'cli-progress'
-import numeral from 'numeral'
+import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'

 export class ApiClient {
-  progressBar: MultiBar
-  client: AxiosInstance
-  storage: Storage
-  logger: Logger
+  instance: AxiosInstance

-  constructor({ logger }: { logger: Logger }) {
-    this.logger = logger
-    this.client = axios.create({
+  constructor() {
+    this.instance = axios.create({
+      baseURL: 'https://iptv-org.github.io/api',
      responseType: 'stream'
    })
-    this.storage = new Storage()
-    this.progressBar = new cliProgress.MultiBar({
-      stopOnComplete: true,
-      hideCursor: true,
-      forceRedraw: true,
-      barsize: 36,
-      format(options, params, payload) {
-        const filename = payload.filename.padEnd(18, ' ')
-        const barsize = options.barsize || 40
-        const percent = (params.progress * 100).toFixed(2)
-        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
-        const total = numeral(params.total).format('0.0 b')
-        const completeSize = Math.round(params.progress * barsize)
-        const incompleteSize = barsize - completeSize
-        const bar =
-          options.barCompleteString && options.barIncompleteString
-            ? options.barCompleteString.substr(0, completeSize) +
-              options.barGlue +
-              options.barIncompleteString.substr(0, incompleteSize)
-            : '-'.repeat(barsize)
-
-        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
-      }
-    })
  }

-  async download(filename: string) {
-    const stream = await this.storage.createStream(`temp/data/${filename}`)
-
-    const bar = this.progressBar.create(0, 0, { filename })
-
-    this.client
-      .get(`https://iptv-org.github.io/api/${filename}`, {
-        onDownloadProgress({ total, loaded, rate }: AxiosProgressEvent) {
-          if (total) bar.setTotal(total)
-          bar.update(loaded, { speed: rate })
-        }
-      })
-      .then((response: AxiosResponse) => {
-        response.data.pipe(stream)
-      })
+  get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
+    return this.instance.get(url, options)
  }
 }
--- a/scripts/core/dataLoader.ts
+++ b/scripts/core/dataLoader.ts
@ -0,0 +1,100 @@
+import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
+import cliProgress, { MultiBar } from 'cli-progress'
+import { Storage } from '@freearhey/core'
+import { ApiClient } from './apiClient'
+import numeral from 'numeral'
+
+export class DataLoader {
+  client: ApiClient
+  storage: Storage
+  progressBar: MultiBar
+
+  constructor(props: DataLoaderProps) {
+    this.client = new ApiClient()
+    this.storage = props.storage
+    this.progressBar = new cliProgress.MultiBar({
+      stopOnComplete: true,
+      hideCursor: true,
+      forceRedraw: true,
+      barsize: 36,
+      format(options, params, payload) {
+        const filename = payload.filename.padEnd(18, ' ')
+        const barsize = options.barsize || 40
+        const percent = (params.progress * 100).toFixed(2)
+        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
+        const total = numeral(params.total).format('0.0 b')
+        const completeSize = Math.round(params.progress * barsize)
+        const incompleteSize = barsize - completeSize
+        const bar =
+          options.barCompleteString && options.barIncompleteString
+            ? options.barCompleteString.substr(0, completeSize) +
+              options.barGlue +
+              options.barIncompleteString.substr(0, incompleteSize)
+            : '-'.repeat(barsize)
+
+        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
+      }
+    })
+  }
+
+  async load(): Promise<DataLoaderData> {
+    const [
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    ] = await Promise.all([
+      this.storage.json('countries.json'),
+      this.storage.json('regions.json'),
+      this.storage.json('subdivisions.json'),
+      this.storage.json('languages.json'),
+      this.storage.json('categories.json'),
+      this.storage.json('blocklist.json'),
+      this.storage.json('channels.json'),
+      this.storage.json('feeds.json'),
+      this.storage.json('timezones.json'),
+      this.storage.json('guides.json'),
+      this.storage.json('streams.json')
+    ])
+
+    return {
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    }
+  }
+
+  async download(filename: string) {
+    if (!this.storage || !this.progressBar) return
+
+    const stream = await this.storage.createStream(filename)
+    const progressBar = this.progressBar.create(0, 0, { filename })
+
+    this.client
+      .get(filename, {
+        responseType: 'stream',
+        onDownloadProgress({ total, loaded, rate }) {
+          if (total) progressBar.setTotal(total)
+          progressBar.update(loaded, { speed: rate })
+        }
+      })
+      .then(response => {
+        response.data.pipe(stream)
+      })
+  }
+}
--- a/scripts/core/dataProcessor.ts
+++ b/scripts/core/dataProcessor.ts
@ -0,0 +1,39 @@
+import { DataLoaderData } from '../types/dataLoader'
+import { Collection } from '@freearhey/core'
+import { Channel, Feed, Guide, Stream } from '../models'
+
+export class DataProcessor {
+  constructor() {}
+
+  process(data: DataLoaderData) {
+    let channels = new Collection(data.channels).map(data => new Channel(data))
+    const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
+
+    const guides = new Collection(data.guides).map(data => new Guide(data))
+    const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
+
+    const streams = new Collection(data.streams).map(data => new Stream(data))
+    const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
+
+    const feeds = new Collection(data.feeds).map(data =>
+      new Feed(data)
+        .withGuides(guidesGroupedByStreamId)
+        .withStreams(streamsGroupedById)
+        .withChannel(channelsKeyById)
+    )
+    const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
+
+    channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
+
+    return {
+      feedsGroupedByChannelId,
+      guidesGroupedByStreamId,
+      streamsGroupedById,
+      channelsKeyById,
+      channels,
+      streams,
+      guides,
+      feeds
+    }
+  }
+}
--- a/scripts/core/index.ts
+++ b/scripts/core/index.ts
@ -1,15 +1,17 @@
-export * from './xml'
-export * from './channelsParser'
-export * from './xmltv'
-export * from './configLoader'
-export * from './grabber'
-export * from './job'
-export * from './queue'
-export * from './guideManager'
-export * from './guide'
 export * from './apiClient'
-export * from './queueCreator'
+export * from './channelsParser'
+export * from './configLoader'
+export * from './dataLoader'
+export * from './dataProcessor'
+export * from './grabber'
+export * from './guide'
+export * from './guideManager'
+export * from './htmlTable'
 export * from './issueLoader'
 export * from './issueParser'
-export * from './htmlTable'
+export * from './job'
 export * from './proxyParser'
+export * from './queue'
+export * from './queueCreator'
+export * from './xml'
+export * from './xmltv'
--- a/scripts/core/queueCreator.ts
+++ b/scripts/core/queueCreator.ts
@ -38,7 +38,6 @@ export class QueueCreator {
    const queue = new Queue()
    for (const channel of this.parsedChannels.all()) {
      if (!channel.site || !channel.site_id || !channel.name) continue
-      if (this.options.lang && channel.lang !== this.options.lang) continue

      const configPath = path.resolve(SITES_DIR, `${channel.site}/${channel.site}.config.js`)
      const config: SiteConfig = await this.configLoader.load(configPath)
--- a/scripts/models/channel.ts
+++ b/scripts/models/channel.ts
@ -1,40 +1,24 @@
-import { Collection } from '@freearhey/core'
-
-type ChannelData = {
-  id: string
-  name: string
-  alt_names: string[]
-  network: string
-  owners: Collection
-  country: string
-  subdivision: string
-  city: string
-  categories: Collection
-  is_nsfw: boolean
-  launched: string
-  closed: string
-  replaced_by: string
-  website: string
-  logo: string
-}
+import { ChannelData, ChannelSearchableData } from '../types/channel'
+import { Collection, Dictionary } from '@freearhey/core'
+import { Stream, Guide, Feed } from './'

 export class Channel {
  id: string
  name: string
-  altNames: Collection
+  altNames?: Collection
  network?: string
-  owners: Collection
+  owners?: Collection
  countryCode: string
  subdivisionCode?: string
  cityName?: string
-  categoryIds: Collection
-  categories?: Collection
+  categoryIds?: Collection
  isNSFW: boolean
  launched?: string
  closed?: string
  replacedBy?: string
  website?: string
-  logo: string
+  logo?: string
+  feeds?: Collection

  constructor(data: ChannelData) {
    this.id = data.id
@ -53,4 +37,77 @@ export class Channel {
    this.website = data.website || undefined
    this.logo = data.logo
  }
+
+  withFeeds(feedsGroupedByChannelId: Dictionary): this {
+    this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
+
+    return this
+  }
+
+  getFeeds(): Collection {
+    if (!this.feeds) return new Collection()
+
+    return this.feeds
+  }
+
+  getGuides(): Collection {
+    let guides = new Collection()
+
+    this.getFeeds().forEach((feed: Feed) => {
+      guides = guides.concat(feed.getGuides())
+    })
+
+    return guides
+  }
+
+  getGuideNames(): Collection {
+    return this.getGuides()
+      .map((guide: Guide) => guide.siteName)
+      .uniq()
+  }
+
+  getStreams(): Collection {
+    let streams = new Collection()
+
+    this.getFeeds().forEach((feed: Feed) => {
+      streams = streams.concat(feed.getStreams())
+    })
+
+    return streams
+  }
+
+  getStreamNames(): Collection {
+    return this.getStreams()
+      .map((stream: Stream) => stream.getName())
+      .uniq()
+  }
+
+  getFeedFullNames(): Collection {
+    return this.getFeeds()
+      .map((feed: Feed) => feed.getFullName())
+      .uniq()
+  }
+
+  getName(): string {
+    return this.name || ''
+  }
+
+  getId(): string {
+    return this.id || ''
+  }
+
+  getAltNames(): Collection {
+    return this.altNames || new Collection()
+  }
+
+  getSearchable(): ChannelSearchableData {
+    return {
+      id: this.getId(),
+      name: this.getName(),
+      altNames: this.getAltNames().all(),
+      guideNames: this.getGuideNames().all(),
+      streamNames: this.getStreamNames().all(),
+      feedFullNames: this.getFeedFullNames().all()
+    }
+  }
 }
--- a/scripts/models/feed.ts
+++ b/scripts/models/feed.ts
@ -1,18 +1,10 @@
-import { Collection } from '@freearhey/core'
-
-type FeedData = {
-  channel: string
-  id: string
-  name: string
-  is_main: boolean
-  broadcast_area: Collection
-  languages: Collection
-  timezones: Collection
-  video_format: string
-}
+import { Collection, Dictionary } from '@freearhey/core'
+import { FeedData } from '../types/feed'
+import { Channel } from './channel'

 export class Feed {
  channelId: string
+  channel?: Channel
  id: string
  name: string
  isMain: boolean
@ -20,6 +12,8 @@ export class Feed {
  languageCodes: Collection
  timezoneIds: Collection
  videoFormat: string
+  guides?: Collection
+  streams?: Collection

  constructor(data: FeedData) {
    this.channelId = data.channel
@ -31,4 +25,48 @@ export class Feed {
    this.timezoneIds = new Collection(data.timezones)
    this.videoFormat = data.video_format
  }
+
+  withChannel(channelsKeyById: Dictionary): this {
+    this.channel = channelsKeyById.get(this.channelId)
+
+    return this
+  }
+
+  withStreams(streamsGroupedById: Dictionary): this {
+    this.streams = new Collection(streamsGroupedById.get(`${this.channelId}@${this.id}`))
+
+    if (this.isMain) {
+      this.streams = this.streams.concat(new Collection(streamsGroupedById.get(this.channelId)))
+    }
+
+    return this
+  }
+
+  withGuides(guidesGroupedByStreamId: Dictionary): this {
+    this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
+
+    if (this.isMain) {
+      this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
+    }
+
+    return this
+  }
+
+  getGuides(): Collection {
+    if (!this.guides) return new Collection()
+
+    return this.guides
+  }
+
+  getStreams(): Collection {
+    if (!this.streams) return new Collection()
+
+    return this.streams
+  }
+
+  getFullName(): string {
+    if (!this.channel) return ''
+
+    return `${this.channel.name} ${this.name}`
+  }
 }
--- a/scripts/models/guide.ts
+++ b/scripts/models/guide.ts
@ -0,0 +1,35 @@
+import type { GuideData } from '../types/guide'
+import { uniqueId } from 'lodash'
+
+export class Guide {
+  channelId?: string
+  feedId?: string
+  siteDomain?: string
+  siteId?: string
+  siteName?: string
+  languageCode?: string
+
+  constructor(data?: GuideData) {
+    if (!data) return
+
+    this.channelId = data.channel
+    this.feedId = data.feed
+    this.siteDomain = data.site
+    this.siteId = data.site_id
+    this.siteName = data.site_name
+    this.languageCode = data.lang
+  }
+
+  getUUID(): string {
+    if (!this.getStreamId() || !this.siteId) return uniqueId()
+
+    return this.getStreamId() + this.siteId
+  }
+
+  getStreamId(): string | undefined {
+    if (!this.channelId) return undefined
+    if (!this.feedId) return this.channelId
+
+    return `${this.channelId}@${this.feedId}`
+  }
+}
--- a/scripts/models/index.ts
+++ b/scripts/models/index.ts
@ -2,3 +2,5 @@ export * from './issue'
 export * from './site'
 export * from './channel'
 export * from './feed'
+export * from './stream'
+export * from './guide'
--- a/scripts/models/stream.ts
+++ b/scripts/models/stream.ts
@ -0,0 +1,58 @@
+import type { StreamData } from '../types/stream'
+import { Feed, Channel } from './index'
+
+export class Stream {
+  name?: string
+  url: string
+  id?: string
+  channelId?: string
+  channel?: Channel
+  feedId?: string
+  feed?: Feed
+  filepath?: string
+  line?: number
+  label?: string
+  verticalResolution?: number
+  isInterlaced?: boolean
+  referrer?: string
+  userAgent?: string
+  groupTitle: string = 'Undefined'
+  removed: boolean = false
+
+  constructor(data: StreamData) {
+    const id = data.channel && data.feed ? [data.channel, data.feed].join('@') : data.channel
+    const { verticalResolution, isInterlaced } = parseQuality(data.quality)
+
+    this.id = id || undefined
+    this.channelId = data.channel || undefined
+    this.feedId = data.feed || undefined
+    this.name = data.name || undefined
+    this.url = data.url
+    this.referrer = data.referrer || undefined
+    this.userAgent = data.user_agent || undefined
+    this.verticalResolution = verticalResolution || undefined
+    this.isInterlaced = isInterlaced || undefined
+    this.label = data.label || undefined
+  }
+
+  getId(): string {
+    return this.id || ''
+  }
+
+  getName(): string {
+    return this.name || ''
+  }
+}
+
+function parseQuality(quality: string | null): {
+  verticalResolution: number | null
+  isInterlaced: boolean | null
+} {
+  if (!quality) return { verticalResolution: null, isInterlaced: null }
+  const [, verticalResolutionString] = quality.match(/^(\d+)/) || [null, undefined]
+  const isInterlaced = /i$/i.test(quality)
+  let verticalResolution = 0
+  if (verticalResolutionString) verticalResolution = parseInt(verticalResolutionString)
+
+  return { verticalResolution, isInterlaced }
+}
--- a/scripts/types/channel.d.ts
+++ b/scripts/types/channel.d.ts
@ -0,0 +1,28 @@
+import { Collection } from '@freearhey/core'
+
+export type ChannelData = {
+  id: string
+  name: string
+  alt_names: string[]
+  network: string
+  owners: Collection
+  country: string
+  subdivision: string
+  city: string
+  categories: Collection
+  is_nsfw: boolean
+  launched: string
+  closed: string
+  replaced_by: string
+  website: string
+  logo: string
+}
+
+export type ChannelSearchableData = {
+  id: string
+  name: string
+  altNames: string[]
+  guideNames: string[]
+  streamNames: string[]
+  feedFullNames: string[]
+}
--- a/scripts/types/dataLoader.d.ts
+++ b/scripts/types/dataLoader.d.ts
@ -0,0 +1,19 @@
+import { Storage } from '@freearhey/core'
+
+export type DataLoaderProps = {
+  storage: Storage
+}
+
+export type DataLoaderData = {
+  countries: object | object[]
+  regions: object | object[]
+  subdivisions: object | object[]
+  languages: object | object[]
+  categories: object | object[]
+  blocklist: object | object[]
+  channels: object | object[]
+  feeds: object | object[]
+  timezones: object | object[]
+  guides: object | object[]
+  streams: object | object[]
+}
--- a/scripts/types/dataProcessor.d.ts
+++ b/scripts/types/dataProcessor.d.ts
@ -0,0 +1,12 @@
+import { Collection, Dictionary } from '@freearhey/core'
+
+export type DataProcessorData = {
+  feedsGroupedByChannelId: Dictionary
+  guidesGroupedByStreamId: Dictionary
+  streamsGroupedById: Dictionary
+  channelsKeyById: Dictionary
+  channels: Collection
+  streams: Collection
+  guides: Collection
+  feeds: Collection
+}
--- a/scripts/types/feed.d.ts
+++ b/scripts/types/feed.d.ts
@ -0,0 +1,12 @@
+import { Collection } from '@freearhey/core'
+
+export type FeedData = {
+  channel: string
+  id: string
+  name: string
+  is_main: boolean
+  broadcast_area: Collection
+  languages: Collection
+  timezones: Collection
+  video_format: string
+}
--- a/scripts/types/guide.d.ts
+++ b/scripts/types/guide.d.ts
@ -0,0 +1,8 @@
+export type GuideData = {
+  channel: string
+  feed: string
+  site: string
+  site_id: string
+  site_name: string
+  lang: string
+}
--- a/scripts/types/stream.d.ts
+++ b/scripts/types/stream.d.ts
@ -0,0 +1,10 @@
+export type StreamData = {
+  channel: string | null
+  feed: string | null
+  name?: string
+  url: string
+  referrer: string | null
+  user_agent: string | null
+  quality: string | null
+  label: string | null
+}
--- a/tests/data/expected/epg_grab/guide_4.xml
+++ b/tests/data/expected/epg_grab/guide_4.xml
@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20221020">
+<channel id="Channel1.us"><display-name>Channel 1</display-name><url>https://example.com</url></channel>
+<channel id="Channel3.us"><display-name>Channel 3</display-name><icon src="https://upload.wikimedia.org/wikipedia/commons/6/64/6%27eren_2015.png"/><url>https://example.com</url></channel>
+<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel1.us"><title lang="fr">Programme1 (example.com)</title></programme>
+<programme start="20221019043000 +0000" stop="20221019071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
+<programme start="20221020043000 +0000" stop="20221020071000 +0000" channel="Channel3.us"><title lang="it">Program1 (example.com)</title></programme>
+</tv>
--- a/tests/data/input/data/blocklist.json
+++ b/tests/data/input/data/blocklist.json
@ -0,0 +1 @@
+[]
--- a/tests/data/input/data/categories.json
+++ b/tests/data/input/data/categories.json
@ -0,0 +1 @@
+[{"id":"auto","name":"Auto"},{"id":"animation","name":"Animation"},{"id":"business","name":"Business"},{"id":"classic","name":"Classic"},{"id":"comedy","name":"Comedy"},{"id":"cooking","name":"Cooking"},{"id":"culture","name":"Culture"},{"id":"documentary","name":"Documentary"},{"id":"education","name":"Education"},{"id":"entertainment","name":"Entertainment"},{"id":"family","name":"Family"},{"id":"general","name":"General"},{"id":"kids","name":"Kids"},{"id":"legislative","name":"Legislative"},{"id":"lifestyle","name":"Lifestyle"},{"id":"movies","name":"Movies"},{"id":"music","name":"Music"},{"id":"news","name":"News"},{"id":"outdoor","name":"Outdoor"},{"id":"relax","name":"Relax"},{"id":"religious","name":"Religious"},{"id":"series","name":"Series"},{"id":"science","name":"Science"},{"id":"shop","name":"Shop"},{"id":"sports","name":"Sports"},{"id":"travel","name":"Travel"},{"id":"weather","name":"Weather"},{"id":"xxx","name":"XXX"}]
--- a/tests/data/input/data/guides.json
+++ b/tests/data/input/data/guides.json
@ -0,0 +1,10 @@
+[
+  {
+    "channel": "CNNInternational.us",
+    "feed": "SD",
+    "site": "9tv.co.il",
+    "site_id": "#",
+    "site_name": "CNN (guide)",
+    "lang": "en"
+  }
+]
--- a/tests/data/input/data/languages.json
+++ b/tests/data/input/data/languages.json
--- a/tests/data/input/data/streams.json
+++ b/tests/data/input/data/streams.json
@ -0,0 +1,10 @@
+[
+  {
+    "channel": "CNNInternational.us",
+    "feed": "SD",
+    "url": "https://live.relentlessinnovations.net:1936/imantv/imantv/playlist.m3u8",
+    "referrer": null,
+    "user_agent": null,
+    "quality": "480p"
+  }
+]
--- a/tests/data/input/data/timezones.json
+++ b/tests/data/input/data/timezones.json
--- a/tests/data/input/channels_edit/example.com.channels.xml
+++ b/tests/data/input/channels_edit/example.com.channels.xml
@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels>
-  <channel site="example.com" lang="" xmltv_id="" site_id="140">CNN International</channel>
+  <channel site="example.com" lang="" xmltv_id="" site_id="140">CNN International Europe</channel>
  <channel site="example.com" lang="" xmltv_id="" site_id="141">CNN International</channel>
  <channel site="example.com" lang="" xmltv_id="" site_id="142">Bravo</channel>
 </channels>
--- a/tests/data/input/epg_grab/example.com/example.com.channels.xml
+++ b/tests/data/input/epg_grab/example.com/example.com.channels.xml
@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<channels site="example.com">
+  <channel lang="en" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
+  <channel lang="en" xmltv_id="Channel2.us" site_id="142" logo="https://i.imgur.com/qmRnD0M.png">Channel 2</channel>
+  <channel lang="fr" xmltv_id="Channel1.us" site_id="140">Channel 1</channel>
+  <channel lang="it" xmltv_id="Channel3.us" site_id="143">Channel 3</channel>
+</channels>
--- a/tests/data/input/epg_grab/example.com/example.com.config.js
+++ b/tests/data/input/epg_grab/example.com/example.com.config.js
@ -0,0 +1,28 @@
+module.exports = {
+  site: 'example.com',
+  days: 2,
+  request: {
+    timeout: 1000
+  },
+  url: 'https://example.com',
+  parser({ channel, date }) {
+    if (channel.xmltv_id === 'Channel2.us') return []
+    else if (channel.xmltv_id === 'Channel1.us' && channel.lang === 'fr') {
+      return [
+        {
+          title: 'Programme1 (example.com)',
+          start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
+          stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
+        }
+      ]
+    }
+
+    return [
+      {
+        title: 'Program1 (example.com)',
+        start: `${date.format('YYYY-MM-DD')}T04:30:00.000Z`,
+        stop: `${date.format('YYYY-MM-DD')}T07:10:00.000Z`
+      }
+    ]
+  }
+}
--- a/tests/commands/epg/grab.test.ts
+++ b/tests/commands/epg/grab.test.ts
@ -82,6 +82,16 @@ describe('epg:grab', () => {
    )
  })

+  it('can grab epg then using a multi-language filter', () => {
+    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/example.com/example.com.channels.xml --output=tests/__data__/output/guides/{site}.xml --lang=fr,it`
+    const stdout = execSync(cmd, { encoding: 'utf8' })
+    if (process.env.DEBUG === 'true') console.log(cmd, stdout)
+
+    expect(content('tests/__data__/output/guides/example.com.xml')).toEqual(
+      content('tests/__data__/expected/epg_grab/guide_4.xml')
+    )
+  })
+
  it('can grab epg using custom channels list', () => {
    const cmd = `${ENV_VAR} npm run grab --- --channels=tests/__data__/input/epg_grab/custom.channels.xml --output=tests/__data__/output/guide.xml`
    const stdout = execSync(cmd, { encoding: 'utf8' })
--- a/yarn.lock
+++ b/yarn.lock
@ -407,6 +407,13 @@
    run-script-os "^1.1.6"
    signale "^1.4.0"

+"@freearhey/search-js@^0.1.2":
+  version "0.1.2"
+  resolved "https://registry.npmjs.org/@freearhey/search-js/-/search-js-0.1.2.tgz"
+  integrity sha512-F2o+xpGCXOK4OsZfKEHfXNNkAZmny2eBnPOp+P0iyV20ja7gJGfTFaEc6okcuEo6OB6P7LnSxTvISkoArFtlfg==
+  dependencies:
+    lodash "^4.17.21"
+
 "@humanfs/core@^0.19.1":
  version "0.19.1"
  resolved "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz"
@ -2887,11 +2894,6 @@ function-bind@^1.1.1:
  resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"
  integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==

-fuse.js@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.npmjs.org/fuse.js/-/fuse.js-7.0.0.tgz"
-  integrity sha512-14F4hBIxqKvD4Zz/XjDc3y94mNZN6pRv3U13Udo0lNLCWRBUsrMv2xwcF/y/Z5sV6+FQW+/ow68cHpm4sunt8Q==
-
 gensync@^1.0.0-beta.2:
  version "1.0.0-beta.2"
  resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz"
				`@ -0,0 +1 @@`
				[{"id":"auto","name":"Auto"},{"id":"animation","name":"Animation"},{"id":"business","name":"Business"},{"id":"classic","name":"Classic"},{"id":"comedy","name":"Comedy"},{"id":"cooking","name":"Cooking"},{"id":"culture","name":"Culture"},{"id":"documentary","name":"Documentary"},{"id":"education","name":"Education"},{"id":"entertainment","name":"Entertainment"},{"id":"family","name":"Family"},{"id":"general","name":"General"},{"id":"kids","name":"Kids"},{"id":"legislative","name":"Legislative"},{"id":"lifestyle","name":"Lifestyle"},{"id":"movies","name":"Movies"},{"id":"music","name":"Music"},{"id":"news","name":"News"},{"id":"outdoor","name":"Outdoor"},{"id":"relax","name":"Relax"},{"id":"religious","name":"Religious"},{"id":"series","name":"Series"},{"id":"science","name":"Science"},{"id":"shop","name":"Shop"},{"id":"sports","name":"Sports"},{"id":"travel","name":"Travel"},{"id":"weather","name":"Weather"},{"id":"xxx","name":"XXX"}]