Update scripts

scripts/commands/api/generate.ts

@@ -1,21 +1,37 @@
-import { Logger, Storage } from '@freearhey/core'
+import { Logger, Storage, Collection } from '@freearhey/core'
-import { API_DIR, STREAMS_DIR } from '../../constants'
+import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
 import { PlaylistParser } from '../../core'
-import { Stream } from '../../models'
+import { Stream, Channel, Feed } from '../../models'
+import { uniqueId } from 'lodash'
 
 async function main() {
   const logger = new Logger()
 
+  logger.info('loading api data...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsData = await dataStorage.json('channels.json')
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
+
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({ storage: streamsStorage })
+  const parser = new PlaylistParser({
+    storage: streamsStorage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
   const files = await streamsStorage.list('**/*.m3u')
   let streams = await parser.parse(files)
   streams = streams
-    .map(data => new Stream(data))
+    .orderBy((stream: Stream) => stream.getId())
-    .orderBy([(stream: Stream) => stream.channel])
     .map((stream: Stream) => stream.toJSON())
-
   logger.info(`found ${streams.count()} streams`)
 
   logger.info('saving to .api/streams.json...')

scripts/commands/api/load.ts

@@ -12,7 +12,9 @@ async function main() {
     client.download('countries.json'),
     client.download('languages.json'),
     client.download('regions.json'),
-    client.download('subdivisions.json')
+    client.download('subdivisions.json'),
+    client.download('feeds.json'),
+    client.download('timezones.json')
   ]
 
   await Promise.all(requests)

scripts/commands/playlist/format.ts

@@ -1,25 +1,36 @@
 import { Logger, Storage, Collection } from '@freearhey/core'
 import { STREAMS_DIR, DATA_DIR } from '../../constants'
 import { PlaylistParser } from '../../core'
-import { Stream, Playlist, Channel } from '../../models'
+import { Stream, Playlist, Channel, Feed } from '../../models'
 import { program } from 'commander'
+import { uniqueId } from 'lodash'
 
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
 
 async function main() {
-  const storage = new Storage(STREAMS_DIR)
+  const streamsStorage = new Storage(STREAMS_DIR)
   const logger = new Logger()
 
-  logger.info('loading channels from api...')
+  logger.info('loading data from api...')
   const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await dataStorage.json('channels.json')
+  const channelsData = await dataStorage.json('channels.json')
-  const groupedChannels = new Collection(channelsContent)
+  const channels = new Collection(channelsData).map(data => new Channel(data))
-    .map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
-    .keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy(feed =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
 
   logger.info('loading streams...')
-  const parser = new PlaylistParser({ storage })
+  const parser = new PlaylistParser({
-  const files = program.args.length ? program.args : await storage.list('**/*.m3u')
+    storage: streamsStorage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
+  const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
   let streams = await parser.parse(files)
 
   logger.info(`found ${streams.count()} streams`)
@@ -35,8 +46,8 @@ async function main() {
 
   logger.info('removing wrong id...')
   streams = streams.map((stream: Stream) => {
-    if (groupedChannels.missing(stream.channel)) {
+    if (!stream.channel || channelsGroupedById.missing(stream.channel.id)) {
-      stream.channel = ''
+      stream.id = ''
     }
 
     return stream
@@ -46,22 +57,22 @@ async function main() {
   streams = streams.orderBy(
     [
       (stream: Stream) => stream.name,
-      (stream: Stream) => parseInt(stream.quality.replace('p', '')),
+      (stream: Stream) => stream.getHorizontalResolution(),
-      (stream: Stream) => stream.label,
+      (stream: Stream) => stream.getLabel(),
       (stream: Stream) => stream.url
     ],
     ['asc', 'desc', 'asc', 'asc']
   )
 
   logger.info('saving...')
-  const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
   for (let filepath of groupedStreams.keys()) {
     const streams = groupedStreams.get(filepath) || []
 
     if (!streams.length) return
 
     const playlist = new Playlist(streams, { public: false })
-    await storage.save(filepath, playlist.toString())
+    await streamsStorage.save(filepath, playlist.toString())
   }
 }
 

scripts/commands/playlist/generate.ts

@@ -1,14 +1,23 @@
-import { Logger, Storage, Collection, File } from '@freearhey/core'
+import { Logger, Storage, Collection } from '@freearhey/core'
 import { PlaylistParser } from '../../core'
-import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
+import {
-import _ from 'lodash'
+  Stream,
+  Category,
+  Channel,
+  Language,
+  Country,
+  Region,
+  Subdivision,
+  Feed,
+  Timezone
+} from '../../models'
+import { uniqueId } from 'lodash'
 import {
   CategoriesGenerator,
   CountriesGenerator,
   LanguagesGenerator,
   RegionsGenerator,
   IndexGenerator,
-  IndexNsfwGenerator,
   IndexCategoryGenerator,
   IndexCountryGenerator,
   IndexLanguageGenerator,
@@ -19,123 +28,134 @@ import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
 async function main() {
   const logger = new Logger()
   const dataStorage = new Storage(DATA_DIR)
-
-  logger.info('loading data from api...')
-  const channelsContent = await dataStorage.json('channels.json')
-  const channels = new Collection(channelsContent).map(data => new Channel(data))
-  const categoriesContent = await dataStorage.json('categories.json')
-  const categories = new Collection(categoriesContent).map(data => new Category(data))
-  const countriesContent = await dataStorage.json('countries.json')
-  const countries = new Collection(countriesContent).map(data => new Country(data))
-  const languagesContent = await dataStorage.json('languages.json')
-  const languages = new Collection(languagesContent).map(data => new Language(data))
-  const regionsContent = await dataStorage.json('regions.json')
-  const regions = new Collection(regionsContent).map(data => new Region(data))
-  const subdivisionsContent = await dataStorage.json('subdivisions.json')
-  const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
-
-  logger.info('loading streams...')
-  let streams = await loadStreams({ channels, categories, languages })
-  let totalStreams = streams.count()
-  streams = streams.uniqBy((stream: Stream) => (stream.channel || _.uniqueId()) + stream.timeshift)
-  logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`)
-
   const generatorsLogger = new Logger({
     stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
   })
 
+  logger.info('loading data from api...')
+  const categoriesData = await dataStorage.json('categories.json')
+  const countriesData = await dataStorage.json('countries.json')
+  const languagesData = await dataStorage.json('languages.json')
+  const regionsData = await dataStorage.json('regions.json')
+  const subdivisionsData = await dataStorage.json('subdivisions.json')
+  const timezonesData = await dataStorage.json('timezones.json')
+  const channelsData = await dataStorage.json('channels.json')
+  const feedsData = await dataStorage.json('feeds.json')
+
+  logger.info('preparing data...')
+  const subdivisions = new Collection(subdivisionsData).map(data => new Subdivision(data))
+  const subdivisionsGroupedByCode = subdivisions.keyBy(
+    (subdivision: Subdivision) => subdivision.code
+  )
+  const subdivisionsGroupedByCountryCode = subdivisions.groupBy(
+    (subdivision: Subdivision) => subdivision.countryCode
+  )
+  let regions = new Collection(regionsData).map(data =>
+    new Region(data).withSubdivisions(subdivisions)
+  )
+  const regionsGroupedByCode = regions.keyBy((region: Region) => region.code)
+  const categories = new Collection(categoriesData).map(data => new Category(data))
+  const categoriesGroupedById = categories.keyBy((category: Category) => category.id)
+  const languages = new Collection(languagesData).map(data => new Language(data))
+  const languagesGroupedByCode = languages.keyBy((language: Language) => language.code)
+  const countries = new Collection(countriesData).map(data =>
+    new Country(data)
+      .withRegions(regions)
+      .withLanguage(languagesGroupedByCode)
+      .withSubdivisions(subdivisionsGroupedByCountryCode)
+  )
+  const countriesGroupedByCode = countries.keyBy((country: Country) => country.code)
+  regions = regions.map((region: Region) => region.withCountries(countriesGroupedByCode))
+
+  const timezones = new Collection(timezonesData).map(data =>
+    new Timezone(data).withCountries(countriesGroupedByCode)
+  )
+  const timezonesGroupedById = timezones.keyBy((timezone: Timezone) => timezone.id)
+
+  const channels = new Collection(channelsData).map(data =>
+    new Channel(data)
+      .withCategories(categoriesGroupedById)
+      .withCountry(countriesGroupedByCode)
+      .withSubdivision(subdivisionsGroupedByCode)
+  )
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data)
+      .withChannel(channelsGroupedById)
+      .withLanguages(languagesGroupedByCode)
+      .withTimezones(timezonesGroupedById)
+      .withBroadcastCountries(
+        countriesGroupedByCode,
+        regionsGroupedByCode,
+        subdivisionsGroupedByCode
+      )
+      .withBroadcastRegions(regions, regionsGroupedByCode)
+      .withBroadcastSubdivisions(subdivisionsGroupedByCode)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
+
+  logger.info('loading streams...')
+  const storage = new Storage(STREAMS_DIR)
+  const parser = new PlaylistParser({
+    storage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
+  const files = await storage.list('**/*.m3u')
+  let streams = await parser.parse(files)
+  const totalStreams = streams.count()
+  streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId())
+  logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`)
+
+  logger.info('sorting streams...')
+  streams = streams.orderBy(
+    [
+      (stream: Stream) => stream.getId(),
+      (stream: Stream) => stream.getHorizontalResolution(),
+      (stream: Stream) => stream.getLabel()
+    ],
+    ['asc', 'asc', 'desc']
+  )
+
   logger.info('generating categories/...')
   await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
+
   logger.info('generating countries/...')
   await new CountriesGenerator({
     countries,
     streams,
-    regions,
-    subdivisions,
     logger: generatorsLogger
   }).generate()
+
   logger.info('generating languages/...')
   await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
+
   logger.info('generating regions/...')
   await new RegionsGenerator({
     streams,
     regions,
-    subdivisions,
     logger: generatorsLogger
   }).generate()
+
   logger.info('generating index.m3u...')
   await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
+
   logger.info('generating index.category.m3u...')
   await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
+
   logger.info('generating index.country.m3u...')
   await new IndexCountryGenerator({
     streams,
-    countries,
-    regions,
-    subdivisions,
     logger: generatorsLogger
   }).generate()
+
   logger.info('generating index.language.m3u...')
   await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
+
   logger.info('generating index.region.m3u...')
   await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
 }
 
 main()
-
-async function loadStreams({
-  channels,
-  categories,
-  languages
-}: {
-  channels: Collection
-  categories: Collection
-  languages: Collection
-}) {
-  const groupedChannels = channels.keyBy(channel => channel.id)
-  const groupedCategories = categories.keyBy(category => category.id)
-  const groupedLanguages = languages.keyBy(language => language.code)
-
-  const storage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({ storage })
-  const files = await storage.list('**/*.m3u')
-  let streams = await parser.parse(files)
-
-  streams = streams
-    .orderBy(
-      [
-        (stream: Stream) => stream.channel,
-        (stream: Stream) => parseInt(stream.quality.replace('p', '')),
-        (stream: Stream) => stream.label
-      ],
-      ['asc', 'asc', 'desc', 'asc']
-    )
-    .map((stream: Stream) => {
-      const channel: Channel | undefined = groupedChannels.get(stream.channel)
-
-      if (channel) {
-        const channelCategories = channel.categories
-          .map((id: string) => groupedCategories.get(id))
-          .filter(Boolean)
-        const channelLanguages = channel.languages
-          .map((id: string) => groupedLanguages.get(id))
-          .filter(Boolean)
-
-        stream.categories = channelCategories
-        stream.languages = channelLanguages
-        stream.broadcastArea = channel.broadcastArea
-        stream.isNSFW = channel.isNSFW
-        if (channel.logo) stream.logo = channel.logo
-      } else {
-        const file = new File(stream.filepath)
-        const [_, countryCode] = file.name().match(/^([a-z]{2})(_|$)/) || [null, null]
-        const defaultBroadcastArea = countryCode ? [`c/${countryCode.toUpperCase()}`] : []
-
-        stream.broadcastArea = new Collection(defaultBroadcastArea)
-      }
-
-      return stream
-    })
-
-  return streams
-}

scripts/commands/playlist/test.ts

@@ -1,7 +1,7 @@
 import { Logger, Storage, Collection } from '@freearhey/core'
-import { ROOT_DIR, STREAMS_DIR } from '../../constants'
+import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
 import { PlaylistParser, StreamTester, CliTable } from '../../core'
-import { Stream } from '../../models'
+import { Stream, Feed, Channel } from '../../models'
 import { program } from 'commander'
 import { eachLimit } from 'async-es'
 import commandExists from 'command-exists'
@@ -38,8 +38,6 @@ const logger = new Logger()
 const tester = new StreamTester()
 
 async function main() {
-  const storage = new Storage(ROOT_DIR)
-
   if (await isOffline()) {
     logger.error(chalk.red('Internet connection is required for the script to work'))
 
@@ -56,9 +54,25 @@ async function main() {
     return
   }
 
+  logger.info('loading channels from api...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsData = await dataStorage.json('channels.json')
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy(feed => feed.channel)
+
   logger.info('loading streams...')
-  const parser = new PlaylistParser({ storage })
+  const rootStorage = new Storage(ROOT_DIR)
-  const files = program.args.length ? program.args : await storage.list(`${STREAMS_DIR}/*.m3u`)
+  const parser = new PlaylistParser({
+    storage: rootStorage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
+  const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)
   streams = await parser.parse(files)
 
   logger.info(`found ${streams.count()} streams`)
@@ -89,7 +103,7 @@ async function main() {
 main()
 
 async function runTest(stream: Stream) {
-  const key = stream.filepath + stream.channel + stream.url
+  const key = stream.filepath + stream.getId() + stream.url
   results[key] = chalk.white('LOADING...')
 
   const result = await tester.test(stream)
@@ -125,11 +139,11 @@ function drawTable() {
       ]
     })
     streams.forEach((stream: Stream, index: number) => {
-      const status = results[stream.filepath + stream.channel + stream.url] || chalk.gray('PENDING')
+      const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING')
 
       const row = {
         '': index,
-        'tvg-id': stream.channel.length > 25 ? stream.channel.slice(0, 22) + '...' : stream.channel,
+        'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(),
         url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url,
         status
       }

scripts/commands/playlist/update.ts

@@ -1,45 +1,63 @@
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
 import { IssueLoader, PlaylistParser } from '../../core'
-import { Stream, Playlist, Channel, Issue } from '../../models'
+import { Stream, Playlist, Channel, Feed, Issue } from '../../models'
 import validUrl from 'valid-url'
+import { uniqueId } from 'lodash'
 
 let processedIssues = new Collection()
-let streams: Collection
-let groupedChannels: Dictionary
-let issues: Collection
 
 async function main() {
   const logger = new Logger({ disabled: true })
   const loader = new IssueLoader()
 
   logger.info('loading issues...')
-  issues = await loader.load()
+  const issues = await loader.load()
 
   logger.info('loading channels from api...')
   const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await dataStorage.json('channels.json')
+  const channelsData = await dataStorage.json('channels.json')
-  groupedChannels = new Collection(channelsContent)
+  const channels = new Collection(channelsData).map(data => new Channel(data))
-    .map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
-    .keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({ storage: streamsStorage })
+  const parser = new PlaylistParser({
+    storage: streamsStorage,
+    feedsGroupedByChannelId,
+    channelsGroupedById
+  })
   const files = await streamsStorage.list('**/*.m3u')
-  streams = await parser.parse(files)
+  const streams = await parser.parse(files)
 
   logger.info('removing broken streams...')
-  await removeStreams(loader)
+  await removeStreams({ streams, issues })
 
   logger.info('edit stream description...')
-  await editStreams(loader)
+  await editStreams({
+    streams,
+    issues,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
 
   logger.info('add new streams...')
-  await addStreams(loader)
+  await addStreams({
+    streams,
+    issues,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
 
   logger.info('saving...')
-  const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
   for (let filepath of groupedStreams.keys()) {
     let streams = groupedStreams.get(filepath) || []
     streams = streams.filter((stream: Stream) => stream.removed === false)
@@ -54,7 +72,7 @@ async function main() {
 
 main()
 
-async function removeStreams(loader: IssueLoader) {
+async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) {
   const requests = issues.filter(
     issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
   )
@@ -62,22 +80,35 @@ async function removeStreams(loader: IssueLoader) {
     const data = issue.data
     if (data.missing('brokenLinks')) return
 
-    const brokenLinks = data.getString('brokenLinks').split(/\r?\n/).filter(Boolean)
+    const brokenLinks = data.getString('brokenLinks') || ''
 
     let changed = false
-    brokenLinks.forEach(link => {
+    brokenLinks
-      const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
+      .split(/\r?\n/)
-      if (found) {
+      .filter(Boolean)
-        found.removed = true
+      .forEach(link => {
-        changed = true
+        const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
-      }
+        if (found) {
-    })
+          found.removed = true
+          changed = true
+        }
+      })
 
     if (changed) processedIssues.add(issue.number)
   })
 }
 
-async function editStreams(loader: IssueLoader) {
+async function editStreams({
+  streams,
+  issues,
+  channelsGroupedById,
+  feedsGroupedByChannelId
+}: {
+  streams: Collection
+  issues: Collection
+  channelsGroupedById: Dictionary
+  feedsGroupedByChannelId: Dictionary
+}) {
   const requests = issues.filter(
     issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
   )
@@ -86,59 +117,110 @@ async function editStreams(loader: IssueLoader) {
 
     if (data.missing('streamUrl')) return
 
-    let stream = streams.first(
+    let stream: Stream = streams.first(
       (_stream: Stream) => _stream.url === data.getString('streamUrl')
-    ) as Stream
+    )
-
     if (!stream) return
 
-    if (data.has('channelId')) {
+    const streamId = data.getString('streamId') || ''
-      const channel = groupedChannels.get(data.getString('channelId'))
+    const [channelId, feedId] = streamId.split('@')
 
-      if (!channel) return
+    if (channelId) {
-
+      stream
-      stream.channel = data.getString('channelId')
+        .setChannelId(channelId)
-      stream.filepath = `${channel.country.toLowerCase()}.m3u`
+        .setFeedId(feedId)
-      stream.line = -1
+        .withChannel(channelsGroupedById)
-      stream.name = channel.name
+        .withFeed(feedsGroupedByChannelId)
+        .updateId()
+        .updateName()
+        .updateFilepath()
     }
 
-    if (data.has('label')) stream.label = data.getString('label')
+    const label = data.getString('label') || ''
-    if (data.has('quality')) stream.quality = data.getString('quality')
+    const quality = data.getString('quality') || ''
-    if (data.has('httpUserAgent')) stream.httpUserAgent = data.getString('httpUserAgent')
+    const httpUserAgent = data.getString('httpUserAgent') || ''
-    if (data.has('httpReferrer')) stream.httpReferrer = data.getString('httpReferrer')
+    const httpReferrer = data.getString('httpReferrer') || ''
+
+    if (data.has('label')) stream.setLabel(label)
+    if (data.has('quality')) stream.setQuality(quality)
+    if (data.has('httpUserAgent')) stream.setHttpUserAgent(httpUserAgent)
+    if (data.has('httpReferrer')) stream.setHttpReferrer(httpReferrer)
 
     processedIssues.add(issue.number)
   })
 }
 
-async function addStreams(loader: IssueLoader) {
+async function addStreams({
+  streams,
+  issues,
+  channelsGroupedById,
+  feedsGroupedByChannelId
+}: {
+  streams: Collection
+  issues: Collection
+  channelsGroupedById: Dictionary
+  feedsGroupedByChannelId: Dictionary
+}) {
   const requests = issues.filter(
     issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
   )
   requests.forEach((issue: Issue) => {
     const data = issue.data
-    if (data.missing('channelId') || data.missing('streamUrl')) return
+    if (data.missing('streamId') || data.missing('streamUrl')) return
     if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return
-    if (!validUrl.isUri(data.getString('streamUrl'))) return
+    const stringUrl = data.getString('streamUrl') || ''
+    if (!isUri(stringUrl)) return
 
-    const channel = groupedChannels.get(data.getString('channelId'))
+    const streamId = data.getString('streamId') || ''
+    const [channelId] = streamId.split('@')
 
+    const channel: Channel = channelsGroupedById.get(channelId)
     if (!channel) return
 
+    const label = data.getString('label') || ''
+    const quality = data.getString('quality') || ''
+    const httpUserAgent = data.getString('httpUserAgent') || ''
+    const httpReferrer = data.getString('httpReferrer') || ''
+
     const stream = new Stream({
-      channel: data.getString('channelId'),
+      tvg: {
-      url: data.getString('streamUrl'),
+        id: streamId,
-      label: data.getString('label'),
+        name: '',
-      quality: data.getString('quality'),
+        url: '',
-      httpUserAgent: data.getString('httpUserAgent'),
+        logo: '',
-      httpReferrer: data.getString('httpReferrer'),
+        rec: '',
-      filepath: `${channel.country.toLowerCase()}.m3u`,
+        shift: ''
+      },
+      name: data.getString('channelName') || channel.name,
+      url: stringUrl,
+      group: {
+        title: ''
+      },
+      http: {
+        'user-agent': httpUserAgent,
+        referrer: httpReferrer
+      },
       line: -1,
-      name: data.getString('channelName') || channel.name
+      raw: '',
+      timeshift: '',
+      catchup: {
+        type: '',
+        source: '',
+        days: ''
+      }
     })
+      .withChannel(channelsGroupedById)
+      .withFeed(feedsGroupedByChannelId)
+      .setLabel(label)
+      .setQuality(quality)
+      .updateName()
+      .updateFilepath()
 
     streams.add(stream)
     processedIssues.add(issue.number)
   })
 }
+
+function isUri(string: string) {
+  return validUrl.isUri(encodeURI(string))
+}

scripts/commands/playlist/validate.ts

@@ -1,9 +1,9 @@
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
 import { PlaylistParser } from '../../core'
-import { Channel, Stream, Blocked } from '../../models'
+import { Channel, Stream, Blocked, Feed } from '../../models'
 import { program } from 'commander'
 import chalk from 'chalk'
-import _ from 'lodash'
+import { uniqueId } from 'lodash'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
 
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
@@ -17,41 +17,52 @@ type LogItem = {
 async function main() {
   const logger = new Logger()
 
-  logger.info(`loading blocklist...`)
+  logger.info('loading data from api...')
   const dataStorage = new Storage(DATA_DIR)
-  const channelsContent = await dataStorage.json('channels.json')
+  const channelsData = await dataStorage.json('channels.json')
-  const channels = new Collection(channelsContent).map(data => new Channel(data))
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
   const blocklistContent = await dataStorage.json('blocklist.json')
   const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
-
+  const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
-  logger.info(`found ${blocklist.count()} records`)
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({ storage: streamsStorage })
+  const parser = new PlaylistParser({
+    storage: streamsStorage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
   const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
   const streams = await parser.parse(files)
-
   logger.info(`found ${streams.count()} streams`)
 
   let errors = new Collection()
   let warnings = new Collection()
-  let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
+  let streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath())
-  for (const filepath of groupedStreams.keys()) {
+  for (const filepath of streamsGroupedByFilepath.keys()) {
-    const streams = groupedStreams.get(filepath)
+    const streams = streamsGroupedByFilepath.get(filepath)
     if (!streams) continue
 
     const log = new Collection()
     const buffer = new Dictionary()
     streams.forEach((stream: Stream) => {
-      const invalidId =
+      if (stream.channelId) {
-        stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
+        const channel = channelsGroupedById.get(stream.channelId)
-      if (invalidId) {
+        if (!channel) {
-        log.add({
+          log.add({
-          type: 'warning',
+            type: 'warning',
-          line: stream.line,
+            line: stream.line,
-          message: `"${stream.channel}" is not in the database`
+            message: `"${stream.id}" is not in the database`
-        })
+          })
+        }
       }
 
       const duplicate = stream.url && buffer.has(stream.url)
@@ -65,19 +76,19 @@ async function main() {
         buffer.set(stream.url, true)
       }
 
-      const blocked = blocklist.first(blocked => stream.channel === blocked.channel)
+      const blocked = stream.channel ? blocklistGroupedByChannelId.get(stream.channel.id) : false
       if (blocked) {
         if (blocked.reason === 'dmca') {
           log.add({
             type: 'error',
             line: stream.line,
-            message: `"${stream.channel}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
+            message: `"${blocked.channelId}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
           })
         } else if (blocked.reason === 'nsfw') {
           log.add({
             type: 'error',
             line: stream.line,
-            message: `"${stream.channel}" is on the blocklist due to NSFW content (${blocked.ref})`
+            message: `"${blocked.channelId}" is on the blocklist due to NSFW content (${blocked.ref})`
           })
         }
       }

scripts/commands/report/create.ts

@@ -1,154 +1,164 @@
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
 import { IssueLoader, PlaylistParser } from '../../core'
-import { Blocked, Channel, Issue, Stream } from '../../models'
+import { Blocked, Channel, Issue, Stream, Feed } from '../../models'
+import { uniqueId } from 'lodash'
 
 async function main() {
   const logger = new Logger()
   const loader = new IssueLoader()
-
+  let report = new Collection()
-  const storage = new Storage(DATA_DIR)
 
   logger.info('loading issues...')
   const issues = await loader.load()
 
+  logger.info('loading data from api...')
+  const dataStorage = new Storage(DATA_DIR)
+  const channelsData = await dataStorage.json('channels.json')
+  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const feedsData = await dataStorage.json('feeds.json')
+  const feeds = new Collection(feedsData).map(data =>
+    new Feed(data).withChannel(channelsGroupedById)
+  )
+  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
+    feed.channel ? feed.channel.id : uniqueId()
+  )
+  const blocklistContent = await dataStorage.json('blocklist.json')
+  const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
+  const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
+
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
-  const parser = new PlaylistParser({ storage: streamsStorage })
+  const parser = new PlaylistParser({
+    storage: streamsStorage,
+    channelsGroupedById,
+    feedsGroupedByChannelId
+  })
   const files = await streamsStorage.list('**/*.m3u')
   const streams = await parser.parse(files)
   const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
-  const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel)
+  const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId)
-
-  logger.info('loading channels from api...')
-  const channelsContent = await storage.json('channels.json')
-  const channelsGroupedById = new Collection(channelsContent)
-    .map(data => new Channel(data))
-    .groupBy((channel: Channel) => channel.id)
-
-  logger.info('loading blocklist from api...')
-  const blocklistContent = await storage.json('blocklist.json')
-  const blocklistGroupedByChannel = new Collection(blocklistContent)
-    .map(data => new Blocked(data))
-    .groupBy((blocked: Blocked) => blocked.channel)
-
-  let report = new Collection()
-
-  logger.info('checking streams:add requests...')
-  const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
-  const addRequestsBuffer = new Dictionary()
-  addRequests.forEach((issue: Issue) => {
-    const channelId = issue.data.getString('channelId') || undefined
-    const streamUrl = issue.data.getString('streamUrl')
-
-    const result = new Dictionary({
-      issueNumber: issue.number,
-      type: 'streams:add',
-      channelId,
-      streamUrl,
-      status: 'pending'
-    })
-
-    if (!channelId) result.set('status', 'missing_id')
-    else if (!streamUrl) result.set('status', 'missing_link')
-    else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked')
-    else if (channelsGroupedById.missing(channelId)) result.set('status', 'wrong_id')
-    else if (streamsGroupedByUrl.has(streamUrl)) result.set('status', 'on_playlist')
-    else if (addRequestsBuffer.has(streamUrl)) result.set('status', 'duplicate')
-    else result.set('status', 'pending')
-
-    addRequestsBuffer.set(streamUrl, true)
-
-    report.add(result.data())
-  })
-
-  logger.info('checking streams:edit requests...')
-  const editRequests = issues.filter(issue => issue.labels.find(label => label === 'streams:edit'))
-  editRequests.forEach((issue: Issue) => {
-    const channelId = issue.data.getString('channelId') || undefined
-    const streamUrl = issue.data.getString('streamUrl') || undefined
-
-    const result = new Dictionary({
-      issueNumber: issue.number,
-      type: 'streams:edit',
-      channelId,
-      streamUrl,
-      status: 'pending'
-    })
-
-    if (!streamUrl) result.set('status', 'missing_link')
-    else if (streamsGroupedByUrl.missing(streamUrl)) result.set('status', 'invalid_link')
-    else if (channelId && channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id')
-
-    report.add(result.data())
-  })
 
   logger.info('checking broken streams reports...')
   const brokenStreamReports = issues.filter(issue =>
-    issue.labels.find(label => label === 'broken stream')
+    issue.labels.find((label: string) => label === 'broken stream')
   )
   brokenStreamReports.forEach((issue: Issue) => {
     const brokenLinks = issue.data.getArray('brokenLinks') || []
 
     if (!brokenLinks.length) {
-      const result = new Dictionary({
+      const result = {
         issueNumber: issue.number,
         type: 'broken stream',
-        channelId: undefined,
+        streamId: undefined,
         streamUrl: undefined,
         status: 'missing_link'
-      })
+      }
 
-      report.add(result.data())
+      report.add(result)
     } else {
       for (const streamUrl of brokenLinks) {
-        const result = new Dictionary({
+        const result = {
           issueNumber: issue.number,
           type: 'broken stream',
-          channelId: undefined,
+          streamId: undefined,
-          streamUrl: undefined,
+          streamUrl: truncate(streamUrl),
           status: 'pending'
-        })
-
-        if (streamsGroupedByUrl.missing(streamUrl)) {
-          result.set('streamUrl', streamUrl)
-          result.set('status', 'wrong_link')
         }
 
-        report.add(result.data())
+        if (streamsGroupedByUrl.missing(streamUrl)) {
+          result.status = 'wrong_link'
+        }
+
+        report.add(result)
       }
     }
   })
 
+  logger.info('checking streams:add requests...')
+  const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
+  const addRequestsBuffer = new Dictionary()
+  addRequests.forEach((issue: Issue) => {
+    const streamId = issue.data.getString('streamId') || ''
+    const streamUrl = issue.data.getString('streamUrl') || ''
+    const [channelId] = streamId.split('@')
+
+    const result = {
+      issueNumber: issue.number,
+      type: 'streams:add',
+      streamId: streamId || undefined,
+      streamUrl: truncate(streamUrl),
+      status: 'pending'
+    }
+
+    if (!channelId) result.status = 'missing_id'
+    else if (!streamUrl) result.status = 'missing_link'
+    else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
+    else if (channelsGroupedById.missing(channelId)) result.status = 'wrong_id'
+    else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
+    else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
+    else result.status = 'pending'
+
+    addRequestsBuffer.set(streamUrl, true)
+
+    report.add(result)
+  })
+
+  logger.info('checking streams:edit requests...')
+  const editRequests = issues.filter(issue =>
+    issue.labels.find((label: string) => label === 'streams:edit')
+  )
+  editRequests.forEach((issue: Issue) => {
+    const streamId = issue.data.getString('streamId') || ''
+    const streamUrl = issue.data.getString('streamUrl') || ''
+    const [channelId] = streamId.split('@')
+
+    const result = {
+      issueNumber: issue.number,
+      type: 'streams:edit',
+      streamId: streamId || undefined,
+      streamUrl: truncate(streamUrl),
+      status: 'pending'
+    }
+
+    if (!streamUrl) result.status = 'missing_link'
+    else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
+    else if (channelId && channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
+
+    report.add(result)
+  })
+
   logger.info('checking channel search requests...')
   const channelSearchRequests = issues.filter(issue =>
-    issue.labels.find(label => label === 'channel search')
+    issue.labels.find((label: string) => label === 'channel search')
   )
   const channelSearchRequestsBuffer = new Dictionary()
   channelSearchRequests.forEach((issue: Issue) => {
-    const channelId = issue.data.getString('channelId')
+    const streamId = issue.data.getString('channelId') || ''
+    const [channelId] = streamId.split('@')
 
-    const result = new Dictionary({
+    const result = {
       issueNumber: issue.number,
       type: 'channel search',
-      channelId,
+      streamId: streamId || undefined,
       streamUrl: undefined,
       status: 'pending'
-    })
+    }
 
-    if (!channelId) result.set('status', 'missing_id')
+    if (!channelId) result.status = 'missing_id'
-    else if (channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id')
+    else if (channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
-    else if (channelSearchRequestsBuffer.has(channelId)) result.set('status', 'duplicate')
+    else if (channelSearchRequestsBuffer.has(channelId)) result.status = 'duplicate'
-    else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked')
+    else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
-    else if (streamsGroupedByChannel.has(channelId)) result.set('status', 'fulfilled')
+    else if (streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled'
     else {
       const channelData = channelsGroupedById.get(channelId)
-      if (channelData.length && channelData[0].closed) result.set('status', 'closed')
+      if (channelData.length && channelData[0].closed) result.status = 'closed'
     }
 
     channelSearchRequestsBuffer.set(channelId, true)
 
-    report.add(result.data())
+    report.add(result)
   })
 
   report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending')
@@ -157,3 +167,10 @@ async function main() {
 }
 
 main()
+
+function truncate(string: string, limit: number = 100) {
+  if (!string) return string
+  if (string.length < limit) return string
+
+  return string.slice(0, limit) + '...'
+}

scripts/core/apiClient.ts

@@ -41,7 +41,7 @@ export class ApiClient {
   }
 
   async download(filename: string) {
-    const stream = await this.storage.createStream(`/temp/data/${filename}`)
+    const stream = await this.storage.createStream(`temp/data/${filename}`)
 
     const bar = this.progressBar.create(0, 0, { filename })
 

scripts/core/cliTable.ts

@@ -1,9 +1,10 @@
 import { Table } from 'console-table-printer'
+import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table'
 
 export class CliTable {
   table: Table
 
-  constructor(options?) {
+  constructor(options?: ComplexOptions | string[]) {
     this.table = new Table(options)
   }
 

scripts/core/issueData.ts

@@ -18,7 +18,7 @@ export class IssueData {
     return Boolean(this._data.get(key))
   }
 
-  getString(key: string): string {
+  getString(key: string): string | undefined {
     const deleteSymbol = '~'
 
     return this._data.get(key) === deleteSymbol ? '' : this._data.get(key)

scripts/core/issueLoader.ts

@@ -16,7 +16,7 @@ export class IssueLoader {
     }
     let issues: object[] = []
     if (TESTING) {
-      issues = (await import('../../tests/__data__/input/issues/all.js')).default
+      issues = (await import('../../tests/__data__/input/playlist_update/issues.js')).default
     } else {
       issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
         owner: OWNER,

scripts/core/issueParser.ts

@@ -3,11 +3,10 @@ import { Issue } from '../models'
 import { IssueData } from './issueData'
 
 const FIELDS = new Dictionary({
+  'Stream ID': 'streamId',
   'Channel ID': 'channelId',
-  'Channel ID (required)': 'channelId',
+  'Feed ID': 'feedId',
   'Stream URL': 'streamUrl',
-  'Stream URL (optional)': 'streamUrl',
-  'Stream URL (required)': 'streamUrl',
   'Broken Link': 'brokenLinks',
   'Broken Links': 'brokenLinks',
   Label: 'label',
@@ -18,8 +17,7 @@ const FIELDS = new Dictionary({
   'HTTP Referrer': 'httpReferrer',
   'What happened to the stream?': 'reason',
   Reason: 'reason',
-  Notes: 'notes',
+  Notes: 'notes'
-  'Notes (optional)': 'notes'
 })
 
 export class IssueParser {
@@ -30,7 +28,7 @@ export class IssueParser {
     fields.forEach((field: string) => {
       const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
       let _label = parsed.shift()
-      _label = _label ? _label.trim() : ''
+      _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
       let _value = parsed.join('\r\n')
       _value = _value ? _value.trim() : ''
 

scripts/core/logParser.ts

@@ -1,4 +1,5 @@
 export type LogItem = {
+  type: string
   filepath: string
   count: number
 }

scripts/core/playlistParser.ts

@@ -1,12 +1,22 @@
-import { Collection, Storage } from '@freearhey/core'
+import { Collection, Storage, Dictionary } from '@freearhey/core'
 import parser from 'iptv-playlist-parser'
 import { Stream } from '../models'
 
+type PlaylistPareserProps = {
+  storage: Storage
+  feedsGroupedByChannelId: Dictionary
+  channelsGroupedById: Dictionary
+}
+
 export class PlaylistParser {
   storage: Storage
+  feedsGroupedByChannelId: Dictionary
+  channelsGroupedById: Dictionary
 
-  constructor({ storage }: { storage: Storage }) {
+  constructor({ storage, feedsGroupedByChannelId, channelsGroupedById }: PlaylistPareserProps) {
     this.storage = storage
+    this.feedsGroupedByChannelId = feedsGroupedByChannelId
+    this.channelsGroupedById = channelsGroupedById
   }
 
   async parse(files: string[]): Promise<Collection> {
@@ -21,41 +31,18 @@ export class PlaylistParser {
   }
 
   async parseFile(filepath: string): Promise<Collection> {
-    const streams = new Collection()
-
     const content = await this.storage.load(filepath)
     const parsed: parser.Playlist = parser.parse(content)
 
-    parsed.items.forEach((item: parser.PlaylistItem) => {
+    const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
-      const { name, label, quality } = parseTitle(item.name)
+      const stream = new Stream(data)
-      const stream = new Stream({
+        .withFeed(this.feedsGroupedByChannelId)
-        channel: item.tvg.id,
+        .withChannel(this.channelsGroupedById)
-        name,
+        .setFilepath(filepath)
-        label,
-        quality,
-        filepath,
-        line: item.line,
-        url: item.url,
-        httpReferrer: item.http.referrer,
-        httpUserAgent: item.http['user-agent']
-      })
 
-      streams.add(stream)
+      return stream
     })
 
     return streams
   }
 }
-
-function parseTitle(title: string): { name: string; label: string; quality: string } {
-  const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
-  title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
-  const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
-  title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
-
-  return { name: title, label, quality }
-}
-
-function escapeRegExp(text) {
-  return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
-}

scripts/core/streamTester.ts

@@ -11,15 +11,15 @@ export class StreamTester {
 
   async test(stream: Stream) {
     if (TESTING) {
-      const results = (await import('../../tests/__data__/input/test_results/all.js')).default
+      const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default
 
       return results[stream.url]
     } else {
       return this.checker.checkStream({
         url: stream.url,
         http: {
-          referrer: stream.httpReferrer,
+          referrer: stream.getHttpReferrer(),
-          'user-agent': stream.httpUserAgent
+          'user-agent': stream.getHttpUserAgent()
         }
       })
     }

scripts/generators/categoriesGenerator.ts

@@ -29,11 +29,7 @@ export class CategoriesGenerator implements Generator {
       const categoryStreams = streams
         .filter((stream: Stream) => stream.hasCategory(category))
         .map((stream: Stream) => {
-          const streamCategories = stream.categories
+          stream.groupTitle = stream.getCategoryNames().join(';')
-            .map((category: Category) => category.name)
-            .sort()
-          const groupTitle = stream.categories ? streamCategories.join(';') : ''
-          stream.groupTitle = groupTitle
 
           return stream
         })
@@ -41,13 +37,17 @@ export class CategoriesGenerator implements Generator {
       const playlist = new Playlist(categoryStreams, { public: true })
       const filepath = `categories/${category.id}.m3u`
       await this.storage.save(filepath, playlist.toString())
-      this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+      this.logger.info(
+        JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() })
+      )
     })
 
-    const undefinedStreams = streams.filter((stream: Stream) => stream.noCategories())
+    const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories())
     const playlist = new Playlist(undefinedStreams, { public: true })
     const filepath = 'categories/undefined.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(
+      JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() })
+    )
   }
 }

scripts/generators/countriesGenerator.ts

@@ -1,12 +1,10 @@
 import { Generator } from './generator'
 import { Collection, Storage, Logger } from '@freearhey/core'
-import { Country, Region, Subdivision, Stream, Playlist } from '../models'
+import { Country, Subdivision, Stream, Playlist } from '../models'
 import { PUBLIC_DIR } from '../constants'
 
 type CountriesGeneratorProps = {
   streams: Collection
-  regions: Collection
-  subdivisions: Collection
   countries: Collection
   logger: Logger
 }
@@ -14,55 +12,37 @@ type CountriesGeneratorProps = {
 export class CountriesGenerator implements Generator {
   streams: Collection
   countries: Collection
-  regions: Collection
-  subdivisions: Collection
   storage: Storage
   logger: Logger
 
-  constructor({ streams, countries, regions, subdivisions, logger }: CountriesGeneratorProps) {
+  constructor({ streams, countries, logger }: CountriesGeneratorProps) {
     this.streams = streams
     this.countries = countries
-    this.regions = regions
-    this.subdivisions = subdivisions
     this.storage = new Storage(PUBLIC_DIR)
     this.logger = logger
   }
 
   async generate(): Promise<void> {
     const streams = this.streams
-      .orderBy([stream => stream.getTitle()])
+      .orderBy((stream: Stream) => stream.getTitle())
       .filter((stream: Stream) => stream.isSFW())
-    const regions = this.regions.filter((region: Region) => region.code !== 'INT')
 
     this.countries.forEach(async (country: Country) => {
-      const countrySubdivisions = this.subdivisions.filter(
+      const countryStreams = streams.filter((stream: Stream) =>
-        (subdivision: Subdivision) => subdivision.country === country.code
+        stream.isBroadcastInCountry(country)
       )
-
-      const countrySubdivisionsCodes = countrySubdivisions.map(
-        (subdivision: Subdivision) => `s/${subdivision.code}`
-      )
-
-      const countryAreaCodes = regions
-        .filter((region: Region) => region.countries.includes(country.code))
-        .map((region: Region) => `r/${region.code}`)
-        .concat(countrySubdivisionsCodes)
-        .add(`c/${country.code}`)
-
-      const countryStreams = streams.filter(stream =>
-        stream.broadcastArea.intersects(countryAreaCodes)
-      )
-
       if (countryStreams.isEmpty()) return
 
       const playlist = new Playlist(countryStreams, { public: true })
       const filepath = `countries/${country.code.toLowerCase()}.m3u`
       await this.storage.save(filepath, playlist.toString())
-      this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+      this.logger.info(
+        JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() })
+      )
 
-      countrySubdivisions.forEach(async (subdivision: Subdivision) => {
+      country.getSubdivisions().forEach(async (subdivision: Subdivision) => {
-        const subdivisionStreams = streams.filter(stream =>
+        const subdivisionStreams = streams.filter((stream: Stream) =>
-          stream.broadcastArea.includes(`s/${subdivision.code}`)
+          stream.isBroadcastInSubdivision(subdivision)
         )
 
         if (subdivisionStreams.isEmpty()) return
@@ -70,16 +50,22 @@ export class CountriesGenerator implements Generator {
         const playlist = new Playlist(subdivisionStreams, { public: true })
         const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u`
         await this.storage.save(filepath, playlist.toString())
-        this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+        this.logger.info(
+          JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() })
+        )
       })
     })
 
-    const internationalStreams = streams.filter(stream => stream.isInternational())
+    const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
-    if (internationalStreams.notEmpty()) {
+    const undefinedPlaylist = new Playlist(undefinedStreams, { public: true })
-      const playlist = new Playlist(internationalStreams, { public: true })
+    const undefinedFilepath = 'countries/undefined.m3u'
-      const filepath = 'countries/int.m3u'
+    await this.storage.save(undefinedFilepath, undefinedPlaylist.toString())
-      await this.storage.save(filepath, playlist.toString())
+    this.logger.info(
-      this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+      JSON.stringify({
-    }
+        type: 'country',
+        filepath: undefinedFilepath,
+        count: undefinedPlaylist.streams.count()
+      })
+    )
   }
 }

scripts/generators/indexCategoryGenerator.ts

@@ -26,14 +26,14 @@ export class IndexCategoryGenerator implements Generator {
 
     let groupedStreams = new Collection()
     streams.forEach((stream: Stream) => {
-      if (stream.noCategories()) {
+      if (!stream.hasCategories()) {
         const streamClone = stream.clone()
         streamClone.groupTitle = 'Undefined'
         groupedStreams.add(streamClone)
         return
       }
 
-      stream.categories.forEach((category: Category) => {
+      stream.getCategories().forEach((category: Category) => {
         const streamClone = stream.clone()
         streamClone.groupTitle = category.name
         groupedStreams.push(streamClone)
@@ -48,6 +48,6 @@ export class IndexCategoryGenerator implements Generator {
     const playlist = new Playlist(groupedStreams, { public: true })
     const filepath = 'index.category.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
   }
 }

scripts/generators/indexCountryGenerator.ts

@@ -1,29 +1,20 @@
 import { Generator } from './generator'
 import { Collection, Storage, Logger } from '@freearhey/core'
-import { Stream, Playlist, Country, Subdivision, Region } from '../models'
+import { Stream, Playlist, Country } from '../models'
 import { PUBLIC_DIR } from '../constants'
 
 type IndexCountryGeneratorProps = {
   streams: Collection
-  regions: Collection
-  countries: Collection
-  subdivisions: Collection
   logger: Logger
 }
 
 export class IndexCountryGenerator implements Generator {
   streams: Collection
-  countries: Collection
-  regions: Collection
-  subdivisions: Collection
   storage: Storage
   logger: Logger
 
-  constructor({ streams, regions, countries, subdivisions, logger }: IndexCountryGeneratorProps) {
+  constructor({ streams, logger }: IndexCountryGeneratorProps) {
     this.streams = streams
-    this.countries = countries
-    this.regions = regions
-    this.subdivisions = subdivisions
     this.storage = new Storage(PUBLIC_DIR)
     this.logger = logger
   }
@@ -32,10 +23,10 @@ export class IndexCountryGenerator implements Generator {
     let groupedStreams = new Collection()
 
     this.streams
-      .orderBy(stream => stream.getTitle())
+      .orderBy((stream: Stream) => stream.getTitle())
-      .filter(stream => stream.isSFW())
+      .filter((stream: Stream) => stream.isSFW())
-      .forEach(stream => {
+      .forEach((stream: Stream) => {
-        if (stream.noBroadcastArea()) {
+        if (!stream.hasBroadcastArea()) {
           const streamClone = stream.clone()
           streamClone.groupTitle = 'Undefined'
           groupedStreams.add(streamClone)
@@ -48,7 +39,7 @@ export class IndexCountryGenerator implements Generator {
           groupedStreams.add(streamClone)
         }
 
-        this.getStreamBroadcastCountries(stream).forEach((country: Country) => {
+        stream.getBroadcastCountries().forEach((country: Country) => {
           const streamClone = stream.clone()
           streamClone.groupTitle = country.name
           groupedStreams.add(streamClone)
@@ -65,40 +56,6 @@ export class IndexCountryGenerator implements Generator {
     const playlist = new Playlist(groupedStreams, { public: true })
     const filepath = 'index.country.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
-  }
-
-  getStreamBroadcastCountries(stream: Stream) {
-    const groupedRegions = this.regions.keyBy((region: Region) => region.code)
-    const groupedCountries = this.countries.keyBy((country: Country) => country.code)
-    const groupedSubdivisions = this.subdivisions.keyBy(
-      (subdivision: Subdivision) => subdivision.code
-    )
-
-    let broadcastCountries = new Collection()
-
-    stream.broadcastArea.forEach(broadcastAreaCode => {
-      const [type, code] = broadcastAreaCode.split('/')
-      switch (type) {
-        case 'c':
-          broadcastCountries.add(code)
-          break
-        case 'r':
-          if (code !== 'INT' && groupedRegions.has(code)) {
-            broadcastCountries = broadcastCountries.concat(groupedRegions.get(code).countries)
-          }
-          break
-        case 's':
-          if (groupedSubdivisions.has(code)) {
-            broadcastCountries.add(groupedSubdivisions.get(code).country)
-          }
-          break
-      }
-    })
-
-    return broadcastCountries
-      .uniq()
-      .map(code => groupedCountries.get(code))
-      .filter(Boolean)
   }
 }

scripts/generators/indexGenerator.ts

@@ -27,6 +27,6 @@ export class IndexGenerator implements Generator {
     const playlist = new Playlist(sfwStreams, { public: true })
     const filepath = 'index.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
   }
 }

scripts/generators/indexLanguageGenerator.ts

@@ -22,17 +22,17 @@ export class IndexLanguageGenerator implements Generator {
   async generate(): Promise<void> {
     let groupedStreams = new Collection()
     this.streams
-      .orderBy(stream => stream.getTitle())
+      .orderBy((stream: Stream) => stream.getTitle())
-      .filter(stream => stream.isSFW())
+      .filter((stream: Stream) => stream.isSFW())
-      .forEach(stream => {
+      .forEach((stream: Stream) => {
-        if (stream.noLanguages()) {
+        if (!stream.hasLanguages()) {
           const streamClone = stream.clone()
           streamClone.groupTitle = 'Undefined'
           groupedStreams.add(streamClone)
           return
         }
 
-        stream.languages.forEach((language: Language) => {
+        stream.getLanguages().forEach((language: Language) => {
           const streamClone = stream.clone()
           streamClone.groupTitle = language.name
           groupedStreams.add(streamClone)
@@ -47,6 +47,6 @@ export class IndexLanguageGenerator implements Generator {
     const playlist = new Playlist(groupedStreams, { public: true })
     const filepath = 'index.language.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
   }
 }

scripts/generators/indexNsfwGenerator.ts

@@ -25,6 +25,6 @@ export class IndexNsfwGenerator implements Generator {
     const playlist = new Playlist(allStreams, { public: true })
     const filepath = 'index.nsfw.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
   }
 }

scripts/generators/indexRegionGenerator.ts

@@ -28,14 +28,14 @@ export class IndexRegionGenerator implements Generator {
       .orderBy((stream: Stream) => stream.getTitle())
       .filter((stream: Stream) => stream.isSFW())
       .forEach((stream: Stream) => {
-        if (stream.noBroadcastArea()) {
+        if (!stream.hasBroadcastArea()) {
           const streamClone = stream.clone()
           streamClone.groupTitle = 'Undefined'
           groupedStreams.push(streamClone)
           return
         }
 
-        this.getStreamRegions(stream).forEach((region: Region) => {
+        stream.getBroadcastRegions().forEach((region: Region) => {
           const streamClone = stream.clone()
           streamClone.groupTitle = region.name
           groupedStreams.push(streamClone)
@@ -50,34 +50,6 @@ export class IndexRegionGenerator implements Generator {
     const playlist = new Playlist(groupedStreams, { public: true })
     const filepath = 'index.region.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
-  }
-
-  getStreamRegions(stream: Stream) {
-    let streamRegions = new Collection()
-    stream.broadcastArea.forEach(broadcastAreaCode => {
-      const [type, code] = broadcastAreaCode.split('/')
-      switch (type) {
-        case 'r':
-          const groupedRegions = this.regions.keyBy((region: Region) => region.code)
-          streamRegions.add(groupedRegions.get(code))
-          break
-        case 's':
-          const [countryCode] = code.split('-')
-          const subdivisionRegions = this.regions.filter((region: Region) =>
-            region.countries.includes(countryCode)
-          )
-          streamRegions = streamRegions.concat(subdivisionRegions)
-          break
-        case 'c':
-          const countryRegions = this.regions.filter((region: Region) =>
-            region.countries.includes(code)
-          )
-          streamRegions = streamRegions.concat(countryRegions)
-          break
-      }
-    })
-
-    return streamRegions
   }
 }

scripts/generators/languagesGenerator.ts

@@ -18,35 +18,40 @@ export class LanguagesGenerator implements Generator {
 
   async generate(): Promise<void> {
     const streams = this.streams
-      .orderBy(stream => stream.getTitle())
+      .orderBy((stream: Stream) => stream.getTitle())
-      .filter(stream => stream.isSFW())
+      .filter((stream: Stream) => stream.isSFW())
 
     let languages = new Collection()
     streams.forEach((stream: Stream) => {
-      languages = languages.concat(stream.languages)
+      languages = languages.concat(stream.getLanguages())
     })
 
     languages
+      .filter(Boolean)
       .uniqBy((language: Language) => language.code)
       .orderBy((language: Language) => language.name)
       .forEach(async (language: Language) => {
-        const languageStreams = streams.filter(stream => stream.hasLanguage(language))
+        const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language))
 
         if (languageStreams.isEmpty()) return
 
         const playlist = new Playlist(languageStreams, { public: true })
         const filepath = `languages/${language.code}.m3u`
         await this.storage.save(filepath, playlist.toString())
-        this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+        this.logger.info(
+          JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() })
+        )
       })
 
-    const undefinedStreams = streams.filter(stream => stream.noLanguages())
+    const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages())
 
     if (undefinedStreams.isEmpty()) return
 
     const playlist = new Playlist(undefinedStreams, { public: true })
     const filepath = 'languages/undefined.m3u'
     await this.storage.save(filepath, playlist.toString())
-    this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+    this.logger.info(
+      JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() })
+    )
   }
 }

scripts/generators/regionsGenerator.ts

@@ -1,53 +1,61 @@
 import { Generator } from './generator'
 import { Collection, Storage, Logger } from '@freearhey/core'
-import { Playlist, Subdivision, Region } from '../models'
+import { Playlist, Region, Stream } from '../models'
 import { PUBLIC_DIR } from '../constants'
 
 type RegionsGeneratorProps = {
   streams: Collection
   regions: Collection
-  subdivisions: Collection
   logger: Logger
 }
 
 export class RegionsGenerator implements Generator {
   streams: Collection
   regions: Collection
-  subdivisions: Collection
   storage: Storage
   logger: Logger
 
-  constructor({ streams, regions, subdivisions, logger }: RegionsGeneratorProps) {
+  constructor({ streams, regions, logger }: RegionsGeneratorProps) {
     this.streams = streams
     this.regions = regions
-    this.subdivisions = subdivisions
     this.storage = new Storage(PUBLIC_DIR)
     this.logger = logger
   }
 
   async generate(): Promise<void> {
     const streams = this.streams
-      .orderBy(stream => stream.getTitle())
+      .orderBy((stream: Stream) => stream.getTitle())
-      .filter(stream => stream.isSFW())
+      .filter((stream: Stream) => stream.isSFW())
 
     this.regions.forEach(async (region: Region) => {
-      if (region.code === 'INT') return
+      if (region.isWorldwide()) return
 
-      const regionSubdivisionsCodes = this.subdivisions
+      const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region))
-        .filter((subdivision: Subdivision) => region.countries.indexOf(subdivision.country) > -1)
-        .map((subdivision: Subdivision) => `s/${subdivision.code}`)
-
-      const regionCodes = region.countries
-        .map((code: string) => `c/${code}`)
-        .concat(regionSubdivisionsCodes)
-        .add(`r/${region.code}`)
-
-      const regionStreams = streams.filter(stream => stream.broadcastArea.intersects(regionCodes))
 
       const playlist = new Playlist(regionStreams, { public: true })
       const filepath = `regions/${region.code.toLowerCase()}.m3u`
       await this.storage.save(filepath, playlist.toString())
-      this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
+      this.logger.info(
+        JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() })
+      )
     })
+
+    const internationalStreams = streams.filter((stream: Stream) => stream.isInternational())
+    const internationalPlaylist = new Playlist(internationalStreams, { public: true })
+    const internationalFilepath = 'regions/int.m3u'
+    await this.storage.save(internationalFilepath, internationalPlaylist.toString())
+    this.logger.info(
+      JSON.stringify({
+        type: 'region',
+        filepath: internationalFilepath,
+        count: internationalPlaylist.streams.count()
+      })
+    )
+
+    const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
+    const playlist = new Playlist(undefinedStreams, { public: true })
+    const filepath = 'regions/undefined.m3u'
+    await this.storage.save(filepath, playlist.toString())
+    this.logger.info(JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }))
   }
 }

scripts/models/blocked.ts

@@ -5,13 +5,13 @@ type BlockedProps = {
 }
 
 export class Blocked {
-  channel: string
+  channelId: string
   reason: string
   ref: string
 
-  constructor({ ref, reason, channel }: BlockedProps) {
+  constructor(data: BlockedProps) {
-    this.channel = channel
+    this.channelId = data.channel
-    this.reason = reason
+    this.reason = data.reason
-    this.ref = ref
+    this.ref = data.ref
   }
 }

scripts/models/broadcastArea.ts

@@ -0,0 +1,11 @@
+type BroadcastAreaProps = {
+  code: string
+}
+
+export class BroadcastArea {
+  code: string
+
+  constructor(data: BroadcastAreaProps) {
+    this.code = data.code
+  }
+}

scripts/models/category.ts

@@ -1,4 +1,4 @@
-type CategoryProps = {
+type CategoryData = {
   id: string
   name: string
 }
@@ -7,8 +7,8 @@ export class Category {
   id: string
   name: string
 
-  constructor({ id, name }: CategoryProps) {
+  constructor(data: CategoryData) {
-    this.id = id
+    this.id = data.id
-    this.name = name
+    this.name = data.name
   }
 }

scripts/models/channel.ts

@@ -1,17 +1,16 @@
-import { Collection } from '@freearhey/core'
+import { Collection, Dictionary } from '@freearhey/core'
+import { Category, Country, Subdivision } from './index'
 
-type ChannelProps = {
+type ChannelData = {
   id: string
   name: string
   alt_names: string[]
   network: string
-  owners: string[]
+  owners: Collection
   country: string
   subdivision: string
   city: string
-  broadcast_area: string[]
+  categories: Collection
-  languages: string[]
-  categories: string[]
   is_nsfw: boolean
   launched: string
   closed: string
@@ -24,56 +23,86 @@ export class Channel {
   id: string
   name: string
   altNames: Collection
-  network: string
+  network?: string
   owners: Collection
-  country: string
+  countryCode: string
-  subdivision: string
+  country?: Country
-  city: string
+  subdivisionCode?: string
-  broadcastArea: Collection
+  subdivision?: Subdivision
-  languages: Collection
+  cityName?: string
-  categories: Collection
+  categoryIds: Collection
+  categories?: Collection
   isNSFW: boolean
-  launched: string
+  launched?: string
-  closed: string
+  closed?: string
-  replacedBy: string
+  replacedBy?: string
-  website: string
+  website?: string
   logo: string
 
-  constructor({
+  constructor(data: ChannelData) {
-    id,
+    this.id = data.id
-    name,
+    this.name = data.name
-    alt_names,
+    this.altNames = new Collection(data.alt_names)
-    network,
+    this.network = data.network || undefined
-    owners,
+    this.owners = new Collection(data.owners)
-    country,
+    this.countryCode = data.country
-    subdivision,
+    this.subdivisionCode = data.subdivision || undefined
-    city,
+    this.cityName = data.city || undefined
-    broadcast_area,
+    this.categoryIds = new Collection(data.categories)
-    languages,
+    this.isNSFW = data.is_nsfw
-    categories,
+    this.launched = data.launched || undefined
-    is_nsfw,
+    this.closed = data.closed || undefined
-    launched,
+    this.replacedBy = data.replaced_by || undefined
-    closed,
+    this.website = data.website || undefined
-    replaced_by,
+    this.logo = data.logo
-    website,
+  }
-    logo
+
-  }: ChannelProps) {
+  withSubdivision(subdivisionsGroupedByCode: Dictionary): this {
-    this.id = id
+    if (!this.subdivisionCode) return this
-    this.name = name
+
-    this.altNames = new Collection(alt_names)
+    this.subdivision = subdivisionsGroupedByCode.get(this.subdivisionCode)
-    this.network = network
+
-    this.owners = new Collection(owners)
+    return this
-    this.country = country
+  }
-    this.subdivision = subdivision
+
-    this.city = city
+  withCountry(countriesGroupedByCode: Dictionary): this {
-    this.broadcastArea = new Collection(broadcast_area)
+    this.country = countriesGroupedByCode.get(this.countryCode)
-    this.languages = new Collection(languages)
+
-    this.categories = new Collection(categories)
+    return this
-    this.isNSFW = is_nsfw
+  }
-    this.launched = launched
+
-    this.closed = closed
+  withCategories(groupedCategories: Dictionary): this {
-    this.replacedBy = replaced_by
+    this.categories = this.categoryIds
-    this.website = website
+      .map((id: string) => groupedCategories.get(id))
-    this.logo = logo
+      .filter(Boolean)
+
+    return this
+  }
+
+  getCountry(): Country | undefined {
+    return this.country
+  }
+
+  getSubdivision(): Subdivision | undefined {
+    return this.subdivision
+  }
+
+  getCategories(): Collection {
+    return this.categories || new Collection()
+  }
+
+  hasCategories(): boolean {
+    return !!this.categories && this.categories.notEmpty()
+  }
+
+  hasCategory(category: Category): boolean {
+    return (
+      !!this.categories &&
+      this.categories.includes((_category: Category) => _category.id === category.id)
+    )
+  }
+
+  isSFW(): boolean {
+    return this.isNSFW === false
   }
 }

scripts/models/country.ts

@@ -1,20 +1,58 @@
-type CountryProps = {
+import { Collection, Dictionary } from '@freearhey/core'
+import { Region, Language } from '.'
+
+type CountryData = {
   code: string
   name: string
-  languages: string[]
+  lang: string
   flag: string
 }
 
 export class Country {
   code: string
   name: string
-  languages: string[]
   flag: string
+  languageCode: string
+  language?: Language
+  subdivisions?: Collection
+  regions?: Collection
 
-  constructor({ code, name, languages, flag }: CountryProps) {
+  constructor(data: CountryData) {
-    this.code = code
+    this.code = data.code
-    this.name = name
+    this.name = data.name
-    this.languages = languages
+    this.flag = data.flag
-    this.flag = flag
+    this.languageCode = data.lang
+  }
+
+  withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this {
+    this.subdivisions = subdivisionsGroupedByCountryCode.get(this.code) || new Collection()
+
+    return this
+  }
+
+  withRegions(regions: Collection): this {
+    this.regions = regions.filter(
+      (region: Region) => region.code !== 'INT' && region.includesCountryCode(this.code)
+    )
+
+    return this
+  }
+
+  withLanguage(languagesGroupedByCode: Dictionary): this {
+    this.language = languagesGroupedByCode.get(this.languageCode)
+
+    return this
+  }
+
+  getLanguage(): Language | undefined {
+    return this.language
+  }
+
+  getRegions(): Collection {
+    return this.regions || new Collection()
+  }
+
+  getSubdivisions(): Collection {
+    return this.subdivisions || new Collection()
   }
 }

scripts/models/feed.ts

@@ -0,0 +1,196 @@
+import { Collection, Dictionary } from '@freearhey/core'
+import { Country, Language, Region, Channel, Subdivision } from './index'
+
+type FeedData = {
+  channel: string
+  id: string
+  name: string
+  is_main: boolean
+  broadcast_area: Collection
+  languages: Collection
+  timezones: Collection
+  video_format: string
+}
+
+export class Feed {
+  channelId: string
+  channel?: Channel
+  id: string
+  name: string
+  isMain: boolean
+  broadcastAreaCodes: Collection
+  broadcastCountryCodes: Collection
+  broadcastCountries?: Collection
+  broadcastRegionCodes: Collection
+  broadcastRegions?: Collection
+  broadcastSubdivisionCodes: Collection
+  broadcastSubdivisions?: Collection
+  languageCodes: Collection
+  languages?: Collection
+  timezoneIds: Collection
+  timezones?: Collection
+  videoFormat: string
+
+  constructor(data: FeedData) {
+    this.channelId = data.channel
+    this.id = data.id
+    this.name = data.name
+    this.isMain = data.is_main
+    this.broadcastAreaCodes = new Collection(data.broadcast_area)
+    this.languageCodes = new Collection(data.languages)
+    this.timezoneIds = new Collection(data.timezones)
+    this.videoFormat = data.video_format
+    this.broadcastCountryCodes = new Collection()
+    this.broadcastRegionCodes = new Collection()
+    this.broadcastSubdivisionCodes = new Collection()
+
+    this.broadcastAreaCodes.forEach((areaCode: string) => {
+      const [type, code] = areaCode.split('/')
+
+      switch (type) {
+        case 'c':
+          this.broadcastCountryCodes.add(code)
+          break
+        case 'r':
+          this.broadcastRegionCodes.add(code)
+          break
+        case 's':
+          this.broadcastSubdivisionCodes.add(code)
+          break
+      }
+    })
+  }
+
+  withChannel(channelsGroupedById: Dictionary): this {
+    this.channel = channelsGroupedById.get(this.channelId)
+
+    return this
+  }
+
+  withLanguages(languagesGroupedByCode: Dictionary): this {
+    this.languages = this.languageCodes
+      .map((code: string) => languagesGroupedByCode.get(code))
+      .filter(Boolean)
+
+    return this
+  }
+
+  withTimezones(timezonesGroupedById: Dictionary): this {
+    this.timezones = this.timezoneIds
+      .map((id: string) => timezonesGroupedById.get(id))
+      .filter(Boolean)
+
+    return this
+  }
+
+  withBroadcastSubdivisions(subdivisionsGroupedByCode: Dictionary): this {
+    this.broadcastSubdivisions = this.broadcastSubdivisionCodes.map((code: string) =>
+      subdivisionsGroupedByCode.get(code)
+    )
+
+    return this
+  }
+
+  withBroadcastCountries(
+    countriesGroupedByCode: Dictionary,
+    regionsGroupedByCode: Dictionary,
+    subdivisionsGroupedByCode: Dictionary
+  ): this {
+    let broadcastCountries = new Collection()
+
+    if (this.isInternational()) {
+      this.broadcastCountries = broadcastCountries
+      return this
+    }
+
+    this.broadcastCountryCodes.forEach((code: string) => {
+      broadcastCountries.add(countriesGroupedByCode.get(code))
+    })
+
+    this.broadcastRegionCodes.forEach((code: string) => {
+      const region: Region = regionsGroupedByCode.get(code)
+      broadcastCountries = broadcastCountries.concat(region.countryCodes)
+    })
+
+    this.broadcastSubdivisionCodes.forEach((code: string) => {
+      const subdivision: Subdivision = subdivisionsGroupedByCode.get(code)
+      broadcastCountries.add(countriesGroupedByCode.get(subdivision.countryCode))
+    })
+
+    this.broadcastCountries = broadcastCountries.uniq().filter(Boolean)
+
+    return this
+  }
+
+  withBroadcastRegions(regions: Collection, regionsGroupedByCode: Dictionary): this {
+    if (!this.broadcastCountries) return this
+    const countriesCodes = this.broadcastCountries.map((country: Country) => country.code)
+
+    const broadcastRegions = regions.filter((region: Region) =>
+      region.countryCodes.intersects(countriesCodes)
+    )
+
+    if (this.isInternational()) broadcastRegions.add(regionsGroupedByCode.get('INT'))
+
+    this.broadcastRegions = broadcastRegions
+
+    return this
+  }
+
+  hasBroadcastArea(): boolean {
+    return (
+      this.isInternational() || (!!this.broadcastCountries && this.broadcastCountries.notEmpty())
+    )
+  }
+
+  getBroadcastCountries(): Collection {
+    return this.broadcastCountries || new Collection()
+  }
+
+  getBroadcastRegions(): Collection {
+    return this.broadcastRegions || new Collection()
+  }
+
+  getTimezones(): Collection {
+    return this.timezones || new Collection()
+  }
+
+  getLanguages(): Collection {
+    return this.languages || new Collection()
+  }
+
+  hasLanguages(): boolean {
+    return !!this.languages && this.languages.notEmpty()
+  }
+
+  hasLanguage(language: Language): boolean {
+    return (
+      !!this.languages &&
+      this.languages.includes((_language: Language) => _language.code === language.code)
+    )
+  }
+
+  isInternational(): boolean {
+    return this.broadcastAreaCodes.includes('r/INT')
+  }
+
+  isBroadcastInSubdivision(subdivision: Subdivision): boolean {
+    if (this.isInternational()) return false
+
+    return this.broadcastSubdivisionCodes.includes(subdivision.code)
+  }
+
+  isBroadcastInCountry(country: Country): boolean {
+    if (this.isInternational()) return false
+
+    return this.getBroadcastCountries().includes(
+      (_country: Country) => _country.code === country.code
+    )
+  }
+
+  isBroadcastInRegion(region: Region): boolean {
+    if (this.isInternational()) return false
+
+    return this.getBroadcastRegions().includes((_region: Region) => _region.code === region.code)
+  }
+}

scripts/models/index.ts

@@ -8,3 +8,6 @@ export * from './language'
 export * from './country'
 export * from './region'
 export * from './subdivision'
+export * from './feed'
+export * from './broadcastArea'
+export * from './timezone'

scripts/models/language.ts

@@ -1,4 +1,4 @@
-type LanguageProps = {
+type LanguageData = {
   code: string
   name: string
 }
@@ -7,8 +7,8 @@ export class Language {
   code: string
   name: string
 
-  constructor({ code, name }: LanguageProps) {
+  constructor(data: LanguageData) {
-    this.code = code
+    this.code = data.code
-    this.name = name
+    this.name = data.name
   }
 }

scripts/models/region.ts

@@ -1,6 +1,7 @@
-import { Collection } from '@freearhey/core'
+import { Collection, Dictionary } from '@freearhey/core'
+import { Subdivision } from '.'
 
-type RegionProps = {
+type RegionData = {
   code: string
   name: string
   countries: string[]
@@ -9,11 +10,43 @@ type RegionProps = {
 export class Region {
   code: string
   name: string
-  countries: Collection
+  countryCodes: Collection
+  countries?: Collection
+  subdivisions?: Collection
 
-  constructor({ code, name, countries }: RegionProps) {
+  constructor(data: RegionData) {
-    this.code = code
+    this.code = data.code
-    this.name = name
+    this.name = data.name
-    this.countries = new Collection(countries)
+    this.countryCodes = new Collection(data.countries)
+  }
+
+  withCountries(countriesGroupedByCode: Dictionary): this {
+    this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
+
+    return this
+  }
+
+  withSubdivisions(subdivisions: Collection): this {
+    this.subdivisions = subdivisions.filter(
+      (subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1
+    )
+
+    return this
+  }
+
+  getSubdivisions(): Collection {
+    return this.subdivisions || new Collection()
+  }
+
+  getCountries(): Collection {
+    return this.countries || new Collection()
+  }
+
+  includesCountryCode(code: string): boolean {
+    return this.countryCodes.includes((countryCode: string) => countryCode === code)
+  }
+
+  isWorldwide(): boolean {
+    return this.code === 'INT'
   }
 }

scripts/models/stream.ts

@@ -1,64 +1,166 @@
-import { URL, Collection } from '@freearhey/core'
+import { URL, Collection, Dictionary } from '@freearhey/core'
-import { Category, Language } from './index'
+import { Feed, Channel, Category, Region, Subdivision, Country, Language } from './index'
-
+import parser from 'iptv-playlist-parser'
-type StreamProps = {
-  name: string
-  url: string
-  filepath: string
-  line: number
-  channel?: string
-  httpReferrer?: string
-  httpUserAgent?: string
-  label?: string
-  quality?: string
-}
 
 export class Stream {
-  channel: string
-  filepath: string
-  line: number
-  httpReferrer: string
-  label: string
   name: string
-  quality: string
   url: string
-  httpUserAgent: string
+  id?: string
-  logo: string
-  broadcastArea: Collection
-  categories: Collection
-  languages: Collection
-  isNSFW: boolean
   groupTitle: string
+  channelId?: string
+  channel?: Channel
+  feedId?: string
+  feed?: Feed
+  filepath?: string
+  line: number
+  label?: string
+  quality?: string
+  httpReferrer?: string
+  httpUserAgent?: string
   removed: boolean = false
 
-  constructor({
+  constructor(data: parser.PlaylistItem) {
-    channel,
+    if (!data.name) throw new Error('"name" property is required')
-    filepath,
+    if (!data.url) throw new Error('"url" property is required')
-    line,
+
-    httpReferrer,
+    const [channelId, feedId] = data.tvg.id.split('@')
-    label,
+    const { name, label, quality } = parseTitle(data.name)
-    name,
+
-    quality,
+    this.id = data.tvg.id || undefined
-    url,
+    this.feedId = feedId || undefined
-    httpUserAgent
+    this.channelId = channelId || undefined
-  }: StreamProps) {
+    this.line = data.line
-    this.channel = channel || ''
+    this.label = label || undefined
-    this.filepath = filepath
-    this.line = line
-    this.httpReferrer = httpReferrer || ''
-    this.label = label || ''
     this.name = name
-    this.quality = quality || ''
+    this.quality = quality || undefined
-    this.url = url
+    this.url = data.url
-    this.httpUserAgent = httpUserAgent || ''
+    this.httpReferrer = data.http.referrer || undefined
-    this.logo = ''
+    this.httpUserAgent = data.http['user-agent'] || undefined
-    this.broadcastArea = new Collection()
-    this.categories = new Collection()
-    this.languages = new Collection()
-    this.isNSFW = false
     this.groupTitle = 'Undefined'
   }
 
+  withChannel(channelsGroupedById: Dictionary): this {
+    if (!this.channelId) return this
+
+    this.channel = channelsGroupedById.get(this.channelId)
+
+    return this
+  }
+
+  withFeed(feedsGroupedByChannelId: Dictionary): this {
+    if (!this.channelId) return this
+
+    const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || []
+    if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId)
+    if (!this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain)
+
+    return this
+  }
+
+  setId(id: string): this {
+    this.id = id
+
+    return this
+  }
+
+  setChannelId(channelId: string): this {
+    this.channelId = channelId
+
+    return this
+  }
+
+  setFeedId(feedId: string | undefined): this {
+    this.feedId = feedId
+
+    return this
+  }
+
+  setLabel(label: string): this {
+    this.label = label
+
+    return this
+  }
+
+  setQuality(quality: string): this {
+    this.quality = quality
+
+    return this
+  }
+
+  setHttpUserAgent(httpUserAgent: string): this {
+    this.httpUserAgent = httpUserAgent
+
+    return this
+  }
+
+  setHttpReferrer(httpReferrer: string): this {
+    this.httpReferrer = httpReferrer
+
+    return this
+  }
+
+  setFilepath(filepath: string): this {
+    this.filepath = filepath
+
+    return this
+  }
+
+  updateFilepath(): this {
+    if (!this.channel) return this
+
+    this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u`
+
+    return this
+  }
+
+  getFilepath(): string {
+    return this.filepath || ''
+  }
+
+  getHttpReferrer(): string {
+    return this.httpReferrer || ''
+  }
+
+  getHttpUserAgent(): string {
+    return this.httpUserAgent || ''
+  }
+
+  getQuality(): string {
+    return this.quality || ''
+  }
+
+  hasQuality(): boolean {
+    return !!this.quality
+  }
+
+  getHorizontalResolution(): number {
+    if (!this.hasQuality()) return 0
+
+    return parseInt(this.getQuality().replace(/p|i/, ''))
+  }
+
+  updateName(): this {
+    if (!this.channel) return this
+
+    this.name = this.channel.name
+    if (this.feed && !this.feed.isMain) {
+      this.name += ` ${this.feed.name}`
+    }
+
+    return this
+  }
+
+  updateId(): this {
+    if (!this.channel) return this
+    if (this.feed) {
+      this.id = `${this.channel.id}@${this.feed.id}`
+    } else {
+      this.id = this.channel.id
+    }
+
+    return this
+  }
+
   normalizeURL() {
     const url = new URL(this.url)
 
@@ -81,36 +183,75 @@ export class Stream {
     return !!this.channel
   }
 
-  hasCategories(): boolean {
+  getBroadcastRegions(): Collection {
-    return this.categories.notEmpty()
+    return this.feed ? this.feed.getBroadcastRegions() : new Collection()
   }
 
-  noCategories(): boolean {
+  getBroadcastCountries(): Collection {
-    return this.categories.isEmpty()
+    return this.feed ? this.feed.getBroadcastCountries() : new Collection()
   }
 
-  hasCategory(category: Category): boolean {
+  hasBroadcastArea(): boolean {
-    return this.categories.includes((_category: Category) => _category.id === category.id)
+    return this.feed ? this.feed.hasBroadcastArea() : false
-  }
-
-  noLanguages(): boolean {
-    return this.languages.isEmpty()
-  }
-
-  hasLanguage(language: Language): boolean {
-    return this.languages.includes((_language: Language) => _language.code === language.code)
-  }
-
-  noBroadcastArea(): boolean {
-    return this.broadcastArea.isEmpty()
-  }
-
-  isInternational(): boolean {
-    return this.broadcastArea.includes('r/INT')
   }
 
   isSFW(): boolean {
-    return this.isNSFW === false
+    return this.channel ? this.channel.isSFW() : true
+  }
+
+  hasCategories(): boolean {
+    return this.channel ? this.channel.hasCategories() : false
+  }
+
+  hasCategory(category: Category): boolean {
+    return this.channel ? this.channel.hasCategory(category) : false
+  }
+
+  getCategoryNames(): string[] {
+    return this.getCategories()
+      .map((category: Category) => category.name)
+      .sort()
+      .all()
+  }
+
+  getCategories(): Collection {
+    return this.channel ? this.channel.getCategories() : new Collection()
+  }
+
+  getLanguages(): Collection {
+    return this.feed ? this.feed.getLanguages() : new Collection()
+  }
+
+  hasLanguages() {
+    return this.feed ? this.feed.hasLanguages() : false
+  }
+
+  hasLanguage(language: Language) {
+    return this.feed ? this.feed.hasLanguage(language) : false
+  }
+
+  getBroadcastAreaCodes(): Collection {
+    return this.feed ? this.feed.broadcastAreaCodes : new Collection()
+  }
+
+  isBroadcastInSubdivision(subdivision: Subdivision): boolean {
+    return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false
+  }
+
+  isBroadcastInCountry(country: Country): boolean {
+    return this.feed ? this.feed.isBroadcastInCountry(country) : false
+  }
+
+  isBroadcastInRegion(region: Region): boolean {
+    return this.feed ? this.feed.isBroadcastInRegion(region) : false
+  }
+
+  isInternational(): boolean {
+    return this.feed ? this.feed.isInternational() : false
+  }
+
+  getLogo(): string {
+    return this?.channel?.logo || ''
   }
 
   getTitle(): string {
@@ -127,15 +268,25 @@ export class Stream {
     return title
   }
 
+  getLabel(): string {
+    return this.label || ''
+  }
+
+  getId(): string {
+    return this.id || ''
+  }
+
   data() {
     return {
+      id: this.id,
       channel: this.channel,
+      feed: this.feed,
       filepath: this.filepath,
-      httpReferrer: this.httpReferrer,
       label: this.label,
       name: this.name,
       quality: this.quality,
       url: this.url,
+      httpReferrer: this.httpReferrer,
       httpUserAgent: this.httpUserAgent,
       line: this.line
     }
@@ -143,7 +294,8 @@ export class Stream {
 
   toJSON() {
     return {
-      channel: this.channel || null,
+      channel: this.channelId || null,
+      feed: this.feedId || null,
       url: this.url,
       referrer: this.httpReferrer || null,
       user_agent: this.httpUserAgent || null
@@ -151,10 +303,10 @@ export class Stream {
   }
 
   toString(options: { public: boolean }) {
-    let output = `#EXTINF:-1 tvg-id="${this.channel}"`
+    let output = `#EXTINF:-1 tvg-id="${this.getId()}"`
 
     if (options.public) {
-      output += ` tvg-logo="${this.logo}" group-title="${this.groupTitle}"`
+      output += ` tvg-logo="${this.getLogo()}" group-title="${this.groupTitle}"`
     }
 
     if (this.httpReferrer) {
@@ -180,3 +332,16 @@ export class Stream {
     return output
   }
 }
+
+function parseTitle(title: string): { name: string; label: string; quality: string } {
+  const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
+  title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
+  const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
+  title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
+
+  return { name: title, label, quality }
+}
+
+function escapeRegExp(text) {
+  return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
+}

scripts/models/subdivision.ts

@@ -1,4 +1,7 @@
-type SubdivisionProps = {
+import { Dictionary } from '@freearhey/core'
+import { Country } from '.'
+
+type SubdivisionData = {
   code: string
   name: string
   country: string
@@ -7,11 +10,18 @@ type SubdivisionProps = {
 export class Subdivision {
   code: string
   name: string
-  country: string
+  countryCode: string
+  country?: Country
 
-  constructor({ code, name, country }: SubdivisionProps) {
+  constructor(data: SubdivisionData) {
-    this.code = code
+    this.code = data.code
-    this.name = name
+    this.name = data.name
-    this.country = country
+    this.countryCode = data.country
+  }
+
+  withCountry(countriesGroupedByCode: Dictionary): this {
+    this.country = countriesGroupedByCode.get(this.countryCode)
+
+    return this
   }
 }

scripts/models/timezone.ts

@@ -0,0 +1,30 @@
+import { Collection, Dictionary } from '@freearhey/core'
+
+type TimezoneData = {
+  id: string
+  utc_offset: string
+  countries: string[]
+}
+
+export class Timezone {
+  id: string
+  utcOffset: string
+  countryCodes: Collection
+  countries?: Collection
+
+  constructor(data: TimezoneData) {
+    this.id = data.id
+    this.utcOffset = data.utc_offset
+    this.countryCodes = new Collection(data.countries)
+  }
+
+  withCountries(countriesGroupedByCode: Dictionary): this {
+    this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
+
+    return this
+  }
+
+  getCountries(): Collection {
+    return this.countries || new Collection()
+  }
+}

scripts/tables/categoryTable.ts

@@ -11,6 +11,7 @@ export class CategoryTable implements Table {
     const dataStorage = new Storage(DATA_DIR)
     const categoriesContent = await dataStorage.json('categories.json')
     const categories = new Collection(categoriesContent).map(data => new Category(data))
+    const categoriesGroupedById = categories.keyBy((category: Category) => category.id)
 
     const parser = new LogParser()
     const logsStorage = new Storage(LOGS_DIR)
@@ -19,13 +20,12 @@ export class CategoryTable implements Table {
     let data = new Collection()
     parser
       .parse(generatorsLog)
-      .filter((logItem: LogItem) => logItem.filepath.includes('categories/'))
+      .filter((logItem: LogItem) => logItem.type === 'category')
       .forEach((logItem: LogItem) => {
         const file = new File(logItem.filepath)
         const categoryId = file.name()
-        const category: Category = categories.first(
+        const category: Category = categoriesGroupedById.get(categoryId)
-          (category: Category) => category.id === categoryId
+
-        )
         data.add([
           category ? category.name : 'ZZ',
           category ? category.name : 'Undefined',

scripts/tables/countryTable.ts

@@ -12,34 +12,31 @@ export class CountryTable implements Table {
 
     const countriesContent = await dataStorage.json('countries.json')
     const countries = new Collection(countriesContent).map(data => new Country(data))
-
+    const countriesGroupedByCode = countries.keyBy((country: Country) => country.code)
     const subdivisionsContent = await dataStorage.json('subdivisions.json')
     const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
+    const subdivisionsGroupedByCode = subdivisions.keyBy(
+      (subdivision: Subdivision) => subdivision.code
+    )
 
     const parser = new LogParser()
     const logsStorage = new Storage(LOGS_DIR)
     const generatorsLog = await logsStorage.load('generators.log')
+    const parsed = parser.parse(generatorsLog)
 
     let data = new Collection()
-    parser
+
-      .parse(generatorsLog)
+    parsed
-      .filter(
+      .filter((logItem: LogItem) => logItem.type === 'subdivision')
-        (logItem: LogItem) =>
-          logItem.filepath.includes('countries/') || logItem.filepath.includes('subdivisions/')
-      )
       .forEach((logItem: LogItem) => {
         const file = new File(logItem.filepath)
         const code = file.name().toUpperCase()
         const [countryCode, subdivisionCode] = code.split('-') || ['', '']
+        const country = countriesGroupedByCode.get(countryCode)
 
-        if (subdivisionCode) {
+        if (country && subdivisionCode) {
-          const subdivision = subdivisions.first(
+          const subdivision = subdivisionsGroupedByCode.get(code)
-            (subdivision: Subdivision) => subdivision.code === code
-          )
           if (subdivision) {
-            const country = countries.first(
-              (country: Country) => country.code === subdivision.country
-            )
             data.add([
               `${country.name}/${subdivision.name}`,
               `      ${subdivision.name}`,
@@ -47,18 +44,28 @@ export class CountryTable implements Table {
               `<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
             ])
           }
-        } else if (countryCode === 'INT') {
+        }
+      })
+
+    parsed
+      .filter((logItem: LogItem) => logItem.type === 'country')
+      .forEach((logItem: LogItem) => {
+        const file = new File(logItem.filepath)
+        const code = file.name().toUpperCase()
+        const [countryCode] = code.split('-') || ['', '']
+        const country = countriesGroupedByCode.get(countryCode)
+
+        if (country) {
           data.add([
-            'ZZ',
+            country.name,
-            '🌍 International',
+            `${country.flag} ${country.name}`,
             logItem.count,
             `<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
           ])
         } else {
-          const country = countries.first((country: Country) => country.code === countryCode)
           data.add([
-            country.name,
+            'ZZ',
-            `${country.flag} ${country.name}`,
+            'Undefined',
             logItem.count,
             `<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
           ])

scripts/tables/languageTable.ts

@@ -11,6 +11,7 @@ export class LanguageTable implements Table {
     const dataStorage = new Storage(DATA_DIR)
     const languagesContent = await dataStorage.json('languages.json')
     const languages = new Collection(languagesContent).map(data => new Language(data))
+    const languagesGroupedByCode = languages.keyBy((language: Language) => language.code)
 
     const parser = new LogParser()
     const logsStorage = new Storage(LOGS_DIR)
@@ -19,13 +20,11 @@ export class LanguageTable implements Table {
     let data = new Collection()
     parser
       .parse(generatorsLog)
-      .filter((logItem: LogItem) => logItem.filepath.includes('languages/'))
+      .filter((logItem: LogItem) => logItem.type === 'language')
       .forEach((logItem: LogItem) => {
         const file = new File(logItem.filepath)
         const languageCode = file.name()
-        const language: Language = languages.first(
+        const language: Language = languagesGroupedByCode.get(languageCode)
-          (language: Language) => language.code === languageCode
-        )
 
         data.add([
           language ? language.name : 'ZZ',

scripts/tables/regionTable.ts

@@ -11,6 +11,7 @@ export class RegionTable implements Table {
     const dataStorage = new Storage(DATA_DIR)
     const regionsContent = await dataStorage.json('regions.json')
     const regions = new Collection(regionsContent).map(data => new Region(data))
+    const regionsGroupedByCode = regions.keyBy((region: Region) => region.code)
 
     const parser = new LogParser()
     const logsStorage = new Storage(LOGS_DIR)
@@ -19,22 +20,35 @@ export class RegionTable implements Table {
     let data = new Collection()
     parser
       .parse(generatorsLog)
-      .filter((logItem: LogItem) => logItem.filepath.includes('regions/'))
+      .filter((logItem: LogItem) => logItem.type === 'region')
       .forEach((logItem: LogItem) => {
         const file = new File(logItem.filepath)
         const regionCode = file.name().toUpperCase()
-        const region: Region = regions.first((region: Region) => region.code === regionCode)
+        const region: Region = regionsGroupedByCode.get(regionCode)
 
         if (region) {
           data.add([
             region.name,
+            region.name,
+            logItem.count,
+            `<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
+          ])
+        } else {
+          data.add([
+            'ZZZ',
+            'Undefined',
             logItem.count,
             `<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
           ])
         }
       })
 
-    data = data.orderBy(item => item[0])
+    data = data
+      .orderBy(item => item[0])
+      .map(item => {
+        item.shift()
+        return item
+      })
 
     const table = new HTMLTable(data.all(), [
       { name: 'Region', align: 'left' },