Update scripts

scripts/commands/api/generate.ts

@@ -1,30 +1,25 @@
-import { Logger, Storage, Collection } from '@freearhey/core'
+import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
+import type { DataProcessorData } from '../../types/dataProcessor'
 import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
-import { PlaylistParser } from '../../core'
+import type { DataLoaderData } from '../../types/dataLoader'
-import { Stream, Channel, Feed } from '../../models'
+import { Logger, Storage } from '@freearhey/core'
-import { uniqueId } from 'lodash'
+import { Stream } from '../../models'
 
 async function main() {
   const logger = new Logger()
 
-  logger.info('loading api data...')
+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const dataLoader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await dataLoader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data)
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data =>
-    new Feed(data).withChannel(channelsGroupedById)
-  )
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
     storage: streamsStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
   const files = await streamsStorage.list('**/*.m3u')

scripts/commands/api/load.ts

@@ -1,23 +1,24 @@
-import { Logger } from '@freearhey/core'
+import { DATA_DIR } from '../../constants'
-import { ApiClient } from '../../core'
+import { Storage } from '@freearhey/core'
+import { DataLoader } from '../../core'
 
 async function main() {
-  const logger = new Logger()
+  const storage = new Storage(DATA_DIR)
-  const client = new ApiClient({ logger })
+  const loader = new DataLoader({ storage })
 
-  const requests = [
+  await Promise.all([
-    client.download('blocklist.json'),
+    loader.download('blocklist.json'),
-    client.download('categories.json'),
+    loader.download('categories.json'),
-    client.download('channels.json'),
+    loader.download('channels.json'),
-    client.download('countries.json'),
+    loader.download('countries.json'),
-    client.download('languages.json'),
+    loader.download('languages.json'),
-    client.download('regions.json'),
+    loader.download('regions.json'),
-    client.download('subdivisions.json'),
+    loader.download('subdivisions.json'),
-    client.download('feeds.json'),
+    loader.download('feeds.json'),
-    client.download('timezones.json')
+    loader.download('timezones.json'),
-  ]
+    loader.download('guides.json'),
-
+    loader.download('streams.json')
-  await Promise.all(requests)
+  ])
 }
 
 main()

scripts/commands/playlist/edit.ts

@@ -0,0 +1,208 @@
+import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
+import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
+import { Channel, Feed, Playlist, Stream } from '../../models'
+import type { ChannelSearchableData } from '../../types/channel'
+import { DataProcessorData } from '../../types/dataProcessor'
+import { DataLoaderData } from '../../types/dataLoader'
+import { select, input } from '@inquirer/prompts'
+import { DATA_DIR } from '../../constants'
+import nodeCleanup from 'node-cleanup'
+import sjs from '@freearhey/search-js'
+import { Command } from 'commander'
+import readline from 'readline'
+
+type ChoiceValue = { type: string; value?: Feed | Channel }
+type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
+
+if (process.platform === 'win32') {
+  readline
+    .createInterface({
+      input: process.stdin,
+      output: process.stdout
+    })
+    .on('SIGINT', function () {
+      process.emit('SIGINT')
+    })
+}
+
+const program = new Command()
+
+program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
+
+const filepath = program.args[0]
+const logger = new Logger()
+const storage = new Storage()
+let parsedStreams = new Collection()
+
+main(filepath)
+nodeCleanup(() => {
+  save(filepath)
+})
+
+export default async function main(filepath: string) {
+  if (!(await storage.exists(filepath))) {
+    throw new Error(`File "${filepath}" does not exists`)
+  }
+
+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
+  const dataStorage = new Storage(DATA_DIR)
+  const loader = new DataLoader({ storage: dataStorage })
+  const data: DataLoaderData = await loader.load()
+  const { channels, channelsKeyById, feedsGroupedByChannelId }: DataProcessorData =
+    processor.process(data)
+
+  logger.info('loading streams...')
+  const parser = new PlaylistParser({ storage, feedsGroupedByChannelId, channelsKeyById })
+  parsedStreams = await parser.parseFile(filepath)
+  const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.id)
+
+  logger.info(
+    `found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)`
+  )
+
+  logger.info('creating search index...')
+  const items = channels.map((channel: Channel) => channel.getSearchable()).all()
+  const searchIndex = sjs.createIndex(items, {
+    searchable: ['name', 'altNames', 'guideNames', 'streamNames', 'feedFullNames']
+  })
+
+  logger.info('starting...\n')
+
+  for (const stream of streamsWithoutId.all()) {
+    try {
+      stream.id = await selectChannel(stream, searchIndex, feedsGroupedByChannelId, channelsKeyById)
+    } catch (err) {
+      logger.info(err.message)
+      break
+    }
+  }
+
+  streamsWithoutId.forEach((stream: Stream) => {
+    if (stream.id === '-') {
+      stream.id = ''
+    }
+  })
+}
+
+async function selectChannel(
+  stream: Stream,
+  searchIndex,
+  feedsGroupedByChannelId: Dictionary,
+  channelsKeyById: Dictionary
+): Promise<string> {
+  const query = escapeRegex(stream.getName())
+  const similarChannels = searchIndex
+    .search(query)
+    .map((item: ChannelSearchableData) => channelsKeyById.get(item.id))
+
+  const url = stream.url.length > 50 ? stream.url.slice(0, 50) + '...' : stream.url
+
+  const selected: ChoiceValue = await select({
+    message: `Select channel ID for "${stream.name}" (${url}):`,
+    choices: getChannelChoises(new Collection(similarChannels)),
+    pageSize: 10
+  })
+
+  switch (selected.type) {
+    case 'skip':
+      return '-'
+    case 'type': {
+      const typedChannelId = await input({ message: '  Channel ID:' })
+      if (!typedChannelId) return ''
+      const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return typedChannelId
+      return [typedChannelId, selectedFeedId].join('@')
+    }
+    case 'channel': {
+      const selectedChannel = selected.value
+      if (!selectedChannel) return ''
+      const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
+      if (selectedFeedId === '-') return selectedChannel.id
+      return [selectedChannel.id, selectedFeedId].join('@')
+    }
+  }
+
+  return ''
+}
+
+async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
+  const channelFeeds = new Collection(feedsGroupedByChannelId.get(channelId)) || new Collection()
+  const choices = getFeedChoises(channelFeeds)
+
+  const selected: ChoiceValue = await select({
+    message: `Select feed ID for "${channelId}":`,
+    choices,
+    pageSize: 10
+  })
+
+  switch (selected.type) {
+    case 'skip':
+      return '-'
+    case 'type':
+      return await input({ message: '  Feed ID:', default: 'SD' })
+    case 'feed':
+      const selectedFeed = selected.value
+      if (!selectedFeed) return ''
+      return selectedFeed.id
+  }
+
+  return ''
+}
+
+function getChannelChoises(channels: Collection): Choice[] {
+  const choises: Choice[] = []
+
+  channels.forEach((channel: Channel) => {
+    const names = new Collection([channel.name, ...channel.altNames.all()]).uniq().join(', ')
+
+    choises.push({
+      value: {
+        type: 'channel',
+        value: channel
+      },
+      name: `${channel.id} (${names})`,
+      short: `${channel.id}`
+    })
+  })
+
+  choises.push({ name: 'Type...', value: { type: 'type' } })
+  choises.push({ name: 'Skip', value: { type: 'skip' } })
+
+  return choises
+}
+
+function getFeedChoises(feeds: Collection): Choice[] {
+  const choises: Choice[] = []
+
+  feeds.forEach((feed: Feed) => {
+    let name = `${feed.id} (${feed.name})`
+    if (feed.isMain) name += ' [main]'
+
+    choises.push({
+      value: {
+        type: 'feed',
+        value: feed
+      },
+      default: feed.isMain,
+      name,
+      short: feed.id
+    })
+  })
+
+  choises.push({ name: 'Type...', value: { type: 'type' } })
+  choises.push({ name: 'Skip', value: { type: 'skip' } })
+
+  return choises
+}
+
+function save(filepath: string) {
+  if (!storage.existsSync(filepath)) return
+  const playlist = new Playlist(parsedStreams)
+  storage.saveSync(filepath, playlist.toString())
+  logger.info(`\nFile '${filepath}' successfully saved`)
+}
+
+function escapeRegex(string: string) {
+  return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
+}

scripts/commands/playlist/format.ts

@@ -1,33 +1,28 @@
-import { Logger, Storage, Collection } from '@freearhey/core'
+import { Logger, Storage } from '@freearhey/core'
 import { STREAMS_DIR, DATA_DIR } from '../../constants'
-import { PlaylistParser } from '../../core'
+import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
-import { Stream, Playlist, Channel, Feed } from '../../models'
+import { Stream, Playlist } from '../../models'
 import { program } from 'commander'
-import { uniqueId } from 'lodash'
+import { DataLoaderData } from '../../types/dataLoader'
+import { DataProcessorData } from '../../types/dataProcessor'
 
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
 
 async function main() {
-  const streamsStorage = new Storage(STREAMS_DIR)
   const logger = new Logger()
 
   logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const loader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await loader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data)
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data =>
-    new Feed(data).withChannel(channelsGroupedById)
-  )
-  const feedsGroupedByChannelId = feeds.groupBy(feed =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
 
   logger.info('loading streams...')
+  const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
     storage: streamsStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
   const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
@@ -46,7 +41,7 @@ async function main() {
 
   logger.info('removing wrong id...')
   streams = streams.map((stream: Stream) => {
-    if (!stream.channel || channelsGroupedById.missing(stream.channel.id)) {
+    if (!stream.channel || channelsKeyById.missing(stream.channel.id)) {
       stream.id = ''
     }
 

scripts/commands/playlist/generate.ts

@@ -1,16 +1,6 @@
-import { Logger, Storage, Collection } from '@freearhey/core'
+import { Logger, Storage } from '@freearhey/core'
-import { PlaylistParser } from '../../core'
+import { PlaylistParser, DataProcessor, DataLoader } from '../../core'
-import {
+import { Stream } from '../../models'
-  Stream,
-  Category,
-  Channel,
-  Language,
-  Country,
-  Region,
-  Subdivision,
-  Feed,
-  Timezone
-} from '../../models'
 import { uniqueId } from 'lodash'
 import {
   CategoriesGenerator,
@@ -24,86 +14,36 @@ import {
   IndexRegionGenerator
 } from '../../generators'
 import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
+import type { DataProcessorData } from '../../types/dataProcessor'
+import type { DataLoaderData } from '../../types/dataLoader'
 
 async function main() {
   const logger = new Logger()
-  const dataStorage = new Storage(DATA_DIR)
   const generatorsLogger = new Logger({
     stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
   })
 
   logger.info('loading data from api...')
-  const categoriesData = await dataStorage.json('categories.json')
+  const processor = new DataProcessor()
-  const countriesData = await dataStorage.json('countries.json')
+  const dataStorage = new Storage(DATA_DIR)
-  const languagesData = await dataStorage.json('languages.json')
+  const loader = new DataLoader({ storage: dataStorage })
-  const regionsData = await dataStorage.json('regions.json')
+  const data: DataLoaderData = await loader.load()
-  const subdivisionsData = await dataStorage.json('subdivisions.json')
+  const {
-  const timezonesData = await dataStorage.json('timezones.json')
+    categories,
-  const channelsData = await dataStorage.json('channels.json')
+    countries,
-  const feedsData = await dataStorage.json('feeds.json')
+    regions,
-
+    channelsKeyById,
-  logger.info('preparing data...')
+    feedsGroupedByChannelId
-  const subdivisions = new Collection(subdivisionsData).map(data => new Subdivision(data))
+  }: DataProcessorData = processor.process(data)
-  const subdivisionsGroupedByCode = subdivisions.keyBy(
-    (subdivision: Subdivision) => subdivision.code
-  )
-  const subdivisionsGroupedByCountryCode = subdivisions.groupBy(
-    (subdivision: Subdivision) => subdivision.countryCode
-  )
-  let regions = new Collection(regionsData).map(data =>
-    new Region(data).withSubdivisions(subdivisions)
-  )
-  const regionsGroupedByCode = regions.keyBy((region: Region) => region.code)
-  const categories = new Collection(categoriesData).map(data => new Category(data))
-  const categoriesGroupedById = categories.keyBy((category: Category) => category.id)
-  const languages = new Collection(languagesData).map(data => new Language(data))
-  const languagesGroupedByCode = languages.keyBy((language: Language) => language.code)
-  const countries = new Collection(countriesData).map(data =>
-    new Country(data)
-      .withRegions(regions)
-      .withLanguage(languagesGroupedByCode)
-      .withSubdivisions(subdivisionsGroupedByCountryCode)
-  )
-  const countriesGroupedByCode = countries.keyBy((country: Country) => country.code)
-  regions = regions.map((region: Region) => region.withCountries(countriesGroupedByCode))
-
-  const timezones = new Collection(timezonesData).map(data =>
-    new Timezone(data).withCountries(countriesGroupedByCode)
-  )
-  const timezonesGroupedById = timezones.keyBy((timezone: Timezone) => timezone.id)
-
-  const channels = new Collection(channelsData).map(data =>
-    new Channel(data)
-      .withCategories(categoriesGroupedById)
-      .withCountry(countriesGroupedByCode)
-      .withSubdivision(subdivisionsGroupedByCode)
-  )
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
-  const feeds = new Collection(feedsData).map(data =>
-    new Feed(data)
-      .withChannel(channelsGroupedById)
-      .withLanguages(languagesGroupedByCode)
-      .withTimezones(timezonesGroupedById)
-      .withBroadcastCountries(
-        countriesGroupedByCode,
-        regionsGroupedByCode,
-        subdivisionsGroupedByCode
-      )
-      .withBroadcastRegions(regions)
-      .withBroadcastSubdivisions(subdivisionsGroupedByCode)
-  )
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
 
   logger.info('loading streams...')
-  const storage = new Storage(STREAMS_DIR)
+  const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
-    storage,
+    storage: streamsStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
-  const files = await storage.list('**/*.m3u')
+  const files = await streamsStorage.list('**/*.m3u')
   let streams = await parser.parse(files)
   const totalStreams = streams.count()
   streams = streams.uniqBy((stream: Stream) =>

scripts/commands/playlist/test.ts

@@ -1,13 +1,15 @@
 import { Logger, Storage, Collection } from '@freearhey/core'
 import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
-import { PlaylistParser, StreamTester, CliTable } from '../../core'
+import { PlaylistParser, StreamTester, CliTable, DataProcessor, DataLoader } from '../../core'
-import { Stream, Feed, Channel } from '../../models'
+import { Stream } from '../../models'
 import { program } from 'commander'
 import { eachLimit } from 'async-es'
 import commandExists from 'command-exists'
 import chalk from 'chalk'
 import os from 'node:os'
 import dns from 'node:dns'
+import type { DataLoaderData } from '../../types/dataLoader'
+import type { DataProcessorData } from '../../types/dataProcessor'
 
 const cpus = os.cpus()
 
@@ -54,22 +56,18 @@ async function main() {
     return
   }
 
-  logger.info('loading channels from api...')
+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const loader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await loader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data)
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data =>
-    new Feed(data).withChannel(channelsGroupedById)
-  )
-  const feedsGroupedByChannelId = feeds.groupBy(feed => feed.channel)
 
   logger.info('loading streams...')
   const rootStorage = new Storage(ROOT_DIR)
   const parser = new PlaylistParser({
     storage: rootStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
   const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)

scripts/commands/playlist/update.ts

@@ -1,38 +1,33 @@
+import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
+import type { DataProcessorData } from '../../types/dataProcessor'
+import { Stream, Playlist, Channel, Issue } from '../../models'
+import type { DataLoaderData } from '../../types/dataLoader'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import { IssueLoader, PlaylistParser } from '../../core'
-import { Stream, Playlist, Channel, Feed, Issue } from '../../models'
 import validUrl from 'valid-url'
-import { uniqueId } from 'lodash'
 
 let processedIssues = new Collection()
 
 async function main() {
   const logger = new Logger({ disabled: true })
-  const loader = new IssueLoader()
+  const issueLoader = new IssueLoader()
 
   logger.info('loading issues...')
-  const issues = await loader.load()
+  const issues = await issueLoader.load()
 
-  logger.info('loading channels from api...')
+  logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const dataLoader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await dataLoader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const { channelsKeyById, feedsGroupedByChannelId }: DataProcessorData = processor.process(data)
-  const feedsData = await dataStorage.json('feeds.json')
-  const feeds = new Collection(feedsData).map(data =>
-    new Feed(data).withChannel(channelsGroupedById)
-  )
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
     storage: streamsStorage,
     feedsGroupedByChannelId,
-    channelsGroupedById
+    channelsKeyById
   })
   const files = await streamsStorage.list('**/*.m3u')
   const streams = await parser.parse(files)
@@ -44,7 +39,7 @@ async function main() {
   await editStreams({
     streams,
     issues,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
 
@@ -52,7 +47,7 @@ async function main() {
   await addStreams({
     streams,
     issues,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
 
@@ -101,12 +96,12 @@ async function removeStreams({ streams, issues }: { streams: Collection; issues:
 async function editStreams({
   streams,
   issues,
-  channelsGroupedById,
+  channelsKeyById,
   feedsGroupedByChannelId
 }: {
   streams: Collection
   issues: Collection
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
   feedsGroupedByChannelId: Dictionary
 }) {
   const requests = issues.filter(
@@ -129,7 +124,7 @@ async function editStreams({
       stream
         .setChannelId(channelId)
         .setFeedId(feedId)
-        .withChannel(channelsGroupedById)
+        .withChannel(channelsKeyById)
         .withFeed(feedsGroupedByChannelId)
         .updateId()
         .updateName()
@@ -143,8 +138,8 @@ async function editStreams({
 
     if (data.has('label')) stream.setLabel(label)
     if (data.has('quality')) stream.setQuality(quality)
-    if (data.has('httpUserAgent')) stream.setHttpUserAgent(httpUserAgent)
+    if (data.has('httpUserAgent')) stream.setUserAgent(httpUserAgent)
-    if (data.has('httpReferrer')) stream.setHttpReferrer(httpReferrer)
+    if (data.has('httpReferrer')) stream.setReferrer(httpReferrer)
 
     processedIssues.add(issue.number)
   })
@@ -153,12 +148,12 @@ async function editStreams({
 async function addStreams({
   streams,
   issues,
-  channelsGroupedById,
+  channelsKeyById,
   feedsGroupedByChannelId
 }: {
   streams: Collection
   issues: Collection
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
   feedsGroupedByChannelId: Dictionary
 }) {
   const requests = issues.filter(
@@ -168,51 +163,32 @@ async function addStreams({
     const data = issue.data
     if (data.missing('streamId') || data.missing('streamUrl')) return
     if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return
-    const stringUrl = data.getString('streamUrl') || ''
+    const streamUrl = data.getString('streamUrl') || ''
-    if (!isUri(stringUrl)) return
+    if (!isUri(streamUrl)) return
 
     const streamId = data.getString('streamId') || ''
-    const [channelId] = streamId.split('@')
+    const [channelId, feedId] = streamId.split('@')
 
-    const channel: Channel = channelsGroupedById.get(channelId)
+    const channel: Channel = channelsKeyById.get(channelId)
     if (!channel) return
 
-    const label = data.getString('label') || ''
+    const label = data.getString('label') || null
-    const quality = data.getString('quality') || ''
+    const quality = data.getString('quality') || null
-    const httpUserAgent = data.getString('httpUserAgent') || ''
+    const httpUserAgent = data.getString('httpUserAgent') || null
-    const httpReferrer = data.getString('httpReferrer') || ''
+    const httpReferrer = data.getString('httpReferrer') || null
 
     const stream = new Stream({
-      tvg: {
+      channel: channelId,
-        id: streamId,
+      feed: feedId,
-        name: '',
-        url: '',
-        logo: '',
-        rec: '',
-        shift: ''
-      },
       name: data.getString('channelName') || channel.name,
-      url: stringUrl,
+      url: streamUrl,
-      group: {
+      user_agent: httpUserAgent,
-        title: ''
+      referrer: httpReferrer,
-      },
+      quality,
-      http: {
+      label
-        'user-agent': httpUserAgent,
-        referrer: httpReferrer
-      },
-      line: -1,
-      raw: '',
-      timeshift: '',
-      catchup: {
-        type: '',
-        source: '',
-        days: ''
-      }
     })
-      .withChannel(channelsGroupedById)
+      .withChannel(channelsKeyById)
       .withFeed(feedsGroupedByChannelId)
-      .setLabel(label)
-      .setQuality(quality)
       .updateName()
       .updateFilepath()
 

scripts/commands/playlist/validate.ts

@@ -1,10 +1,11 @@
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
-import { PlaylistParser } from '../../core'
+import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
-import { Channel, Stream, Blocked, Feed } from '../../models'
+import { DataProcessorData } from '../../types/dataProcessor'
+import { DATA_DIR, STREAMS_DIR } from '../../constants'
+import { DataLoaderData } from '../../types/dataLoader'
+import { BlocklistRecord, Stream } from '../../models'
 import { program } from 'commander'
 import chalk from 'chalk'
-import { uniqueId } from 'lodash'
-import { DATA_DIR, STREAMS_DIR } from '../../constants'
 
 program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
 
@@ -18,26 +19,21 @@ async function main() {
   const logger = new Logger()
 
   logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const loader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await loader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const {
-  const feedsData = await dataStorage.json('feeds.json')
+    channelsKeyById,
-  const feeds = new Collection(feedsData).map(data =>
+    feedsGroupedByChannelId,
-    new Feed(data).withChannel(channelsGroupedById)
+    blocklistRecordsGroupedByChannelId
-  )
+  }: DataProcessorData = processor.process(data)
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
-  const blocklistContent = await dataStorage.json('blocklist.json')
-  const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
-  const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
     storage: streamsStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
   const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
@@ -55,11 +51,11 @@ async function main() {
     const buffer = new Dictionary()
     streams.forEach((stream: Stream) => {
       if (stream.channelId) {
-        const channel = channelsGroupedById.get(stream.channelId)
+        const channel = channelsKeyById.get(stream.channelId)
         if (!channel) {
           log.add({
             type: 'warning',
-            line: stream.line,
+            line: stream.getLine(),
             message: `"${stream.id}" is not in the database`
           })
         }
@@ -69,29 +65,32 @@ async function main() {
       if (duplicate) {
         log.add({
           type: 'warning',
-          line: stream.line,
+          line: stream.getLine(),
           message: `"${stream.url}" is already on the playlist`
         })
       } else {
         buffer.set(stream.url, true)
       }
 
-      const blocked = stream.channel ? blocklistGroupedByChannelId.get(stream.channel.id) : false
+      const blocklistRecords = stream.channel
-      if (blocked) {
+        ? new Collection(blocklistRecordsGroupedByChannelId.get(stream.channel.id))
-        if (blocked.reason === 'dmca') {
+        : new Collection()
+
+      blocklistRecords.forEach((blocklistRecord: BlocklistRecord) => {
+        if (blocklistRecord.reason === 'dmca') {
           log.add({
             type: 'error',
-            line: stream.line,
+            line: stream.getLine(),
-            message: `"${blocked.channelId}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
+            message: `"${blocklistRecord.channelId}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})`
           })
-        } else if (blocked.reason === 'nsfw') {
+        } else if (blocklistRecord.reason === 'nsfw') {
           log.add({
             type: 'error',
-            line: stream.line,
+            line: stream.getLine(),
-            message: `"${blocked.channelId}" is on the blocklist due to NSFW content (${blocked.ref})`
+            message: `"${blocklistRecord.channelId}" is on the blocklist due to NSFW content (${blocklistRecord.ref})`
           })
         }
-      }
+      })
     })
 
     if (log.notEmpty()) {

scripts/commands/report/create.ts

@@ -1,44 +1,41 @@
+import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
 import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
+import { DataProcessorData } from '../../types/dataProcessor'
 import { DATA_DIR, STREAMS_DIR } from '../../constants'
-import { IssueLoader, PlaylistParser } from '../../core'
+import { DataLoaderData } from '../../types/dataLoader'
-import { Blocked, Channel, Issue, Stream, Feed } from '../../models'
+import { Issue, Stream } from '../../models'
-import { uniqueId } from 'lodash'
 
 async function main() {
   const logger = new Logger()
-  const loader = new IssueLoader()
+  const issueLoader = new IssueLoader()
   let report = new Collection()
 
   logger.info('loading issues...')
-  const issues = await loader.load()
+  const issues = await issueLoader.load()
 
   logger.info('loading data from api...')
+  const processor = new DataProcessor()
   const dataStorage = new Storage(DATA_DIR)
-  const channelsData = await dataStorage.json('channels.json')
+  const dataLoader = new DataLoader({ storage: dataStorage })
-  const channels = new Collection(channelsData).map(data => new Channel(data))
+  const data: DataLoaderData = await dataLoader.load()
-  const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
+  const {
-  const feedsData = await dataStorage.json('feeds.json')
+    channelsKeyById,
-  const feeds = new Collection(feedsData).map(data =>
+    feedsGroupedByChannelId,
-    new Feed(data).withChannel(channelsGroupedById)
+    blocklistRecordsGroupedByChannelId
-  )
+  }: DataProcessorData = processor.process(data)
-  const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
-    feed.channel ? feed.channel.id : uniqueId()
-  )
-  const blocklistContent = await dataStorage.json('blocklist.json')
-  const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
-  const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
 
   logger.info('loading streams...')
   const streamsStorage = new Storage(STREAMS_DIR)
   const parser = new PlaylistParser({
     storage: streamsStorage,
-    channelsGroupedById,
+    channelsKeyById,
     feedsGroupedByChannelId
   })
   const files = await streamsStorage.list('**/*.m3u')
   const streams = await parser.parse(files)
   const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
   const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId)
+  const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
 
   logger.info('checking broken streams reports...')
   const brokenStreamReports = issues.filter(issue =>
@@ -94,8 +91,8 @@ async function main() {
 
     if (!channelId) result.status = 'missing_id'
     else if (!streamUrl) result.status = 'missing_link'
-    else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
+    else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
-    else if (channelsGroupedById.missing(channelId)) result.status = 'wrong_id'
+    else if (channelsKeyById.missing(channelId)) result.status = 'wrong_id'
     else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
     else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
     else result.status = 'pending'
@@ -124,7 +121,7 @@ async function main() {
 
     if (!streamUrl) result.status = 'missing_link'
     else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
-    else if (channelId && channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
+    else if (channelId && channelsKeyById.missing(channelId)) result.status = 'invalid_id'
 
     report.add(result)
   })
@@ -147,16 +144,16 @@ async function main() {
     }
 
     if (!channelId) result.status = 'missing_id'
-    else if (channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
+    else if (channelsKeyById.missing(channelId)) result.status = 'invalid_id'
-    else if (channelSearchRequestsBuffer.has(channelId)) result.status = 'duplicate'
+    else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate'
-    else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
+    else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
-    else if (streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled'
+    else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled'
     else {
-      const channelData = channelsGroupedById.get(channelId)
+      const channelData = channelsKeyById.get(channelId)
       if (channelData.length && channelData[0].closed) result.status = 'closed'
     }
 
-    channelSearchRequestsBuffer.set(channelId, true)
+    channelSearchRequestsBuffer.set(streamId, true)
 
     report.add(result)
   })

scripts/core/apiClient.ts

@@ -1,59 +1,16 @@
-import { Logger, Storage } from '@freearhey/core'
+import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'
-import axios, { AxiosInstance, AxiosResponse, AxiosProgressEvent } from 'axios'
-import cliProgress, { MultiBar } from 'cli-progress'
-import numeral from 'numeral'
 
 export class ApiClient {
-  progressBar: MultiBar
+  instance: AxiosInstance
-  client: AxiosInstance
-  storage: Storage
-  logger: Logger
 
-  constructor({ logger }: { logger: Logger }) {
+  constructor() {
-    this.logger = logger
+    this.instance = axios.create({
-    this.client = axios.create({
+      baseURL: 'https://iptv-org.github.io/api',
       responseType: 'stream'
     })
-    this.storage = new Storage()
-    this.progressBar = new cliProgress.MultiBar({
-      stopOnComplete: true,
-      hideCursor: true,
-      forceRedraw: true,
-      barsize: 36,
-      format(options, params, payload) {
-        const filename = payload.filename.padEnd(18, ' ')
-        const barsize = options.barsize || 40
-        const percent = (params.progress * 100).toFixed(2)
-        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
-        const total = numeral(params.total).format('0.0 b')
-        const completeSize = Math.round(params.progress * barsize)
-        const incompleteSize = barsize - completeSize
-        const bar =
-          options.barCompleteString && options.barIncompleteString
-            ? options.barCompleteString.substr(0, completeSize) +
-              options.barGlue +
-              options.barIncompleteString.substr(0, incompleteSize)
-            : '-'.repeat(barsize)
-
-        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
-      }
-    })
   }
 
-  async download(filename: string) {
+  get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
-    const stream = await this.storage.createStream(`temp/data/${filename}`)
+    return this.instance.get(url, options)
-
-    const bar = this.progressBar.create(0, 0, { filename })
-
-    this.client
-      .get(`https://iptv-org.github.io/api/${filename}`, {
-        onDownloadProgress({ total, loaded, rate }: AxiosProgressEvent) {
-          if (total) bar.setTotal(total)
-          bar.update(loaded, { speed: rate })
-        }
-      })
-      .then((response: AxiosResponse) => {
-        response.data.pipe(stream)
-      })
   }
 }

scripts/core/dataLoader.ts

@@ -0,0 +1,100 @@
+import { ApiClient } from './apiClient'
+import { Storage } from '@freearhey/core'
+import cliProgress, { MultiBar } from 'cli-progress'
+import numeral from 'numeral'
+import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
+
+export class DataLoader {
+  client: ApiClient
+  storage: Storage
+  progressBar: MultiBar
+
+  constructor(props: DataLoaderProps) {
+    this.client = new ApiClient()
+    this.storage = props.storage
+    this.progressBar = new cliProgress.MultiBar({
+      stopOnComplete: true,
+      hideCursor: true,
+      forceRedraw: true,
+      barsize: 36,
+      format(options, params, payload) {
+        const filename = payload.filename.padEnd(18, ' ')
+        const barsize = options.barsize || 40
+        const percent = (params.progress * 100).toFixed(2)
+        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
+        const total = numeral(params.total).format('0.0 b')
+        const completeSize = Math.round(params.progress * barsize)
+        const incompleteSize = barsize - completeSize
+        const bar =
+          options.barCompleteString && options.barIncompleteString
+            ? options.barCompleteString.substr(0, completeSize) +
+              options.barGlue +
+              options.barIncompleteString.substr(0, incompleteSize)
+            : '-'.repeat(barsize)
+
+        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
+      }
+    })
+  }
+
+  async load(): Promise<DataLoaderData> {
+    const [
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    ] = await Promise.all([
+      this.storage.json('countries.json'),
+      this.storage.json('regions.json'),
+      this.storage.json('subdivisions.json'),
+      this.storage.json('languages.json'),
+      this.storage.json('categories.json'),
+      this.storage.json('blocklist.json'),
+      this.storage.json('channels.json'),
+      this.storage.json('feeds.json'),
+      this.storage.json('timezones.json'),
+      this.storage.json('guides.json'),
+      this.storage.json('streams.json')
+    ])
+
+    return {
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    }
+  }
+
+  async download(filename: string) {
+    if (!this.storage || !this.progressBar) return
+
+    const stream = await this.storage.createStream(filename)
+    const progressBar = this.progressBar.create(0, 0, { filename })
+
+    this.client
+      .get(filename, {
+        responseType: 'stream',
+        onDownloadProgress({ total, loaded, rate }) {
+          if (total) progressBar.setTotal(total)
+          progressBar.update(loaded, { speed: rate })
+        }
+      })
+      .then(response => {
+        response.data.pipe(stream)
+      })
+  }
+}

scripts/core/dataProcessor.ts

@@ -0,0 +1,110 @@
+import { DataLoaderData } from '../types/dataLoader'
+import { Collection } from '@freearhey/core'
+import {
+  BlocklistRecord,
+  Subdivision,
+  Category,
+  Language,
+  Timezone,
+  Channel,
+  Country,
+  Region,
+  Stream,
+  Guide,
+  Feed
+} from '../models'
+
+export class DataProcessor {
+  constructor() {}
+
+  process(data: DataLoaderData) {
+    const categories = new Collection(data.categories).map(data => new Category(data))
+    const categoriesKeyById = categories.keyBy((category: Category) => category.id)
+
+    const subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data))
+    const subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
+    const subdivisionsGroupedByCountryCode = subdivisions.groupBy(
+      (subdivision: Subdivision) => subdivision.countryCode
+    )
+
+    let regions = new Collection(data.regions).map(data => new Region(data))
+    const regionsKeyByCode = regions.keyBy((region: Region) => region.code)
+
+    const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data))
+    const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy(
+      (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId
+    )
+
+    const streams = new Collection(data.streams).map(data => new Stream(data))
+    const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
+
+    const guides = new Collection(data.guides).map(data => new Guide(data))
+    const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
+
+    const languages = new Collection(data.languages).map(data => new Language(data))
+    const languagesKeyByCode = languages.keyBy((language: Language) => language.code)
+
+    const countries = new Collection(data.countries).map(data =>
+      new Country(data)
+        .withRegions(regions)
+        .withLanguage(languagesKeyByCode)
+        .withSubdivisions(subdivisionsGroupedByCountryCode)
+    )
+    const countriesKeyByCode = countries.keyBy((country: Country) => country.code)
+
+    regions = regions.map((region: Region) => region.withCountries(countriesKeyByCode))
+
+    const timezones = new Collection(data.timezones).map(data =>
+      new Timezone(data).withCountries(countriesKeyByCode)
+    )
+    const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id)
+
+    let channels = new Collection(data.channels).map(data =>
+      new Channel(data)
+        .withCategories(categoriesKeyById)
+        .withCountry(countriesKeyByCode)
+        .withSubdivision(subdivisionsKeyByCode)
+        .withCategories(categoriesKeyById)
+    )
+    const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
+
+    let feeds = new Collection(data.feeds).map(data =>
+      new Feed(data)
+        .withChannel(channelsKeyById)
+        .withLanguages(languagesKeyByCode)
+        .withTimezones(timezonesKeyById)
+        .withBroadcastCountries(countriesKeyByCode, regionsKeyByCode, subdivisionsKeyByCode)
+        .withBroadcastRegions(regions)
+        .withBroadcastSubdivisions(subdivisionsKeyByCode)
+    )
+    const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
+
+    channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
+
+    return {
+      blocklistRecordsGroupedByChannelId,
+      subdivisionsGroupedByCountryCode,
+      feedsGroupedByChannelId,
+      guidesGroupedByStreamId,
+      subdivisionsKeyByCode,
+      countriesKeyByCode,
+      languagesKeyByCode,
+      streamsGroupedById,
+      categoriesKeyById,
+      timezonesKeyById,
+      regionsKeyByCode,
+      blocklistRecords,
+      channelsKeyById,
+      subdivisions,
+      categories,
+      countries,
+      languages,
+      timezones,
+      channels,
+      regions,
+      streams,
+      guides,
+      feeds
+    }
+  }
+}

scripts/core/index.ts

@@ -1,11 +1,13 @@
-export * from './playlistParser'
+export * from './apiClient'
-export * from './numberParser'
+export * from './cliTable'
-export * from './logParser'
+export * from './dataProcessor'
-export * from './markdown'
+export * from './dataLoader'
+export * from './htmlTable'
+export * from './issueData'
 export * from './issueLoader'
 export * from './issueParser'
-export * from './htmlTable'
+export * from './logParser'
-export * from './apiClient'
+export * from './markdown'
-export * from './issueData'
+export * from './numberParser'
+export * from './playlistParser'
 export * from './streamTester'
-export * from './cliTable'

scripts/core/playlistParser.ts

@@ -5,18 +5,18 @@ import { Stream } from '../models'
 type PlaylistPareserProps = {
   storage: Storage
   feedsGroupedByChannelId: Dictionary
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
 }
 
 export class PlaylistParser {
   storage: Storage
   feedsGroupedByChannelId: Dictionary
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
 
-  constructor({ storage, feedsGroupedByChannelId, channelsGroupedById }: PlaylistPareserProps) {
+  constructor({ storage, feedsGroupedByChannelId, channelsKeyById }: PlaylistPareserProps) {
     this.storage = storage
     this.feedsGroupedByChannelId = feedsGroupedByChannelId
-    this.channelsGroupedById = channelsGroupedById
+    this.channelsKeyById = channelsKeyById
   }
 
   async parse(files: string[]): Promise<Collection> {
@@ -35,9 +35,10 @@ export class PlaylistParser {
     const parsed: parser.Playlist = parser.parse(content)
 
     const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
-      const stream = new Stream(data)
+      const stream = new Stream()
+        .fromPlaylistItem(data)
         .withFeed(this.feedsGroupedByChannelId)
-        .withChannel(this.channelsGroupedById)
+        .withChannel(this.channelsKeyById)
         .setFilepath(filepath)
 
       return stream

scripts/models/blocked.ts

@@ -1,17 +0,0 @@
-type BlockedProps = {
-  channel: string
-  reason: string
-  ref: string
-}
-
-export class Blocked {
-  channelId: string
-  reason: string
-  ref: string
-
-  constructor(data: BlockedProps) {
-    this.channelId = data.channel
-    this.reason = data.reason
-    this.ref = data.ref
-  }
-}

scripts/models/blocklistRecord.ts

@@ -0,0 +1,15 @@
+import type { BlocklistRecordData } from '../types/blocklistRecord'
+
+export class BlocklistRecord {
+  channelId: string
+  reason: string
+  ref: string
+
+  constructor(data?: BlocklistRecordData) {
+    if (!data) return
+
+    this.channelId = data.channel
+    this.reason = data.reason
+    this.ref = data.ref
+  }
+}

scripts/models/category.ts

@@ -1,7 +1,4 @@
-type CategoryData = {
+import type { CategoryData, CategorySerializedData } from '../types/category'
-  id: string
-  name: string
-}
 
 export class Category {
   id: string
@@ -11,4 +8,11 @@ export class Category {
     this.id = data.id
     this.name = data.name
   }
+
+  serialize(): CategorySerializedData {
+    return {
+      id: this.id,
+      name: this.name
+    }
+  }
 }

scripts/models/channel.ts

@@ -1,23 +1,6 @@
 import { Collection, Dictionary } from '@freearhey/core'
-import { Category, Country, Subdivision } from './index'
+import { Category, Country, Feed, Guide, Stream, Subdivision } from './index'
-
+import type { ChannelData, ChannelSearchableData, ChannelSerializedData } from '../types/channel'
-type ChannelData = {
-  id: string
-  name: string
-  alt_names: string[]
-  network: string
-  owners: Collection
-  country: string
-  subdivision: string
-  city: string
-  categories: Collection
-  is_nsfw: boolean
-  launched: string
-  closed: string
-  replaced_by: string
-  website: string
-  logo: string
-}
 
 export class Channel {
   id: string
@@ -31,15 +14,18 @@ export class Channel {
   subdivision?: Subdivision
   cityName?: string
   categoryIds: Collection
-  categories?: Collection
+  categories: Collection = new Collection()
   isNSFW: boolean
   launched?: string
   closed?: string
   replacedBy?: string
   website?: string
   logo: string
+  feeds?: Collection
+
+  constructor(data?: ChannelData) {
+    if (!data) return
 
-  constructor(data: ChannelData) {
     this.id = data.id
     this.name = data.name
     this.altNames = new Collection(data.alt_names)
@@ -57,28 +43,34 @@ export class Channel {
     this.logo = data.logo
   }
 
-  withSubdivision(subdivisionsGroupedByCode: Dictionary): this {
+  withSubdivision(subdivisionsKeyByCode: Dictionary): this {
     if (!this.subdivisionCode) return this
 
-    this.subdivision = subdivisionsGroupedByCode.get(this.subdivisionCode)
+    this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode)
 
     return this
   }
 
-  withCountry(countriesGroupedByCode: Dictionary): this {
+  withCountry(countriesKeyByCode: Dictionary): this {
-    this.country = countriesGroupedByCode.get(this.countryCode)
+    this.country = countriesKeyByCode.get(this.countryCode)
 
     return this
   }
 
-  withCategories(groupedCategories: Dictionary): this {
+  withCategories(categoriesKeyById: Dictionary): this {
     this.categories = this.categoryIds
-      .map((id: string) => groupedCategories.get(id))
+      .map((id: string) => categoriesKeyById.get(id))
       .filter(Boolean)
 
     return this
   }
 
+  withFeeds(feedsGroupedByChannelId: Dictionary): this {
+    this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
+
+    return this
+  }
+
   getCountry(): Country | undefined {
     return this.country
   }
@@ -102,7 +94,106 @@ export class Channel {
     )
   }
 
+  getFeeds(): Collection {
+    if (!this.feeds) return new Collection()
+
+    return this.feeds
+  }
+
+  getGuides(): Collection {
+    let guides = new Collection()
+
+    this.getFeeds().forEach((feed: Feed) => {
+      guides = guides.concat(feed.getGuides())
+    })
+
+    return guides
+  }
+
+  getGuideNames(): Collection {
+    return this.getGuides()
+      .map((guide: Guide) => guide.siteName)
+      .uniq()
+  }
+
+  getStreams(): Collection {
+    let streams = new Collection()
+
+    this.getFeeds().forEach((feed: Feed) => {
+      streams = streams.concat(feed.getStreams())
+    })
+
+    return streams
+  }
+
+  getStreamNames(): Collection {
+    return this.getStreams()
+      .map((stream: Stream) => stream.getName())
+      .uniq()
+  }
+
+  getFeedFullNames(): Collection {
+    return this.getFeeds()
+      .map((feed: Feed) => feed.getFullName())
+      .uniq()
+  }
+
   isSFW(): boolean {
     return this.isNSFW === false
   }
+
+  getSearchable(): ChannelSearchableData {
+    return {
+      id: this.id,
+      name: this.name,
+      altNames: this.altNames.all(),
+      guideNames: this.getGuideNames().all(),
+      streamNames: this.getStreamNames().all(),
+      feedFullNames: this.getFeedFullNames().all()
+    }
+  }
+
+  serialize(): ChannelSerializedData {
+    return {
+      id: this.id,
+      name: this.name,
+      altNames: this.altNames.all(),
+      network: this.network,
+      owners: this.owners.all(),
+      countryCode: this.countryCode,
+      country: this.country ? this.country.serialize() : undefined,
+      subdivisionCode: this.subdivisionCode,
+      subdivision: this.subdivision ? this.subdivision.serialize() : undefined,
+      cityName: this.cityName,
+      categoryIds: this.categoryIds.all(),
+      categories: this.categories.map((category: Category) => category.serialize()).all(),
+      isNSFW: this.isNSFW,
+      launched: this.launched,
+      closed: this.closed,
+      replacedBy: this.replacedBy,
+      website: this.website,
+      logo: this.logo
+    }
+  }
+
+  deserialize(data: ChannelSerializedData): this {
+    this.id = data.id
+    this.name = data.name
+    this.altNames = new Collection(data.altNames)
+    this.network = data.network
+    this.owners = new Collection(data.owners)
+    this.countryCode = data.countryCode
+    this.country = data.country ? new Country().deserialize(data.country) : undefined
+    this.subdivisionCode = data.subdivisionCode
+    this.cityName = data.cityName
+    this.categoryIds = new Collection(data.categoryIds)
+    this.isNSFW = data.isNSFW
+    this.launched = data.launched
+    this.closed = data.closed
+    this.replacedBy = data.replacedBy
+    this.website = data.website
+    this.logo = data.logo
+
+    return this
+  }
 }

scripts/models/country.ts

@@ -1,12 +1,8 @@
 import { Collection, Dictionary } from '@freearhey/core'
-import { Region, Language } from '.'
+import { Region, Language, Subdivision } from '.'
-
+import type { CountryData, CountrySerializedData } from '../types/country'
-type CountryData = {
+import { SubdivisionSerializedData } from '../types/subdivision'
-  code: string
+import { RegionSerializedData } from '../types/region'
-  name: string
-  lang: string
-  flag: string
-}
 
 export class Country {
   code: string
@@ -17,7 +13,9 @@ export class Country {
   subdivisions?: Collection
   regions?: Collection
 
-  constructor(data: CountryData) {
+  constructor(data?: CountryData) {
+    if (!data) return
+
     this.code = data.code
     this.name = data.name
     this.flag = data.flag
@@ -38,8 +36,8 @@ export class Country {
     return this
   }
 
-  withLanguage(languagesGroupedByCode: Dictionary): this {
+  withLanguage(languagesKeyByCode: Dictionary): this {
-    this.language = languagesGroupedByCode.get(this.languageCode)
+    this.language = languagesKeyByCode.get(this.languageCode)
 
     return this
   }
@@ -55,4 +53,34 @@ export class Country {
   getSubdivisions(): Collection {
     return this.subdivisions || new Collection()
   }
+
+  serialize(): CountrySerializedData {
+    return {
+      code: this.code,
+      name: this.name,
+      flag: this.flag,
+      languageCode: this.languageCode,
+      language: this.language ? this.language.serialize() : null,
+      subdivisions: this.subdivisions
+        ? this.subdivisions.map((subdivision: Subdivision) => subdivision.serialize()).all()
+        : [],
+      regions: this.regions ? this.regions.map((region: Region) => region.serialize()).all() : []
+    }
+  }
+
+  deserialize(data: CountrySerializedData): this {
+    this.code = data.code
+    this.name = data.name
+    this.flag = data.flag
+    this.languageCode = data.languageCode
+    this.language = data.language ? new Language().deserialize(data.language) : undefined
+    this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
+      new Subdivision().deserialize(data)
+    )
+    this.regions = new Collection(data.regions).map((data: RegionSerializedData) =>
+      new Region().deserialize(data)
+    )
+
+    return this
+  }
 }

scripts/models/feed.ts

@@ -1,16 +1,6 @@
 import { Collection, Dictionary } from '@freearhey/core'
 import { Country, Language, Region, Channel, Subdivision } from './index'
-
+import type { FeedData } from '../types/feed'
-type FeedData = {
-  channel: string
-  id: string
-  name: string
-  is_main: boolean
-  broadcast_area: Collection
-  languages: Collection
-  timezones: Collection
-  video_format: string
-}
 
 export class Feed {
   channelId: string
@@ -30,6 +20,8 @@ export class Feed {
   timezoneIds: Collection
   timezones?: Collection
   videoFormat: string
+  guides?: Collection
+  streams?: Collection
 
   constructor(data: FeedData) {
     this.channelId = data.channel
@@ -61,40 +53,58 @@ export class Feed {
     })
   }
 
-  withChannel(channelsGroupedById: Dictionary): this {
+  withChannel(channelsKeyById: Dictionary): this {
-    this.channel = channelsGroupedById.get(this.channelId)
+    this.channel = channelsKeyById.get(this.channelId)
 
     return this
   }
 
-  withLanguages(languagesGroupedByCode: Dictionary): this {
+  withStreams(streamsGroupedById: Dictionary): this {
+    this.streams = new Collection(streamsGroupedById.get(`${this.channelId}@${this.id}`))
+
+    if (this.isMain) {
+      this.streams = this.streams.concat(new Collection(streamsGroupedById.get(this.channelId)))
+    }
+
+    return this
+  }
+
+  withGuides(guidesGroupedByStreamId: Dictionary): this {
+    this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
+
+    if (this.isMain) {
+      this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
+    }
+
+    return this
+  }
+
+  withLanguages(languagesKeyByCode: Dictionary): this {
     this.languages = this.languageCodes
-      .map((code: string) => languagesGroupedByCode.get(code))
+      .map((code: string) => languagesKeyByCode.get(code))
       .filter(Boolean)
 
     return this
   }
 
-  withTimezones(timezonesGroupedById: Dictionary): this {
+  withTimezones(timezonesKeyById: Dictionary): this {
-    this.timezones = this.timezoneIds
+    this.timezones = this.timezoneIds.map((id: string) => timezonesKeyById.get(id)).filter(Boolean)
-      .map((id: string) => timezonesGroupedById.get(id))
-      .filter(Boolean)
 
     return this
   }
 
-  withBroadcastSubdivisions(subdivisionsGroupedByCode: Dictionary): this {
+  withBroadcastSubdivisions(subdivisionsKeyByCode: Dictionary): this {
     this.broadcastSubdivisions = this.broadcastSubdivisionCodes.map((code: string) =>
-      subdivisionsGroupedByCode.get(code)
+      subdivisionsKeyByCode.get(code)
     )
 
     return this
   }
 
   withBroadcastCountries(
-    countriesGroupedByCode: Dictionary,
+    countriesKeyByCode: Dictionary,
-    regionsGroupedByCode: Dictionary,
+    regionsKeyByCode: Dictionary,
-    subdivisionsGroupedByCode: Dictionary
+    subdivisionsKeyByCode: Dictionary
   ): this {
     let broadcastCountries = new Collection()
 
@@ -104,22 +114,22 @@ export class Feed {
     }
 
     this.broadcastCountryCodes.forEach((code: string) => {
-      broadcastCountries.add(countriesGroupedByCode.get(code))
+      broadcastCountries.add(countriesKeyByCode.get(code))
     })
 
     this.broadcastRegionCodes.forEach((code: string) => {
-      const region: Region = regionsGroupedByCode.get(code)
+      const region: Region = regionsKeyByCode.get(code)
       if (region) {
         region.countryCodes.forEach((countryCode: string) => {
-          broadcastCountries.add(countriesGroupedByCode.get(countryCode))
+          broadcastCountries.add(countriesKeyByCode.get(countryCode))
         })
       }
     })
 
     this.broadcastSubdivisionCodes.forEach((code: string) => {
-      const subdivision: Subdivision = subdivisionsGroupedByCode.get(code)
+      const subdivision: Subdivision = subdivisionsKeyByCode.get(code)
       if (subdivision) {
-        broadcastCountries.add(countriesGroupedByCode.get(subdivision.countryCode))
+        broadcastCountries.add(countriesKeyByCode.get(subdivision.countryCode))
       }
     })
 
@@ -197,4 +207,22 @@ export class Feed {
 
     return this.getBroadcastRegions().includes((_region: Region) => _region.code === region.code)
   }
+
+  getGuides(): Collection {
+    if (!this.guides) return new Collection()
+
+    return this.guides
+  }
+
+  getStreams(): Collection {
+    if (!this.streams) return new Collection()
+
+    return this.streams
+  }
+
+  getFullName(): string {
+    if (!this.channel) return ''
+
+    return `${this.channel.name} ${this.name}`
+  }
 }

scripts/models/guide.ts

@@ -0,0 +1,54 @@
+import type { GuideData, GuideSerializedData } from '../types/guide'
+
+export class Guide {
+  channelId?: string
+  feedId?: string
+  siteDomain: string
+  siteId: string
+  siteName: string
+  languageCode: string
+
+  constructor(data?: GuideData) {
+    if (!data) return
+
+    this.channelId = data.channel
+    this.feedId = data.feed
+    this.siteDomain = data.site
+    this.siteId = data.site_id
+    this.siteName = data.site_name
+    this.languageCode = data.lang
+  }
+
+  getUUID(): string {
+    return this.getStreamId() + this.siteId
+  }
+
+  getStreamId(): string | undefined {
+    if (!this.channelId) return undefined
+    if (!this.feedId) return this.channelId
+
+    return `${this.channelId}@${this.feedId}`
+  }
+
+  serialize(): GuideSerializedData {
+    return {
+      channelId: this.channelId,
+      feedId: this.feedId,
+      siteDomain: this.siteDomain,
+      siteId: this.siteId,
+      siteName: this.siteName,
+      languageCode: this.languageCode
+    }
+  }
+
+  deserialize(data: GuideSerializedData): this {
+    this.channelId = data.channelId
+    this.feedId = data.feedId
+    this.siteDomain = data.siteDomain
+    this.siteId = data.siteId
+    this.siteName = data.siteName
+    this.languageCode = data.languageCode
+
+    return this
+  }
+}

scripts/models/index.ts

@@ -1,13 +1,14 @@
-export * from './issue'
+export * from './blocklistRecord'
-export * from './playlist'
+export * from './broadcastArea'
-export * from './blocked'
-export * from './stream'
 export * from './category'
 export * from './channel'
-export * from './language'
 export * from './country'
-export * from './region'
-export * from './subdivision'
 export * from './feed'
-export * from './broadcastArea'
+export * from './guide'
+export * from './issue'
+export * from './language'
+export * from './playlist'
+export * from './region'
+export * from './stream'
+export * from './subdivision'
 export * from './timezone'

scripts/models/language.ts

@@ -1,14 +1,27 @@
-type LanguageData = {
+import type { LanguageData, LanguageSerializedData } from '../types/language'
-  code: string
-  name: string
-}
 
 export class Language {
   code: string
   name: string
 
-  constructor(data: LanguageData) {
+  constructor(data?: LanguageData) {
+    if (!data) return
+
     this.code = data.code
     this.name = data.name
   }
+
+  serialize(): LanguageSerializedData {
+    return {
+      code: this.code,
+      name: this.name
+    }
+  }
+
+  deserialize(data: LanguageSerializedData): this {
+    this.code = data.code
+    this.name = data.name
+
+    return this
+  }
 }

scripts/models/region.ts

@@ -1,27 +1,26 @@
 import { Collection, Dictionary } from '@freearhey/core'
-import { Subdivision } from '.'
+import { Country, Subdivision } from '.'
-
+import type { RegionData, RegionSerializedData } from '../types/region'
-type RegionData = {
+import { CountrySerializedData } from '../types/country'
-  code: string
+import { SubdivisionSerializedData } from '../types/subdivision'
-  name: string
-  countries: string[]
-}
 
 export class Region {
   code: string
   name: string
   countryCodes: Collection
-  countries?: Collection
+  countries: Collection = new Collection()
-  subdivisions?: Collection
+  subdivisions: Collection = new Collection()
+
+  constructor(data?: RegionData) {
+    if (!data) return
 
-  constructor(data: RegionData) {
     this.code = data.code
     this.name = data.name
     this.countryCodes = new Collection(data.countries)
   }
 
-  withCountries(countriesGroupedByCode: Dictionary): this {
+  withCountries(countriesKeyByCode: Dictionary): this {
-    this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
+    this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
 
     return this
   }
@@ -35,11 +34,11 @@ export class Region {
   }
 
   getSubdivisions(): Collection {
-    return this.subdivisions || new Collection()
+    return this.subdivisions
   }
 
   getCountries(): Collection {
-    return this.countries || new Collection()
+    return this.countries
   }
 
   includesCountryCode(code: string): boolean {
@@ -49,4 +48,30 @@ export class Region {
   isWorldwide(): boolean {
     return this.code === 'INT'
   }
+
+  serialize(): RegionSerializedData {
+    return {
+      code: this.code,
+      name: this.name,
+      countryCodes: this.countryCodes.all(),
+      countries: this.countries.map((country: Country) => country.serialize()).all(),
+      subdivisions: this.subdivisions
+        .map((subdivision: Subdivision) => subdivision.serialize())
+        .all()
+    }
+  }
+
+  deserialize(data: RegionSerializedData): this {
+    this.code = data.code
+    this.name = data.name
+    this.countryCodes = new Collection(data.countryCodes)
+    this.countries = new Collection(data.countries).map((data: CountrySerializedData) =>
+      new Country().deserialize(data)
+    )
+    this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
+      new Subdivision().deserialize(data)
+    )
+
+    return this
+  }
 }

scripts/models/stream.ts

@@ -1,26 +1,45 @@
-import { URL, Collection, Dictionary } from '@freearhey/core'
 import { Feed, Channel, Category, Region, Subdivision, Country, Language } from './index'
+import { URL, Collection, Dictionary } from '@freearhey/core'
+import type { StreamData } from '../types/stream'
 import parser from 'iptv-playlist-parser'
 
 export class Stream {
-  name: string
+  name?: string
   url: string
   id?: string
-  groupTitle: string
   channelId?: string
   channel?: Channel
   feedId?: string
   feed?: Feed
   filepath?: string
-  line: number
+  line?: number
   label?: string
   verticalResolution?: number
   isInterlaced?: boolean
-  httpReferrer?: string
+  referrer?: string
-  httpUserAgent?: string
+  userAgent?: string
+  groupTitle: string = 'Undefined'
   removed: boolean = false
 
-  constructor(data: parser.PlaylistItem) {
+  constructor(data?: StreamData) {
+    if (!data) return
+
+    const id = data.channel && data.feed ? [data.channel, data.feed].join('@') : data.channel
+    const { verticalResolution, isInterlaced } = parseQuality(data.quality)
+
+    this.id = id || undefined
+    this.channelId = data.channel || undefined
+    this.feedId = data.feed || undefined
+    this.name = data.name || undefined
+    this.url = data.url
+    this.referrer = data.referrer || undefined
+    this.userAgent = data.user_agent || undefined
+    this.verticalResolution = verticalResolution || undefined
+    this.isInterlaced = isInterlaced || undefined
+    this.label = data.label || undefined
+  }
+
+  fromPlaylistItem(data: parser.PlaylistItem): this {
     if (!data.name) throw new Error('"name" property is required')
     if (!data.url) throw new Error('"url" property is required')
 
@@ -37,15 +56,16 @@ export class Stream {
     this.verticalResolution = verticalResolution || undefined
     this.isInterlaced = isInterlaced || undefined
     this.url = data.url
-    this.httpReferrer = data.http.referrer || undefined
+    this.referrer = data.http.referrer || undefined
-    this.httpUserAgent = data.http['user-agent'] || undefined
+    this.userAgent = data.http['user-agent'] || undefined
-    this.groupTitle = 'Undefined'
+
+    return this
   }
 
-  withChannel(channelsGroupedById: Dictionary): this {
+  withChannel(channelsKeyById: Dictionary): this {
     if (!this.channelId) return this
 
-    this.channel = channelsGroupedById.get(this.channelId)
+    this.channel = channelsKeyById.get(this.channelId)
 
     return this
   }
@@ -93,18 +113,22 @@ export class Stream {
     return this
   }
 
-  setHttpUserAgent(httpUserAgent: string): this {
+  setUserAgent(userAgent: string): this {
-    this.httpUserAgent = httpUserAgent
+    this.userAgent = userAgent
 
     return this
   }
 
-  setHttpReferrer(httpReferrer: string): this {
+  setReferrer(referrer: string): this {
-    this.httpReferrer = httpReferrer
+    this.referrer = referrer
 
     return this
   }
 
+  getLine(): number {
+    return this.line || -1
+  }
+
   setFilepath(filepath: string): this {
     this.filepath = filepath
 
@@ -133,12 +157,12 @@ export class Stream {
     return this.filepath || ''
   }
 
-  getHttpReferrer(): string {
+  getReferrer(): string {
-    return this.httpReferrer || ''
+    return this.referrer || ''
   }
 
-  getHttpUserAgent(): string {
+  getUserAgent(): string {
-    return this.httpUserAgent || ''
+    return this.userAgent || ''
   }
 
   getQuality(): string {
@@ -198,14 +222,6 @@ export class Stream {
     return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
   }
 
-  hasName(): boolean {
-    return !!this.name
-  }
-
-  noName(): boolean {
-    return !this.name
-  }
-
   hasChannel() {
     return !!this.channel
   }
@@ -281,8 +297,12 @@ export class Stream {
     return this?.channel?.logo || ''
   }
 
+  getName(): string {
+    return this.name || ''
+  }
+
   getTitle(): string {
-    let title = `${this.name}`
+    let title = `${this.getName()}`
 
     if (this.getQuality()) {
       title += ` (${this.getQuality()})`
@@ -303,30 +323,13 @@ export class Stream {
     return this.id || ''
   }
 
-  data() {
-    return {
-      id: this.id,
-      channel: this.channel,
-      feed: this.feed,
-      filepath: this.filepath,
-      label: this.label,
-      name: this.name,
-      verticalResolution: this.verticalResolution,
-      isInterlaced: this.isInterlaced,
-      url: this.url,
-      httpReferrer: this.httpReferrer,
-      httpUserAgent: this.httpUserAgent,
-      line: this.line
-    }
-  }
-
   toJSON() {
     return {
       channel: this.channelId || null,
       feed: this.feedId || null,
       url: this.url,
-      referrer: this.httpReferrer || null,
+      referrer: this.referrer || null,
-      user_agent: this.httpUserAgent || null,
+      user_agent: this.userAgent || null,
       quality: this.getQuality() || null
     }
   }
@@ -338,22 +341,22 @@ export class Stream {
       output += ` tvg-logo="${this.getLogo()}" group-title="${this.groupTitle}"`
     }
 
-    if (this.httpReferrer) {
+    if (this.referrer) {
-      output += ` http-referrer="${this.httpReferrer}"`
+      output += ` http-referrer="${this.referrer}"`
     }
 
-    if (this.httpUserAgent) {
+    if (this.userAgent) {
-      output += ` http-user-agent="${this.httpUserAgent}"`
+      output += ` http-user-agent="${this.userAgent}"`
     }
 
     output += `,${this.getTitle()}`
 
-    if (this.httpReferrer) {
+    if (this.referrer) {
-      output += `\n#EXTVLCOPT:http-referrer=${this.httpReferrer}`
+      output += `\n#EXTVLCOPT:http-referrer=${this.referrer}`
     }
 
-    if (this.httpUserAgent) {
+    if (this.userAgent) {
-      output += `\n#EXTVLCOPT:http-user-agent=${this.httpUserAgent}`
+      output += `\n#EXTVLCOPT:http-user-agent=${this.userAgent}`
     }
 
     output += `\n${this.url}`
@@ -379,7 +382,11 @@ function escapeRegExp(text) {
   return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
 }
 
-function parseQuality(quality: string): { verticalResolution: number; isInterlaced: boolean } {
+function parseQuality(quality: string | null): {
+  verticalResolution: number | null
+  isInterlaced: boolean | null
+} {
+  if (!quality) return { verticalResolution: null, isInterlaced: null }
   let [, verticalResolutionString] = quality.match(/^(\d+)/) || [null, undefined]
   const isInterlaced = /i$/i.test(quality)
   let verticalResolution = 0

scripts/models/subdivision.ts

@@ -1,26 +1,41 @@
+import { SubdivisionData, SubdivisionSerializedData } from '../types/subdivision'
 import { Dictionary } from '@freearhey/core'
 import { Country } from '.'
 
-type SubdivisionData = {
-  code: string
-  name: string
-  country: string
-}
-
 export class Subdivision {
   code: string
   name: string
   countryCode: string
   country?: Country
 
-  constructor(data: SubdivisionData) {
+  constructor(data?: SubdivisionData) {
+    if (!data) return
+
     this.code = data.code
     this.name = data.name
     this.countryCode = data.country
   }
 
-  withCountry(countriesGroupedByCode: Dictionary): this {
+  withCountry(countriesKeyByCode: Dictionary): this {
-    this.country = countriesGroupedByCode.get(this.countryCode)
+    this.country = countriesKeyByCode.get(this.countryCode)
+
+    return this
+  }
+
+  serialize(): SubdivisionSerializedData {
+    return {
+      code: this.code,
+      name: this.name,
+      countryCode: this.code,
+      country: this.country ? this.country.serialize() : undefined
+    }
+  }
+
+  deserialize(data: SubdivisionSerializedData): this {
+    this.code = data.code
+    this.name = data.name
+    this.countryCode = data.countryCode
+    this.country = data.country ? new Country().deserialize(data.country) : undefined
 
     return this
   }

scripts/models/timezone.ts

@@ -18,8 +18,8 @@ export class Timezone {
     this.countryCodes = new Collection(data.countries)
   }
 
-  withCountries(countriesGroupedByCode: Dictionary): this {
+  withCountries(countriesKeyByCode: Dictionary): this {
-    this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
+    this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
 
     return this
   }

scripts/types/blocklistRecord.d.ts

@@ -0,0 +1,5 @@
+export type BlocklistRecordData = {
+  channel: string
+  reason: string
+  ref: string
+}

scripts/types/category.d.ts

@@ -0,0 +1,9 @@
+export type CategorySerializedData = {
+  id: string
+  name: string
+}
+
+export type CategoryData = {
+  id: string
+  name: string
+}

scripts/types/channel.d.ts

@@ -0,0 +1,52 @@
+import { Collection } from '@freearhey/core'
+import type { CountrySerializedData } from './country'
+import type { SubdivisionSerializedData } from './subdivision'
+import type { CategorySerializedData } from './category'
+
+export type ChannelSerializedData = {
+  id: string
+  name: string
+  altNames: string[]
+  network?: string
+  owners: string[]
+  countryCode: string
+  country?: CountrySerializedData
+  subdivisionCode?: string
+  subdivision?: SubdivisionSerializedData
+  cityName?: string
+  categoryIds: string[]
+  categories?: CategorySerializedData[]
+  isNSFW: boolean
+  launched?: string
+  closed?: string
+  replacedBy?: string
+  website?: string
+  logo: string
+}
+
+export type ChannelData = {
+  id: string
+  name: string
+  alt_names: string[]
+  network: string
+  owners: Collection
+  country: string
+  subdivision: string
+  city: string
+  categories: Collection
+  is_nsfw: boolean
+  launched: string
+  closed: string
+  replaced_by: string
+  website: string
+  logo: string
+}
+
+export type ChannelSearchableData = {
+  id: string
+  name: string
+  altNames: string[]
+  guideNames: string[]
+  streamNames: string[]
+  feedFullNames: string[]
+}

scripts/types/country.d.ts

@@ -0,0 +1,20 @@
+import type { LanguageSerializedData } from './language'
+import type { SubdivisionSerializedData } from './subdivision'
+import type { RegionSerializedData } from './region'
+
+export type CountrySerializedData = {
+  code: string
+  name: string
+  flag: string
+  languageCode: string
+  language: LanguageSerializedData | null
+  subdivisions: SubdivisionSerializedData[]
+  regions: RegionSerializedData[]
+}
+
+export type CountryData = {
+  code: string
+  name: string
+  lang: string
+  flag: string
+}

scripts/types/dataLoader.d.ts

@@ -0,0 +1,19 @@
+import { Storage } from '@freearhey/core'
+
+export type DataLoaderProps = {
+  storage: Storage
+}
+
+export type DataLoaderData = {
+  countries: object | object[]
+  regions: object | object[]
+  subdivisions: object | object[]
+  languages: object | object[]
+  categories: object | object[]
+  blocklist: object | object[]
+  channels: object | object[]
+  feeds: object | object[]
+  timezones: object | object[]
+  guides: object | object[]
+  streams: object | object[]
+}

scripts/types/dataProcessor.d.ts

@@ -0,0 +1,27 @@
+import { Collection, Dictionary } from '@freearhey/core'
+
+export type DataProcessorData = {
+  blocklistRecordsGroupedByChannelId: Dictionary
+  subdivisionsGroupedByCountryCode: Dictionary
+  feedsGroupedByChannelId: Dictionary
+  guidesGroupedByStreamId: Dictionary
+  subdivisionsKeyByCode: Dictionary
+  countriesKeyByCode: Dictionary
+  languagesKeyByCode: Dictionary
+  streamsGroupedById: Dictionary
+  categoriesKeyById: Dictionary
+  timezonesKeyById: Dictionary
+  regionsKeyByCode: Dictionary
+  blocklistRecords: Collection
+  channelsKeyById: Dictionary
+  subdivisions: Collection
+  categories: Collection
+  countries: Collection
+  languages: Collection
+  timezones: Collection
+  channels: Collection
+  regions: Collection
+  streams: Collection
+  guides: Collection
+  feeds: Collection
+}

scripts/types/feed.d.ts

@@ -0,0 +1,12 @@
+import { Collection } from '@freearhey/core'
+
+export type FeedData = {
+  channel: string
+  id: string
+  name: string
+  is_main: boolean
+  broadcast_area: Collection
+  languages: Collection
+  timezones: Collection
+  video_format: string
+}

scripts/types/guide.d.ts

@@ -0,0 +1,17 @@
+export type GuideSerializedData = {
+  channelId?: string
+  feedId?: string
+  siteDomain: string
+  siteId: string
+  siteName: string
+  languageCode: string
+}
+
+export type GuideData = {
+  channel: string
+  feed: string
+  site: string
+  site_id: string
+  site_name: string
+  lang: string
+}

scripts/types/language.d.ts

@@ -0,0 +1,9 @@
+export type LanguageSerializedData = {
+  code: string
+  name: string
+}
+
+export type LanguageData = {
+  code: string
+  name: string
+}

scripts/types/region.d.ts

@@ -0,0 +1,13 @@
+export type RegionSerializedData = {
+  code: string
+  name: string
+  countryCodes: string[]
+  countries?: CountrySerializedData[]
+  subdivisions?: SubdivisionSerializedData[]
+}
+
+export type RegionData = {
+  code: string
+  name: string
+  countries: string[]
+}

scripts/types/stream.d.ts

@@ -0,0 +1,10 @@
+export type StreamData = {
+  channel: string | null
+  feed: string | null
+  name: string | null
+  url: string
+  referrer: string | null
+  user_agent: string | null
+  quality: string | null
+  label: string | null
+}

scripts/types/subdivision.d.ts

@@ -0,0 +1,12 @@
+export type SubdivisionSerializedData = {
+  code: string
+  name: string
+  countryCode: string
+  country?: CountrySerializedData
+}
+
+export type SubdivisionData = {
+  code: string
+  name: string
+  country: string
+}