Update scripts

scripts/core/apiClient.ts

@@ -1,59 +1,16 @@
-import { Logger, Storage } from '@freearhey/core'
-import axios, { AxiosInstance, AxiosResponse, AxiosProgressEvent } from 'axios'
-import cliProgress, { MultiBar } from 'cli-progress'
-import numeral from 'numeral'
+import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'
 
 export class ApiClient {
-  progressBar: MultiBar
-  client: AxiosInstance
-  storage: Storage
-  logger: Logger
+  instance: AxiosInstance
 
-  constructor({ logger }: { logger: Logger }) {
-    this.logger = logger
-    this.client = axios.create({
+  constructor() {
+    this.instance = axios.create({
+      baseURL: 'https://iptv-org.github.io/api',
       responseType: 'stream'
     })
-    this.storage = new Storage()
-    this.progressBar = new cliProgress.MultiBar({
-      stopOnComplete: true,
-      hideCursor: true,
-      forceRedraw: true,
-      barsize: 36,
-      format(options, params, payload) {
-        const filename = payload.filename.padEnd(18, ' ')
-        const barsize = options.barsize || 40
-        const percent = (params.progress * 100).toFixed(2)
-        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
-        const total = numeral(params.total).format('0.0 b')
-        const completeSize = Math.round(params.progress * barsize)
-        const incompleteSize = barsize - completeSize
-        const bar =
-          options.barCompleteString && options.barIncompleteString
-            ? options.barCompleteString.substr(0, completeSize) +
-              options.barGlue +
-              options.barIncompleteString.substr(0, incompleteSize)
-            : '-'.repeat(barsize)
-
-        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
-      }
-    })
   }
 
-  async download(filename: string) {
-    const stream = await this.storage.createStream(`temp/data/${filename}`)
-
-    const bar = this.progressBar.create(0, 0, { filename })
-
-    this.client
-      .get(`https://iptv-org.github.io/api/${filename}`, {
-        onDownloadProgress({ total, loaded, rate }: AxiosProgressEvent) {
-          if (total) bar.setTotal(total)
-          bar.update(loaded, { speed: rate })
-        }
-      })
-      .then((response: AxiosResponse) => {
-        response.data.pipe(stream)
-      })
+  get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
+    return this.instance.get(url, options)
   }
 }

scripts/core/dataLoader.ts

@@ -0,0 +1,100 @@
+import { ApiClient } from './apiClient'
+import { Storage } from '@freearhey/core'
+import cliProgress, { MultiBar } from 'cli-progress'
+import numeral from 'numeral'
+import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
+
+export class DataLoader {
+  client: ApiClient
+  storage: Storage
+  progressBar: MultiBar
+
+  constructor(props: DataLoaderProps) {
+    this.client = new ApiClient()
+    this.storage = props.storage
+    this.progressBar = new cliProgress.MultiBar({
+      stopOnComplete: true,
+      hideCursor: true,
+      forceRedraw: true,
+      barsize: 36,
+      format(options, params, payload) {
+        const filename = payload.filename.padEnd(18, ' ')
+        const barsize = options.barsize || 40
+        const percent = (params.progress * 100).toFixed(2)
+        const speed = payload.speed ? numeral(payload.speed).format('0.0 b') + '/s' : 'N/A'
+        const total = numeral(params.total).format('0.0 b')
+        const completeSize = Math.round(params.progress * barsize)
+        const incompleteSize = barsize - completeSize
+        const bar =
+          options.barCompleteString && options.barIncompleteString
+            ? options.barCompleteString.substr(0, completeSize) +
+              options.barGlue +
+              options.barIncompleteString.substr(0, incompleteSize)
+            : '-'.repeat(barsize)
+
+        return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
+      }
+    })
+  }
+
+  async load(): Promise<DataLoaderData> {
+    const [
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    ] = await Promise.all([
+      this.storage.json('countries.json'),
+      this.storage.json('regions.json'),
+      this.storage.json('subdivisions.json'),
+      this.storage.json('languages.json'),
+      this.storage.json('categories.json'),
+      this.storage.json('blocklist.json'),
+      this.storage.json('channels.json'),
+      this.storage.json('feeds.json'),
+      this.storage.json('timezones.json'),
+      this.storage.json('guides.json'),
+      this.storage.json('streams.json')
+    ])
+
+    return {
+      countries,
+      regions,
+      subdivisions,
+      languages,
+      categories,
+      blocklist,
+      channels,
+      feeds,
+      timezones,
+      guides,
+      streams
+    }
+  }
+
+  async download(filename: string) {
+    if (!this.storage || !this.progressBar) return
+
+    const stream = await this.storage.createStream(filename)
+    const progressBar = this.progressBar.create(0, 0, { filename })
+
+    this.client
+      .get(filename, {
+        responseType: 'stream',
+        onDownloadProgress({ total, loaded, rate }) {
+          if (total) progressBar.setTotal(total)
+          progressBar.update(loaded, { speed: rate })
+        }
+      })
+      .then(response => {
+        response.data.pipe(stream)
+      })
+  }
+}

scripts/core/dataProcessor.ts

@@ -0,0 +1,110 @@
+import { DataLoaderData } from '../types/dataLoader'
+import { Collection } from '@freearhey/core'
+import {
+  BlocklistRecord,
+  Subdivision,
+  Category,
+  Language,
+  Timezone,
+  Channel,
+  Country,
+  Region,
+  Stream,
+  Guide,
+  Feed
+} from '../models'
+
+export class DataProcessor {
+  constructor() {}
+
+  process(data: DataLoaderData) {
+    const categories = new Collection(data.categories).map(data => new Category(data))
+    const categoriesKeyById = categories.keyBy((category: Category) => category.id)
+
+    const subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data))
+    const subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
+    const subdivisionsGroupedByCountryCode = subdivisions.groupBy(
+      (subdivision: Subdivision) => subdivision.countryCode
+    )
+
+    let regions = new Collection(data.regions).map(data => new Region(data))
+    const regionsKeyByCode = regions.keyBy((region: Region) => region.code)
+
+    const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data))
+    const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy(
+      (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId
+    )
+
+    const streams = new Collection(data.streams).map(data => new Stream(data))
+    const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
+
+    const guides = new Collection(data.guides).map(data => new Guide(data))
+    const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
+
+    const languages = new Collection(data.languages).map(data => new Language(data))
+    const languagesKeyByCode = languages.keyBy((language: Language) => language.code)
+
+    const countries = new Collection(data.countries).map(data =>
+      new Country(data)
+        .withRegions(regions)
+        .withLanguage(languagesKeyByCode)
+        .withSubdivisions(subdivisionsGroupedByCountryCode)
+    )
+    const countriesKeyByCode = countries.keyBy((country: Country) => country.code)
+
+    regions = regions.map((region: Region) => region.withCountries(countriesKeyByCode))
+
+    const timezones = new Collection(data.timezones).map(data =>
+      new Timezone(data).withCountries(countriesKeyByCode)
+    )
+    const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id)
+
+    let channels = new Collection(data.channels).map(data =>
+      new Channel(data)
+        .withCategories(categoriesKeyById)
+        .withCountry(countriesKeyByCode)
+        .withSubdivision(subdivisionsKeyByCode)
+        .withCategories(categoriesKeyById)
+    )
+    const channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
+
+    let feeds = new Collection(data.feeds).map(data =>
+      new Feed(data)
+        .withChannel(channelsKeyById)
+        .withLanguages(languagesKeyByCode)
+        .withTimezones(timezonesKeyById)
+        .withBroadcastCountries(countriesKeyByCode, regionsKeyByCode, subdivisionsKeyByCode)
+        .withBroadcastRegions(regions)
+        .withBroadcastSubdivisions(subdivisionsKeyByCode)
+    )
+    const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
+
+    channels = channels.map((channel: Channel) => channel.withFeeds(feedsGroupedByChannelId))
+
+    return {
+      blocklistRecordsGroupedByChannelId,
+      subdivisionsGroupedByCountryCode,
+      feedsGroupedByChannelId,
+      guidesGroupedByStreamId,
+      subdivisionsKeyByCode,
+      countriesKeyByCode,
+      languagesKeyByCode,
+      streamsGroupedById,
+      categoriesKeyById,
+      timezonesKeyById,
+      regionsKeyByCode,
+      blocklistRecords,
+      channelsKeyById,
+      subdivisions,
+      categories,
+      countries,
+      languages,
+      timezones,
+      channels,
+      regions,
+      streams,
+      guides,
+      feeds
+    }
+  }
+}

scripts/core/index.ts

@@ -1,11 +1,13 @@
-export * from './playlistParser'
-export * from './numberParser'
-export * from './logParser'
-export * from './markdown'
+export * from './apiClient'
+export * from './cliTable'
+export * from './dataProcessor'
+export * from './dataLoader'
+export * from './htmlTable'
+export * from './issueData'
 export * from './issueLoader'
 export * from './issueParser'
-export * from './htmlTable'
-export * from './apiClient'
-export * from './issueData'
+export * from './logParser'
+export * from './markdown'
+export * from './numberParser'
+export * from './playlistParser'
 export * from './streamTester'
-export * from './cliTable'

scripts/core/playlistParser.ts

@@ -5,18 +5,18 @@ import { Stream } from '../models'
 type PlaylistPareserProps = {
   storage: Storage
   feedsGroupedByChannelId: Dictionary
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
 }
 
 export class PlaylistParser {
   storage: Storage
   feedsGroupedByChannelId: Dictionary
-  channelsGroupedById: Dictionary
+  channelsKeyById: Dictionary
 
-  constructor({ storage, feedsGroupedByChannelId, channelsGroupedById }: PlaylistPareserProps) {
+  constructor({ storage, feedsGroupedByChannelId, channelsKeyById }: PlaylistPareserProps) {
     this.storage = storage
     this.feedsGroupedByChannelId = feedsGroupedByChannelId
-    this.channelsGroupedById = channelsGroupedById
+    this.channelsKeyById = channelsKeyById
   }
 
   async parse(files: string[]): Promise<Collection> {
@@ -35,9 +35,10 @@ export class PlaylistParser {
     const parsed: parser.Playlist = parser.parse(content)
 
     const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
-      const stream = new Stream(data)
+      const stream = new Stream()
+        .fromPlaylistItem(data)
         .withFeed(this.feedsGroupedByChannelId)
-        .withChannel(this.channelsGroupedById)
+        .withChannel(this.channelsKeyById)
         .setFilepath(filepath)
 
       return stream