From 5dd131e2d37e17e00759408fe8bc9d5b4b6699c4 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Wed, 2 Apr 2025 07:13:39 +0300 Subject: [PATCH] Update scripts --- scripts/commands/api/generate.ts | 11 +- scripts/commands/api/load.ts | 1 + scripts/commands/channels/.gitignore | 1 - scripts/commands/channels/edit.ts | 216 ++++++++++++++++---------- scripts/commands/channels/validate.ts | 18 +-- scripts/commands/sites/init.ts | 4 +- scripts/commands/sites/update.ts | 23 ++- scripts/core/apiChannel.ts | 79 ---------- scripts/core/index.ts | 1 - scripts/core/issueLoader.ts | 21 +-- scripts/core/queueCreator.ts | 9 +- scripts/models/channel.ts | 56 +++++++ scripts/models/feed.ts | 34 ++++ scripts/models/index.ts | 2 + scripts/templates/_readme.md | 2 +- 15 files changed, 274 insertions(+), 204 deletions(-) delete mode 100644 scripts/commands/channels/.gitignore delete mode 100644 scripts/core/apiChannel.ts create mode 100644 scripts/models/channel.ts create mode 100644 scripts/models/feed.ts diff --git a/scripts/commands/api/generate.ts b/scripts/commands/api/generate.ts index 43634f8c..b43bc84b 100644 --- a/scripts/commands/api/generate.ts +++ b/scripts/commands/api/generate.ts @@ -2,10 +2,11 @@ import { Logger, Storage, Collection } from '@freearhey/core' import { ChannelsParser } from '../../core' import path from 'path' import { SITES_DIR, API_DIR } from '../../constants' -import { Channel } from 'epg-grabber' +import epgGrabber from 'epg-grabber' type OutputItem = { channel: string | null + feed: string | null site: string site_id: string site_name: string @@ -31,9 +32,13 @@ async function main() { logger.info(` found ${parsedChannels.count()} channel(s)`) - const output = parsedChannels.map((channel: Channel): OutputItem => { + const output = parsedChannels.map((channel: epgGrabber.Channel): OutputItem => { + const xmltv_id = channel.xmltv_id || '' + const [channelId, feedId] = xmltv_id.split('@') + return { - channel: channel.xmltv_id || null, + channel: channelId || null, + feed: feedId || null, site: channel.site || '', site_id: channel.site_id || '', site_name: channel.name, diff --git a/scripts/commands/api/load.ts b/scripts/commands/api/load.ts index 9e0cd0a6..28d19912 100644 --- a/scripts/commands/api/load.ts +++ b/scripts/commands/api/load.ts @@ -7,6 +7,7 @@ async function main() { const requests = [ client.download('channels.json'), + client.download('feeds.json'), client.download('countries.json'), client.download('regions.json'), client.download('subdivisions.json') diff --git a/scripts/commands/channels/.gitignore b/scripts/commands/channels/.gitignore deleted file mode 100644 index c7b0476d..00000000 --- a/scripts/commands/channels/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/replace.ts \ No newline at end of file diff --git a/scripts/commands/channels/edit.ts b/scripts/commands/channels/edit.ts index 484ba08e..87e81999 100644 --- a/scripts/commands/channels/edit.ts +++ b/scripts/commands/channels/edit.ts @@ -1,12 +1,16 @@ +import { Storage, Collection, Logger, Dictionary } from '@freearhey/core' +import { select, input } from '@inquirer/prompts' +import { ChannelsParser, XML } from '../../core' +import { Channel, Feed } from '../../models' import { DATA_DIR } from '../../constants' -import { Storage, Collection, Logger } from '@freearhey/core' -import { ChannelsParser, XML, ApiChannel } from '../../core' -import { Channel } from 'epg-grabber' import nodeCleanup from 'node-cleanup' -import { program } from 'commander' -import inquirer, { QuestionCollection } from 'inquirer' -import Fuse from 'fuse.js' +import epgGrabber from 'epg-grabber' +import { Command } from 'commander' import readline from 'readline' +import Fuse from 'fuse.js' + +type ChoiceValue = { type: string; value?: Feed | Channel } +type Choice = { name: string; short?: string; value: ChoiceValue } if (process.platform === 'win32') { readline @@ -19,105 +23,159 @@ if (process.platform === 'win32') { }) } +const program = new Command() + program.argument('', 'Path to *.channels.xml file to edit').parse(process.argv) const filepath = program.args[0] - const logger = new Logger() const storage = new Storage() -let channels = new Collection() +let parsedChannels = new Collection() -async function main() { +main(filepath) +nodeCleanup(() => { + save(filepath) +}) + +export default async function main(filepath: string) { if (!(await storage.exists(filepath))) { throw new Error(`File "${filepath}" does not exists`) } const parser = new ChannelsParser({ storage }) - channels = await parser.parse(filepath) + parsedChannels = await parser.parse(filepath) const dataStorage = new Storage(DATA_DIR) - const channelsContent = await dataStorage.json('channels.json') - const searchIndex = new Fuse(channelsContent, { keys: ['name', 'alt_names'], threshold: 0.4 }) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const feedsData = await dataStorage.json('feeds.json') + const feeds = new Collection(feedsData).map(data => new Feed(data)) + const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId) - for (const channel of channels.all()) { + const searchIndex: Fuse = new Fuse(channels.all(), { + keys: ['name', 'alt_names'], + threshold: 0.4 + }) + + for (const channel of parsedChannels.all()) { if (channel.xmltv_id) continue - const question: QuestionCollection = { - name: 'option', - message: `Select xmltv_id for "${channel.name}" (${channel.site_id}):`, - type: 'list', - choices: getOptions(searchIndex, channel), - pageSize: 10 + try { + channel.xmltv_id = await selectChannel(channel, searchIndex, feedsGroupedByChannelId) + } catch { + break } - - await inquirer.prompt(question).then(async selected => { - switch (selected.option) { - case 'Type...': - const input = await getInput(channel) - channel.xmltv_id = input.xmltv_id - break - case 'Skip': - channel.xmltv_id = '-' - break - default: - const [, xmltv_id] = selected.option - .replace(/ \[.*\]/, '') - .split('|') - .map((i: string) => i.trim()) - channel.xmltv_id = xmltv_id - break - } - }) } - channels.forEach((channel: Channel) => { + parsedChannels.forEach((channel: epgGrabber.Channel) => { if (channel.xmltv_id === '-') { channel.xmltv_id = '' } }) } -main() +async function selectChannel( + channel: epgGrabber.Channel, + searchIndex: Fuse, + feedsGroupedByChannelId: Dictionary +): Promise { + const similarChannels = searchIndex + .search(channel.name) + .map((result: { item: Channel }) => result.item) -function save() { + const selected: ChoiceValue = await select({ + message: `Select channel ID for "${channel.name}" (${channel.site_id}):`, + choices: getChannelChoises(new Collection(similarChannels)), + pageSize: 10 + }) + + switch (selected.type) { + case 'skip': + return '-' + case 'type': { + const typedChannelId = await input({ message: ' Channel ID:' }) + const typedFeedId = await input({ message: ' Feed ID:', default: 'SD' }) + return [typedChannelId, typedFeedId].join('@') + } + case 'channel': { + const selectedChannel = selected.value + if (!selectedChannel) return '' + const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId) + return [selectedChannel.id, selectedFeedId].join('@') + } + } + + return '' +} + +async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise { + const channelFeeds = feedsGroupedByChannelId.get(channelId) || [] + if (channelFeeds.length <= 1) return '' + + const selected: ChoiceValue = await select({ + message: `Select feed ID for "${channelId}":`, + choices: getFeedChoises(channelFeeds), + pageSize: 10 + }) + + switch (selected.type) { + case 'type': + return await input({ message: ' Feed ID:' }) + case 'feed': + const selectedFeed = selected.value + if (!selectedFeed) return '' + return selectedFeed.id + } + + return '' +} + +function getChannelChoises(channels: Collection): Choice[] { + const choises: Choice[] = [] + + channels.forEach((channel: Channel) => { + const names = [channel.name, ...channel.altNames.all()].join(', ') + + choises.push({ + value: { + type: 'channel', + value: channel + }, + name: `${channel.id} (${names})`, + short: `${channel.id}` + }) + }) + + choises.push({ name: 'Type...', value: { type: 'type' } }) + choises.push({ name: 'Skip', value: { type: 'skip' } }) + + return choises +} + +function getFeedChoises(feeds: Collection): Choice[] { + const choises: Choice[] = [] + + feeds.forEach((feed: Feed) => { + let name = `${feed.id} (${feed.name})` + if (feed.isMain) name += ' [main]' + + choises.push({ + value: { + type: 'feed', + value: feed + }, + name, + short: feed.id + }) + }) + + choises.push({ name: 'Type...', value: { type: 'type' } }) + + return choises +} + +function save(filepath: string) { if (!storage.existsSync(filepath)) return - - const xml = new XML(channels) - + const xml = new XML(parsedChannels) storage.saveSync(filepath, xml.toString()) - logger.info(`\nFile '${filepath}' successfully saved`) } - -nodeCleanup(() => { - save() -}) - -async function getInput(channel: Channel) { - const name = channel.name.trim() - const input = await inquirer.prompt([ - { - name: 'xmltv_id', - message: ' xmltv_id:', - type: 'input' - } - ]) - - return { name, xmltv_id: input['xmltv_id'] } -} - -function getOptions(index, channel: Channel) { - const similar = index.search(channel.name).map(result => new ApiChannel(result.item)) - - const variants = new Collection() - similar.forEach((_channel: ApiChannel) => { - const altNames = _channel.altNames.notEmpty() ? ` (${_channel.altNames.join(',')})` : '' - const closed = _channel.closed ? ` [closed:${_channel.closed}]` : '' - const replacedBy = _channel.replacedBy ? `[replaced_by:${_channel.replacedBy}]` : '' - - variants.add(`${_channel.name}${altNames} | ${_channel.id}${closed}${replacedBy}`) - }) - variants.add('Type...') - variants.add('Skip') - - return variants.all() -} diff --git a/scripts/commands/channels/validate.ts b/scripts/commands/channels/validate.ts index 05b78b78..47e6088c 100644 --- a/scripts/commands/channels/validate.ts +++ b/scripts/commands/channels/validate.ts @@ -1,10 +1,11 @@ import { Storage, Collection, Dictionary, File } from '@freearhey/core' -import { ChannelsParser, ApiChannel } from '../../core' +import { ChannelsParser } from '../../core' +import { Channel } from '../../models' import { program } from 'commander' import chalk from 'chalk' import langs from 'langs' import { DATA_DIR } from '../../constants' -import { Channel } from 'epg-grabber' +import epgGrabber from 'epg-grabber' program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv) @@ -21,8 +22,9 @@ async function main() { const parser = new ChannelsParser({ storage: new Storage() }) const dataStorage = new Storage(DATA_DIR) - const channelsContent = await dataStorage.json('channels.json') - const channels = new Collection(channelsContent).map(data => new ApiChannel(data)) + const channelsData = await dataStorage.json('channels.json') + const channels = new Collection(channelsData).map(data => new Channel(data)) + const channelsGroupedById = channels.groupBy((channel: Channel) => channel.id) let totalFiles = 0 let totalErrors = 0 @@ -37,7 +39,7 @@ async function main() { const bufferBySiteId = new Dictionary() const errors: ValidationError[] = [] - parsedChannels.forEach((channel: Channel) => { + parsedChannels.forEach((channel: epgGrabber.Channel) => { const bufferId: string = channel.site_id if (bufferBySiteId.missing(bufferId)) { bufferBySiteId.set(bufferId, true) @@ -52,10 +54,8 @@ async function main() { } if (!channel.xmltv_id) return - - const foundChannel = channels.first( - (_channel: ApiChannel) => _channel.id === channel.xmltv_id - ) + const [channelId] = channel.xmltv_id.split('@') + const foundChannel = channelsGroupedById.get(channelId) if (!foundChannel) { errors.push({ type: 'wrong_xmltv_id', ...channel }) totalErrors++ diff --git a/scripts/commands/sites/init.ts b/scripts/commands/sites/init.ts index bf34e1ad..44df0c95 100644 --- a/scripts/commands/sites/init.ts +++ b/scripts/commands/sites/init.ts @@ -1,8 +1,8 @@ import { Logger, Storage } from '@freearhey/core' -import { program } from 'commander' import { SITES_DIR } from '../../constants' -import fs from 'fs-extra' import { pathToFileURL } from 'node:url' +import { program } from 'commander' +import fs from 'fs-extra' program.argument('', 'Domain name of the site').parse(process.argv) diff --git a/scripts/commands/sites/update.ts b/scripts/commands/sites/update.ts index 9ffc0b4b..a2cf5cd7 100644 --- a/scripts/commands/sites/update.ts +++ b/scripts/commands/sites/update.ts @@ -1,8 +1,8 @@ -import { Channel } from 'epg-grabber' -import { Logger, Storage, Collection } from '@freearhey/core' import { IssueLoader, HTMLTable, ChannelsParser } from '../../core' -import { Issue, Site } from '../../models' +import { Logger, Storage, Collection } from '@freearhey/core' import { SITES_DIR, ROOT_DIR } from '../../constants' +import { Issue, Site } from '../../models' +import { Channel } from 'epg-grabber' async function main() { const logger = new Logger({ disabled: true }) @@ -15,11 +15,17 @@ async function main() { const folders = await sitesStorage.list('*/') logger.info('loading issues...') - const issues = await loadIssues(loader) + const issues = await loader.load() logger.info('putting the data together...') + const brokenGuideReports = issues.filter(issue => + issue.labels.find((label: string) => label === 'broken guide') + ) for (const domain of folders) { - const filteredIssues = issues.filter((issue: Issue) => domain === issue.data.get('site')) + const filteredIssues = brokenGuideReports.filter( + (issue: Issue) => domain === issue.data.get('site') + ) + const site = new Site({ domain, issues: filteredIssues @@ -62,10 +68,3 @@ async function main() { } main() - -async function loadIssues(loader: IssueLoader) { - const issuesWithStatusWarning = await loader.load({ labels: ['broken guide', 'status:warning'] }) - const issuesWithStatusDown = await loader.load({ labels: ['broken guide', 'status:down'] }) - - return issuesWithStatusWarning.concat(issuesWithStatusDown) -} diff --git a/scripts/core/apiChannel.ts b/scripts/core/apiChannel.ts deleted file mode 100644 index fd1fcd69..00000000 --- a/scripts/core/apiChannel.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Collection } from '@freearhey/core' - -type ApiChannelProps = { - id: string - name: string - alt_names: string[] - network: string - owners: string[] - country: string - subdivision: string - city: string - broadcast_area: string[] - languages: string[] - categories: string[] - is_nsfw: boolean - launched: string - closed: string - replaced_by: string - website: string - logo: string -} - -export class ApiChannel { - id: string - name: string - altNames: Collection - network: string - owners: Collection - country: string - subdivision: string - city: string - broadcastArea: Collection - languages: Collection - categories: Collection - isNSFW: boolean - launched: string - closed: string - replacedBy: string - website: string - logo: string - - constructor({ - id, - name, - alt_names, - network, - owners, - country, - subdivision, - city, - broadcast_area, - languages, - categories, - is_nsfw, - launched, - closed, - replaced_by, - website, - logo - }: ApiChannelProps) { - this.id = id - this.name = name - this.altNames = new Collection(alt_names) - this.network = network - this.owners = new Collection(owners) - this.country = country - this.subdivision = subdivision - this.city = city - this.broadcastArea = new Collection(broadcast_area) - this.languages = new Collection(languages) - this.categories = new Collection(categories) - this.isNSFW = is_nsfw - this.launched = launched - this.closed = closed - this.replacedBy = replaced_by - this.website = website - this.logo = logo - } -} diff --git a/scripts/core/index.ts b/scripts/core/index.ts index db3e75a5..91a684ff 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -7,7 +7,6 @@ export * from './job' export * from './queue' export * from './guideManager' export * from './guide' -export * from './apiChannel' export * from './apiClient' export * from './queueCreator' export * from './issueLoader' diff --git a/scripts/core/issueLoader.ts b/scripts/core/issueLoader.ts index 8d9b6479..4aa1cceb 100644 --- a/scripts/core/issueLoader.ts +++ b/scripts/core/issueLoader.ts @@ -1,27 +1,22 @@ -import { Collection } from '@freearhey/core' import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods' import { paginateRest } from '@octokit/plugin-paginate-rest' +import { TESTING, OWNER, REPO } from '../constants' +import { Collection } from '@freearhey/core' import { Octokit } from '@octokit/core' import { IssueParser } from './' -import { TESTING, OWNER, REPO } from '../constants' const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods) const octokit = new CustomOctokit() export class IssueLoader { - async load({ labels }: { labels: string[] | string }) { - labels = Array.isArray(labels) ? labels.join(',') : labels + async load(props?: { labels: string[] | string }) { + let labels = '' + if (props && props.labels) { + labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels + } let issues: object[] = [] if (TESTING) { - switch (labels) { - case 'broken guide,status:warning': - issues = (await import('../../tests/__data__/input/issues/broken_guide_warning.mjs')) - .default - break - case 'broken guide,status:down': - issues = (await import('../../tests/__data__/input/issues/broken_guide_down.mjs')).default - break - } + issues = (await import('../../tests/__data__/input/sites_update/issues.mjs')).default } else { issues = await octokit.paginate(octokit.rest.issues.listForRepo, { owner: OWNER, diff --git a/scripts/core/queueCreator.ts b/scripts/core/queueCreator.ts index 1a906f9d..d1ebf381 100644 --- a/scripts/core/queueCreator.ts +++ b/scripts/core/queueCreator.ts @@ -1,9 +1,10 @@ import { Storage, Collection, DateTime, Logger } from '@freearhey/core' -import { ChannelsParser, ConfigLoader, ApiChannel, Queue } from './' +import { ChannelsParser, ConfigLoader, Queue } from './' import { SITES_DIR, DATA_DIR } from '../constants' import { SiteConfig } from 'epg-grabber' import path from 'path' import { GrabOptions } from '../commands/epg/grab' +import { Channel } from '../models' type QueueCreatorProps = { logger: Logger @@ -32,7 +33,7 @@ export class QueueCreator { async create(): Promise { const channelsContent = await this.dataStorage.json('channels.json') - const channels = new Collection(channelsContent).map(data => new ApiChannel(data)) + const channels = new Collection(channelsContent).map(data => new Channel(data)) const queue = new Queue() for (const channel of this.parsedChannels.all()) { @@ -44,8 +45,8 @@ export class QueueCreator { if (channel.xmltv_id) { if (!channel.icon) { - const found: ApiChannel = channels.first( - (_channel: ApiChannel) => _channel.id === channel.xmltv_id + const found: Channel = channels.first( + (_channel: Channel) => _channel.id === channel.xmltv_id ) if (found) { diff --git a/scripts/models/channel.ts b/scripts/models/channel.ts new file mode 100644 index 00000000..281c3914 --- /dev/null +++ b/scripts/models/channel.ts @@ -0,0 +1,56 @@ +import { Collection } from '@freearhey/core' + +type ChannelData = { + id: string + name: string + alt_names: string[] + network: string + owners: Collection + country: string + subdivision: string + city: string + categories: Collection + is_nsfw: boolean + launched: string + closed: string + replaced_by: string + website: string + logo: string +} + +export class Channel { + id: string + name: string + altNames: Collection + network?: string + owners: Collection + countryCode: string + subdivisionCode?: string + cityName?: string + categoryIds: Collection + categories?: Collection + isNSFW: boolean + launched?: string + closed?: string + replacedBy?: string + website?: string + logo: string + + constructor(data: ChannelData) { + this.id = data.id + this.name = data.name + this.altNames = new Collection(data.alt_names) + this.network = data.network || undefined + this.owners = new Collection(data.owners) + this.countryCode = data.country + this.subdivisionCode = data.subdivision || undefined + this.cityName = data.city || undefined + this.categoryIds = new Collection(data.categories) + this.isNSFW = data.is_nsfw + this.launched = data.launched || undefined + this.closed = data.closed || undefined + this.replacedBy = data.replaced_by || undefined + this.website = data.website || undefined + this.logo = data.logo + } +} diff --git a/scripts/models/feed.ts b/scripts/models/feed.ts new file mode 100644 index 00000000..4941d4f7 --- /dev/null +++ b/scripts/models/feed.ts @@ -0,0 +1,34 @@ +import { Collection } from '@freearhey/core' + +type FeedData = { + channel: string + id: string + name: string + is_main: boolean + broadcast_area: Collection + languages: Collection + timezones: Collection + video_format: string +} + +export class Feed { + channelId: string + id: string + name: string + isMain: boolean + broadcastAreaCodes: Collection + languageCodes: Collection + timezoneIds: Collection + videoFormat: string + + constructor(data: FeedData) { + this.channelId = data.channel + this.id = data.id + this.name = data.name + this.isMain = data.is_main + this.broadcastAreaCodes = new Collection(data.broadcast_area) + this.languageCodes = new Collection(data.languages) + this.timezoneIds = new Collection(data.timezones) + this.videoFormat = data.video_format + } +} diff --git a/scripts/models/index.ts b/scripts/models/index.ts index f8188a40..1b602c54 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -1,2 +1,4 @@ export * from './issue' export * from './site' +export * from './channel' +export * from './feed' diff --git a/scripts/templates/_readme.md b/scripts/templates/_readme.md index 3260ab62..4aa94527 100644 --- a/scripts/templates/_readme.md +++ b/scripts/templates/_readme.md @@ -1,6 +1,6 @@ # -https://example.com +https:// ### Download the guide