mirror of
https://github.com/iptv-org/epg.git
synced 2025-05-09 08:30:06 -04:00
Update scripts
This commit is contained in:
freearhey
parent
d681fcb3d5
commit
8e363d0e83
19 changed files with 562 additions and 141 deletions
|
|
@ -1,19 +1,24 @@
|
|||
import { Logger } from '@freearhey/core'
|
||||
import { ApiClient } from '../../core'
|
||||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage } from '@freearhey/core'
|
||||
import { DataLoader } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
const client = new ApiClient({ logger })
|
||||
const storage = new Storage(DATA_DIR)
|
||||
const loader = new DataLoader({ storage })
|
||||
|
||||
const requests = [
|
||||
client.download('channels.json'),
|
||||
client.download('feeds.json'),
|
||||
client.download('countries.json'),
|
||||
client.download('regions.json'),
|
||||
client.download('subdivisions.json')
|
||||
]
|
||||
|
||||
await Promise.all(requests)
|
||||
await Promise.all([
|
||||
loader.download('blocklist.json'),
|
||||
loader.download('categories.json'),
|
||||
loader.download('channels.json'),
|
||||
loader.download('countries.json'),
|
||||
loader.download('languages.json'),
|
||||
loader.download('regions.json'),
|
||||
loader.download('subdivisions.json'),
|
||||
loader.download('feeds.json'),
|
||||
loader.download('timezones.json'),
|
||||
loader.download('guides.json'),
|
||||
loader.download('streams.json')
|
||||
])
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ import { ChannelsParser, XML } from '../../core'
|
|||
import { Channel, Feed } from '../../models'
|
||||
import { DATA_DIR } from '../../constants'
|
||||
import nodeCleanup from 'node-cleanup'
|
||||
import epgGrabber from 'epg-grabber'
|
||||
import { Command } from 'commander'
|
||||
import readline from 'readline'
|
||||
import Fuse from 'fuse.js'
|
||||
import sjs from '@freearhey/search-js'
|
||||
import { DataProcessor, DataLoader } from '../../core'
|
||||
import type { DataLoaderData } from '../../types/dataLoader'
|
||||
import type { DataProcessorData } from '../../types/dataProcessor'
|
||||
import epgGrabber from 'epg-grabber'
|
||||
import { ChannelSearchableData } from '../../types/channel'
|
||||
|
||||
type ChoiceValue = { type: string; value?: Feed | Channel }
|
||||
type Choice = { name: string; short?: string; value: ChoiceValue }
|
||||
type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
readline
|
||||
|
|
@ -42,31 +46,48 @@ export default async function main(filepath: string) {
|
|||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
logger.info('loading data from api...')
|
||||
const processor = new DataProcessor()
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const loader = new DataLoader({ storage: dataStorage })
|
||||
const data: DataLoaderData = await loader.load()
|
||||
const { feedsGroupedByChannelId, channels, channelsKeyById }: DataProcessorData =
|
||||
processor.process(data)
|
||||
|
||||
logger.info('loading channels...')
|
||||
const parser = new ChannelsParser({ storage })
|
||||
parsedChannels = await parser.parse(filepath)
|
||||
const parsedChannelsWithoutId = parsedChannels.filter(
|
||||
(channel: epgGrabber.Channel) => !channel.xmltv_id
|
||||
)
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsData = await dataStorage.json('channels.json')
|
||||
const channels = new Collection(channelsData).map(data => new Channel(data))
|
||||
const feedsData = await dataStorage.json('feeds.json')
|
||||
const feeds = new Collection(feedsData).map(data => new Feed(data))
|
||||
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
|
||||
logger.info(
|
||||
`found ${parsedChannels.count()} channels (including ${parsedChannelsWithoutId.count()} without ID)`
|
||||
)
|
||||
|
||||
const searchIndex: Fuse<Channel> = new Fuse(channels.all(), {
|
||||
keys: ['name', 'alt_names'],
|
||||
threshold: 0.4
|
||||
logger.info('creating search index...')
|
||||
const items = channels.map((channel: Channel) => channel.getSearchable()).all()
|
||||
const searchIndex = sjs.createIndex(items, {
|
||||
searchable: ['name', 'altNames', 'guideNames', 'streamNames', 'feedFullNames']
|
||||
})
|
||||
|
||||
for (const channel of parsedChannels.all()) {
|
||||
if (channel.xmltv_id) continue
|
||||
logger.info('starting...\n')
|
||||
|
||||
for (const parsedChannel of parsedChannelsWithoutId.all()) {
|
||||
try {
|
||||
channel.xmltv_id = await selectChannel(channel, searchIndex, feedsGroupedByChannelId)
|
||||
} catch {
|
||||
parsedChannel.xmltv_id = await selectChannel(
|
||||
parsedChannel,
|
||||
searchIndex,
|
||||
feedsGroupedByChannelId,
|
||||
channelsKeyById
|
||||
)
|
||||
} catch (err) {
|
||||
logger.info(err.message)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
parsedChannels.forEach((channel: epgGrabber.Channel) => {
|
||||
parsedChannelsWithoutId.forEach((channel: epgGrabber.Channel) => {
|
||||
if (channel.xmltv_id === '-') {
|
||||
channel.xmltv_id = ''
|
||||
}
|
||||
|
|
@ -75,12 +96,14 @@ export default async function main(filepath: string) {
|
|||
|
||||
async function selectChannel(
|
||||
channel: epgGrabber.Channel,
|
||||
searchIndex: Fuse<Channel>,
|
||||
feedsGroupedByChannelId: Dictionary
|
||||
searchIndex,
|
||||
feedsGroupedByChannelId: Dictionary,
|
||||
channelsKeyById: Dictionary
|
||||
): Promise<string> {
|
||||
const query = escapeRegex(channel.name)
|
||||
const similarChannels = searchIndex
|
||||
.search(channel.name)
|
||||
.map((result: { item: Channel }) => result.item)
|
||||
.search(query)
|
||||
.map((item: ChannelSearchableData) => channelsKeyById.get(item.id))
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select channel ID for "${channel.name}" (${channel.site_id}):`,
|
||||
|
|
@ -93,13 +116,16 @@ async function selectChannel(
|
|||
return '-'
|
||||
case 'type': {
|
||||
const typedChannelId = await input({ message: ' Channel ID:' })
|
||||
const typedFeedId = await input({ message: ' Feed ID:', default: 'SD' })
|
||||
return [typedChannelId, typedFeedId].join('@')
|
||||
if (!typedChannelId) return ''
|
||||
const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
|
||||
if (selectedFeedId === '-') return typedChannelId
|
||||
return [typedChannelId, selectedFeedId].join('@')
|
||||
}
|
||||
case 'channel': {
|
||||
const selectedChannel = selected.value
|
||||
if (!selectedChannel) return ''
|
||||
const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
|
||||
if (selectedFeedId === '-') return selectedChannel.id
|
||||
return [selectedChannel.id, selectedFeedId].join('@')
|
||||
}
|
||||
}
|
||||
|
|
@ -108,18 +134,22 @@ async function selectChannel(
|
|||
}
|
||||
|
||||
async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
|
||||
const channelFeeds = feedsGroupedByChannelId.get(channelId) || []
|
||||
if (channelFeeds.length <= 1) return ''
|
||||
const channelFeeds = feedsGroupedByChannelId.has(channelId)
|
||||
? new Collection(feedsGroupedByChannelId.get(channelId))
|
||||
: new Collection()
|
||||
const choices = getFeedChoises(channelFeeds)
|
||||
|
||||
const selected: ChoiceValue = await select({
|
||||
message: `Select feed ID for "${channelId}":`,
|
||||
choices: getFeedChoises(channelFeeds),
|
||||
choices,
|
||||
pageSize: 10
|
||||
})
|
||||
|
||||
switch (selected.type) {
|
||||
case 'skip':
|
||||
return '-'
|
||||
case 'type':
|
||||
return await input({ message: ' Feed ID:' })
|
||||
return await input({ message: ' Feed ID:', default: 'SD' })
|
||||
case 'feed':
|
||||
const selectedFeed = selected.value
|
||||
if (!selectedFeed) return ''
|
||||
|
|
@ -133,7 +163,7 @@ function getChannelChoises(channels: Collection): Choice[] {
|
|||
const choises: Choice[] = []
|
||||
|
||||
channels.forEach((channel: Channel) => {
|
||||
const names = [channel.name, ...channel.altNames.all()].join(', ')
|
||||
const names = new Collection([channel.name, ...channel.getAltNames().all()]).uniq().join(', ')
|
||||
|
||||
choises.push({
|
||||
value: {
|
||||
|
|
@ -163,12 +193,14 @@ function getFeedChoises(feeds: Collection): Choice[] {
|
|||
type: 'feed',
|
||||
value: feed
|
||||
},
|
||||
default: feed.isMain,
|
||||
name,
|
||||
short: feed.id
|
||||
})
|
||||
})
|
||||
|
||||
choises.push({ name: 'Type...', value: { type: 'type' } })
|
||||
choises.push({ name: 'Skip', value: { type: 'skip' } })
|
||||
|
||||
return choises
|
||||
}
|
||||
|
|
@ -179,3 +211,7 @@ function save(filepath: string) {
|
|||
storage.saveSync(filepath, xml.toString())
|
||||
logger.info(`\nFile '${filepath}' successfully saved`)
|
||||
}
|
||||
|
||||
function escapeRegex(string: string) {
|
||||
return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ program
|
|||
)
|
||||
)
|
||||
.addOption(new Option('-o, --output <path>', 'Path to output file').default('guide.xml'))
|
||||
.addOption(new Option('-l, --lang <code>', 'Filter channels by language (ISO 639-1 code)'))
|
||||
.addOption(new Option('-l, --lang <codes>', 'Filter channels by languages (ISO 639-1 codes)'))
|
||||
.addOption(
|
||||
new Option('-t, --timeout <milliseconds>', 'Override the default timeout for each request').env(
|
||||
'TIMEOUT'
|
||||
|
|
@ -90,7 +90,11 @@ async function main() {
|
|||
parsedChannels = parsedChannels.concat(await parser.parse(filepath))
|
||||
}
|
||||
if (options.lang) {
|
||||
parsedChannels = parsedChannels.filter((channel: Channel) => channel.lang === options.lang)
|
||||
parsedChannels = parsedChannels.filter((channel: Channel) => {
|
||||
if (!options.lang || !channel.lang) return true
|
||||
|
||||
return options.lang.includes(channel.lang)
|
||||
})
|
||||
}
|
||||
logger.info(` found ${parsedChannels.count()} channel(s)`)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue