mirror of
https://github.com/iptv-org/epg.git
synced 2025-05-09 08:30:06 -04:00
Merge branch 'master' into patch-2025.01.2
This commit is contained in:
freearhey
commit
a0a48e24ec
313 changed files with 126643 additions and 36500 deletions
123
scripts/commands/channels/edit.ts
Normal file
123
scripts/commands/channels/edit.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage, Collection, Logger } from '@freearhey/core'
|
||||
import { ChannelsParser, XML, ApiChannel } from '../../core'
|
||||
import { Channel } from 'epg-grabber'
|
||||
import nodeCleanup from 'node-cleanup'
|
||||
import { program } from 'commander'
|
||||
import inquirer, { QuestionCollection } from 'inquirer'
|
||||
import Fuse from 'fuse.js'
|
||||
import readline from 'readline'
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
readline
|
||||
.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
})
|
||||
.on('SIGINT', function () {
|
||||
process.emit('SIGINT')
|
||||
})
|
||||
}
|
||||
|
||||
program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
let channels = new Collection()
|
||||
|
||||
async function main() {
|
||||
if (!(await storage.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
const parser = new ChannelsParser({ storage })
|
||||
channels = await parser.parse(filepath)
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const searchIndex = new Fuse(channelsContent, { keys: ['name', 'alt_names'], threshold: 0.4 })
|
||||
|
||||
for (const channel of channels.all()) {
|
||||
if (channel.xmltv_id) continue
|
||||
const question: QuestionCollection = {
|
||||
name: 'option',
|
||||
message: `Select xmltv_id for "${channel.name}" (${channel.site_id}):`,
|
||||
type: 'list',
|
||||
choices: getOptions(searchIndex, channel),
|
||||
pageSize: 10
|
||||
}
|
||||
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Type...':
|
||||
const input = await getInput(channel)
|
||||
channel.xmltv_id = input.xmltv_id
|
||||
break
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
default:
|
||||
const [, xmltv_id] = selected.option
|
||||
.replace(/ \[.*\]/, '')
|
||||
.split('|')
|
||||
.map((i: string) => i.trim())
|
||||
channel.xmltv_id = xmltv_id
|
||||
break
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
channels.forEach((channel: Channel) => {
|
||||
if (channel.xmltv_id === '-') {
|
||||
channel.xmltv_id = ''
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function save() {
|
||||
if (!storage.existsSync(filepath)) return
|
||||
|
||||
const xml = new XML(channels)
|
||||
|
||||
storage.saveSync(filepath, xml.toString())
|
||||
|
||||
logger.info(`\nFile '${filepath}' successfully saved`)
|
||||
}
|
||||
|
||||
nodeCleanup(() => {
|
||||
save()
|
||||
})
|
||||
|
||||
async function getInput(channel: Channel) {
|
||||
const name = channel.name.trim()
|
||||
const input = await inquirer.prompt([
|
||||
{
|
||||
name: 'xmltv_id',
|
||||
message: ' xmltv_id:',
|
||||
type: 'input'
|
||||
}
|
||||
])
|
||||
|
||||
return { name, xmltv_id: input['xmltv_id'] }
|
||||
}
|
||||
|
||||
function getOptions(index, channel: Channel) {
|
||||
const similar = index.search(channel.name).map(result => new ApiChannel(result.item))
|
||||
|
||||
const variants = new Collection()
|
||||
similar.forEach((_channel: ApiChannel) => {
|
||||
const altNames = _channel.altNames.notEmpty() ? ` (${_channel.altNames.join(',')})` : ''
|
||||
const closed = _channel.closed ? ` [closed:${_channel.closed}]` : ''
|
||||
const replacedBy = _channel.replacedBy ? `[replaced_by:${_channel.replacedBy}]` : ''
|
||||
|
||||
variants.add(`${_channel.name}${altNames} | ${_channel.id}${closed}${replacedBy}`)
|
||||
})
|
||||
variants.add('Type...')
|
||||
variants.add('Skip')
|
||||
|
||||
return variants.all()
|
||||
}
|
||||
|
|
@ -1,179 +0,0 @@
|
|||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage, Collection, Dictionary, Logger } from '@freearhey/core'
|
||||
import { ChannelsParser, XML, ApiChannel } from '../../core'
|
||||
import { Channel } from 'epg-grabber'
|
||||
import { transliterate } from 'transliteration'
|
||||
import nodeCleanup from 'node-cleanup'
|
||||
import { program } from 'commander'
|
||||
import inquirer, { QuestionCollection } from 'inquirer'
|
||||
import sj from '@freearhey/search-js'
|
||||
|
||||
program
|
||||
.argument('<filepath>', 'Path to *.channels.xml file to edit')
|
||||
.option('-c, --country <name>', 'Default country (ISO 3166 code)', 'US')
|
||||
.parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
const programOptions = program.opts()
|
||||
const defaultCountry = programOptions.country.toLowerCase()
|
||||
const newLabel = ' [new]'
|
||||
|
||||
let options = new Collection()
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
if (!(await storage.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
const parser = new ChannelsParser({ storage })
|
||||
|
||||
const parsedChannels = await parser.parse(filepath)
|
||||
options = parsedChannels.map((channel: Channel): { channel: Channel; delete: boolean } => {
|
||||
return {
|
||||
channel,
|
||||
delete: false
|
||||
}
|
||||
})
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
|
||||
const channelsIndex = sj.createIndex(channelsContent)
|
||||
|
||||
const buffer = new Dictionary()
|
||||
for (const option of options.all()) {
|
||||
const channel: Channel = option.channel
|
||||
if (channel.xmltv_id) {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.set(`${channel.xmltv_id}/${channel.lang}`, true)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const choices = getOptions(channelsIndex, channel)
|
||||
const question: QuestionCollection = {
|
||||
name: 'option',
|
||||
message: `Choose xmltv_id for "${channel.name}" (${channel.site_id}):`,
|
||||
type: 'list',
|
||||
choices,
|
||||
pageSize: 10
|
||||
}
|
||||
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Overwrite':
|
||||
const input = await getInput(channel)
|
||||
channel.xmltv_id = input.xmltv_id
|
||||
break
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
default:
|
||||
const [, xmltv_id] = selected.option
|
||||
.replace(/ \[.*\]/, '')
|
||||
.split('|')
|
||||
.map((i: string) => i.trim().replace(newLabel, ''))
|
||||
channel.xmltv_id = xmltv_id
|
||||
break
|
||||
}
|
||||
|
||||
const found = buffer.has(`${channel.xmltv_id}/${channel.lang}`)
|
||||
if (found) {
|
||||
const question: QuestionCollection = {
|
||||
name: 'option',
|
||||
message: `"${channel.xmltv_id}" already on the list. Choose an option:`,
|
||||
type: 'list',
|
||||
choices: ['Skip', 'Add', 'Delete'],
|
||||
pageSize: 5
|
||||
}
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
case 'Delete':
|
||||
option.delete = true
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
})
|
||||
} else {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.set(`${channel.xmltv_id}/${channel.lang}`, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function save() {
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
|
||||
if (!storage.existsSync(filepath)) return
|
||||
|
||||
const channels = options
|
||||
.filter((option: { channel: Channel; delete: boolean }) => !option.delete)
|
||||
.map((option: { channel: Channel; delete: boolean }) => option.channel)
|
||||
|
||||
const xml = new XML(channels)
|
||||
|
||||
storage.saveSync(filepath, xml.toString())
|
||||
|
||||
logger.info(`\nFile '${filepath}' successfully saved`)
|
||||
}
|
||||
|
||||
nodeCleanup(() => {
|
||||
save()
|
||||
})
|
||||
|
||||
async function getInput(channel: Channel) {
|
||||
const name = channel.name.trim()
|
||||
const input = await inquirer.prompt([
|
||||
{
|
||||
name: 'xmltv_id',
|
||||
message: ' ID:',
|
||||
type: 'input',
|
||||
default: generateCode(name, defaultCountry)
|
||||
}
|
||||
])
|
||||
|
||||
return { name, xmltv_id: input['xmltv_id'] }
|
||||
}
|
||||
|
||||
function getOptions(channelsIndex, channel: Channel) {
|
||||
const channelId = generateCode(channel.name, defaultCountry)
|
||||
const query = channel.name
|
||||
.replace(/\s(SD|TV|HD|SD\/HD|HDTV)$/i, '')
|
||||
.replace(/(\(|\)|,)/gi, '')
|
||||
.replace(/-/gi, ' ')
|
||||
.replace(/\+/gi, '')
|
||||
const similar = channelsIndex.search(query).map(item => new ApiChannel(item))
|
||||
|
||||
const variants = new Collection()
|
||||
variants.add(`${channel.name.trim()} | ${channelId}${newLabel}`)
|
||||
similar.forEach((_channel: ApiChannel) => {
|
||||
const altNames = _channel.altNames.notEmpty() ? ` (${_channel.altNames.join(',')})` : ''
|
||||
const closed = _channel.closed ? ` [closed:${_channel.closed}]` : ''
|
||||
const replacedBy = _channel.replacedBy ? `[replaced_by:${_channel.replacedBy}]` : ''
|
||||
|
||||
variants.add(`${_channel.name}${altNames} | ${_channel.id}${closed}${replacedBy}`)
|
||||
})
|
||||
variants.add('Overwrite')
|
||||
variants.add('Skip')
|
||||
|
||||
return variants.all()
|
||||
}
|
||||
|
||||
function generateCode(name: string, country: string) {
|
||||
const channelId: string = transliterate(name)
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/^&/gi, 'And')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
|
||||
return `${channelId}.${country}`
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import chalk from 'chalk'
|
||||
import libxml, { ValidationError } from 'libxmljs2'
|
||||
import { program } from 'commander'
|
||||
import { Storage, File } from '@freearhey/core'
|
||||
import { XmlDocument, XsdValidator, XmlValidateError, ErrorDetail } from 'libxml2-wasm'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
|
|
@ -23,12 +23,12 @@ const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
|||
</xs:element>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv)
|
||||
program.argument('[filepath]', 'Path to *.channels.xml files to check').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
let errors: ValidationError[] = []
|
||||
let errors: ErrorDetail[] = []
|
||||
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
|
|
@ -37,23 +37,28 @@ async function main() {
|
|||
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
let localErrors: ValidationError[] = []
|
||||
let localErrors: ErrorDetail[] = []
|
||||
|
||||
try {
|
||||
const xsdDoc = libxml.parseXml(xsd)
|
||||
const doc = libxml.parseXml(xml)
|
||||
const schema = XmlDocument.fromString(xsd)
|
||||
const validator = XsdValidator.fromDoc(schema)
|
||||
const doc = XmlDocument.fromString(xml)
|
||||
|
||||
if (!doc.validate(xsdDoc)) {
|
||||
localErrors = doc.validationErrors
|
||||
}
|
||||
} catch (error) {
|
||||
localErrors.push(error)
|
||||
validator.validate(doc)
|
||||
|
||||
schema.dispose()
|
||||
validator.dispose()
|
||||
doc.dispose()
|
||||
} catch (_error) {
|
||||
const error = _error as XmlValidateError
|
||||
|
||||
localErrors = localErrors.concat(error.details)
|
||||
}
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach((error: ValidationError) => {
|
||||
const position = `${error.line}:${error.column}`
|
||||
localErrors.forEach((error: ErrorDetail) => {
|
||||
const position = `${error.line}:${error.col}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { Storage, Collection, Dictionary, File, Logger } from '@freearhey/core'
|
||||
import { Storage, Collection, Dictionary, File } from '@freearhey/core'
|
||||
import { ChannelsParser, ApiChannel } from '../../core'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
|
|
@ -6,15 +6,7 @@ import langs from 'langs'
|
|||
import { DATA_DIR } from '../../constants'
|
||||
import { Channel } from 'epg-grabber'
|
||||
|
||||
program
|
||||
.option(
|
||||
'-c, --channels <path>',
|
||||
'Path to channels.xml file to validate',
|
||||
'sites/**/*.channels.xml'
|
||||
)
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
program.argument('[filepath]', 'Path to *.channels.xml files to validate').parse(process.argv)
|
||||
|
||||
type ValidationError = {
|
||||
type: 'duplicate' | 'wrong_xmltv_id' | 'wrong_lang'
|
||||
|
|
@ -26,11 +18,6 @@ type ValidationError = {
|
|||
}
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info('options:')
|
||||
logger.tree(options)
|
||||
|
||||
const parser = new ChannelsParser({ storage: new Storage() })
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
|
@ -39,8 +26,9 @@ async function main() {
|
|||
|
||||
let totalFiles = 0
|
||||
let totalErrors = 0
|
||||
|
||||
const storage = new Storage()
|
||||
const files: string[] = await storage.list(options.channels)
|
||||
const files = program.args.length ? program.args : await storage.list('sites/**/*.channels.xml')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
|
@ -50,9 +38,9 @@ async function main() {
|
|||
const bufferBySiteId = new Dictionary()
|
||||
const errors: ValidationError[] = []
|
||||
parsedChannels.forEach((channel: Channel) => {
|
||||
const bufferSiteId: string = `${channel.site_id}:${channel.lang}`
|
||||
if (bufferBySiteId.missing(bufferSiteId)) {
|
||||
bufferBySiteId.set(bufferSiteId, true)
|
||||
const bufferId: string = channel.site_id
|
||||
if (bufferBySiteId.missing(bufferId)) {
|
||||
bufferBySiteId.set(bufferId, true)
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel })
|
||||
totalErrors++
|
||||
|
|
@ -72,16 +60,6 @@ async function main() {
|
|||
errors.push({ type: 'wrong_xmltv_id', ...channel })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
// if (foundChannel && foundChannel.replacedBy) {
|
||||
// errors.push({ type: 'replaced', ...channel })
|
||||
// totalErrors++
|
||||
// }
|
||||
|
||||
// if (foundChannel && foundChannel.closed && !foundChannel.replacedBy) {
|
||||
// errors.push({ type: 'closed', ...channel })
|
||||
// totalErrors++
|
||||
// }
|
||||
})
|
||||
|
||||
if (errors.length) {
|
||||
|
|
|
|||
45
scripts/commands/sites/init.ts
Normal file
45
scripts/commands/sites/init.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import { Logger, Storage } from '@freearhey/core'
|
||||
import { program } from 'commander'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
import fs from 'fs-extra'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
program.argument('<site>', 'Domain name of the site').parse(process.argv)
|
||||
|
||||
const domain = program.args[0]
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(SITES_DIR)
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info(`Initializing "${domain}"...\r\n`)
|
||||
|
||||
const dir = domain
|
||||
if (await storage.exists(dir)) {
|
||||
throw new Error(`Folder "${dir}" already exists`)
|
||||
}
|
||||
|
||||
await storage.createDir(dir)
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.test.js"...`)
|
||||
const testTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_test.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.test.js`, testTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/${domain}.config.js"...`)
|
||||
const configTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_config.js'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/${domain}.config.js`, configTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info(`Creating "${dir}/readme.md"...`)
|
||||
const readmeTemplate = fs.readFileSync(pathToFileURL('scripts/templates/_readme.md'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
await storage.save(`${dir}/readme.md`, readmeTemplate.replace(/<DOMAIN>/g, domain))
|
||||
|
||||
logger.info('\r\nDone')
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -41,7 +41,7 @@ export class ApiClient {
|
|||
}
|
||||
|
||||
async download(filename: string) {
|
||||
const stream = await this.storage.createStream(`/temp/data/${filename}`)
|
||||
const stream = await this.storage.createStream(`temp/data/${filename}`)
|
||||
|
||||
const bar = this.progressBar.create(0, 0, { filename })
|
||||
|
||||
|
|
|
|||
16
scripts/templates/_config.js
Normal file
16
scripts/templates/_config.js
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
module.exports = {
|
||||
site: '<DOMAIN>',
|
||||
url({ channel, date }) {
|
||||
return `https://example.com/api/${channel.site_id}/${date.format('YYYY-MM-DD')}`
|
||||
},
|
||||
parser({ content }) {
|
||||
try {
|
||||
return JSON.parse(content)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
},
|
||||
channels() {
|
||||
return []
|
||||
}
|
||||
}
|
||||
21
scripts/templates/_readme.md
Normal file
21
scripts/templates/_readme.md
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# <DOMAIN>
|
||||
|
||||
https://example.com
|
||||
|
||||
### Download the guide
|
||||
|
||||
```sh
|
||||
npm run grab --- --site=<DOMAIN>
|
||||
```
|
||||
|
||||
### Update channel list
|
||||
|
||||
```sh
|
||||
npm run channels:parse --- --config=./sites/<DOMAIN>/<DOMAIN>.config.js --output=./sites/<DOMAIN>/<DOMAIN>.channels.xml
|
||||
```
|
||||
|
||||
### Test
|
||||
|
||||
```sh
|
||||
npm test --- <DOMAIN>
|
||||
```
|
||||
42
scripts/templates/_test.js
Normal file
42
scripts/templates/_test.js
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
const { parser, url } = require('./<DOMAIN>.config.js')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
dayjs.extend(customParseFormat)
|
||||
dayjs.extend(utc)
|
||||
|
||||
const date = dayjs.utc('2025-01-12', 'YYYY-MM-DD').startOf('d')
|
||||
const channel = { site_id: 'bbc1', xmltv_id: 'BBCOne.uk' }
|
||||
|
||||
it('can generate valid url', () => {
|
||||
expect(url({ channel, date })).toBe('https://example.com/api/bbc1/2025-01-12')
|
||||
})
|
||||
|
||||
it('can parse response', () => {
|
||||
const content =
|
||||
'[{"title":"Program 1","start":"2025-01-12T00:00:00.000Z","stop":"2025-01-12T00:30:00.000Z"},{"title":"Program 2","start":"2025-01-12T00:30:00.000Z","stop":"2025-01-12T01:00:00.000Z"}]'
|
||||
|
||||
const results = parser({ content })
|
||||
|
||||
expect(results.length).toBe(2)
|
||||
expect(results[0]).toMatchObject({
|
||||
title: 'Program 1',
|
||||
start: '2025-01-12T00:00:00.000Z',
|
||||
stop: '2025-01-12T00:30:00.000Z'
|
||||
})
|
||||
expect(results[1]).toMatchObject({
|
||||
title: 'Program 2',
|
||||
start: '2025-01-12T00:30:00.000Z',
|
||||
stop: '2025-01-12T01:00:00.000Z'
|
||||
})
|
||||
})
|
||||
|
||||
it('can handle empty guide', () => {
|
||||
const result = parser({
|
||||
date,
|
||||
channel,
|
||||
content: ''
|
||||
})
|
||||
|
||||
expect(result).toMatchObject([])
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue