mirror of
https://github.com/iptv-org/epg.git
synced 2025-05-09 16:40:07 -04:00
Update scripts
This commit is contained in:
freearhey
parent
b7214db4fb
commit
ca254a6df0
37 changed files with 1091 additions and 915 deletions
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
mkdir -p scripts/tmp/data
|
||||
curl -L -o scripts/tmp/data/channels.json https://iptv-org.github.io/api/channels.json
|
||||
curl -L -o scripts/tmp/data/countries.json https://iptv-org.github.io/api/countries.json
|
||||
curl -L -o scripts/tmp/data/regions.json https://iptv-org.github.io/api/regions.json
|
||||
curl -L -o scripts/tmp/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json
|
||||
18
scripts/commands/api/load.ts
Normal file
18
scripts/commands/api/load.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { Logger } from '@freearhey/core'
|
||||
import { ApiClient } from '../../core'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
const client = new ApiClient({ logger })
|
||||
|
||||
const requests = [
|
||||
client.download('channels.json'),
|
||||
client.download('countries.json'),
|
||||
client.download('regions.json'),
|
||||
client.download('subdivisions.json')
|
||||
]
|
||||
|
||||
await Promise.all(requests)
|
||||
}
|
||||
|
||||
main()
|
||||
1
scripts/commands/channels/.gitignore
vendored
Normal file
1
scripts/commands/channels/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
/replace.ts
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
const { api, parser, xml, file, logger } = require('../../core')
|
||||
const { transliterate } = require('transliteration')
|
||||
const nodeCleanup = require('node-cleanup')
|
||||
const { program } = require('commander')
|
||||
const inquirer = require('inquirer')
|
||||
|
||||
program
|
||||
.argument('<filepath>', 'Path to *.channels.xml file to edit')
|
||||
.option('-c, --country <name>', 'Source country', 'us')
|
||||
.parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
const options = program.opts()
|
||||
const defaultCountry = options.country
|
||||
const newLabel = ` [new]`
|
||||
|
||||
let site
|
||||
let channels = []
|
||||
|
||||
async function main() {
|
||||
if (!(await file.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
return
|
||||
}
|
||||
|
||||
let result = await parser.parseChannels(filepath)
|
||||
site = result.site
|
||||
channels = result.channels
|
||||
channels = channels.map(c => {
|
||||
c.xmltv_id = c.xmltv_id
|
||||
return c
|
||||
})
|
||||
await api.channels.load()
|
||||
const buffer = []
|
||||
for (const channel of channels) {
|
||||
if (channel.xmltv_id) {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.push(`${channel.xmltv_id}/${channel.lang}`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
let choices = await getOptions(channel)
|
||||
const question = {
|
||||
name: 'option',
|
||||
message: `Choose an option:`,
|
||||
type: 'list',
|
||||
choices,
|
||||
pageSize: 10
|
||||
}
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Overwrite':
|
||||
const input = await getInput(channel)
|
||||
channel.xmltv_id = input.xmltv_id
|
||||
break
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
default:
|
||||
const [name, xmltv_id] = selected.option
|
||||
.replace(/ \[.*\]/, '')
|
||||
.split('|')
|
||||
.map(i => i.trim().replace(newLabel, ''))
|
||||
channel.xmltv_id = xmltv_id
|
||||
break
|
||||
}
|
||||
|
||||
const found = buffer.includes(`${channel.xmltv_id}/${channel.lang}`)
|
||||
if (found) {
|
||||
const question = {
|
||||
name: 'option',
|
||||
message: `"${channel.xmltv_id}" already on the list. Choose an option:`,
|
||||
type: 'list',
|
||||
choices: ['Skip', 'Add', 'Delete'],
|
||||
pageSize: 5
|
||||
}
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
case 'Delete':
|
||||
channel.delete = true
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
})
|
||||
} else {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.push(`${channel.xmltv_id}/${channel.lang}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function save() {
|
||||
if (!file.existsSync(filepath)) return
|
||||
|
||||
channels = channels.filter(c => !c.delete)
|
||||
|
||||
const output = xml.create(channels, site)
|
||||
|
||||
file.writeSync(filepath, output)
|
||||
|
||||
logger.info(`\nFile '${filepath}' successfully saved`)
|
||||
}
|
||||
|
||||
nodeCleanup(() => {
|
||||
save()
|
||||
})
|
||||
|
||||
async function getInput(channel) {
|
||||
const name = channel.name.trim()
|
||||
const input = await inquirer.prompt([
|
||||
{
|
||||
name: 'xmltv_id',
|
||||
message: ' ID:',
|
||||
type: 'input',
|
||||
default: generateCode(name, defaultCountry)
|
||||
}
|
||||
])
|
||||
|
||||
return { name, xmltv_id: input['xmltv_id'] }
|
||||
}
|
||||
|
||||
async function getOptions(channel) {
|
||||
const channels = await api.channels.all()
|
||||
const channelId = generateCode(channel.name, defaultCountry)
|
||||
const similar = await getSimilar(channels, channelId)
|
||||
let variants = []
|
||||
variants.push(`${channel.name.trim()} | ${channelId}${newLabel}`)
|
||||
similar.forEach(i => {
|
||||
let alt_names = i.alt_names.length ? ` (${i.alt_names.join(',')})` : ''
|
||||
let closed = i.closed ? `[closed:${i.closed}]` : ``
|
||||
let replaced_by = i.replaced_by ? `[replaced_by:${i.replaced_by}]` : ''
|
||||
variants.push(`${i.name}${alt_names} | ${i.id} ${closed}${replaced_by}[api]`)
|
||||
})
|
||||
variants.push(`Overwrite`)
|
||||
variants.push(`Skip`)
|
||||
|
||||
return variants
|
||||
}
|
||||
|
||||
async function getSimilar(list, channelId) {
|
||||
const normChannelId = channelId.split('.')[0].slice(0, 8).toLowerCase()
|
||||
return list.filter(i => i.id.split('.')[0].toLowerCase().startsWith(normChannelId))
|
||||
}
|
||||
|
||||
function generateCode(name, country) {
|
||||
const id = transliterate(name)
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/^\&/gi, 'And')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
|
||||
return `${id}.${country}`
|
||||
}
|
||||
181
scripts/commands/channels/editor.ts
Normal file
181
scripts/commands/channels/editor.ts
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import { DATA_DIR } from '../../constants'
|
||||
import { Storage, Collection, Dictionary, Logger } from '@freearhey/core'
|
||||
import { ChannelsParser, XML, ApiChannel } from '../../core'
|
||||
import { Channel } from 'epg-grabber'
|
||||
import { transliterate } from 'transliteration'
|
||||
import nodeCleanup from 'node-cleanup'
|
||||
import { program } from 'commander'
|
||||
import inquirer, { QuestionCollection } from 'inquirer'
|
||||
|
||||
program
|
||||
.argument('<filepath>', 'Path to *.channels.xml file to edit')
|
||||
.option('-c, --country <name>', 'Source country', 'us')
|
||||
.parse(process.argv)
|
||||
|
||||
const filepath = program.args[0]
|
||||
const programOptions = program.opts()
|
||||
const defaultCountry = programOptions.country
|
||||
const newLabel = ` [new]`
|
||||
|
||||
let site: string
|
||||
let options = new Collection()
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
|
||||
if (!(await storage.exists(filepath))) {
|
||||
throw new Error(`File "${filepath}" does not exists`)
|
||||
}
|
||||
|
||||
const parser = new ChannelsParser({ storage })
|
||||
|
||||
const parsedChannels = await parser.parse(filepath)
|
||||
options = parsedChannels.map((channel: Channel) => {
|
||||
return {
|
||||
channel,
|
||||
delete: false
|
||||
}
|
||||
})
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const channels = new Collection(channelsContent).map(data => new ApiChannel(data))
|
||||
|
||||
const buffer = new Dictionary()
|
||||
options.forEach(async (option: { channel: Channel; delete: boolean }) => {
|
||||
const channel = option.channel
|
||||
if (channel.xmltv_id) {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.set(`${channel.xmltv_id}/${channel.lang}`, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
let choices = getOptions(channels, channel)
|
||||
const question: QuestionCollection = {
|
||||
name: 'option',
|
||||
message: `Choose an option:`,
|
||||
type: 'list',
|
||||
choices,
|
||||
pageSize: 10
|
||||
}
|
||||
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Overwrite':
|
||||
const input = await getInput(channel)
|
||||
channel.xmltv_id = input.xmltv_id
|
||||
break
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
default:
|
||||
const [, xmltv_id] = selected.option
|
||||
.replace(/ \[.*\]/, '')
|
||||
.split('|')
|
||||
.map((i: string) => i.trim().replace(newLabel, ''))
|
||||
channel.xmltv_id = xmltv_id
|
||||
break
|
||||
}
|
||||
|
||||
const found = buffer.has(`${channel.xmltv_id}/${channel.lang}`)
|
||||
if (found) {
|
||||
const question: QuestionCollection = {
|
||||
name: 'option',
|
||||
message: `"${channel.xmltv_id}" already on the list. Choose an option:`,
|
||||
type: 'list',
|
||||
choices: ['Skip', 'Add', 'Delete'],
|
||||
pageSize: 5
|
||||
}
|
||||
await inquirer.prompt(question).then(async selected => {
|
||||
switch (selected.option) {
|
||||
case 'Skip':
|
||||
channel.xmltv_id = '-'
|
||||
break
|
||||
case 'Delete':
|
||||
option.delete = true
|
||||
break
|
||||
default:
|
||||
break
|
||||
}
|
||||
})
|
||||
} else {
|
||||
if (channel.xmltv_id !== '-') {
|
||||
buffer.set(`${channel.xmltv_id}/${channel.lang}`, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function save() {
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
|
||||
if (!storage.existsSync(filepath)) return
|
||||
|
||||
const channels = options
|
||||
.filter((option: { channel: Channel; delete: boolean }) => !option.delete)
|
||||
.map((option: { channel: Channel; delete: boolean }) => option.channel)
|
||||
|
||||
const xml = new XML(channels, site)
|
||||
|
||||
storage.saveSync(filepath, xml.toString())
|
||||
|
||||
logger.info(`\nFile '${filepath}' successfully saved`)
|
||||
}
|
||||
|
||||
nodeCleanup(() => {
|
||||
save()
|
||||
})
|
||||
|
||||
async function getInput(channel: Channel) {
|
||||
const name = channel.name.trim()
|
||||
const input = await inquirer.prompt([
|
||||
{
|
||||
name: 'xmltv_id',
|
||||
message: ' ID:',
|
||||
type: 'input',
|
||||
default: generateCode(name, defaultCountry)
|
||||
}
|
||||
])
|
||||
|
||||
return { name, xmltv_id: input['xmltv_id'] }
|
||||
}
|
||||
|
||||
function getOptions(channels: Collection, channel: Channel) {
|
||||
const channelId = generateCode(channel.name, defaultCountry)
|
||||
const similar = getSimilar(channels, channelId)
|
||||
|
||||
const variants = new Collection()
|
||||
variants.add(`${channel.name.trim()} | ${channelId}${newLabel}`)
|
||||
similar.forEach((_channel: ApiChannel) => {
|
||||
const altNames = _channel.altNames.notEmpty() ? ` (${_channel.altNames.join(',')})` : ''
|
||||
const closed = _channel.closed ? `[closed:${_channel.closed}]` : ``
|
||||
const replacedBy = _channel.replacedBy ? `[replaced_by:${_channel.replacedBy}]` : ''
|
||||
|
||||
variants.add(`${_channel.name}${altNames} | ${_channel.id} ${closed}${replacedBy}[api]`)
|
||||
})
|
||||
variants.add(`Overwrite`)
|
||||
variants.add(`Skip`)
|
||||
|
||||
return variants.all()
|
||||
}
|
||||
|
||||
function getSimilar(channels: Collection, channelId: string) {
|
||||
const normChannelId = channelId.split('.')[0].slice(0, 8).toLowerCase()
|
||||
|
||||
return channels.filter((channel: ApiChannel) =>
|
||||
channel.id.split('.')[0].toLowerCase().startsWith(normChannelId)
|
||||
)
|
||||
}
|
||||
|
||||
function generateCode(name: string, country: string) {
|
||||
const channelId: string = transliterate(name)
|
||||
.replace(/\+/gi, 'Plus')
|
||||
.replace(/^\&/gi, 'And')
|
||||
.replace(/[^a-z\d]+/gi, '')
|
||||
|
||||
return `${channelId}.${country}`
|
||||
}
|
||||
|
|
@ -1,18 +1,10 @@
|
|||
const chalk = require('chalk')
|
||||
const libxml = require('libxmljs2')
|
||||
const { program } = require('commander')
|
||||
const { logger, file } = require('../../core')
|
||||
import chalk from 'chalk'
|
||||
import libxml, { ValidationError } from 'libxmljs2'
|
||||
import { program } from 'commander'
|
||||
import { Logger, Storage, File } from '@freearhey/core'
|
||||
|
||||
const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
|
||||
<xs:element name="site">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
<xs:element ref="channels"/>
|
||||
</xs:sequence>
|
||||
<xs:attribute name="site" use="required" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
<xs:element name="channels">
|
||||
<xs:complexType>
|
||||
<xs:sequence>
|
||||
|
|
@ -22,43 +14,53 @@ const xsd = `<?xml version="1.0" encoding="UTF-8"?>
|
|||
</xs:element>
|
||||
<xs:element name="channel">
|
||||
<xs:complexType mixed="true">
|
||||
<xs:attribute name="site" use="required" type="xs:string"/>
|
||||
<xs:attribute name="lang" use="required" type="xs:string"/>
|
||||
<xs:attribute name="site_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="xmltv_id" use="required" type="xs:string"/>
|
||||
<xs:attribute name="logo" type="xs:string"/>
|
||||
</xs:complexType>
|
||||
</xs:element>
|
||||
</xs:schema>`
|
||||
|
||||
program.argument('<filepath>', 'Path to file to validate').parse(process.argv)
|
||||
program
|
||||
.option(
|
||||
'-c, --channels <path>',
|
||||
'Path to channels.xml file to validate',
|
||||
'sites/**/*.channels.xml'
|
||||
)
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
|
||||
async function main() {
|
||||
if (!program.args.length) {
|
||||
logger.error('required argument "filepath" not specified')
|
||||
}
|
||||
const logger = new Logger()
|
||||
const storage = new Storage()
|
||||
|
||||
let errors = []
|
||||
logger.info('options:')
|
||||
logger.tree(options)
|
||||
|
||||
for (const filepath of program.args) {
|
||||
if (!filepath.endsWith('.xml')) continue
|
||||
let errors: ValidationError[] = []
|
||||
|
||||
const xml = await file.read(filepath)
|
||||
let files: string[] = await storage.list(options.channels)
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
let localErrors = []
|
||||
const xml = await storage.load(filepath)
|
||||
|
||||
try {
|
||||
const xsdDoc = libxml.parseXml(xsd)
|
||||
const doc = libxml.parseXml(xml)
|
||||
let localErrors: ValidationError[] = []
|
||||
|
||||
if (!doc.validate(xsdDoc)) {
|
||||
localErrors = doc.validationErrors
|
||||
}
|
||||
} catch (error) {
|
||||
localErrors.push(error)
|
||||
const xsdDoc = libxml.parseXml(xsd)
|
||||
const doc = libxml.parseXml(xml)
|
||||
|
||||
if (!doc.validate(xsdDoc)) {
|
||||
localErrors = doc.validationErrors
|
||||
}
|
||||
|
||||
if (localErrors.length) {
|
||||
console.log(`\n${chalk.underline(filepath)}`)
|
||||
localErrors.forEach(error => {
|
||||
localErrors.forEach((error: ValidationError) => {
|
||||
const position = `${error.line}:${error.column}`
|
||||
console.log(` ${chalk.gray(position.padEnd(4, ' '))} ${error.message.trim()}`)
|
||||
})
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
const { logger, file, xml, parser } = require('../../core')
|
||||
const { Command } = require('commander')
|
||||
const path = require('path')
|
||||
const _ = require('lodash')
|
||||
|
||||
const program = new Command()
|
||||
program
|
||||
.requiredOption('-c, --config <config>', 'Config file')
|
||||
.option('-s, --set [args...]', 'Set custom arguments', [])
|
||||
.option('-o, --output <output>', 'Output file')
|
||||
.option('--clean', 'Delete the previous *.channels.xml if exists')
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
|
||||
async function main() {
|
||||
const config = require(path.resolve(options.config))
|
||||
const dir = file.dirname(options.config)
|
||||
const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
|
||||
|
||||
let channels = []
|
||||
if (!options.clean && (await file.exists(outputFilepath))) {
|
||||
let result = await parser.parseChannels(outputFilepath)
|
||||
|
||||
channels = result.channels
|
||||
}
|
||||
|
||||
const args = {}
|
||||
options.set.forEach(arg => {
|
||||
const [key, value] = arg.split(':')
|
||||
args[key] = value
|
||||
})
|
||||
|
||||
let parsedChannels = config.channels(args)
|
||||
if (isPromise(parsedChannels)) {
|
||||
parsedChannels = await parsedChannels
|
||||
}
|
||||
parsedChannels = parsedChannels.map(c => {
|
||||
c.lang = c.lang || 'en'
|
||||
|
||||
return c
|
||||
})
|
||||
|
||||
channels = channels.concat(parsedChannels)
|
||||
|
||||
channels = _.uniqBy(channels, c => c.site_id + c.lang)
|
||||
|
||||
channels = _.sortBy(channels, [
|
||||
'lang',
|
||||
c => (c.xmltv_id ? c.xmltv_id.toLowerCase() : '_'),
|
||||
'site_id'
|
||||
])
|
||||
|
||||
const output = xml.create(channels, config.site)
|
||||
|
||||
await file.write(outputFilepath, output)
|
||||
|
||||
logger.info(`File '${outputFilepath}' successfully saved`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function isPromise(promise) {
|
||||
return !!promise && typeof promise.then === 'function'
|
||||
}
|
||||
76
scripts/commands/channels/parse.ts
Normal file
76
scripts/commands/channels/parse.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { Logger, File, Collection, Storage } from '@freearhey/core'
|
||||
import { ChannelsParser, XML } from '../../core'
|
||||
import { Channel } from 'epg-grabber'
|
||||
import { Command, OptionValues } from 'commander'
|
||||
import path from 'path'
|
||||
|
||||
const program = new Command()
|
||||
program
|
||||
.requiredOption('-c, --config <config>', 'Config file')
|
||||
.option('-s, --set [args...]', 'Set custom arguments')
|
||||
.option('-o, --output <output>', 'Output file')
|
||||
.option('--clean', 'Delete the previous *.channels.xml if exists')
|
||||
.parse(process.argv)
|
||||
|
||||
type ParseOptions = {
|
||||
config: string
|
||||
set?: string
|
||||
output?: string
|
||||
clean?: boolean
|
||||
}
|
||||
|
||||
const options: ParseOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage()
|
||||
const parser = new ChannelsParser({ storage })
|
||||
const logger = new Logger()
|
||||
const file = new File(options.config)
|
||||
const dir = file.dirname()
|
||||
const config = require(path.resolve(options.config))
|
||||
const outputFilepath = options.output || `${dir}/${config.site}.channels.xml`
|
||||
|
||||
let channels = new Collection()
|
||||
if (!options.clean && (await storage.exists(outputFilepath))) {
|
||||
channels = await parser.parse(outputFilepath)
|
||||
}
|
||||
|
||||
const args: {
|
||||
[key: string]: any
|
||||
} = {}
|
||||
|
||||
if (Array.isArray(options.set)) {
|
||||
options.set.forEach((arg: string) => {
|
||||
const [key, value] = arg.split(':')
|
||||
args[key] = value
|
||||
})
|
||||
}
|
||||
|
||||
let parsedChannels = config.channels(args)
|
||||
if (isPromise(parsedChannels)) {
|
||||
parsedChannels = await parsedChannels
|
||||
}
|
||||
|
||||
channels = channels
|
||||
.mergeBy(
|
||||
new Collection(parsedChannels),
|
||||
(channel: Channel) => channel.site_id.toString() + channel.lang
|
||||
)
|
||||
.orderBy([
|
||||
(channel: Channel) => channel.lang,
|
||||
(channel: Channel) => (channel.xmltv_id ? channel.xmltv_id.toLowerCase() : '_'),
|
||||
(channel: Channel) => channel.site_id
|
||||
])
|
||||
|
||||
const xml = new XML(channels, config.site)
|
||||
|
||||
await storage.save(outputFilepath, xml.toString())
|
||||
|
||||
logger.info(`File '${outputFilepath}' successfully saved`)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
function isPromise(promise: any) {
|
||||
return !!promise && typeof promise.then === 'function'
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
const { parser, logger, api } = require('../../core')
|
||||
const { program } = require('commander')
|
||||
const chalk = require('chalk')
|
||||
const langs = require('langs')
|
||||
|
||||
program.argument('<filepath>', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
await api.channels.load()
|
||||
|
||||
const stats = {
|
||||
files: 0,
|
||||
errors: 0
|
||||
}
|
||||
|
||||
if (!program.args.length) {
|
||||
logger.error('required argument "filepath" not specified')
|
||||
}
|
||||
|
||||
for (const filepath of program.args) {
|
||||
if (!filepath.endsWith('.xml')) continue
|
||||
|
||||
const { site, channels } = await parser.parseChannels(filepath)
|
||||
|
||||
const bufferById = {}
|
||||
const bufferBySiteId = {}
|
||||
const errors = []
|
||||
for (const channel of channels) {
|
||||
if (!bufferById[channel.xmltv_id + channel.lang]) {
|
||||
bufferById[channel.xmltv_id + channel.lang] = channel
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel })
|
||||
stats.errors++
|
||||
}
|
||||
|
||||
if (!bufferBySiteId[channel.site_id + channel.lang]) {
|
||||
bufferBySiteId[channel.site_id + channel.lang] = channel
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel })
|
||||
stats.errors++
|
||||
}
|
||||
|
||||
if (!api.channels.find({ id: channel.xmltv_id })) {
|
||||
errors.push({ type: 'wrong_xmltv_id', ...channel })
|
||||
stats.errors++
|
||||
}
|
||||
|
||||
if (!langs.where('1', channel.lang)) {
|
||||
errors.push({ type: 'wrong_lang', ...channel })
|
||||
stats.errors++
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.underline(filepath))
|
||||
console.table(errors, ['type', 'lang', 'xmltv_id', 'site_id', 'name'])
|
||||
console.log()
|
||||
stats.files++
|
||||
}
|
||||
}
|
||||
|
||||
if (stats.errors > 0) {
|
||||
console.log(chalk.red(`${stats.errors} error(s) in ${stats.files} file(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
95
scripts/commands/channels/validate.ts
Normal file
95
scripts/commands/channels/validate.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { Storage, Collection, Dictionary, File, Logger } from '@freearhey/core'
|
||||
import { ChannelsParser, ApiChannel } from '../../core'
|
||||
import { program } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
import langs from 'langs'
|
||||
import { DATA_DIR } from '../../constants'
|
||||
import { Channel } from 'epg-grabber'
|
||||
|
||||
program
|
||||
.option(
|
||||
'-c, --channels <path>',
|
||||
'Path to channels.xml file to validate',
|
||||
'sites/**/*.channels.xml'
|
||||
)
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
|
||||
type ValidationError = {
|
||||
type: 'duplicate' | 'wrong_xmltv_id' | 'wrong_lang'
|
||||
name: string
|
||||
lang?: string
|
||||
xmltv_id?: string
|
||||
site_id?: string
|
||||
logo?: string
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info('options:')
|
||||
logger.tree(options)
|
||||
|
||||
const parser = new ChannelsParser({ storage: new Storage() })
|
||||
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const channels = new Collection(channelsContent).map(data => new ApiChannel(data))
|
||||
|
||||
let totalFiles = 0
|
||||
let totalErrors = 0
|
||||
const storage = new Storage()
|
||||
let files: string[] = await storage.list(options.channels)
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'xml') continue
|
||||
|
||||
const parsedChannels = await parser.parse(filepath)
|
||||
|
||||
const bufferById = new Dictionary()
|
||||
const bufferBySiteId = new Dictionary()
|
||||
const errors: ValidationError[] = []
|
||||
parsedChannels.forEach((channel: Channel) => {
|
||||
const bufferId: string = `${channel.xmltv_id}:${channel.lang}`
|
||||
if (bufferById.missing(bufferId)) {
|
||||
bufferById.set(bufferId, true)
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
const bufferSiteId: string = `${channel.site_id}:${channel.lang}`
|
||||
if (bufferBySiteId.missing(bufferSiteId)) {
|
||||
bufferBySiteId.set(bufferSiteId, true)
|
||||
} else {
|
||||
errors.push({ type: 'duplicate', ...channel })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (channels.missing((_channel: ApiChannel) => _channel.id === channel.xmltv_id)) {
|
||||
errors.push({ type: 'wrong_xmltv_id', ...channel })
|
||||
totalErrors++
|
||||
}
|
||||
|
||||
if (!langs.where('1', channel.lang)) {
|
||||
errors.push({ type: 'wrong_lang', ...channel })
|
||||
totalErrors++
|
||||
}
|
||||
})
|
||||
|
||||
if (errors.length) {
|
||||
console.log(chalk.underline(filepath))
|
||||
console.table(errors, ['type', 'lang', 'xmltv_id', 'site_id', 'name'])
|
||||
console.log()
|
||||
totalFiles++
|
||||
}
|
||||
}
|
||||
|
||||
if (totalErrors > 0) {
|
||||
console.log(chalk.red(`${totalErrors} error(s) in ${totalFiles} file(s)`))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
const { program } = require('commander')
|
||||
const _ = require('lodash')
|
||||
const { EPGGrabber, generateXMLTV, Channel, Program } = require('epg-grabber')
|
||||
const { db, logger, date, timer, file, parser, api, zip } = require('../../core')
|
||||
const path = require('path')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const CronJob = require('cron').CronJob
|
||||
|
||||
dayjs.extend(utc)
|
||||
|
||||
const BASE_DIR = process.env.BASE_DIR || '.'
|
||||
const CURR_DATE = process.env.CURR_DATE || new Date()
|
||||
|
||||
program
|
||||
.requiredOption('-s, --site <name>', 'Name of the site to parse')
|
||||
.option('-l, --lang <code>', 'Filter channels by language (ISO 639-2 code)')
|
||||
.option('-o, --output <path>', 'Path to output file')
|
||||
.option('--days <days>', 'Override the number of days for which the program will be loaded')
|
||||
.option('--cron <expression>', 'Schedule a script run')
|
||||
.option('--gzip', 'Create a compressed version of the guide as well', false)
|
||||
.parse(process.argv)
|
||||
|
||||
const options = program.opts()
|
||||
|
||||
options.output = options.output || file.resolve(`${BASE_DIR}/guides/{lang}/{site}.xml`)
|
||||
options.config = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}.config.js`)
|
||||
options.channels = file.resolve(`${BASE_DIR}/sites/${options.site}/${options.site}*.channels.xml`)
|
||||
|
||||
let channels = []
|
||||
let programs = []
|
||||
let runIndex = 0
|
||||
|
||||
async function main() {
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('settings:')
|
||||
for (let prop in options) {
|
||||
logger.info(` ${prop}: ${options[prop]}`)
|
||||
}
|
||||
|
||||
const config = await loadConfig(options.config)
|
||||
const queue = await createQueue(options.channels, config)
|
||||
const outputPath = options.output
|
||||
|
||||
if (options.cron) {
|
||||
const job = new CronJob(options.cron, function () {
|
||||
runJob(config, queue, outputPath)
|
||||
})
|
||||
job.start()
|
||||
} else {
|
||||
await runJob(config, queue, outputPath)
|
||||
}
|
||||
}
|
||||
|
||||
async function loadConfig(configPath) {
|
||||
let config = require(file.resolve(configPath))
|
||||
config = _.merge(config, {})
|
||||
config.days = config.days || 1
|
||||
|
||||
logger.info('config:')
|
||||
logConfig(config)
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
function logConfig(config, level = 1) {
|
||||
let padLeft = ' '.repeat(level)
|
||||
for (let prop in config) {
|
||||
if (typeof config[prop] === 'string' || typeof config[prop] === 'number') {
|
||||
logger.info(`${padLeft}${prop}: ${config[prop]}`)
|
||||
} else if (typeof config[prop] === 'object') {
|
||||
level++
|
||||
logger.info(`${padLeft}${prop}:`)
|
||||
logConfig(config[prop], level)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function runJob(config, queue, outputPath) {
|
||||
runIndex++
|
||||
logger.info(`run #${runIndex}:`)
|
||||
|
||||
timer.start()
|
||||
|
||||
await grab(queue, config)
|
||||
|
||||
await save(outputPath, channels, programs)
|
||||
|
||||
logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`)
|
||||
}
|
||||
|
||||
async function grab(queue, config) {
|
||||
const grabber = new EPGGrabber(config)
|
||||
const total = queue.length
|
||||
|
||||
let i = 1
|
||||
for (const item of queue) {
|
||||
let channel = item.channel
|
||||
let date = item.date
|
||||
channels.push(item.channel)
|
||||
await grabber
|
||||
.grab(channel, date, (data, err) => {
|
||||
logger.info(
|
||||
` [${i}/${total}] ${channel.site} (${channel.lang}) - ${channel.xmltv_id} - ${dayjs
|
||||
.utc(data.date)
|
||||
.format('MMM D, YYYY')} (${data.programs.length} programs)`
|
||||
)
|
||||
if (i < total) i++
|
||||
|
||||
if (err) {
|
||||
logger.info(` ERR: ${err.message}`)
|
||||
}
|
||||
})
|
||||
.then(results => {
|
||||
programs = programs.concat(results)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function createQueue(channelsPath, config) {
|
||||
logger.info('creating queue...')
|
||||
let queue = {}
|
||||
await api.channels.load().catch(logger.error)
|
||||
const files = await file.list(channelsPath).catch(logger.error)
|
||||
const utcDate = date.getUTC(CURR_DATE)
|
||||
const days = options.days ? parseInt(options.days) : config.days
|
||||
for (const filepath of files) {
|
||||
logger.info(` loading "${filepath}"...`)
|
||||
try {
|
||||
const dir = file.dirname(filepath)
|
||||
const { channels } = await parser.parseChannels(filepath)
|
||||
const filename = file.basename(filepath)
|
||||
const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
|
||||
for (const channel of channels) {
|
||||
if (!channel.site || !channel.xmltv_id) continue
|
||||
if (options.lang && channel.lang !== options.lang) continue
|
||||
const found = api.channels.find({ id: channel.xmltv_id })
|
||||
if (found) {
|
||||
channel.logo = found.logo
|
||||
}
|
||||
for (const d of dates) {
|
||||
const dateString = d.toJSON()
|
||||
const key = `${channel.site}:${channel.lang}:${channel.xmltv_id}:${dateString}`
|
||||
if (!queue[key]) {
|
||||
queue[key] = {
|
||||
channel,
|
||||
date: dateString,
|
||||
config,
|
||||
error: null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
queue = Object.values(queue)
|
||||
|
||||
logger.info(` added ${queue.length} items`)
|
||||
|
||||
return queue
|
||||
}
|
||||
|
||||
async function save(template, parsedChannels, programs = []) {
|
||||
const variables = file.templateVariables(template)
|
||||
|
||||
const groups = _.groupBy(parsedChannels, channel => {
|
||||
let groupId = ''
|
||||
for (let key in channel) {
|
||||
if (variables.includes(key)) {
|
||||
groupId += channel[key]
|
||||
}
|
||||
}
|
||||
|
||||
return groupId
|
||||
})
|
||||
|
||||
for (let groupId in groups) {
|
||||
const channels = groups[groupId]
|
||||
|
||||
let output = {
|
||||
channels,
|
||||
programs: [],
|
||||
date: CURR_DATE
|
||||
}
|
||||
|
||||
for (let program of programs) {
|
||||
let programLang = program.titles[0].lang
|
||||
let channel = channels.find(c => c.xmltv_id === program.channel && c.lang === programLang)
|
||||
if (!channel) continue
|
||||
|
||||
output.programs.push(new Program(program, channel))
|
||||
}
|
||||
|
||||
output.channels = _.sortBy(output.channels, 'xmltv_id')
|
||||
output.channels = _.uniqBy(output.channels, 'xmltv_id')
|
||||
|
||||
output.programs = _.sortBy(output.programs, ['channel', 'start'])
|
||||
output.programs = _.uniqBy(output.programs, p => p.channel + p.start)
|
||||
|
||||
const outputPath = file.templateFormat(template, output.channels[0])
|
||||
const xmlFilepath = outputPath
|
||||
const xmltv = generateXMLTV(output)
|
||||
logger.info(` saving to "${xmlFilepath}"...`)
|
||||
await file.create(xmlFilepath, xmltv)
|
||||
|
||||
if (options.gzip) {
|
||||
const gzFilepath = `${outputPath}.gz`
|
||||
const compressed = await zip.compress(xmltv)
|
||||
logger.info(` saving to "${gzFilepath}"...`)
|
||||
await file.create(gzFilepath, compressed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
115
scripts/commands/epg/grab.ts
Normal file
115
scripts/commands/epg/grab.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import { Logger, Timer, Storage, Collection } from '@freearhey/core'
|
||||
import { program } from 'commander'
|
||||
import { CronJob } from 'cron'
|
||||
import { Queue, Job, ChannelsParser } from '../../core'
|
||||
import { Channel } from 'epg-grabber'
|
||||
import path from 'path'
|
||||
import { SITES_DIR } from '../../constants'
|
||||
|
||||
program
|
||||
.option('-s, --site <name>', 'Name of the site to parse')
|
||||
.option(
|
||||
'-c, --channels <path>',
|
||||
'Path to *.channels.xml file (required if the "--site" attribute is not specified)'
|
||||
)
|
||||
.option('-o, --output <path>', 'Path to output file', 'guide.xml')
|
||||
.option('-l, --lang <code>', 'Filter channels by language (ISO 639-2 code)')
|
||||
.option('-t, --timeout <milliseconds>', 'Override the default timeout for each request')
|
||||
.option(
|
||||
'--days <days>',
|
||||
'Override the number of days for which the program will be loaded (defaults to the value from the site config)',
|
||||
value => parseInt(value)
|
||||
)
|
||||
.option(
|
||||
'--maxConnections <number>',
|
||||
'Limit on the number of concurrent requests',
|
||||
value => parseInt(value),
|
||||
1
|
||||
)
|
||||
.option('--cron <expression>', 'Schedule a script run (example: "0 0 * * *")')
|
||||
.option('--gzip', 'Create a compressed version of the guide as well', false)
|
||||
.parse(process.argv)
|
||||
|
||||
export type GrabOptions = {
|
||||
site?: string
|
||||
channels?: string
|
||||
output: string
|
||||
gzip: boolean
|
||||
maxConnections: number
|
||||
timeout?: string
|
||||
lang?: string
|
||||
days?: number
|
||||
cron?: string
|
||||
}
|
||||
|
||||
const options: GrabOptions = program.opts()
|
||||
|
||||
async function main() {
|
||||
if (!options.site && !options.channels)
|
||||
throw new Error('One of the arguments must be presented: `--site` or `--channels`')
|
||||
|
||||
const logger = new Logger()
|
||||
|
||||
logger.start('staring...')
|
||||
|
||||
logger.info('config:')
|
||||
logger.tree(options)
|
||||
|
||||
logger.info(`loading channels...`)
|
||||
const storage = new Storage()
|
||||
const parser = new ChannelsParser({ storage })
|
||||
|
||||
let files: string[] = []
|
||||
if (options.site) {
|
||||
files = await storage.list(path.join(SITES_DIR, `${options.site}/*.channels.xml`))
|
||||
} else if (options.channels) {
|
||||
files = await storage.list(options.channels)
|
||||
}
|
||||
|
||||
let parsedChannels = new Collection()
|
||||
for (let filepath of files) {
|
||||
parsedChannels = parsedChannels.concat(await parser.parse(filepath))
|
||||
}
|
||||
if (options.lang) {
|
||||
parsedChannels = parsedChannels.filter((channel: Channel) => channel.lang === options.lang)
|
||||
}
|
||||
logger.info(` found ${parsedChannels.count()} channels`)
|
||||
|
||||
logger.info('creating queue...')
|
||||
const queue = new Queue({
|
||||
parsedChannels,
|
||||
logger,
|
||||
options
|
||||
})
|
||||
await queue.create()
|
||||
logger.info(` added ${queue.size()} items`)
|
||||
|
||||
const job = new Job({
|
||||
queue,
|
||||
logger,
|
||||
options
|
||||
})
|
||||
|
||||
let runIndex = 1
|
||||
if (options.cron) {
|
||||
const cronJob = new CronJob(options.cron, async () => {
|
||||
logger.info(`run #${runIndex}:`)
|
||||
const timer = new Timer()
|
||||
timer.start()
|
||||
await job.run()
|
||||
runIndex++
|
||||
logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`)
|
||||
})
|
||||
cronJob.start()
|
||||
} else {
|
||||
logger.info(`run #${runIndex}:`)
|
||||
const timer = new Timer()
|
||||
timer.start()
|
||||
await job.run()
|
||||
logger.success(` done in ${timer.format('HH[h] mm[m] ss[s]')}`)
|
||||
}
|
||||
|
||||
logger.info('finished')
|
||||
}
|
||||
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue