iptv-org-mirror/epg
1
0
Fork 0
mirror of https://github.com/iptv-org/epg.git synced 2025-05-10 00:50:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Aleksandr Statciuk 2022-01-09 20:14:41 +03:00
parent cea7fc2cbb
commit cfdf85bd01
7 changed files with 199 additions and 161 deletions
Download patch file Download diff file Expand all files Collapse all files

299
scripts/commands/generate-guides.js
View file

@ -4,21 +4,26 @@ const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc') const utc = require('dayjs/plugin/utc')
dayjs.extend(utc) dayjs.extend(utc)
let programs = {} let channels = []
let programs = []
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs' const DB_DIR = process.env.DB_DIR || 'scripts/database'
const PUBLIC_PATH = process.env.PUBLIC_PATH || '.' const PUBLIC_DIR = process.env.PUBLIC_DIR || '.gh-pages'
async function main() { async function main() {
await setUp() await setUp()
await generateGuideXML() await generateChannelsJson()
} }
main() main()
async function createChannelsJson() { async function setUp() {
logger.info('Creating channels.json...') channels = await db.channels.find({})
}
async function generateChannelsJson() {
logger.info('Generating channels.json...')
let items = channels let items = channels
items = _.sortBy(items, item => item.name) items = _.sortBy(items, item => item.name)
@ -47,180 +52,174 @@ async function createChannelsJson() {
items = Object.values(buffer) items = Object.values(buffer)
await file.create(`${OUTPUT_PATH}/channels.json`, JSON.stringify(items, null, 2)) await file.create(`${PUBLIC_DIR}/api/channels.json`, JSON.stringify(items))
} }
async function createProgramsJson() { // async function createProgramsJson() {
logger.info('Creating programs.json...') // logger.info('Creating programs.json...')
let items = programs // let items = programs
items = _.sortBy(items, ['channel', 'start']) // items = _.sortBy(items, ['channel', 'start'])
items = _.groupBy(items, 'channel') // items = _.groupBy(items, 'channel')
for (let channel in items) { // for (let channel in items) {
let programs = items[channel] // let programs = items[channel]
programs = Object.values(_.groupBy(programs, i => i.site))[0] // programs = Object.values(_.groupBy(programs, i => i.site))[0]
let slots = _.groupBy(programs, i => `${i.start}_${i.stop}`) // let slots = _.groupBy(programs, i => `${i.start}_${i.stop}`)
for (let slotId in slots) { // for (let slotId in slots) {
let program = { // let program = {
channel, // channel,
site: null, // site: null,
title: [], // title: [],
description: [], // description: [],
categories: [], // categories: [],
icons: [], // icons: [],
start: null, // start: null,
stop: null // stop: null
} // }
slots[slotId].forEach(item => { // slots[slotId].forEach(item => {
program.site = item.site // program.site = item.site
if (item.title) program.title.push({ lang: item.lang, value: item.title }) // if (item.title) program.title.push({ lang: item.lang, value: item.title })
if (item.description) // if (item.description)
program.description.push({ // program.description.push({
lang: item.lang, // lang: item.lang,
value: item.description // value: item.description
}) // })
if (item.category) program.categories.push({ lang: item.lang, value: item.category }) // if (item.category) program.categories.push({ lang: item.lang, value: item.category })
if (item.icon) program.icons.push(item.icon) // if (item.icon) program.icons.push(item.icon)
program.start = item.start // program.start = item.start
program.stop = item.stop // program.stop = item.stop
}) // })
slots[slotId] = program // slots[slotId] = program
} // }
items[channel] = Object.values(slots) // items[channel] = Object.values(slots)
} // }
// console.log(items)
await file.create(`${OUTPUT_PATH}/programs.json`, JSON.stringify(items, null, 2)) // await file.create(`${PUBLIC_PATH}/api/programs.json`, JSON.stringify(items, null, 2))
} // }
async function generateGuideXML() { // async function generateGuideXML() {
logger.info(`Generating guide.xml...`) // logger.info(`Generating guide.xml...`)
const channels = Object.keys(programs) // const channels = Object.keys(programs)
let items = await db.find({ xmltv_id: { $in: channels } }) // let items = await db.find({ xmltv_id: { $in: channels } })
items = _.sortBy(items, item => item.name) // items = _.sortBy(items, item => item.name)
let buffer = {} // let buffer = {}
items.forEach(item => { // items.forEach(item => {
if (!buffer[item.xmltv_id]) { // if (!buffer[item.xmltv_id]) {
const countryCode = item.xmltv_id.split('.')[1] // const countryCode = item.xmltv_id.split('.')[1]
buffer[item.xmltv_id] = { // buffer[item.xmltv_id] = {
id: item.xmltv_id, // id: item.xmltv_id,
display_name: [item.name], // display_name: [item.name],
logo: item.logo || null, // logo: item.logo || null,
country: countryCode ? countryCode.toUpperCase() : null, // country: countryCode ? countryCode.toUpperCase() : null,
site: `https://${programs[item.xmltv_id][0].site}` // site: `https://${programs[item.xmltv_id][0].site}`
} // }
} else { // } else {
if (!buffer[item.xmltv_id].logo && item.logo) { // if (!buffer[item.xmltv_id].logo && item.logo) {
buffer[item.xmltv_id].logo = item.logo // buffer[item.xmltv_id].logo = item.logo
} // }
if (!buffer[item.xmltv_id].display_name.includes(item.name)) { // if (!buffer[item.xmltv_id].display_name.includes(item.name)) {
buffer[item.xmltv_id].display_name.push(item.name) // buffer[item.xmltv_id].display_name.push(item.name)
} // }
} // }
}) // })
items = Object.values(buffer) // items = Object.values(buffer)
let outputProgs = [] // let outputProgs = []
for (let ip of Object.values(programs)) { // for (let ip of Object.values(programs)) {
outputProgs = outputProgs.concat(ip) // outputProgs = outputProgs.concat(ip)
} // }
const xml = convertToXMLTV({ channels: items, programs: outputProgs }) // const xml = convertToXMLTV({ channels: items, programs: outputProgs })
await file.write('./guide.xml', xml) // await file.write('./guide.xml', xml)
} // }
async function setUp() { // function convertToXMLTV({ channels, programs }) {
const f = await file.read(`${PUBLIC_PATH}/scripts/output/programs.json`) // let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n`
programs = JSON.parse(f) // for (let channel of channels) {
} // output += `<channel id="${escapeString(channel.id)}">`
// channel.display_name.forEach(displayName => {
// output += `<display-name>${escapeString(displayName)}</display-name>`
// })
// if (channel.logo) {
// const logo = escapeString(channel.logo)
// output += `<icon src="${logo}"/>`
// }
// output += `<url>${channel.site}</url>`
// output += `</channel>\r\n`
// }
function convertToXMLTV({ channels, programs }) { // for (let program of programs) {
let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n` // if (!program) continue
for (let channel of channels) {
output += `<channel id="${escapeString(channel.id)}">`
channel.display_name.forEach(displayName => {
output += `<display-name>${escapeString(displayName)}</display-name>`
})
if (channel.logo) {
const logo = escapeString(channel.logo)
output += `<icon src="${logo}"/>`
}
output += `<url>${channel.site}</url>`
output += `</channel>\r\n`
}
for (let program of programs) { // const start = program.start ? dayjs.unix(program.start).utc().format('YYYYMMDDHHmmss ZZ') : ''
if (!program) continue // const stop = program.stop ? dayjs.unix(program.stop).utc().format('YYYYMMDDHHmmss ZZ') : ''
// const icon = escapeString(program.icon)
const start = program.start ? dayjs.unix(program.start).utc().format('YYYYMMDDHHmmss ZZ') : '' // if (start && stop) {
const stop = program.stop ? dayjs.unix(program.stop).utc().format('YYYYMMDDHHmmss ZZ') : '' // output += `<programme start="${start}" stop="${stop}" channel="${escapeString(
const icon = escapeString(program.icon) // program.channel
// )}">`
if (start && stop) { // program.title.forEach(title => {
output += `<programme start="${start}" stop="${stop}" channel="${escapeString( // output += `<title lang="${title.lang}">${escapeString(title.value)}</title>`
program.channel // })
)}">`
program.title.forEach(title => { // program.description.forEach(description => {
output += `<title lang="${title.lang}">${escapeString(title.value)}</title>` // output += `<desc lang="${description.lang}">${escapeString(description.value)}</desc>`
}) // })
program.description.forEach(description => { // program.categories.forEach(category => {
output += `<desc lang="${description.lang}">${escapeString(description.value)}</desc>` // output += `<category lang="${category.lang}">${escapeString(category.value)}</category>`
}) // })
program.categories.forEach(category => { // program.icons.forEach(icon => {
output += `<category lang="${category.lang}">${escapeString(category.value)}</category>` // output += `<icon src="${icon}"/>`
}) // })
program.icons.forEach(icon => { // output += '</programme>\r\n'
output += `<icon src="${icon}"/>` // }
}) // }
output += '</programme>\r\n' // output += '</tv>'
}
}
output += '</tv>' // return output
// }
return output // function escapeString(string, defaultValue = '') {
} // if (!string) return defaultValue
function escapeString(string, defaultValue = '') { // const regex = new RegExp(
if (!string) return defaultValue // '((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
// 'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
// 'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
// '|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
// 'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
// '[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
// 'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
// '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
// 'g'
// )
const regex = new RegExp( // string = String(string || '').replace(regex, '')
'((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
'|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
'[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
'(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
'g'
)
string = String(string || '').replace(regex, '') // return string
// .replace(/&/g, '&amp;')
return string // .replace(/</g, '&lt;')
.replace(/&/g, '&amp;') // .replace(/>/g, '&gt;')
.replace(/</g, '&lt;') // .replace(/"/g, '&quot;')
.replace(/>/g, '&gt;') // .replace(/'/g, '&apos;')
.replace(/"/g, '&quot;') // .replace(/\n|\r/g, ' ')
.replace(/'/g, '&apos;') // .replace(/ +/g, ' ')
.replace(/\n|\r/g, ' ') // .trim()
.replace(/ +/g, ' ') // }
.trim()
}

4
scripts/commands/load-cluster.js
View file

@ -8,13 +8,13 @@ const options = program
.parse(process.argv) .parse(process.argv)
.opts() .opts()
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs' const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
async function main() { async function main() {
logger.info('Starting...') logger.info('Starting...')
timer.start() timer.start()
const clusterLog = `${LOGS_PATH}/load-cluster/cluster_${options.clusterId}.log` const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
logger.info(`Loading cluster: ${options.clusterId}`) logger.info(`Loading cluster: ${options.clusterId}`)
logger.info(`Creating '${clusterLog}'...`) logger.info(`Creating '${clusterLog}'...`)
await file.create(clusterLog) await file.create(clusterLog)

4
scripts/commands/save-results.js
View file

@ -1,11 +1,11 @@
const { db, logger, file, parser } = require('../core') const { db, logger, file, parser } = require('../core')
const _ = require('lodash') const _ = require('lodash')
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs' const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
async function main() { async function main() {
db.programs.reset() db.programs.reset()
const files = await file.list(`${LOGS_PATH}/load-cluster/cluster_*.log`) const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`)
for (const filepath of files) { for (const filepath of files) {
const results = await parser.parseLogs(filepath) const results = await parser.parseLogs(filepath)
results.forEach(result => { results.forEach(result => {

1
tests/__data__/input/database/channels.db Normal file
View file

@ -0,0 +1 @@
{"lang":"ca","xmltv_id":"AndorraTV.ad","site_id":"atv","name":"Andorra TV","site":"andorradifusio.ad","channelsPath":"sites/andorradifusio.ad/andorradifusio.ad_ad.channels.xml","configPath":"sites/andorradifusio.ad/andorradifusio.ad.config.js","cluster_id":1,"_id":"K1kaxwsWVjsRIZL6"}

23
tests/__data__/input/database/programs.db Normal file
View file

@ -0,0 +1,23 @@
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732000,"stop":1641732300,"site":"andorradifusio.ad","_id":"1SNKtqTU4MWuSlc4"}
{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600,"site":"andorradifusio.ad","_id":"2Ijpxgj73SQDWe2h"}
{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641766800,"stop":1641767400,"site":"andorradifusio.ad","_id":"656oToxGrnZgxI9e"}
{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736200,"stop":1641736800,"site":"andorradifusio.ad","_id":"6vbUqroQtpLHVlvm"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641743400,"stop":1641750900,"site":"andorradifusio.ad","_id":"876l6GHzYXQSOa8i"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641741900,"stop":1641743400,"site":"andorradifusio.ad","_id":"8X3lJJIsn3GK7YBN"}
{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641729600,"stop":1641730800,"site":"andorradifusio.ad","_id":"9cN1SblkKwDdb5Y7"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732300,"stop":1641733800,"site":"andorradifusio.ad","_id":"D9cSGXpT8qTgFBg9"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736800,"stop":1641738300,"site":"andorradifusio.ad","_id":"E4Nu7gXml4Ue1bN6"}
{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200,"site":"andorradifusio.ad","_id":"E7Y7eiu5bbifr2F8"}
{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641772800,"stop":1641776400,"site":"andorradifusio.ad","_id":"OrlfYaCHaXvvJXIK"}
{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757500,"stop":1641759000,"site":"andorradifusio.ad","_id":"XKEYb3MHbmHZaD5A"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641738300,"stop":1641741900,"site":"andorradifusio.ad","_id":"Z535Ou887Zj7dEUB"}
{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641761100,"stop":1641763800,"site":"andorradifusio.ad","_id":"amdpu94GcCGZDkUW"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641730800,"stop":1641732000,"site":"andorradifusio.ad","_id":"ewwNaH8scbFl75W9"}
{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641763800,"stop":1641766800,"site":"andorradifusio.ad","_id":"jGh8MthqsqDZZ33C"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641715200,"stop":1641718800,"site":"andorradifusio.ad","_id":"jXjG1SkfkxLYUu5E"}
{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641759000,"stop":1641761100,"site":"andorradifusio.ad","_id":"omUQMThgK6h0xVrD"}
{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641767400,"stop":1641772800,"site":"andorradifusio.ad","_id":"owJmOhLT8OmAqq3B"}
{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641750900,"stop":1641753600,"site":"andorradifusio.ad","_id":"pL7Tr5hsKOvaYSHk"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641733800,"stop":1641736200,"site":"andorradifusio.ad","_id":"qsLuxMRcSMvKab3N"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757200,"stop":1641757500,"site":"andorradifusio.ad","_id":"uPKYB5PCOt67vvqO"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641753600,"stop":1641757200,"site":"andorradifusio.ad","_id":"xpnrcHZrrMiD4QcL"}

21
tests/commands/generate-guides.test.js Normal file
View file

@ -0,0 +1,21 @@
const fs = require('fs')
const path = require('path')
const { execSync } = require('child_process')
beforeEach(() => {
fs.rmdirSync('tests/__data__/output', { recursive: true })
fs.mkdirSync('tests/__data__/output')
fs.mkdirSync('tests/__data__/output/api')
})
it('can generate channels.json', () => {
const result = execSync(
'PUBLIC_DIR=tests/__data__/output DB_DIR=tests/__data__/input/database node scripts/commands/generate-guides.js',
{ encoding: 'utf8' }
)
const json = fs.readFileSync(path.resolve('tests/__data__/output/api/channels.json'), {
encoding: 'utf8'
})
const parsed = JSON.parse(json)
expect(parsed[0]).toMatchObject({})
})

8
tests/commands/load-cluster.test.js
View file

@ -5,17 +5,11 @@ const { execSync } = require('child_process')
beforeEach(() => { beforeEach(() => {
fs.rmdirSync('tests/__data__/output', { recursive: true }) fs.rmdirSync('tests/__data__/output', { recursive: true })
fs.mkdirSync('tests/__data__/output') fs.mkdirSync('tests/__data__/output')
fs.copyFileSync('tests/__data__/input/channels.db', 'tests/__data__/temp/channels.db')
})
afterEach(() => {
fs.rmdirSync('tests/__data__/temp', { recursive: true })
fs.mkdirSync('tests/__data__/temp')
}) })
it('can load cluster', () => { it('can load cluster', () => {
const result = execSync( const result = execSync(
'DB_DIR=tests/__data__/temp LOGS_PATH=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1', 'DB_DIR=tests/__data__/input/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1',
{ encoding: 'utf8' } { encoding: 'utf8' }
) )
const logs = fs.readFileSync( const logs = fs.readFileSync(