This commit is contained in:
Aleksandr Statciuk 2022-01-09 20:14:41 +03:00
parent cea7fc2cbb
commit cfdf85bd01
7 changed files with 199 additions and 161 deletions

View file

@ -4,21 +4,26 @@ const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
dayjs.extend(utc)
let programs = {}
let channels = []
let programs = []
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs'
const PUBLIC_PATH = process.env.PUBLIC_PATH || '.'
const DB_DIR = process.env.DB_DIR || 'scripts/database'
const PUBLIC_DIR = process.env.PUBLIC_DIR || '.gh-pages'
async function main() {
await setUp()
await generateGuideXML()
await generateChannelsJson()
}
main()
async function createChannelsJson() {
logger.info('Creating channels.json...')
async function setUp() {
channels = await db.channels.find({})
}
async function generateChannelsJson() {
logger.info('Generating channels.json...')
let items = channels
items = _.sortBy(items, item => item.name)
@ -47,180 +52,174 @@ async function createChannelsJson() {
items = Object.values(buffer)
await file.create(`${OUTPUT_PATH}/channels.json`, JSON.stringify(items, null, 2))
await file.create(`${PUBLIC_DIR}/api/channels.json`, JSON.stringify(items))
}
async function createProgramsJson() {
logger.info('Creating programs.json...')
// async function createProgramsJson() {
// logger.info('Creating programs.json...')
let items = programs
// let items = programs
items = _.sortBy(items, ['channel', 'start'])
items = _.groupBy(items, 'channel')
// items = _.sortBy(items, ['channel', 'start'])
// items = _.groupBy(items, 'channel')
for (let channel in items) {
let programs = items[channel]
programs = Object.values(_.groupBy(programs, i => i.site))[0]
let slots = _.groupBy(programs, i => `${i.start}_${i.stop}`)
// for (let channel in items) {
// let programs = items[channel]
// programs = Object.values(_.groupBy(programs, i => i.site))[0]
// let slots = _.groupBy(programs, i => `${i.start}_${i.stop}`)
for (let slotId in slots) {
let program = {
channel,
site: null,
title: [],
description: [],
categories: [],
icons: [],
start: null,
stop: null
}
// for (let slotId in slots) {
// let program = {
// channel,
// site: null,
// title: [],
// description: [],
// categories: [],
// icons: [],
// start: null,
// stop: null
// }
slots[slotId].forEach(item => {
program.site = item.site
if (item.title) program.title.push({ lang: item.lang, value: item.title })
if (item.description)
program.description.push({
lang: item.lang,
value: item.description
})
if (item.category) program.categories.push({ lang: item.lang, value: item.category })
if (item.icon) program.icons.push(item.icon)
program.start = item.start
program.stop = item.stop
})
// slots[slotId].forEach(item => {
// program.site = item.site
// if (item.title) program.title.push({ lang: item.lang, value: item.title })
// if (item.description)
// program.description.push({
// lang: item.lang,
// value: item.description
// })
// if (item.category) program.categories.push({ lang: item.lang, value: item.category })
// if (item.icon) program.icons.push(item.icon)
// program.start = item.start
// program.stop = item.stop
// })
slots[slotId] = program
}
// slots[slotId] = program
// }
items[channel] = Object.values(slots)
}
// console.log(items)
// items[channel] = Object.values(slots)
// }
await file.create(`${OUTPUT_PATH}/programs.json`, JSON.stringify(items, null, 2))
}
// await file.create(`${PUBLIC_PATH}/api/programs.json`, JSON.stringify(items, null, 2))
// }
async function generateGuideXML() {
logger.info(`Generating guide.xml...`)
// async function generateGuideXML() {
// logger.info(`Generating guide.xml...`)
const channels = Object.keys(programs)
let items = await db.find({ xmltv_id: { $in: channels } })
items = _.sortBy(items, item => item.name)
// const channels = Object.keys(programs)
// let items = await db.find({ xmltv_id: { $in: channels } })
// items = _.sortBy(items, item => item.name)
let buffer = {}
items.forEach(item => {
if (!buffer[item.xmltv_id]) {
const countryCode = item.xmltv_id.split('.')[1]
// let buffer = {}
// items.forEach(item => {
// if (!buffer[item.xmltv_id]) {
// const countryCode = item.xmltv_id.split('.')[1]
buffer[item.xmltv_id] = {
id: item.xmltv_id,
display_name: [item.name],
logo: item.logo || null,
country: countryCode ? countryCode.toUpperCase() : null,
site: `https://${programs[item.xmltv_id][0].site}`
}
} else {
if (!buffer[item.xmltv_id].logo && item.logo) {
buffer[item.xmltv_id].logo = item.logo
}
// buffer[item.xmltv_id] = {
// id: item.xmltv_id,
// display_name: [item.name],
// logo: item.logo || null,
// country: countryCode ? countryCode.toUpperCase() : null,
// site: `https://${programs[item.xmltv_id][0].site}`
// }
// } else {
// if (!buffer[item.xmltv_id].logo && item.logo) {
// buffer[item.xmltv_id].logo = item.logo
// }
if (!buffer[item.xmltv_id].display_name.includes(item.name)) {
buffer[item.xmltv_id].display_name.push(item.name)
}
}
})
// if (!buffer[item.xmltv_id].display_name.includes(item.name)) {
// buffer[item.xmltv_id].display_name.push(item.name)
// }
// }
// })
items = Object.values(buffer)
// items = Object.values(buffer)
let outputProgs = []
for (let ip of Object.values(programs)) {
outputProgs = outputProgs.concat(ip)
}
// let outputProgs = []
// for (let ip of Object.values(programs)) {
// outputProgs = outputProgs.concat(ip)
// }
const xml = convertToXMLTV({ channels: items, programs: outputProgs })
await file.write('./guide.xml', xml)
}
// const xml = convertToXMLTV({ channels: items, programs: outputProgs })
// await file.write('./guide.xml', xml)
// }
async function setUp() {
const f = await file.read(`${PUBLIC_PATH}/scripts/output/programs.json`)
programs = JSON.parse(f)
}
// function convertToXMLTV({ channels, programs }) {
// let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n`
// for (let channel of channels) {
// output += `<channel id="${escapeString(channel.id)}">`
// channel.display_name.forEach(displayName => {
// output += `<display-name>${escapeString(displayName)}</display-name>`
// })
// if (channel.logo) {
// const logo = escapeString(channel.logo)
// output += `<icon src="${logo}"/>`
// }
// output += `<url>${channel.site}</url>`
// output += `</channel>\r\n`
// }
function convertToXMLTV({ channels, programs }) {
let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n`
for (let channel of channels) {
output += `<channel id="${escapeString(channel.id)}">`
channel.display_name.forEach(displayName => {
output += `<display-name>${escapeString(displayName)}</display-name>`
})
if (channel.logo) {
const logo = escapeString(channel.logo)
output += `<icon src="${logo}"/>`
}
output += `<url>${channel.site}</url>`
output += `</channel>\r\n`
}
// for (let program of programs) {
// if (!program) continue
for (let program of programs) {
if (!program) continue
// const start = program.start ? dayjs.unix(program.start).utc().format('YYYYMMDDHHmmss ZZ') : ''
// const stop = program.stop ? dayjs.unix(program.stop).utc().format('YYYYMMDDHHmmss ZZ') : ''
// const icon = escapeString(program.icon)
const start = program.start ? dayjs.unix(program.start).utc().format('YYYYMMDDHHmmss ZZ') : ''
const stop = program.stop ? dayjs.unix(program.stop).utc().format('YYYYMMDDHHmmss ZZ') : ''
const icon = escapeString(program.icon)
// if (start && stop) {
// output += `<programme start="${start}" stop="${stop}" channel="${escapeString(
// program.channel
// )}">`
if (start && stop) {
output += `<programme start="${start}" stop="${stop}" channel="${escapeString(
program.channel
)}">`
// program.title.forEach(title => {
// output += `<title lang="${title.lang}">${escapeString(title.value)}</title>`
// })
program.title.forEach(title => {
output += `<title lang="${title.lang}">${escapeString(title.value)}</title>`
})
// program.description.forEach(description => {
// output += `<desc lang="${description.lang}">${escapeString(description.value)}</desc>`
// })
program.description.forEach(description => {
output += `<desc lang="${description.lang}">${escapeString(description.value)}</desc>`
})
// program.categories.forEach(category => {
// output += `<category lang="${category.lang}">${escapeString(category.value)}</category>`
// })
program.categories.forEach(category => {
output += `<category lang="${category.lang}">${escapeString(category.value)}</category>`
})
// program.icons.forEach(icon => {
// output += `<icon src="${icon}"/>`
// })
program.icons.forEach(icon => {
output += `<icon src="${icon}"/>`
})
// output += '</programme>\r\n'
// }
// }
output += '</programme>\r\n'
}
}
// output += '</tv>'
output += '</tv>'
// return output
// }
return output
}
// function escapeString(string, defaultValue = '') {
// if (!string) return defaultValue
function escapeString(string, defaultValue = '') {
if (!string) return defaultValue
// const regex = new RegExp(
// '((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
// 'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
// 'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
// '|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
// 'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
// '[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
// 'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
// '(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
// 'g'
// )
const regex = new RegExp(
'((?:[\0-\x08\x0B\f\x0E-\x1F\uFFFD\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]))|([\\x7F-\\x84]|[\\x86-\\x9F]|[\\uFDD0-\\uFDEF]|(?:\\uD83F[\\uDFFE\\uDFFF])|(?:\\uD87F[\\uDF' +
'FE\\uDFFF])|(?:\\uD8BF[\\uDFFE\\uDFFF])|(?:\\uD8FF[\\uDFFE\\uDFFF])|(?:\\uD93F[\\uDFFE\\uD' +
'FFF])|(?:\\uD97F[\\uDFFE\\uDFFF])|(?:\\uD9BF[\\uDFFE\\uDFFF])|(?:\\uD9FF[\\uDFFE\\uDFFF])' +
'|(?:\\uDA3F[\\uDFFE\\uDFFF])|(?:\\uDA7F[\\uDFFE\\uDFFF])|(?:\\uDABF[\\uDFFE\\uDFFF])|(?:\\' +
'uDAFF[\\uDFFE\\uDFFF])|(?:\\uDB3F[\\uDFFE\\uDFFF])|(?:\\uDB7F[\\uDFFE\\uDFFF])|(?:\\uDBBF' +
'[\\uDFFE\\uDFFF])|(?:\\uDBFF[\\uDFFE\\uDFFF])(?:[\\0-\\t\\x0B\\f\\x0E-\\u2027\\u202A-\\uD7FF\\' +
'uE000-\\uFFFF]|[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|' +
'(?:[^\\uD800-\\uDBFF]|^)[\\uDC00-\\uDFFF]))',
'g'
)
// string = String(string || '').replace(regex, '')
string = String(string || '').replace(regex, '')
return string
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;')
.replace(/\n|\r/g, ' ')
.replace(/ +/g, ' ')
.trim()
}
// return string
// .replace(/&/g, '&amp;')
// .replace(/</g, '&lt;')
// .replace(/>/g, '&gt;')
// .replace(/"/g, '&quot;')
// .replace(/'/g, '&apos;')
// .replace(/\n|\r/g, ' ')
// .replace(/ +/g, ' ')
// .trim()
// }

View file

@ -8,13 +8,13 @@ const options = program
.parse(process.argv)
.opts()
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs'
const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
async function main() {
logger.info('Starting...')
timer.start()
const clusterLog = `${LOGS_PATH}/load-cluster/cluster_${options.clusterId}.log`
const clusterLog = `${LOGS_DIR}/load-cluster/cluster_${options.clusterId}.log`
logger.info(`Loading cluster: ${options.clusterId}`)
logger.info(`Creating '${clusterLog}'...`)
await file.create(clusterLog)

View file

@ -1,11 +1,11 @@
const { db, logger, file, parser } = require('../core')
const _ = require('lodash')
const LOGS_PATH = process.env.LOGS_PATH || 'scripts/logs'
const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs'
async function main() {
db.programs.reset()
const files = await file.list(`${LOGS_PATH}/load-cluster/cluster_*.log`)
const files = await file.list(`${LOGS_DIR}/load-cluster/cluster_*.log`)
for (const filepath of files) {
const results = await parser.parseLogs(filepath)
results.forEach(result => {

View file

@ -0,0 +1 @@
{"lang":"ca","xmltv_id":"AndorraTV.ad","site_id":"atv","name":"Andorra TV","site":"andorradifusio.ad","channelsPath":"sites/andorradifusio.ad/andorradifusio.ad_ad.channels.xml","configPath":"sites/andorradifusio.ad/andorradifusio.ad.config.js","cluster_id":1,"_id":"K1kaxwsWVjsRIZL6"}

View file

@ -0,0 +1,23 @@
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732000,"stop":1641732300,"site":"andorradifusio.ad","_id":"1SNKtqTU4MWuSlc4"}
{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641718800,"stop":1641729600,"site":"andorradifusio.ad","_id":"2Ijpxgj73SQDWe2h"}
{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641766800,"stop":1641767400,"site":"andorradifusio.ad","_id":"656oToxGrnZgxI9e"}
{"title":"La Terre vue du Sport","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736200,"stop":1641736800,"site":"andorradifusio.ad","_id":"6vbUqroQtpLHVlvm"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641743400,"stop":1641750900,"site":"andorradifusio.ad","_id":"876l6GHzYXQSOa8i"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641741900,"stop":1641743400,"site":"andorradifusio.ad","_id":"8X3lJJIsn3GK7YBN"}
{"title":"Andorra Actualitat (RNA)","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641729600,"stop":1641730800,"site":"andorradifusio.ad","_id":"9cN1SblkKwDdb5Y7"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641732300,"stop":1641733800,"site":"andorradifusio.ad","_id":"D9cSGXpT8qTgFBg9"}
{"title":"Informatiu migdia","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641736800,"stop":1641738300,"site":"andorradifusio.ad","_id":"E4Nu7gXml4Ue1bN6"}
{"title":"InfoNeu ","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641711600,"stop":1641715200,"site":"andorradifusio.ad","_id":"E7Y7eiu5bbifr2F8"}
{"title":"Àrea Andorra Difusió","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641772800,"stop":1641776400,"site":"andorradifusio.ad","_id":"OrlfYaCHaXvvJXIK"}
{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757500,"stop":1641759000,"site":"andorradifusio.ad","_id":"XKEYb3MHbmHZaD5A"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641738300,"stop":1641741900,"site":"andorradifusio.ad","_id":"Z535Ou887Zj7dEUB"}
{"title":"Memòries d'arxiu: 10 anys d'ATV","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641761100,"stop":1641763800,"site":"andorradifusio.ad","_id":"amdpu94GcCGZDkUW"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641730800,"stop":1641732000,"site":"andorradifusio.ad","_id":"ewwNaH8scbFl75W9"}
{"title":"El cafè dels matins","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641763800,"stop":1641766800,"site":"andorradifusio.ad","_id":"jGh8MthqsqDZZ33C"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641715200,"stop":1641718800,"site":"andorradifusio.ad","_id":"jXjG1SkfkxLYUu5E"}
{"title":"Recull setmanal","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641759000,"stop":1641761100,"site":"andorradifusio.ad","_id":"omUQMThgK6h0xVrD"}
{"title":"Informatiu vespre","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641767400,"stop":1641772800,"site":"andorradifusio.ad","_id":"owJmOhLT8OmAqq3B"}
{"title":"La rotonda","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641750900,"stop":1641753600,"site":"andorradifusio.ad","_id":"pL7Tr5hsKOvaYSHk"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641733800,"stop":1641736200,"site":"andorradifusio.ad","_id":"qsLuxMRcSMvKab3N"}
{"title":"El Trànsit","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641757200,"stop":1641757500,"site":"andorradifusio.ad","_id":"uPKYB5PCOt67vvqO"}
{"title":"Club Piolet","description":null,"category":null,"icon":null,"channel":"AndorraTV.ad","lang":"ca","start":1641753600,"stop":1641757200,"site":"andorradifusio.ad","_id":"xpnrcHZrrMiD4QcL"}

View file

@ -0,0 +1,21 @@
const fs = require('fs')
const path = require('path')
const { execSync } = require('child_process')
beforeEach(() => {
fs.rmdirSync('tests/__data__/output', { recursive: true })
fs.mkdirSync('tests/__data__/output')
fs.mkdirSync('tests/__data__/output/api')
})
it('can generate channels.json', () => {
const result = execSync(
'PUBLIC_DIR=tests/__data__/output DB_DIR=tests/__data__/input/database node scripts/commands/generate-guides.js',
{ encoding: 'utf8' }
)
const json = fs.readFileSync(path.resolve('tests/__data__/output/api/channels.json'), {
encoding: 'utf8'
})
const parsed = JSON.parse(json)
expect(parsed[0]).toMatchObject({})
})

View file

@ -5,17 +5,11 @@ const { execSync } = require('child_process')
beforeEach(() => {
fs.rmdirSync('tests/__data__/output', { recursive: true })
fs.mkdirSync('tests/__data__/output')
fs.copyFileSync('tests/__data__/input/channels.db', 'tests/__data__/temp/channels.db')
})
afterEach(() => {
fs.rmdirSync('tests/__data__/temp', { recursive: true })
fs.mkdirSync('tests/__data__/temp')
})
it('can load cluster', () => {
const result = execSync(
'DB_DIR=tests/__data__/temp LOGS_PATH=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1',
'DB_DIR=tests/__data__/input/database LOGS_DIR=tests/__data__/output/logs node scripts/commands/load-cluster.js --cluster-id=1',
{ encoding: 'utf8' }
)
const logs = fs.readFileSync(