iptv-org-mirror/epg
1
0
Fork 0
mirror of https://github.com/iptv-org/epg.git synced 2025-05-09 08:30:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
freearhey 2021-03-10 21:31:32 +03:00
parent ce8565f133
commit 96eb13d2e6
6 changed files with 1470 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

1118
.gh-pages/guide_ru.xml Normal file
View file

File diff suppressed because it is too large Load diff

69
bin/epg-grabber/index.js
View file

@ -3,6 +3,7 @@
const fs = require('fs')
const path = require('path')
const axios = require('axios')
const axiosDelayAdapter = require('axios-delay').default
const utils = require('./utils')
const { Command } = require('commander')
const program = new Command()
@ -22,61 +23,47 @@ program
const options = program.opts()
const config = utils.parseConfig(options.config)
const sites = utils.loadSites(options.sites)
return console.log(config)
const client = axios.create({
adapter: axiosDelayAdapter(axios.defaults.adapter),
headers: { 'User-Agent': config.userAgent }
})
const sites = {
'tv.yandex.ru': {
url: function ({ date, channel }) {
return `https://tv.yandex.ru/channel/${channel.site_id}?date=${date.format('YYYY-MM-DD')}`
},
parser: function ({ channel, content }) {
const initialState = content.match(/window.__INITIAL_STATE__ = (.*);/i)[1]
const data = JSON.parse(initialState, null, 2)
const programs = data.channel.schedule.events.map(i => {
return {
title: i.title,
description: i.program.description,
start: i.start,
stop: i.finish,
lang: 'ru',
channel: channel['xmltv_id']
}
})
return programs
}
}
}
function main() {
async function main() {
const d = dayjs.utc()
const dates = Array.from({ length: config.days }, (_, i) => d.add(i, 'd'))
const channels = config.channels
const promises = []
const requests = []
channels.forEach(channel => {
const site = sites[channel.site]
dates.forEach(date => {
const url = site.url({ date, channel })
const promise = axios.get(url).then(response => {
return site.parser({ channel, content: response.data })
const promise = client.get(url).catch(console.log)
requests.push({
url,
site,
channel,
promise
})
promises.push(promise)
})
})
Promise.allSettled(promises).then(results => {
let programs = []
results.forEach(result => {
if (result.status === 'fulfilled') {
programs = programs.concat(result.value)
}
})
let programs = []
for (let request of requests) {
const progs = await request.promise
.then(response => {
const channel = request.channel
console.log(`${channel.site} - ${channel.xmltv_id}`)
const xml = utils.convertToXMLTV({ channels, programs })
fs.writeFileSync(path.resolve(__dirname, config.filename), xml)
})
return request.site.parser({ channel, content: response.data })
})
.then(utils.sleep(3000))
programs = programs.concat(progs)
}
const xml = utils.convertToXMLTV({ channels, programs })
fs.writeFileSync(path.resolve(__dirname, config.filename), xml)
}
main()

32
bin/epg-grabber/utils.js
View file

@ -2,16 +2,15 @@ const fs = require('fs')
const path = require('path')
const convert = require('xml-js')
const dayjs = require('dayjs')
const glob = require('glob')
const utils = {}
utils.convertToXMLTV = function ({ channels, programs }) {
let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>`
let output = '<?xml version="1.0" encoding="UTF-8" ?><tv>'
for (let channel of channels) {
output += `
<channel id="${channel['xmltv_id']}">
<display-name>${channel.name}</display-name>
</channel>`
<channel id="${channel['xmltv_id']}"><display-name>${channel.name}</display-name></channel>`
}
for (let program of programs) {
@ -19,14 +18,13 @@ utils.convertToXMLTV = function ({ channels, programs }) {
const stop = dayjs(program.stop).format('YYYYMMDDHHmmss ZZ')
output += `
<programme start="${start}" stop="${stop}" channel="${program.channel}">
<title lang="${program.lang}">${program.title}</title>`
<programme start="${start}" stop="${stop}" channel="${program.channel}"><title lang="${program.lang}">${program.title}</title>`
if (program.category) {
output += `<category lang="${program.lang}">${program.category}</category>`
}
output += `</programme>`
output += '</programme>'
}
output += '</tv>'
@ -34,8 +32,8 @@ utils.convertToXMLTV = function ({ channels, programs }) {
return output
}
utils.parseConfig = function (config) {
const xml = fs.readFileSync(path.resolve(process.cwd(), config), {
utils.parseConfig = function (configPath) {
const xml = fs.readFileSync(path.resolve(process.cwd(), configPath), {
encoding: 'utf-8'
})
const result = convert.xml2js(xml)
@ -66,4 +64,20 @@ utils.getElementText = function (name, elements) {
return el ? el.elements.find(el => el.type === 'text').text : null
}
utils.loadSites = function (sitesPath) {
const sites = {}
glob.sync(`${sitesPath}/*.js`).forEach(function (file) {
const name = path.parse(file).name
sites[name] = require(path.resolve(file))
})
return sites
}
utils.sleep = function (ms) {
return function (x) {
return new Promise(resolve => setTimeout(() => resolve(x), ms))
}
}
module.exports = utils

21
config/ru/sites/tv.yandex.ru.js Normal file
View file

@ -0,0 +1,21 @@
module.exports = {
url: function ({ date, channel }) {
return `https://tv.yandex.ru/channel/${channel.site_id}?date=${date.format('YYYY-MM-DD')}`
},
parser: function ({ channel, content }) {
const initialState = content.match(/window.__INITIAL_STATE__ = (.*);/i)[1]
const data = JSON.parse(initialState, null, 2)
const programs = data.channel.schedule.events.map(i => {
return {
title: i.title,
description: i.program.description,
start: i.start,
stop: i.finish,
lang: 'ru',
channel: channel['xmltv_id']
}
})
return programs
}
}

277
package-lock.json generated
View file

@ -7,8 +7,10 @@
"license": "MIT",
"dependencies": {
"axios": "^0.21.1",
"axios-delay": "^1.0.0-rc6",
"commander": "^7.1.0",
"dayjs": "^1.10.4",
"glob": "^7.1.6",
"xml-js": "^1.6.11"
}
},
@ -20,6 +22,49 @@
"follow-redirects": "^1.10.0"
}
},
"node_modules/axios-delay": {
"version": "1.0.0-rc6",
"resolved": "https://registry.npmjs.org/axios-delay/-/axios-delay-1.0.0-rc6.tgz",
"integrity": "sha512-l52QkvPb3/T1b5NqyjhXt3LU5lkYOSn0AFb9tXI8qApBv9kGvLQj0PG69s7T9aVtJDMsKakekIMNhJMyXzTJZg==",
"dependencies": {
"axios": "^0.18.0"
}
},
"node_modules/axios-delay/node_modules/axios": {
"version": "0.18.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz",
"integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==",
"deprecated": "Critical security vulnerability fixed in v0.21.1. For more information, see https://github.com/axios/axios/pull/3410",
"dependencies": {
"follow-redirects": "1.5.10",
"is-buffer": "^2.0.2"
}
},
"node_modules/axios-delay/node_modules/follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
"integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
"dependencies": {
"debug": "=3.1.0"
},
"engines": {
"node": ">=4.0"
}
},
"node_modules/balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"node_modules/brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dependencies": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"node_modules/commander": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-7.1.0.tgz",
@ -28,11 +73,24 @@
"node": ">= 10"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"node_modules/dayjs": {
"version": "1.10.4",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.4.tgz",
"integrity": "sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw=="
},
"node_modules/debug": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"dependencies": {
"ms": "2.0.0"
}
},
"node_modules/follow-redirects": {
"version": "1.13.3",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz",
@ -52,11 +110,108 @@
}
}
},
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"node_modules/glob": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
},
"engines": {
"node": "*"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dependencies": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"node_modules/is-buffer": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
"integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"engines": {
"node": ">=4"
}
},
"node_modules/minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dependencies": {
"brace-expansion": "^1.1.7"
},
"engines": {
"node": "*"
}
},
"node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dependencies": {
"wrappy": "1"
}
},
"node_modules/path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"node_modules/wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"node_modules/xml-js": {
"version": "1.6.11",
"resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",
@ -78,26 +233,148 @@
"follow-redirects": "^1.10.0"
}
},
"axios-delay": {
"version": "1.0.0-rc6",
"resolved": "https://registry.npmjs.org/axios-delay/-/axios-delay-1.0.0-rc6.tgz",
"integrity": "sha512-l52QkvPb3/T1b5NqyjhXt3LU5lkYOSn0AFb9tXI8qApBv9kGvLQj0PG69s7T9aVtJDMsKakekIMNhJMyXzTJZg==",
"requires": {
"axios": "^0.18.0"
},
"dependencies": {
"axios": {
"version": "0.18.1",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz",
"integrity": "sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g==",
"requires": {
"follow-redirects": "1.5.10",
"is-buffer": "^2.0.2"
}
},
"follow-redirects": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
"integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==",
"requires": {
"debug": "=3.1.0"
}
}
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
}
},
"commander": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-7.1.0.tgz",
"integrity": "sha512-pRxBna3MJe6HKnBGsDyMv8ETbptw3axEdYHoqNh7gu5oDcew8fs0xnivZGm06Ogk8zGAJ9VX+OPEr2GXEQK4dg=="
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
},
"dayjs": {
"version": "1.10.4",
"resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.4.tgz",
"integrity": "sha512-RI/Hh4kqRc1UKLOAf/T5zdMMX5DQIlDxwUe3wSyMMnEbGunnpENCdbUgM+dW7kXidZqCttBrmw7BhN4TMddkCw=="
},
"debug": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"requires": {
"ms": "2.0.0"
}
},
"follow-redirects": {
"version": "1.13.3",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.3.tgz",
"integrity": "sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA=="
},
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
},
"glob": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^3.0.4",
"once": "^1.3.0",
"path-is-absolute": "^1.0.0"
}
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"requires": {
"once": "^1.3.0",
"wrappy": "1"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"is-buffer": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
"integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": {
"brace-expansion": "^1.1.7"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
},
"sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"xml-js": {
"version": "1.6.11",
"resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",

4
package.json
View file

@ -1,15 +1,17 @@
{
"name": "epg",
"scripts": {
"update": "./bin/epg-grabber/index.js --config=config/ru/config.xml"
"update": "./bin/epg-grabber/index.js --config=config/ru/config.xml --sites=config/ru/sites"
},
"private": true,
"author": "Arhey",
"license": "MIT",
"dependencies": {
"axios": "^0.21.1",
"axios-delay": "^1.0.0-rc6",
"commander": "^7.1.0",
"dayjs": "^1.10.4",
"glob": "^7.1.6",
"xml-js": "^1.6.11"
}
}