mirror of
https://github.com/iptv-org/epg.git
synced 2025-05-10 00:50:09 -04:00
Merge branch 'snrt.ma' of https://github.com/fraudiay79/epg into pr/2534
This commit is contained in:
freearhey
commit
73a80fbb3b
24 changed files with 3829 additions and 635 deletions
17
package-lock.json
generated
17
package-lock.json
generated
|
|
@ -11,6 +11,7 @@
|
|||
"@alex_neo/jest-expect-message": "^1.0.5",
|
||||
"@freearhey/core": "^0.3.1",
|
||||
"@freearhey/search-js": "^0.1.1",
|
||||
"@ntlab/sfetch": "^1.0.0",
|
||||
"@octokit/core": "^4.1.0",
|
||||
"@types/cli-progress": "^3.11.3",
|
||||
"@types/fs-extra": "^11.0.2",
|
||||
|
|
@ -1885,6 +1886,14 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@ntlab/sfetch": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz",
|
||||
"integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==",
|
||||
"dependencies": {
|
||||
"axios": "^1.7.9"
|
||||
}
|
||||
},
|
||||
"node_modules/@octokit/auth-token": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz",
|
||||
|
|
@ -10340,6 +10349,14 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"@ntlab/sfetch": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz",
|
||||
"integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==",
|
||||
"requires": {
|
||||
"axios": "^1.7.9"
|
||||
}
|
||||
},
|
||||
"@octokit/auth-token": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz",
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
"@alex_neo/jest-expect-message": "^1.0.5",
|
||||
"@freearhey/core": "^0.3.1",
|
||||
"@freearhey/search-js": "^0.1.1",
|
||||
"@ntlab/sfetch": "^1.0.0",
|
||||
"@octokit/core": "^4.1.0",
|
||||
"@types/cli-progress": "^3.11.3",
|
||||
"@types/fs-extra": "^11.0.2",
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ module.exports = {
|
|||
},
|
||||
async channels() {
|
||||
const axios = require('axios')
|
||||
const cheerio = require('cheerio')
|
||||
const result = await axios
|
||||
.get(
|
||||
`https://api.firstmedia.com/api/content/tv-guide/list?date=${dayjs().format(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<channels>
|
||||
<channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar#world">I24NEWS عربى</channel>
|
||||
<channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en#usa">I24NEWS English (USA)</channel>
|
||||
<channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishWorld.il" site_id="en#world">I24NEWS English (World)</channel>
|
||||
<channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr#world">I24NEWS Français</channel>
|
||||
<channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar">I24NEWS عربى</channel>
|
||||
<channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en">I24NEWS English (USA)</channel>
|
||||
<channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr">I24NEWS Français</channel>
|
||||
<channel site="i24news.tv" lang="he" xmltv_id="i24NEWSHebrew.il" site_id="he">I24NEWS עברית</channel>
|
||||
</channels>
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ module.exports = {
|
|||
site: 'i24news.tv',
|
||||
days: 2,
|
||||
url: function ({ channel }) {
|
||||
const [lang, region] = channel.site_id.split('#')
|
||||
|
||||
return `https://api.i24news.tv/v2/${lang}/schedules/${region}`
|
||||
return `https://api.i24news.tv/v2/${channel.site_id}/schedules`
|
||||
},
|
||||
parser: function ({ content, date }) {
|
||||
let programs = []
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ dayjs.extend(utc)
|
|||
|
||||
const date = dayjs.utc('2022-03-06', 'YYYY-MM-DD').startOf('d')
|
||||
const channel = {
|
||||
site_id: 'ar#',
|
||||
site_id: 'ar',
|
||||
xmltv_id: 'I24NewsArabic.il'
|
||||
}
|
||||
|
||||
it('can generate valid url', () => {
|
||||
expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules/world')
|
||||
expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules')
|
||||
})
|
||||
|
||||
it('can parse response', () => {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ dayjs.extend(utc)
|
|||
|
||||
const date = dayjs.utc('2022-10-29', 'YYYY-MM-DD').startOf('d')
|
||||
const channel = {
|
||||
site_id: 'default_builtin_channelgroup1#yle-tv1',
|
||||
site_id: '1#yle-tv1',
|
||||
xmltv_id: 'YleTV1.fi'
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,23 @@
|
|||
const _ = require('lodash')
|
||||
const axios = require('axios')
|
||||
const cheerio = require('cheerio')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:mncvision.id')
|
||||
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
doFetch
|
||||
.setCheckResult(false)
|
||||
.setDebugger(debug)
|
||||
|
||||
const languages = { en: 'english', id: 'indonesia' }
|
||||
const cookies = {}
|
||||
const timeout = 30000
|
||||
const nworker = 25
|
||||
|
||||
module.exports = {
|
||||
site: 'mncvision.id',
|
||||
|
|
@ -55,8 +59,6 @@ module.exports = {
|
|||
return await parseItems(content, date, cookies[channel.lang])
|
||||
},
|
||||
async channels({ lang = 'id' }) {
|
||||
const axios = require('axios')
|
||||
const cheerio = require('cheerio')
|
||||
const result = await axios
|
||||
.get('https://www.mncvision.id/schedule')
|
||||
.then(response => response.data)
|
||||
|
|
@ -117,42 +119,31 @@ async function parseItems(content, date, cookies) {
|
|||
const $ = cheerio.load(content)
|
||||
const items = $('tr[valign="top"]').toArray()
|
||||
if (items.length) {
|
||||
const workers = []
|
||||
const n = Math.min(nworker, items.length)
|
||||
while (workers.length < n) {
|
||||
const worker = () => {
|
||||
if (items.length) {
|
||||
const $item = $(items.shift())
|
||||
const done = (description = null) => {
|
||||
const start = parseStart($item, date)
|
||||
const duration = parseDuration($item)
|
||||
const stop = start.add(duration, 'm')
|
||||
programs.push({
|
||||
title: parseTitle($item),
|
||||
season: parseSeason($item),
|
||||
episode: parseEpisode($item),
|
||||
description,
|
||||
start,
|
||||
stop
|
||||
})
|
||||
worker()
|
||||
}
|
||||
loadDescription($item, cookies)
|
||||
.then(description => done(description))
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
const queues = []
|
||||
for (const item of items) {
|
||||
const $item = $(item)
|
||||
const url = $item.find('a').attr('href')
|
||||
const headers = {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
Cookie: cookies,
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
queues.push({ i: $item, url, params: { headers, timeout } })
|
||||
}
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
await doFetch(queues, (queue, res) => {
|
||||
const $item = queue.i
|
||||
const $page = cheerio.load(res)
|
||||
const description = $page('.synopsis').text().trim()
|
||||
const start = parseStart($item, date)
|
||||
const duration = parseDuration($item)
|
||||
const stop = start.add(duration, 'm')
|
||||
programs.push({
|
||||
title: parseTitle($item),
|
||||
season: parseSeason($item),
|
||||
episode: parseEpisode($item),
|
||||
description: description && description !== '-' ? description : null,
|
||||
start,
|
||||
stop
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -168,24 +159,6 @@ function loadLangCookies(channel) {
|
|||
.catch(error => console.error(error.message))
|
||||
}
|
||||
|
||||
async function loadDescription($item, cookies) {
|
||||
const url = $item.find('a').attr('href')
|
||||
if (!url) return null
|
||||
const content = await axios
|
||||
.get(url, {
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest', Cookie: cookies },
|
||||
timeout
|
||||
})
|
||||
.then(r => r.data)
|
||||
.catch(error => console.error(error.message))
|
||||
if (!content) return null
|
||||
|
||||
const $page = cheerio.load(content)
|
||||
const description = $page('.synopsis').text().trim()
|
||||
|
||||
return description !== '-' ? description : null
|
||||
}
|
||||
|
||||
function parseCookies(headers) {
|
||||
const cookies = []
|
||||
if (Array.isArray(headers['set-cookie'])) {
|
||||
|
|
|
|||
|
|
@ -3,15 +3,17 @@ const dayjs = require('dayjs')
|
|||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:mytelly.co.uk')
|
||||
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
doFetch.setDebugger(debug)
|
||||
|
||||
const detailedGuide = true
|
||||
const tz = 'Europe/London'
|
||||
const nworker = 25
|
||||
|
||||
module.exports = {
|
||||
site: 'mytelly.co.uk',
|
||||
|
|
@ -108,8 +110,7 @@ module.exports = {
|
|||
},
|
||||
async channels() {
|
||||
const channels = {}
|
||||
const axios = require('axios')
|
||||
const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }]
|
||||
const queues = [{ t: 'p', method: 'post', url: 'https://www.mytelly.co.uk/getform' }]
|
||||
await doFetch(queues, (queue, res) => {
|
||||
// process form -> provider
|
||||
if (queue.t === 'p') {
|
||||
|
|
@ -118,7 +119,7 @@ module.exports = {
|
|||
.forEach(el => {
|
||||
const opt = $(el)
|
||||
const provider = opt.attr('value')
|
||||
queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
|
||||
queues.push({ t: 'r', method: 'post', url: 'https://www.mytelly.co.uk/getregions', params: { provider } })
|
||||
})
|
||||
}
|
||||
// process provider -> region
|
||||
|
|
@ -134,7 +135,7 @@ module.exports = {
|
|||
u_time: now.format('HHmm'),
|
||||
is_mobile: 1
|
||||
}
|
||||
queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
|
||||
queues.push({ t: 's', method: 'post', url: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
|
||||
}
|
||||
}
|
||||
// process schedule -> channels
|
||||
|
|
@ -191,67 +192,3 @@ function parseText($item) {
|
|||
|
||||
return text
|
||||
}
|
||||
|
||||
async function doFetch(queues, cb) {
|
||||
const axios = require('axios')
|
||||
|
||||
let n = Math.min(nworker, queues.length)
|
||||
const workers = []
|
||||
const adjustWorker = () => {
|
||||
if (queues.length > workers.length && workers.length < nworker) {
|
||||
let nw = Math.min(nworker, queues.length)
|
||||
if (n < nw) {
|
||||
n = nw
|
||||
createWorker()
|
||||
}
|
||||
}
|
||||
}
|
||||
const createWorker = () => {
|
||||
while (workers.length < n) {
|
||||
startWorker()
|
||||
}
|
||||
}
|
||||
const startWorker = () => {
|
||||
const worker = () => {
|
||||
if (queues.length) {
|
||||
const queue = queues.shift()
|
||||
const done = res => {
|
||||
if (res) {
|
||||
cb(queue, res)
|
||||
adjustWorker()
|
||||
}
|
||||
worker()
|
||||
}
|
||||
const url = typeof queue === 'string' ? queue : queue.u
|
||||
const params = typeof queue === 'object' && queue.params ? queue.params : {}
|
||||
const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
|
||||
debug(`fetch %s with %s`, url, JSON.stringify(params))
|
||||
if (method === 'post') {
|
||||
axios
|
||||
.post(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
} else {
|
||||
axios
|
||||
.get(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
}
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
}
|
||||
createWorker()
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,29 +3,22 @@ const dayjs = require('dayjs')
|
|||
const utc = require('dayjs/plugin/utc')
|
||||
|
||||
let apiVersion
|
||||
let isApiVersionFetched = false
|
||||
|
||||
;(async () => {
|
||||
try {
|
||||
await fetchApiVersion()
|
||||
isApiVersionFetched = true
|
||||
} catch (error) {
|
||||
console.error('Error during script initialization:', error)
|
||||
}
|
||||
})()
|
||||
|
||||
dayjs.extend(utc)
|
||||
|
||||
module.exports = {
|
||||
site: 'pickx.be',
|
||||
days: 2,
|
||||
apiVersion: function () {
|
||||
setApiVersion: function (version) {
|
||||
apiVersion = version
|
||||
},
|
||||
getApiVersion: function () {
|
||||
return apiVersion
|
||||
},
|
||||
fetchApiVersion: fetchApiVersion, // Export fetchApiVersion
|
||||
fetchApiVersion: fetchApiVersion,
|
||||
url: async function ({ channel, date }) {
|
||||
while (!isApiVersionFetched) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100)) // Wait for 100 milliseconds
|
||||
if (!apiVersion) {
|
||||
await fetchApiVersion()
|
||||
}
|
||||
return `https://px-epg.azureedge.net/airings/${apiVersion}/${date.format(
|
||||
'YYYY-MM-DD'
|
||||
|
|
@ -116,7 +109,7 @@ module.exports = {
|
|||
}`
|
||||
}
|
||||
const result = await axios
|
||||
.post('https://api.proximusmwc.be/tiams/v2/graphql', query)
|
||||
.post('https://api.proximusmwc.be/tiams/v3/graphql', query)
|
||||
.then(r => r.data)
|
||||
.catch(console.error)
|
||||
|
||||
|
|
@ -140,53 +133,38 @@ function fetchApiVersion() {
|
|||
return new Promise(async (resolve, reject) => {
|
||||
try {
|
||||
// you'll never find what happened here :)
|
||||
// load pickx bundle and get react version hash (regex).
|
||||
// load the pickx page and get the hash from the MWC configuration.
|
||||
// it's not the best way to get the version but it's the only way to get it.
|
||||
|
||||
// find bundle version
|
||||
const minBundleVer = "https://www.pickx.be/minimal-bundle-version"
|
||||
const bundleVerData = await axios.get(minBundleVer, {
|
||||
headers: {
|
||||
Origin: 'https://www.pickx.be',
|
||||
Referer: 'https://www.pickx.be/'
|
||||
}
|
||||
const hashUrl = 'https://www.pickx.be/nl/televisie/tv-gids';
|
||||
|
||||
const hashData = await axios.get(hashUrl)
|
||||
.then(r => {
|
||||
const re = /"hashes":\["(.*)"\]/
|
||||
const match = r.data.match(re)
|
||||
if (match && match[1]) {
|
||||
return match[1]
|
||||
} else {
|
||||
throw new Error('React app version hash not found')
|
||||
}
|
||||
})
|
||||
.catch(console.error);
|
||||
|
||||
const versionUrl = `https://www.pickx.be/api/s-${hashData}`
|
||||
|
||||
const response = await axios.get(versionUrl, {
|
||||
headers: {
|
||||
Origin: 'https://www.pickx.be',
|
||||
Referer: 'https://www.pickx.be/'
|
||||
}
|
||||
})
|
||||
|
||||
if (bundleVerData.status !== 200) {
|
||||
console.error(`Failed to fetch bundle version. Status: ${bundleVerData.status}`)
|
||||
reject(`Failed to fetch bundle version. Status: ${bundleVerData.status}`)
|
||||
if (response.status === 200) {
|
||||
apiVersion = response.data.version
|
||||
resolve()
|
||||
} else {
|
||||
const bundleVer = bundleVerData.data.version
|
||||
// get the minified JS app bundle
|
||||
const bundleUrl = `https://components.pickx.be/pxReactPlayer/${bundleVer}/bundle.min.js`
|
||||
|
||||
// now, find the react hash inside the bundle URL
|
||||
const bundle = await axios.get(bundleUrl).then(r => {
|
||||
const re = /REACT_APP_VERSION_HASH:"([^"]+)"/
|
||||
const match = r.data.match(re)
|
||||
if (match && match[1]) {
|
||||
return match[1]
|
||||
} else {
|
||||
throw new Error('React app version hash not found')
|
||||
}
|
||||
}).catch(console.error)
|
||||
|
||||
const versionUrl = `https://www.pickx.be/api/s-${bundle.replace('/REACT_APP_VERSION_HASH:"', '')}`
|
||||
|
||||
const response = await axios.get(versionUrl, {
|
||||
headers: {
|
||||
Origin: 'https://www.pickx.be',
|
||||
Referer: 'https://www.pickx.be/'
|
||||
}
|
||||
})
|
||||
|
||||
if (response.status === 200) {
|
||||
apiVersion = response.data.version
|
||||
resolve()
|
||||
} else {
|
||||
console.error(`Failed to fetch API version. Status: ${response.status}`)
|
||||
reject(`Failed to fetch API version. Status: ${response.status}`)
|
||||
}
|
||||
console.error(`Failed to fetch API version. Status: ${response.status}`)
|
||||
reject(`Failed to fetch API version. Status: ${response.status}`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error during fetchApiVersion:', error)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,20 @@
|
|||
const { parser, url, request, fetchApiVersion, apiVersion } = require('./pickx.be.config.js')
|
||||
jest.mock('./pickx.be.config.js', () => {
|
||||
const originalModule = jest.requireActual('./pickx.be.config.js')
|
||||
return {
|
||||
...originalModule,
|
||||
fetchApiVersion: jest.fn(() => Promise.resolve())
|
||||
}
|
||||
})
|
||||
|
||||
const {
|
||||
parser,
|
||||
url,
|
||||
request,
|
||||
fetchApiVersion,
|
||||
setApiVersion,
|
||||
getApiVersion
|
||||
} = require('./pickx.be.config.js')
|
||||
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const dayjs = require('dayjs')
|
||||
|
|
@ -13,12 +29,14 @@ const channel = {
|
|||
xmltv_id: 'Vedia.be'
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
setApiVersion('mockedApiVersion')
|
||||
})
|
||||
|
||||
it('can generate valid url', async () => {
|
||||
await fetchApiVersion()
|
||||
const generatedUrl = await url({ channel, date })
|
||||
const resolvedApiVersion = apiVersion()
|
||||
expect(generatedUrl).toBe(
|
||||
`https://px-epg.azureedge.net/airings/${resolvedApiVersion}/2023-12-13/channel/UID0118?timezone=Europe%2FBrussels`
|
||||
`https://px-epg.azureedge.net/airings/mockedApiVersion/2023-12-13/channel/UID0118?timezone=Europe%2FBrussels`
|
||||
)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -4,16 +4,19 @@ const dayjs = require('dayjs')
|
|||
const timezone = require('dayjs/plugin/timezone')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:rotana.net')
|
||||
|
||||
dayjs.extend(timezone)
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
const tz = 'Asia/Riyadh'
|
||||
const nworker = 25
|
||||
doFetch
|
||||
.setCheckResult(false)
|
||||
.setDebugger(debug)
|
||||
|
||||
const headers = {
|
||||
const tz = 'Asia/Riyadh'
|
||||
const defaultHeaders = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 OPR/104.0.0.0'
|
||||
}
|
||||
|
|
@ -26,7 +29,7 @@ module.exports = {
|
|||
return `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&tz=`
|
||||
},
|
||||
request: {
|
||||
headers,
|
||||
headers: defaultHeaders,
|
||||
timeout: 15000
|
||||
},
|
||||
async parser({ content, headers, channel, date }) {
|
||||
|
|
@ -37,31 +40,20 @@ module.exports = {
|
|||
|
||||
const items = parseItems(content, date)
|
||||
if (items.length) {
|
||||
const workers = []
|
||||
const n = Math.min(nworker, items.length)
|
||||
while (workers.length < n) {
|
||||
const worker = () => {
|
||||
if (items.length) {
|
||||
const item = items.shift()
|
||||
parseProgram(item, channel)
|
||||
.then(() => {
|
||||
programs.push(item)
|
||||
worker()
|
||||
})
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
const queues = []
|
||||
for (const item of items) {
|
||||
const url = `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&itemId=${item.program}`
|
||||
const params = {
|
||||
headers: {
|
||||
...defaultHeaders,
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
cookie: cookies[channel.lang],
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
queues.push({ i: item, url, params })
|
||||
}
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
await doFetch(queues, (queue, res) => {
|
||||
programs.push(parseProgram(queue.i, res))
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -83,68 +75,56 @@ module.exports = {
|
|||
}
|
||||
}
|
||||
|
||||
async function parseProgram(item, channel) {
|
||||
if (item.program) {
|
||||
const url = `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&itemId=${item.program}`
|
||||
const params = {
|
||||
headers: Object.assign({}, headers, { 'X-Requested-With': 'XMLHttpRequest' }),
|
||||
Cookie: cookies[channel.lang]
|
||||
}
|
||||
debug(`fetching description ${url}`)
|
||||
const result = await axios
|
||||
.get(url, params)
|
||||
.then(response => response.data)
|
||||
.catch(console.error)
|
||||
|
||||
const $ = cheerio.load(result)
|
||||
const details = $('.trending-info .row div > span')
|
||||
if (details.length) {
|
||||
for (const el of details[0].children) {
|
||||
switch (el.constructor.name) {
|
||||
case 'Text':
|
||||
if (item.description === undefined) {
|
||||
const desc = $(el).text().trim()
|
||||
if (desc) {
|
||||
item.description = desc
|
||||
}
|
||||
function parseProgram(item, result) {
|
||||
const $ = cheerio.load(result)
|
||||
const details = $('.trending-info .row div > span')
|
||||
if (details.length) {
|
||||
for (const el of details[0].children) {
|
||||
switch (el.constructor.name) {
|
||||
case 'Text':
|
||||
if (item.description === undefined) {
|
||||
const desc = $(el).text().trim()
|
||||
if (desc) {
|
||||
item.description = desc
|
||||
}
|
||||
break;
|
||||
case 'Element':
|
||||
if (el.name === 'span') {
|
||||
const [k, v] = $(el).text().split(':').map(a => a.trim())
|
||||
switch (k) {
|
||||
case 'Category':
|
||||
case 'التصنيف':
|
||||
item.category = v;
|
||||
break;
|
||||
case 'Country':
|
||||
case 'البلد':
|
||||
item.country = v;
|
||||
break;
|
||||
case 'Director':
|
||||
case 'المخرج':
|
||||
item.director = v;
|
||||
break;
|
||||
case 'Language':
|
||||
case 'اللغة':
|
||||
item.language = v;
|
||||
break;
|
||||
case 'Release Year':
|
||||
case 'سنة الإصدار':
|
||||
item.date = v;
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'Element':
|
||||
if (el.name === 'span') {
|
||||
const [k, v] = $(el).text().split(':').map(a => a.trim())
|
||||
switch (k) {
|
||||
case 'Category':
|
||||
case 'التصنيف':
|
||||
item.category = v;
|
||||
break;
|
||||
case 'Country':
|
||||
case 'البلد':
|
||||
item.country = v;
|
||||
break;
|
||||
case 'Director':
|
||||
case 'المخرج':
|
||||
item.director = v;
|
||||
break;
|
||||
case 'Language':
|
||||
case 'اللغة':
|
||||
item.language = v;
|
||||
break;
|
||||
case 'Release Year':
|
||||
case 'سنة الإصدار':
|
||||
item.date = v;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
const img = $('.row > div > img')
|
||||
if (img.length) {
|
||||
item.image = img.attr('src')
|
||||
}
|
||||
delete item.program
|
||||
}
|
||||
const img = $('.row > div > img')
|
||||
if (img.length) {
|
||||
item.image = img.attr('src')
|
||||
}
|
||||
delete item.program
|
||||
return item
|
||||
}
|
||||
|
||||
function parseItems(content, date) {
|
||||
|
|
|
|||
|
|
@ -52,12 +52,11 @@ it('can generate valid arabic url', () => {
|
|||
})
|
||||
|
||||
it('can parse english response', async () => {
|
||||
let result = await parser({
|
||||
const result = (await parser({
|
||||
channel,
|
||||
date,
|
||||
content: fs.readFileSync(path.join(__dirname, '/__data__/content_en.html'))
|
||||
})
|
||||
result = result.map(a => {
|
||||
})).map(a => {
|
||||
a.start = a.start.toJSON()
|
||||
a.stop = a.stop.toJSON()
|
||||
return a
|
||||
|
|
@ -76,12 +75,11 @@ it('can parse english response', async () => {
|
|||
})
|
||||
|
||||
it('can parse arabic response', async () => {
|
||||
let result = await parser({
|
||||
const result = (await parser({
|
||||
channel: channelAr,
|
||||
date,
|
||||
content: fs.readFileSync(path.join(__dirname, '/__data__/content_ar.html'))
|
||||
})
|
||||
result = result.map(a => {
|
||||
})).map(a => {
|
||||
a.start = a.start.toJSON()
|
||||
a.stop = a.stop.toJSON()
|
||||
return a
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
const cheerio = require('cheerio')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:sky.com')
|
||||
|
||||
dayjs.extend(utc)
|
||||
|
||||
const nworker = 10
|
||||
doFetch.setDebugger(debug)
|
||||
|
||||
module.exports = {
|
||||
site: 'sky.com',
|
||||
|
|
@ -48,7 +49,7 @@ module.exports = {
|
|||
},
|
||||
async channels() {
|
||||
const channels = {}
|
||||
const queues = [{ t: 'r', u: 'https://www.sky.com/tv-guide' }]
|
||||
const queues = [{ t: 'r', url: 'https://www.sky.com/tv-guide' }]
|
||||
await doFetch(queues, (queue, res) => {
|
||||
// process regions
|
||||
if (queue.t === 'r') {
|
||||
|
|
@ -56,7 +57,7 @@ module.exports = {
|
|||
const initialData = JSON.parse(decodeURIComponent($('#initialData').text()))
|
||||
initialData.state.epgData.regions
|
||||
.forEach(region => {
|
||||
queues.push({ t: 'c', u: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
|
||||
queues.push({ t: 'c', url: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
|
||||
})
|
||||
}
|
||||
// process channels
|
||||
|
|
@ -78,64 +79,3 @@ module.exports = {
|
|||
return Object.values(channels)
|
||||
}
|
||||
}
|
||||
|
||||
async function doFetch(queues, cb) {
|
||||
const axios = require('axios')
|
||||
let n = Math.min(nworker, queues.length)
|
||||
const workers = []
|
||||
const adjustWorker = () => {
|
||||
if (queues.length > workers.length && workers.length < nworker) {
|
||||
let nw = Math.min(nworker, queues.length)
|
||||
if (n < nw) {
|
||||
n = nw
|
||||
createWorker()
|
||||
}
|
||||
}
|
||||
}
|
||||
const createWorker = () => {
|
||||
while (workers.length < n) {
|
||||
startWorker()
|
||||
}
|
||||
}
|
||||
const startWorker = () => {
|
||||
const worker = () => {
|
||||
if (queues.length) {
|
||||
const queue = queues.shift()
|
||||
const done = (res, headers) => {
|
||||
if (res) {
|
||||
cb(queue, res, headers)
|
||||
adjustWorker()
|
||||
}
|
||||
worker()
|
||||
}
|
||||
const url = typeof queue === 'string' ? queue : queue.u
|
||||
const params = typeof queue === 'object' && queue.params ? queue.params : {}
|
||||
const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
|
||||
if (typeof debug === 'function') {
|
||||
debug(`fetch %s with %s`, url, JSON.stringify(params))
|
||||
}
|
||||
axios[method](url, params)
|
||||
.then(response => {
|
||||
done(response.data, response.headers)
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(`Unable to fetch ${url}: ${err.message}!`)
|
||||
done()
|
||||
})
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
}
|
||||
createWorker()
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
3500
sites/snrt.ma/__data__/content.html
Normal file
3500
sites/snrt.ma/__data__/content.html
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,21 +1,21 @@
|
|||
# snrt.ma
|
||||
|
||||
https://www.snrt.ma/ar/node/1208
|
||||
|
||||
### Download the guide
|
||||
|
||||
```sh
|
||||
npm run grab --- --site=snrt.ma
|
||||
```
|
||||
|
||||
### Update channel list
|
||||
|
||||
```sh
|
||||
npm run channels:parse --- --config=./sites/snrt.ma/snrt.ma.config.js --output=./sites/snrt.ma/snrt.ma.channels.xml
|
||||
```
|
||||
|
||||
### Test
|
||||
|
||||
```sh
|
||||
npm test --- snrt.ma
|
||||
# snrt.ma
|
||||
|
||||
https://www.snrt.ma/ar/node/1208
|
||||
|
||||
### Download the guide
|
||||
|
||||
```sh
|
||||
npm run grab --- --site=snrt.ma
|
||||
```
|
||||
|
||||
### Update channel list
|
||||
|
||||
```sh
|
||||
npm run channels:parse --- --config=./sites/snrt.ma/snrt.ma.config.js --output=./sites/snrt.ma/snrt.ma.channels.xml
|
||||
```
|
||||
|
||||
### Test
|
||||
|
||||
```sh
|
||||
npm test --- snrt.ma
|
||||
```
|
||||
|
|
@ -1,44 +1,45 @@
|
|||
const { parser, url } = require('./snrt.ma.config.js')
|
||||
const cheerio = require('cheerio')
|
||||
const { DateTime } = require('luxon')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
const date = dayjs.utc('2024-12-19', 'YYYY-MM-DD').startOf('d')
|
||||
const channel = { site_id: '1208', xmltv_id: 'AlAoula.ma', lang: 'ar' }
|
||||
|
||||
it('can generate valid url', () => {
|
||||
expect(url({ channel })).toBe('https://www.snrt.ma/ar/node/1208')
|
||||
})
|
||||
|
||||
it('can parse response', () => {
|
||||
const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.html'))
|
||||
const results = parser({ date, content }).map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
|
||||
expect(results).toMatchObject({
|
||||
start: '2024-12-19T07:00:00.000Z',
|
||||
stop: '2024-12-19T07:10:00.000Z',
|
||||
title: 'ﺍﻟﺴﻼﻡ ﺍﻟﻮﻃﻨﻲ + ﺍﻟﻘﺮﺁﻥ ﺍﻟﻜﺮﻳﻢ',
|
||||
category: 'القرآن الكريم'
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
it('can handle empty guide', () => {
|
||||
const result = parser({
|
||||
date,
|
||||
channel: channel,
|
||||
content: '<!DOCTYPE html><html lang="ar" dir="rtl"><head></head><body></body></html>'
|
||||
})
|
||||
expect(result).toMatchObject([])
|
||||
const { parser, url } = require('./snrt.ma.config.js')
|
||||
const cheerio = require('cheerio')
|
||||
const { DateTime } = require('luxon')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
|
||||
const date = dayjs.utc('2024-12-19', 'YYYY-MM-DD').startOf('d')
|
||||
const channel = { site_id: '1208', xmltv_id: 'AlAoula.ma', lang: 'ar' }
|
||||
|
||||
it('can generate valid url', () => {
|
||||
expect(url({ channel })).toBe('https://www.snrt.ma/ar/node/1208')
|
||||
})
|
||||
|
||||
it('can parse response', () => {
|
||||
const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.html'))
|
||||
const results = parser({ date, content }).map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
|
||||
expect(results[0]).toMatchObject({
|
||||
"category": "القرآن الكريم",
|
||||
"description": "",
|
||||
"start": "2024-12-19T06:00:00.000Z",
|
||||
"stop": "2024-12-19T06:10:00.000Z",
|
||||
"stop": "2024-12-19T06:30:00.000Z",
|
||||
"title": "ﺍﻟﺴﻼﻡ ﺍﻟﻮﻃﻨﻲ + ﺍﻟﻘﺮﺁﻥ ﺍﻟﻜﺮﻳﻢ"
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
it('can handle empty guide', () => {
|
||||
const result = parser({
|
||||
date,
|
||||
channel: channel,
|
||||
content: '<!DOCTYPE html><html lang="ar" dir="rtl"><head></head><body></body></html>'
|
||||
})
|
||||
expect(result).toMatchObject([])
|
||||
})
|
||||
|
|
@ -1,14 +1,16 @@
|
|||
const axios = require('axios')
|
||||
const cheerio = require('cheerio')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:startimestv.com')
|
||||
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
const nworker = 5
|
||||
doFetch
|
||||
.setDebugger(debug)
|
||||
.setMaxWorker(5)
|
||||
|
||||
module.exports = {
|
||||
site: 'startimestv.com',
|
||||
|
|
@ -46,7 +48,7 @@ module.exports = {
|
|||
},
|
||||
async channels() {
|
||||
const channels = {}
|
||||
const queues = [{ t: 'a', u: 'https://www.startimestv.com/tv_guide.html' }]
|
||||
const queues = [{ t: 'a', url: 'https://www.startimestv.com/tv_guide.html' }]
|
||||
await doFetch(queues, (queue, res) => {
|
||||
// process area-id
|
||||
if (queue.t === 'a') {
|
||||
|
|
@ -57,7 +59,7 @@ module.exports = {
|
|||
const areaId = dd.attr('area-id')
|
||||
queues.push({
|
||||
t: 's',
|
||||
u: 'https://www.startimestv.com/tv_guide.html',
|
||||
url: 'https://www.startimestv.com/tv_guide.html',
|
||||
params: {
|
||||
headers: {
|
||||
cookie: `default_areaID=${areaId}`
|
||||
|
|
@ -110,66 +112,3 @@ function parseText($item) {
|
|||
|
||||
return text
|
||||
}
|
||||
|
||||
async function doFetch(queues, cb) {
|
||||
const axios = require('axios')
|
||||
|
||||
let n = Math.min(nworker, queues.length)
|
||||
const workers = []
|
||||
const adjustWorker = () => {
|
||||
if (queues.length > workers.length && workers.length < nworker) {
|
||||
let nw = Math.min(nworker, queues.length)
|
||||
if (n < nw) {
|
||||
n = nw
|
||||
createWorker()
|
||||
}
|
||||
}
|
||||
}
|
||||
const createWorker = () => {
|
||||
while (workers.length < n) {
|
||||
startWorker()
|
||||
}
|
||||
}
|
||||
const startWorker = () => {
|
||||
const worker = () => {
|
||||
if (queues.length) {
|
||||
const queue = queues.shift()
|
||||
const done = res => {
|
||||
if (res) {
|
||||
cb(queue, res)
|
||||
adjustWorker()
|
||||
}
|
||||
worker()
|
||||
}
|
||||
const url = typeof queue === 'string' ? queue : queue.u
|
||||
const params = typeof queue === 'object' && queue.params ? queue.params : {}
|
||||
const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
|
||||
debug(`fetch %s with %s`, url, JSON.stringify(params))
|
||||
if (method === 'post') {
|
||||
axios
|
||||
.post(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
} else {
|
||||
axios
|
||||
.get(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
}
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
}
|
||||
createWorker()
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,22 @@
|
|||
const dayjs = require('dayjs')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:tv.yandex.ru')
|
||||
|
||||
doFetch
|
||||
.setDebugger(debug)
|
||||
.setMaxWorker(10)
|
||||
|
||||
// enable to fetch guide description but its take a longer time
|
||||
const detailedGuide = true
|
||||
const nworker = 10
|
||||
|
||||
// update this data by heading to https://tv.yandex.ru and change the values accordingly
|
||||
const cookies = {
|
||||
i: 'dkim62pClrWWC4CShVQYMpVw1ELNVw4XJdL/lzT4E2r05IgcST1GtCA4ho/UyGgW2AO4qftDfZzGX2OHqCzwY7GUkpM=',
|
||||
spravka: 'dD0xNzMyNjgzMTEwO2k9MTgwLjI0OC41OS40MDtEPTkyOUM2MkQ0Mzc3OUNBMUFCNzg3NTIyMEQ4OEJBMEVBMzQ2RUNGNUU5Q0FEQUM5RUVDMTFCNjc1ODA2MThEQTQ3RTY3RTUyRUNBRDdBMTY2OTY1MjMzRDU1QjNGMTc1MDA0NDM3MjBGMUNGQTM5RjA3OUQwRjE2MzQxMUNFOTgxQ0E0RjNGRjRGODNCMEM1QjlGNTg5RkI4NDk0NEM2QjNDQUQ5NkJGRTBFNTVCQ0Y1OTEzMEY0O3U9MTczMjY4MzExMDY3MTA1MzIzNDtoPTA1YWJmMTY0ZmI2MGViNTBhMDUwZWUwMThmYWNiYjhm',
|
||||
i: 'eIUfSP+/mzQWXcH+Cuz8o1vY+D2K8fhBd6Sj0xvbPZeO4l3cY+BvMp8fFIuM17l6UE1Z5+R2a18lP00ex9iYVJ+VT+c=',
|
||||
spravka: 'dD0xNzM0MjA0NjM4O2k9MTI1LjE2NC4xNDkuMjAwO0Q9QTVCQ0IyOTI5RDQxNkU5NkEyOTcwMTNDMzZGMDAzNjRDNTFFNDM4QkE2Q0IyOTJDRjhCOTZDRDIzODdBQzk2MzRFRDc5QTk2Qjc2OEI1MUY5MTM5M0QzNkY3OEQ2OUY3OTUwNkQ3RjBCOEJGOEJDMjAwMTQ0RDUwRkFCMDNEQzJFMDI2OEI5OTk5OUJBNEFERUYwOEQ1MjUwQTE0QTI3RDU1MEQwM0U0O3U9MTczNDIwNDYzODUyNDYyNzg1NDtoPTIxNTc0ZTc2MDQ1ZjcwMDBkYmY0NTVkM2Q2ZWMyM2Y1',
|
||||
yandexuid: '1197179041732383499',
|
||||
yashr: '4682342911732383504',
|
||||
yuidss: '1197179041732383499',
|
||||
user_display: 930,
|
||||
user_display: 824,
|
||||
}
|
||||
const headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 OPR/114.0.0.0',
|
||||
|
|
@ -91,32 +95,35 @@ async function fetchSchedules({ date, content = null }) {
|
|||
|
||||
let mainApi
|
||||
// parse content as schedules and add to queue if more requests is needed
|
||||
const f = (data, src) => {
|
||||
const f = (src, res, headers) => {
|
||||
if (src) {
|
||||
fetches.push(src)
|
||||
}
|
||||
const [q, s] = parseContent(data, date)
|
||||
if (headers) {
|
||||
parseCookies(headers)
|
||||
}
|
||||
const [q, s] = parseContent(res, date)
|
||||
if (!mainApi) {
|
||||
mainApi = true
|
||||
if (caches.region) {
|
||||
queues.push(getUrl(date, caches.region))
|
||||
queues.push(getQueue(getUrl(date, caches.region), src))
|
||||
}
|
||||
}
|
||||
for (const url of q) {
|
||||
if (fetches.indexOf(url) < 0) {
|
||||
queues.push(url)
|
||||
queues.push(getQueue(url, src))
|
||||
}
|
||||
}
|
||||
schedules.push(...s)
|
||||
}
|
||||
// is main html already fetched?
|
||||
if (content) {
|
||||
f(content)
|
||||
f(url, content)
|
||||
} else {
|
||||
queues.push(url)
|
||||
queues.push(getQueue(url, 'https://tv.yandex.ru/'))
|
||||
}
|
||||
// fetch all queues
|
||||
await doFetch(queues, url, f)
|
||||
await doFetch(queues, f)
|
||||
|
||||
return schedules
|
||||
}
|
||||
|
|
@ -129,17 +136,20 @@ async function fetchPrograms({ schedules, date, channel }) {
|
|||
queues.push(
|
||||
...schedule.events
|
||||
.filter(event => date.isSame(event.start, 'day'))
|
||||
.map(event => getUrl(null, caches.region, null, event))
|
||||
.map(event => getQueue(getUrl(null, caches.region, null, event), 'https://tv.yandex.ru/'))
|
||||
)
|
||||
})
|
||||
await doFetch(queues, getUrl(date), content => {
|
||||
await doFetch(queues, (queue, res, headers) => {
|
||||
if (headers) {
|
||||
parseCookies(headers)
|
||||
}
|
||||
// is it a program?
|
||||
if (content?.program) {
|
||||
if (res?.program) {
|
||||
let updated = false
|
||||
schedules.forEach(schedule => {
|
||||
schedule.events.forEach(event => {
|
||||
if (event.channelFamilyId === content.channelFamilyId && event.id === content.id) {
|
||||
Object.assign(event, content)
|
||||
if (event.channelFamilyId === res.channelFamilyId && event.id === res.id) {
|
||||
Object.assign(event, res)
|
||||
updated = true
|
||||
return true
|
||||
}
|
||||
|
|
@ -152,61 +162,6 @@ async function fetchPrograms({ schedules, date, channel }) {
|
|||
})
|
||||
}
|
||||
|
||||
async function doFetch(queues, referer, cb) {
|
||||
if (queues.length) {
|
||||
const workers = []
|
||||
let n = Math.min(nworker, queues.length)
|
||||
while (workers.length < n) {
|
||||
const worker = () => {
|
||||
if (queues.length) {
|
||||
const url = queues.shift()
|
||||
debug(`Fetching ${url}`)
|
||||
const data = {
|
||||
'Origin': 'https://tv.yandex.ru',
|
||||
}
|
||||
if (referer) {
|
||||
data['Referer'] = referer
|
||||
}
|
||||
if (url.indexOf('api') > 0) {
|
||||
data['X-Requested-With'] = 'XMLHttpRequest'
|
||||
}
|
||||
const headers = getHeaders(data)
|
||||
doRequest(url, { headers })
|
||||
.then(res => {
|
||||
cb(res, url)
|
||||
worker()
|
||||
})
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
}
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function doRequest(url, params) {
|
||||
const axios = require('axios')
|
||||
const content = await axios
|
||||
.get(url, params)
|
||||
.then(response => {
|
||||
parseCookies(response.headers)
|
||||
return response.data
|
||||
})
|
||||
.catch(err => console.error(err.message))
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
function parseContent(content, date, checkOnly = false) {
|
||||
const queues = []
|
||||
const schedules = []
|
||||
|
|
@ -307,4 +262,21 @@ function getUrl(date, region = null, page = null, event = null) {
|
|||
url += `${url.indexOf('?') < 0 ? '?' : '&'}limit=${page.limit}`
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
function getQueue(url, referer) {
|
||||
const data = {
|
||||
'Origin': 'https://tv.yandex.ru',
|
||||
}
|
||||
if (referer) {
|
||||
data['Referer'] = referer
|
||||
}
|
||||
if (url.indexOf('api') > 0) {
|
||||
data['X-Requested-With'] = 'XMLHttpRequest'
|
||||
}
|
||||
const headers = getHeaders(data)
|
||||
return {
|
||||
url,
|
||||
params: { headers }
|
||||
}
|
||||
}
|
||||
|
|
@ -52,12 +52,12 @@ it('can generate valid url', () => {
|
|||
it('can generate valid request headers', () => {
|
||||
expect(request.headers).toMatchObject({
|
||||
Cookie:
|
||||
'i=dkim62pClrWWC4CShVQYMpVw1ELNVw4XJdL/lzT4E2r05IgcST1GtCA4ho/UyGgW2AO4qftDfZzGX2OHqCzwY7GUkpM=; ' +
|
||||
'spravka=dD0xNzMyNjgzMTEwO2k9MTgwLjI0OC41OS40MDtEPTkyOUM2MkQ0Mzc3OUNBMUFCNzg3NTIyMEQ4OEJBMEVBMzQ2RUNGNUU5Q0FEQUM5RUVDMTFCNjc1ODA2MThEQTQ3RTY3RTUyRUNBRDdBMTY2OTY1MjMzRDU1QjNGMTc1MDA0NDM3MjBGMUNGQTM5RjA3OUQwRjE2MzQxMUNFOTgxQ0E0RjNGRjRGODNCMEM1QjlGNTg5RkI4NDk0NEM2QjNDQUQ5NkJGRTBFNTVCQ0Y1OTEzMEY0O3U9MTczMjY4MzExMDY3MTA1MzIzNDtoPTA1YWJmMTY0ZmI2MGViNTBhMDUwZWUwMThmYWNiYjhm; ' +
|
||||
'i=eIUfSP+/mzQWXcH+Cuz8o1vY+D2K8fhBd6Sj0xvbPZeO4l3cY+BvMp8fFIuM17l6UE1Z5+R2a18lP00ex9iYVJ+VT+c=; ' +
|
||||
'spravka=dD0xNzM0MjA0NjM4O2k9MTI1LjE2NC4xNDkuMjAwO0Q9QTVCQ0IyOTI5RDQxNkU5NkEyOTcwMTNDMzZGMDAzNjRDNTFFNDM4QkE2Q0IyOTJDRjhCOTZDRDIzODdBQzk2MzRFRDc5QTk2Qjc2OEI1MUY5MTM5M0QzNkY3OEQ2OUY3OTUwNkQ3RjBCOEJGOEJDMjAwMTQ0RDUwRkFCMDNEQzJFMDI2OEI5OTk5OUJBNEFERUYwOEQ1MjUwQTE0QTI3RDU1MEQwM0U0O3U9MTczNDIwNDYzODUyNDYyNzg1NDtoPTIxNTc0ZTc2MDQ1ZjcwMDBkYmY0NTVkM2Q2ZWMyM2Y1; ' +
|
||||
'yandexuid=1197179041732383499; ' +
|
||||
'yashr=4682342911732383504; ' +
|
||||
'yuidss=1197179041732383499; ' +
|
||||
'user_display=930'
|
||||
'user_display=824'
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const doFetch = require('@ntlab/sfetch')
|
||||
const debug = require('debug')('site:virgintvgo.virginmedia.com')
|
||||
|
||||
dayjs.extend(utc)
|
||||
|
||||
doFetch.setDebugger(debug)
|
||||
|
||||
const detailedGuide = true
|
||||
const nworker = 25
|
||||
|
||||
module.exports = {
|
||||
site: 'virgintvgo.virginmedia.com',
|
||||
|
|
@ -110,66 +112,3 @@ module.exports = {
|
|||
return channels
|
||||
}
|
||||
}
|
||||
|
||||
async function doFetch(queues, cb) {
|
||||
const axios = require('axios')
|
||||
|
||||
let n = Math.min(nworker, queues.length)
|
||||
const workers = []
|
||||
const adjustWorker = () => {
|
||||
if (queues.length > workers.length && workers.length < nworker) {
|
||||
let nw = Math.min(nworker, queues.length)
|
||||
if (n < nw) {
|
||||
n = nw
|
||||
createWorker()
|
||||
}
|
||||
}
|
||||
}
|
||||
const createWorker = () => {
|
||||
while (workers.length < n) {
|
||||
startWorker()
|
||||
}
|
||||
}
|
||||
const startWorker = () => {
|
||||
const worker = () => {
|
||||
if (queues.length) {
|
||||
const queue = queues.shift()
|
||||
const done = res => {
|
||||
if (res) {
|
||||
cb(queue, res)
|
||||
adjustWorker()
|
||||
}
|
||||
worker()
|
||||
}
|
||||
const url = typeof queue === 'string' ? queue : queue.u
|
||||
const params = typeof queue === 'object' && queue.params ? queue.params : {}
|
||||
const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
|
||||
debug(`fetch %s with %s`, url, JSON.stringify(params))
|
||||
if (method === 'post') {
|
||||
axios
|
||||
.post(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
} else {
|
||||
axios
|
||||
.get(url, params)
|
||||
.then(response => done(response.data))
|
||||
.catch(console.error)
|
||||
}
|
||||
} else {
|
||||
workers.splice(workers.indexOf(worker), 1)
|
||||
}
|
||||
}
|
||||
workers.push(worker)
|
||||
worker()
|
||||
}
|
||||
createWorker()
|
||||
await new Promise(resolve => {
|
||||
const interval = setInterval(() => {
|
||||
if (workers.length === 0) {
|
||||
clearInterval(interval)
|
||||
resolve()
|
||||
}
|
||||
}, 500)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,13 @@
|
|||
const axios = require('axios')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const timezone = require('dayjs/plugin/timezone')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
const cheerio = require('cheerio')
|
||||
|
||||
dayjs.extend(utc)
|
||||
dayjs.extend(timezone)
|
||||
dayjs.extend(customParseFormat)
|
||||
|
||||
const languages = { en: 'ENG', id: 'IND' }
|
||||
const tz = 'Asia/Jakarta'
|
||||
|
||||
module.exports = {
|
||||
site: 'visionplus.id',
|
||||
|
|
@ -22,7 +19,7 @@ module.exports = {
|
|||
'YYYY-MM-DD'
|
||||
)}T00%3A00%3A00Z&view=cd-events-grid-view`
|
||||
},
|
||||
parser({ content, channel, date }) {
|
||||
parser({ content, channel }) {
|
||||
const programs = []
|
||||
const json = JSON.parse(content)
|
||||
if (Array.isArray(json.evs)) {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
const { parser, url, request } = require('./visionplus.id.config.js')
|
||||
const { parser, url } = require('./visionplus.id.config.js')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const dayjs = require('dayjs')
|
||||
const utc = require('dayjs/plugin/utc')
|
||||
const customParseFormat = require('dayjs/plugin/customParseFormat')
|
||||
|
||||
dayjs.extend(customParseFormat)
|
||||
dayjs.extend(utc)
|
||||
|
||||
|
|
@ -17,7 +18,6 @@ const channel = {
|
|||
}
|
||||
const channelId = { ...channel, lang: 'id' }
|
||||
|
||||
|
||||
it('can generate valid url', () => {
|
||||
expect(url({ channel, date })).toBe(
|
||||
'https://www.visionplus.id/managetv/tvinfo/events/schedule?language=ENG&serviceId=00000000000000000079&start=2024-11-24T00%3A00%3A00Z&end=2024-11-25T00%3A00%3A00Z&view=cd-events-grid-view'
|
||||
|
|
@ -30,11 +30,11 @@ it('can generate valid url', () => {
|
|||
it('can parse response', () => {
|
||||
let content = fs.readFileSync(path.resolve(__dirname, '__data__/content_en.json'))
|
||||
let results = parser({ content, channel, date })
|
||||
results = results.map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
.map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
|
||||
expect(results.length).toBe(1)
|
||||
expect(results[0]).toMatchObject({
|
||||
|
|
@ -48,11 +48,11 @@ it('can parse response', () => {
|
|||
|
||||
content = fs.readFileSync(path.resolve(__dirname, '__data__/content_id.json'))
|
||||
results = parser({ content, channel: channelId, date })
|
||||
results = results.map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
.map(p => {
|
||||
p.start = p.start.toJSON()
|
||||
p.stop = p.stop.toJSON()
|
||||
return p
|
||||
})
|
||||
|
||||
expect(results.length).toBe(1)
|
||||
expect(results[0]).toMatchObject({
|
||||
|
|
|
|||
|
|
@ -758,6 +758,13 @@
|
|||
dependencies:
|
||||
semver "^7.3.5"
|
||||
|
||||
"@ntlab/sfetch@^1.0.0":
|
||||
version "1.0.0"
|
||||
resolved "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz"
|
||||
integrity sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==
|
||||
dependencies:
|
||||
axios "^1.7.9"
|
||||
|
||||
"@octokit/auth-token@^3.0.0":
|
||||
version "3.0.2"
|
||||
resolved "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz"
|
||||
|
|
@ -1354,7 +1361,7 @@ axios-mock-adapter@^1.20.0:
|
|||
is-blob "^2.1.0"
|
||||
is-buffer "^2.0.5"
|
||||
|
||||
axios@^1.5.1, axios@^1.6.1, "axios@>= 0.9.0", axios@>=0.20.0:
|
||||
axios@^1.5.1, axios@^1.6.1, axios@^1.7.9, "axios@>= 0.9.0", axios@>=0.20.0:
|
||||
version "1.7.9"
|
||||
resolved "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz"
|
||||
integrity sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue