diff --git a/sites/mytelly.co.uk/mytelly.co.uk.config.js b/sites/mytelly.co.uk/mytelly.co.uk.config.js index ca76c70b..ef28ef78 100644 --- a/sites/mytelly.co.uk/mytelly.co.uk.config.js +++ b/sites/mytelly.co.uk/mytelly.co.uk.config.js @@ -3,6 +3,7 @@ const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const timezone = require('dayjs/plugin/timezone') const customParseFormat = require('dayjs/plugin/customParseFormat') +const debug = require('debug')('site:mytelly.co.uk') dayjs.extend(utc) dayjs.extend(timezone) @@ -62,11 +63,8 @@ module.exports = { async channels() { const channels = {} const axios = require('axios') - const queues = [{ t: 'p', u: 'https://www.mytelly.co.uk/getform' }] - - let n = Math.min(nworker, queues.length) - const workers = [] - const cb = (queue, res) => { + const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }] + await doFetch(queues, (queue, res) => { // process form -> provider if (queue.t === 'p') { const $ = cheerio.load(res) @@ -74,7 +72,7 @@ module.exports = { .forEach(el => { const opt = $(el) const provider = opt.attr('value') - queues.push({ t: 'r', u: 'https://www.mytelly.co.uk/getregions', params: { provider } }) + queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } }) }) } // process provider -> region @@ -90,7 +88,7 @@ module.exports = { u_time: now.format('HHmm'), is_mobile: 1 } - queues.push({ t: 's', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params }) + queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params }) } } // process schedule -> channels @@ -111,48 +109,6 @@ module.exports = { } }) } - // increase worker - if (queues.length > workers.length && workers.length < nworker) { - let nw = Math.min(nworker, queues.length) - if (n < nw) { - n = nw - createWorker() - } - } - } - const createWorker = () => { - while (workers.length < n) { - startWorker() - } - } - const startWorker = () => { - const worker = () => { - if (queues.length) { - const q = queues.shift() - axios - .post(q.u, q.params || {}) - .then(response => { - if (response.data) { - cb(q, response.data) - } - worker() - }) - .catch(console.error) - } else { - workers.splice(workers.indexOf(worker), 1) - } - } - workers.push(worker) - worker() - } - createWorker() - await new Promise(resolve => { - const interval = setInterval(() => { - if (workers.length === 0) { - clearInterval(interval) - resolve() - } - }, 500) }) return Object.values(channels) @@ -166,3 +122,67 @@ function parseText($item) { .replace(/ /g, ' ') .trim() } + +async function doFetch(queues, cb) { + const axios = require('axios') + + let n = Math.min(nworker, queues.length) + const workers = [] + const adjustWorker = () => { + if (queues.length > workers.length && workers.length < nworker) { + let nw = Math.min(nworker, queues.length) + if (n < nw) { + n = nw + createWorker() + } + } + } + const createWorker = () => { + while (workers.length < n) { + startWorker() + } + } + const startWorker = () => { + const worker = () => { + if (queues.length) { + const queue = queues.shift() + const done = res => { + if (res) { + cb(queue, res) + adjustWorker() + } + worker() + } + const url = typeof queue === 'string' ? queue : queue.u + const params = typeof queue === 'object' && queue.params ? queue.params : {} + const method = typeof queue === 'object' && queue.m ? queue.m : 'get' + debug(`fetch %s with %s`, url, JSON.stringify(params)) + if (method === 'post') { + axios + .post(url, params) + .then(response => done(response.data)) + .catch(console.error) + } else { + axios + .get(url, params) + .then(response => done(response.data)) + .catch(console.error) + } + } else { + workers.splice(workers.indexOf(worker), 1) + } + } + workers.push(worker) + worker() + } + createWorker() + await new Promise(resolve => { + const interval = setInterval(() => { + if (workers.length === 0) { + clearInterval(interval) + resolve() + } + }, 500) + }) +} +