diff --git a/package.json b/package.json index 3d3a5213..5f15d386 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@alex_neo/jest-expect-message": "^1.0.5", "@freearhey/core": "^0.3.1", "@freearhey/search-js": "^0.1.1", + "@ntlab/sfetch": "^1.0.0", "@octokit/core": "^4.1.0", "@types/cli-progress": "^3.11.3", "@types/fs-extra": "^11.0.2", diff --git a/scripts/core/fetch.js b/scripts/core/fetch.js deleted file mode 100644 index a8e7ae81..00000000 --- a/scripts/core/fetch.js +++ /dev/null @@ -1,109 +0,0 @@ -const axios = require('axios') - -/** - * A callback when fetch queue is completely done. - * - * To check for successful operation simply check if res is not undefined. - * - * @callback completeCallback - * @param {string|object} queue Fetched queue which is complete - * @param {string|object} res Response content returned by axios - * @param {object} headers Response headers returned by axios - */ - -/** - * @type {number} - */ -let nworker = 25 - -/** - * @type {boolean} - */ -let checkResult = true - -/** - * @type {any} - */ -let debug - -/** - * Queued url fetch. - * - * @param {array} queues The queues - * @param {completeCallback} cb Queue completion callback - */ -async function doFetch(queues, cb) { - let n = Math.min(nworker, queues.length) - const workers = [] - const adjustWorker = () => { - if (queues.length > workers.length && workers.length < nworker) { - let nw = Math.min(nworker, queues.length) - if (n < nw) { - n = nw - createWorker() - } - } - } - const createWorker = () => { - while (workers.length < n) { - startWorker() - } - } - const startWorker = () => { - const worker = () => { - if (queues.length) { - const queue = queues.shift() - const done = (res, headers) => { - if ((checkResult && res) || !checkResult) { - cb(queue, res, headers) - adjustWorker() - } - worker() - } - const url = typeof queue === 'string' ? queue : queue.u - const params = typeof queue === 'object' && queue.params ? queue.params : {} - const method = typeof queue === 'object' && queue.m ? queue.m : 'get' - if (typeof debug === 'function') { - debug(`fetch %s with %s`, url, JSON.stringify(params)) - } - axios[method](url, params) - .then(response => { - done(response.data, response.headers) - }) - .catch(err => { - console.error(`Unable to fetch ${url}: ${err.message}!`) - done() - }) - } else { - workers.splice(workers.indexOf(worker), 1) - } - } - workers.push(worker) - worker() - } - createWorker() - await new Promise(resolve => { - const interval = setInterval(() => { - if (workers.length === 0) { - clearInterval(interval) - resolve() - } - }, 500) - }) -} - -module.exports = doFetch -Object.assign(module.exports, { - setMaxWorker(n) { - nworker = n - return module.exports - }, - setCheckResult(enabled) { - checkResult = enabled - return module.exports - }, - setDebugger(dbg) { - debug = dbg - return module.exports - } -}) \ No newline at end of file diff --git a/sites/mncvision.id/mncvision.id.config.js b/sites/mncvision.id/mncvision.id.config.js index 46994b9a..ba944503 100644 --- a/sites/mncvision.id/mncvision.id.config.js +++ b/sites/mncvision.id/mncvision.id.config.js @@ -4,7 +4,7 @@ const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const timezone = require('dayjs/plugin/timezone') const customParseFormat = require('dayjs/plugin/customParseFormat') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:mncvision.id') dayjs.extend(utc) @@ -127,7 +127,7 @@ async function parseItems(content, date, cookies) { 'X-Requested-With': 'XMLHttpRequest', Cookie: cookies, } - queues.push({ i: $item, u: url, params: { headers, timeout } }) + queues.push({ i: $item, url, params: { headers, timeout } }) } await doFetch(queues, (queue, res) => { const $item = queue.i diff --git a/sites/mytelly.co.uk/mytelly.co.uk.config.js b/sites/mytelly.co.uk/mytelly.co.uk.config.js index 0a42f01e..282f8dc4 100644 --- a/sites/mytelly.co.uk/mytelly.co.uk.config.js +++ b/sites/mytelly.co.uk/mytelly.co.uk.config.js @@ -3,7 +3,7 @@ const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const timezone = require('dayjs/plugin/timezone') const customParseFormat = require('dayjs/plugin/customParseFormat') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:mytelly.co.uk') dayjs.extend(utc) @@ -110,7 +110,7 @@ module.exports = { }, async channels() { const channels = {} - const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }] + const queues = [{ t: 'p', method: 'post', url: 'https://www.mytelly.co.uk/getform' }] await doFetch(queues, (queue, res) => { // process form -> provider if (queue.t === 'p') { @@ -119,7 +119,7 @@ module.exports = { .forEach(el => { const opt = $(el) const provider = opt.attr('value') - queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } }) + queues.push({ t: 'r', method: 'post', url: 'https://www.mytelly.co.uk/getregions', params: { provider } }) }) } // process provider -> region @@ -135,7 +135,7 @@ module.exports = { u_time: now.format('HHmm'), is_mobile: 1 } - queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params }) + queues.push({ t: 's', method: 'post', url: 'https://www.mytelly.co.uk/tv-guide/schedule', params }) } } // process schedule -> channels diff --git a/sites/rotana.net/rotana.net.config.js b/sites/rotana.net/rotana.net.config.js index 6237d756..9eb0b930 100644 --- a/sites/rotana.net/rotana.net.config.js +++ b/sites/rotana.net/rotana.net.config.js @@ -4,7 +4,7 @@ const dayjs = require('dayjs') const timezone = require('dayjs/plugin/timezone') const utc = require('dayjs/plugin/utc') const customParseFormat = require('dayjs/plugin/customParseFormat') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:rotana.net') dayjs.extend(timezone) @@ -50,7 +50,7 @@ module.exports = { cookie: cookies[channel.lang], } } - queues.push({ i: item, u: url, params }) + queues.push({ i: item, url, params }) } await doFetch(queues, (queue, res) => { programs.push(parseProgram(queue.i, res)) diff --git a/sites/sky.com/sky.com.config.js b/sites/sky.com/sky.com.config.js index cf24b538..cf024178 100644 --- a/sites/sky.com/sky.com.config.js +++ b/sites/sky.com/sky.com.config.js @@ -1,7 +1,7 @@ const cheerio = require('cheerio') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:sky.com') dayjs.extend(utc) @@ -49,7 +49,7 @@ module.exports = { }, async channels() { const channels = {} - const queues = [{ t: 'r', u: 'https://www.sky.com/tv-guide' }] + const queues = [{ t: 'r', url: 'https://www.sky.com/tv-guide' }] await doFetch(queues, (queue, res) => { // process regions if (queue.t === 'r') { @@ -57,7 +57,7 @@ module.exports = { const initialData = JSON.parse(decodeURIComponent($('#initialData').text())) initialData.state.epgData.regions .forEach(region => { - queues.push({ t: 'c', u: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` }) + queues.push({ t: 'c', url: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` }) }) } // process channels diff --git a/sites/startimestv.com/startimestv.com.config.js b/sites/startimestv.com/startimestv.com.config.js index e84f433c..83b6a4d4 100644 --- a/sites/startimestv.com/startimestv.com.config.js +++ b/sites/startimestv.com/startimestv.com.config.js @@ -2,7 +2,7 @@ const cheerio = require('cheerio') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const customParseFormat = require('dayjs/plugin/customParseFormat') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:startimestv.com') dayjs.extend(utc) @@ -48,7 +48,7 @@ module.exports = { }, async channels() { const channels = {} - const queues = [{ t: 'a', u: 'https://www.startimestv.com/tv_guide.html' }] + const queues = [{ t: 'a', url: 'https://www.startimestv.com/tv_guide.html' }] await doFetch(queues, (queue, res) => { // process area-id if (queue.t === 'a') { @@ -59,7 +59,7 @@ module.exports = { const areaId = dd.attr('area-id') queues.push({ t: 's', - u: 'https://www.startimestv.com/tv_guide.html', + url: 'https://www.startimestv.com/tv_guide.html', params: { headers: { cookie: `default_areaID=${areaId}` diff --git a/sites/tv.yandex.ru/tv.yandex.ru.config.js b/sites/tv.yandex.ru/tv.yandex.ru.config.js index 99c908f8..e12df5d0 100644 --- a/sites/tv.yandex.ru/tv.yandex.ru.config.js +++ b/sites/tv.yandex.ru/tv.yandex.ru.config.js @@ -1,5 +1,5 @@ const dayjs = require('dayjs') -const doFetch = require('../../scripts/core/fetch') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:tv.yandex.ru') doFetch @@ -276,7 +276,7 @@ function getQueue(url, referer) { } const headers = getHeaders(data) return { - u: url, + url, params: { headers } } } \ No newline at end of file diff --git a/sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js b/sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js index 75cc2fcd..a26c23fd 100644 --- a/sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js +++ b/sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js @@ -1,11 +1,13 @@ const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') +const doFetch = require('@ntlab/sfetch') const debug = require('debug')('site:virgintvgo.virginmedia.com') dayjs.extend(utc) +doFetch.setDebugger(debug) + const detailedGuide = true -const nworker = 25 module.exports = { site: 'virgintvgo.virginmedia.com', @@ -110,66 +112,3 @@ module.exports = { return channels } } - -async function doFetch(queues, cb) { - const axios = require('axios') - - let n = Math.min(nworker, queues.length) - const workers = [] - const adjustWorker = () => { - if (queues.length > workers.length && workers.length < nworker) { - let nw = Math.min(nworker, queues.length) - if (n < nw) { - n = nw - createWorker() - } - } - } - const createWorker = () => { - while (workers.length < n) { - startWorker() - } - } - const startWorker = () => { - const worker = () => { - if (queues.length) { - const queue = queues.shift() - const done = res => { - if (res) { - cb(queue, res) - adjustWorker() - } - worker() - } - const url = typeof queue === 'string' ? queue : queue.u - const params = typeof queue === 'object' && queue.params ? queue.params : {} - const method = typeof queue === 'object' && queue.m ? queue.m : 'get' - debug(`fetch %s with %s`, url, JSON.stringify(params)) - if (method === 'post') { - axios - .post(url, params) - .then(response => done(response.data)) - .catch(console.error) - } else { - axios - .get(url, params) - .then(response => done(response.data)) - .catch(console.error) - } - } else { - workers.splice(workers.indexOf(worker), 1) - } - } - workers.push(worker) - worker() - } - createWorker() - await new Promise(resolve => { - const interval = setInterval(() => { - if (workers.length === 0) { - clearInterval(interval) - resolve() - } - }, 500) - }) -}