Simultaneous fetch is now external package.

Signed-off-by: Toha <tohenk@yahoo.com>

package.json

@@ -30,6 +30,7 @@
     "@alex_neo/jest-expect-message": "^1.0.5",
     "@freearhey/core": "^0.3.1",
     "@freearhey/search-js": "^0.1.1",
+    "@ntlab/sfetch": "^1.0.0",
     "@octokit/core": "^4.1.0",
     "@types/cli-progress": "^3.11.3",
     "@types/fs-extra": "^11.0.2",

scripts/core/fetch.js

@@ -1,109 +0,0 @@
-const axios = require('axios')
-
-/**
- * A callback when fetch queue is completely done.
- *
- * To check for successful operation simply check if res is not undefined.
- *
- * @callback completeCallback
- * @param {string|object} queue Fetched queue which is complete
- * @param {string|object} res Response content returned by axios
- * @param {object} headers Response headers returned by axios
- */
-
-/**
- * @type {number}
- */
-let nworker = 25
-
-/**
- * @type {boolean}
- */
-let checkResult = true
-
-/**
- * @type {any}
- */
-let debug
-
-/**
- * Queued url fetch.
- *
- * @param {array<string>} queues The queues
- * @param {completeCallback} cb Queue completion callback
- */
-async function doFetch(queues, cb) {
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = (res, headers) => {
-          if ((checkResult && res) || !checkResult) {
-            cb(queue, res, headers)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        if (typeof debug === 'function') {
-          debug(`fetch %s with %s`, url, JSON.stringify(params))
-        }
-        axios[method](url, params)
-          .then(response => {
-            done(response.data, response.headers)
-          })
-          .catch(err => {
-            console.error(`Unable to fetch ${url}: ${err.message}!`)
-            done()
-          })
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}
-
-module.exports = doFetch
-Object.assign(module.exports, {
-  setMaxWorker(n) {
-    nworker = n
-    return module.exports
-  },
-  setCheckResult(enabled) {
-    checkResult = enabled
-    return module.exports
-  },
-  setDebugger(dbg) {
-    debug = dbg
-    return module.exports
-  }
-})

sites/mncvision.id/mncvision.id.config.js

@@ -4,7 +4,7 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:mncvision.id')
 
 dayjs.extend(utc)
@@ -127,7 +127,7 @@ async function parseItems(content, date, cookies) {
         'X-Requested-With': 'XMLHttpRequest',
         Cookie: cookies,
       }
-      queues.push({ i: $item, u: url, params: { headers, timeout } })
+      queues.push({ i: $item, url, params: { headers, timeout } })
     }
     await doFetch(queues, (queue, res) => {
       const $item = queue.i

sites/mytelly.co.uk/mytelly.co.uk.config.js

@@ -3,7 +3,7 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:mytelly.co.uk')
 
 dayjs.extend(utc)
@@ -110,7 +110,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }]
+    const queues = [{ t: 'p', method: 'post', url: 'https://www.mytelly.co.uk/getform' }]
     await doFetch(queues, (queue, res) => {
       // process form -> provider
       if (queue.t === 'p') {
@@ -119,7 +119,7 @@ module.exports = {
           .forEach(el => {
             const opt = $(el)
             const provider = opt.attr('value')
-            queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
+            queues.push({ t: 'r', method: 'post', url: 'https://www.mytelly.co.uk/getregions', params: { provider } })
           })
       }
       // process provider -> region
@@ -135,7 +135,7 @@ module.exports = {
             u_time: now.format('HHmm'),
             is_mobile: 1
           }
-          queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
+          queues.push({ t: 's', method: 'post', url: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
         }
       }
       // process schedule -> channels

sites/rotana.net/rotana.net.config.js

@@ -4,7 +4,7 @@ const dayjs = require('dayjs')
 const timezone = require('dayjs/plugin/timezone')
 const utc = require('dayjs/plugin/utc')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:rotana.net')
 
 dayjs.extend(timezone)
@@ -50,7 +50,7 @@ module.exports = {
             cookie: cookies[channel.lang],
           }
         }
-        queues.push({ i: item, u: url, params })
+        queues.push({ i: item, url, params })
       }
       await doFetch(queues, (queue, res) => {
         programs.push(parseProgram(queue.i, res))

sites/sky.com/sky.com.config.js

@@ -1,7 +1,7 @@
 const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:sky.com')
 
 dayjs.extend(utc)
@@ -49,7 +49,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const queues = [{ t: 'r', u: 'https://www.sky.com/tv-guide' }]
+    const queues = [{ t: 'r', url: 'https://www.sky.com/tv-guide' }]
     await doFetch(queues, (queue, res) => {
       // process regions
       if (queue.t === 'r') {
@@ -57,7 +57,7 @@ module.exports = {
         const initialData = JSON.parse(decodeURIComponent($('#initialData').text()))
         initialData.state.epgData.regions
           .forEach(region => {
-            queues.push({ t: 'c', u: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
+            queues.push({ t: 'c', url: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
           })
       }
       // process channels

sites/startimestv.com/startimestv.com.config.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:startimestv.com')
 
 dayjs.extend(utc)
@@ -48,7 +48,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const queues = [{ t: 'a', u: 'https://www.startimestv.com/tv_guide.html' }]
+    const queues = [{ t: 'a', url: 'https://www.startimestv.com/tv_guide.html' }]
     await doFetch(queues, (queue, res) => {
       // process area-id
       if (queue.t === 'a') {
@@ -59,7 +59,7 @@ module.exports = {
             const areaId = dd.attr('area-id')
             queues.push({
               t: 's',
-              u: 'https://www.startimestv.com/tv_guide.html',
+              url: 'https://www.startimestv.com/tv_guide.html',
               params: {
                 headers: {
                   cookie: `default_areaID=${areaId}`

sites/tv.yandex.ru/tv.yandex.ru.config.js

@@ -1,5 +1,5 @@
 const dayjs = require('dayjs')
-const doFetch = require('../../scripts/core/fetch')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:tv.yandex.ru')
 
 doFetch
@@ -276,7 +276,7 @@ function getQueue(url, referer) {
   }
   const headers = getHeaders(data)
   return {
-    u: url,
+    url,
     params: { headers }
   }
 }

sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js

@@ -1,11 +1,13 @@
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:virgintvgo.virginmedia.com')
 
 dayjs.extend(utc)
 
+doFetch.setDebugger(debug)
+
 const detailedGuide = true
-const nworker = 25
 
 module.exports = {
   site: 'virgintvgo.virginmedia.com',
@@ -110,66 +112,3 @@ module.exports = {
     return channels
   }
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = res => {
-          if (res) {
-            cb(queue, res)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        debug(`fetch %s with %s`, url, JSON.stringify(params))
-        if (method === 'post') {
-          axios
-            .post(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        } else {
-          axios
-            .get(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        }
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}