Refactor fetch worker as async function.

Signed-off-by: Toha <tohenk@yahoo.com>

sites/mytelly.co.uk/mytelly.co.uk.config.js

@@ -3,6 +3,7 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const debug = require('debug')('site:mytelly.co.uk')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
@@ -62,11 +63,8 @@ module.exports = {
   async channels() {
     const channels = {}
     const axios = require('axios')
-    const queues = [{ t: 'p', u: 'https://www.mytelly.co.uk/getform' }]
+    const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }]
-
+    await doFetch(queues, (queue, res) => {
-    let n = Math.min(nworker, queues.length)
-    const workers = []
-    const cb = (queue, res) => {
       // process form -> provider
       if (queue.t === 'p') {
         const $ = cheerio.load(res)
@@ -74,7 +72,7 @@ module.exports = {
           .forEach(el => {
             const opt = $(el)
             const provider = opt.attr('value')
-            queues.push({ t: 'r', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
+            queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
           })
       }
       // process provider -> region
@@ -90,7 +88,7 @@ module.exports = {
             u_time: now.format('HHmm'),
             is_mobile: 1
           }
-          queues.push({ t: 's', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
+          queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
         }
       }
       // process schedule -> channels
@@ -111,7 +109,26 @@ module.exports = {
             }
           })
       }
-      // increase worker
+    })
+
+    return Object.values(channels)
+  }
+}
+
+function parseText($item) {
+  return $item.text()
+    .replace(/\t/g, '')
+    .replace(/\n/g, ' ')
+    .replace(/  /g, ' ')
+    .trim()
+}
+
+async function doFetch(queues, cb) {
+  const axios = require('axios')
+
+  let n = Math.min(nworker, queues.length)
+  const workers = []
+  const adjustWorker = () => {
     if (queues.length > workers.length && workers.length < nworker) {
       let nw = Math.min(nworker, queues.length)
       if (n < nw) {
@@ -128,16 +145,29 @@ module.exports = {
   const startWorker = () => {
     const worker = () => {
       if (queues.length) {
-          const q = queues.shift()
+        const queue = queues.shift()
-          axios
+        const done = res => {
-            .post(q.u, q.params || {})
+          if (res) {
-            .then(response => {
+            cb(queue, res)
-              if (response.data) {
+            adjustWorker()
-                cb(q, response.data)
           }
           worker()
-            })
+        }
+        const url = typeof queue === 'string' ? queue : queue.u
+        const params = typeof queue === 'object' && queue.params ? queue.params : {}
+        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
+        debug(`fetch %s with %s`, url, JSON.stringify(params))
+        if (method === 'post') {
+          axios
+            .post(url, params)
+            .then(response => done(response.data))
             .catch(console.error)
+        } else {
+          axios
+            .get(url, params)
+            .then(response => done(response.data))
+            .catch(console.error)
+        }
       } else {
         workers.splice(workers.indexOf(worker), 1)
       }
@@ -154,15 +184,5 @@ module.exports = {
       }
     }, 500)
   })
-
-    return Object.values(channels)
-  }
 }
 
-function parseText($item) {
-  return $item.text()
-    .replace(/\t/g, '')
-    .replace(/\n/g, ' ')
-    .replace(/  /g, ' ')
-    .trim()
-}