Merge branch 'master' into bellezaemporium/fixes/ziggogo

package-lock.json

@@ -11,6 +11,7 @@
         "@alex_neo/jest-expect-message": "^1.0.5",
         "@freearhey/core": "^0.3.1",
         "@freearhey/search-js": "^0.1.1",
+        "@ntlab/sfetch": "^1.0.0",
         "@octokit/core": "^4.1.0",
         "@types/cli-progress": "^3.11.3",
         "@types/fs-extra": "^11.0.2",
@@ -1885,6 +1886,14 @@
         "node": ">=10"
       }
     },
+    "node_modules/@ntlab/sfetch": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz",
+      "integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==",
+      "dependencies": {
+        "axios": "^1.7.9"
+      }
+    },
     "node_modules/@octokit/auth-token": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz",
@@ -10340,6 +10349,14 @@
         }
       }
     },
+    "@ntlab/sfetch": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz",
+      "integrity": "sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==",
+      "requires": {
+        "axios": "^1.7.9"
+      }
+    },
     "@octokit/auth-token": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz",

package.json

@@ -30,6 +30,7 @@
     "@alex_neo/jest-expect-message": "^1.0.5",
     "@freearhey/core": "^0.3.1",
     "@freearhey/search-js": "^0.1.1",
+    "@ntlab/sfetch": "^1.0.0",
     "@octokit/core": "^4.1.0",
     "@types/cli-progress": "^3.11.3",
     "@types/fs-extra": "^11.0.2",

sites/firstmedia.com/firstmedia.com.config.js

@@ -43,7 +43,6 @@ module.exports = {
   },
   async channels() {
     const axios = require('axios')
-    const cheerio = require('cheerio')
     const result = await axios
       .get(
         `https://api.firstmedia.com/api/content/tv-guide/list?date=${dayjs().format(

sites/i24news.tv/i24news.tv.channels.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <channels>
-  <channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar#world">I24NEWS عربى</channel>
+  <channel site="i24news.tv" lang="ar" xmltv_id="i24NEWSArabic.il" site_id="ar">I24NEWS عربى</channel>
-  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en#usa">I24NEWS English (USA)</channel>
+  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishUSA.il" site_id="en">I24NEWS English (USA)</channel>
-  <channel site="i24news.tv" lang="en" xmltv_id="i24NEWSEnglishWorld.il" site_id="en#world">I24NEWS English (World)</channel>
+  <channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr">I24NEWS Français</channel>
-  <channel site="i24news.tv" lang="fr" xmltv_id="i24NEWSFrench.il" site_id="fr#world">I24NEWS Français</channel>
+  <channel site="i24news.tv" lang="he" xmltv_id="i24NEWSHebrew.il" site_id="he">I24NEWS עברית</channel>
 </channels>

sites/i24news.tv/i24news.tv.config.js

@@ -11,9 +11,7 @@ module.exports = {
   site: 'i24news.tv',
   days: 2,
   url: function ({ channel }) {
-    const [lang, region] = channel.site_id.split('#')
+    return `https://api.i24news.tv/v2/${channel.site_id}/schedules`
-
-    return `https://api.i24news.tv/v2/${lang}/schedules/${region}`
   },
   parser: function ({ content, date }) {
     let programs = []

sites/i24news.tv/i24news.tv.test.js

@@ -7,12 +7,12 @@ dayjs.extend(utc)
 
 const date = dayjs.utc('2022-03-06', 'YYYY-MM-DD').startOf('d')
 const channel = {
-  site_id: 'ar#',
+  site_id: 'ar',
   xmltv_id: 'I24NewsArabic.il'
 }
 
 it('can generate valid url', () => {
-  expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules/world')
+  expect(url({ channel })).toBe('https://api.i24news.tv/v2/ar/schedules')
 })
 
 it('can parse response', () => {

sites/iltalehti.fi/iltalehti.fi.test.js

@@ -9,7 +9,7 @@ dayjs.extend(utc)
 
 const date = dayjs.utc('2022-10-29', 'YYYY-MM-DD').startOf('d')
 const channel = {
-  site_id: 'default_builtin_channelgroup1#yle-tv1',
+  site_id: '1#yle-tv1',
   xmltv_id: 'YleTV1.fi'
 }
 

sites/mncvision.id/mncvision.id.config.js

@@ -1,19 +1,23 @@
-const _ = require('lodash')
 const axios = require('axios')
 const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
+const debug = require('debug')('site:mncvision.id')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
 dayjs.extend(customParseFormat)
 
+doFetch
+  .setCheckResult(false)
+  .setDebugger(debug)
+
 const languages = { en: 'english', id: 'indonesia' }
 const cookies = {}
 const timeout = 30000
-const nworker = 25
 
 module.exports = {
   site: 'mncvision.id',
@@ -55,8 +59,6 @@ module.exports = {
     return await parseItems(content, date, cookies[channel.lang])
   },
   async channels({ lang = 'id' }) {
-    const axios = require('axios')
-    const cheerio = require('cheerio')
     const result = await axios
       .get('https://www.mncvision.id/schedule')
       .then(response => response.data)
@@ -117,13 +119,20 @@ async function parseItems(content, date, cookies) {
   const $ = cheerio.load(content)
   const items = $('tr[valign="top"]').toArray()
   if (items.length) {
-    const workers = []
+    const queues = []
-    const n = Math.min(nworker, items.length)
+    for (const item of items) {
-    while (workers.length < n) {
+      const $item = $(item)
-      const worker = () => {
+      const url = $item.find('a').attr('href')
-        if (items.length) {
+      const headers = {
-          const $item = $(items.shift())
+        'X-Requested-With': 'XMLHttpRequest',
-          const done = (description = null) => {
+        Cookie: cookies,
+      }
+      queues.push({ i: $item, url, params: { headers, timeout } })
+    }
+    await doFetch(queues, (queue, res) => {
+      const $item = queue.i
+      const $page = cheerio.load(res)
+      const description = $page('.synopsis').text().trim()
       const start = parseStart($item, date)
       const duration = parseDuration($item)
       const stop = start.add(duration, 'm')
@@ -131,28 +140,10 @@ async function parseItems(content, date, cookies) {
         title: parseTitle($item),
         season: parseSeason($item),
         episode: parseEpisode($item),
-              description,
+        description: description && description !== '-' ? description : null,
         start,
         stop
       })
-            worker()
-          }
-          loadDescription($item, cookies)
-            .then(description => done(description))
-        } else {
-          workers.splice(workers.indexOf(worker), 1)
-        }
-      }
-      workers.push(worker)
-      worker()
-    }
-    await new Promise(resolve => {
-      const interval = setInterval(() => {
-        if (workers.length === 0) {
-          clearInterval(interval)
-          resolve()
-        }
-      }, 500)
     })
   }
 
@@ -168,24 +159,6 @@ function loadLangCookies(channel) {
     .catch(error => console.error(error.message))
 }
 
-async function loadDescription($item, cookies) {
-  const url = $item.find('a').attr('href')
-  if (!url) return null
-  const content = await axios
-    .get(url, {
-      headers: { 'X-Requested-With': 'XMLHttpRequest', Cookie: cookies },
-      timeout
-    })
-    .then(r => r.data)
-    .catch(error => console.error(error.message))
-  if (!content) return null
-
-  const $page = cheerio.load(content)
-  const description = $page('.synopsis').text().trim()
-
-  return description !== '-' ? description : null
-}
-
 function parseCookies(headers) {
   const cookies = []
   if (Array.isArray(headers['set-cookie'])) {

sites/mytelly.co.uk/mytelly.co.uk.config.js

@@ -3,15 +3,17 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:mytelly.co.uk')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
 dayjs.extend(customParseFormat)
 
+doFetch.setDebugger(debug)
+
 const detailedGuide = true
 const tz = 'Europe/London'
-const nworker = 25
 
 module.exports = {
   site: 'mytelly.co.uk',
@@ -108,8 +110,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const axios = require('axios')
+    const queues = [{ t: 'p', method: 'post', url: 'https://www.mytelly.co.uk/getform' }]
-    const queues = [{ t: 'p', m: 'post', u: 'https://www.mytelly.co.uk/getform' }]
     await doFetch(queues, (queue, res) => {
       // process form -> provider
       if (queue.t === 'p') {
@@ -118,7 +119,7 @@ module.exports = {
           .forEach(el => {
             const opt = $(el)
             const provider = opt.attr('value')
-            queues.push({ t: 'r', m: 'post', u: 'https://www.mytelly.co.uk/getregions', params: { provider } })
+            queues.push({ t: 'r', method: 'post', url: 'https://www.mytelly.co.uk/getregions', params: { provider } })
           })
       }
       // process provider -> region
@@ -134,7 +135,7 @@ module.exports = {
             u_time: now.format('HHmm'),
             is_mobile: 1
           }
-          queues.push({ t: 's', m: 'post', u: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
+          queues.push({ t: 's', method: 'post', url: 'https://www.mytelly.co.uk/tv-guide/schedule', params })
         }
       }
       // process schedule -> channels
@@ -191,67 +192,3 @@ function parseText($item) {
 
   return text
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = res => {
-          if (res) {
-            cb(queue, res)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        debug(`fetch %s with %s`, url, JSON.stringify(params))
-        if (method === 'post') {
-          axios
-            .post(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        } else {
-          axios
-            .get(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        }
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}
-

sites/pickx.be/pickx.be.config.js

@@ -3,29 +3,22 @@ const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 
 let apiVersion
-let isApiVersionFetched = false
-
-;(async () => {
-  try {
-    await fetchApiVersion()
-    isApiVersionFetched = true
-  } catch (error) {
-    console.error('Error during script initialization:', error)
-  }
-})()
 
 dayjs.extend(utc)
 
 module.exports = {
   site: 'pickx.be',
   days: 2,
-  apiVersion: function () {
+  setApiVersion: function (version) {
+    apiVersion = version
+  },
+  getApiVersion: function () {
     return apiVersion
   },
-  fetchApiVersion: fetchApiVersion, // Export fetchApiVersion
+  fetchApiVersion: fetchApiVersion,
   url: async function ({ channel, date }) {
-    while (!isApiVersionFetched) {
+    if (!apiVersion) {
-      await new Promise(resolve => setTimeout(resolve, 100)) // Wait for 100 milliseconds
+      await fetchApiVersion()
     }
     return `https://px-epg.azureedge.net/airings/${apiVersion}/${date.format(
       'YYYY-MM-DD'
@@ -116,7 +109,7 @@ module.exports = {
         }`
     }
     const result = await axios
-      .post('https://api.proximusmwc.be/tiams/v2/graphql', query)
+      .post('https://api.proximusmwc.be/tiams/v3/graphql', query)
       .then(r => r.data)
       .catch(console.error)
 
@@ -140,38 +133,24 @@ function fetchApiVersion() {
   return new Promise(async (resolve, reject) => {
     try {
       // you'll never find what happened here :)
-      // load pickx bundle and get react version hash (regex).
+      // load the pickx page and get the hash from the MWC configuration.
       // it's not the best way to get the version but it's the only way to get it.
 
-      // find bundle version
+      const hashUrl = 'https://www.pickx.be/nl/televisie/tv-gids';
-      const minBundleVer = "https://www.pickx.be/minimal-bundle-version"
-      const bundleVerData = await axios.get(minBundleVer, {
-          headers: {
-            Origin: 'https://www.pickx.be',
-            Referer: 'https://www.pickx.be/'
-          }
-      })
 
-      if (bundleVerData.status !== 200) {
+      const hashData = await axios.get(hashUrl)
-        console.error(`Failed to fetch bundle version. Status: ${bundleVerData.status}`)
+      .then(r => {
-        reject(`Failed to fetch bundle version. Status: ${bundleVerData.status}`)
+        const re = /"hashes":\["(.*)"\]/
-      } else {
-        const bundleVer = bundleVerData.data.version
-        // get the minified JS app bundle
-        const bundleUrl = `https://components.pickx.be/pxReactPlayer/${bundleVer}/bundle.min.js`
-
-        // now, find the react hash inside the bundle URL
-        const bundle = await axios.get(bundleUrl).then(r => {
-          const re = /REACT_APP_VERSION_HASH:"([^"]+)"/
         const match = r.data.match(re)
         if (match && match[1]) {
           return match[1]
         } else {
           throw new Error('React app version hash not found')
         }
-        }).catch(console.error)
+      })
+      .catch(console.error);
 
-        const versionUrl = `https://www.pickx.be/api/s-${bundle.replace('/REACT_APP_VERSION_HASH:"', '')}`
+      const versionUrl = `https://www.pickx.be/api/s-${hashData}`
    
       const response = await axios.get(versionUrl, {
         headers: {
@@ -187,7 +166,6 @@ function fetchApiVersion() {
         console.error(`Failed to fetch API version. Status: ${response.status}`)
         reject(`Failed to fetch API version. Status: ${response.status}`)
       }
-      }
     } catch (error) {
       console.error('Error during fetchApiVersion:', error)
       reject(error)

sites/pickx.be/pickx.be.test.js

@@ -1,4 +1,20 @@
-const { parser, url, request, fetchApiVersion, apiVersion } = require('./pickx.be.config.js')
+jest.mock('./pickx.be.config.js', () => {
+  const originalModule = jest.requireActual('./pickx.be.config.js')
+  return {
+    ...originalModule,
+    fetchApiVersion: jest.fn(() => Promise.resolve())
+  }
+})
+
+const {
+  parser,
+  url,
+  request,
+  fetchApiVersion,
+  setApiVersion,
+  getApiVersion
+} = require('./pickx.be.config.js')
+
 const fs = require('fs')
 const path = require('path')
 const dayjs = require('dayjs')
@@ -13,12 +29,14 @@ const channel = {
   xmltv_id: 'Vedia.be'
 }
 
+beforeEach(() => {
+  setApiVersion('mockedApiVersion')
+})
+
 it('can generate valid url', async () => {
-  await fetchApiVersion()
   const generatedUrl = await url({ channel, date })
-  const resolvedApiVersion = apiVersion()
   expect(generatedUrl).toBe(
-    `https://px-epg.azureedge.net/airings/${resolvedApiVersion}/2023-12-13/channel/UID0118?timezone=Europe%2FBrussels`
+    `https://px-epg.azureedge.net/airings/mockedApiVersion/2023-12-13/channel/UID0118?timezone=Europe%2FBrussels`
   )
 })
 

sites/rotana.net/rotana.net.config.js

@@ -4,16 +4,19 @@ const dayjs = require('dayjs')
 const timezone = require('dayjs/plugin/timezone')
 const utc = require('dayjs/plugin/utc')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:rotana.net')
 
 dayjs.extend(timezone)
 dayjs.extend(utc)
 dayjs.extend(customParseFormat)
 
-const tz = 'Asia/Riyadh'
+doFetch
-const nworker = 25
+  .setCheckResult(false)
+  .setDebugger(debug)
 
-const headers = {
+const tz = 'Asia/Riyadh'
+const defaultHeaders = {
   'User-Agent':
     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 OPR/104.0.0.0'
 }
@@ -26,7 +29,7 @@ module.exports = {
     return `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&tz=`
   },
   request: {
-    headers,
+    headers: defaultHeaders,
     timeout: 15000
   },
   async parser({ content, headers, channel, date }) {
@@ -37,31 +40,20 @@ module.exports = {
 
     const items = parseItems(content, date)
     if (items.length) {
-      const workers = []
+      const queues = []
-      const n = Math.min(nworker, items.length)
+      for (const item of items) {
-      while (workers.length < n) {
+        const url = `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&itemId=${item.program}`
-        const worker = () => {
+        const params = {
-          if (items.length) {
+          headers: {
-            const item = items.shift()
+            ...defaultHeaders,
-            parseProgram(item, channel)
+            'X-Requested-With': 'XMLHttpRequest',
-              .then(() => {
+            cookie: cookies[channel.lang],
-                programs.push(item)
-                worker()
-              })
-          } else {
-            workers.splice(workers.indexOf(worker), 1)
           }
         }
-        workers.push(worker)
+        queues.push({ i: item, url, params })
-        worker()
       }
-      await new Promise(resolve => {
+      await doFetch(queues, (queue, res) => {
-        const interval = setInterval(() => {
+        programs.push(parseProgram(queue.i, res))
-          if (workers.length === 0) {
-            clearInterval(interval)
-            resolve()
-          }
-        }, 500)
       })
     }
 
@@ -83,19 +75,7 @@ module.exports = {
   }
 }
 
-async function parseProgram(item, channel) {
+function parseProgram(item, result) {
-  if (item.program) {
-    const url = `https://rotana.net/${channel.lang}/streams?channel=${channel.site_id}&itemId=${item.program}`
-    const params = {
-      headers: Object.assign({}, headers, { 'X-Requested-With': 'XMLHttpRequest' }),
-      Cookie: cookies[channel.lang]
-    }
-    debug(`fetching description ${url}`)
-    const result = await axios
-      .get(url, params)
-      .then(response => response.data)
-      .catch(console.error)
-
   const $ = cheerio.load(result)
   const details = $('.trending-info .row div > span')
   if (details.length) {
@@ -144,7 +124,7 @@ async function parseProgram(item, channel) {
     item.image = img.attr('src')
   }
   delete item.program
-  }
+  return item
 }
 
 function parseItems(content, date) {

sites/rotana.net/rotana.net.test.js

@@ -52,12 +52,11 @@ it('can generate valid arabic url', () => {
 })
 
 it('can parse english response', async () => {
-  let result = await parser({
+  const result = (await parser({
     channel,
     date,
     content: fs.readFileSync(path.join(__dirname, '/__data__/content_en.html'))
-  })
+  })).map(a => {
-  result = result.map(a => {
     a.start = a.start.toJSON()
     a.stop = a.stop.toJSON()
     return a
@@ -76,12 +75,11 @@ it('can parse english response', async () => {
 })
 
 it('can parse arabic response', async () => {
-  let result = await parser({
+  const result = (await parser({
     channel: channelAr,
     date,
     content: fs.readFileSync(path.join(__dirname, '/__data__/content_ar.html'))
-  })
+  })).map(a => {
-  result = result.map(a => {
     a.start = a.start.toJSON()
     a.stop = a.stop.toJSON()
     return a

sites/sky.com/sky.com.config.js

@@ -1,11 +1,12 @@
 const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:sky.com')
 
 dayjs.extend(utc)
 
-const nworker = 10
+doFetch.setDebugger(debug)
 
 module.exports = {
   site: 'sky.com',
@@ -48,7 +49,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const queues = [{ t: 'r', u: 'https://www.sky.com/tv-guide' }]
+    const queues = [{ t: 'r', url: 'https://www.sky.com/tv-guide' }]
     await doFetch(queues, (queue, res) => {
       // process regions
       if (queue.t === 'r') {
@@ -56,7 +57,7 @@ module.exports = {
         const initialData = JSON.parse(decodeURIComponent($('#initialData').text()))
         initialData.state.epgData.regions
           .forEach(region => {
-            queues.push({ t: 'c', u: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
+            queues.push({ t: 'c', url: `https://awk.epgsky.com/hawk/linear/services/${region.bouquet}/${region.subBouquet}` })
           })
       }
       // process channels
@@ -78,64 +79,3 @@ module.exports = {
     return Object.values(channels)
   }
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = (res, headers) => {
-          if (res) {
-            cb(queue, res, headers)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        if (typeof debug === 'function') {
-          debug(`fetch %s with %s`, url, JSON.stringify(params))
-        }
-        axios[method](url, params)
-          .then(response => {
-            done(response.data, response.headers)
-          })
-          .catch(err => {
-            console.error(`Unable to fetch ${url}: ${err.message}!`)
-            done()
-          })
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}

sites/startimestv.com/startimestv.com.config.js

@@ -1,14 +1,16 @@
-const axios = require('axios')
 const cheerio = require('cheerio')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:startimestv.com')
 
 dayjs.extend(utc)
 dayjs.extend(customParseFormat)
 
-const nworker = 5
+doFetch
+  .setDebugger(debug)
+  .setMaxWorker(5)
 
 module.exports = {
   site: 'startimestv.com',
@@ -46,7 +48,7 @@ module.exports = {
   },
   async channels() {
     const channels = {}
-    const queues = [{ t: 'a', u: 'https://www.startimestv.com/tv_guide.html' }]
+    const queues = [{ t: 'a', url: 'https://www.startimestv.com/tv_guide.html' }]
     await doFetch(queues, (queue, res) => {
       // process area-id
       if (queue.t === 'a') {
@@ -57,7 +59,7 @@ module.exports = {
             const areaId = dd.attr('area-id')
             queues.push({
               t: 's',
-              u: 'https://www.startimestv.com/tv_guide.html',
+              url: 'https://www.startimestv.com/tv_guide.html',
               params: {
                 headers: {
                   cookie: `default_areaID=${areaId}`
@@ -110,66 +112,3 @@ function parseText($item) {
 
   return text
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = res => {
-          if (res) {
-            cb(queue, res)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        debug(`fetch %s with %s`, url, JSON.stringify(params))
-        if (method === 'post') {
-          axios
-            .post(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        } else {
-          axios
-            .get(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        }
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}

sites/tv.yandex.ru/tv.yandex.ru.config.js

@@ -1,18 +1,22 @@
 const dayjs = require('dayjs')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:tv.yandex.ru')
 
+doFetch
+  .setDebugger(debug)
+  .setMaxWorker(10)
+
 // enable to fetch guide description but its take a longer time
 const detailedGuide = true
-const nworker = 10
 
 // update this data by heading to https://tv.yandex.ru and change the values accordingly
 const cookies = {
-  i: 'dkim62pClrWWC4CShVQYMpVw1ELNVw4XJdL/lzT4E2r05IgcST1GtCA4ho/UyGgW2AO4qftDfZzGX2OHqCzwY7GUkpM=',
+  i: 'eIUfSP+/mzQWXcH+Cuz8o1vY+D2K8fhBd6Sj0xvbPZeO4l3cY+BvMp8fFIuM17l6UE1Z5+R2a18lP00ex9iYVJ+VT+c=',
-  spravka: 'dD0xNzMyNjgzMTEwO2k9MTgwLjI0OC41OS40MDtEPTkyOUM2MkQ0Mzc3OUNBMUFCNzg3NTIyMEQ4OEJBMEVBMzQ2RUNGNUU5Q0FEQUM5RUVDMTFCNjc1ODA2MThEQTQ3RTY3RTUyRUNBRDdBMTY2OTY1MjMzRDU1QjNGMTc1MDA0NDM3MjBGMUNGQTM5RjA3OUQwRjE2MzQxMUNFOTgxQ0E0RjNGRjRGODNCMEM1QjlGNTg5RkI4NDk0NEM2QjNDQUQ5NkJGRTBFNTVCQ0Y1OTEzMEY0O3U9MTczMjY4MzExMDY3MTA1MzIzNDtoPTA1YWJmMTY0ZmI2MGViNTBhMDUwZWUwMThmYWNiYjhm',
+  spravka: 'dD0xNzM0MjA0NjM4O2k9MTI1LjE2NC4xNDkuMjAwO0Q9QTVCQ0IyOTI5RDQxNkU5NkEyOTcwMTNDMzZGMDAzNjRDNTFFNDM4QkE2Q0IyOTJDRjhCOTZDRDIzODdBQzk2MzRFRDc5QTk2Qjc2OEI1MUY5MTM5M0QzNkY3OEQ2OUY3OTUwNkQ3RjBCOEJGOEJDMjAwMTQ0RDUwRkFCMDNEQzJFMDI2OEI5OTk5OUJBNEFERUYwOEQ1MjUwQTE0QTI3RDU1MEQwM0U0O3U9MTczNDIwNDYzODUyNDYyNzg1NDtoPTIxNTc0ZTc2MDQ1ZjcwMDBkYmY0NTVkM2Q2ZWMyM2Y1',
   yandexuid: '1197179041732383499',
   yashr: '4682342911732383504',
   yuidss: '1197179041732383499',
-  user_display: 930,
+  user_display: 824,
 }
 const headers = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 OPR/114.0.0.0',
@@ -91,32 +95,35 @@ async function fetchSchedules({ date, content = null }) {
 
   let mainApi
   // parse content as schedules and add to queue if more requests is needed
-  const f = (data, src) => {
+  const f = (src, res, headers) => {
     if (src) {
       fetches.push(src)
     }
-    const [q, s] = parseContent(data, date)
+    if (headers) {
+      parseCookies(headers)
+    }
+    const [q, s] = parseContent(res, date)
     if (!mainApi) {
       mainApi = true
       if (caches.region) {
-        queues.push(getUrl(date, caches.region))
+        queues.push(getQueue(getUrl(date, caches.region), src))
       }
     }
     for (const url of q) {
       if (fetches.indexOf(url) < 0) {
-        queues.push(url)
+        queues.push(getQueue(url, src))
       }
     }
     schedules.push(...s)
   }
   // is main html already fetched?
   if (content) {
-    f(content)
+    f(url, content)
   } else {
-    queues.push(url)
+    queues.push(getQueue(url, 'https://tv.yandex.ru/'))
   }
   // fetch all queues
-  await doFetch(queues, url, f)
+  await doFetch(queues, f)
 
   return schedules
 }
@@ -129,17 +136,20 @@ async function fetchPrograms({ schedules, date, channel }) {
       queues.push(
         ...schedule.events
           .filter(event => date.isSame(event.start, 'day'))
-          .map(event => getUrl(null, caches.region, null, event))
+          .map(event => getQueue(getUrl(null, caches.region, null, event), 'https://tv.yandex.ru/'))
       )
     })
-  await doFetch(queues, getUrl(date), content => {
+  await doFetch(queues, (queue, res, headers) => {
+    if (headers) {
+      parseCookies(headers)
+    }
     // is it a program?
-    if (content?.program) {
+    if (res?.program) {
       let updated = false
       schedules.forEach(schedule => {
         schedule.events.forEach(event => {
-          if (event.channelFamilyId === content.channelFamilyId && event.id === content.id) {
+          if (event.channelFamilyId === res.channelFamilyId && event.id === res.id) {
-            Object.assign(event, content)
+            Object.assign(event, res)
             updated = true
             return true
           }
@@ -152,61 +162,6 @@ async function fetchPrograms({ schedules, date, channel }) {
   })
 }
 
-async function doFetch(queues, referer, cb) {
-  if (queues.length) {
-    const workers = []
-    let n = Math.min(nworker, queues.length)
-    while (workers.length < n) {
-      const worker = () => {
-        if (queues.length) {
-          const url = queues.shift()
-          debug(`Fetching ${url}`)
-          const data = {
-            'Origin': 'https://tv.yandex.ru',
-          }
-          if (referer) {
-            data['Referer'] = referer
-          }
-          if (url.indexOf('api') > 0) {
-            data['X-Requested-With'] = 'XMLHttpRequest'
-          }
-          const headers = getHeaders(data)
-          doRequest(url, { headers })
-            .then(res => {
-              cb(res, url)
-              worker()
-            })
-        } else {
-          workers.splice(workers.indexOf(worker), 1)
-        }
-      }
-      workers.push(worker)
-      worker()
-    }
-    await new Promise(resolve => {
-      const interval = setInterval(() => {
-        if (workers.length === 0) {
-          clearInterval(interval)
-          resolve()
-        }
-      }, 500)
-    })
-  }
-}
-
-async function doRequest(url, params) {
-  const axios = require('axios')
-  const content = await axios
-    .get(url, params)
-    .then(response => {
-      parseCookies(response.headers)
-      return response.data
-    })
-    .catch(err => console.error(err.message))
-
-  return content
-}
-
 function parseContent(content, date, checkOnly = false) {
   const queues = []
   const schedules = []
@@ -308,3 +263,20 @@ function getUrl(date, region = null, page = null, event = null) {
   }
   return url
 }
+
+function getQueue(url, referer) {
+  const data = {
+    'Origin': 'https://tv.yandex.ru',
+  }
+  if (referer) {
+    data['Referer'] = referer
+  }
+  if (url.indexOf('api') > 0) {
+    data['X-Requested-With'] = 'XMLHttpRequest'
+  }
+  const headers = getHeaders(data)
+  return {
+    url,
+    params: { headers }
+  }
+}

sites/tv.yandex.ru/tv.yandex.ru.test.js

@@ -52,12 +52,12 @@ it('can generate valid url', () => {
 it('can generate valid request headers', () => {
   expect(request.headers).toMatchObject({
     Cookie:
-      'i=dkim62pClrWWC4CShVQYMpVw1ELNVw4XJdL/lzT4E2r05IgcST1GtCA4ho/UyGgW2AO4qftDfZzGX2OHqCzwY7GUkpM=; ' +
+      'i=eIUfSP+/mzQWXcH+Cuz8o1vY+D2K8fhBd6Sj0xvbPZeO4l3cY+BvMp8fFIuM17l6UE1Z5+R2a18lP00ex9iYVJ+VT+c=; ' +
-      'spravka=dD0xNzMyNjgzMTEwO2k9MTgwLjI0OC41OS40MDtEPTkyOUM2MkQ0Mzc3OUNBMUFCNzg3NTIyMEQ4OEJBMEVBMzQ2RUNGNUU5Q0FEQUM5RUVDMTFCNjc1ODA2MThEQTQ3RTY3RTUyRUNBRDdBMTY2OTY1MjMzRDU1QjNGMTc1MDA0NDM3MjBGMUNGQTM5RjA3OUQwRjE2MzQxMUNFOTgxQ0E0RjNGRjRGODNCMEM1QjlGNTg5RkI4NDk0NEM2QjNDQUQ5NkJGRTBFNTVCQ0Y1OTEzMEY0O3U9MTczMjY4MzExMDY3MTA1MzIzNDtoPTA1YWJmMTY0ZmI2MGViNTBhMDUwZWUwMThmYWNiYjhm; ' +
+      'spravka=dD0xNzM0MjA0NjM4O2k9MTI1LjE2NC4xNDkuMjAwO0Q9QTVCQ0IyOTI5RDQxNkU5NkEyOTcwMTNDMzZGMDAzNjRDNTFFNDM4QkE2Q0IyOTJDRjhCOTZDRDIzODdBQzk2MzRFRDc5QTk2Qjc2OEI1MUY5MTM5M0QzNkY3OEQ2OUY3OTUwNkQ3RjBCOEJGOEJDMjAwMTQ0RDUwRkFCMDNEQzJFMDI2OEI5OTk5OUJBNEFERUYwOEQ1MjUwQTE0QTI3RDU1MEQwM0U0O3U9MTczNDIwNDYzODUyNDYyNzg1NDtoPTIxNTc0ZTc2MDQ1ZjcwMDBkYmY0NTVkM2Q2ZWMyM2Y1; ' +
       'yandexuid=1197179041732383499; ' +
       'yashr=4682342911732383504; ' +
       'yuidss=1197179041732383499; ' +
-      'user_display=930'
+      'user_display=824'
   })
 })
 

sites/virgintvgo.virginmedia.com/virgintvgo.virginmedia.com.config.js

@@ -1,11 +1,13 @@
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const doFetch = require('@ntlab/sfetch')
 const debug = require('debug')('site:virgintvgo.virginmedia.com')
 
 dayjs.extend(utc)
 
+doFetch.setDebugger(debug)
+
 const detailedGuide = true
-const nworker = 25
 
 module.exports = {
   site: 'virgintvgo.virginmedia.com',
@@ -110,66 +112,3 @@ module.exports = {
     return channels
   }
 }
-
-async function doFetch(queues, cb) {
-  const axios = require('axios')
-
-  let n = Math.min(nworker, queues.length)
-  const workers = []
-  const adjustWorker = () => {
-    if (queues.length > workers.length && workers.length < nworker) {
-      let nw = Math.min(nworker, queues.length)
-      if (n < nw) {
-        n = nw
-        createWorker()
-      }
-    }
-  }
-  const createWorker = () => {
-    while (workers.length < n) {
-      startWorker()
-    }
-  }
-  const startWorker = () => {
-    const worker = () => {
-      if (queues.length) {
-        const queue = queues.shift()
-        const done = res => {
-          if (res) {
-            cb(queue, res)
-            adjustWorker()
-          }
-          worker()
-        }
-        const url = typeof queue === 'string' ? queue : queue.u
-        const params = typeof queue === 'object' && queue.params ? queue.params : {}
-        const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
-        debug(`fetch %s with %s`, url, JSON.stringify(params))
-        if (method === 'post') {
-          axios
-            .post(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        } else {
-          axios
-            .get(url, params)
-            .then(response => done(response.data))
-            .catch(console.error)
-        }
-      } else {
-        workers.splice(workers.indexOf(worker), 1)
-      }
-    }
-    workers.push(worker)
-    worker()
-  }
-  createWorker()
-  await new Promise(resolve => {
-    const interval = setInterval(() => {
-      if (workers.length === 0) {
-        clearInterval(interval)
-        resolve()
-      }
-    }, 500)
-  })
-}

sites/visionplus.id/visionplus.id.config.js

@@ -1,16 +1,13 @@
-const axios = require('axios')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
-const cheerio = require('cheerio')
 
 dayjs.extend(utc)
 dayjs.extend(timezone)
 dayjs.extend(customParseFormat)
 
 const languages = { en: 'ENG', id: 'IND' }
-const tz = 'Asia/Jakarta'
 
 module.exports = {
   site: 'visionplus.id',
@@ -22,7 +19,7 @@ module.exports = {
       'YYYY-MM-DD'
     )}T00%3A00%3A00Z&view=cd-events-grid-view`
   },
-  parser({ content, channel, date }) {
+  parser({ content, channel }) {
     const programs = []
     const json = JSON.parse(content)
     if (Array.isArray(json.evs)) {

sites/visionplus.id/visionplus.id.test.js

@@ -1,9 +1,10 @@
-const { parser, url, request } = require('./visionplus.id.config.js')
+const { parser, url } = require('./visionplus.id.config.js')
 const fs = require('fs')
 const path = require('path')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const customParseFormat = require('dayjs/plugin/customParseFormat')
+
 dayjs.extend(customParseFormat)
 dayjs.extend(utc)
 
@@ -17,7 +18,6 @@ const channel = {
 }
 const channelId = { ...channel,  lang: 'id' }
 
-
 it('can generate valid url', () => {
   expect(url({ channel, date })).toBe(
     'https://www.visionplus.id/managetv/tvinfo/events/schedule?language=ENG&serviceId=00000000000000000079&start=2024-11-24T00%3A00%3A00Z&end=2024-11-25T00%3A00%3A00Z&view=cd-events-grid-view'
@@ -30,7 +30,7 @@ it('can generate valid url', () => {
 it('can parse response', () => {
   let content = fs.readFileSync(path.resolve(__dirname, '__data__/content_en.json'))
   let results = parser({ content, channel, date })
-  results = results.map(p => {
+    .map(p => {
       p.start = p.start.toJSON()
       p.stop = p.stop.toJSON()
       return p
@@ -48,7 +48,7 @@ it('can parse response', () => {
 
   content = fs.readFileSync(path.resolve(__dirname, '__data__/content_id.json'))
   results = parser({ content, channel: channelId, date })
-  results = results.map(p => {
+    .map(p => {
       p.start = p.start.toJSON()
       p.stop = p.stop.toJSON()
       return p

yarn.lock

@@ -758,6 +758,13 @@
   dependencies:
     semver "^7.3.5"
 
+"@ntlab/sfetch@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.npmjs.org/@ntlab/sfetch/-/sfetch-1.0.0.tgz"
+  integrity sha512-AWrC43z1TncvB7S7dl9Wn8xZpCqdKFBfXqaN3BXPfJeS3gxV9Fm86eAsW95YdXTOgPWbCC/GAgVuXi6Aot6DkQ==
+  dependencies:
+    axios "^1.7.9"
+
 "@octokit/auth-token@^3.0.0":
   version "3.0.2"
   resolved "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.2.tgz"
@@ -1354,7 +1361,7 @@ axios-mock-adapter@^1.20.0:
     is-blob "^2.1.0"
     is-buffer "^2.0.5"
 
-axios@^1.5.1, axios@^1.6.1, "axios@>= 0.9.0", axios@>=0.20.0:
+axios@^1.5.1, axios@^1.6.1, axios@^1.7.9, "axios@>= 0.9.0", axios@>=0.20.0:
   version "1.7.9"
   resolved "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz"
   integrity sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==