Merge pull request #677 from iptv-org/cache-response

Cache response

.github/workflows/_load.yml

@@ -32,7 +32,7 @@ jobs:
           cache: 'npm'
       - run: npm install
       - run: CHANNELS_PATH=sites/${{inputs.site}}/*.channels.xml npm run queue:create -- --max-clusters=1 --days=2
-      - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cluster-id=1
+      - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1
       - run: npm run programs:save
       - run: npm run guides:update
       - uses: tibdex/github-app-token@v1

package-lock.json

@@ -14,7 +14,7 @@
         "commander": "^8.2.0",
         "csv-parser": "^3.0.0",
         "dayjs": "^1.10.8",
-        "epg-grabber": "^0.24.0",
+        "epg-grabber": "^0.25.1",
         "epg-parser": "^0.1.6",
         "form-data": "^4.0.0",
         "fs-extra": "^10.0.1",
@@ -1270,6 +1270,18 @@
         "follow-redirects": "^1.14.0"
       }
     },
+    "node_modules/axios-cache-adapter": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz",
+      "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==",
+      "dependencies": {
+        "cache-control-esm": "1.0.0",
+        "md5": "^2.2.1"
+      },
+      "peerDependencies": {
+        "axios": "~0.21.1"
+      }
+    },
     "node_modules/axios-cookiejar-support": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
@@ -1529,6 +1541,11 @@
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
     },
+    "node_modules/cache-control-esm": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz",
+      "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g=="
+    },
     "node_modules/cacheable-lookup": {
       "version": "5.0.4",
       "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
@@ -1621,6 +1638,14 @@
       "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
       "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA=="
     },
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/cheerio": {
       "version": "1.0.0-rc.10",
       "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz",
@@ -1875,6 +1900,14 @@
         "node": ">= 8"
       }
     },
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
+      "engines": {
+        "node": "*"
+      }
+    },
     "node_modules/css-select": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
@@ -2213,11 +2246,12 @@
       }
     },
     "node_modules/epg-grabber": {
-      "version": "0.24.0",
+      "version": "0.25.1",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz",
-      "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==",
+      "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==",
       "dependencies": {
         "axios": "^0.21.1",
+        "axios-cache-adapter": "^2.7.3",
         "axios-cookiejar-support": "^1.0.1",
         "axios-mock-adapter": "^1.20.0",
         "commander": "^7.1.0",
@@ -4072,6 +4106,21 @@
         "markdown-include": "bin/cli.js"
       }
     },
+    "node_modules/md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "dependencies": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
+    "node_modules/md5/node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
     "node_modules/merge-stream": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@@ -6890,6 +6939,15 @@
         "follow-redirects": "^1.14.0"
       }
     },
+    "axios-cache-adapter": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz",
+      "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==",
+      "requires": {
+        "cache-control-esm": "1.0.0",
+        "md5": "^2.2.1"
+      }
+    },
     "axios-cookiejar-support": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
@@ -7075,6 +7133,11 @@
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
     },
+    "cache-control-esm": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz",
+      "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g=="
+    },
     "cacheable-lookup": {
       "version": "5.0.4",
       "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
@@ -7138,6 +7201,11 @@
       "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
       "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA=="
     },
+    "charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
+    },
     "cheerio": {
       "version": "1.0.0-rc.10",
       "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz",
@@ -7348,6 +7416,11 @@
         "which": "^2.0.1"
       }
     },
+    "crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
+    },
     "css-select": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
@@ -7591,11 +7664,12 @@
       "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
     },
     "epg-grabber": {
-      "version": "0.24.0",
+      "version": "0.25.1",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz",
-      "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==",
+      "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==",
       "requires": {
         "axios": "^0.21.1",
+        "axios-cache-adapter": "^2.7.3",
         "axios-cookiejar-support": "^1.0.1",
         "axios-mock-adapter": "^1.20.0",
         "commander": "^7.1.0",
@@ -8976,6 +9050,23 @@
         "q": "^1.2.0"
       }
     },
+    "md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "requires": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "1.1.6",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+          "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+        }
+      }
+    },
     "merge-stream": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",

package.json

@@ -36,7 +36,7 @@
     "commander": "^8.2.0",
     "csv-parser": "^3.0.0",
     "dayjs": "^1.10.8",
-    "epg-grabber": "^0.24.0",
+    "epg-grabber": "^0.25.1",
     "epg-parser": "^0.1.6",
     "form-data": "^4.0.0",
     "fs-extra": "^10.0.1",

scripts/commands/cluster/load.js

@@ -1,5 +1,5 @@
 const _ = require('lodash')
-const grabber = require('epg-grabber')
+const EPGGrabber = require('epg-grabber')
 const { program } = require('commander')
 const { db, logger, timer, file, parser } = require('../../core')
 
@@ -11,6 +11,11 @@ const options = program
     'Set a timeout for each request (in mileseconds)',
     parser.parseNumber
   )
+  .option(
+    '--cache-max-age <cacheMaxAge>',
+    'Maximum time for storing each request (in milliseconds)',
+    parser.parseNumber
+  )
   .option('--debug', 'Enable debug mode', false)
   .parse(process.argv)
   .opts()
@@ -33,18 +38,20 @@ async function main() {
   logger.info('Loading...')
   const results = {}
   let i = 1
-  for (const item of items) {
+  let config = require(file.resolve(items[0].configPath))
-    let config = require(file.resolve(item.configPath))
+  config = _.merge(config, {
-
+    debug: options.debug,
-    config = _.merge(config, {
+    delay: options.delay,
-      debug: options.debug,
+    request: {
-      delay: options.delay,
+      timeout: options.timeout,
-      request: {
+      cache: {
-        timeout: options.timeout
+        maxAge: options.cacheMaxAge
       }
-    })
+    }
-
+  })
-    await grabber.grab(item.channel, item.date, config, async (data, err) => {
+  const grabber = new EPGGrabber(config)
+  for (const item of items) {
+    await grabber.grab(item.channel, item.date, async (data, err) => {
       logger.info(
         `[${i}/${total}] ${item.channel.site} (${item.channel.lang}) - ${
           item.channel.xmltv_id