Merge pull request #677 from iptv-org/cache-response

Cache response
2025-05-10 00:50:09 -04:00 · 2022-03-29 20:24:26 +03:00 · 2022-03-29 20:24:26 +03:00 · b6991bf45f
commit b6991bf45f
parent 966815d6df 9e506438aa
4 changed files with 119 additions and 21 deletions
--- a/.github/workflows/_load.yml
+++ b/.github/workflows/_load.yml
@ -32,7 +32,7 @@ jobs:
          cache: 'npm'
      - run: npm install
      - run: CHANNELS_PATH=sites/${{inputs.site}}/*.channels.xml npm run queue:create -- --max-clusters=1 --days=2
-      - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cluster-id=1
+      - run: NODE_OPTIONS=--insecure-http-parser npm run cluster:load -- --timeout=30000 --cache-max-age=21600000 --cluster-id=1
      - run: npm run programs:save
      - run: npm run guides:update
      - uses: tibdex/github-app-token@v1
--- a/package-lock.json
+++ b/package-lock.json
@ -14,7 +14,7 @@
        "commander": "^8.2.0",
        "csv-parser": "^3.0.0",
        "dayjs": "^1.10.8",
-        "epg-grabber": "^0.24.0",
+        "epg-grabber": "^0.25.1",
        "epg-parser": "^0.1.6",
        "form-data": "^4.0.0",
        "fs-extra": "^10.0.1",
@ -1270,6 +1270,18 @@
        "follow-redirects": "^1.14.0"
      }
    },
+    "node_modules/axios-cache-adapter": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz",
+      "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==",
+      "dependencies": {
+        "cache-control-esm": "1.0.0",
+        "md5": "^2.2.1"
+      },
+      "peerDependencies": {
+        "axios": "~0.21.1"
+      }
+    },
    "node_modules/axios-cookiejar-support": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
@ -1529,6 +1541,11 @@
      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
    },
+    "node_modules/cache-control-esm": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz",
+      "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g=="
+    },
    "node_modules/cacheable-lookup": {
      "version": "5.0.4",
      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
@ -1621,6 +1638,14 @@
      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA=="
    },
+    "node_modules/charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=",
+      "engines": {
+        "node": "*"
+      }
+    },
    "node_modules/cheerio": {
      "version": "1.0.0-rc.10",
      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz",
@ -1875,6 +1900,14 @@
        "node": ">= 8"
      }
    },
+    "node_modules/crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=",
+      "engines": {
+        "node": "*"
+      }
+    },
    "node_modules/css-select": {
      "version": "4.1.3",
      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
@ -2213,11 +2246,12 @@
      }
    },
    "node_modules/epg-grabber": {
-      "version": "0.24.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz",
-      "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==",
+      "version": "0.25.1",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz",
+      "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==",
      "dependencies": {
        "axios": "^0.21.1",
+        "axios-cache-adapter": "^2.7.3",
        "axios-cookiejar-support": "^1.0.1",
        "axios-mock-adapter": "^1.20.0",
        "commander": "^7.1.0",
@ -4072,6 +4106,21 @@
        "markdown-include": "bin/cli.js"
      }
    },
+    "node_modules/md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "dependencies": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      }
+    },
+    "node_modules/md5/node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+    },
    "node_modules/merge-stream": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
@ -6890,6 +6939,15 @@
        "follow-redirects": "^1.14.0"
      }
    },
+    "axios-cache-adapter": {
+      "version": "2.7.3",
+      "resolved": "https://registry.npmjs.org/axios-cache-adapter/-/axios-cache-adapter-2.7.3.tgz",
+      "integrity": "sha512-A+ZKJ9lhpjthOEp4Z3QR/a9xC4du1ALaAsejgRGrH9ef6kSDxdFrhRpulqsh9khsEnwXxGfgpUuDp1YXMNMEiQ==",
+      "requires": {
+        "cache-control-esm": "1.0.0",
+        "md5": "^2.2.1"
+      }
+    },
    "axios-cookiejar-support": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
@ -7075,6 +7133,11 @@
      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
    },
+    "cache-control-esm": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/cache-control-esm/-/cache-control-esm-1.0.0.tgz",
+      "integrity": "sha512-Fa3UV4+eIk4EOih8FTV6EEsVKO0W5XWtNs6FC3InTfVz+EjurjPfDXY5wZDo/lxjDxg5RjNcurLyxEJBcEUx9g=="
+    },
    "cacheable-lookup": {
      "version": "5.0.4",
      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
@ -7138,6 +7201,11 @@
      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
      "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA=="
    },
+    "charenc": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
+      "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc="
+    },
    "cheerio": {
      "version": "1.0.0-rc.10",
      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.10.tgz",
@ -7348,6 +7416,11 @@
        "which": "^2.0.1"
      }
    },
+    "crypt": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
+      "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs="
+    },
    "css-select": {
      "version": "4.1.3",
      "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.1.3.tgz",
@ -7591,11 +7664,12 @@
      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
    },
    "epg-grabber": {
-      "version": "0.24.0",
-      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.24.0.tgz",
-      "integrity": "sha512-TrFKHF3kUjffJDCASBv/gyjlIcahbIJf02TgmgKzFQjSZYCLhqR87U7MXbWkioftdM5BLoQKbuaI/OQGbIJsBQ==",
+      "version": "0.25.1",
+      "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.25.1.tgz",
+      "integrity": "sha512-4Lv6FrVQ5R7ntWXkpl4tor8QIVlIdTtOeH4r/x2DAQWp7ZwT/IdZ6EUSoJ4lYSY63ezA7nMxrxdJLrMNUDU4+A==",
      "requires": {
        "axios": "^0.21.1",
+        "axios-cache-adapter": "^2.7.3",
        "axios-cookiejar-support": "^1.0.1",
        "axios-mock-adapter": "^1.20.0",
        "commander": "^7.1.0",
@ -8976,6 +9050,23 @@
        "q": "^1.2.0"
      }
    },
+    "md5": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
+      "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
+      "requires": {
+        "charenc": "0.0.2",
+        "crypt": "0.0.2",
+        "is-buffer": "~1.1.6"
+      },
+      "dependencies": {
+        "is-buffer": {
+          "version": "1.1.6",
+          "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+          "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
+        }
+      }
+    },
    "merge-stream": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
--- a/package.json
+++ b/package.json
@ -36,7 +36,7 @@
    "commander": "^8.2.0",
    "csv-parser": "^3.0.0",
    "dayjs": "^1.10.8",
-    "epg-grabber": "^0.24.0",
+    "epg-grabber": "^0.25.1",
    "epg-parser": "^0.1.6",
    "form-data": "^4.0.0",
    "fs-extra": "^10.0.1",
--- a/scripts/commands/cluster/load.js
+++ b/scripts/commands/cluster/load.js
@ -1,5 +1,5 @@
 const _ = require('lodash')
-const grabber = require('epg-grabber')
+const EPGGrabber = require('epg-grabber')
 const { program } = require('commander')
 const { db, logger, timer, file, parser } = require('../../core')

@ -11,6 +11,11 @@ const options = program
    'Set a timeout for each request (in mileseconds)',
    parser.parseNumber
  )
+  .option(
+    '--cache-max-age <cacheMaxAge>',
+    'Maximum time for storing each request (in milliseconds)',
+    parser.parseNumber
+  )
  .option('--debug', 'Enable debug mode', false)
  .parse(process.argv)
  .opts()
@ -33,18 +38,20 @@ async function main() {
  logger.info('Loading...')
  const results = {}
  let i = 1
-  for (const item of items) {
-    let config = require(file.resolve(item.configPath))
-
-    config = _.merge(config, {
-      debug: options.debug,
-      delay: options.delay,
-      request: {
-        timeout: options.timeout
+  let config = require(file.resolve(items[0].configPath))
+  config = _.merge(config, {
+    debug: options.debug,
+    delay: options.delay,
+    request: {
+      timeout: options.timeout,
+      cache: {
+        maxAge: options.cacheMaxAge
      }
-    })
-
-    await grabber.grab(item.channel, item.date, config, async (data, err) => {
+    }
+  })
+  const grabber = new EPGGrabber(config)
+  for (const item of items) {
+    await grabber.grab(item.channel, item.date, async (data, err) => {
      logger.info(
        `[${i}/${total}] ${item.channel.site} (${item.channel.lang}) - ${
          item.channel.xmltv_id