wip

bin/epg-grabber/index.js

@@ -3,6 +3,7 @@
 const fs = require('fs')
 const path = require('path')
 const axios = require('axios')
+const axiosDelayAdapter = require('axios-delay').default
 const utils = require('./utils')
 const { Command } = require('commander')
 const program = new Command()
@@ -22,61 +23,47 @@ program
 const options = program.opts()
 
 const config = utils.parseConfig(options.config)
+const sites = utils.loadSites(options.sites)
 
-return console.log(config)
+const client = axios.create({
+  adapter: axiosDelayAdapter(axios.defaults.adapter),
+  headers: { 'User-Agent': config.userAgent }
+})
 
-const sites = {
-  'tv.yandex.ru': {
-    url: function ({ date, channel }) {
-      return `https://tv.yandex.ru/channel/${channel.site_id}?date=${date.format('YYYY-MM-DD')}`
-    },
-    parser: function ({ channel, content }) {
-      const initialState = content.match(/window.__INITIAL_STATE__ = (.*);/i)[1]
-      const data = JSON.parse(initialState, null, 2)
-      const programs = data.channel.schedule.events.map(i => {
-        return {
-          title: i.title,
-          description: i.program.description,
-          start: i.start,
-          stop: i.finish,
-          lang: 'ru',
-          channel: channel['xmltv_id']
-        }
-      })
-
-      return programs
-    }
-  }
-}
-
-function main() {
+async function main() {
   const d = dayjs.utc()
   const dates = Array.from({ length: config.days }, (_, i) => d.add(i, 'd'))
   const channels = config.channels
-  const promises = []
+  const requests = []
   channels.forEach(channel => {
     const site = sites[channel.site]
     dates.forEach(date => {
       const url = site.url({ date, channel })
-      const promise = axios.get(url).then(response => {
-        return site.parser({ channel, content: response.data })
+      const promise = client.get(url).catch(console.log)
+
+      requests.push({
+        url,
+        site,
+        channel,
+        promise
       })
-
-      promises.push(promise)
     })
   })
 
-  Promise.allSettled(promises).then(results => {
-    let programs = []
-    results.forEach(result => {
-      if (result.status === 'fulfilled') {
-        programs = programs.concat(result.value)
-      }
-    })
+  let programs = []
+  for (let request of requests) {
+    const progs = await request.promise
+      .then(response => {
+        const channel = request.channel
+        console.log(`${channel.site} - ${channel.xmltv_id}`)
 
-    const xml = utils.convertToXMLTV({ channels, programs })
-    fs.writeFileSync(path.resolve(__dirname, config.filename), xml)
-  })
+        return request.site.parser({ channel, content: response.data })
+      })
+      .then(utils.sleep(3000))
+    programs = programs.concat(progs)
+  }
+  const xml = utils.convertToXMLTV({ channels, programs })
+  fs.writeFileSync(path.resolve(__dirname, config.filename), xml)
 }
 
 main()

bin/epg-grabber/utils.js

@@ -2,16 +2,15 @@ const fs = require('fs')
 const path = require('path')
 const convert = require('xml-js')
 const dayjs = require('dayjs')
+const glob = require('glob')
 
 const utils = {}
 utils.convertToXMLTV = function ({ channels, programs }) {
-  let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>`
+  let output = '<?xml version="1.0" encoding="UTF-8" ?><tv>'
 
   for (let channel of channels) {
     output += `
-<channel id="${channel['xmltv_id']}">
-  <display-name>${channel.name}</display-name>
-</channel>`
+<channel id="${channel['xmltv_id']}"><display-name>${channel.name}</display-name></channel>`
   }
 
   for (let program of programs) {
@@ -19,14 +18,13 @@ utils.convertToXMLTV = function ({ channels, programs }) {
     const stop = dayjs(program.stop).format('YYYYMMDDHHmmss ZZ')
 
     output += `
-<programme start="${start}" stop="${stop}" channel="${program.channel}">
-  <title lang="${program.lang}">${program.title}</title>`
+<programme start="${start}" stop="${stop}" channel="${program.channel}"><title lang="${program.lang}">${program.title}</title>`
 
     if (program.category) {
       output += `<category lang="${program.lang}">${program.category}</category>`
     }
 
-    output += `</programme>`
+    output += '</programme>'
   }
 
   output += '</tv>'
@@ -34,8 +32,8 @@ utils.convertToXMLTV = function ({ channels, programs }) {
   return output
 }
 
-utils.parseConfig = function (config) {
-  const xml = fs.readFileSync(path.resolve(process.cwd(), config), {
+utils.parseConfig = function (configPath) {
+  const xml = fs.readFileSync(path.resolve(process.cwd(), configPath), {
     encoding: 'utf-8'
   })
   const result = convert.xml2js(xml)
@@ -66,4 +64,20 @@ utils.getElementText = function (name, elements) {
   return el ? el.elements.find(el => el.type === 'text').text : null
 }
 
+utils.loadSites = function (sitesPath) {
+  const sites = {}
+  glob.sync(`${sitesPath}/*.js`).forEach(function (file) {
+    const name = path.parse(file).name
+    sites[name] = require(path.resolve(file))
+  })
+
+  return sites
+}
+
+utils.sleep = function (ms) {
+  return function (x) {
+    return new Promise(resolve => setTimeout(() => resolve(x), ms))
+  }
+}
+
 module.exports = utils