diff --git a/package-lock.json b/package-lock.json index f0555882..bb7a3629 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "commander": "^8.2.0", "csv-parser": "^3.0.0", "dayjs": "^1.10.4", - "epg-grabber": "^0.20.0", + "epg-grabber": "^0.21.0", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "fs-extra": "^10.0.1", @@ -2198,9 +2198,9 @@ } }, "node_modules/epg-grabber": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.20.0.tgz", - "integrity": "sha512-6xmz1AfYKDduEjZqQlYyacR41fb8ITDymORyOcKmTdCs9XXrofYL84lULkEmhsNfksAuPf5hMef/uFptbDh/GA==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.21.0.tgz", + "integrity": "sha512-8H+xUp521vFKkL7m3vefWpC00UQULYHfyWm+TNGXsFPYXSQ4Mke0647Vzp5RmnH1tHcIzxGON/s580tdM56WTw==", "dependencies": { "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", @@ -7556,9 +7556,9 @@ "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" }, "epg-grabber": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.20.0.tgz", - "integrity": "sha512-6xmz1AfYKDduEjZqQlYyacR41fb8ITDymORyOcKmTdCs9XXrofYL84lULkEmhsNfksAuPf5hMef/uFptbDh/GA==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.21.0.tgz", + "integrity": "sha512-8H+xUp521vFKkL7m3vefWpC00UQULYHfyWm+TNGXsFPYXSQ4Mke0647Vzp5RmnH1tHcIzxGON/s580tdM56WTw==", "requires": { "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", diff --git a/package.json b/package.json index 157617f0..5bb6fa47 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "commander": "^8.2.0", "csv-parser": "^3.0.0", "dayjs": "^1.10.4", - "epg-grabber": "^0.20.0", + "epg-grabber": "^0.21.0", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "fs-extra": "^10.0.1", diff --git a/sites/mncvision.id/mncvision.id.config.js b/sites/mncvision.id/mncvision.id.config.js index 20331b04..28ec6b11 100644 --- a/sites/mncvision.id/mncvision.id.config.js +++ b/sites/mncvision.id/mncvision.id.config.js @@ -12,7 +12,7 @@ dayjs.extend(customParseFormat) module.exports = { site: 'mncvision.id', - url: `https://mncvision.id/schedule/table`, + url: 'https://mncvision.id/schedule/table', request: { method: 'POST', data: function ({ channel, date }) { @@ -30,21 +30,33 @@ module.exports = { 'Content-Type': 'multipart/form-data; boundary=X-EPG-BOUNDARY' } }, - async parser({ content, date }) { + async parser({ content, date, headers }) { const programs = [] - const items = parseItems(content) + + let items = parseItems(content) + const pages = parsePages(content) + const cookies = headers && headers['set-cookie'] ? headers['set-cookie'].join(';') : '' + for (let url of pages) { + const nextContent = await axios + .get(url, { + headers: { + Cookie: cookies + } + }) + .then(r => r.data) + .catch(console.log) + items = items.concat(parseItems(nextContent)) + } + for (const item of items) { - const title = parseTitle(item) const start = parseStart(item, date) const duration = parseDuration(item) const stop = start.add(duration, 'm') - const description = await loadDescription(item) - programs.push({ - title, - description, - start: start.toJSON(), - stop: stop.toJSON() + title: parseTitle(item), + description: await loadDescription(item), + start, + stop }) } @@ -117,3 +129,16 @@ function parseItems(content) { return $('tr[valign="top"]').toArray() } + +function parsePages(content) { + const $ = cheerio.load(content) + const links = $('#schedule > div.schedule_search_result_container > div.box.well > a').toArray() + + const pages = {} + for (let link of links) { + const url = $(link).attr('href') + pages[url] = true + } + + return Object.keys(pages) +} diff --git a/sites/mncvision.id/mncvision.id.test.js b/sites/mncvision.id/mncvision.id.test.js index 6055d9d1..e7a40388 100644 --- a/sites/mncvision.id/mncvision.id.test.js +++ b/sites/mncvision.id/mncvision.id.test.js @@ -1,5 +1,5 @@ // node ./scripts/channels.js --config=./sites/mncvision.id/mncvision.id.config.js --output=./sites/mncvision.id/mncvision.id_id.channels.xml -// npx epg-grabber --config=sites/mncvision.id/mncvision.id.config.js --channels=sites/mncvision.id/mncvision.id_id.channels.xml --output=.gh-pages/guides/id/mncvision.id.epg.xml --days=2 +// npx epg-grabber --config=sites/mncvision.id/mncvision.id.config.js --channels=sites/mncvision.id/mncvision.id_id.channels.xml --output=guide.xml --days=2 const { parser, url, request } = require('./mncvision.id.config.js') const axios = require('axios') @@ -16,7 +16,11 @@ const channel = { site_id: '203', xmltv_id: 'AnimalPlanetSoutheastAsia.us' } -const content = `
Jam Tayang Program Acara Durasi
00:00African Wild S1: Seals01:00
` +const setCookie = [ + 's1nd0vL=0qpsmm7dpjmi7nt8d2h5epf16rmgg8a8; expires=Sat, 05-Mar-2022 15:44:22 GMT; Max-Age=7200; path=/; HttpOnly' +] +const content0 = `

Jadwal Tayang Cari

Jadwal Tayang Channel: 41 Tanggal: 2022-03-05
Jam Tayang Program Acara Durasi
00:00Hey Duggee S3, Ep 2200:07
page: 12
` +const content50 = `

Jadwal Tayang Cari

Jadwal Tayang Channel: 41 Tanggal: 2022-03-05
Jam Tayang Program Acara Durasi
08:25Hey Duggee S1, Ep 4600:07
page: 12
` it('can generate valid url', () => { expect(url).toBe('https://mncvision.id/schedule/table') @@ -34,45 +38,45 @@ it('can generate valid request data', () => { }) it('can parse response', done => { - axios.get.mockImplementation(() => - Promise.resolve({ - data: `
+ axios.get.mockImplementation((url, options = {}) => { + if (url === 'https://www.mncvision.id/schedule/detail/2022030500000041/Hey-Duggee-S3-Ep-22/1') { + return Promise.resolve({ + data: `
Nikmati suasana kehidupan koloni anjing laut di kawasan pantai barat Afrika Selatan.
` - }) - ) + }) + } else if ( + url === 'https://www.mncvision.id/schedule/table/startno/50' && + options.headers && + options.headers['Cookie'] === setCookie.join(';') + ) { + return Promise.resolve({ + data: content50 + }) + } - parser({ date, channel, content }) + return Promise.resolve({ data: '' }) + }) + + parser({ date, content: content0, headers: { 'set-cookie': setCookie } }) .then(result => { + result = result.map(p => { + p.start = p.start.toJSON() + p.stop = p.stop.toJSON() + return p + }) + expect(result).toMatchObject([ { start: '2021-11-11T17:00:00.000Z', - stop: '2021-11-11T18:00:00.000Z', - title: 'African Wild S1: Seals', + stop: '2021-11-11T17:07:00.000Z', + title: 'Hey Duggee S3, Ep 22', description: 'Nikmati suasana kehidupan koloni anjing laut di kawasan pantai barat Afrika Selatan.' - } - ]) - done() - }) - .catch(error => { - done(error) - }) -}) - -it('can parse response with empty description', done => { - axios.get.mockImplementation(() => - Promise.resolve({ - data: `
-
` - }) - ) - - parser({ date, channel, content }) - .then(result => { - expect(result).toMatchObject([ + }, { - start: '2021-11-11T17:00:00.000Z', - stop: '2021-11-11T18:00:00.000Z', - title: 'African Wild S1: Seals', + start: '2021-11-12T01:25:00.000Z', + stop: '2021-11-12T01:32:00.000Z', + title: 'Hey Duggee S1, Ep 46', description: null } ])