From 5937d8dc46bea42ef70c0361d8e4c760f40c1878 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Oct 2022 18:38:39 +0300 Subject: [PATCH 1/3] Create __data__/ --- sites/tvguide.com/__data__/content.json | 159 +++++++++++++++++++++ sites/tvguide.com/__data__/no-content.json | 20 +++ sites/tvguide.com/__data__/program.json | 91 ++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 sites/tvguide.com/__data__/content.json create mode 100644 sites/tvguide.com/__data__/no-content.json create mode 100644 sites/tvguide.com/__data__/program.json diff --git a/sites/tvguide.com/__data__/content.json b/sites/tvguide.com/__data__/content.json new file mode 100644 index 00000000..d0ad4a3e --- /dev/null +++ b/sites/tvguide.com/__data__/content.json @@ -0,0 +1,159 @@ +{ + "data": { + "duration": "1440", + "providerId": "9100001138", + "startTime": "1667088000", + "items": [ + { + "channel": { + "fullName": "CBS", + "name": "CBS", + "number": null, + "sourceId": 9200018514, + "legacySourceId": 5943, + "networkName": "CBS", + "networkId": 1, + "logo": "/provider/1/4/1-685389140.png" + }, + "programSchedules": [ + { + "airingAttrib": 524817, + "catId": 2, + "startTime": 1667088000, + "endTime": 1667097000, + "programId": 6060455538, + "title": "NWSL Soccer Playoff", + "rating": null, + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060455538/web" + }, + { + "airingAttrib": 558098, + "catId": 5, + "startTime": 1667097000, + "endTime": 1667098800, + "programId": 6060248599, + "title": "Young Sheldon", + "rating": "TV-PG", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060248599/web" + }, + { + "airingAttrib": 524288, + "catId": 5, + "startTime": 1667098800, + "endTime": 1667102400, + "programId": 1030182907, + "title": "Local Programming", + "rating": "TV-G", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/1030182907/web" + }, + { + "airingAttrib": 524288, + "catId": 5, + "startTime": 1667102400, + "endTime": 1667120400, + "programId": 1030182907, + "title": "Local Programming", + "rating": "TV-G", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/1030182907/web" + }, + { + "airingAttrib": 524288, + "catId": 5, + "startTime": 1667120400, + "endTime": 1667134800, + "programId": 1030182907, + "title": "Local Programming", + "rating": "TV-G", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/1030182907/web" + }, + { + "airingAttrib": 557588, + "catId": 4, + "startTime": 1667134800, + "endTime": 1667140200, + "programId": 6060613824, + "title": "CBS Sunday Morning", + "rating": "TV-PG", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060613824/web" + }, + { + "airingAttrib": 557588, + "catId": 4, + "startTime": 1667140200, + "endTime": 1667143800, + "programId": 6060613861, + "title": "Face the Nation", + "rating": "TV-PG", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060613861/web" + }, + { + "airingAttrib": 524288, + "catId": 5, + "startTime": 1667143800, + "endTime": 1667145600, + "programId": 1030182907, + "title": "Local Programming", + "rating": "TV-G", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/1030182907/web" + }, + { + "airingAttrib": 558097, + "catId": 2, + "startTime": 1667145600, + "endTime": 1667149200, + "programId": 6003572942, + "title": "The NFL Today", + "rating": null, + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6003572942/web" + }, + { + "airingAttrib": 524817, + "catId": 2, + "startTime": 1667149200, + "endTime": 1667160000, + "programId": 6000129053, + "title": "NFL Football", + "rating": null, + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6000129053/web" + }, + { + "airingAttrib": 524817, + "catId": 2, + "startTime": 1667160000, + "endTime": 1667170800, + "programId": 6000129053, + "title": "NFL Football", + "rating": null, + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6000129053/web" + }, + { + "airingAttrib": 558100, + "catId": 4, + "startTime": 1667170800, + "endTime": 1667174400, + "programId": 6003572892, + "title": "60 Minutes", + "rating": "TV-PG", + "programDetails": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6003572892/web" + } + ] + } + ] + }, + "links": { + "self": { + "href": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1667088000&duration=1440&channelSourceIds=9200018514" + }, + "prev": { + "href": null + }, + "next": { + "href": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1667088000&duration=1440&channelSourceIds=9200018514&offset=1667089440&limit=1667088000" + } + }, + "meta": { + "componentName": null, + "componentDisplayName": null, + "componentType": null + } +} \ No newline at end of file diff --git a/sites/tvguide.com/__data__/no-content.json b/sites/tvguide.com/__data__/no-content.json new file mode 100644 index 00000000..d67154ca --- /dev/null +++ b/sites/tvguide.com/__data__/no-content.json @@ -0,0 +1,20 @@ +{ + "errors": [ + { + "code": 400, + "reason": "Not Found", + "message": "Bad Request" + } + ], + "links": { + "self": { + "href": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/100001138/web?start=1667088000&duration=1440&channelSourceIds=9200018514" + } + }, + "meta": { + "componentName": null, + "componentDisplayName": null, + "componentType": null + } +} + \ No newline at end of file diff --git a/sites/tvguide.com/__data__/program.json b/sites/tvguide.com/__data__/program.json new file mode 100644 index 00000000..e0697a72 --- /dev/null +++ b/sites/tvguide.com/__data__/program.json @@ -0,0 +1,91 @@ +{ + "data": { + "item": { + "id": 6060613824, + "name": "CBS Sunday Morning", + "parentId": 1030037352, + "description": "The Backseat Lovers perform on the \"Saturday Sessions\"; and Daisy Ryan guests on \"The Dish.\" Also: comedian Fortune Feimster.", + "isSportsEvent": false, + "rating": null, + "tvRating": "TV-PG", + "episodeTitle": "10-30-2022", + "releaseYear": 2022, + "seoUrl": null, + "categoryId": 4, + "subCategoryId": 0, + "episodeAirDate": "/Date(1667088000000)/", + "episodeNumber": 248, + "seasonNumber": 40, + "mcoId": 1030037352, + "title": "CBS Sunday Morning", + "type": "show", + "slug": "cbs-sunday-morning", + "typeId": 1, + "images": [ + { + "id": "1-10744155154", + "provider": "Tivo", + "imageType": { + "typeId": 1, + "typeName": "showcard", + "providerTypeName": "Vertical Showcard (3:4)" + }, + "bucketType": "catalog", + "bucketPath": "/provider/1/1/1-10744155154.jpg", + "filename": "", + "width": 2160, + "height": 2880 + }, + { + "id": "1-10744155148", + "provider": "Tivo", + "imageType": { + "typeId": 5, + "typeName": "key art", + "providerTypeName": "Key Art" + }, + "bucketType": "catalog", + "bucketPath": "/provider/1/5/1-10744155148.jpg", + "filename": "", + "width": 3000, + "height": 1688 + } + ], + "genres": [ + { + "id": 19, + "name": "Talk & Interview", + "genres": [ + "interview", + "talk" + ] + }, + { + "id": 99, + "name": "Other", + "genres": [ + "newsmagazine", + "profile" + ] + } + ], + "duration": null, + "metacriticSummary": { + "url": null, + "score": 0, + "reviewCount": 0 + }, + "video": null + } + }, + "links": { + "self": { + "href": "https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060613824/web" + } + }, + "meta": { + "componentName": null, + "componentDisplayName": null, + "componentType": null + } +} \ No newline at end of file From 344b4e0a66513db9fd86739d8234c4054e380ec0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Oct 2022 18:38:56 +0300 Subject: [PATCH 2/3] Update tvguide.com.test.js --- sites/tvguide.com/tvguide.com.test.js | 61 +++++++++++++++++++-------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/sites/tvguide.com/tvguide.com.test.js b/sites/tvguide.com/tvguide.com.test.js index cfd2ee87..8720125c 100644 --- a/sites/tvguide.com/tvguide.com.test.js +++ b/sites/tvguide.com/tvguide.com.test.js @@ -1,47 +1,74 @@ // npx epg-grabber --config=sites/tvguide.com/tvguide.com.config.js --channels=sites/tvguide.com/tvguide.com_us.channels.xml --output=guide.xml --days=2 const { parser, url } = require('./tvguide.com.config.js') +const fs = require('fs') +const path = require('path') +const axios = require('axios') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const customParseFormat = require('dayjs/plugin/customParseFormat') dayjs.extend(customParseFormat) dayjs.extend(utc) -const date = dayjs.utc('2022-10-04', 'YYYY-MM-DD').startOf('d') +jest.mock('axios') + +const date = dayjs.utc('2022-10-30', 'YYYY-MM-DD').startOf('d') const channel = { - site_id: '9100001138#9233011874', - xmltv_id: 'ABCEast.us' + site_id: '9100001138#9200018514', + xmltv_id: 'CBSEast.us' } it('can generate valid url', () => { expect(url({ date, channel })).toBe( - 'https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1664841600&duration=1439&channelSourceIds=9233011874' + 'https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1667088000&duration=1440&channelSourceIds=9200018514' ) }) -it('can parse response', () => { - const content = `{"data":{"duration":"1439","providerId":"9100001138","startTime":"1664841600","items":[{"channel":{"fullName":"ABC","name":"ABC","number":null,"sourceId":9233011874,"legacySourceId":5942,"networkName":"ABC","networkId":2,"logo":"/provider/1/4/1-4124037679.png"},"programSchedules":[{"airingAttrib":533524,"catId":5,"startTime":1664841600,"endTime":1664848800,"programId":6033556709,"title":"Bachelor in Paradise","rating":"TV-14","programDetails":"https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6033556709/web"}]}]}}` +it('can parse response', async () => { + const content = fs.readFileSync(path.resolve(__dirname, '__data__/content.json')) - const result = parser({ date, channel, content }).map(p => { + axios.get.mockImplementation(url => { + if ( + url === + 'https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/programdetails/6060613824/web' + ) { + return Promise.resolve({ + data: JSON.parse(fs.readFileSync(path.resolve(__dirname, '__data__/program.json'))) + }) + } else { + return Promise.resolve({ data: '' }) + } + }) + + let results = await parser({ content, channel, date }) + results = results.map(p => { p.start = p.start.toJSON() p.stop = p.stop.toJSON() return p }) - expect(result).toMatchObject([ - { - start: '2022-10-04T00:00:00.000Z', - stop: '2022-10-04T02:00:00.000Z', - title: 'Bachelor in Paradise' + expect(results[5]).toMatchObject({ + start: '2022-10-30T13:00:00.000Z', + stop: '2022-10-30T14:30:00.000Z', + title: 'CBS Sunday Morning', + sub_title: '10-30-2022', + description: + 'The Backseat Lovers perform on the "Saturday Sessions"; and Daisy Ryan guests on "The Dish." Also: comedian Fortune Feimster.', + categories: ['Talk & Interview', 'Other'], + season: 40, + episode: 248, + rating: { + system: 'MPA', + value: 'TV-PG' } - ]) + }) }) -it('can handle empty guide', () => { - const result = parser({ +it('can handle empty guide', async () => { + const results = await parser({ date, channel, - content: `{"data":{"duration":"1439","providerId":"9100001138","startTime":"1664841600","items":[]},"links":{"self":{"href":"https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1664841600&duration=1439&channelSourceIds=923301187"},"prev":{"href":null},"next":{"href":"https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/9100001138/web?start=1664841600&duration=1439&channelSourceIds=923301187&offset=1664843039&limit=1664841600"}},"meta":{"componentName":null,"componentDisplayName":null,"componentType":null}}` + content: fs.readFileSync(path.resolve(__dirname, '__data__/no-content.json')) }) - expect(result).toMatchObject([]) + expect(results).toMatchObject([]) }) From 34597ae3eced60eaf3fe5af429c4c10837529187 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Oct 2022 18:39:05 +0300 Subject: [PATCH 3/3] Update tvguide.com.config.js --- sites/tvguide.com/tvguide.com.config.js | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/sites/tvguide.com/tvguide.com.config.js b/sites/tvguide.com/tvguide.com.config.js index 14f72092..271bac6e 100644 --- a/sites/tvguide.com/tvguide.com.config.js +++ b/sites/tvguide.com/tvguide.com.config.js @@ -1,3 +1,4 @@ +const axios = require('axios') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const timezone = require('dayjs/plugin/timezone') @@ -8,37 +9,62 @@ dayjs.extend(timezone) module.exports = { site: 'tvguide.com', url: function ({ date, channel }) { - const parts = channel.site_id.split('#') - const start = date.startOf('d') - const duration = date.endOf('d').diff(start, 'm') - const url = `https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/${ - parts[0] - }/web?start=${start.unix()}&duration=${duration}&channelSourceIds=${parts[1]}` + const [providerId, channelSourceIds] = channel.site_id.split('#') + const url = `https://cmg-prod.apigee.net/v1/xapi/tvschedules/tvguide/${providerId}/web?start=${date + .startOf('d') + .unix()}&duration=1440&channelSourceIds=${channelSourceIds}` return url }, - parser: function ({ content }) { + async parser({ content, channel }) { const programs = [] const items = parseItems(content) - items.forEach(item => { + for (let item of items) { + const details = await loadProgramDetails(item) programs.push({ title: item.title, + sub_title: details.episodeTitle, + description: details.description, + season: details.seasonNumber, + episode: details.episodeNumber, + rating: parseRating(item), + categories: parseCategories(details), start: parseTime(item.startTime), stop: parseTime(item.endTime) }) - }) + } return programs } } +function parseRating(item) { + return item.rating ? { system: 'MPA', value: item.rating } : null +} + +function parseCategories(details) { + return Array.isArray(details.genres) ? details.genres.map(g => g.name) : [] +} + function parseTime(timestamp) { return dayjs.unix(timestamp) } function parseItems(content) { const data = JSON.parse(content) - if (!Array.isArray(data.data.items) || !data.data.items.length) return [] + if (!data.data || !Array.isArray(data.data.items) || !data.data.items.length) return [] return data.data.items[0].programSchedules } + +async function loadProgramDetails(item) { + const data = await axios + .get(item.programDetails) + .then(r => r.data) + .catch(err => { + console.log(err.message) + }) + if (!data || !data.data || !data.data.item) return {} + + return data.data.item +}