From b5f8f41547c88aba038bbdfd425a0472a1162784 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 31 Oct 2021 04:44:25 +0300 Subject: [PATCH 1/5] Upgrade epg-grabber package --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 29041a46..c833e9fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "cheerio": "^1.0.0-rc.10", "commander": "^8.2.0", "dayjs": "^1.10.4", - "epg-grabber": "^0.11.0", + "epg-grabber": "^0.12.0", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "glob": "^7.2.0", @@ -1852,9 +1852,9 @@ "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==" }, "node_modules/epg-grabber": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.11.0.tgz", - "integrity": "sha512-ybVzyTwv04C2BSkA6CBC1SxFAEkpg5Yw7w9AVbhI33iZmywclhIZoBHKslXJrVH15Zlm4bTDtEcA/OMhFaL9lA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.12.0.tgz", + "integrity": "sha512-wMF/GVau74aYhjtYy+YkVpPrH+2niEBGY3oHxp79GISZGmCCn42I/gN5nec/r52m30IwwHod39tOg3tfQisMfQ==", "dependencies": { "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", @@ -5851,9 +5851,9 @@ "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==" }, "epg-grabber": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.11.0.tgz", - "integrity": "sha512-ybVzyTwv04C2BSkA6CBC1SxFAEkpg5Yw7w9AVbhI33iZmywclhIZoBHKslXJrVH15Zlm4bTDtEcA/OMhFaL9lA==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.12.0.tgz", + "integrity": "sha512-wMF/GVau74aYhjtYy+YkVpPrH+2niEBGY3oHxp79GISZGmCCn42I/gN5nec/r52m30IwwHod39tOg3tfQisMfQ==", "requires": { "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", diff --git a/package.json b/package.json index 034c6537..eb683050 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "cheerio": "^1.0.0-rc.10", "commander": "^8.2.0", "dayjs": "^1.10.4", - "epg-grabber": "^0.11.0", + "epg-grabber": "^0.12.0", "epg-parser": "^0.1.6", "form-data": "^4.0.0", "glob": "^7.2.0", From a5cafa487869d448fa3bfb318199bef05d049de8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 31 Oct 2021 04:44:30 +0300 Subject: [PATCH 2/5] Create .gitignore --- sites/directv.com/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 sites/directv.com/.gitignore diff --git a/sites/directv.com/.gitignore b/sites/directv.com/.gitignore new file mode 100644 index 00000000..7dcdeb69 --- /dev/null +++ b/sites/directv.com/.gitignore @@ -0,0 +1,2 @@ +channels.json +*.crawler.js \ No newline at end of file From 40f1890efb35ca8820e54284c03bfd0977a607f3 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 31 Oct 2021 05:54:45 +0300 Subject: [PATCH 3/5] Create directv.com.test.js --- sites/directv.com/directv.com.test.js | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 sites/directv.com/directv.com.test.js diff --git a/sites/directv.com/directv.com.test.js b/sites/directv.com/directv.com.test.js new file mode 100644 index 00000000..a3c24e6f --- /dev/null +++ b/sites/directv.com/directv.com.test.js @@ -0,0 +1,50 @@ +// npx epg-grabber --config=sites/directv.com/directv.com.config.js --channels=sites/directv.com/directv.com_us.channels.xml --days=1 --output=.gh-pages/guides/us/directv.com.epg.xml + +const { parser, url, logo } = require('./directv.com.config.js') +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') +const customParseFormat = require('dayjs/plugin/customParseFormat') +dayjs.extend(customParseFormat) +dayjs.extend(utc) + +const date = dayjs.utc('2021-10-24', 'YYYY-MM-DD').startOf('d') +const channel = { + site_id: '15', + xmltv_id: 'WTAP.us', + logo: 'https://www.directv.com/images/logos/channels/dark/large/875.png' +} +const content = `{"schedule":[{"secLiveStreaming":"N","chNum":15,"authCode":"NA","chRec":true,"chCall":"WTAP","chId":2073,"secondaryChannelId":0,"chHd":true,"secondary":false,"blackOut":false,"chAdult":false,"chCat":["HDTV Channels","Local Channels"],"chLogoId":875,"detailsLinkUrl":"/Channels/Parkersburg-WV-WTAP-NBC-15-A3-HD-15","schedules":[{"primaryImageUrl":"/db_photos/default/TV/tv.jpg","restartAllowed":false,"subcategoryList":["Series","Reality"],"gridViewPrimaryImageUrl":"/db_photos/default/TV/tv_p.jpg","rating":"TVPG","description":null,"title":"Home Sweet Home","episodeNumber":3,"duration":60,"price":0,"repeat":false,"lookBack":false,"tvAdvisory":["L"],"dimension":"2D","ltd":"","programID":"EP039886740003","blackoutCode":"NA","airTime":"2021-10-30T00:00:00.000+0000","secLiveStreaming":"N","prOrd":0,"episodeTitle":"Art Is My God","authCode":"NA","format":"HD","seasonNumber":1,"listViewPrimaryImageUrl":"/db_photos/default/TV/tv_l.jpg","eventCode":"","mainCategory":"TV","hd":1,"liveStreaming":"N"}],"chKey":"2073_1476352800000","chName":"Parkersburg, WV WTAP NBC 15 A3 HD","chDesc":"NBC television services from WTAPDT-TV, 15, Parkersburg, WV.","liveStreaming":"N","digitalAdInsertableLive":false}],"reporting":{"channelschedules":{"success":false,"reportingData":"reporting for app/json/channelschedules/channelschedules not implemented yet"}},"messagekeys":null,"contingencies":[]}` + +it('can generate valid url', () => { + const result = url({ date, channel }) + expect(result).toBe( + 'https://www.directv.com/json/channelschedule?channels=15&startTime=2021-10-24T00:00:00Z&hours=24' + ) +}) + +it('can get logo url', () => { + const result = logo({ channel }) + expect(result).toBe('https://www.directv.com/images/logos/channels/dark/large/875.png') +}) + +it('can parse response', () => { + const result = parser({ date, channel, content }) + expect(result).toMatchObject([ + { + start: dayjs.utc('Sat, 30 Oct 2021 00:00:00 GMT'), + stop: dayjs.utc('Sat, 30 Oct 2021 01:00:00 GMT'), + title: 'Home Sweet Home', + category: ['Series', 'Reality'], + description: null + } + ]) +}) + +it('can handle empty guide', () => { + const result = parser({ + date, + channel, + content: `{"errors":[{"text":"Service failure: see errors or BulkOperationErrors for details","field":"","reason":"INTERNAL_SERVER_ERROR"}],"statusCode":500,"apiResponse":{"messages":"NOTE: see res.contingencies for size-filtered message values"},"reporting":{"channelschedules":{"success":false,"reportingData":"reporting for app/json/channelschedules/channelschedules not implemented yet"}},"messagekeys":null,"contingencies":[{"key":"ent_ep_guide_backend_unavailable_error_message","value":"Due to technical issues the guide is currently unavailable, please check back to soon.","level":"ERROR"}]}` + }) + expect(result).toMatchObject([]) +}) From 566b5b561dd2c2b3d44f8d6b9003694d3c6de610 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 31 Oct 2021 05:54:49 +0300 Subject: [PATCH 4/5] Create directv.com.config.js --- sites/directv.com/directv.com.config.js | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 sites/directv.com/directv.com.config.js diff --git a/sites/directv.com/directv.com.config.js b/sites/directv.com/directv.com.config.js new file mode 100644 index 00000000..168e2f71 --- /dev/null +++ b/sites/directv.com/directv.com.config.js @@ -0,0 +1,47 @@ +const dayjs = require('dayjs') +const utc = require('dayjs/plugin/utc') + +dayjs.extend(utc) + +module.exports = { + request: { + timeout: 15000 + }, + site: 'directv.com', + url({ channel, date }) { + return `https://www.directv.com/json/channelschedule?channels=${ + channel.site_id + }&startTime=${date.format()}&hours=24` + }, + logo({ channel }) { + return channel.logo + }, + parser({ content }) { + const programs = [] + const items = parseItems(content) + items.forEach(item => { + if (item.programID === '-1') return + const start = parseStart(item) + const stop = start.add(item.duration, 'm') + programs.push({ + title: item.title, + description: item.description, + category: item.subcategoryList, + start, + stop + }) + }) + + return programs + } +} + +function parseStart(item) { + return dayjs.utc(item.airTime) +} + +function parseItems(content) { + const data = JSON.parse(content) + + return data && data.schedule && data.schedule[0] ? data.schedule[0].schedules : [] +} From 7bb8deae8cae61bd16545bb4c08ae8246c77420a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 31 Oct 2021 05:55:20 +0300 Subject: [PATCH 5/5] Create directv.com_us.channels.xml --- sites/directv.com/directv.com_us.channels.xml | 382 ++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 sites/directv.com/directv.com_us.channels.xml diff --git a/sites/directv.com/directv.com_us.channels.xml b/sites/directv.com/directv.com_us.channels.xml new file mode 100644 index 00000000..f3966f81 --- /dev/null +++ b/sites/directv.com/directv.com_us.channels.xml @@ -0,0 +1,382 @@ + + + + 5 StarMax + 88 Films + 88 TV + AccuWeather Network + ActionMax East + Adult Monthly Offers + A&E East + Altitude Sports + AMC East + American Heroes Channel USA + America's Value Channel + Animal Planet East + Antena 3 Internacional + Aqui + Arirang World + Aspire TV + Atreseries + AT&T SportsNet Rocky Mountain + AT&T SportsNet Rocky Mountain West + AWE + AXS TV + Azteca US Este + BabyFirst TV + Baby TV Latin America + Bandamax Estados Unidos + Bang U + BBC America East + BBC World News Americas + Best Channel + BET East + BET Her East + Bloomberg TV US + Boomerang US + Bravo East + Brazzers TV Monthly Offer + Brazzers TV XX + Buzz TV + BYU TV + Canal 6 Nacional + Canal 22 Internacional + Caracol TV Internacional + Cartoon Network East + Cartoon Network West + Cash + CBS Sports Network USA + CCTV 4 America + Celebrity Shopping TV + Centroamérica TV + CGTN America + Channel Neco International + Charming China + Cine Estelar + CineLatino + Cinema One Global + Cinemax East + Cinemax West + Cine Mexicano + Cine Nostalgia + Cine Sony + CMT East + CNBC US + CNBC World + CNN en Español + CNN USA + Comedy Central East + Comedy TV + Cooking Channel + Crime + Investigation Network USA + C-SPAN + C-SPAN 2 + CTI TV International + CTN + CTS + Daystar TV + De Película Clásico + De Película Estados Unidos + Destination America + Discovery Channel East + Discovery en Español + Discovery Familia + Discovery Family + Discovery Life Channel + Disney Channel East + Disney Channel West + Disney Junior East + Disney XD East + DIY Network USA + Dom Kino + EBS America + Ecuador TV + Ecuavisa Internacional + E! East + Eleven Sports USA + Enlace + ESPN 2 US + ESPN Deportes + ESPNews + ESPNU + ESPN US + Estrella TV East + EWTN US + Family Entertainment TV + Film 24h + Flix East + FM + Food Network East + Foro TV Estados Unidos + Fox Business + Fox Deportes + Fox Life USA + Fox News Channel + Freeform East + Free Speech TV + FS1 + FS2 + Fuse East + FX East + FX Movie Channel + FXX East + FYI East + Galavisión Este + Game Show Network East + GEB + Gem Shopping Network + Get It Infomercial + GMA Life TV + GMA Pinoy TV USA & Canada + Go4It + God TV US + Golf Channel US + Gol TV + Great American Country + Guangzhou TV + Hallmark Channel East + Hallmark Movies & Mysteries East + HBO 2 East + HBO 2 West + HBO Comedy East + HBO East + HBO Family East + HBO Family West + HBO Latino Este + HBO Signature East + HBO West + HBO Zone East + HDNet Movies + HGTV East + Hillsong Channel + History East + History en Español + HITN + HLN + Hola! TV Estados Unidos + Honvietv + Hope Channel North America + HSN + Hustler HD USA + Hustler TV Monthly Offer + I-Cable Finance Info Channel + I-Cable News Channel + Idea Channel + IFC East + Impact Network + Informercial 1 + Informercial 77 + Informercial 82 + Informercial 87 + Informercial 91 + Informercial Secret + INSP + Investigation Discovery East + ION TV East + ION TV West + IVC + JBS + Jewelry TV + Justice Central TV + Kapatid TV 5 + Karusel International + KBS Korea + KBS World + Lifetime East + Lifetime Movies East + Link TV + Logo East + Longhorn Network + Look Network + Mall TV + MASN + MavTV + MaxLatino + MBC + MBN Plus + Mediaset Italia + Mega TV + Mercury Media Channel + MGM HD USA + MLB Network + MoreMax East + Motortrend + MovieMax + MSG + MSNBC + MTV 2 East + MTV Classic East + MTV East + MTV Live USA + MTV Tr3s East + Muzika Pervogo + Myx USA + NASA TV Public + Nat Geo Mundo + National Geographic East + National Geographic Korea + National Geographic Wild + NBA TV + NBC Sports Bay Area + NBC Sports California + NBC Sports Washington + NBC Universo Este + NESN + Newsmax TV + NewsNation East + NHL Network + Nickelodeon East + Nickelodeon West + Nick Jr East + Nicktoons East + Nippon TV + NRB TV + NTN 24 USA + NTV America + Olympic Channel USA + Once México + One America News Network + OPM TV + Oprah Winfrey Network East + Outdoor Channel + Ovation + Oxygen East + Paramount Network East + Pasiones Estados Unidos + PBS East + Penthouse TV + Penthouse TV Monthly Offer + Perú Mágico + Perviy kanal America + Phoenix Hong Kong + Phoenix InfoNews Channel + Phoenix North America Chinese Channel + Pinoy Box Office Global + Playboy TV Monthly Offer + Playboy TV USA + Playboy TV USA HD + Pop East + Prime TV + QVC 2 + QVC 3 + QVC US + Rai Italia Nord America + Rai News 24 + Rai World Premium + RCN Nuestra Tele Internacional + Reality Kings TV + Reelz + Revolt + RFD-TV + Rossiya 24 + RT America + RTR Planeta USA + RTVi USA + Sale Network + Sari Sari Channel + SBN Domestic + SBS + SBS Plus + SBTN + S Channel + Science + Scientology Network + ShopHQ + Shop LC + Shorts TV US + Showtime 2 East + Showtime East + Showtime Extreme East + Showtime Next East + Showtime Showcase East + Showtime West + Showtime Women East + Sho x Bet East + Smithsonian Channel East + Sony Movies USA + Spectrum SportsNet + Sportsman Channel + SportsNet New York + SporTV + Starz Cinema East + Starz Comedy East + Starz East + Starz Edge East + Starz Encore Action East + Starz Encore Black East + Starz Encore Classic East + Starz Encore East + Starz Encore Español Este + Starz Encore Family East + Starz Encore Suspense East + Starz Encore West + Starz Encore Westerns East + Starz In Black East + Starz Kids & Family East + Starz West + Sur Perú + Syfy East + TAN TV + TBN US + TBS East + TCM US + TCT + TeenNick East + TeleCentro + Telefe Internacional + Telemundo Este + Telemundo Oeste + Tennis Channel + TFC USA West + The Movie Channel East + The Movie Channel West + The Movie Channel Xtra East + The Weather Channel + The Word Network + ThrillerMax East + TLC East + TNT East + Travel Channel East + Tru TV East + TUDN Estados Unidos + TV Chile + TVE Internacional América + TVG + TV Globo Internacional Américas + TViet Network + TV Japan + TV Land East + TV One + TyC Sports Internacional + UniMás Oeste + Universal Kids East + Universal Living Faith Network + Univisión Este + Univisión Oeste + Univision Tlnovelas + Up TV + USA Network East + Valu + Venevisión + VGN TV + VH1 East + Vice East + Video Rola + ViendoMovies + Vien Thao TV + Vietface TV + Vietv Network + Vivid TV + Vivid TV Monthly Offer + V me + Vremya + WAPA America + We TV East + World Harvest TV + Wow TV + Xtra + Yes Network + YTN + ZooMoo Latinoamérica + + \ No newline at end of file