From 7cce6d737e7e34f72aefd1f8b076842ee66873bc Mon Sep 17 00:00:00 2001 From: Toha Date: Sat, 11 Jan 2025 20:32:46 +0700 Subject: [PATCH] Update web.magentatv.de. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test: ```sh npm test --- web.magentatv.de > test > run-script-os web.magentatv.de > test:win32 > SET "TZ=Pacific/Nauru" && npx jest --runInBand web.magentatv.de PASS sites/web.magentatv.de/web.magentatv.de.test.js √ can generate valid url (2 ms) √ can generate valid request method (1 ms) √ can generate valid request headers (2 ms) √ can generate valid request data (3 ms) √ can parse response (3 ms) √ can handle empty guide Test Suites: 1 passed, 1 total Tests: 6 passed, 6 total Snapshots: 0 total Time: 0.599 s, estimated 1 s Ran all test suites matching /web.magentatv.de/i. ``` Grab: ```sh npm run grab --- --site=web.magentatv.de > grab > npx tsx scripts/commands/epg/grab.ts --site=web.magentatv.de starting... config: output: guide.xml maxConnections: 1 gzip: false site: web.magentatv.de loading channels... found 348 channel(s) run #1: [1/696] web.magentatv.de (de) - 132 - Jan 11, 2025 (9 programs) [2/696] web.magentatv.de (de) - 132 - Jan 12, 2025 (11 programs) ... [695/696] web.magentatv.de (de) - SkyOne.de - Jan 12, 2025 (38 programs) [696/696] web.magentatv.de (de) - SkySport9.de - Jan 11, 2025 (2 programs) saving to "guide.xml"... done in 00h 04m 59s ``` Signed-off-by: Toha --- .../web.magentatv.de.config.js | 41 ++++++------------- .../web.magentatv.de/web.magentatv.de.test.js | 26 +++++++++++- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/sites/web.magentatv.de/web.magentatv.de.config.js b/sites/web.magentatv.de/web.magentatv.de.config.js index aadff1cf..371df0c1 100644 --- a/sites/web.magentatv.de/web.magentatv.de.config.js +++ b/sites/web.magentatv.de/web.magentatv.de.config.js @@ -5,7 +5,7 @@ const customParseFormat = require('dayjs/plugin/customParseFormat') const { upperCase } = require('lodash') let X_CSRFTOKEN -let COOKIE +let Cookie const cookiesToExtract = ['JSESSIONID', 'CSESSIONID', 'CSRFSESSION'] dayjs.extend(utc) @@ -17,10 +17,9 @@ module.exports = { url: 'https://api.prod.sngtv.magentatv.de/EPG/JSON/PlayBillList', request: { method: 'POST', - headers: function () { - return setHeaders() + async headers() { + return await setHeaders() }, - data({ channel, date }) { return { count: -1, @@ -40,8 +39,8 @@ module.exports = { } } }, - parser: function ({ content }) { - let programs = [] + parser({ content }) { + const programs = [] const items = parseItems(content) items.forEach(item => { programs.push({ @@ -166,23 +165,13 @@ function parseItems(content) { async function fetchCookieAndToken() { // Only fetch the cookies and csrfToken if they are not already set - if (X_CSRFTOKEN && COOKIE) { + if (X_CSRFTOKEN && Cookie) { return } try { const response = await axios.request({ url: 'https://api.prod.sngtv.magentatv.de/EPG/JSON/Authenticate', - headers: { - accept: 'application/json, text/javascript, */*; q=0.01', - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'x-requested-with': 'XMLHttpRequest', - Referer: 'https://web.magentatv.de/', - 'Referrer-Policy': 'strict-origin-when-cross-origin' - }, params: { SID: 'firstup', T: 'Windows_chrome_118' @@ -191,10 +180,9 @@ async function fetchCookieAndToken() { data: '{"terminalid":"00:00:00:00:00:00","mac":"00:00:00:00:00:00","terminaltype":"WEBTV","utcEnable":1,"timezone":"Etc/GMT0","userType":3,"terminalvendor":"Unknown"}', }) - // Extract the cookies specified in cookiesToExtract const setCookieHeader = response.headers['set-cookie'] || [] - let extractedCookies = [] + const extractedCookies = [] cookiesToExtract.forEach(cookieName => { const regex = new RegExp(`${cookieName}=(.+?)(;|$)`) const match = setCookieHeader.find(header => regex.test(header)) @@ -205,7 +193,6 @@ async function fetchCookieAndToken() { } }) - // check if we recieved a csrfToken only then store the values if (!response.data.csrfToken) { console.log('csrfToken not found in the response.') @@ -213,19 +200,15 @@ async function fetchCookieAndToken() { } X_CSRFTOKEN = response.data.csrfToken - COOKIE = extractedCookies.join(' ') + Cookie = extractedCookies.join(' ') } catch(error) { console.error(error) } } -function setHeaders() { - return fetchCookieAndToken().then(() => { - return { - X_CSRFTOKEN: X_CSRFTOKEN, - 'Content-Type': 'application/json', - Cookie: COOKIE - } - }) +async function setHeaders() { + await fetchCookieAndToken() + + return { X_CSRFTOKEN, Cookie } } diff --git a/sites/web.magentatv.de/web.magentatv.de.test.js b/sites/web.magentatv.de/web.magentatv.de.test.js index 95332580..71f56672 100644 --- a/sites/web.magentatv.de/web.magentatv.de.test.js +++ b/sites/web.magentatv.de/web.magentatv.de.test.js @@ -1,18 +1,43 @@ const { parser, url, request } = require('./web.magentatv.de.config.js') const fs = require('fs') const path = require('path') +const axios = require('axios') const dayjs = require('dayjs') const utc = require('dayjs/plugin/utc') const customParseFormat = require('dayjs/plugin/customParseFormat') + dayjs.extend(customParseFormat) dayjs.extend(utc) +jest.mock('axios') + const date = dayjs.utc('2022-03-09', 'YYYY-MM-DD').startOf('d') const channel = { site_id: '255', xmltv_id: '13thStreet.de' } +axios.request.mockImplementation(req => { + const result = {} + if (req.url === 'https://api.prod.sngtv.magentatv.de/EPG/JSON/Authenticate') { + Object.assign(result, { + headers: { + 'set-cookie': [ + 'JSESSIONID=2147EBA9C59BCDC33822CFD2764E5C0B; Path=/EPG; HttpOnly; SameSite=None; Secure', + 'JSESSIONID=2147EBA9C59BCDC33822CFD2764E5C0B; Path=/EPG/; HttpOnly; SameSite=None; Secure', + 'CSESSIONID=1CF187ABCA12ED1B01ADF84C691048ED; Path=/EPG/; Secure; HttpOnly; SameSite=None', + 'CSRFSESSION=ea2329ba213271192bffd77c2fa276086a8e828c1a4ee379; Path=/EPG/; SameSite=None; Secure' + ] + }, + data: { + csrfToken: '6f678415702493d2c28813747c413aa05c87d8f87ecf05fe' + } + }) + } + + return Promise.resolve(result) +}) + it('can generate valid url', () => { expect(url).toBe('https://api.prod.sngtv.magentatv.de/EPG/JSON/PlayBillList') }) @@ -30,7 +55,6 @@ it('can generate valid request headers', async () => { expect(headers.Cookie).toMatch(/JSESSIONID=[\dA-F]+;/i) expect(headers.Cookie).toMatch(/CSESSIONID=[\dA-F]+;/i) expect(headers.Cookie).toMatch(/CSRFSESSION=[\dA-F]+;/i) - expect(headers.Cookie).toMatch(/JSESSIONID=[\dA-F]+;/i) expect(headers.X_CSRFTOKEN).toMatch(/[\dA-F]/i) })