From c52c2fbddbd95b59a951f5885b5dc685e443c086 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:22:42 +0300 Subject: [PATCH 01/15] Update update-readme.js --- scripts/commands/update-readme.js | 98 ++++++++++--------------------- 1 file changed, 30 insertions(+), 68 deletions(-) diff --git a/scripts/commands/update-readme.js b/scripts/commands/update-readme.js index 826f96c5..e942884e 100644 --- a/scripts/commands/update-readme.js +++ b/scripts/commands/update-readme.js @@ -1,37 +1,44 @@ -const { file, markdown, parser, logger, api } = require('../core') +const { file, markdown, parser, logger, api, table } = require('../core') const { program } = require('commander') const _ = require('lodash') -const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs' +const CHANNELS_PATH = process.env.CHANNELS_PATH || 'sites/**/*.channels.xml' const options = program .option('-c, --config ', 'Set path to config file', '.readme/config.json') .parse(process.argv) .opts() -const statuses = { - 0: '🟒', - 1: 'πŸ”΄' -} - async function main() { await api.countries.load() - await api.subdivisions.load() - const records = await getLogRecords() - await generateCountriesTable(records) - await generateUSStatesTable(records) - await generateCanadaProvincesTable(records) + const files = await file.list(CHANNELS_PATH) + const items = [] + for (const filepath of files) { + const { site, channels } = await parser.parseChannels(filepath) + const filename = file.basename(filepath) + const [__, suffix] = filename.match(/\_(.*)\.channels\.xml$/) || [null, null] + const [code] = suffix.split('-') + + items.push({ + code, + site, + count: channels.length, + group: `${suffix}/${site}` + }) + } + + await generateCountriesTable(items) await updateReadme() } main() async function generateCountriesTable(items = []) { - logger.info('Generating countries table...') + logger.info('generating countries table...') let rows = [] for (const item of items) { - const country = api.countries.find({ code: item.code }) + const country = api.countries.find({ code: item.code.toUpperCase() }) if (!country) continue rows.push({ @@ -39,70 +46,25 @@ async function generateCountriesTable(items = []) { name: country.name, channels: item.count, epg: `https://iptv-org.github.io/epg/guides/${item.group}.epg.xml`, - status: statuses[item.status] + status: `${item.site}` }) } rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc']) rows = _.groupBy(rows, 'name') - const table = markdown.createTable(rows, ['Country', 'Channels', 'EPG', 'Status']) + const output = table.create(rows, [ + 'Country', + 'Channels', + 'EPG', + 'Status                                                   ' + ]) - await file.create('./.readme/_countries.md', table) -} - -async function generateUSStatesTable(items = []) { - logger.info('Generating US states table...') - - let rows = [] - for (const item of items) { - if (!item.code.startsWith('US-')) continue - const state = api.subdivisions.find({ code: item.code }) - if (!state) continue - - rows.push({ - name: state.name, - channels: item.count, - epg: `https://iptv-org.github.io/epg/guides/${item.group}.epg.xml`, - status: statuses[item.status] - }) - } - - rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc']) - rows = _.groupBy(rows, 'name') - - const table = markdown.createTable(rows, ['State', 'Channels', 'EPG', 'Status']) - - await file.create('./.readme/_us-states.md', table) -} - -async function generateCanadaProvincesTable(items = []) { - logger.info('Generating Canada provinces table...') - - let rows = [] - for (const item of items) { - if (!item.code.startsWith('CA-')) continue - const province = api.subdivisions.find({ code: item.code }) - if (!province) continue - - rows.push({ - name: province.name, - channels: item.count, - epg: `https://iptv-org.github.io/epg/guides/${item.group}.epg.xml`, - status: statuses[item.status] - }) - } - - rows = _.orderBy(rows, ['name', 'channels'], ['asc', 'desc']) - rows = _.groupBy(rows, 'name') - - const table = markdown.createTable(rows, ['Province', 'Channels', 'EPG', 'Status']) - - await file.create('./.readme/_ca-provinces.md', table) + await file.create('./.readme/_countries.md', output) } async function updateReadme() { - logger.info('Updating README.md...') + logger.info('updating readme.md...') const config = require(file.resolve(options.config)) await file.createDir(file.dirname(config.build)) From 77046998bf12d244954c286f9cc5d861342ae401 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:22:59 +0300 Subject: [PATCH 02/15] Update index.js --- scripts/core/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/core/index.js b/scripts/core/index.js index 19b46793..ccbdd057 100644 --- a/scripts/core/index.js +++ b/scripts/core/index.js @@ -6,3 +6,4 @@ exports.timer = require('./timer') exports.markdown = require('./markdown') exports.api = require('./api') exports.date = require('./date') +exports.table = require('./table') From c51bfdeb50e99f50822229acaa5b246a1b2e6702 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:10 +0300 Subject: [PATCH 03/15] Update markdown.js --- scripts/core/markdown.js | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/scripts/core/markdown.js b/scripts/core/markdown.js index 3d735b18..2729f0b7 100644 --- a/scripts/core/markdown.js +++ b/scripts/core/markdown.js @@ -3,38 +3,6 @@ const file = require('./file') const markdown = {} -markdown.createTable = function (data, cols) { - let output = '\n' - - output += ' \n ' - for (let column of cols) { - output += `` - } - output += '\n \n' - - output += ' \n' - for (let groupId in data) { - const group = data[groupId] - for (let [i, item] of group.entries()) { - const rowspan = group.length > 1 ? ` rowspan="${group.length}"` : '' - output += ' ' - if (i === 0) { - const name = item.flag ? `${item.flag} ${item.name}` : item.name - output += `` - } - output += `` - output += `` - output += `` - output += '\n' - } - } - output += ' \n' - - output += '
${column}
${name}${item.channels}${item.epg}${item.status}
' - - return output -} - markdown.compile = function (filepath) { markdownInclude.compileFiles(file.resolve(filepath)) } From 24dac3eff700deeaf17a08e1aa196d2fe992691f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:17 +0300 Subject: [PATCH 04/15] Create table.js --- scripts/core/table.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 scripts/core/table.js diff --git a/scripts/core/table.js b/scripts/core/table.js new file mode 100644 index 00000000..f9315f1e --- /dev/null +++ b/scripts/core/table.js @@ -0,0 +1,35 @@ +const table = {} + +table.create = function (data, cols) { + let output = '\n' + + output += ' \n ' + for (let column of cols) { + output += `` + } + output += '\n \n' + + output += ' \n' + for (let groupId in data) { + const group = data[groupId] + for (let [i, item] of group.entries()) { + const rowspan = group.length > 1 ? ` rowspan="${group.length}"` : '' + output += ' ' + if (i === 0) { + const name = item.flag ? `${item.flag} ${item.name}` : item.name + output += `` + } + output += `` + output += `` + output += `` + output += '\n' + } + } + output += ' \n' + + output += '
${column}
${name}${item.channels}${item.epg}${item.status}
' + + return output +} + +module.exports = table From 127b7f2a3131d8c5e8a466af834e9935f5790732 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:22 +0300 Subject: [PATCH 05/15] Update queue.db --- tests/__data__/expected/database/queue.db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__data__/expected/database/queue.db b/tests/__data__/expected/database/queue.db index 4680eb70..34ab32c8 100644 --- a/tests/__data__/expected/database/queue.db +++ b/tests/__data__/expected/database/queue.db @@ -1,2 +1,2 @@ -{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"configPath":"tests/__data__/input/sites/example.com.config.js","date":"2022-01-30T00:00:00.000Z","groups":["ca-nl/example.com"],"error":null,"cluster_id":1,"_id":"TYDwYLsrkmPtTLT2"} -{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"configPath":"tests/__data__/input/sites/example.com.config.js","date":"2022-01-31T00:00:00.000Z","groups":["ca-nl/example.com"],"error":null,"cluster_id":1,"_id":"98cKRthEhMmKEnwx"} +{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"date":"2022-02-03T00:00:00.000Z","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ca/example.com"],"error":null,"cluster_id":1,"_id":"vOpwztzvJ5pFSVws"} +{"channel":{"lang":"ru","xmltv_id":"CNNInternationalEurope.us","site_id":"140","site":"example.com"},"date":"2022-02-04T00:00:00.000Z","configPath":"tests/__data__/input/sites/example.com.config.js","groups":["ca/example.com"],"error":null,"cluster_id":1,"_id":"sP2A0zQSOoVg0BS1"} From 6b0ef6049dace17073dd7754b3adc158eb71db10 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:26 +0300 Subject: [PATCH 06/15] Delete example.com.log --- tests/__data__/expected/logs/errors/ca-nl/example.com.log | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/__data__/expected/logs/errors/ca-nl/example.com.log diff --git a/tests/__data__/expected/logs/errors/ca-nl/example.com.log b/tests/__data__/expected/logs/errors/ca-nl/example.com.log deleted file mode 100644 index eb0f2aac..00000000 --- a/tests/__data__/expected/logs/errors/ca-nl/example.com.log +++ /dev/null @@ -1 +0,0 @@ -{"xmltv_id":"CNNInternationalEurope2.us","site":"example.com","site_id":"141","lang":"en","error":"The channel has the wrong xmltv_id"} From 90b0c62492d67ede558d5c178af0960f5cb3540d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:29 +0300 Subject: [PATCH 07/15] Create example.com.log --- tests/__data__/expected/logs/errors/ca/example.com.log | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/__data__/expected/logs/errors/ca/example.com.log diff --git a/tests/__data__/expected/logs/errors/ca/example.com.log b/tests/__data__/expected/logs/errors/ca/example.com.log new file mode 100644 index 00000000..eb0f2aac --- /dev/null +++ b/tests/__data__/expected/logs/errors/ca/example.com.log @@ -0,0 +1 @@ +{"xmltv_id":"CNNInternationalEurope2.us","site":"example.com","site_id":"141","lang":"en","error":"The channel has the wrong xmltv_id"} From 13ce4a2d04b00e8dea071d5252eb44bfd0c20fd2 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:35 +0300 Subject: [PATCH 08/15] Update readme.md --- tests/__data__/expected/readme.md | 35 +++++-------------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/tests/__data__/expected/readme.md b/tests/__data__/expected/readme.md index 66277c4f..f7606f4a 100644 --- a/tests/__data__/expected/readme.md +++ b/tests/__data__/expected/readme.md @@ -1,7 +1,5 @@ # EPG -![auto-update](https://github.com/iptv-org/epg/actions/workflows/auto-update.yml/badge.svg) - EPG (Electronic Program Guide) for thousands of TV channels collected from different sources. ## Usage @@ -11,40 +9,17 @@ To load a program guide, all you need to do is copy the link to one or more of t - + - - - + +
CountryChannelsEPGStatus
CountryChannelsEPGStatus                                                   
πŸ‡ΏπŸ‡¦ South Africa1https://iptv-org.github.io/epg/guides/za/dstv.com.epg.xml🟒
πŸ‡ΊπŸ‡Έ United States372https://iptv-org.github.io/epg/guides/us/tvtv.us.epg.xml🟒
74https://iptv-org.github.io/epg/guides/us/magticom.ge.epg.xml🟒
πŸ‡¨πŸ‡¦ Canada2https://iptv-org.github.io/epg/guides/ca/example.com.epg.xmlexample.com
πŸ‡ͺπŸ‡ͺ Estonia1https://iptv-org.github.io/epg/guides/ee-en/ignore.com.epg.xmlignore.com
-### US States +## EPG Codes - - - - - - - - - - -
StateChannelsEPGStatus
Puerto Rico14https://iptv-org.github.io/epg/guides/us-pr/tvtv.us.epg.xml🟒
7https://iptv-org.github.io/epg/guides/us-pr/gatotv.com.epg.xmlπŸ”΄
0https://iptv-org.github.io/epg/guides/us-pr/directv.com.epg.xml🟒
- -### Provinces of Canada - - - - - - - - - -
ProvinceChannelsEPGStatus
Newfoundland and Labrador1https://iptv-org.github.io/epg/guides/ca-nl/tvtv.us.epg.xml🟒
+πŸ“‹  [iptv-org.github.io](https://iptv-org.github.io/) ## API From ef25d5febc728991df64a760b660a0008cad3df7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:39 +0300 Subject: [PATCH 09/15] Delete example.com_ca-nl.channels.xml --- tests/__data__/input/sites/example.com_ca-nl.channels.xml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/__data__/input/sites/example.com_ca-nl.channels.xml diff --git a/tests/__data__/input/sites/example.com_ca-nl.channels.xml b/tests/__data__/input/sites/example.com_ca-nl.channels.xml deleted file mode 100644 index c3f20f08..00000000 --- a/tests/__data__/input/sites/example.com_ca-nl.channels.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - CNN International Europe - CNN International Europe 2 - - \ No newline at end of file From 92d586c957d51db920eca109c5f3a455bcd051fd Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:45 +0300 Subject: [PATCH 10/15] Create example.com_ca.channels.xml --- tests/__data__/input/sites/example.com_ca.channels.xml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/__data__/input/sites/example.com_ca.channels.xml diff --git a/tests/__data__/input/sites/example.com_ca.channels.xml b/tests/__data__/input/sites/example.com_ca.channels.xml new file mode 100644 index 00000000..c3f20f08 --- /dev/null +++ b/tests/__data__/input/sites/example.com_ca.channels.xml @@ -0,0 +1,7 @@ + + + + CNN International Europe + CNN International Europe 2 + + \ No newline at end of file From 5bfc8894f3b2fdedd0d20767f63be9dea1678120 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:50 +0300 Subject: [PATCH 11/15] Create ignore.com_ee-en.channels.xml --- tests/__data__/input/sites/ignore.com_ee-en.channels.xml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/__data__/input/sites/ignore.com_ee-en.channels.xml diff --git a/tests/__data__/input/sites/ignore.com_ee-en.channels.xml b/tests/__data__/input/sites/ignore.com_ee-en.channels.xml new file mode 100644 index 00000000..ec8ad844 --- /dev/null +++ b/tests/__data__/input/sites/ignore.com_ee-en.channels.xml @@ -0,0 +1,6 @@ + + + + 1+2 + + \ No newline at end of file From 8c504fc4152134826ba83847ae17df933c438ebc Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:54 +0300 Subject: [PATCH 12/15] Delete ignore.com_en-ee.channels.xml --- tests/__data__/input/sites/ignore.com_en-ee.channels.xml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 tests/__data__/input/sites/ignore.com_en-ee.channels.xml diff --git a/tests/__data__/input/sites/ignore.com_en-ee.channels.xml b/tests/__data__/input/sites/ignore.com_en-ee.channels.xml deleted file mode 100644 index ec8ad844..00000000 --- a/tests/__data__/input/sites/ignore.com_en-ee.channels.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - 1+2 - - \ No newline at end of file From 8040f70a3a375a42c1867eb42f0ce3561dedeae4 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:23:58 +0300 Subject: [PATCH 13/15] Update create-queue.test.js --- tests/commands/create-queue.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/create-queue.test.js b/tests/commands/create-queue.test.js index 1ea09c3f..80ed0508 100644 --- a/tests/commands/create-queue.test.js +++ b/tests/commands/create-queue.test.js @@ -36,8 +36,8 @@ it('can create queue', () => { }) it('can log errors', () => { - let output = content('tests/__data__/output/logs/errors/ca-nl/example.com.log') - let expected = content('tests/__data__/expected/logs/errors/ca-nl/example.com.log') + let output = content('tests/__data__/output/logs/errors/ca/example.com.log') + let expected = content('tests/__data__/expected/logs/errors/ca/example.com.log') expect(output).toEqual(expected) }) From 2e21c854447c286020d60be49a8ea378924c819d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:24:04 +0300 Subject: [PATCH 14/15] Update update-readme.test.js --- tests/commands/update-readme.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/commands/update-readme.test.js b/tests/commands/update-readme.test.js index 1c8df660..108517c5 100644 --- a/tests/commands/update-readme.test.js +++ b/tests/commands/update-readme.test.js @@ -7,7 +7,7 @@ beforeEach(() => { fs.mkdirSync('tests/__data__/output') const stdout = execSync( - 'LOGS_DIR=tests/__data__/input/logs DATA_DIR=tests/__data__/input/data node scripts/commands/update-readme.js --config=tests/__data__/input/_readme.json', + 'CHANNELS_PATH=tests/__data__/input/sites/*.channels.xml DATA_DIR=tests/__data__/input/data node scripts/commands/update-readme.js --config=tests/__data__/input/_readme.json', { encoding: 'utf8' } ) }) From f37a5b3d9b665619af760a4db664bbe73089961d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Thu, 3 Feb 2022 06:33:21 +0300 Subject: [PATCH 15/15] Update package.json --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8a0c3051..856e4900 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,11 @@ "scripts": { "lint": "node scripts/commands/lint.js", "validate": "node scripts/commands/validate.js", - "act": "act workflow_dispatch", "test": "npx jest --runInBand", "test:commands": "npx jest --runInBand -- commands", - "test:sites": "npx jest --runInBand -- sites" + "test:sites": "npx jest --runInBand -- sites", + "act": "act workflow_dispatch", + "update-readme": "node scripts/commands/update-readme.js" }, "private": true, "author": "Arhey",