mirror of
https://github.com/iptv-org/epg.git
synced 2025-05-09 16:40:07 -04:00
Created commands/create-database.js
This commit is contained in:
Aleksandr Statciuk
parent
5c95098fea
commit
f5dbc9376e
12 changed files with 29999 additions and 1 deletions
180
package-lock.json
generated
180
package-lock.json
generated
|
|
@ -21,6 +21,8 @@
|
|||
"lodash": "^4.17.21",
|
||||
"markdown-include": "^0.4.3",
|
||||
"mockdate": "^3.0.5",
|
||||
"mz": "^2.7.0",
|
||||
"nedb-promises": "^5.0.3",
|
||||
"parse-duration": "^1.0.0",
|
||||
"pdf-parse": "^1.1.1",
|
||||
"srcset": "^4.0.0",
|
||||
|
|
@ -850,6 +852,26 @@
|
|||
"node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@seald-io/binary-search-tree": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.2.tgz",
|
||||
"integrity": "sha512-+pYGvPFAk7wUR+ONMOlc6A+LUN4kOCFwyPLjyaeS7wVibADPHWYJNYsNtyIAwjF1AXQkuaXElnIc4XjKt55QZA=="
|
||||
},
|
||||
"node_modules/@seald-io/nedb": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-2.2.0.tgz",
|
||||
"integrity": "sha512-whkcx3hpcowNhoSEbIsrfe8TXxDwyj8SJJut2EqF7DSX2GGqQlL7Ix/vzwwOo4FJolzDhZD2DaUTVKmTQS3Rog==",
|
||||
"dependencies": {
|
||||
"@seald-io/binary-search-tree": "^1.0.2",
|
||||
"async": "0.2.10",
|
||||
"localforage": "^1.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@seald-io/nedb/node_modules/async": {
|
||||
"version": "0.2.10",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
|
||||
"integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E="
|
||||
},
|
||||
"node_modules/@sindresorhus/is": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.2.0.tgz",
|
||||
|
|
@ -1139,6 +1161,11 @@
|
|||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/any-promise": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
|
||||
"integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
|
||||
},
|
||||
"node_modules/anymatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz",
|
||||
|
|
@ -2439,6 +2466,11 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/immediate": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
|
||||
"integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
|
||||
},
|
||||
"node_modules/import-local": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.3.tgz",
|
||||
|
|
@ -3395,6 +3427,22 @@
|
|||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lie": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
|
||||
"integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
|
||||
"dependencies": {
|
||||
"immediate": "~3.0.5"
|
||||
}
|
||||
},
|
||||
"node_modules/localforage": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz",
|
||||
"integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==",
|
||||
"dependencies": {
|
||||
"lie": "3.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/locate-path": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||
|
|
@ -3563,11 +3611,29 @@
|
|||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node_modules/mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
"integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==",
|
||||
"dependencies": {
|
||||
"any-promise": "^1.0.0",
|
||||
"object-assign": "^4.0.1",
|
||||
"thenify-all": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/natural-compare": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
|
||||
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc="
|
||||
},
|
||||
"node_modules/nedb-promises": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/nedb-promises/-/nedb-promises-5.0.3.tgz",
|
||||
"integrity": "sha512-hqQZi/T3dMJXQraFohVdatNX1uPDTGuVrqMiEnwA0dJxlC90xctOFG+jeUzPy9hYQGqCV+CpkSBkvUvTJdUApQ==",
|
||||
"dependencies": {
|
||||
"@seald-io/nedb": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-ensure": {
|
||||
"version": "0.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz",
|
||||
|
|
@ -3637,6 +3703,14 @@
|
|||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
|
||||
"integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ=="
|
||||
},
|
||||
"node_modules/object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
|
|
@ -4312,6 +4386,25 @@
|
|||
"resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
|
||||
"integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
|
||||
},
|
||||
"node_modules/thenify": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
|
||||
"integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==",
|
||||
"dependencies": {
|
||||
"any-promise": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/thenify-all": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
|
||||
"integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=",
|
||||
"dependencies": {
|
||||
"thenify": ">= 3.1.0 < 4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/throat": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/throat/-/throat-6.0.1.tgz",
|
||||
|
|
@ -5386,6 +5479,28 @@
|
|||
"chalk": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"@seald-io/binary-search-tree": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@seald-io/binary-search-tree/-/binary-search-tree-1.0.2.tgz",
|
||||
"integrity": "sha512-+pYGvPFAk7wUR+ONMOlc6A+LUN4kOCFwyPLjyaeS7wVibADPHWYJNYsNtyIAwjF1AXQkuaXElnIc4XjKt55QZA=="
|
||||
},
|
||||
"@seald-io/nedb": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@seald-io/nedb/-/nedb-2.2.0.tgz",
|
||||
"integrity": "sha512-whkcx3hpcowNhoSEbIsrfe8TXxDwyj8SJJut2EqF7DSX2GGqQlL7Ix/vzwwOo4FJolzDhZD2DaUTVKmTQS3Rog==",
|
||||
"requires": {
|
||||
"@seald-io/binary-search-tree": "^1.0.2",
|
||||
"async": "0.2.10",
|
||||
"localforage": "^1.9.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"async": {
|
||||
"version": "0.2.10",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
|
||||
"integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@sindresorhus/is": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.2.0.tgz",
|
||||
|
|
@ -5632,6 +5747,11 @@
|
|||
"color-convert": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"any-promise": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
|
||||
"integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
|
||||
},
|
||||
"anymatch": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz",
|
||||
|
|
@ -6600,6 +6720,11 @@
|
|||
"safer-buffer": ">= 2.1.2 < 3"
|
||||
}
|
||||
},
|
||||
"immediate": {
|
||||
"version": "3.0.6",
|
||||
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
|
||||
"integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
|
||||
},
|
||||
"import-local": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.3.tgz",
|
||||
|
|
@ -7314,6 +7439,22 @@
|
|||
"type-check": "~0.3.2"
|
||||
}
|
||||
},
|
||||
"lie": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
|
||||
"integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
|
||||
"requires": {
|
||||
"immediate": "~3.0.5"
|
||||
}
|
||||
},
|
||||
"localforage": {
|
||||
"version": "1.10.0",
|
||||
"resolved": "https://registry.npmjs.org/localforage/-/localforage-1.10.0.tgz",
|
||||
"integrity": "sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==",
|
||||
"requires": {
|
||||
"lie": "3.1.1"
|
||||
}
|
||||
},
|
||||
"locate-path": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||
|
|
@ -7446,11 +7587,29 @@
|
|||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"mz": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
|
||||
"integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==",
|
||||
"requires": {
|
||||
"any-promise": "^1.0.0",
|
||||
"object-assign": "^4.0.1",
|
||||
"thenify-all": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"natural-compare": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
|
||||
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc="
|
||||
},
|
||||
"nedb-promises": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/nedb-promises/-/nedb-promises-5.0.3.tgz",
|
||||
"integrity": "sha512-hqQZi/T3dMJXQraFohVdatNX1uPDTGuVrqMiEnwA0dJxlC90xctOFG+jeUzPy9hYQGqCV+CpkSBkvUvTJdUApQ==",
|
||||
"requires": {
|
||||
"@seald-io/nedb": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node-ensure": {
|
||||
"version": "0.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz",
|
||||
|
|
@ -7502,6 +7661,11 @@
|
|||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.0.tgz",
|
||||
"integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ=="
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
|
||||
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
|
|
@ -8000,6 +8164,22 @@
|
|||
"resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz",
|
||||
"integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="
|
||||
},
|
||||
"thenify": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz",
|
||||
"integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==",
|
||||
"requires": {
|
||||
"any-promise": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"thenify-all": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz",
|
||||
"integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=",
|
||||
"requires": {
|
||||
"thenify": ">= 3.1.0 < 4"
|
||||
}
|
||||
},
|
||||
"throat": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/throat/-/throat-6.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
"author": "Arhey",
|
||||
"license": "MIT",
|
||||
"jest": {
|
||||
"testRegex": "sites/(.*?/)?.*test.js$"
|
||||
"testRegex": "(sites|tests)/(.*?/)?.*test.js$"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^0.21.1",
|
||||
|
|
@ -25,6 +25,8 @@
|
|||
"lodash": "^4.17.21",
|
||||
"markdown-include": "^0.4.3",
|
||||
"mockdate": "^3.0.5",
|
||||
"mz": "^2.7.0",
|
||||
"nedb-promises": "^5.0.3",
|
||||
"parse-duration": "^1.0.0",
|
||||
"pdf-parse": "^1.1.1",
|
||||
"srcset": "^4.0.0",
|
||||
|
|
|
|||
29512
scripts/channels.db
Normal file
29512
scripts/channels.db
Normal file
File diff suppressed because it is too large
Load diff
63
scripts/commands/create-database.js
Normal file
63
scripts/commands/create-database.js
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
const { db, file, parser, logger } = require('../core')
|
||||
const { program } = require('commander')
|
||||
const _ = require('lodash')
|
||||
|
||||
const options = program
|
||||
.option(
|
||||
'--max-clusters <max-clusters>',
|
||||
'Set maximum number of clusters',
|
||||
parser.parseNumber,
|
||||
200
|
||||
)
|
||||
.option('--channels <channels>', 'Set path to channels.xml file', 'sites/**/*.channels.xml')
|
||||
.parse(process.argv)
|
||||
.opts()
|
||||
|
||||
const channels = []
|
||||
|
||||
async function main() {
|
||||
logger.info('Starting...')
|
||||
logger.info(`Number of clusters: ${options.maxClusters}`)
|
||||
|
||||
await loadChannels()
|
||||
await saveToDatabase()
|
||||
|
||||
logger.info('Done')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
async function loadChannels() {
|
||||
logger.info(`Loading channels...`)
|
||||
|
||||
const files = await file.list(options.channels)
|
||||
for (const filepath of files) {
|
||||
const items = await parser.parseChannels(filepath)
|
||||
for (const item of items) {
|
||||
item.filepath = filepath
|
||||
channels.push(item)
|
||||
}
|
||||
}
|
||||
logger.info(`Found ${channels.length} channels`)
|
||||
}
|
||||
|
||||
async function saveToDatabase() {
|
||||
logger.info('Saving to the database...')
|
||||
|
||||
await db.reset()
|
||||
const chunks = split(_.shuffle(channels), options.maxClusters)
|
||||
for (const [i, chunk] of chunks.entries()) {
|
||||
for (const item of chunk) {
|
||||
item.cluster_id = i + 1
|
||||
await db.insert(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function split(arr, n) {
|
||||
let result = []
|
||||
for (let i = n; i > 0; i--) {
|
||||
result.push(arr.splice(0, Math.ceil(arr.length / i)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
61
scripts/core/db.js
Normal file
61
scripts/core/db.js
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
const Database = require('nedb-promises')
|
||||
const file = require('./file')
|
||||
|
||||
const DB_FILEPATH = process.env.DB_FILEPATH || './scripts/channels.db'
|
||||
|
||||
const nedb = Database.create({
|
||||
filename: file.resolve(DB_FILEPATH),
|
||||
autoload: true,
|
||||
onload(err) {
|
||||
if (err) console.error(err)
|
||||
},
|
||||
compareStrings: (a, b) => {
|
||||
a = a.replace(/\s/g, '_')
|
||||
b = b.replace(/\s/g, '_')
|
||||
|
||||
return a.localeCompare(b, undefined, {
|
||||
sensitivity: 'accent',
|
||||
numeric: true
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const db = {}
|
||||
|
||||
db.removeIndex = function (field) {
|
||||
return nedb.removeIndex(field)
|
||||
}
|
||||
|
||||
db.addIndex = function (options) {
|
||||
return nedb.ensureIndex(options)
|
||||
}
|
||||
|
||||
db.compact = function () {
|
||||
return nedb.persistence.compactDatafile()
|
||||
}
|
||||
|
||||
db.reset = function () {
|
||||
return file.clear(DB_FILEPATH)
|
||||
}
|
||||
|
||||
db.count = function (query) {
|
||||
return nedb.count(query)
|
||||
}
|
||||
|
||||
db.insert = function (doc) {
|
||||
return nedb.insert(doc)
|
||||
}
|
||||
|
||||
db.update = function (query, update) {
|
||||
return nedb.update(query, update)
|
||||
}
|
||||
|
||||
db.find = function (query) {
|
||||
return nedb.find(query)
|
||||
}
|
||||
|
||||
db.remove = function (query, options) {
|
||||
return nedb.remove(query, options)
|
||||
}
|
||||
|
||||
module.exports = db
|
||||
67
scripts/core/file.js
Normal file
67
scripts/core/file.js
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
const path = require('path')
|
||||
const glob = require('glob')
|
||||
const fs = require('mz/fs')
|
||||
|
||||
const file = {}
|
||||
|
||||
file.list = function (pattern) {
|
||||
return new Promise(resolve => {
|
||||
glob(pattern, function (err, files) {
|
||||
resolve(files)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
file.getFilename = function (filepath) {
|
||||
return path.parse(filepath).name
|
||||
}
|
||||
|
||||
file.createDir = async function (dir) {
|
||||
if (await file.exists(dir)) return
|
||||
|
||||
return fs.mkdir(dir, { recursive: true }).catch(console.error)
|
||||
}
|
||||
|
||||
file.exists = function (filepath) {
|
||||
return fs.exists(path.resolve(filepath))
|
||||
}
|
||||
|
||||
file.read = function (filepath) {
|
||||
return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
|
||||
}
|
||||
|
||||
file.append = function (filepath, data) {
|
||||
return fs.appendFile(path.resolve(filepath), data).catch(console.error)
|
||||
}
|
||||
|
||||
file.create = function (filepath, data = '') {
|
||||
filepath = path.resolve(filepath)
|
||||
const dir = path.dirname(filepath)
|
||||
|
||||
return file
|
||||
.createDir(dir)
|
||||
.then(() => fs.writeFile(filepath, data, { encoding: 'utf8', flag: 'w' }))
|
||||
.catch(console.error)
|
||||
}
|
||||
|
||||
file.write = function (filepath, data = '') {
|
||||
return fs.writeFile(path.resolve(filepath), data).catch(console.error)
|
||||
}
|
||||
|
||||
file.clear = function (filepath) {
|
||||
return file.write(filepath, '')
|
||||
}
|
||||
|
||||
file.resolve = function (filepath) {
|
||||
return path.resolve(filepath)
|
||||
}
|
||||
|
||||
file.dirname = function (filepath) {
|
||||
return path.dirname(filepath)
|
||||
}
|
||||
|
||||
file.basename = function (filepath) {
|
||||
return path.basename(filepath)
|
||||
}
|
||||
|
||||
module.exports = file
|
||||
4
scripts/core/index.js
Normal file
4
scripts/core/index.js
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
exports.db = require('./db')
|
||||
exports.logger = require('./logger')
|
||||
exports.file = require('./file')
|
||||
exports.parser = require('./parser')
|
||||
42
scripts/core/logger.js
Normal file
42
scripts/core/logger.js
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
const { createLogger, format, transports, addColors } = require('winston')
|
||||
const { combine, timestamp, printf } = format
|
||||
|
||||
const consoleFormat = ({ level, message, timestamp }) => {
|
||||
if (typeof message === 'object') return JSON.stringify(message)
|
||||
return message
|
||||
}
|
||||
|
||||
const config = {
|
||||
levels: {
|
||||
error: 0,
|
||||
warn: 1,
|
||||
info: 2,
|
||||
failed: 3,
|
||||
success: 4,
|
||||
http: 5,
|
||||
verbose: 6,
|
||||
debug: 7,
|
||||
silly: 8
|
||||
},
|
||||
colors: {
|
||||
info: 'white',
|
||||
success: 'green',
|
||||
failed: 'red'
|
||||
}
|
||||
}
|
||||
|
||||
const t = [
|
||||
new transports.Console({
|
||||
format: format.combine(format.printf(consoleFormat))
|
||||
})
|
||||
]
|
||||
|
||||
const logger = createLogger({
|
||||
transports: t,
|
||||
levels: config.levels,
|
||||
level: 'verbose'
|
||||
})
|
||||
|
||||
addColors(config.colors)
|
||||
|
||||
module.exports = logger
|
||||
31
scripts/core/parser.js
Normal file
31
scripts/core/parser.js
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
const logger = require('./logger')
|
||||
const file = require('./file')
|
||||
const grabber = require('epg-grabber')
|
||||
|
||||
const parser = {}
|
||||
|
||||
parser.parseChannels = async function(filepath) {
|
||||
const content = await file.read(filepath)
|
||||
const channels = grabber.parseChannels(content)
|
||||
|
||||
return channels
|
||||
}
|
||||
|
||||
parser.parseLogs = async function (filepath) {
|
||||
const content = await file.read(filepath)
|
||||
if (!content) return []
|
||||
const lines = content.split('\n')
|
||||
|
||||
return lines.map(line => (line ? JSON.parse(line) : null)).filter(l => l)
|
||||
}
|
||||
|
||||
parser.parseNumber = function (string) {
|
||||
const parsed = parseInt(string)
|
||||
if (isNaN(parsed)) {
|
||||
throw new Error('scripts/core/parser.js:parseNumber() Input value is not a number')
|
||||
}
|
||||
|
||||
return parsed
|
||||
}
|
||||
|
||||
module.exports = parser
|
||||
6
tests/__data__/input/site.channels.xml
Normal file
6
tests/__data__/input/site.channels.xml
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<site site="andorradifusio.ad">
|
||||
<channels>
|
||||
<channel lang="ca" xmltv_id="AndorraTV.ad" site_id="atv">Andorra TV</channel>
|
||||
</channels>
|
||||
</site>
|
||||
1
tests/__data__/output/test.db
Normal file
1
tests/__data__/output/test.db
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"lang":"ca","xmltv_id":"AndorraTV.ad","site_id":"atv","name":"Andorra TV","site":"andorradifusio.ad","filepath":"tests/__data__/input/site.channels.xml","cluster_id":1,"_id":"2lf1xEfDjyxPsXhO"}
|
||||
29
tests/commands/create-database.test.js
Normal file
29
tests/commands/create-database.test.js
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const { execSync } = require('child_process')
|
||||
|
||||
beforeEach(() => {
|
||||
fs.rmdirSync('tests/__data__/output', { recursive: true })
|
||||
fs.mkdirSync('tests/__data__/output')
|
||||
})
|
||||
|
||||
it('can create database', () => {
|
||||
const results = execSync(
|
||||
'DB_FILEPATH=tests/__data__/output/test.db node scripts/commands/create-database.js --channels=tests/__data__/input/site.channels.xml --max-clusters=1',
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
|
||||
const database = fs.readFileSync(path.resolve('tests/__data__/output/test.db'), {
|
||||
encoding: 'utf8'
|
||||
})
|
||||
const item = database.split('\n').find(i => i.includes('AndorraTV.ad'))
|
||||
expect(JSON.parse(item)).toMatchObject({
|
||||
name: 'Andorra TV',
|
||||
lang: 'ca',
|
||||
xmltv_id: 'AndorraTV.ad',
|
||||
site_id: 'atv',
|
||||
site: 'andorradifusio.ad',
|
||||
filepath: 'tests/__data__/input/site.channels.xml',
|
||||
cluster_id: 1
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue