mirror of
https://github.com/iptv-org/database.git
synced 2025-05-09 19:20:01 -04:00
Update scripts
This commit is contained in:
freearhey
parent
66ec908b6e
commit
179ef6a41d
28 changed files with 958 additions and 866 deletions
|
|
@ -1,105 +0,0 @@
|
|||
const csv2json = require('csvtojson')
|
||||
const chalk = require('chalk')
|
||||
const logger = require('./logger')
|
||||
const fs = require('mz/fs')
|
||||
const {
|
||||
Parser,
|
||||
transforms: { flatten },
|
||||
formatters: { stringQuoteOnlyIfNecessary }
|
||||
} = require('json2csv')
|
||||
|
||||
const csv2jsonOptions = {
|
||||
checkColumn: true,
|
||||
trim: true,
|
||||
delimiter: ',',
|
||||
eol: '\r\n',
|
||||
colParser: {
|
||||
alt_names: listParser,
|
||||
network: nullable,
|
||||
owners: listParser,
|
||||
subdivision: nullable,
|
||||
city: nullable,
|
||||
broadcast_area: listParser,
|
||||
languages: listParser,
|
||||
categories: listParser,
|
||||
is_nsfw: boolParser,
|
||||
launched: nullable,
|
||||
closed: nullable,
|
||||
replaced_by: nullable,
|
||||
website: nullable,
|
||||
logo: nullable,
|
||||
countries: listParser
|
||||
}
|
||||
}
|
||||
|
||||
const json2csv = new Parser({
|
||||
transforms: [flattenArray, formatBool],
|
||||
formatters: {
|
||||
string: stringQuoteOnlyIfNecessary()
|
||||
},
|
||||
eol: '\r\n'
|
||||
})
|
||||
|
||||
const csv = {}
|
||||
|
||||
csv.fromFile = async function (filepath) {
|
||||
return csv2json(csv2jsonOptions).fromFile(filepath)
|
||||
}
|
||||
|
||||
csv.fromString = async function (filepath) {
|
||||
return csv2json(csv2jsonOptions).fromString(filepath)
|
||||
}
|
||||
|
||||
csv.save = async function (filepath, data) {
|
||||
const string = json2csv.parse(data)
|
||||
|
||||
return fs.writeFile(filepath, string)
|
||||
}
|
||||
|
||||
csv.saveSync = function (filepath, data) {
|
||||
const string = json2csv.parse(data)
|
||||
|
||||
return fs.writeFileSync(filepath, string)
|
||||
}
|
||||
|
||||
module.exports = csv
|
||||
|
||||
function flattenArray(item) {
|
||||
for (let prop in item) {
|
||||
const value = item[prop]
|
||||
item[prop] = Array.isArray(value) ? value.join(';') : value
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function formatBool(item) {
|
||||
for (let prop in item) {
|
||||
if (item[prop] === false) {
|
||||
item[prop] = 'FALSE'
|
||||
} else if (item[prop] === true) {
|
||||
item[prop] = 'TRUE'
|
||||
}
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function listParser(value) {
|
||||
return value.split(';').filter(i => i)
|
||||
}
|
||||
|
||||
function boolParser(value) {
|
||||
switch (value) {
|
||||
case 'TRUE':
|
||||
return true
|
||||
case 'FALSE':
|
||||
return false
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function nullable(value) {
|
||||
return value === '' ? null : value
|
||||
}
|
||||
44
scripts/core/csv.ts
Normal file
44
scripts/core/csv.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import { Parser } from '@json2csv/plainjs'
|
||||
import { stringQuoteOnlyIfNecessary } from '@json2csv/formatters'
|
||||
|
||||
export class CSV {
|
||||
items: Collection
|
||||
|
||||
constructor({ items }: { items: Collection }) {
|
||||
this.items = items
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
const parser = new Parser({
|
||||
transforms: [flattenArray, formatBool],
|
||||
formatters: {
|
||||
string: stringQuoteOnlyIfNecessary()
|
||||
},
|
||||
eol: '\r\n'
|
||||
})
|
||||
|
||||
return parser.parse(this.items.all())
|
||||
}
|
||||
}
|
||||
|
||||
function flattenArray(item: { [key: string]: string[] | string | boolean }) {
|
||||
for (const prop in item) {
|
||||
const value = item[prop]
|
||||
item[prop] = Array.isArray(value) ? value.join(';') : value
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
||||
function formatBool(item: { [key: string]: string[] | string | boolean }) {
|
||||
for (const prop in item) {
|
||||
if (item[prop] === false) {
|
||||
item[prop] = 'FALSE'
|
||||
} else if (item[prop] === true) {
|
||||
item[prop] = 'TRUE'
|
||||
}
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
53
scripts/core/csvParser.ts
Normal file
53
scripts/core/csvParser.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import csv2json from 'csvtojson'
|
||||
|
||||
const opts = {
|
||||
checkColumn: true,
|
||||
trim: true,
|
||||
delimiter: ',',
|
||||
eol: '\r\n',
|
||||
colParser: {
|
||||
alt_names: listParser,
|
||||
network: nullable,
|
||||
owners: listParser,
|
||||
subdivision: nullable,
|
||||
city: nullable,
|
||||
broadcast_area: listParser,
|
||||
languages: listParser,
|
||||
categories: listParser,
|
||||
is_nsfw: boolParser,
|
||||
launched: nullable,
|
||||
closed: nullable,
|
||||
replaced_by: nullable,
|
||||
website: nullable,
|
||||
logo: nullable,
|
||||
countries: listParser
|
||||
}
|
||||
}
|
||||
|
||||
export class CSVParser {
|
||||
async parse(data: string): Promise<Collection> {
|
||||
const items = await csv2json(opts).fromString(data)
|
||||
|
||||
return new Collection(items)
|
||||
}
|
||||
}
|
||||
|
||||
function listParser(value: string) {
|
||||
return value.split(';').filter(i => i)
|
||||
}
|
||||
|
||||
function boolParser(value: string) {
|
||||
switch (value) {
|
||||
case 'TRUE':
|
||||
return true
|
||||
case 'FALSE':
|
||||
return false
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function nullable(value: string) {
|
||||
return value === '' ? null : value
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
const path = require('path')
|
||||
const glob = require('glob')
|
||||
const fs = require('mz/fs')
|
||||
const crlf = require('crlf')
|
||||
|
||||
const file = {}
|
||||
|
||||
file.list = function (pattern) {
|
||||
return new Promise(resolve => {
|
||||
glob(pattern, function (err, files) {
|
||||
resolve(files)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
file.getFilename = function (filepath) {
|
||||
return path.parse(filepath).name
|
||||
}
|
||||
|
||||
file.createDir = async function (dir) {
|
||||
if (await file.exists(dir)) return
|
||||
|
||||
return fs.mkdir(dir, { recursive: true }).catch(console.error)
|
||||
}
|
||||
|
||||
file.exists = function (filepath) {
|
||||
return fs.exists(path.resolve(filepath))
|
||||
}
|
||||
|
||||
file.read = function (filepath) {
|
||||
return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
|
||||
}
|
||||
|
||||
file.append = function (filepath, data) {
|
||||
return fs.appendFile(path.resolve(filepath), data).catch(console.error)
|
||||
}
|
||||
|
||||
file.create = function (filepath, data = '') {
|
||||
filepath = path.resolve(filepath)
|
||||
const dir = path.dirname(filepath)
|
||||
|
||||
return file
|
||||
.createDir(dir)
|
||||
.then(() => file.write(filepath, data))
|
||||
.catch(console.error)
|
||||
}
|
||||
|
||||
file.write = function (filepath, data = '') {
|
||||
return fs.writeFile(path.resolve(filepath), data, { encoding: 'utf8' }).catch(console.error)
|
||||
}
|
||||
|
||||
file.clear = async function (filepath) {
|
||||
if (await file.exists(filepath)) return file.write(filepath, '')
|
||||
return true
|
||||
}
|
||||
|
||||
file.resolve = function (filepath) {
|
||||
return path.resolve(filepath)
|
||||
}
|
||||
|
||||
file.dirname = function (filepath) {
|
||||
return path.dirname(filepath)
|
||||
}
|
||||
|
||||
file.basename = function (filepath) {
|
||||
return path.basename(filepath)
|
||||
}
|
||||
|
||||
file.eol = function (filepath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
crlf.get(filepath, null, function (err, endingType) {
|
||||
if (err) reject(err)
|
||||
resolve(endingType)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = file
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
exports.csv = require('./csv')
|
||||
exports.file = require('./file')
|
||||
exports.logger = require('./logger')
|
||||
4
scripts/core/index.ts
Normal file
4
scripts/core/index.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
export * from './csv'
|
||||
export * from './issueParser'
|
||||
export * from './issueLoader'
|
||||
export * from './csvParser'
|
||||
49
scripts/core/issueLoader.ts
Normal file
49
scripts/core/issueLoader.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { Collection } from '@freearhey/core'
|
||||
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
|
||||
import { paginateRest } from '@octokit/plugin-paginate-rest'
|
||||
import { Octokit } from '@octokit/core'
|
||||
import { IssueParser } from './'
|
||||
import { TESTING, OWNER, REPO } from '../constants'
|
||||
|
||||
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
|
||||
const octokit = new CustomOctokit()
|
||||
|
||||
export class IssueLoader {
|
||||
async load({ labels }: { labels: string[] | string }) {
|
||||
labels = Array.isArray(labels) ? labels.join(',') : labels
|
||||
let issues: object[] = []
|
||||
if (TESTING) {
|
||||
switch (labels) {
|
||||
case 'channels:add,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_add_approved.js')
|
||||
break
|
||||
case 'channels:edit,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_edit_approved.js')
|
||||
break
|
||||
case 'channels:remove,approved':
|
||||
issues = require('../../tests/__data__/input/issues/channels_remove_approved.js')
|
||||
break
|
||||
case 'blocklist:add,approved':
|
||||
issues = require('../../tests/__data__/input/issues/blocklist_add_approved.js')
|
||||
break
|
||||
case 'blocklist:remove,approved':
|
||||
issues = require('../../tests/__data__/input/issues/blocklist_remove_approved.js')
|
||||
break
|
||||
}
|
||||
} else {
|
||||
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
|
||||
owner: OWNER,
|
||||
repo: REPO,
|
||||
per_page: 100,
|
||||
labels,
|
||||
headers: {
|
||||
'X-GitHub-Api-Version': '2022-11-28'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const parser = new IssueParser()
|
||||
|
||||
return new Collection(issues).map(parser.parse)
|
||||
}
|
||||
}
|
||||
66
scripts/core/issueParser.ts
Normal file
66
scripts/core/issueParser.ts
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import { Dictionary } from '@freearhey/core'
|
||||
import { Issue } from '../models'
|
||||
|
||||
const FIELDS = new Dictionary({
|
||||
'Channel ID': 'channel_id',
|
||||
'Channel ID (required)': 'channel_id',
|
||||
'Channel ID (optional)': 'channel_id',
|
||||
'Channel Name': 'name',
|
||||
'Alternative Names': 'alt_names',
|
||||
'Alternative Names (optional)': 'alt_names',
|
||||
Network: 'network',
|
||||
'Network (optional)': 'network',
|
||||
Owners: 'owners',
|
||||
'Owners (optional)': 'owners',
|
||||
Country: 'country',
|
||||
Subdivision: 'subdivision',
|
||||
'Subdivision (optional)': 'subdivision',
|
||||
City: 'city',
|
||||
'City (optional)': 'city',
|
||||
'Broadcast Area': 'broadcast_area',
|
||||
Languages: 'languages',
|
||||
Categories: 'categories',
|
||||
'Categories (optional)': 'categories',
|
||||
NSFW: 'is_nsfw',
|
||||
Launched: 'launched',
|
||||
'Launched (optional)': 'launched',
|
||||
Closed: 'closed',
|
||||
'Closed (optional)': 'closed',
|
||||
'Replaced By': 'replaced_by',
|
||||
'Replaced By (optional)': 'replaced_by',
|
||||
Website: 'website',
|
||||
'Website (optional)': 'website',
|
||||
Logo: 'logo',
|
||||
Reason: 'reason',
|
||||
Notes: 'notes',
|
||||
'Notes (optional)': 'notes',
|
||||
Reference: 'ref',
|
||||
'Reference (optional)': 'ref',
|
||||
'Reference (required)': 'ref'
|
||||
})
|
||||
|
||||
export class IssueParser {
|
||||
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
|
||||
const fields = issue.body.split('###')
|
||||
|
||||
const data = new Dictionary()
|
||||
fields.forEach((field: string) => {
|
||||
let [_label, , _value] = field.split(/\r?\n/)
|
||||
_label = _label ? _label.trim() : ''
|
||||
_value = _value ? _value.trim() : ''
|
||||
|
||||
if (!_label || !_value) return data
|
||||
|
||||
const id: string = FIELDS.get(_label)
|
||||
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
|
||||
|
||||
if (!id) return
|
||||
|
||||
data.set(id, value)
|
||||
})
|
||||
|
||||
const labels = issue.labels.map(label => label.name)
|
||||
|
||||
return new Issue({ number: issue.number, labels, data })
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
const { Signale } = require('signale')
|
||||
|
||||
const options = {}
|
||||
|
||||
const logger = new Signale(options)
|
||||
|
||||
logger.config({
|
||||
displayLabel: false,
|
||||
displayScope: false,
|
||||
displayBadge: false
|
||||
})
|
||||
|
||||
module.exports = logger
|
||||
Loading…
Add table
Add a link
Reference in a new issue