mirror of
https://github.com/iptv-org/database.git
synced 2025-05-10 03:30:01 -04:00
Update scripts
This commit is contained in:
freearhey
parent
77680e2dc9
commit
075c53143e
14 changed files with 520 additions and 132 deletions
|
|
@ -1,36 +1,49 @@
|
|||
import { CSV, IssueLoader, CSVParser, IDCreator, Issue, IssueData } from '../core'
|
||||
import { Channel, Blocked } from '../models'
|
||||
import { CSV, IssueLoader, CSVParser, Issue, IssueData } from '../core'
|
||||
import { Channel, Blocked, Feed } from '../models'
|
||||
import { DATA_DIR } from '../constants'
|
||||
import { Storage, Collection } from '@freearhey/core'
|
||||
import { createChannelId, createFeedId } from '../utils'
|
||||
|
||||
let blocklist = new Collection()
|
||||
let channels = new Collection()
|
||||
let feeds = new Collection()
|
||||
let issues = new Collection()
|
||||
const processedIssues = new Collection()
|
||||
|
||||
async function main() {
|
||||
const idCreator = new IDCreator()
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const parser = new CSVParser()
|
||||
|
||||
const _channels = await dataStorage.load('channels.csv')
|
||||
channels = (await parser.parse(_channels)).map(data => new Channel(data))
|
||||
|
||||
const _blocklist = await dataStorage.load('blocklist.csv')
|
||||
blocklist = (await parser.parse(_blocklist)).map(data => new Blocked(data))
|
||||
|
||||
const loader = new IssueLoader()
|
||||
|
||||
await removeChannels({ loader })
|
||||
await editChannels({ loader, idCreator })
|
||||
await addChannels({ loader, idCreator })
|
||||
await blockChannels({ loader })
|
||||
await unblockChannels({ loader })
|
||||
issues = await loader.load()
|
||||
|
||||
channels = sortBy(channels, 'id')
|
||||
const channelsCSV = await dataStorage.load('channels.csv')
|
||||
channels = (await parser.parse(channelsCSV)).map(data => new Channel(data))
|
||||
|
||||
const feedsCSV = await dataStorage.load('feeds.csv')
|
||||
feeds = (await parser.parse(feedsCSV)).map(data => new Feed(data))
|
||||
|
||||
const blocklistCSV = await dataStorage.load('blocklist.csv')
|
||||
blocklist = (await parser.parse(blocklistCSV)).map(data => new Blocked(data))
|
||||
|
||||
await removeFeeds()
|
||||
await removeChannels()
|
||||
await editFeeds()
|
||||
await editChannels()
|
||||
await addFeeds()
|
||||
await addChannels()
|
||||
await blockChannels()
|
||||
await unblockChannels()
|
||||
|
||||
channels = channels.sortBy(channel => channel.id.toLowerCase())
|
||||
const channelsOutput = new CSV({ items: channels }).toString()
|
||||
await dataStorage.save('channels.csv', channelsOutput)
|
||||
|
||||
blocklist = sortBy(blocklist, 'channel')
|
||||
feeds = feeds.sortBy(feed => `${feed.channel}@${feed.id}`.toLowerCase())
|
||||
const feedsOutput = new CSV({ items: feeds }).toString()
|
||||
await dataStorage.save('feeds.csv', feedsOutput)
|
||||
|
||||
blocklist = blocklist.sortBy(blocked => blocked.channel.toLowerCase())
|
||||
const blocklistOutput = new CSV({ items: blocklist }).toString()
|
||||
await dataStorage.save('blocklist.csv', blocklistOutput)
|
||||
|
||||
|
|
@ -40,21 +53,139 @@ async function main() {
|
|||
|
||||
main()
|
||||
|
||||
function sortBy(channels: Collection, key: string) {
|
||||
const items = channels.all().sort((a, b) => {
|
||||
const normA = a[key].toLowerCase()
|
||||
const normB = b[key].toLowerCase()
|
||||
if (normA < normB) return -1
|
||||
if (normA > normB) return 1
|
||||
return 0
|
||||
})
|
||||
async function removeFeeds() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('feeds:remove') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
return new Collection(items)
|
||||
requests.forEach((issue: Issue) => {
|
||||
if (issue.data.missing('channel_id') || issue.data.missing('feed_id')) return
|
||||
|
||||
const found = feeds.first(
|
||||
(feed: Feed) =>
|
||||
feed.channel === issue.data.getString('channel_id') &&
|
||||
feed.id === issue.data.getString('feed_id')
|
||||
)
|
||||
if (!found) return
|
||||
|
||||
feeds.remove((feed: Feed) => feed.channel === found.channel && feed.id === found.id)
|
||||
|
||||
onFeedRemoval(found.channel, found.id)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function removeChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['channels:remove,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
async function editFeeds() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('feeds:edit') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data: IssueData = issue.data
|
||||
if (data.missing('channel_id') || data.missing('feed_id')) return
|
||||
|
||||
const found: Feed = feeds.first(
|
||||
(feed: Feed) =>
|
||||
feed.channel === data.getString('channel_id') && feed.id === data.getString('feed_id')
|
||||
)
|
||||
if (!found) return
|
||||
|
||||
let channelId: string | undefined = found.channel
|
||||
let feedId: string | undefined = found.id
|
||||
if (data.has('feed_name')) {
|
||||
const name = data.getString('feed_name') || found.name
|
||||
if (name) {
|
||||
feedId = createFeedId(name)
|
||||
if (feedId) onFeedIdChange(found.channel, found.id, feedId)
|
||||
}
|
||||
}
|
||||
|
||||
if (data.has('is_main')) {
|
||||
const isMain = data.getBoolean('is_main') || false
|
||||
if (isMain) onFeedNewMain(channelId, feedId)
|
||||
}
|
||||
|
||||
if (!feedId || !channelId) return
|
||||
|
||||
const updated = new Feed({
|
||||
channel: channelId,
|
||||
id: feedId,
|
||||
name: data.getString('feed_name'),
|
||||
is_main: data.getBoolean('is_main'),
|
||||
broadcast_area: data.getArray('broadcast_area'),
|
||||
timezones: data.getArray('timezones'),
|
||||
languages: data.getArray('languages'),
|
||||
video_format: data.getString('video_format'),
|
||||
launched: data.getString('launched'),
|
||||
closed: data.getString('closed'),
|
||||
replaced_by: data.getString('replaced_by')
|
||||
})
|
||||
|
||||
found.merge(updated)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function addFeeds() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('feeds:add') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data: IssueData = issue.data
|
||||
|
||||
if (
|
||||
data.missing('channel_id') ||
|
||||
data.missing('feed_name') ||
|
||||
data.missing('is_main') ||
|
||||
data.missing('broadcast_area') ||
|
||||
data.missing('timezones') ||
|
||||
data.missing('languages') ||
|
||||
data.missing('video_format')
|
||||
)
|
||||
return
|
||||
|
||||
const channelId = data.getString('channel_id')
|
||||
const feedName = data.getString('feed_name') || 'SD'
|
||||
const feedId = createFeedId(feedName)
|
||||
if (!channelId || !feedId) return
|
||||
|
||||
const found: Feed = feeds.first(
|
||||
(feed: Feed) => feed.channel === channelId && feed.id === feedId
|
||||
)
|
||||
if (found) return
|
||||
|
||||
const isMain = data.getBoolean('is_main') || false
|
||||
if (isMain) onFeedNewMain(channelId, feedId)
|
||||
|
||||
feeds.push(
|
||||
new Feed({
|
||||
channel: channelId,
|
||||
id: feedId,
|
||||
name: feedName,
|
||||
is_main: data.getBoolean('is_main'),
|
||||
broadcast_area: data.getArray('broadcast_area'),
|
||||
timezones: data.getArray('timezones'),
|
||||
languages: data.getArray('languages'),
|
||||
video_format: data.getString('video_format'),
|
||||
launched: data.getString('launched'),
|
||||
closed: data.getString('closed'),
|
||||
replaced_by: data.getString('replaced_by')
|
||||
})
|
||||
)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function removeChannels() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('channels:remove') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
if (issue.data.missing('channel_id')) return
|
||||
|
||||
const found = channels.first(
|
||||
|
|
@ -64,13 +195,18 @@ async function removeChannels({ loader }: { loader: IssueLoader }) {
|
|||
|
||||
channels.remove((channel: Channel) => channel.id === found.id)
|
||||
|
||||
onChannelRemoval(found.id)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
|
||||
const issues = await loader.load({ labels: ['channels:edit,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
async function editChannels() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('channels:edit') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data: IssueData = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
|
|
@ -79,20 +215,21 @@ async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCrea
|
|||
)
|
||||
if (!found) return
|
||||
|
||||
let channelId = found.id
|
||||
if (data.has('name') || data.has('country')) {
|
||||
const name = data.getString('name') || found.name
|
||||
let channelId: string | undefined = found.id
|
||||
if (data.has('channel_name') || data.has('country')) {
|
||||
const name = data.getString('channel_name') || found.name
|
||||
const country = data.getString('country') || found.country
|
||||
if (name && country) {
|
||||
channelId = idCreator.create(name, country)
|
||||
updateBlocklistId(found.id, channelId)
|
||||
updateChannelReplacedBy(found.id, channelId)
|
||||
channelId = createChannelId(name, country)
|
||||
if (channelId) onChannelIdChange(found.id, channelId)
|
||||
}
|
||||
}
|
||||
|
||||
if (!channelId) return
|
||||
|
||||
const updated = new Channel({
|
||||
id: channelId,
|
||||
name: data.getString('name'),
|
||||
name: data.getString('channel_name'),
|
||||
alt_names: data.getArray('alt_names'),
|
||||
network: data.getString('network'),
|
||||
owners: data.getArray('owners'),
|
||||
|
|
@ -116,16 +253,29 @@ async function editChannels({ loader, idCreator }: { loader: IssueLoader; idCrea
|
|||
})
|
||||
}
|
||||
|
||||
async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreator: IDCreator }) {
|
||||
const issues = await loader.load({ labels: ['channels:add,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
async function addChannels() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('channels:add') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data: IssueData = issue.data
|
||||
const name = data.getString('name')
|
||||
const country = data.getString('country')
|
||||
|
||||
if (!name || !country) return
|
||||
if (
|
||||
data.missing('channel_name') ||
|
||||
data.missing('country') ||
|
||||
data.missing('is_nsfw') ||
|
||||
data.missing('logo') ||
|
||||
data.missing('feed_name') ||
|
||||
data.missing('broadcast_area') ||
|
||||
data.missing('timezones') ||
|
||||
data.missing('languages') ||
|
||||
data.missing('video_format')
|
||||
)
|
||||
return
|
||||
|
||||
const channelId = idCreator.create(name, country)
|
||||
const channelId = createChannelId(data.getString('channel_name'), data.getString('country'))
|
||||
if (!channelId) return
|
||||
|
||||
const found: Channel = channels.first((channel: Channel) => channel.id === channelId)
|
||||
if (found) return
|
||||
|
|
@ -133,7 +283,7 @@ async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreat
|
|||
channels.push(
|
||||
new Channel({
|
||||
id: channelId,
|
||||
name: data.getString('name'),
|
||||
name: data.getString('channel_name'),
|
||||
alt_names: data.getArray('alt_names'),
|
||||
network: data.getString('network'),
|
||||
owners: data.getArray('owners'),
|
||||
|
|
@ -152,13 +302,34 @@ async function addChannels({ loader, idCreator }: { loader: IssueLoader; idCreat
|
|||
})
|
||||
)
|
||||
|
||||
const feedName = data.getString('feed_name') || 'SD'
|
||||
|
||||
feeds.push(
|
||||
new Feed({
|
||||
channel: channelId,
|
||||
id: createFeedId(feedName),
|
||||
name: feedName,
|
||||
is_main: true,
|
||||
broadcast_area: data.getArray('broadcast_area'),
|
||||
timezones: data.getArray('timezones'),
|
||||
languages: data.getArray('languages'),
|
||||
video_format: data.getString('video_format'),
|
||||
launched: data.getString('launched'),
|
||||
closed: data.getString('closed'),
|
||||
replaced_by: data.getString('replaced_by')
|
||||
})
|
||||
)
|
||||
|
||||
processedIssues.push(issue)
|
||||
})
|
||||
}
|
||||
|
||||
async function unblockChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['blocklist:remove,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
async function unblockChannels() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('blocklist:remove') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
|
|
@ -173,9 +344,12 @@ async function unblockChannels({ loader }: { loader: IssueLoader }) {
|
|||
})
|
||||
}
|
||||
|
||||
async function blockChannels({ loader }: { loader: IssueLoader }) {
|
||||
const issues = await loader.load({ labels: ['blocklist:add,approved'] })
|
||||
issues.forEach((issue: Issue) => {
|
||||
async function blockChannels() {
|
||||
const requests = issues.filter(
|
||||
issue => issue.labels.includes('blocklist:add') && issue.labels.includes('approved')
|
||||
)
|
||||
|
||||
requests.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id')) return
|
||||
|
||||
|
|
@ -201,22 +375,77 @@ async function blockChannels({ loader }: { loader: IssueLoader }) {
|
|||
})
|
||||
}
|
||||
|
||||
function updateBlocklistId(oldId: string, newId: string) {
|
||||
const filtered: Collection = blocklist.filter((blocked: Blocked) => blocked.channel === oldId)
|
||||
if (filtered.isEmpty()) return
|
||||
function onFeedIdChange(channelId: string, feedId: string, newFeedId: string) {
|
||||
channels.forEach((channel: Channel) => {
|
||||
if (channel.replaced_by && channel.replaced_by === `${channelId}@${feedId}`) {
|
||||
channel.replaced_by = `${channelId}@${newFeedId}`
|
||||
}
|
||||
})
|
||||
|
||||
filtered.forEach(item => {
|
||||
item.channel = newId
|
||||
feeds.forEach((feed: Feed) => {
|
||||
if (feed.replaced_by && feed.replaced_by === `${channelId}@${feedId}`) {
|
||||
feed.replaced_by = `${channelId}@${newFeedId}`
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function updateChannelReplacedBy(channelId: string, newReplacedBy: string) {
|
||||
const filtered: Collection = channels.filter(
|
||||
(channel: Channel) => channel.replaced_by === channelId
|
||||
)
|
||||
if (filtered.isEmpty()) return
|
||||
|
||||
filtered.forEach(item => {
|
||||
item.replaced_by = newReplacedBy
|
||||
function onFeedNewMain(channelId: string, feedId: string) {
|
||||
feeds.forEach((feed: Feed) => {
|
||||
if (feed.channel === channelId && feed.id !== feedId && feed.is_main === true) {
|
||||
feed.is_main = false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function onFeedRemoval(channelId: string, feedId: string) {
|
||||
channels.forEach((channel: Channel) => {
|
||||
if (channel.replaced_by && channel.replaced_by === `${channelId}@${feedId}`) {
|
||||
channel.replaced_by = ''
|
||||
}
|
||||
})
|
||||
|
||||
feeds.forEach((feed: Feed) => {
|
||||
if (feed.replaced_by && feed.replaced_by === `${channelId}@${feedId}`) {
|
||||
feed.replaced_by = ''
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function onChannelIdChange(channelId: string, newChannelId: string) {
|
||||
channels.forEach((channel: Channel) => {
|
||||
if (channel.replaced_by && channel.replaced_by.includes(channelId)) {
|
||||
channel.replaced_by = channel.replaced_by.replace(channelId, newChannelId)
|
||||
}
|
||||
})
|
||||
|
||||
feeds.forEach((feed: Feed) => {
|
||||
if (feed.channel === channelId) {
|
||||
feed.channel = newChannelId
|
||||
}
|
||||
|
||||
if (feed.replaced_by && feed.replaced_by.includes(channelId)) {
|
||||
feed.replaced_by = feed.replaced_by.replace(channelId, newChannelId)
|
||||
}
|
||||
})
|
||||
|
||||
blocklist.forEach((blocked: Blocked) => {
|
||||
if (blocked.channel === channelId) {
|
||||
blocked.channel = newChannelId
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function onChannelRemoval(channelId: string) {
|
||||
channels.forEach((channel: Channel) => {
|
||||
if (channel.replaced_by && channel.replaced_by.includes(channelId)) {
|
||||
channel.replaced_by = ''
|
||||
}
|
||||
})
|
||||
|
||||
feeds.remove((feed: Feed) => feed.channel === channelId)
|
||||
feeds.forEach((feed: Feed) => {
|
||||
if (feed.replaced_by && feed.replaced_by.includes(channelId)) {
|
||||
feed.replaced_by = ''
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ import { DATA_DIR } from '../constants'
|
|||
import schemesData from '../schemes'
|
||||
import { program } from 'commander'
|
||||
import Joi from 'joi'
|
||||
import { CSVParser, IDCreator } from '../core'
|
||||
import { CSVParser } from '../core'
|
||||
import chalk from 'chalk'
|
||||
import { createChannelId } from '../utils'
|
||||
|
||||
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
|
|
@ -42,11 +43,15 @@ async function main() {
|
|||
|
||||
let grouped
|
||||
switch (filename) {
|
||||
case 'feeds':
|
||||
grouped = data.keyBy(item => item.channel + item.id)
|
||||
break
|
||||
case 'blocklist':
|
||||
grouped = data.keyBy(item => item.channel + item.ref)
|
||||
break
|
||||
case 'categories':
|
||||
case 'channels':
|
||||
case 'timezones':
|
||||
grouped = data.keyBy(item => item.id)
|
||||
break
|
||||
default:
|
||||
|
|
@ -91,6 +96,17 @@ async function main() {
|
|||
)
|
||||
}
|
||||
break
|
||||
case 'feeds':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['channel', 'id']))
|
||||
fileErrors = fileErrors.concat(validateMainFeeds(rowsCopy))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
fileErrors = fileErrors.concat(validateChannel(row.channel, i))
|
||||
fileErrors = fileErrors.concat(validateTimezones(row, i))
|
||||
fileErrors = fileErrors.concat(
|
||||
checkValue(i, row, 'id', 'replaced_by', buffer.get('channels'))
|
||||
)
|
||||
}
|
||||
break
|
||||
case 'blocklist':
|
||||
fileErrors = fileErrors.concat(findDuplicatesBy(rowsCopy, ['channel', 'ref']))
|
||||
for (const [i, row] of rowsCopy.entries()) {
|
||||
|
|
@ -201,7 +217,7 @@ function findDuplicatesBy(rows: { [key: string]: string }[], keys: string[]) {
|
|||
const buffer = new Dictionary()
|
||||
|
||||
rows.forEach((row, i) => {
|
||||
const normId = keys.map(key => row[key].toLowerCase()).join()
|
||||
const normId = keys.map(key => row[key].toString().toLowerCase()).join()
|
||||
if (buffer.has(normId)) {
|
||||
const fieldsList = keys.map(key => `${key} "${row[key]}"`).join(' and ')
|
||||
errors.push({
|
||||
|
|
@ -216,10 +232,31 @@ function findDuplicatesBy(rows: { [key: string]: string }[], keys: string[]) {
|
|||
return errors
|
||||
}
|
||||
|
||||
function validateMainFeeds(rows: { [key: string]: string }[]) {
|
||||
const errors = new Collection()
|
||||
const buffer = new Dictionary()
|
||||
|
||||
rows.forEach((row, i) => {
|
||||
const normId = `${row.channel}${row.is_main}`
|
||||
if (buffer.has(normId)) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `entry with the channel "${row.channel}" and is_main "true" already exists`
|
||||
})
|
||||
}
|
||||
|
||||
if (row.is_main) {
|
||||
buffer.set(normId, true)
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function validateChannelId(row: { [key: string]: string }, i: number) {
|
||||
const errors = new Collection()
|
||||
|
||||
const expectedId = new IDCreator().create(row.name, row.country)
|
||||
const expectedId = createChannelId(row.name, row.country)
|
||||
|
||||
if (expectedId !== row.id) {
|
||||
errors.push({
|
||||
|
|
@ -254,6 +291,22 @@ function validateChannelBroadcastArea(row: { [key: string]: string[] }, i: numbe
|
|||
return errors
|
||||
}
|
||||
|
||||
function validateTimezones(row: { [key: string]: string[] }, i: number) {
|
||||
const errors = new Collection()
|
||||
const timezones = buffer.get('timezones')
|
||||
|
||||
row.timezones.forEach((timezone: string) => {
|
||||
if (timezones.missing(timezone)) {
|
||||
errors.push({
|
||||
line: i + 2,
|
||||
message: `"${row.channel}@${row.id}" has the wrong timezone "${timezone}"`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
function handleError(message: string) {
|
||||
logger.error(chalk.red(message))
|
||||
process.exit(1)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue