iptv-org-mirror/iptv
1
0
Fork 0
mirror of https://github.com/iptv-org/iptv.git synced 2025-05-12 18:10:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update scripts

Browse source
This commit is contained in:
freearhey 2023-09-17 04:08:50 +03:00
parent 89b87ad5c8
commit bb2935878d
16 changed files with 248 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

67
scripts/commands/playlist/format.ts Normal file
View file

@ -0,0 +1,67 @@
import { STREAMS_DIR, DATA_DIR } from '../../constants'
import { Storage, Logger, PlaylistParser, Collection } from '../../core'
import { Stream, Playlist, Channel } from '../../models'
import { program } from 'commander'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() {
const storage = new Storage(STREAMS_DIR)
const logger = new Logger()
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
logger.info('loading streams...')
const parser = new PlaylistParser({ storage })
const files = program.args.length ? program.args : await storage.list('**/*.m3u')
let streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('removing duplicates...')
streams = streams.uniqBy(stream => stream.url)
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
if (groupedChannels.missing(stream.channel)) {
stream.channel = ''
}
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
[
(stream: Stream) => stream.name,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (let filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
if (!streams.length) return
const playlist = new Playlist(streams, { public: false })
await storage.save(filepath, playlist.toString())
}
}
main()

48
scripts/commands/playlist/generate.ts
View file

@ -1,6 +1,5 @@
import { File, Storage } from '../../core'
import { File, PlaylistParser, Storage } from '../../core'
import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
import { Database } from '../../core/database'
import { Collection } from '../../core/collection'
import { Logger } from '../../core/logger'
import _ from 'lodash'
@ -16,32 +15,29 @@ import {
IndexLanguageGenerator,
IndexRegionGenerator
} from '../../generators'
import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants'
import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
async function main() {
const logger = new Logger()
const dataStorage = new Storage(DATA_DIR)
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
logger.info('loading data from api...')
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const categoriesContent = await storage.json('categories.json')
const categoriesContent = await dataStorage.json('categories.json')
const categories = new Collection(categoriesContent).map(data => new Category(data))
const countriesContent = await storage.json('countries.json')
const countriesContent = await dataStorage.json('countries.json')
const countries = new Collection(countriesContent).map(data => new Country(data))
const languagesContent = await storage.json('languages.json')
const languagesContent = await dataStorage.json('languages.json')
const languages = new Collection(languagesContent).map(data => new Language(data))
const regionsContent = await storage.json('regions.json')
const regionsContent = await dataStorage.json('regions.json')
const regions = new Collection(regionsContent).map(data => new Region(data))
const subdivisionsContent = await storage.json('subdivisions.json')
const subdivisionsContent = await dataStorage.json('subdivisions.json')
const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
logger.info('loading streams...')
const streams = await loadStreams({ channels, categories, languages })
logger.info(`found ${streams.count()} streams`)
const generatorsLogger = new Logger({
stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
@ -49,7 +45,6 @@ async function main() {
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
logger.info('generating countries/...')
await new CountriesGenerator({
countries,
@ -58,10 +53,8 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating languages/...')
await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating regions/...')
await new RegionsGenerator({
streams,
@ -69,16 +62,12 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.m3u...')
await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.nsfw.m3u...')
await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.category.m3u...')
await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
@ -87,10 +76,8 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.region.m3u...')
await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
}
@ -110,11 +97,12 @@ async function loadStreams({
const groupedCategories = categories.keyBy(category => category.id)
const groupedLanguages = languages.keyBy(language => language.code)
const db = new Database(DB_DIR)
const dbStreams = await db.load('streams.db')
const docs = await dbStreams.find({})
const streams = new Collection(docs as any[])
.map((data: any) => new Stream(data))
const storage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage })
const files = await storage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc'])
.uniqBy((stream: Stream) => stream.channel || _.uniqueId())
.map((stream: Stream) => {

58
scripts/commands/playlist/update.ts
View file

@ -1,6 +1,6 @@
import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants'
import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core'
import { Stream, Playlist, Channel } from '../../models'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core'
import { Stream, Playlist, Channel, Issue } from '../../models'
let processedIssues = new Collection()
let streams: Collection
@ -10,19 +10,19 @@ async function main() {
const logger = new Logger({ disabled: true })
const loader = new IssueLoader()
logger.info('loading streams...')
const db = new Database(DB_DIR)
const docs = await db.load('streams.db')
const dbStreams = await docs.find({})
streams = new Collection(dbStreams as any[]).map(data => new Stream(data))
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = await streamsStorage.list('**/*.m3u')
streams = await parser.parse(files)
logger.info('removing broken streams...')
await removeStreams(loader)
@ -32,25 +32,7 @@ async function main() {
logger.info('add new streams...')
await addStreams(loader)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
[
(stream: Stream) => stream.name,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const streamsStorage = new Storage(STREAMS_DIR)
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (let filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
@ -69,19 +51,22 @@ main()
async function removeStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:remove', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('stream_url')) return
const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url'))
if (removed.notEmpty()) {
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
}
})
}
async function editStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:edit', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('stream_url')) return
let stream = streams.first(
@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) {
streams.remove((_stream: Stream) => _stream.channel === stream.channel)
streams.add(stream)
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
})
}
async function addStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:add', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('channel_id') || data.missing('stream_url')) return
if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return
@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) {
})
streams.add(stream)
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
})
}

102
scripts/commands/playlist/validate.ts
View file

@ -5,7 +5,6 @@ import chalk from 'chalk'
import { transliterate } from 'transliteration'
import _ from 'lodash'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import path from 'path'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
@ -19,73 +18,70 @@ async function main() {
const logger = new Logger()
logger.info(`loading blocklist...`)
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const blocklistContent = await storage.json('blocklist.json')
const blocklistContent = await dataStorage.json('blocklist.json')
const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
logger.info(`found ${blocklist.count()} records`)
let errors = new Collection()
let warnings = new Collection()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
for (const filepath of files) {
const file = new File(filepath)
if (file.extension() !== 'm3u') continue
const streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
let errors = new Collection()
let warnings = new Collection()
let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath)
if (!streams) continue
const file = new File(filepath)
const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, '']
const log = new Collection()
const buffer = new Dictionary()
try {
const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
const playlist = await parser.parse(relativeFilepath)
playlist.streams.forEach((stream: Stream) => {
const channelNotInDatabase =
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
if (channelNotInDatabase) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.channel}" is not in the database`
})
}
streams.forEach((stream: Stream) => {
const channelNotInDatabase =
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
if (channelNotInDatabase) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.channel}" is not in the database`
})
}
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
if (alreadyOnPlaylist) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
if (alreadyOnPlaylist) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
const channelId = generateChannelId(stream.name, countryCode)
const blocked = blocklist.first(
blocked =>
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
channelId.toLowerCase() === blocked.channel.toLowerCase()
)
if (blocked) {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
})
}
})
} catch (error) {
log.add({
type: 'error',
line: 0,
message: error.message.toLowerCase()
})
}
const channelId = generateChannelId(stream.name, countryCode)
const blocked = blocklist.first(
blocked =>
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
channelId.toLowerCase() === blocked.channel.toLowerCase()
)
if (blocked) {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
})
}
})
if (log.notEmpty()) {
logger.info(`\n${chalk.underline(filepath)}`)