iptv-org-mirror/iptv
1
0
Fork 0
mirror of https://github.com/iptv-org/iptv.git synced 2025-05-11 01:20:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update scripts

Browse source
This commit is contained in:
freearhey 2025-03-29 11:39:46 +03:00
parent 74b3cff1d2
commit 02ec7e6f76
42 changed files with 1317 additions and 694 deletions
Download patch file Download diff file Expand all files Collapse all files

30
scripts/commands/api/generate.ts
View file

@ -1,21 +1,37 @@
import { Logger, Storage } from '@freearhey/core'
import { API_DIR, STREAMS_DIR } from '../../constants'
import { Logger, Storage, Collection } from '@freearhey/core'
import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import { PlaylistParser } from '../../core'
import { Stream } from '../../models'
import { Stream, Channel, Feed } from '../../models'
import { uniqueId } from 'lodash'
async function main() {
const logger = new Logger()
logger.info('loading api data...')
const dataStorage = new Storage(DATA_DIR)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
feed.channel ? feed.channel.id : uniqueId()
)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const parser = new PlaylistParser({
storage: streamsStorage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.map(data => new Stream(data))
.orderBy([(stream: Stream) => stream.channel])
.orderBy((stream: Stream) => stream.getId())
.map((stream: Stream) => stream.toJSON())
logger.info(`found ${streams.count()} streams`)
logger.info('saving to .api/streams.json...')

4
scripts/commands/api/load.ts
View file

@ -12,7 +12,9 @@ async function main() {
client.download('countries.json'),
client.download('languages.json'),
client.download('regions.json'),
client.download('subdivisions.json')
client.download('subdivisions.json'),
client.download('feeds.json'),
client.download('timezones.json')
]
await Promise.all(requests)

41
scripts/commands/playlist/format.ts
View file

@ -1,25 +1,36 @@
import { Logger, Storage, Collection } from '@freearhey/core'
import { STREAMS_DIR, DATA_DIR } from '../../constants'
import { PlaylistParser } from '../../core'
import { Stream, Playlist, Channel } from '../../models'
import { Stream, Playlist, Channel, Feed } from '../../models'
import { program } from 'commander'
import { uniqueId } from 'lodash'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() {
const storage = new Storage(STREAMS_DIR)
const streamsStorage = new Storage(STREAMS_DIR)
const logger = new Logger()
logger.info('loading channels from api...')
logger.info('loading data from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy(feed =>
feed.channel ? feed.channel.id : uniqueId()
)
logger.info('loading streams...')
const parser = new PlaylistParser({ storage })
const files = program.args.length ? program.args : await storage.list('**/*.m3u')
const parser = new PlaylistParser({
storage: streamsStorage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
@ -35,8 +46,8 @@ async function main() {
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
if (groupedChannels.missing(stream.channel)) {
stream.channel = ''
if (!stream.channel || channelsGroupedById.missing(stream.channel.id)) {
stream.id = ''
}
return stream
@ -46,22 +57,22 @@ async function main() {
streams = streams.orderBy(
[
(stream: Stream) => stream.name,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label,
(stream: Stream) => stream.getHorizontalResolution(),
(stream: Stream) => stream.getLabel(),
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (let filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
if (!streams.length) return
const playlist = new Playlist(streams, { public: false })
await storage.save(filepath, playlist.toString())
await streamsStorage.save(filepath, playlist.toString())
}
}

196
scripts/commands/playlist/generate.ts
View file

@ -1,14 +1,23 @@
import { Logger, Storage, Collection, File } from '@freearhey/core'
import { Logger, Storage, Collection } from '@freearhey/core'
import { PlaylistParser } from '../../core'
import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
import _ from 'lodash'
import {
Stream,
Category,
Channel,
Language,
Country,
Region,
Subdivision,
Feed,
Timezone
} from '../../models'
import { uniqueId } from 'lodash'
import {
CategoriesGenerator,
CountriesGenerator,
LanguagesGenerator,
RegionsGenerator,
IndexGenerator,
IndexNsfwGenerator,
IndexCategoryGenerator,
IndexCountryGenerator,
IndexLanguageGenerator,
@ -19,123 +28,134 @@ import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
async function main() {
const logger = new Logger()
const dataStorage = new Storage(DATA_DIR)
logger.info('loading data from api...')
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const categoriesContent = await dataStorage.json('categories.json')
const categories = new Collection(categoriesContent).map(data => new Category(data))
const countriesContent = await dataStorage.json('countries.json')
const countries = new Collection(countriesContent).map(data => new Country(data))
const languagesContent = await dataStorage.json('languages.json')
const languages = new Collection(languagesContent).map(data => new Language(data))
const regionsContent = await dataStorage.json('regions.json')
const regions = new Collection(regionsContent).map(data => new Region(data))
const subdivisionsContent = await dataStorage.json('subdivisions.json')
const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
logger.info('loading streams...')
let streams = await loadStreams({ channels, categories, languages })
let totalStreams = streams.count()
streams = streams.uniqBy((stream: Stream) => (stream.channel || _.uniqueId()) + stream.timeshift)
logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`)
const generatorsLogger = new Logger({
stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
})
logger.info('loading data from api...')
const categoriesData = await dataStorage.json('categories.json')
const countriesData = await dataStorage.json('countries.json')
const languagesData = await dataStorage.json('languages.json')
const regionsData = await dataStorage.json('regions.json')
const subdivisionsData = await dataStorage.json('subdivisions.json')
const timezonesData = await dataStorage.json('timezones.json')
const channelsData = await dataStorage.json('channels.json')
const feedsData = await dataStorage.json('feeds.json')
logger.info('preparing data...')
const subdivisions = new Collection(subdivisionsData).map(data => new Subdivision(data))
const subdivisionsGroupedByCode = subdivisions.keyBy(
(subdivision: Subdivision) => subdivision.code
)
const subdivisionsGroupedByCountryCode = subdivisions.groupBy(
(subdivision: Subdivision) => subdivision.countryCode
)
let regions = new Collection(regionsData).map(data =>
new Region(data).withSubdivisions(subdivisions)
)
const regionsGroupedByCode = regions.keyBy((region: Region) => region.code)
const categories = new Collection(categoriesData).map(data => new Category(data))
const categoriesGroupedById = categories.keyBy((category: Category) => category.id)
const languages = new Collection(languagesData).map(data => new Language(data))
const languagesGroupedByCode = languages.keyBy((language: Language) => language.code)
const countries = new Collection(countriesData).map(data =>
new Country(data)
.withRegions(regions)
.withLanguage(languagesGroupedByCode)
.withSubdivisions(subdivisionsGroupedByCountryCode)
)
const countriesGroupedByCode = countries.keyBy((country: Country) => country.code)
regions = regions.map((region: Region) => region.withCountries(countriesGroupedByCode))
const timezones = new Collection(timezonesData).map(data =>
new Timezone(data).withCountries(countriesGroupedByCode)
)
const timezonesGroupedById = timezones.keyBy((timezone: Timezone) => timezone.id)
const channels = new Collection(channelsData).map(data =>
new Channel(data)
.withCategories(categoriesGroupedById)
.withCountry(countriesGroupedByCode)
.withSubdivision(subdivisionsGroupedByCode)
)
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feeds = new Collection(feedsData).map(data =>
new Feed(data)
.withChannel(channelsGroupedById)
.withLanguages(languagesGroupedByCode)
.withTimezones(timezonesGroupedById)
.withBroadcastCountries(
countriesGroupedByCode,
regionsGroupedByCode,
subdivisionsGroupedByCode
)
.withBroadcastRegions(regions, regionsGroupedByCode)
.withBroadcastSubdivisions(subdivisionsGroupedByCode)
)
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
feed.channel ? feed.channel.id : uniqueId()
)
logger.info('loading streams...')
const storage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = await storage.list('**/*.m3u')
let streams = await parser.parse(files)
const totalStreams = streams.count()
streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId())
logger.info(`found ${totalStreams} streams (including ${streams.count()} unique)`)
logger.info('sorting streams...')
streams = streams.orderBy(
[
(stream: Stream) => stream.getId(),
(stream: Stream) => stream.getHorizontalResolution(),
(stream: Stream) => stream.getLabel()
],
['asc', 'asc', 'desc']
)
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
logger.info('generating countries/...')
await new CountriesGenerator({
countries,
streams,
regions,
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating languages/...')
await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating regions/...')
await new RegionsGenerator({
streams,
regions,
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.m3u...')
await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.category.m3u...')
await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
countries,
regions,
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.region.m3u...')
await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
}
main()
async function loadStreams({
channels,
categories,
languages
}: {
channels: Collection
categories: Collection
languages: Collection
}) {
const groupedChannels = channels.keyBy(channel => channel.id)
const groupedCategories = categories.keyBy(category => category.id)
const groupedLanguages = languages.keyBy(language => language.code)
const storage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage })
const files = await storage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.orderBy(
[
(stream: Stream) => stream.channel,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label
],
['asc', 'asc', 'desc', 'asc']
)
.map((stream: Stream) => {
const channel: Channel | undefined = groupedChannels.get(stream.channel)
if (channel) {
const channelCategories = channel.categories
.map((id: string) => groupedCategories.get(id))
.filter(Boolean)
const channelLanguages = channel.languages
.map((id: string) => groupedLanguages.get(id))
.filter(Boolean)
stream.categories = channelCategories
stream.languages = channelLanguages
stream.broadcastArea = channel.broadcastArea
stream.isNSFW = channel.isNSFW
if (channel.logo) stream.logo = channel.logo
} else {
const file = new File(stream.filepath)
const [_, countryCode] = file.name().match(/^([a-z]{2})(_|$)/) || [null, null]
const defaultBroadcastArea = countryCode ? [`c/${countryCode.toUpperCase()}`] : []
stream.broadcastArea = new Collection(defaultBroadcastArea)
}
return stream
})
return streams
}

32
scripts/commands/playlist/test.ts
View file

@ -1,7 +1,7 @@
import { Logger, Storage, Collection } from '@freearhey/core'
import { ROOT_DIR, STREAMS_DIR } from '../../constants'
import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import { PlaylistParser, StreamTester, CliTable } from '../../core'
import { Stream } from '../../models'
import { Stream, Feed, Channel } from '../../models'
import { program } from 'commander'
import { eachLimit } from 'async-es'
import commandExists from 'command-exists'
@ -38,8 +38,6 @@ const logger = new Logger()
const tester = new StreamTester()
async function main() {
const storage = new Storage(ROOT_DIR)
if (await isOffline()) {
logger.error(chalk.red('Internet connection is required for the script to work'))
@ -56,9 +54,25 @@ async function main() {
return
}
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy(feed => feed.channel)
logger.info('loading streams...')
const parser = new PlaylistParser({ storage })
const files = program.args.length ? program.args : await storage.list(`${STREAMS_DIR}/*.m3u`)
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)
streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
@ -89,7 +103,7 @@ async function main() {
main()
async function runTest(stream: Stream) {
const key = stream.filepath + stream.channel + stream.url
const key = stream.filepath + stream.getId() + stream.url
results[key] = chalk.white('LOADING...')
const result = await tester.test(stream)
@ -125,11 +139,11 @@ function drawTable() {
]
})
streams.forEach((stream: Stream, index: number) => {
const status = results[stream.filepath + stream.channel + stream.url] || chalk.gray('PENDING')
const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING')
const row = {
'': index,
'tvg-id': stream.channel.length > 25 ? stream.channel.slice(0, 22) + '...' : stream.channel,
'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(),
url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url,
status
}

186
scripts/commands/playlist/update.ts
View file

@ -1,45 +1,63 @@
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { IssueLoader, PlaylistParser } from '../../core'
import { Stream, Playlist, Channel, Issue } from '../../models'
import { Stream, Playlist, Channel, Feed, Issue } from '../../models'
import validUrl from 'valid-url'
import { uniqueId } from 'lodash'
let processedIssues = new Collection()
let streams: Collection
let groupedChannels: Dictionary
let issues: Collection
async function main() {
const logger = new Logger({ disabled: true })
const loader = new IssueLoader()
logger.info('loading issues...')
issues = await loader.load()
const issues = await loader.load()
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
feed.channel ? feed.channel.id : uniqueId()
)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const parser = new PlaylistParser({
storage: streamsStorage,
feedsGroupedByChannelId,
channelsGroupedById
})
const files = await streamsStorage.list('**/*.m3u')
streams = await parser.parse(files)
const streams = await parser.parse(files)
logger.info('removing broken streams...')
await removeStreams(loader)
await removeStreams({ streams, issues })
logger.info('edit stream description...')
await editStreams(loader)
await editStreams({
streams,
issues,
channelsGroupedById,
feedsGroupedByChannelId
})
logger.info('add new streams...')
await addStreams(loader)
await addStreams({
streams,
issues,
channelsGroupedById,
feedsGroupedByChannelId
})
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (let filepath of groupedStreams.keys()) {
let streams = groupedStreams.get(filepath) || []
streams = streams.filter((stream: Stream) => stream.removed === false)
@ -54,7 +72,7 @@ async function main() {
main()
async function removeStreams(loader: IssueLoader) {
async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) {
const requests = issues.filter(
issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
)
@ -62,22 +80,35 @@ async function removeStreams(loader: IssueLoader) {
const data = issue.data
if (data.missing('brokenLinks')) return
const brokenLinks = data.getString('brokenLinks').split(/\r?\n/).filter(Boolean)
const brokenLinks = data.getString('brokenLinks') || ''
let changed = false
brokenLinks.forEach(link => {
const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
if (found) {
found.removed = true
changed = true
}
})
brokenLinks
.split(/\r?\n/)
.filter(Boolean)
.forEach(link => {
const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
if (found) {
found.removed = true
changed = true
}
})
if (changed) processedIssues.add(issue.number)
})
}
async function editStreams(loader: IssueLoader) {
async function editStreams({
streams,
issues,
channelsGroupedById,
feedsGroupedByChannelId
}: {
streams: Collection
issues: Collection
channelsGroupedById: Dictionary
feedsGroupedByChannelId: Dictionary
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
)
@ -86,59 +117,110 @@ async function editStreams(loader: IssueLoader) {
if (data.missing('streamUrl')) return
let stream = streams.first(
let stream: Stream = streams.first(
(_stream: Stream) => _stream.url === data.getString('streamUrl')
) as Stream
)
if (!stream) return
if (data.has('channelId')) {
const channel = groupedChannels.get(data.getString('channelId'))
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
if (!channel) return
stream.channel = data.getString('channelId')
stream.filepath = `${channel.country.toLowerCase()}.m3u`
stream.line = -1
stream.name = channel.name
if (channelId) {
stream
.setChannelId(channelId)
.setFeedId(feedId)
.withChannel(channelsGroupedById)
.withFeed(feedsGroupedByChannelId)
.updateId()
.updateName()
.updateFilepath()
}
if (data.has('label')) stream.label = data.getString('label')
if (data.has('quality')) stream.quality = data.getString('quality')
if (data.has('httpUserAgent')) stream.httpUserAgent = data.getString('httpUserAgent')
if (data.has('httpReferrer')) stream.httpReferrer = data.getString('httpReferrer')
const label = data.getString('label') || ''
const quality = data.getString('quality') || ''
const httpUserAgent = data.getString('httpUserAgent') || ''
const httpReferrer = data.getString('httpReferrer') || ''
if (data.has('label')) stream.setLabel(label)
if (data.has('quality')) stream.setQuality(quality)
if (data.has('httpUserAgent')) stream.setHttpUserAgent(httpUserAgent)
if (data.has('httpReferrer')) stream.setHttpReferrer(httpReferrer)
processedIssues.add(issue.number)
})
}
async function addStreams(loader: IssueLoader) {
async function addStreams({
streams,
issues,
channelsGroupedById,
feedsGroupedByChannelId
}: {
streams: Collection
issues: Collection
channelsGroupedById: Dictionary
feedsGroupedByChannelId: Dictionary
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('channelId') || data.missing('streamUrl')) return
if (data.missing('streamId') || data.missing('streamUrl')) return
if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return
if (!validUrl.isUri(data.getString('streamUrl'))) return
const stringUrl = data.getString('streamUrl') || ''
if (!isUri(stringUrl)) return
const channel = groupedChannels.get(data.getString('channelId'))
const streamId = data.getString('streamId') || ''
const [channelId] = streamId.split('@')
const channel: Channel = channelsGroupedById.get(channelId)
if (!channel) return
const label = data.getString('label') || ''
const quality = data.getString('quality') || ''
const httpUserAgent = data.getString('httpUserAgent') || ''
const httpReferrer = data.getString('httpReferrer') || ''
const stream = new Stream({
channel: data.getString('channelId'),
url: data.getString('streamUrl'),
label: data.getString('label'),
quality: data.getString('quality'),
httpUserAgent: data.getString('httpUserAgent'),
httpReferrer: data.getString('httpReferrer'),
filepath: `${channel.country.toLowerCase()}.m3u`,
tvg: {
id: streamId,
name: '',
url: '',
logo: '',
rec: '',
shift: ''
},
name: data.getString('channelName') || channel.name,
url: stringUrl,
group: {
title: ''
},
http: {
'user-agent': httpUserAgent,
referrer: httpReferrer
},
line: -1,
name: data.getString('channelName') || channel.name
raw: '',
timeshift: '',
catchup: {
type: '',
source: '',
days: ''
}
})
.withChannel(channelsGroupedById)
.withFeed(feedsGroupedByChannelId)
.setLabel(label)
.setQuality(quality)
.updateName()
.updateFilepath()
streams.add(stream)
processedIssues.add(issue.number)
})
}
function isUri(string: string) {
return validUrl.isUri(encodeURI(string))
}

57
scripts/commands/playlist/validate.ts
View file

@ -1,9 +1,9 @@
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { PlaylistParser } from '../../core'
import { Channel, Stream, Blocked } from '../../models'
import { Channel, Stream, Blocked, Feed } from '../../models'
import { program } from 'commander'
import chalk from 'chalk'
import _ from 'lodash'
import { uniqueId } from 'lodash'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
@ -17,41 +17,52 @@ type LogItem = {
async function main() {
const logger = new Logger()
logger.info(`loading blocklist...`)
logger.info('loading data from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
feed.channel ? feed.channel.id : uniqueId()
)
const blocklistContent = await dataStorage.json('blocklist.json')
const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
logger.info(`found ${blocklist.count()} records`)
const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const parser = new PlaylistParser({
storage: streamsStorage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
let errors = new Collection()
let warnings = new Collection()
let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath)
let streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of streamsGroupedByFilepath.keys()) {
const streams = streamsGroupedByFilepath.get(filepath)
if (!streams) continue
const log = new Collection()
const buffer = new Dictionary()
streams.forEach((stream: Stream) => {
const invalidId =
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
if (invalidId) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.channel}" is not in the database`
})
if (stream.channelId) {
const channel = channelsGroupedById.get(stream.channelId)
if (!channel) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.id}" is not in the database`
})
}
}
const duplicate = stream.url && buffer.has(stream.url)
@ -65,19 +76,19 @@ async function main() {
buffer.set(stream.url, true)
}
const blocked = blocklist.first(blocked => stream.channel === blocked.channel)
const blocked = stream.channel ? blocklistGroupedByChannelId.get(stream.channel.id) : false
if (blocked) {
if (blocked.reason === 'dmca') {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.channel}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
message: `"${blocked.channelId}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
})
} else if (blocked.reason === 'nsfw') {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.channel}" is on the blocklist due to NSFW content (${blocked.ref})`
message: `"${blocked.channelId}" is on the blocklist due to NSFW content (${blocked.ref})`
})
}
}

205
scripts/commands/report/create.ts
View file

@ -1,154 +1,164 @@
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { IssueLoader, PlaylistParser } from '../../core'
import { Blocked, Channel, Issue, Stream } from '../../models'
import { Blocked, Channel, Issue, Stream, Feed } from '../../models'
import { uniqueId } from 'lodash'
async function main() {
const logger = new Logger()
const loader = new IssueLoader()
const storage = new Storage(DATA_DIR)
let report = new Collection()
logger.info('loading issues...')
const issues = await loader.load()
logger.info('loading data from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsData = await dataStorage.json('channels.json')
const channels = new Collection(channelsData).map(data => new Channel(data))
const channelsGroupedById = channels.keyBy((channel: Channel) => channel.id)
const feedsData = await dataStorage.json('feeds.json')
const feeds = new Collection(feedsData).map(data =>
new Feed(data).withChannel(channelsGroupedById)
)
const feedsGroupedByChannelId = feeds.groupBy((feed: Feed) =>
feed.channel ? feed.channel.id : uniqueId()
)
const blocklistContent = await dataStorage.json('blocklist.json')
const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
const blocklistGroupedByChannelId = blocklist.keyBy((blocked: Blocked) => blocked.channelId)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const parser = new PlaylistParser({
storage: streamsStorage,
channelsGroupedById,
feedsGroupedByChannelId
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel)
logger.info('loading channels from api...')
const channelsContent = await storage.json('channels.json')
const channelsGroupedById = new Collection(channelsContent)
.map(data => new Channel(data))
.groupBy((channel: Channel) => channel.id)
logger.info('loading blocklist from api...')
const blocklistContent = await storage.json('blocklist.json')
const blocklistGroupedByChannel = new Collection(blocklistContent)
.map(data => new Blocked(data))
.groupBy((blocked: Blocked) => blocked.channel)
let report = new Collection()
logger.info('checking streams:add requests...')
const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
const addRequestsBuffer = new Dictionary()
addRequests.forEach((issue: Issue) => {
const channelId = issue.data.getString('channelId') || undefined
const streamUrl = issue.data.getString('streamUrl')
const result = new Dictionary({
issueNumber: issue.number,
type: 'streams:add',
channelId,
streamUrl,
status: 'pending'
})
if (!channelId) result.set('status', 'missing_id')
else if (!streamUrl) result.set('status', 'missing_link')
else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked')
else if (channelsGroupedById.missing(channelId)) result.set('status', 'wrong_id')
else if (streamsGroupedByUrl.has(streamUrl)) result.set('status', 'on_playlist')
else if (addRequestsBuffer.has(streamUrl)) result.set('status', 'duplicate')
else result.set('status', 'pending')
addRequestsBuffer.set(streamUrl, true)
report.add(result.data())
})
logger.info('checking streams:edit requests...')
const editRequests = issues.filter(issue => issue.labels.find(label => label === 'streams:edit'))
editRequests.forEach((issue: Issue) => {
const channelId = issue.data.getString('channelId') || undefined
const streamUrl = issue.data.getString('streamUrl') || undefined
const result = new Dictionary({
issueNumber: issue.number,
type: 'streams:edit',
channelId,
streamUrl,
status: 'pending'
})
if (!streamUrl) result.set('status', 'missing_link')
else if (streamsGroupedByUrl.missing(streamUrl)) result.set('status', 'invalid_link')
else if (channelId && channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id')
report.add(result.data())
})
const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId)
logger.info('checking broken streams reports...')
const brokenStreamReports = issues.filter(issue =>
issue.labels.find(label => label === 'broken stream')
issue.labels.find((label: string) => label === 'broken stream')
)
brokenStreamReports.forEach((issue: Issue) => {
const brokenLinks = issue.data.getArray('brokenLinks') || []
if (!brokenLinks.length) {
const result = new Dictionary({
const result = {
issueNumber: issue.number,
type: 'broken stream',
channelId: undefined,
streamId: undefined,
streamUrl: undefined,
status: 'missing_link'
})
}
report.add(result.data())
report.add(result)
} else {
for (const streamUrl of brokenLinks) {
const result = new Dictionary({
const result = {
issueNumber: issue.number,
type: 'broken stream',
channelId: undefined,
streamUrl: undefined,
streamId: undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
})
if (streamsGroupedByUrl.missing(streamUrl)) {
result.set('streamUrl', streamUrl)
result.set('status', 'wrong_link')
}
report.add(result.data())
if (streamsGroupedByUrl.missing(streamUrl)) {
result.status = 'wrong_link'
}
report.add(result)
}
}
})
logger.info('checking streams:add requests...')
const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
const addRequestsBuffer = new Dictionary()
addRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:add',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!channelId) result.status = 'missing_id'
else if (!streamUrl) result.status = 'missing_link'
else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (channelsGroupedById.missing(channelId)) result.status = 'wrong_id'
else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
else result.status = 'pending'
addRequestsBuffer.set(streamUrl, true)
report.add(result)
})
logger.info('checking streams:edit requests...')
const editRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'streams:edit')
)
editRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:edit',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!streamUrl) result.status = 'missing_link'
else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
else if (channelId && channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
report.add(result)
})
logger.info('checking channel search requests...')
const channelSearchRequests = issues.filter(issue =>
issue.labels.find(label => label === 'channel search')
issue.labels.find((label: string) => label === 'channel search')
)
const channelSearchRequestsBuffer = new Dictionary()
channelSearchRequests.forEach((issue: Issue) => {
const channelId = issue.data.getString('channelId')
const streamId = issue.data.getString('channelId') || ''
const [channelId] = streamId.split('@')
const result = new Dictionary({
const result = {
issueNumber: issue.number,
type: 'channel search',
channelId,
streamId: streamId || undefined,
streamUrl: undefined,
status: 'pending'
})
}
if (!channelId) result.set('status', 'missing_id')
else if (channelsGroupedById.missing(channelId)) result.set('status', 'invalid_id')
else if (channelSearchRequestsBuffer.has(channelId)) result.set('status', 'duplicate')
else if (blocklistGroupedByChannel.has(channelId)) result.set('status', 'blocked')
else if (streamsGroupedByChannel.has(channelId)) result.set('status', 'fulfilled')
if (!channelId) result.status = 'missing_id'
else if (channelsGroupedById.missing(channelId)) result.status = 'invalid_id'
else if (channelSearchRequestsBuffer.has(channelId)) result.status = 'duplicate'
else if (blocklistGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled'
else {
const channelData = channelsGroupedById.get(channelId)
if (channelData.length && channelData[0].closed) result.set('status', 'closed')
if (channelData.length && channelData[0].closed) result.status = 'closed'
}
channelSearchRequestsBuffer.set(channelId, true)
report.add(result.data())
report.add(result)
})
report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending')
@ -157,3 +167,10 @@ async function main() {
}
main()
function truncate(string: string, limit: number = 100) {
if (!string) return string
if (string.length < limit) return string
return string.slice(0, limit) + '...'
}

2
scripts/core/apiClient.ts
View file

@ -41,7 +41,7 @@ export class ApiClient {
}
async download(filename: string) {
const stream = await this.storage.createStream(`/temp/data/${filename}`)
const stream = await this.storage.createStream(`temp/data/${filename}`)
const bar = this.progressBar.create(0, 0, { filename })

3
scripts/core/cliTable.ts
View file

@ -1,9 +1,10 @@
import { Table } from 'console-table-printer'
import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table'
export class CliTable {
table: Table
constructor(options?) {
constructor(options?: ComplexOptions | string[]) {
this.table = new Table(options)
}

2
scripts/core/issueData.ts
View file

@ -18,7 +18,7 @@ export class IssueData {
return Boolean(this._data.get(key))
}
getString(key: string): string {
getString(key: string): string | undefined {
const deleteSymbol = '~'
return this._data.get(key) === deleteSymbol ? '' : this._data.get(key)

2
scripts/core/issueLoader.ts
View file

@ -16,7 +16,7 @@ export class IssueLoader {
}
let issues: object[] = []
if (TESTING) {
issues = (await import('../../tests/__data__/input/issues/all.js')).default
issues = (await import('../../tests/__data__/input/playlist_update/issues.js')).default
} else {
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
owner: OWNER,

10
scripts/core/issueParser.ts
View file

@ -3,11 +3,10 @@ import { Issue } from '../models'
import { IssueData } from './issueData'
const FIELDS = new Dictionary({
'Stream ID': 'streamId',
'Channel ID': 'channelId',
'Channel ID (required)': 'channelId',
'Feed ID': 'feedId',
'Stream URL': 'streamUrl',
'Stream URL (optional)': 'streamUrl',
'Stream URL (required)': 'streamUrl',
'Broken Link': 'brokenLinks',
'Broken Links': 'brokenLinks',
Label: 'label',
@ -18,8 +17,7 @@ const FIELDS = new Dictionary({
'HTTP Referrer': 'httpReferrer',
'What happened to the stream?': 'reason',
Reason: 'reason',
Notes: 'notes',
'Notes (optional)': 'notes'
Notes: 'notes'
})
export class IssueParser {
@ -30,7 +28,7 @@ export class IssueParser {
fields.forEach((field: string) => {
const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
let _label = parsed.shift()
_label = _label ? _label.trim() : ''
_label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
let _value = parsed.join('\r\n')
_value = _value ? _value.trim() : ''

1
scripts/core/logParser.ts
View file

@ -1,4 +1,5 @@
export type LogItem = {
type: string
filepath: string
count: number
}

49
scripts/core/playlistParser.ts
View file

@ -1,12 +1,22 @@
import { Collection, Storage } from '@freearhey/core'
import { Collection, Storage, Dictionary } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { Stream } from '../models'
type PlaylistPareserProps = {
storage: Storage
feedsGroupedByChannelId: Dictionary
channelsGroupedById: Dictionary
}
export class PlaylistParser {
storage: Storage
feedsGroupedByChannelId: Dictionary
channelsGroupedById: Dictionary
constructor({ storage }: { storage: Storage }) {
constructor({ storage, feedsGroupedByChannelId, channelsGroupedById }: PlaylistPareserProps) {
this.storage = storage
this.feedsGroupedByChannelId = feedsGroupedByChannelId
this.channelsGroupedById = channelsGroupedById
}
async parse(files: string[]): Promise<Collection> {
@ -21,41 +31,18 @@ export class PlaylistParser {
}
async parseFile(filepath: string): Promise<Collection> {
const streams = new Collection()
const content = await this.storage.load(filepath)
const parsed: parser.Playlist = parser.parse(content)
parsed.items.forEach((item: parser.PlaylistItem) => {
const { name, label, quality } = parseTitle(item.name)
const stream = new Stream({
channel: item.tvg.id,
name,
label,
quality,
filepath,
line: item.line,
url: item.url,
httpReferrer: item.http.referrer,
httpUserAgent: item.http['user-agent']
})
const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
const stream = new Stream(data)
.withFeed(this.feedsGroupedByChannelId)
.withChannel(this.channelsGroupedById)
.setFilepath(filepath)
streams.add(stream)
return stream
})
return streams
}
}
function parseTitle(title: string): { name: string; label: string; quality: string } {
const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
return { name: title, label, quality }
}
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}

6
scripts/core/streamTester.ts
View file

@ -11,15 +11,15 @@ export class StreamTester {
async test(stream: Stream) {
if (TESTING) {
const results = (await import('../../tests/__data__/input/test_results/all.js')).default
const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default
return results[stream.url]
} else {
return this.checker.checkStream({
url: stream.url,
http: {
referrer: stream.httpReferrer,
'user-agent': stream.httpUserAgent
referrer: stream.getHttpReferrer(),
'user-agent': stream.getHttpUserAgent()
}
})
}

16
scripts/generators/categoriesGenerator.ts
View file

@ -29,11 +29,7 @@ export class CategoriesGenerator implements Generator {
const categoryStreams = streams
.filter((stream: Stream) => stream.hasCategory(category))
.map((stream: Stream) => {
const streamCategories = stream.categories
.map((category: Category) => category.name)
.sort()
const groupTitle = stream.categories ? streamCategories.join(';') : ''
stream.groupTitle = groupTitle
stream.groupTitle = stream.getCategoryNames().join(';')
return stream
})
@ -41,13 +37,17 @@ export class CategoriesGenerator implements Generator {
const playlist = new Playlist(categoryStreams, { public: true })
const filepath = `categories/${category.id}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() })
)
})
const undefinedStreams = streams.filter((stream: Stream) => stream.noCategories())
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories())
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'categories/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() })
)
}
}

64
scripts/generators/countriesGenerator.ts
View file

@ -1,12 +1,10 @@
import { Generator } from './generator'
import { Collection, Storage, Logger } from '@freearhey/core'
import { Country, Region, Subdivision, Stream, Playlist } from '../models'
import { Country, Subdivision, Stream, Playlist } from '../models'
import { PUBLIC_DIR } from '../constants'
type CountriesGeneratorProps = {
streams: Collection
regions: Collection
subdivisions: Collection
countries: Collection
logger: Logger
}
@ -14,55 +12,37 @@ type CountriesGeneratorProps = {
export class CountriesGenerator implements Generator {
streams: Collection
countries: Collection
regions: Collection
subdivisions: Collection
storage: Storage
logger: Logger
constructor({ streams, countries, regions, subdivisions, logger }: CountriesGeneratorProps) {
constructor({ streams, countries, logger }: CountriesGeneratorProps) {
this.streams = streams
this.countries = countries
this.regions = regions
this.subdivisions = subdivisions
this.storage = new Storage(PUBLIC_DIR)
this.logger = logger
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy([stream => stream.getTitle()])
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
const regions = this.regions.filter((region: Region) => region.code !== 'INT')
this.countries.forEach(async (country: Country) => {
const countrySubdivisions = this.subdivisions.filter(
(subdivision: Subdivision) => subdivision.country === country.code
const countryStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInCountry(country)
)
const countrySubdivisionsCodes = countrySubdivisions.map(
(subdivision: Subdivision) => `s/${subdivision.code}`
)
const countryAreaCodes = regions
.filter((region: Region) => region.countries.includes(country.code))
.map((region: Region) => `r/${region.code}`)
.concat(countrySubdivisionsCodes)
.add(`c/${country.code}`)
const countryStreams = streams.filter(stream =>
stream.broadcastArea.intersects(countryAreaCodes)
)
if (countryStreams.isEmpty()) return
const playlist = new Playlist(countryStreams, { public: true })
const filepath = `countries/${country.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() })
)
countrySubdivisions.forEach(async (subdivision: Subdivision) => {
const subdivisionStreams = streams.filter(stream =>
stream.broadcastArea.includes(`s/${subdivision.code}`)
country.getSubdivisions().forEach(async (subdivision: Subdivision) => {
const subdivisionStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInSubdivision(subdivision)
)
if (subdivisionStreams.isEmpty()) return
@ -70,16 +50,22 @@ export class CountriesGenerator implements Generator {
const playlist = new Playlist(subdivisionStreams, { public: true })
const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() })
)
})
})
const internationalStreams = streams.filter(stream => stream.isInternational())
if (internationalStreams.notEmpty()) {
const playlist = new Playlist(internationalStreams, { public: true })
const filepath = 'countries/int.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
}
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
const undefinedPlaylist = new Playlist(undefinedStreams, { public: true })
const undefinedFilepath = 'countries/undefined.m3u'
await this.storage.save(undefinedFilepath, undefinedPlaylist.toString())
this.logger.info(
JSON.stringify({
type: 'country',
filepath: undefinedFilepath,
count: undefinedPlaylist.streams.count()
})
)
}
}

6
scripts/generators/indexCategoryGenerator.ts
View file

@ -26,14 +26,14 @@ export class IndexCategoryGenerator implements Generator {
let groupedStreams = new Collection()
streams.forEach((stream: Stream) => {
if (stream.noCategories()) {
if (!stream.hasCategories()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.categories.forEach((category: Category) => {
stream.getCategories().forEach((category: Category) => {
const streamClone = stream.clone()
streamClone.groupTitle = category.name
groupedStreams.push(streamClone)
@ -48,6 +48,6 @@ export class IndexCategoryGenerator implements Generator {
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.category.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

59
scripts/generators/indexCountryGenerator.ts
View file

@ -1,29 +1,20 @@
import { Generator } from './generator'
import { Collection, Storage, Logger } from '@freearhey/core'
import { Stream, Playlist, Country, Subdivision, Region } from '../models'
import { Stream, Playlist, Country } from '../models'
import { PUBLIC_DIR } from '../constants'
type IndexCountryGeneratorProps = {
streams: Collection
regions: Collection
countries: Collection
subdivisions: Collection
logger: Logger
}
export class IndexCountryGenerator implements Generator {
streams: Collection
countries: Collection
regions: Collection
subdivisions: Collection
storage: Storage
logger: Logger
constructor({ streams, regions, countries, subdivisions, logger }: IndexCountryGeneratorProps) {
constructor({ streams, logger }: IndexCountryGeneratorProps) {
this.streams = streams
this.countries = countries
this.regions = regions
this.subdivisions = subdivisions
this.storage = new Storage(PUBLIC_DIR)
this.logger = logger
}
@ -32,10 +23,10 @@ export class IndexCountryGenerator implements Generator {
let groupedStreams = new Collection()
this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
.forEach(stream => {
if (stream.noBroadcastArea()) {
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasBroadcastArea()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
@ -48,7 +39,7 @@ export class IndexCountryGenerator implements Generator {
groupedStreams.add(streamClone)
}
this.getStreamBroadcastCountries(stream).forEach((country: Country) => {
stream.getBroadcastCountries().forEach((country: Country) => {
const streamClone = stream.clone()
streamClone.groupTitle = country.name
groupedStreams.add(streamClone)
@ -65,40 +56,6 @@ export class IndexCountryGenerator implements Generator {
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.country.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
}
getStreamBroadcastCountries(stream: Stream) {
const groupedRegions = this.regions.keyBy((region: Region) => region.code)
const groupedCountries = this.countries.keyBy((country: Country) => country.code)
const groupedSubdivisions = this.subdivisions.keyBy(
(subdivision: Subdivision) => subdivision.code
)
let broadcastCountries = new Collection()
stream.broadcastArea.forEach(broadcastAreaCode => {
const [type, code] = broadcastAreaCode.split('/')
switch (type) {
case 'c':
broadcastCountries.add(code)
break
case 'r':
if (code !== 'INT' && groupedRegions.has(code)) {
broadcastCountries = broadcastCountries.concat(groupedRegions.get(code).countries)
}
break
case 's':
if (groupedSubdivisions.has(code)) {
broadcastCountries.add(groupedSubdivisions.get(code).country)
}
break
}
})
return broadcastCountries
.uniq()
.map(code => groupedCountries.get(code))
.filter(Boolean)
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

2
scripts/generators/indexGenerator.ts
View file

@ -27,6 +27,6 @@ export class IndexGenerator implements Generator {
const playlist = new Playlist(sfwStreams, { public: true })
const filepath = 'index.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

12
scripts/generators/indexLanguageGenerator.ts
View file

@ -22,17 +22,17 @@ export class IndexLanguageGenerator implements Generator {
async generate(): Promise<void> {
let groupedStreams = new Collection()
this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
.forEach(stream => {
if (stream.noLanguages()) {
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasLanguages()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.languages.forEach((language: Language) => {
stream.getLanguages().forEach((language: Language) => {
const streamClone = stream.clone()
streamClone.groupTitle = language.name
groupedStreams.add(streamClone)
@ -47,6 +47,6 @@ export class IndexLanguageGenerator implements Generator {
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.language.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

2
scripts/generators/indexNsfwGenerator.ts
View file

@ -25,6 +25,6 @@ export class IndexNsfwGenerator implements Generator {
const playlist = new Playlist(allStreams, { public: true })
const filepath = 'index.nsfw.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

34
scripts/generators/indexRegionGenerator.ts
View file

@ -28,14 +28,14 @@ export class IndexRegionGenerator implements Generator {
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (stream.noBroadcastArea()) {
if (!stream.hasBroadcastArea()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.push(streamClone)
return
}
this.getStreamRegions(stream).forEach((region: Region) => {
stream.getBroadcastRegions().forEach((region: Region) => {
const streamClone = stream.clone()
streamClone.groupTitle = region.name
groupedStreams.push(streamClone)
@ -50,34 +50,6 @@ export class IndexRegionGenerator implements Generator {
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.region.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
}
getStreamRegions(stream: Stream) {
let streamRegions = new Collection()
stream.broadcastArea.forEach(broadcastAreaCode => {
const [type, code] = broadcastAreaCode.split('/')
switch (type) {
case 'r':
const groupedRegions = this.regions.keyBy((region: Region) => region.code)
streamRegions.add(groupedRegions.get(code))
break
case 's':
const [countryCode] = code.split('-')
const subdivisionRegions = this.regions.filter((region: Region) =>
region.countries.includes(countryCode)
)
streamRegions = streamRegions.concat(subdivisionRegions)
break
case 'c':
const countryRegions = this.regions.filter((region: Region) =>
region.countries.includes(code)
)
streamRegions = streamRegions.concat(countryRegions)
break
}
})
return streamRegions
this.logger.info(JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }))
}
}

19
scripts/generators/languagesGenerator.ts
View file

@ -18,35 +18,40 @@ export class LanguagesGenerator implements Generator {
async generate(): Promise<void> {
const streams = this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
let languages = new Collection()
streams.forEach((stream: Stream) => {
languages = languages.concat(stream.languages)
languages = languages.concat(stream.getLanguages())
})
languages
.filter(Boolean)
.uniqBy((language: Language) => language.code)
.orderBy((language: Language) => language.name)
.forEach(async (language: Language) => {
const languageStreams = streams.filter(stream => stream.hasLanguage(language))
const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language))
if (languageStreams.isEmpty()) return
const playlist = new Playlist(languageStreams, { public: true })
const filepath = `languages/${language.code}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() })
)
})
const undefinedStreams = streams.filter(stream => stream.noLanguages())
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages())
if (undefinedStreams.isEmpty()) return
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'languages/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() })
)
}
}

46
scripts/generators/regionsGenerator.ts
View file

@ -1,53 +1,61 @@
import { Generator } from './generator'
import { Collection, Storage, Logger } from '@freearhey/core'
import { Playlist, Subdivision, Region } from '../models'
import { Playlist, Region, Stream } from '../models'
import { PUBLIC_DIR } from '../constants'
type RegionsGeneratorProps = {
streams: Collection
regions: Collection
subdivisions: Collection
logger: Logger
}
export class RegionsGenerator implements Generator {
streams: Collection
regions: Collection
subdivisions: Collection
storage: Storage
logger: Logger
constructor({ streams, regions, subdivisions, logger }: RegionsGeneratorProps) {
constructor({ streams, regions, logger }: RegionsGeneratorProps) {
this.streams = streams
this.regions = regions
this.subdivisions = subdivisions
this.storage = new Storage(PUBLIC_DIR)
this.logger = logger
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
this.regions.forEach(async (region: Region) => {
if (region.code === 'INT') return
if (region.isWorldwide()) return
const regionSubdivisionsCodes = this.subdivisions
.filter((subdivision: Subdivision) => region.countries.indexOf(subdivision.country) > -1)
.map((subdivision: Subdivision) => `s/${subdivision.code}`)
const regionCodes = region.countries
.map((code: string) => `c/${code}`)
.concat(regionSubdivisionsCodes)
.add(`r/${region.code}`)
const regionStreams = streams.filter(stream => stream.broadcastArea.intersects(regionCodes))
const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region))
const playlist = new Playlist(regionStreams, { public: true })
const filepath = `regions/${region.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ filepath, count: playlist.streams.count() }))
this.logger.info(
JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() })
)
})
const internationalStreams = streams.filter((stream: Stream) => stream.isInternational())
const internationalPlaylist = new Playlist(internationalStreams, { public: true })
const internationalFilepath = 'regions/int.m3u'
await this.storage.save(internationalFilepath, internationalPlaylist.toString())
this.logger.info(
JSON.stringify({
type: 'region',
filepath: internationalFilepath,
count: internationalPlaylist.streams.count()
})
)
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'regions/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logger.info(JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }))
}
}

10
scripts/models/blocked.ts
View file

@ -5,13 +5,13 @@ type BlockedProps = {
}
export class Blocked {
channel: string
channelId: string
reason: string
ref: string
constructor({ ref, reason, channel }: BlockedProps) {
this.channel = channel
this.reason = reason
this.ref = ref
constructor(data: BlockedProps) {
this.channelId = data.channel
this.reason = data.reason
this.ref = data.ref
}
}

11
scripts/models/broadcastArea.ts Normal file
View file

@ -0,0 +1,11 @@
type BroadcastAreaProps = {
code: string
}
export class BroadcastArea {
code: string
constructor(data: BroadcastAreaProps) {
this.code = data.code
}
}

8
scripts/models/category.ts
View file

@ -1,4 +1,4 @@
type CategoryProps = {
type CategoryData = {
id: string
name: string
}
@ -7,8 +7,8 @@ export class Category {
id: string
name: string
constructor({ id, name }: CategoryProps) {
this.id = id
this.name = name
constructor(data: CategoryData) {
this.id = data.id
this.name = data.name
}
}

135
scripts/models/channel.ts
View file

@ -1,17 +1,16 @@
import { Collection } from '@freearhey/core'
import { Collection, Dictionary } from '@freearhey/core'
import { Category, Country, Subdivision } from './index'
type ChannelProps = {
type ChannelData = {
id: string
name: string
alt_names: string[]
network: string
owners: string[]
owners: Collection
country: string
subdivision: string
city: string
broadcast_area: string[]
languages: string[]
categories: string[]
categories: Collection
is_nsfw: boolean
launched: string
closed: string
@ -24,56 +23,86 @@ export class Channel {
id: string
name: string
altNames: Collection
network: string
network?: string
owners: Collection
country: string
subdivision: string
city: string
broadcastArea: Collection
languages: Collection
categories: Collection
countryCode: string
country?: Country
subdivisionCode?: string
subdivision?: Subdivision
cityName?: string
categoryIds: Collection
categories?: Collection
isNSFW: boolean
launched: string
closed: string
replacedBy: string
website: string
launched?: string
closed?: string
replacedBy?: string
website?: string
logo: string
constructor({
id,
name,
alt_names,
network,
owners,
country,
subdivision,
city,
broadcast_area,
languages,
categories,
is_nsfw,
launched,
closed,
replaced_by,
website,
logo
}: ChannelProps) {
this.id = id
this.name = name
this.altNames = new Collection(alt_names)
this.network = network
this.owners = new Collection(owners)
this.country = country
this.subdivision = subdivision
this.city = city
this.broadcastArea = new Collection(broadcast_area)
this.languages = new Collection(languages)
this.categories = new Collection(categories)
this.isNSFW = is_nsfw
this.launched = launched
this.closed = closed
this.replacedBy = replaced_by
this.website = website
this.logo = logo
constructor(data: ChannelData) {
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.alt_names)
this.network = data.network || undefined
this.owners = new Collection(data.owners)
this.countryCode = data.country
this.subdivisionCode = data.subdivision || undefined
this.cityName = data.city || undefined
this.categoryIds = new Collection(data.categories)
this.isNSFW = data.is_nsfw
this.launched = data.launched || undefined
this.closed = data.closed || undefined
this.replacedBy = data.replaced_by || undefined
this.website = data.website || undefined
this.logo = data.logo
}
withSubdivision(subdivisionsGroupedByCode: Dictionary): this {
if (!this.subdivisionCode) return this
this.subdivision = subdivisionsGroupedByCode.get(this.subdivisionCode)
return this
}
withCountry(countriesGroupedByCode: Dictionary): this {
this.country = countriesGroupedByCode.get(this.countryCode)
return this
}
withCategories(groupedCategories: Dictionary): this {
this.categories = this.categoryIds
.map((id: string) => groupedCategories.get(id))
.filter(Boolean)
return this
}
getCountry(): Country | undefined {
return this.country
}
getSubdivision(): Subdivision | undefined {
return this.subdivision
}
getCategories(): Collection {
return this.categories || new Collection()
}
hasCategories(): boolean {
return !!this.categories && this.categories.notEmpty()
}
hasCategory(category: Category): boolean {
return (
!!this.categories &&
this.categories.includes((_category: Category) => _category.id === category.id)
)
}
isSFW(): boolean {
return this.isNSFW === false
}
}

54
scripts/models/country.ts
View file

@ -1,20 +1,58 @@
type CountryProps = {
import { Collection, Dictionary } from '@freearhey/core'
import { Region, Language } from '.'
type CountryData = {
code: string
name: string
languages: string[]
lang: string
flag: string
}
export class Country {
code: string
name: string
languages: string[]
flag: string
languageCode: string
language?: Language
subdivisions?: Collection
regions?: Collection
constructor({ code, name, languages, flag }: CountryProps) {
this.code = code
this.name = name
this.languages = languages
this.flag = flag
constructor(data: CountryData) {
this.code = data.code
this.name = data.name
this.flag = data.flag
this.languageCode = data.lang
}
withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this {
this.subdivisions = subdivisionsGroupedByCountryCode.get(this.code) || new Collection()
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter(
(region: Region) => region.code !== 'INT' && region.includesCountryCode(this.code)
)
return this
}
withLanguage(languagesGroupedByCode: Dictionary): this {
this.language = languagesGroupedByCode.get(this.languageCode)
return this
}
getLanguage(): Language | undefined {
return this.language
}
getRegions(): Collection {
return this.regions || new Collection()
}
getSubdivisions(): Collection {
return this.subdivisions || new Collection()
}
}

196
scripts/models/feed.ts Normal file
View file

@ -0,0 +1,196 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Country, Language, Region, Channel, Subdivision } from './index'
type FeedData = {
channel: string
id: string
name: string
is_main: boolean
broadcast_area: Collection
languages: Collection
timezones: Collection
video_format: string
}
export class Feed {
channelId: string
channel?: Channel
id: string
name: string
isMain: boolean
broadcastAreaCodes: Collection
broadcastCountryCodes: Collection
broadcastCountries?: Collection
broadcastRegionCodes: Collection
broadcastRegions?: Collection
broadcastSubdivisionCodes: Collection
broadcastSubdivisions?: Collection
languageCodes: Collection
languages?: Collection
timezoneIds: Collection
timezones?: Collection
videoFormat: string
constructor(data: FeedData) {
this.channelId = data.channel
this.id = data.id
this.name = data.name
this.isMain = data.is_main
this.broadcastAreaCodes = new Collection(data.broadcast_area)
this.languageCodes = new Collection(data.languages)
this.timezoneIds = new Collection(data.timezones)
this.videoFormat = data.video_format
this.broadcastCountryCodes = new Collection()
this.broadcastRegionCodes = new Collection()
this.broadcastSubdivisionCodes = new Collection()
this.broadcastAreaCodes.forEach((areaCode: string) => {
const [type, code] = areaCode.split('/')
switch (type) {
case 'c':
this.broadcastCountryCodes.add(code)
break
case 'r':
this.broadcastRegionCodes.add(code)
break
case 's':
this.broadcastSubdivisionCodes.add(code)
break
}
})
}
withChannel(channelsGroupedById: Dictionary): this {
this.channel = channelsGroupedById.get(this.channelId)
return this
}
withLanguages(languagesGroupedByCode: Dictionary): this {
this.languages = this.languageCodes
.map((code: string) => languagesGroupedByCode.get(code))
.filter(Boolean)
return this
}
withTimezones(timezonesGroupedById: Dictionary): this {
this.timezones = this.timezoneIds
.map((id: string) => timezonesGroupedById.get(id))
.filter(Boolean)
return this
}
withBroadcastSubdivisions(subdivisionsGroupedByCode: Dictionary): this {
this.broadcastSubdivisions = this.broadcastSubdivisionCodes.map((code: string) =>
subdivisionsGroupedByCode.get(code)
)
return this
}
withBroadcastCountries(
countriesGroupedByCode: Dictionary,
regionsGroupedByCode: Dictionary,
subdivisionsGroupedByCode: Dictionary
): this {
let broadcastCountries = new Collection()
if (this.isInternational()) {
this.broadcastCountries = broadcastCountries
return this
}
this.broadcastCountryCodes.forEach((code: string) => {
broadcastCountries.add(countriesGroupedByCode.get(code))
})
this.broadcastRegionCodes.forEach((code: string) => {
const region: Region = regionsGroupedByCode.get(code)
broadcastCountries = broadcastCountries.concat(region.countryCodes)
})
this.broadcastSubdivisionCodes.forEach((code: string) => {
const subdivision: Subdivision = subdivisionsGroupedByCode.get(code)
broadcastCountries.add(countriesGroupedByCode.get(subdivision.countryCode))
})
this.broadcastCountries = broadcastCountries.uniq().filter(Boolean)
return this
}
withBroadcastRegions(regions: Collection, regionsGroupedByCode: Dictionary): this {
if (!this.broadcastCountries) return this
const countriesCodes = this.broadcastCountries.map((country: Country) => country.code)
const broadcastRegions = regions.filter((region: Region) =>
region.countryCodes.intersects(countriesCodes)
)
if (this.isInternational()) broadcastRegions.add(regionsGroupedByCode.get('INT'))
this.broadcastRegions = broadcastRegions
return this
}
hasBroadcastArea(): boolean {
return (
this.isInternational() || (!!this.broadcastCountries && this.broadcastCountries.notEmpty())
)
}
getBroadcastCountries(): Collection {
return this.broadcastCountries || new Collection()
}
getBroadcastRegions(): Collection {
return this.broadcastRegions || new Collection()
}
getTimezones(): Collection {
return this.timezones || new Collection()
}
getLanguages(): Collection {
return this.languages || new Collection()
}
hasLanguages(): boolean {
return !!this.languages && this.languages.notEmpty()
}
hasLanguage(language: Language): boolean {
return (
!!this.languages &&
this.languages.includes((_language: Language) => _language.code === language.code)
)
}
isInternational(): boolean {
return this.broadcastAreaCodes.includes('r/INT')
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
if (this.isInternational()) return false
return this.broadcastSubdivisionCodes.includes(subdivision.code)
}
isBroadcastInCountry(country: Country): boolean {
if (this.isInternational()) return false
return this.getBroadcastCountries().includes(
(_country: Country) => _country.code === country.code
)
}
isBroadcastInRegion(region: Region): boolean {
if (this.isInternational()) return false
return this.getBroadcastRegions().includes((_region: Region) => _region.code === region.code)
}
}

3
scripts/models/index.ts
View file

@ -8,3 +8,6 @@ export * from './language'
export * from './country'
export * from './region'
export * from './subdivision'
export * from './feed'
export * from './broadcastArea'
export * from './timezone'

8
scripts/models/language.ts
View file

@ -1,4 +1,4 @@
type LanguageProps = {
type LanguageData = {
code: string
name: string
}
@ -7,8 +7,8 @@ export class Language {
code: string
name: string
constructor({ code, name }: LanguageProps) {
this.code = code
this.name = name
constructor(data: LanguageData) {
this.code = data.code
this.name = data.name
}
}

47
scripts/models/region.ts
View file

@ -1,6 +1,7 @@
import { Collection } from '@freearhey/core'
import { Collection, Dictionary } from '@freearhey/core'
import { Subdivision } from '.'
type RegionProps = {
type RegionData = {
code: string
name: string
countries: string[]
@ -9,11 +10,43 @@ type RegionProps = {
export class Region {
code: string
name: string
countries: Collection
countryCodes: Collection
countries?: Collection
subdivisions?: Collection
constructor({ code, name, countries }: RegionProps) {
this.code = code
this.name = name
this.countries = new Collection(countries)
constructor(data: RegionData) {
this.code = data.code
this.name = data.name
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesGroupedByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
return this
}
withSubdivisions(subdivisions: Collection): this {
this.subdivisions = subdivisions.filter(
(subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1
)
return this
}
getSubdivisions(): Collection {
return this.subdivisions || new Collection()
}
getCountries(): Collection {
return this.countries || new Collection()
}
includesCountryCode(code: string): boolean {
return this.countryCodes.includes((countryCode: string) => countryCode === code)
}
isWorldwide(): boolean {
return this.code === 'INT'
}
}

319
scripts/models/stream.ts
View file

@ -1,64 +1,166 @@
import { URL, Collection } from '@freearhey/core'
import { Category, Language } from './index'
type StreamProps = {
name: string
url: string
filepath: string
line: number
channel?: string
httpReferrer?: string
httpUserAgent?: string
label?: string
quality?: string
}
import { URL, Collection, Dictionary } from '@freearhey/core'
import { Feed, Channel, Category, Region, Subdivision, Country, Language } from './index'
import parser from 'iptv-playlist-parser'
export class Stream {
channel: string
filepath: string
line: number
httpReferrer: string
label: string
name: string
quality: string
url: string
httpUserAgent: string
logo: string
broadcastArea: Collection
categories: Collection
languages: Collection
isNSFW: boolean
id?: string
groupTitle: string
channelId?: string
channel?: Channel
feedId?: string
feed?: Feed
filepath?: string
line: number
label?: string
quality?: string
httpReferrer?: string
httpUserAgent?: string
removed: boolean = false
constructor({
channel,
filepath,
line,
httpReferrer,
label,
name,
quality,
url,
httpUserAgent
}: StreamProps) {
this.channel = channel || ''
this.filepath = filepath
this.line = line
this.httpReferrer = httpReferrer || ''
this.label = label || ''
constructor(data: parser.PlaylistItem) {
if (!data.name) throw new Error('"name" property is required')
if (!data.url) throw new Error('"url" property is required')
const [channelId, feedId] = data.tvg.id.split('@')
const { name, label, quality } = parseTitle(data.name)
this.id = data.tvg.id || undefined
this.feedId = feedId || undefined
this.channelId = channelId || undefined
this.line = data.line
this.label = label || undefined
this.name = name
this.quality = quality || ''
this.url = url
this.httpUserAgent = httpUserAgent || ''
this.logo = ''
this.broadcastArea = new Collection()
this.categories = new Collection()
this.languages = new Collection()
this.isNSFW = false
this.quality = quality || undefined
this.url = data.url
this.httpReferrer = data.http.referrer || undefined
this.httpUserAgent = data.http['user-agent'] || undefined
this.groupTitle = 'Undefined'
}
withChannel(channelsGroupedById: Dictionary): this {
if (!this.channelId) return this
this.channel = channelsGroupedById.get(this.channelId)
return this
}
withFeed(feedsGroupedByChannelId: Dictionary): this {
if (!this.channelId) return this
const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || []
if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId)
if (!this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain)
return this
}
setId(id: string): this {
this.id = id
return this
}
setChannelId(channelId: string): this {
this.channelId = channelId
return this
}
setFeedId(feedId: string | undefined): this {
this.feedId = feedId
return this
}
setLabel(label: string): this {
this.label = label
return this
}
setQuality(quality: string): this {
this.quality = quality
return this
}
setHttpUserAgent(httpUserAgent: string): this {
this.httpUserAgent = httpUserAgent
return this
}
setHttpReferrer(httpReferrer: string): this {
this.httpReferrer = httpReferrer
return this
}
setFilepath(filepath: string): this {
this.filepath = filepath
return this
}
updateFilepath(): this {
if (!this.channel) return this
this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u`
return this
}
getFilepath(): string {
return this.filepath || ''
}
getHttpReferrer(): string {
return this.httpReferrer || ''
}
getHttpUserAgent(): string {
return this.httpUserAgent || ''
}
getQuality(): string {
return this.quality || ''
}
hasQuality(): boolean {
return !!this.quality
}
getHorizontalResolution(): number {
if (!this.hasQuality()) return 0
return parseInt(this.getQuality().replace(/p|i/, ''))
}
updateName(): this {
if (!this.channel) return this
this.name = this.channel.name
if (this.feed && !this.feed.isMain) {
this.name += ` ${this.feed.name}`
}
return this
}
updateId(): this {
if (!this.channel) return this
if (this.feed) {
this.id = `${this.channel.id}@${this.feed.id}`
} else {
this.id = this.channel.id
}
return this
}
normalizeURL() {
const url = new URL(this.url)
@ -81,36 +183,75 @@ export class Stream {
return !!this.channel
}
hasCategories(): boolean {
return this.categories.notEmpty()
getBroadcastRegions(): Collection {
return this.feed ? this.feed.getBroadcastRegions() : new Collection()
}
noCategories(): boolean {
return this.categories.isEmpty()
getBroadcastCountries(): Collection {
return this.feed ? this.feed.getBroadcastCountries() : new Collection()
}
hasCategory(category: Category): boolean {
return this.categories.includes((_category: Category) => _category.id === category.id)
}
noLanguages(): boolean {
return this.languages.isEmpty()
}
hasLanguage(language: Language): boolean {
return this.languages.includes((_language: Language) => _language.code === language.code)
}
noBroadcastArea(): boolean {
return this.broadcastArea.isEmpty()
}
isInternational(): boolean {
return this.broadcastArea.includes('r/INT')
hasBroadcastArea(): boolean {
return this.feed ? this.feed.hasBroadcastArea() : false
}
isSFW(): boolean {
return this.isNSFW === false
return this.channel ? this.channel.isSFW() : true
}
hasCategories(): boolean {
return this.channel ? this.channel.hasCategories() : false
}
hasCategory(category: Category): boolean {
return this.channel ? this.channel.hasCategory(category) : false
}
getCategoryNames(): string[] {
return this.getCategories()
.map((category: Category) => category.name)
.sort()
.all()
}
getCategories(): Collection {
return this.channel ? this.channel.getCategories() : new Collection()
}
getLanguages(): Collection {
return this.feed ? this.feed.getLanguages() : new Collection()
}
hasLanguages() {
return this.feed ? this.feed.hasLanguages() : false
}
hasLanguage(language: Language) {
return this.feed ? this.feed.hasLanguage(language) : false
}
getBroadcastAreaCodes(): Collection {
return this.feed ? this.feed.broadcastAreaCodes : new Collection()
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false
}
isBroadcastInCountry(country: Country): boolean {
return this.feed ? this.feed.isBroadcastInCountry(country) : false
}
isBroadcastInRegion(region: Region): boolean {
return this.feed ? this.feed.isBroadcastInRegion(region) : false
}
isInternational(): boolean {
return this.feed ? this.feed.isInternational() : false
}
getLogo(): string {
return this?.channel?.logo || ''
}
getTitle(): string {
@ -127,15 +268,25 @@ export class Stream {
return title
}
getLabel(): string {
return this.label || ''
}
getId(): string {
return this.id || ''
}
data() {
return {
id: this.id,
channel: this.channel,
feed: this.feed,
filepath: this.filepath,
httpReferrer: this.httpReferrer,
label: this.label,
name: this.name,
quality: this.quality,
url: this.url,
httpReferrer: this.httpReferrer,
httpUserAgent: this.httpUserAgent,
line: this.line
}
@ -143,7 +294,8 @@ export class Stream {
toJSON() {
return {
channel: this.channel || null,
channel: this.channelId || null,
feed: this.feedId || null,
url: this.url,
referrer: this.httpReferrer || null,
user_agent: this.httpUserAgent || null
@ -151,10 +303,10 @@ export class Stream {
}
toString(options: { public: boolean }) {
let output = `#EXTINF:-1 tvg-id="${this.channel}"`
let output = `#EXTINF:-1 tvg-id="${this.getId()}"`
if (options.public) {
output += ` tvg-logo="${this.logo}" group-title="${this.groupTitle}"`
output += ` tvg-logo="${this.getLogo()}" group-title="${this.groupTitle}"`
}
if (this.httpReferrer) {
@ -180,3 +332,16 @@ export class Stream {
return output
}
}
function parseTitle(title: string): { name: string; label: string; quality: string } {
const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
return { name: title, label, quality }
}
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}

22
scripts/models/subdivision.ts
View file

@ -1,4 +1,7 @@
type SubdivisionProps = {
import { Dictionary } from '@freearhey/core'
import { Country } from '.'
type SubdivisionData = {
code: string
name: string
country: string
@ -7,11 +10,18 @@ type SubdivisionProps = {
export class Subdivision {
code: string
name: string
country: string
countryCode: string
country?: Country
constructor({ code, name, country }: SubdivisionProps) {
this.code = code
this.name = name
this.country = country
constructor(data: SubdivisionData) {
this.code = data.code
this.name = data.name
this.countryCode = data.country
}
withCountry(countriesGroupedByCode: Dictionary): this {
this.country = countriesGroupedByCode.get(this.countryCode)
return this
}
}

30
scripts/models/timezone.ts Normal file
View file

@ -0,0 +1,30 @@
import { Collection, Dictionary } from '@freearhey/core'
type TimezoneData = {
id: string
utc_offset: string
countries: string[]
}
export class Timezone {
id: string
utcOffset: string
countryCodes: Collection
countries?: Collection
constructor(data: TimezoneData) {
this.id = data.id
this.utcOffset = data.utc_offset
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesGroupedByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesGroupedByCode.get(code))
return this
}
getCountries(): Collection {
return this.countries || new Collection()
}
}

8
scripts/tables/categoryTable.ts
View file

@ -11,6 +11,7 @@ export class CategoryTable implements Table {
const dataStorage = new Storage(DATA_DIR)
const categoriesContent = await dataStorage.json('categories.json')
const categories = new Collection(categoriesContent).map(data => new Category(data))
const categoriesGroupedById = categories.keyBy((category: Category) => category.id)
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
@ -19,13 +20,12 @@ export class CategoryTable implements Table {
let data = new Collection()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.filepath.includes('categories/'))
.filter((logItem: LogItem) => logItem.type === 'category')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const categoryId = file.name()
const category: Category = categories.first(
(category: Category) => category.id === categoryId
)
const category: Category = categoriesGroupedById.get(categoryId)
data.add([
category ? category.name : 'ZZ',
category ? category.name : 'Undefined',

47
scripts/tables/countryTable.ts
View file

@ -12,34 +12,31 @@ export class CountryTable implements Table {
const countriesContent = await dataStorage.json('countries.json')
const countries = new Collection(countriesContent).map(data => new Country(data))
const countriesGroupedByCode = countries.keyBy((country: Country) => country.code)
const subdivisionsContent = await dataStorage.json('subdivisions.json')
const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
const subdivisionsGroupedByCode = subdivisions.keyBy(
(subdivision: Subdivision) => subdivision.code
)
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
let data = new Collection()
parser
.parse(generatorsLog)
.filter(
(logItem: LogItem) =>
logItem.filepath.includes('countries/') || logItem.filepath.includes('subdivisions/')
)
parsed
.filter((logItem: LogItem) => logItem.type === 'subdivision')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const code = file.name().toUpperCase()
const [countryCode, subdivisionCode] = code.split('-') || ['', '']
const country = countriesGroupedByCode.get(countryCode)
if (subdivisionCode) {
const subdivision = subdivisions.first(
(subdivision: Subdivision) => subdivision.code === code
)
if (country && subdivisionCode) {
const subdivision = subdivisionsGroupedByCode.get(code)
if (subdivision) {
const country = countries.first(
(country: Country) => country.code === subdivision.country
)
data.add([
`${country.name}/${subdivision.name}`,
`&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;${subdivision.name}`,
@ -47,18 +44,28 @@ export class CountryTable implements Table {
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
}
} else if (countryCode === 'INT') {
}
})
parsed
.filter((logItem: LogItem) => logItem.type === 'country')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const code = file.name().toUpperCase()
const [countryCode] = code.split('-') || ['', '']
const country = countriesGroupedByCode.get(countryCode)
if (country) {
data.add([
'ZZ',
'🌍 International',
country.name,
`${country.flag} ${country.name}`,
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
} else {
const country = countries.first((country: Country) => country.code === countryCode)
data.add([
country.name,
`${country.flag} ${country.name}`,
'ZZ',
'Undefined',
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])

7
scripts/tables/languageTable.ts
View file

@ -11,6 +11,7 @@ export class LanguageTable implements Table {
const dataStorage = new Storage(DATA_DIR)
const languagesContent = await dataStorage.json('languages.json')
const languages = new Collection(languagesContent).map(data => new Language(data))
const languagesGroupedByCode = languages.keyBy((language: Language) => language.code)
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
@ -19,13 +20,11 @@ export class LanguageTable implements Table {
let data = new Collection()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.filepath.includes('languages/'))
.filter((logItem: LogItem) => logItem.type === 'language')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const languageCode = file.name()
const language: Language = languages.first(
(language: Language) => language.code === languageCode
)
const language: Language = languagesGroupedByCode.get(languageCode)
data.add([
language ? language.name : 'ZZ',

20
scripts/tables/regionTable.ts
View file

@ -11,6 +11,7 @@ export class RegionTable implements Table {
const dataStorage = new Storage(DATA_DIR)
const regionsContent = await dataStorage.json('regions.json')
const regions = new Collection(regionsContent).map(data => new Region(data))
const regionsGroupedByCode = regions.keyBy((region: Region) => region.code)
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
@ -19,22 +20,35 @@ export class RegionTable implements Table {
let data = new Collection()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.filepath.includes('regions/'))
.filter((logItem: LogItem) => logItem.type === 'region')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const regionCode = file.name().toUpperCase()
const region: Region = regions.first((region: Region) => region.code === regionCode)
const region: Region = regionsGroupedByCode.get(regionCode)
if (region) {
data.add([
region.name,
region.name,
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
} else {
data.add([
'ZZZ',
'Undefined',
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
}
})
data = data.orderBy(item => item[0])
data = data
.orderBy(item => item[0])
.map(item => {
item.shift()
return item
})
const table = new HTMLTable(data.all(), [
{ name: 'Region', align: 'left' },