mirror of
https://github.com/iptv-org/iptv.git
synced 2025-05-13 02:20:03 -04:00
Merge branch 'master' into remove-duplicates-from-generated-playlists
This commit is contained in:
Aleksandr Statciuk
GitHub
commit
0e6482cf3e
127 changed files with 5814 additions and 6052 deletions
148
scripts/clean.js
Normal file
148
scripts/clean.js
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
const { program } = require('commander')
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
const axios = require('axios')
|
||||
const ProgressBar = require('progress')
|
||||
const https = require('https')
|
||||
const chalk = require('chalk')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Debug mode')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
}),
|
||||
validateStatus: function (status) {
|
||||
return status !== 404
|
||||
}
|
||||
})
|
||||
|
||||
const ignore = ['Geo-blocked', 'Not 24/7']
|
||||
|
||||
const stats = { broken: 0 }
|
||||
|
||||
async function main() {
|
||||
console.info(`\nStarting...`)
|
||||
console.time('Process completed in')
|
||||
if (config.debug) {
|
||||
console.info(chalk.yellow(`INFO: Debug mode enabled\n`))
|
||||
}
|
||||
const playlists = parseIndex()
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url).then(checkStatus).then(savePlaylist).then(done)
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
}
|
||||
|
||||
async function checkStatus(playlist) {
|
||||
let bar
|
||||
if (!config.debug) {
|
||||
bar = new ProgressBar(' Testing: [:bar] :current/:total (:percent) ', {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
}
|
||||
const results = []
|
||||
const total = playlist.channels.length
|
||||
for (const [index, channel] of playlist.channels.entries()) {
|
||||
const current = index + 1
|
||||
const counter = chalk.gray(`[${current}/${total}]`)
|
||||
if (bar) bar.tick()
|
||||
if (
|
||||
(channel.status && ignore.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())) ||
|
||||
(!channel.url.startsWith('http://') && !channel.url.startsWith('https://'))
|
||||
) {
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
} else {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(() => {
|
||||
clearTimeout(timeout)
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
if (err.response && err.response.status === 404) {
|
||||
//console.error(err)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.red('offline')} ${chalk.white(channel.url)}`)
|
||||
stats.broken++
|
||||
}
|
||||
} else {
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
playlist.channels = results
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString({ raw: true })
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated. Removed ${stats.broken} broken links.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.timeEnd('Process completed in')
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -9,10 +9,10 @@ const https = require('https')
|
|||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Debug mode')
|
||||
.option('-r --resolution', 'Parse stream resolution')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--resolution', 'Turn on resolution parser')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 0)
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
|
|
@ -20,21 +20,20 @@ const config = program.opts()
|
|||
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 20000,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
})
|
||||
})
|
||||
|
||||
let globalBuffer = []
|
||||
|
||||
async function main() {
|
||||
console.info('Starting...')
|
||||
console.time('Done in')
|
||||
|
||||
const playlists = parseIndex()
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url)
|
||||
.then(addToBuffer)
|
||||
.then(removeDuplicates)
|
||||
.then(sortChannels)
|
||||
.then(filterChannels)
|
||||
.then(detectResolution)
|
||||
|
|
@ -42,20 +41,11 @@ async function main() {
|
|||
.then(done)
|
||||
}
|
||||
|
||||
if (playlists.length) {
|
||||
await loadPlaylist('channels/unsorted.m3u')
|
||||
.then(removeUnsortedDuplicates)
|
||||
.then(filterChannels)
|
||||
.then(sortChannels)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
console.info(`\nParsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils
|
||||
.filterPlaylists(playlists, config.country, config.exclude)
|
||||
|
|
@ -70,13 +60,6 @@ async function loadPlaylist(url) {
|
|||
return parser.parsePlaylist(url)
|
||||
}
|
||||
|
||||
async function addToBuffer(playlist) {
|
||||
if (playlist.url === 'channels/unsorted.m3u') return playlist
|
||||
globalBuffer = globalBuffer.concat(playlist.channels)
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function sortChannels(playlist) {
|
||||
console.info(` Sorting channels...`)
|
||||
playlist.channels = utils.sortBy(playlist.channels, ['name', 'url'])
|
||||
|
|
@ -94,39 +77,35 @@ async function filterChannels(playlist) {
|
|||
return playlist
|
||||
}
|
||||
|
||||
async function removeDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
let buffer = {}
|
||||
const channels = playlist.channels.filter(i => {
|
||||
const url = utils.removeProtocol(i.url)
|
||||
const result = typeof buffer[url] === 'undefined'
|
||||
if (result) {
|
||||
buffer[url] = true
|
||||
}
|
||||
|
||||
return result
|
||||
})
|
||||
|
||||
playlist.channels = channels
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function detectResolution(playlist) {
|
||||
if (!config.resolution) return playlist
|
||||
const bar = new ProgressBar(' Detecting resolution: [:bar] :current/:total (:percent) ', {
|
||||
console.log(' Detecting resolution...')
|
||||
const bar = new ProgressBar(' Progress: [:bar] :current/:total (:percent) ', {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
const results = []
|
||||
for (const channel of playlist.channels) {
|
||||
bar.tick()
|
||||
const url = channel.url
|
||||
const response = await instance
|
||||
.get(url)
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {})
|
||||
if (response) {
|
||||
if (response.status === 200) {
|
||||
if (!channel.resolution.height) {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
const response = await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(res => {
|
||||
clearTimeout(timeout)
|
||||
|
||||
return res
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
})
|
||||
|
||||
if (response && response.status === 200) {
|
||||
if (/^#EXTM3U/.test(response.data)) {
|
||||
const resolution = parseResolution(response.data)
|
||||
if (resolution) {
|
||||
|
|
@ -159,30 +138,9 @@ function parseResolution(string) {
|
|||
: undefined
|
||||
}
|
||||
|
||||
async function removeUnsortedDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
// locally
|
||||
let buffer = {}
|
||||
let channels = playlist.channels.filter(i => {
|
||||
const url = utils.removeProtocol(i.url)
|
||||
const result = typeof buffer[url] === 'undefined'
|
||||
if (result) buffer[url] = true
|
||||
|
||||
return result
|
||||
})
|
||||
// globally
|
||||
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
|
||||
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
|
||||
if (channels.length === playlist.channels.length) return playlist
|
||||
|
||||
playlist.channels = channels
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString(true)
|
||||
const output = playlist.toString()
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
|
|
@ -200,7 +158,7 @@ async function done() {
|
|||
}
|
||||
|
||||
function finish() {
|
||||
console.info('Done.')
|
||||
console.timeEnd('Done in')
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ class Playlist {
|
|||
.filter(channel => channel.url)
|
||||
}
|
||||
|
||||
toString(short = false) {
|
||||
toString(options = {}) {
|
||||
const config = { raw: false, ...options }
|
||||
let parts = ['#EXTM3U']
|
||||
for (let key in this.header.attrs) {
|
||||
let value = this.header.attrs[key]
|
||||
|
|
@ -43,7 +44,7 @@ class Playlist {
|
|||
|
||||
let output = `${parts.join(' ')}\n`
|
||||
for (let channel of this.channels) {
|
||||
output += channel.toString(short)
|
||||
output += channel.toString(config.raw)
|
||||
}
|
||||
|
||||
return output
|
||||
|
|
@ -77,6 +78,7 @@ class Channel {
|
|||
this.countries = this.parseCountries(data.tvg.country)
|
||||
this.languages = this.parseLanguages(data.tvg.language)
|
||||
this.category = this.parseCategory(data.group.title)
|
||||
this.raw = data.raw
|
||||
}
|
||||
|
||||
parseCountries(string) {
|
||||
|
|
@ -181,15 +183,13 @@ class Channel {
|
|||
return ''
|
||||
}
|
||||
|
||||
getInfo(short = false) {
|
||||
toString(raw = false) {
|
||||
if (raw) return this.raw + '\n'
|
||||
|
||||
this.tvg.country = this.tvg.country.toUpperCase()
|
||||
|
||||
let info = `-1 tvg-id="${this.tvgId}" tvg-name="${this.tvgName}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"`
|
||||
|
||||
if (!short) {
|
||||
info += ` tvg-url="${this.tvgUrl}"`
|
||||
}
|
||||
|
||||
info += ` group-title="${this.category}",${this.name}`
|
||||
|
||||
if (this.resolution.height) {
|
||||
|
|
|
|||
110
scripts/remove-duplicates.js
Normal file
110
scripts/remove-duplicates.js
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
|
||||
let globalBuffer = []
|
||||
|
||||
async function main() {
|
||||
const playlists = parseIndex()
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url)
|
||||
.then(addToBuffer)
|
||||
.then(removeDuplicates)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
}
|
||||
|
||||
if (playlists.length) {
|
||||
await loadPlaylist('channels/unsorted.m3u')
|
||||
.then(removeUnsortedDuplicates)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
}
|
||||
|
||||
async function addToBuffer(playlist) {
|
||||
if (playlist.url === 'channels/unsorted.m3u') return playlist
|
||||
globalBuffer = globalBuffer.concat(playlist.channels)
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function removeDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
let buffer = {}
|
||||
const channels = playlist.channels.filter(i => {
|
||||
const url = utils.removeProtocol(i.url)
|
||||
const result = typeof buffer[url] === 'undefined'
|
||||
if (result) {
|
||||
buffer[url] = true
|
||||
}
|
||||
|
||||
return result
|
||||
})
|
||||
|
||||
playlist.channels = channels
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function removeUnsortedDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
// locally
|
||||
let buffer = {}
|
||||
let channels = playlist.channels.filter(i => {
|
||||
const url = utils.removeProtocol(i.url)
|
||||
const result = typeof buffer[url] === 'undefined'
|
||||
if (result) buffer[url] = true
|
||||
|
||||
return result
|
||||
})
|
||||
// globally
|
||||
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
|
||||
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
|
||||
if (channels.length === playlist.channels.length) return playlist
|
||||
|
||||
playlist.channels = channels
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString({ raw: true })
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.info('Done.')
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
const { program } = require('commander')
|
||||
const utils = require('./utils')
|
||||
const parser = require('./parser')
|
||||
const axios = require('axios')
|
||||
const https = require('https')
|
||||
const ProgressBar = require('progress')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Debug mode')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 20000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
}),
|
||||
validateStatus: function (status) {
|
||||
return (status >= 200 && status < 300) || status === 403
|
||||
}
|
||||
})
|
||||
|
||||
let stats = {
|
||||
playlists: 0,
|
||||
channels: 0,
|
||||
failures: 0
|
||||
}
|
||||
|
||||
async function test() {
|
||||
let items = parser.parseIndex()
|
||||
items = utils.filterPlaylists(items, config.country, config.exclude)
|
||||
|
||||
for (const item of items) {
|
||||
const playlist = parser.parsePlaylist(item.url)
|
||||
const bar = new ProgressBar(`Processing '${item.url}'...:current/:total`, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
|
||||
stats.playlists++
|
||||
|
||||
for (let channel of playlist.channels) {
|
||||
bar.tick()
|
||||
stats.channels++
|
||||
|
||||
if (channel.url.startsWith('rtmp://')) continue
|
||||
|
||||
await instance
|
||||
.get(channel.url)
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(error => {
|
||||
if (error.response) {
|
||||
stats.failures++
|
||||
utils.writeToLog(item.url, error.message, channel.url)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if (stats.failures === 0) {
|
||||
console.log(`\nOK (${stats.playlists} playlists, ${stats.channels} channels)`)
|
||||
} else {
|
||||
console.log(
|
||||
`\nFAILURES! (${stats.playlists} playlists, ${stats.channels} channels, ${stats.failures} failures)
|
||||
\n\nCheck the "error.log" file to see which links failed.`
|
||||
)
|
||||
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Test is running...\n')
|
||||
|
||||
test()
|
||||
Loading…
Add table
Add a link
Reference in a new issue