Update scripts

scripts/commands/db/update.ts

@@ -0,0 +1,411 @@
+import { CSV, IssueLoader, CSVParser, Issue, IssueData } from '../../core'
+import { createChannelId, createFeedId } from '../../utils'
+import { Channel, Feed, BlocklistRecord } from '../../models'
+import { Storage, Collection, Logger } from '@freearhey/core'
+import { DATA_DIR } from '../../constants'
+import { DataLoader } from '../../core/dataLoader'
+import { DataLoaderData } from '../../types/dataLoader'
+
+const processedIssues = new Collection()
+const dataStorage = new Storage(DATA_DIR)
+const logger = new Logger({ level: -999 })
+
+async function main() {
+  const parser = new CSVParser()
+  const issueLoader = new IssueLoader()
+  const dataLoader = new DataLoader({ storage: dataStorage })
+
+  logger.info('loading issues...')
+  const issues = await issueLoader.load()
+
+  logger.info('loading data...')
+  const data = await dataLoader.load()
+
+  logger.info('processing issues...')
+  await removeFeeds(issues, data)
+  await removeChannels(issues, data)
+  await editFeeds(issues, data)
+  await editChannels(issues, data)
+  await addFeeds(issues, data)
+  await addChannels(issues, data)
+  await blockChannels(issues, data)
+  await unblockChannels(issues, data)
+
+  logger.info('saving data...')
+  await save(data)
+
+  const output = processedIssues.map((issue: Issue) => `closes #${issue.number}`).join(', ')
+  process.stdout.write(`OUTPUT=${output}`)
+}
+
+main()
+
+async function save(data: DataLoaderData) {
+  const channels = data.channels
+    .sortBy((channel: Channel) => channel.id.toLowerCase())
+    .map((channel: Channel) => channel.data())
+  const channelsOutput = new CSV({ items: channels }).toString()
+  await dataStorage.save('channels.csv', channelsOutput)
+
+  const feeds = data.feeds
+    .sortBy((feed: Feed) => `${feed.getStreamId()}`.toLowerCase())
+    .map((feed: Feed) => feed.data())
+  const feedsOutput = new CSV({ items: feeds }).toString()
+  await dataStorage.save('feeds.csv', feedsOutput)
+
+  const blocklistRecords = data.blocklistRecords
+    .sortBy((blocklistRecord: BlocklistRecord) => blocklistRecord.channelId.toLowerCase())
+    .map((blocklistRecord: BlocklistRecord) => blocklistRecord.data())
+  const blocklistOutput = new CSV({ items: blocklistRecords }).toString()
+  await dataStorage.save('blocklist.csv', blocklistOutput)
+}
+
+async function removeFeeds(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('feeds:remove') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (issueData.missing('channel_id') || issueData.missing('feed_id')) return
+
+    const found: Feed = data.feeds.first(
+      (feed: Feed) =>
+        feed.channelId === issueData.getString('channel_id') &&
+        feed.id === issueData.getString('feed_id')
+    )
+    if (!found) return
+
+    data.feeds.remove((feed: Feed) => feed.channelId === found.channelId && feed.id === found.id)
+
+    onFeedRemoval(found.channelId, found.id, data)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function editFeeds(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('feeds:edit') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+    if (issueData.missing('channel_id') || issueData.missing('feed_id')) return
+
+    const found: Feed = data.feeds.first(
+      (feed: Feed) =>
+        feed.channelId === issueData.getString('channel_id') &&
+        feed.id === issueData.getString('feed_id')
+    )
+    if (!found) return
+
+    let channelId: string | undefined = found.channelId
+    let feedId: string | undefined = found.id
+    if (issueData.has('feed_name')) {
+      const name = issueData.getString('feed_name') || found.name
+      if (name) {
+        feedId = createFeedId(name)
+        if (feedId) onFeedIdChange(found.channelId, found.id, feedId, data)
+      }
+    }
+
+    if (issueData.has('is_main')) {
+      const isMain = issueData.getBoolean('is_main') || false
+      if (isMain) onFeedNewMain(channelId, feedId, data)
+    }
+
+    if (!feedId || !channelId) return
+
+    found.update(issueData).setId(feedId)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function addFeeds(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('feeds:add') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (
+      issueData.missing('channel_id') ||
+      issueData.missing('feed_name') ||
+      issueData.missing('is_main') ||
+      issueData.missing('broadcast_area') ||
+      issueData.missing('timezones') ||
+      issueData.missing('languages') ||
+      issueData.missing('video_format')
+    )
+      return
+
+    const channelId = issueData.getString('channel_id')
+    const feedName = issueData.getString('feed_name') || 'SD'
+    const feedId = createFeedId(feedName)
+    if (!channelId || !feedId) return
+
+    const found: Feed = data.feeds.first(
+      (feed: Feed) => feed.channelId === channelId && feed.id === feedId
+    )
+    if (found) return
+
+    const isMain = issueData.getBoolean('is_main') || false
+    if (isMain) onFeedNewMain(channelId, feedId, data)
+
+    const newFeed = new Feed({
+      channel: channelId,
+      id: feedId,
+      name: feedName,
+      is_main: issueData.getBoolean('is_main') || false,
+      broadcast_area: issueData.getArray('broadcast_area') || [],
+      timezones: issueData.getArray('timezones') || [],
+      languages: issueData.getArray('languages') || [],
+      video_format: issueData.getString('video_format')
+    })
+
+    data.feeds.add(newFeed)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function removeChannels(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('channels:remove') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (issueData.missing('channel_id')) return
+
+    const found = data.channels.first(
+      (channel: Channel) => channel.id === issueData.getString('channel_id')
+    )
+    if (!found) return
+
+    data.channels.remove((channel: Channel) => channel.id === found.id)
+
+    onChannelRemoval(found.id, data)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function editChannels(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('channels:edit') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (issueData.missing('channel_id')) return
+
+    const found: Channel = data.channels.first(
+      (channel: Channel) => channel.id === issueData.getString('channel_id')
+    )
+    if (!found) return
+
+    let channelId: string | undefined = found.id
+    if (issueData.has('channel_name') || issueData.has('country')) {
+      const name = issueData.getString('channel_name') || found.name
+      const country = issueData.getString('country') || found.countryCode
+      if (name && country) {
+        channelId = createChannelId(name, country)
+        if (channelId) onChannelIdChange(found.id, channelId, data)
+      }
+    }
+
+    if (!channelId) return
+
+    found.update(issueData).setId(channelId)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function addChannels(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('channels:add') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (
+      issueData.missing('channel_name') ||
+      issueData.missing('country') ||
+      issueData.missing('is_nsfw') ||
+      issueData.missing('logo') ||
+      issueData.missing('feed_name') ||
+      issueData.missing('broadcast_area') ||
+      issueData.missing('timezones') ||
+      issueData.missing('languages') ||
+      issueData.missing('video_format')
+    )
+      return
+
+    const channelId = createChannelId(
+      issueData.getString('channel_name'),
+      issueData.getString('country')
+    )
+    if (!channelId) return
+
+    const found: Channel = data.channels.first((channel: Channel) => channel.id === channelId)
+    if (found) return
+
+    const newChannel = new Channel({
+      id: channelId,
+      name: issueData.getString('channel_name') || '',
+      alt_names: issueData.getArray('alt_names'),
+      network: issueData.getString('network'),
+      owners: issueData.getArray('owners'),
+      country: issueData.getString('country') || '',
+      subdivision: issueData.getString('subdivision'),
+      city: issueData.getString('city'),
+      categories: issueData.getArray('categories'),
+      is_nsfw: issueData.getBoolean('is_nsfw') || false,
+      launched: issueData.getString('launched'),
+      closed: issueData.getString('closed'),
+      replaced_by: issueData.getString('replaced_by'),
+      website: issueData.getString('website'),
+      logo: issueData.getString('logo') || ''
+    })
+    data.channels.add(newChannel)
+
+    const feedName = issueData.getString('feed_name') || 'SD'
+    const newFeed = new Feed({
+      channel: channelId,
+      id: createFeedId(feedName),
+      name: feedName,
+      is_main: true,
+      broadcast_area: issueData.getArray('broadcast_area') || [],
+      timezones: issueData.getArray('timezones') || [],
+      languages: issueData.getArray('languages') || [],
+      video_format: issueData.getString('video_format')
+    })
+    data.feeds.add(newFeed)
+
+    processedIssues.push(issue)
+  })
+}
+
+async function unblockChannels(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('blocklist:remove') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (issueData.missing('channel_id')) return
+
+    const found: BlocklistRecord = data.blocklistRecords.first(
+      (blocklistRecord: BlocklistRecord) =>
+        blocklistRecord.channelId === issueData.getString('channel_id')
+    )
+    if (!found) return
+
+    data.blocklistRecords.remove(
+      (blocklistRecord: BlocklistRecord) => blocklistRecord.channelId === found.channelId
+    )
+
+    processedIssues.push(issue)
+  })
+}
+
+async function blockChannels(issues: Collection, data: DataLoaderData) {
+  const requests = issues.filter(
+    issue => issue.labels.includes('blocklist:add') && issue.labels.includes('approved')
+  )
+
+  requests.forEach((issue: Issue) => {
+    const issueData: IssueData = issue.data
+
+    if (issueData.missing('channel_id')) return
+
+    const found: BlocklistRecord = data.blocklistRecords.first(
+      (blocklistRecord: BlocklistRecord) =>
+        blocklistRecord.channelId === issueData.getString('channel_id')
+    )
+    if (found) return
+
+    const channel = issueData.getString('channel_id')
+    const reason = issueData.getString('reason')?.toLowerCase()
+    const ref = issueData.getString('ref')
+    if (!channel || !reason || !ref) return
+
+    const newBlocklistRecord = new BlocklistRecord({
+      channel,
+      reason,
+      ref
+    })
+    data.blocklistRecords.add(newBlocklistRecord)
+
+    processedIssues.push(issue)
+  })
+}
+
+function onFeedIdChange(
+  channelId: string,
+  feedId: string,
+  newFeedId: string,
+  data: DataLoaderData
+) {
+  data.channels.forEach((channel: Channel) => {
+    if (channel.replacedBy && channel.replacedBy === `${channelId}@${feedId}`) {
+      channel.replacedBy = `${channelId}@${newFeedId}`
+    }
+  })
+}
+
+function onFeedNewMain(channelId: string, feedId: string, data: DataLoaderData) {
+  data.feeds.forEach((feed: Feed) => {
+    if (feed.channelId === channelId && feed.id !== feedId && feed.isMain === true) {
+      feed.isMain = false
+    }
+  })
+}
+
+function onFeedRemoval(channelId: string, feedId: string, data: DataLoaderData) {
+  data.channels.forEach((channel: Channel) => {
+    if (channel.replacedBy && channel.replacedBy === `${channelId}@${feedId}`) {
+      channel.replacedBy = ''
+    }
+  })
+}
+
+function onChannelIdChange(channelId: string, newChannelId: string, data: DataLoaderData) {
+  data.channels.forEach((channel: Channel) => {
+    if (channel.replacedBy && channel.replacedBy.includes(channelId)) {
+      channel.replacedBy = channel.replacedBy.replace(channelId, newChannelId)
+    }
+  })
+
+  data.feeds.forEach((feed: Feed) => {
+    if (feed.channelId === channelId) {
+      feed.channelId = newChannelId
+    }
+  })
+
+  data.blocklistRecords.forEach((blocklistRecord: BlocklistRecord) => {
+    if (blocklistRecord.channelId === channelId) {
+      blocklistRecord.channelId = newChannelId
+    }
+  })
+}
+
+function onChannelRemoval(channelId: string, data: DataLoaderData) {
+  data.channels.forEach((channel: Channel) => {
+    if (channel.replacedBy && channel.replacedBy.includes(channelId)) {
+      channel.replacedBy = ''
+    }
+  })
+
+  data.feeds.remove((feed: Feed) => feed.channelId === channelId)
+}