Update auto-update.yml

.github/workflows/auto-update.yml

@@ -1,212 +1,125 @@
 name: auto-update
 on:
   workflow_dispatch:
-  schedule:
-    - cron: '0 0 * * *'
+  # schedule:
+  #   - cron: '0 0,12 * * *'
 jobs:
-  delete-old-branch:
+  setup:
     runs-on: ubuntu-latest
-    continue-on-error: true
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: ${{ github.ref }}
-      - name: Delete Branch
-        uses: dawidd6/action-delete-branch@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          branches: 'bot/auto-update'
-  create-branch:
-    runs-on: ubuntu-latest
-    needs: delete-old-branch
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: ${{ github.ref }}
-      - name: Create Branch
-        uses: peterjgrainger/action-create-branch@v2.0.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          branch: 'bot/auto-update'
-  create-matrix:
-    runs-on: ubuntu-latest
-    needs: create-branch
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: bot/auto-update
-      - name: Setup Node
-        uses: actions/setup-node@v2
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        if: ${{ !env.ACT }}
         with:
           node-version: '14'
           cache: 'npm'
-      - name: Install Dependencies
-        run: npm install
-      - name: Create Matrix
-        id: set-matrix
-        run: node scripts/create-matrix.js
-  grab:
+      - run: npm install
+      - run: node scripts/commands/create-database.js --max-clusters=30000
+      - run: node scripts/commands/create-matrix.js
+        id: create-matrix
+      - uses: actions/upload-artifact@v2
+        with:
+          name: database
+          path: scripts/database
+    outputs:
+      matrix: ${{ steps.create-matrix.outputs.matrix }}
+  load:
     runs-on: ubuntu-latest
-    needs: create-matrix
+    needs: setup
     continue-on-error: true
     strategy:
       fail-fast: false
-      matrix: ${{fromJSON(needs.create-matrix.outputs.matrix)}}
+      # matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
+      matrix:
+        cluster_id: []
+        include:
+          - cluster_id: 1
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
         with:
-          ref: bot/auto-update
-      - name: Setup Node
-        uses: actions/setup-node@v2
+          name: database
+          path: scripts/database
+      - uses: FedericoCarboni/setup-ffmpeg@v1
+      - uses: actions/setup-node@v2
+        if: ${{ !env.ACT }}
         with:
           node-version: '14'
-          cache: 'npm'
-      - name: Install Dependencies
-        run: npm install
-      - name: Run Grabber
-        run: NODE_OPTIONS=--insecure-http-parser npx epg-grabber --config=sites/${{ matrix.guide.site }}/${{ matrix.guide.site }}.config.js --channels=sites/${{ matrix.guide.site }}/${{ matrix.guide.site }}_${{ matrix.guide.country }}.channels.xml --output=.gh-pages/guides/${{ matrix.guide.country }}/${{ matrix.guide.site }}.epg.xml --log=logs/${{ matrix.guide.site }}_${{ matrix.guide.country }}.log --days=2 --timeout=30000
-      - name: Upload Guide
-        uses: actions/upload-artifact@v2
-        with:
-          name: .gh-pages
-          path: .gh-pages/
-          if-no-files-found: error
-      - name: Upload Log
-        uses: actions/upload-artifact@v2
+      - run: npm install
+      - run: NODE_OPTIONS=--insecure-http-parser node scripts/commands/load-cluster.js --days=2 --timeout=30000 --cluster-id=${{ matrix.cluster_id }}
+      - uses: actions/upload-artifact@v2
         with:
           name: logs
-          path: logs/
-          if-no-files-found: error
-  deploy:
-    needs: grab
+          path: scripts/logs
+  update:
     runs-on: ubuntu-latest
+    needs: load
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+      - run: echo "::set-output name=branch_name::$(date +'bot/auto-update-%s')"
+        id: create-branch-name
+      - run: git config user.name 'iptv-bot[bot]'
+      - run: git config user.email '84861620+iptv-bot[bot]@users.noreply.github.com'
+      - run: git checkout -b ${{ steps.create-branch-name.outputs.branch_name }}
+      - run: curl -L -o scripts/data/codes.json https://iptv-org.github.io/epg/codes.json
+      - uses: actions/download-artifact@v2
         with:
-          ref: bot/auto-update
-      - name: Download Artifacts
-        uses: actions/download-artifact@v2
-      - name: Generate Token
-        uses: tibdex/github-app-token@v1
-        id: generate-token
+          name: database
+          path: scripts/database
+      - uses: actions/download-artifact@v2
         with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
-      - name: Deploy to GitHub Pages
-        if: ${{ github.ref == 'refs/heads/master' }}
-        uses: JamesIves/github-pages-deploy-action@4.1.1
-        with:
-          branch: gh-pages
-          folder: .gh-pages/guides
-          target-folder: guides
-          clean: false
-          token: ${{ steps.generate-token.outputs.token }}
-          git-config-name: iptv-bot
-          git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com
-          commit-message: '[Bot] Deploy to GitHub Pages'
-  update-codes:
-    runs-on: ubuntu-latest
-    needs: deploy
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: bot/auto-update
-      - name: Download Artifacts
-        uses: actions/download-artifact@v2
-      - name: Setup Node
-        uses: actions/setup-node@v2
+          name: logs
+          path: scripts/logs
+      - uses: actions/setup-node@v2
+        if: ${{ !env.ACT }}
         with:
           node-version: '14'
-          cache: 'npm'
-      - name: Install Dependencies
-        run: npm install
-      - name: Update Codes
-        run: npm run update-codes
-      - name: Generate Token
-        uses: tibdex/github-app-token@v1
-        id: generate-token
+      - run: npm install
+      - run: node scripts/commands/save-results.js
+      - uses: actions/upload-artifact@v2
+        with:
+          name: database
+          path: scripts/database
+      - run: node scripts/commands/update-api.js
+      - run: node scripts/commands/update-guides.js
+      - uses: actions/upload-artifact@v2
+        with:
+          name: logs
+          path: scripts/logs
+      - run: node scripts/commands/update-readme.js
+      - run: git add README.md
+      - run: git commit -m "[Bot] Update README.md"
+      - run: git push -u origin ${{ steps.create-branch-name.outputs.branch_name }}
+      - uses: tibdex/github-app-token@v1
+        if: ${{ !env.ACT }}
+        id: create-app-token
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}
-      - name: Deploy to GitHub Pages
-        if: ${{ github.ref == 'refs/heads/master' }}
-        uses: JamesIves/github-pages-deploy-action@4.1.1
+      - uses: repo-sync/pull-request@v2
+        if: ${{ !env.ACT }}
+        id: pull-request
         with:
-          branch: gh-pages
-          folder: .gh-pages
-          clean: false
-          token: ${{ steps.generate-token.outputs.token }}
-          git-config-name: iptv-bot
-          git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com
-          commit-message: '[Bot] Deploy to GitHub Pages'
-  update-readme:
-    runs-on: ubuntu-latest
-    needs: deploy
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: bot/auto-update
-      - name: Download Artifacts
-        uses: actions/download-artifact@v2
-      - name: Setup Node
-        uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-          cache: 'npm'
-      - name: Install Dependencies
-        run: npm install
-      - name: Update README.md
-        run: npx ts-node scripts/update-readme.ts
-      - name: Commit Changes
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          commit_message: '[Bot] Update README.md'
-          commit_user_name: iptv-bot
-          commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
-          commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
-          branch: bot/auto-update
-          file_pattern: README.md
-  pull-request:
-    if: ${{ github.ref == 'refs/heads/master' }}
-    needs: update-readme
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          ref: bot/auto-update
-      - name: Generate Token
-        uses: tibdex/github-app-token@v1
-        id: generate-token
-        with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
-      - name: Create Pull Request
-        id: pr
-        uses: repo-sync/pull-request@v2
-        with:
-          source_branch: 'bot/auto-update'
+          github_token: ${{ steps.create-app-token.outputs.token }}
+          source_branch: ${{ steps.create-branch-name.outputs.branch_name }}
           destination_branch: 'master'
-          pr_title: '[Bot] Update guides'
+          pr_title: '[Bot] Daily update'
           pr_body: |
             This pull request is created by [auto-update][1] workflow.
 
-            [1]: https://github.com/iptv-org/epg/actions/runs/${{ github.run_id }}
-          github_token: ${{ steps.generate-token.outputs.token }}
-      - name: Merge Pull Request
-        if: ${{ steps.pr.outputs.pr_number != 0 }}
-        uses: juliangruber/merge-pull-request-action@v1
+            [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
+      - uses: juliangruber/merge-pull-request-action@v1
+        if: ${{ github.ref == 'refs/heads/master' }}
         with:
           github-token: ${{ secrets.PAT }}
-          number: ${{ steps.pr.outputs.pr_number }}
+          number: ${{ steps.pull-request.outputs.pr_number }}
           method: squash
+      - uses: JamesIves/github-pages-deploy-action@4.1.1
+        if: ${{ github.ref == 'refs/heads/master' }}
+        with:
+          branch: gh-pages
+          folder: .gh-pages
+          token: ${{ steps.create-app-token.outputs.token }}
+          git-config-name: iptv-bot[bot]
+          git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com
+          commit-message: '[Bot] Deploy to GitHub Pages'

.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 logs/
-guide.xml
+guide.xml
+.artifacts

package.json

@@ -2,6 +2,7 @@
   "name": "epg",
   "scripts": {
     "update-codes": "node scripts/update-codes.js",
+    "act": "act workflow_dispatch -W .github/workflows/auto-update.yml --artifact-server-path ./.artifacts",
     "test": "jest"
   },
   "private": true,

scripts/commands/create-database.js

@@ -7,7 +7,7 @@ const options = program
     '--max-clusters <max-clusters>',
     'Set maximum number of clusters',
     parser.parseNumber,
-    200
+    256
   )
   .option('--channels <channels>', 'Set path to channels.xml file', 'sites/**/*.channels.xml')
   .parse(process.argv)