From 421fa4580c558975cb0460ab378c974da190e78d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sat, 15 Jan 2022 02:01:37 +0300 Subject: [PATCH] Update auto-update.yml --- .github/workflows/auto-update.yml | 265 ++++++++++------------------ .gitignore | 3 +- package.json | 1 + scripts/commands/create-database.js | 2 +- 4 files changed, 93 insertions(+), 178 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 9a718210..16d351ac 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -1,212 +1,125 @@ name: auto-update on: workflow_dispatch: - schedule: - - cron: '0 0 * * *' + # schedule: + # - cron: '0 0,12 * * *' jobs: - delete-old-branch: + setup: runs-on: ubuntu-latest - continue-on-error: true steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} - - name: Delete Branch - uses: dawidd6/action-delete-branch@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branches: 'bot/auto-update' - create-branch: - runs-on: ubuntu-latest - needs: delete-old-branch - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} - - name: Create Branch - uses: peterjgrainger/action-create-branch@v2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: 'bot/auto-update' - create-matrix: - runs-on: ubuntu-latest - needs: create-branch - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Setup Node - uses: actions/setup-node@v2 + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + if: ${{ !env.ACT }} with: node-version: '14' cache: 'npm' - - name: Install Dependencies - run: npm install - - name: Create Matrix - id: set-matrix - run: node scripts/create-matrix.js - grab: + - run: npm install + - run: node scripts/commands/create-database.js --max-clusters=30000 + - run: node scripts/commands/create-matrix.js + id: create-matrix + - uses: actions/upload-artifact@v2 + with: + name: database + path: scripts/database + outputs: + matrix: ${{ steps.create-matrix.outputs.matrix }} + load: runs-on: ubuntu-latest - needs: create-matrix + needs: setup continue-on-error: true strategy: fail-fast: false - matrix: ${{fromJSON(needs.create-matrix.outputs.matrix)}} + # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + matrix: + cluster_id: [] + include: + - cluster_id: 1 steps: - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 with: - ref: bot/auto-update - - name: Setup Node - uses: actions/setup-node@v2 + name: database + path: scripts/database + - uses: FedericoCarboni/setup-ffmpeg@v1 + - uses: actions/setup-node@v2 + if: ${{ !env.ACT }} with: node-version: '14' - cache: 'npm' - - name: Install Dependencies - run: npm install - - name: Run Grabber - run: NODE_OPTIONS=--insecure-http-parser npx epg-grabber --config=sites/${{ matrix.guide.site }}/${{ matrix.guide.site }}.config.js --channels=sites/${{ matrix.guide.site }}/${{ matrix.guide.site }}_${{ matrix.guide.country }}.channels.xml --output=.gh-pages/guides/${{ matrix.guide.country }}/${{ matrix.guide.site }}.epg.xml --log=logs/${{ matrix.guide.site }}_${{ matrix.guide.country }}.log --days=2 --timeout=30000 - - name: Upload Guide - uses: actions/upload-artifact@v2 - with: - name: .gh-pages - path: .gh-pages/ - if-no-files-found: error - - name: Upload Log - uses: actions/upload-artifact@v2 + - run: npm install + - run: NODE_OPTIONS=--insecure-http-parser node scripts/commands/load-cluster.js --days=2 --timeout=30000 --cluster-id=${{ matrix.cluster_id }} + - uses: actions/upload-artifact@v2 with: name: logs - path: logs/ - if-no-files-found: error - deploy: - needs: grab + path: scripts/logs + update: runs-on: ubuntu-latest + needs: load steps: - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + - run: echo "::set-output name=branch_name::$(date +'bot/auto-update-%s')" + id: create-branch-name + - run: git config user.name 'iptv-bot[bot]' + - run: git config user.email '84861620+iptv-bot[bot]@users.noreply.github.com' + - run: git checkout -b ${{ steps.create-branch-name.outputs.branch_name }} + - run: curl -L -o scripts/data/codes.json https://iptv-org.github.io/epg/codes.json + - uses: actions/download-artifact@v2 with: - ref: bot/auto-update - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Generate Token - uses: tibdex/github-app-token@v1 - id: generate-token + name: database + path: scripts/database + - uses: actions/download-artifact@v2 with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Deploy to GitHub Pages - if: ${{ github.ref == 'refs/heads/master' }} - uses: JamesIves/github-pages-deploy-action@4.1.1 - with: - branch: gh-pages - folder: .gh-pages/guides - target-folder: guides - clean: false - token: ${{ steps.generate-token.outputs.token }} - git-config-name: iptv-bot - git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit-message: '[Bot] Deploy to GitHub Pages' - update-codes: - runs-on: ubuntu-latest - needs: deploy - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Setup Node - uses: actions/setup-node@v2 + name: logs + path: scripts/logs + - uses: actions/setup-node@v2 + if: ${{ !env.ACT }} with: node-version: '14' - cache: 'npm' - - name: Install Dependencies - run: npm install - - name: Update Codes - run: npm run update-codes - - name: Generate Token - uses: tibdex/github-app-token@v1 - id: generate-token + - run: npm install + - run: node scripts/commands/save-results.js + - uses: actions/upload-artifact@v2 + with: + name: database + path: scripts/database + - run: node scripts/commands/update-api.js + - run: node scripts/commands/update-guides.js + - uses: actions/upload-artifact@v2 + with: + name: logs + path: scripts/logs + - run: node scripts/commands/update-readme.js + - run: git add README.md + - run: git commit -m "[Bot] Update README.md" + - run: git push -u origin ${{ steps.create-branch-name.outputs.branch_name }} + - uses: tibdex/github-app-token@v1 + if: ${{ !env.ACT }} + id: create-app-token with: app_id: ${{ secrets.APP_ID }} private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Deploy to GitHub Pages - if: ${{ github.ref == 'refs/heads/master' }} - uses: JamesIves/github-pages-deploy-action@4.1.1 + - uses: repo-sync/pull-request@v2 + if: ${{ !env.ACT }} + id: pull-request with: - branch: gh-pages - folder: .gh-pages - clean: false - token: ${{ steps.generate-token.outputs.token }} - git-config-name: iptv-bot - git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit-message: '[Bot] Deploy to GitHub Pages' - update-readme: - runs-on: ubuntu-latest - needs: deploy - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '14' - cache: 'npm' - - name: Install Dependencies - run: npm install - - name: Update README.md - run: npx ts-node scripts/update-readme.ts - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Update README.md' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: README.md - pull-request: - if: ${{ github.ref == 'refs/heads/master' }} - needs: update-readme - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Generate Token - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Create Pull Request - id: pr - uses: repo-sync/pull-request@v2 - with: - source_branch: 'bot/auto-update' + github_token: ${{ steps.create-app-token.outputs.token }} + source_branch: ${{ steps.create-branch-name.outputs.branch_name }} destination_branch: 'master' - pr_title: '[Bot] Update guides' + pr_title: '[Bot] Daily update' pr_body: | This pull request is created by [auto-update][1] workflow. - [1]: https://github.com/iptv-org/epg/actions/runs/${{ github.run_id }} - github_token: ${{ steps.generate-token.outputs.token }} - - name: Merge Pull Request - if: ${{ steps.pr.outputs.pr_number != 0 }} - uses: juliangruber/merge-pull-request-action@v1 + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + - uses: juliangruber/merge-pull-request-action@v1 + if: ${{ github.ref == 'refs/heads/master' }} with: github-token: ${{ secrets.PAT }} - number: ${{ steps.pr.outputs.pr_number }} + number: ${{ steps.pull-request.outputs.pr_number }} method: squash + - uses: JamesIves/github-pages-deploy-action@4.1.1 + if: ${{ github.ref == 'refs/heads/master' }} + with: + branch: gh-pages + folder: .gh-pages + token: ${{ steps.create-app-token.outputs.token }} + git-config-name: iptv-bot[bot] + git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit-message: '[Bot] Deploy to GitHub Pages' diff --git a/.gitignore b/.gitignore index 34d14dc9..285927eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ logs/ -guide.xml \ No newline at end of file +guide.xml +.artifacts \ No newline at end of file diff --git a/package.json b/package.json index 606e9cdb..79e9a1a0 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "epg", "scripts": { "update-codes": "node scripts/update-codes.js", + "act": "act workflow_dispatch -W .github/workflows/auto-update.yml --artifact-server-path ./.artifacts", "test": "jest" }, "private": true, diff --git a/scripts/commands/create-database.js b/scripts/commands/create-database.js index 1ad9e214..d8d473d6 100644 --- a/scripts/commands/create-database.js +++ b/scripts/commands/create-database.js @@ -7,7 +7,7 @@ const options = program '--max-clusters ', 'Set maximum number of clusters', parser.parseNumber, - 200 + 256 ) .option('--channels ', 'Set path to channels.xml file', 'sites/**/*.channels.xml') .parse(process.argv)