Check Expired Domains From Filterlists #69
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Check Expired Domains From Filterlists" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| number_of_KAD_jobs: | |
| description: "Quantity of KAD jobs" | |
| required: true | |
| default: 4 | |
| number_of_KADh_jobs: | |
| description: "Quantity of KADhosts jobs" | |
| required: true | |
| default: 2 | |
| # schedule: | |
| # - cron: "55 23 * * 6" | |
| jobs: | |
| KAD_S: | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| outputs: | |
| KAD-strategy-matrix: ${{ steps.generate.outputs.KAD-strategy-matrix }} | |
| KADh-strategy-matrix: ${{ steps.generate.outputs.KADh-strategy-matrix }} | |
| env: | |
| NUMBER_OF_KAD_JOBS: ${{ github.event.inputs.number_of_kad_jobs }} | |
| NUMBER_OF_KADHOSTS_JOBS: ${{ github.event.inputs.number_of_KADh_jobs }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| echo "$NUMBER_OF_KAD_JOBS" | |
| numberParts="$NUMBER_OF_KAD_JOBS" ./scripts/CI/CI_LETS_DO_THIS.sh KAD | |
| numberParts="$NUMBER_OF_KADHOSTS_JOBS" ./scripts/CI/CI_LETS_DO_THIS.sh KADhosts | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_KAD_S | |
| path: | | |
| split/KAD_* | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_KADH_S | |
| path: | | |
| split/KADhosts_* | |
| - id: generate | |
| run: | | |
| KAD_STRATEGY_MATRIX=$(node -e "let r=[]; for(let i = 1; i <= process.env.NUMBER_OF_KAD_JOBS; i++) { r.push(i) }; console.log(JSON.stringify(r));") | |
| echo "KAD-strategy-matrix=$KAD_STRATEGY_MATRIX" >> $GITHUB_OUTPUT | |
| KADH_STRATEGY_MATRIX=$(node -e "let r=[]; for(let i = 1; i <= process.env.NUMBER_OF_KADHOSTS_JOBS; i++) { r.push(i) }; console.log(JSON.stringify(r));") | |
| echo "KADh-strategy-matrix=$KADH_STRATEGY_MATRIX" >> $GITHUB_OUTPUT | |
| KAD: | |
| needs: KAD_S | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| strategy: | |
| matrix: | |
| KAD: ${{ fromJson(needs.KAD_S.outputs.KAD-strategy-matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: E_KAD_S | |
| path: split | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| KADfile=KAD_"$(./scripts/CI/getFileNumber.sh ${{ matrix.KAD }} )" | |
| ./scripts/CI/CI_LETS_DO_THIS.sh "$KADfile" | |
| echo 'E_KADS<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh $KADfile)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| echo "currentKADName=E_$KADfile" >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.currentKADName }} | |
| path: | | |
| ${{ env.E_KADS }} | |
| KADhosts: | |
| needs: KAD_S | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| strategy: | |
| matrix: | |
| KADH: ${{ fromJson(needs.KAD_S.outputs.KADh-strategy-matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: E_KADH_S | |
| path: split | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| KADHfile=KADhosts_"$(./scripts/CI/getFileNumber.sh ${{ matrix.KADH }} )" | |
| ./scripts/CI/CI_LETS_DO_THIS.sh "$KADHfile" | |
| echo 'E_KADHS<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh $KADHfile)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| echo "currentKADHName=E_$KADHfile" >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.currentKADHName }} | |
| path: | | |
| ${{ env.E_KADHS }} | |
| PAF: | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| ./scripts/CI/CI_LETS_DO_THIS.sh PAF_C | |
| echo 'E_PAF<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh \ | |
| PAF_arrows \ | |
| PAF_backgrounds_self-advertising \ | |
| PAF_contact_feedback_widgets \ | |
| PAF_e_newspaper \ | |
| PAF_newsletters \ | |
| PAF_other_widgets \ | |
| PAF_pop-ups \ | |
| PAF_push \ | |
| PAF_scrolling_videos \ | |
| PAF_backgrounds_self-adv_supp \ | |
| PAF_comeback_titles \ | |
| PAF_contact_feedback_widgets_supp \ | |
| PAF_newsletters_supp \ | |
| PAF_other_elements_supp \ | |
| PAF_pop-ups_supp \ | |
| PAF_push_supp \ | |
| PAF_scrolling_videos_supp \ | |
| PAF_tagged_internal_links)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_PAF | |
| path: | | |
| ${{ env.E_PAF }} | |
| SocialPASS: | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| ./scripts/CI/CI_LETS_DO_THIS.sh Social_C | |
| ./scripts/CI/CI_LETS_DO_THIS.sh PASS | |
| echo 'E_SOCIAL<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh polish_rss_filters polish_rss_filters_supp adblock_social_list social_filters_uB_AG)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_SOCIAL | |
| path: | | |
| ${{ env.E_SOCIAL }} | |
| Cookies: | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| ./scripts/CI/CI_LETS_DO_THIS.sh Cookies_C | |
| echo 'E_COOKIES<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh adblock_cookies cookies_uB_AG)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_COOKIES | |
| path: | | |
| ${{ env.E_COOKIES }} | |
| KAD_limit: | |
| needs: [KAD] | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dawidd6/action-download-artifact@v2 | |
| with: | |
| name: E_KAD_.* | |
| path: expired-domains | |
| name_is_regexp: true | |
| run_id: ${{ github.run_id }} | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| ./scripts/CI/CI_LETS_DO_THIS_KAD_limit.sh | |
| echo 'E_KAD<<EOF' >> $GITHUB_ENV | |
| echo "$(./scripts/CI/getFileListForED.sh KAD)" >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: E_KAD | |
| path: | | |
| ${{ env.E_KAD }} | |
| deploy: | |
| needs: [KAD_limit, KADhosts, PAF, SocialPASS, Cookies] | |
| runs-on: ubuntu-latest | |
| container: ghcr.io/filtersheroes/expired_domains_image:latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| ## See https://github.com/actions/download-artifact/issues/68 | |
| ## Unfortunately currently it's not possible to skip creating extra directory | |
| ## when we want to download all artifacts at once without specyfing name. | |
| ## Bad joke... | |
| - continue-on-error: true | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: E_KAD | |
| path: expired-domains | |
| - continue-on-error: true | |
| uses: dawidd6/action-download-artifact@v2 | |
| with: | |
| name: E_KADH_.* | |
| path: expired-domains | |
| name_is_regexp: true | |
| search_artifacts: true | |
| - continue-on-error: true | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: E_PAF | |
| path: expired-domains | |
| - continue-on-error: true | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: E_SOCIAL | |
| path: expired-domains | |
| - continue-on-error: true | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: E_COOKIES | |
| path: expired-domains | |
| - run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| ./scripts/CI/cleanup_push.sh |