fix: Refactor Midlothian Council scraper to use house number and postcode #2275
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Councils | |
| on: | |
| workflow_dispatch: | |
| push: | |
| # Trigger unless only the wiki directory changed | |
| paths-ignore: | |
| - "wiki/**" | |
| - "**/**.md" | |
| - "uk_bin_collection_api_server/**" | |
| branches: [ "master" ] | |
| pull_request: | |
| # Trigger unless only the wiki directory changed | |
| paths-ignore: | |
| - "wiki/**" | |
| - "**/**.md" | |
| - "uk_bin_collection_api_server/**" | |
| branches: [ "master" ] | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| build: | |
| if: "!startsWith(github.event.head_commit.message, 'bump:')" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [3.11, 3.12] | |
| poetry-version: [1.8.3] | |
| services: | |
| selenium: | |
| image: selenium/standalone-chrome:latest | |
| options: --shm-size=2gb --name selenium --hostname selenium | |
| ports: | |
| - 4444:4444 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install poetry | |
| run: pipx install poetry==${{ matrix.poetry-version }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| #cache: 'poetry' | |
| - name: Install | |
| run: make install | |
| - name: Lint JSON | |
| run: jq empty uk_bin_collection/tests/input.json | |
| - name: Check Parity of Councils / input.json / Feature file | |
| run: | | |
| repo=${{ github.event.pull_request.head.repo.full_name || 'robbrad/UKBinCollectionData' }} | |
| branch=${{ github.event.pull_request.head.ref || 'master' }} | |
| make parity-check repo=$repo branch=$branch | |
| - name: Get all councils files that have changed | |
| id: changed-council-files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| files: | | |
| uk_bin_collection/uk_bin_collection/councils/**.py | |
| - name: Get all councils | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed-council-files.outputs.all_changed_files }} | |
| run: | | |
| IFS=' ' read -ra FILES <<< "$ALL_CHANGED_FILES" | |
| COUNCIL_TESTS="" | |
| for file in "${FILES[@]}"; do | |
| FILENAME=$(basename "$file" .py) | |
| if [ -z "$COUNCIL_TESTS" ]; then | |
| COUNCIL_TESTS="$FILENAME" | |
| else | |
| COUNCIL_TESTS="$COUNCIL_TESTS or $FILENAME" | |
| fi | |
| done | |
| echo "COUNCIL_TESTS=${COUNCIL_TESTS}" >> $GITHUB_ENV | |
| - name: Run integration tests | |
| env: | |
| HEADLESS: True | |
| run: make matrix=${{ matrix.python-version }} councils="${{ env.COUNCIL_TESTS }}" integration-tests | |
| continue-on-error: true | |
| - name: Run unit tests | |
| run: make unit-tests | |
| continue-on-error: true | |
| #- name: Upload test coverage to Codecov | |
| # uses: codecov/codecov-action@v4 | |
| # with: | |
| # gcov_ignore: uk_bin_collection/tests/** | |
| - name: Upload test results to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| fail_ci_if_error: false | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: coverage.xml | |
| - name: Get Allure history | |
| uses: actions/checkout@v4 | |
| if: github.ref == 'refs/heads/master' | |
| continue-on-error: true | |
| with: | |
| ref: gh-pages | |
| path: gh-pages | |
| - name: Allure report action from marketplace | |
| uses: simple-elf/allure-report-action@master | |
| if: github.ref == 'refs/heads/master' | |
| with: | |
| allure_results: build/${{ matrix.python-version }}/allure-results | |
| subfolder: ${{ matrix.python-version }} | |
| allure_history: allure-history | |
| keep_reports: 20 | |
| - name: Tar report | |
| if: github.ref == 'refs/heads/master' | |
| run: tar -cvf allure_history_${{ matrix.python-version }}.tar allure-history/${{ matrix.python-version }} | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| if: github.ref == 'refs/heads/master' | |
| with: | |
| name: allure_history_${{ matrix.python-version }} | |
| path: allure_history_${{ matrix.python-version }}.tar | |
| deploy: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.ref == 'refs/heads/master' | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| name: Download 3.11 artifacts | |
| with: | |
| name: allure_history_3.11 | |
| path: allure-history/tars | |
| - uses: actions/download-artifact@v4 | |
| name: Download 3.12 artifacts | |
| with: | |
| name: allure_history_3.12 | |
| path: allure-history/tars | |
| - name: Untar reports | |
| run: for i in allure-history/tars/*.tar; do tar -xvf "$i" allure-history ;done | |
| - name: Remove tar reports | |
| run: rm -rf allure-history/tars | |
| - name: Display structure of downloaded files | |
| run: ls -R | |
| - name: Deploy | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| PERSONAL_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PUBLISH_BRANCH: gh-pages | |
| PUBLISH_DIR: allure-history |