Nightly upstream testing #697
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly upstream testing | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Daily “At 00:00” UTC | |
| workflow_dispatch: # allows you to trigger the workflow run manually | |
| # Required shell entrypoint to have properly activated conda environments | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| jobs: | |
| build: | |
| # This build step should be similar to the deploy build, to make sure we actually test | |
| # the future deployable | |
| name: Build the jar on ubuntu | |
| runs-on: ubuntu-latest | |
| if: github.repository == 'dask-contrib/dask-sql' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }} | |
| - name: Set up Python | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| python-version: "3.8" | |
| channel-priority: strict | |
| activate-environment: dask-sql | |
| environment-file: continuous_integration/environment-3.8-jdk11-dev.yaml | |
| - name: Install dependencies and build the jar | |
| run: | | |
| python setup.py build_ext | |
| - name: Upload the jar | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: jar | |
| path: dask_sql/jar/DaskSQL.jar | |
| test-dev: | |
| name: "Test upstream dev (${{ matrix.os }}, java: ${{ matrix.java }}, python: ${{ matrix.python }})" | |
| needs: build | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-jdk${{ matrix.java }}-dev.yaml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java: [8, 11] | |
| os: [ubuntu-latest, windows-latest] | |
| python: ["3.8", "3.9", "3.10"] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # Fetch all history for all branches and tags. | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-v1-jdk${{ matrix.java }}-${{ hashFiles('**/pom.xml') }} | |
| - name: Set up Python | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| python-version: ${{ matrix.python }} | |
| channel-priority: strict | |
| channels: dask/label/dev,conda-forge,nodefaults | |
| activate-environment: dask-sql | |
| environment-file: ${{ env.CONDA_FILE }} | |
| - name: Download the pre-build jar | |
| uses: actions/download-artifact@v1 | |
| with: | |
| name: jar | |
| path: dask_sql/jar/ | |
| - name: Install hive testing dependencies for Linux | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| mamba install -c conda-forge sasl>=0.3.1 | |
| docker pull bde2020/hive:2.3.2-postgresql-metastore | |
| docker pull bde2020/hive-metastore-postgresql:2.3.0 | |
| - name: Set proper JAVA_HOME for Windows | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| echo "JAVA_HOME=${{ env.CONDA }}\envs\dask-sql\Library" >> $GITHUB_ENV | |
| - name: Install upstream dev Dask / dask-ml | |
| run: | | |
| mamba update dask | |
| python -m pip install --no-deps git+https://github.com/dask/dask-ml | |
| - name: Test with pytest | |
| run: | | |
| pytest --junitxml=junit/test-results.xml --cov-report=xml -n auto tests --dist loadfile | |
| cluster-dev: | |
| name: "Test upstream dev in a dask cluster" | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }} | |
| - name: Set up Python | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| use-mamba: true | |
| python-version: "3.9" | |
| channel-priority: strict | |
| channels: dask/label/dev,conda-forge,nodefaults | |
| activate-environment: dask-sql | |
| environment-file: continuous_integration/environment-3.9-jdk11-dev.yaml | |
| - name: Download the pre-build jar | |
| uses: actions/download-artifact@v1 | |
| with: | |
| name: jar | |
| path: dask_sql/jar/ | |
| - name: Install cluster dependencies | |
| run: | | |
| mamba install python-blosc lz4 -c conda-forge | |
| which python | |
| pip list | |
| mamba list | |
| - name: Install upstream dev dask-ml | |
| run: | | |
| mamba update dask | |
| python -m pip install --no-deps git+https://github.com/dask/dask-ml | |
| - name: run a dask cluster | |
| run: | | |
| docker-compose -f .github/cluster-upstream.yml up -d | |
| # periodically ping logs until a connection has been established; assume failure after 2 minutes | |
| timeout 2m bash -c 'until docker logs dask-worker 2>&1 | grep -q "Starting established connection"; do sleep 1; done' | |
| docker logs dask-scheduler | |
| docker logs dask-worker | |
| - name: Test with pytest while running an independent dask cluster | |
| run: | | |
| DASK_SQL_TEST_SCHEDULER="tcp://127.0.0.1:8786" pytest --junitxml=junit/test-cluster-results.xml --cov-report=xml -n auto tests --dist loadfile | |
| import-dev: | |
| name: "Test importing with bare requirements and upstream dev" | |
| needs: build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-v1-jdk11-${{ hashFiles('**/pom.xml') }} | |
| - name: Set up Python | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| python-version: "3.8" | |
| mamba-version: "*" | |
| channels: dask/label/dev,conda-forge,nodefaults | |
| channel-priority: strict | |
| - name: Download the pre-build jar | |
| uses: actions/download-artifact@v1 | |
| with: | |
| name: jar | |
| path: dask_sql/jar/ | |
| - name: Install upstream dev Dask / dask-ml | |
| if: needs.detect-ci-trigger.outputs.triggered == 'true' | |
| run: | | |
| mamba update dask | |
| python -m pip install --no-deps git+https://github.com/dask/dask-ml | |
| - name: Install dependencies and nothing else | |
| run: | | |
| pip install -e . | |
| which python | |
| pip list | |
| mamba list | |
| - name: Try to import dask-sql | |
| run: | | |
| python -c "import dask_sql; print('ok')" | |
| report-failures: | |
| name: Open issue for upstream dev failures | |
| needs: [test-dev, cluster-dev] | |
| if: | | |
| always() | |
| && ( | |
| needs.test-dev.result == 'failure' || needs.cluster-dev.result == 'failure' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Report failures | |
| uses: actions/github-script@v3 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const title = "⚠️ Upstream CI failed ⚠️" | |
| const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` | |
| const issue_body = `[Workflow Run URL](${workflow_url})` | |
| // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures | |
| const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){ | |
| repository(owner: $owner, name: $name) { | |
| issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) { | |
| edges { | |
| node { | |
| body | |
| id | |
| number | |
| } | |
| } | |
| } | |
| } | |
| }`; | |
| const variables = { | |
| owner: context.repo.owner, | |
| name: context.repo.repo, | |
| label: 'upstream', | |
| creator: "github-actions[bot]" | |
| } | |
| const result = await github.graphql(query, variables) | |
| // If no issue is open, create a new issue, | |
| // else update the body of the existing issue. | |
| if (result.repository.issues.edges.length === 0) { | |
| github.issues.create({ | |
| owner: variables.owner, | |
| repo: variables.name, | |
| body: issue_body, | |
| title: title, | |
| labels: [variables.label] | |
| }) | |
| } else { | |
| github.issues.update({ | |
| owner: variables.owner, | |
| repo: variables.name, | |
| issue_number: result.repository.issues.edges[0].node.number, | |
| body: issue_body | |
| }) | |
| } |