From 431fc8d03f5065c2a4c01595446ee5706303386b Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Mon, 2 Aug 2021 00:48:55 -0400 Subject: [PATCH] try this again --- .github/workflows/build.yml | 66 ++++++++ .github/workflows/integration-slim.yml | 171 +++++++++++++++++++++ .github/workflows/integration.yml | 105 +------------ .github/workflows/{ci.yml => main.yml} | 7 +- scripts/{build-wheels.sh => build-dist.sh} | 0 scripts/build-sdists.sh | 23 --- 6 files changed, 245 insertions(+), 127 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/integration-slim.yml rename .github/workflows/{ci.yml => main.yml} (93%) rename scripts/{build-wheels.sh => build-dist.sh} (100%) delete mode 100755 scripts/build-sdists.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..9c5cd567350 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,66 @@ +# what? - +# why? - +# when? - + +name: Build + +on: + push: + branches: + - "main" + - "develop" + - "*.latest" + - "releases/*" + workflow_dispatch: + pull_request: + +jobs: + build: + name: ${{ matrix.python-version }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - name: Check out the repository + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Upgrade pip and setuptools + run: | + pip install --upgrade pip setuptools + pip --version + + - name: Build dist + shell: bash + run: | + ./scripts/build-dist.sh + + - name: Install wheel dists + run: | + pip install --force-reinstall --find-links=dist/ dist/*.whl + + - name: Check wheel dist + run: | + dbt --version + + - name: Install source dists + run: | + pip install --force-reinstall --find-links=dist/ dist/*.gz + + - name: Check source dist + run: | + dbt --version + + - uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/ + diff --git a/.github/workflows/integration-slim.yml b/.github/workflows/integration-slim.yml new file mode 100644 index 00000000000..a9bded08ce1 --- /dev/null +++ b/.github/workflows/integration-slim.yml @@ -0,0 +1,171 @@ +# what? - This workflow runs integration tests for supported OS and python versions and core adapters. +# It will only run tests for related file changes. Supports a `test all` label and a `test ${adapter}` +# label to manually run all or other tests. Requires secrets to run against different warehouses. +# why? - This checks the functionality of dbt from a user's perspective, provides code coverage, and +# attempts to prevent functional regressions. +# when? - This workflow will run for all pull requests and forked pull requests. This workflow will run +# for pull requests, and will run for forked pull requests when the `ok to test` label is present. + +# TODO: check if user signed cla maybe? +name: Integration Tests (slim) + +on: + pull_request_target: + +permissions: read-all + +jobs: + pre-checks: + runs-on: ubuntu-latest + outputs: + changes: ${{ steps.check-changes.outputs.changes }} + steps: + - name: Check out the repository + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Check if relevant files changed + uses: dorny/paths-filter@v2 + id: check-changes + with: + filters: | + postgres: + - 'core/**' + - 'plugins/postgres/**' + snowflake: + - 'core/**' + - 'plugins/snowflake/**' + bigquery: + - 'core/**' + - 'plugins/bigquery/**' + redshift: + - 'core/**' + - 'plugins/redshift/**' + - 'plugins/postgres/**' + + test: + name: ${{ matrix.adapter }} ${{ matrix.python-version }} / ${{ matrix.os }} + + if: >- + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + + needs: pre-checks + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + adapter: [postgres, snowflake, redshift, bigquery] + python-version: [3.8] + tox-cmd: ["tox -- -v --color=yes"] + + env: + TOXENV: integration-${{ matrix.adapter }} + + steps: + - name: Check out the repository + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Upgrade pip + run: | + pip install --upgrade pip + pip --version + + - name: Install tox + run: | + pip install tox + + - name: Run tox + if: >- + matrix.adapter == 'postgres' && + ( + contains(jobs.pre-checks.outputs.changes, 'postgres' || + contains(github.event.pull_request.labels.*.name, 'test all') || + contains(github.event.pull_request.labels.*.name, 'test postgres') + ) + env: + POSTGRES_TEST_HOST: ${{ secrets.POSTGRES_TEST_HOST }} + POSTGRES_TEST_PORT: ${{ secrets.POSTGRES_TEST_PORT }} + POSTGRES_TEST_USER: ${{ secrets.POSTGRES_TEST_USER }} + POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }} + POSTGRES_TEST_DATABASE: ${{ secrets.POSTGRES_TEST_DATABASE }} + run: ${{ matrix.tox-cmd }} + + - name: Run tox + if: >- + matrix.adapter == 'redshift' && + ( + contains(jobs.pre-checks.outputs.changes, 'redshift' || + contains(github.event.pull_request.labels.*.name, 'test all') || + contains(github.event.pull_request.labels.*.name, 'test redshift') + ) + env: + REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }} + REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }} + REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} + REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} + REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + run: ${{ matrix.tox-cmd }} + + - name: Run tox + if: >- + matrix.adapter == 'snowflake' && + ( + contains(jobs.pre-checks.outputs.changes, 'snowflake' || + contains(github.event.pull_request.labels.*.name, 'test all') || + contains(github.event.pull_request.labels.*.name, 'test snowflake') + ) + env: + SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} + SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} + SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} + SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} + SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }} + SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }} + SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }} + SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }} + SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }} + SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} + SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }} + SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} + run: ${{ matrix.tox-cmd }} + + - name: Run tox + if: >- + matrix.adapter == 'bigquery' && + ( + contains(jobs.pre-checks.outputs.changes, 'bigquery' || + contains(github.event.pull_request.labels.*.name, 'test all') || + contains(github.event.pull_request.labels.*.name, 'test bigquery') + ) + env: + BIGQUERY_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }} + BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} + run: ${{ matrix.tox-cmd }} + + post-checks: + runs-on: ubuntu-latest + needs: test + steps: + - name: Needs permission PR comment + if: >- + needs.test.result == 'skipped' && + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != github.repository + + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: | + "You do not have permissions to run integration tests, @dbt-labs/core "\ + "needs to label this PR with `ok to test` in order to run integration tests!" + check_for_duplicate_msg: true diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0c075e30299..d5e34c9cea4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,8 +1,8 @@ -# what? - This workflow runs integration tests for supported OS and python versions and core adapters. +# what? - This workflow runs all integration tests for supported OS and python versions and core adapters. +# Requires secrets to run against different warehouses. # why? - This checks the functionality of dbt from a user's perspective, provides code coverage, and # attempts to prevent functional regressions. -# when? - This workflow will run on ever push to a protected branch, for all pull requests, and -# when manually triggered. +# when? - This workflow will run on every push to a protected branch and when manually triggered. name: Integration Tests @@ -13,79 +13,12 @@ on: - "develop" - "*.latest" - "releases/*" - pull_request_target: + - "github-actions" workflow_dispatch: -permissions: read-all - jobs: - pre-checks: - runs-on: ubuntu-latest - outputs: - has-permission: ${{ steps.check-user.outputs.has-permission }} - pr-labels: ${{ steps.get-pr-labels.outputs.pr-labels || [] }} - paths: ${{ steps.check-changes.outputs }} - steps: - - name: Check user permission - id: check-user - uses: scherermichael-oss/action-has-permission@master - with: - required-permission: write - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Get PR labels - id: get-pr-labels - if: github.event_name == 'pull_request_target' - run: >- - echo "::set-output pr-labels=${{ github.event.pull_request.labels.*.name }}" - - name: Check if relevant files changed - uses: dorny/paths-filter@v2 - id: check-changes - with: - filters: | - postgres: - - 'core/**' - - 'plugins/postgres/**' - snowflake: - - 'core/**' - - 'plugins/snowflake/**' - bigquery: - - 'core/**' - - 'plugins/bigquery/**' - redshift: - - 'core/**' - - 'plugins/redshift/**' - - 'plugins/postgres/**' - test: name: ${{ matrix.adapter }} ${{ matrix.python-version }} / ${{ matrix.os }} - - # For PRs, only run python3.9 and ubuntu-latest and: - # 1. all push and workflow dispatch events, or - # 2. when user has repo write permissions, only run tests for relevant code changes, or - # 3. if PR and has an "ok to test" label, only run tests for relevant code changes, or - # 4. if PR and has a "test all" label, or - # 5. if PR and has a "test ${adapter}" label, only run tests for that adapter - if: >- - ( - github.event_name != 'pull_request_target' && - !contains(['macos-latest', 'windows-latest'], matrix.os) && - !contains([3.6, 3.7, 3.8], matrix.python-version) - ) && ( - contains(['push', 'workflow_dispatch'], github.event_name) || - - (jobs.pre-checks.outputs.has-permission && - contains(jobs.pre-checks.outputs.paths.changes, matrix.adapter)) || - - (contains(jobs.pre-checks.outputs.pr-labels, 'ok to test') && - contains(jobs.pre-checks.outputs.paths.changes, matrix.adapter)) || - - contains(jobs.pre-checks.outputs.pr-labels, 'test all') || - - contains(jobs.pre-checks.outputs.pr-labels, format('test {0}', matrix.adapter)) - ) - - needs: pre-checks runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -93,20 +26,13 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] adapter: [postgres, snowflake, redshift, bigquery] python-version: [3.6, 3.7, 3.8, 3.9] - tox-cmd: "tox -- -v --color=yes" + tox-cmd: ["tox -- -v --color=yes"] env: TOXENV: integration-${{ matrix.adapter }} steps: - - name: Check out the repository (pull request) - if: github.event_name == 'pull_request_target' - uses: actions/checkout@v2 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Check out the repository (push, workflow_dispatch) - if: contains(['push', 'workflow_dispatch'], github.event_name) + - name: Check out the repository uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -166,22 +92,3 @@ jobs: BIGQUERY_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }} BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} run: ${{ matrix.tox-cmd }} - - post-checks: - runs-on: ubuntu-latest - needs: test - steps: - - name: Needs permission PR comment - if: >- - needs.test.result == 'skipped' && - github.event_name == 'pull_request_target' && - github.event.pull_request.head.repo.full_name != github.repository - - uses: unsplash/comment-on-pr@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - msg: | - "You do not have permissions to run integration tests, @dbt-labs/core "\ - "needs to label this PR with `ok to test` in order to run integration tests!" - check_for_duplicate_msg: true diff --git a/.github/workflows/ci.yml b/.github/workflows/main.yml similarity index 93% rename from .github/workflows/ci.yml rename to .github/workflows/main.yml index 7ecdf37f701..7cddcfb3880 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,6 @@ # when? - This workflow will run on ever push to a protected branch, for all pull requests, and # when manually triggered. -# TODO add caching? name: CI on: @@ -17,8 +16,6 @@ on: pull_request: workflow_dispatch: -permissions: read-all - jobs: tests: name: ${{ matrix.tox-env }} ${{ matrix.python-version }} / ${{ matrix.os }} @@ -26,8 +23,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - tox-env: [unit] + os: [ubuntu-latest] #, macos-latest, windows-latest] + tox-env: [unit, mypy] python-version: [3.6, 3.7, 3.8, 3.9] include: - os: ubuntu-latest diff --git a/scripts/build-wheels.sh b/scripts/build-dist.sh similarity index 100% rename from scripts/build-wheels.sh rename to scripts/build-dist.sh diff --git a/scripts/build-sdists.sh b/scripts/build-sdists.sh deleted file mode 100755 index 8d0db621b23..00000000000 --- a/scripts/build-sdists.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -eo pipefail - -DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )" - -echo $SCRIPTPATH - -set -x - -rm -rf "$DBT_PATH"/dist -mkdir -p "$DBT_PATH"/dist - -for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake -do - rm -rf "$DBT_PATH"/"$SUBPATH"/dist - cd "$DBT_PATH"/"$SUBPATH" - python setup.py sdist - cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/ -done - -cd "$DBT_PATH" -python setup.py sdist - -set +x