Merge pull request #513 from dbt-labs/er/tox

Er/tox

.circleci/config.yml

@@ -13,11 +13,11 @@ jobs:
       - run:
           name: "Run Tests - Postgres"
           environment:
-            POSTGRES_TEST_HOST: localhost
-            POSTGRES_TEST_USER: root
-            POSTGRES_TEST_PASS: ''
-            POSTGRES_TEST_PORT: 5432
-            POSTGRES_TEST_DBNAME: circle_test
+            POSTGRES_HOST: localhost
+            POSTGRES_USER: root
+            DBT_ENV_SECRET_POSTGRES_PASS: ''
+            POSTGRES_PORT: 5432
+            POSTGRES_DATABASE: circle_test
           command: ./run_test.sh postgres
       - store_artifacts:
           path: ./integration_tests/logs

.github/workflows/local_only.yml

@@ -0,0 +1,83 @@
+# **what?**
+# Run tests for packages not supported for cloud testing
+#
+# **why?**
+# To ensure that packages works as expected with all supported adapters
+
+# **when?**
+# On push, PR or manually called
+
+
+name: Package Integration Tests - Local Only
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+env:
+    PYTHON_VERSION: "3.11"
+    POSTGRES_HOST: "localhost"
+    POSTGRES_USER: "root"
+    POSTGRES_PORT: "5432"
+    POSTGRES_DATABASE: "postgres_test"
+    DBT_ENV_SECRET_POSTGRES_PASS: "password"  # this isn't actually a secret since it only runs on the runner
+
+jobs:
+    run-tests:
+        runs-on: ubuntu-latest
+        services:
+            postgres:
+                image: postgres
+                env:
+                    POSTGRES_USER: ${{ env.POSTGRES_USER }}
+                    POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }}
+                    POSTGRES_DB: ${{ env.POSTGRES_DATABASE }}
+                    POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+                ports:
+                    - 5432:5432
+        strategy:
+            fail-fast: false
+            matrix:
+                # these adapters are tested in this repo but are not tested as part of the dbt Cloud images.
+                # This list should include anything not listed in supported_adapters.env
+                adapter: [duckdb, postgres]
+
+        steps:
+            - name: "Checkout ${{ github.event.repository }} "
+              uses: actions/checkout@v4
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install ${{ matrix.adapter }}"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install dbt-${{ matrix.adapter }}
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Run integration tests with tox on ${{ matrix.adapter }}"
+              run: |
+                  tox -e dbt_integration_${{ matrix.adapter }}
+              env:
+                # postgres
+                POSTGRES_HOST: ${{ env.POSTGRES_HOST }}
+                POSTGRES_USER: ${{ env.POSTGRES_USER }}
+                DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }}
+                POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
+                POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }}
+                POSTGRES_SCHEMA: "integration_tests_postgres_${{ github.run_number }}"
+                # duckdb - needs no vars

integration_tests/profiles.yml

@@ -1,5 +1,5 @@
 
-# HEY! This file is used in the integration tests with CircleCI.
+# HEY! This file is used in the integration tests with CI.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
@@ -16,7 +16,7 @@ integration_tests:
       pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
       port: "{{ env_var('POSTGRES_PORT') | as_number }}"
       dbname: "{{ env_var('POSTGRES_DATABASE') }}"
-      schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_test') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}"
       threads: 5
 
     redshift:
@@ -26,7 +26,7 @@ integration_tests:
       pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
       dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
       port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
-      schema: "{{ env_var('REDSHIFT_SCHEMA') }}"
+      schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}"
       threads: 5
 
     bigquery:
@@ -35,7 +35,7 @@ integration_tests:
       keyfile_json:
         "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}"
       project: "{{ env_var('BIGQUERY_PROJECT') }}"
-      dataset: "{{ env_var('BIGQUERY_SCHEMA') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}"
       threads: 10
 
     snowflake:
@@ -46,7 +46,7 @@ integration_tests:
       role: "{{ env_var('SNOWFLAKE_ROLE') }}"
       database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
       warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
-      schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}"
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}"
       threads: 10
 
     databricks:

integration_tests/ci/sample.profiles.yml → integration_tests_2/profiles.yml

@@ -1,5 +1,5 @@
 
-# HEY! This file is used in the integration tests with CircleCI.
+# HEY! This file is used in the integration tests with CI.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
@@ -11,46 +11,47 @@ integration_tests:
   outputs:
     postgres:
       type: postgres
-      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
-      user: "{{ env_var('POSTGRES_TEST_USER') }}"
-      pass: "{{ env_var('POSTGRES_TEST_PASS') }}"
-      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
-      dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
-      schema: dbt_project_evaluator_integration_tests_postgres
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}"
       threads: 5
 
     redshift:
       type: redshift
-      host: "{{ env_var('REDSHIFT_TEST_HOST') }}"
-      user: "{{ env_var('REDSHIFT_TEST_USER') }}"
-      pass: "{{ env_var('REDSHIFT_TEST_PASS') }}"
-      dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}"
-      port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}"
-      schema: dbt_project_evaluator_integration_tests_redshift
+      host: "{{ env_var('REDSHIFT_HOST') }}"
+      user: "{{ env_var('REDSHIFT_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}"
+      dbname: "{{ env_var('REDSHIFT_DATABASE') }}"
+      port: "{{ env_var('REDSHIFT_PORT') | as_number }}"
+      schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}"
       threads: 5
 
     bigquery:
       type: bigquery
       method: service-account
-      keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}"
-      project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}"
-      schema: dbt_project_evaluator_integration_tests_bigquery
+      keyfile_json:
+        "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}"
+      project: "{{ env_var('BIGQUERY_PROJECT') }}"
+      dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}"
       threads: 10
 
     snowflake:
       type: snowflake
-      account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}"
-      user: "{{ env_var('SNOWFLAKE_TEST_USER') }}"
-      password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}"
-      role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}"
-      database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}"
-      warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}"
-      schema: dbt_project_evaluator_integration_tests_snowflake
+      account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ env_var('SNOWFLAKE_USER') }}"
+      password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}"
+      role: "{{ env_var('SNOWFLAKE_ROLE') }}"
+      database: "{{ env_var('SNOWFLAKE_DATABASE') }}"
+      warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}"
+      schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}"
       threads: 10
 
     databricks:
       type: databricks
-      schema: dbt_project_evaluator_integration_tests_databricks
+      schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}"
       host: "{{ env_var('DATABRICKS_TEST_HOST') }}"
       http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}"
       token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}"

run_test.sh

@@ -13,12 +13,6 @@ fi
 . $VENV
 
 cd integration_tests
-
-if [[ ! -e ~/.dbt/profiles.yml ]]; then
-    mkdir -p ~/.dbt
-    cp ci/sample.profiles.yml ~/.dbt/profiles.yml
-fi
-
 dbt deps --target $1 || exit 1
 dbt build -x --target $1 --full-refresh || exit 1
 

run_tox_tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+echo "Running tests for the first project"
+cd integration_tests
+dbt deps --target $1 || exit 1
+dbt build -x --target $1 --full-refresh || exit 1
+
+echo "Running tests for the second project"
+cd ../integration_tests_2
+dbt deps --target $1 || exit 1
+dbt seed --full-refresh --target $1 || exit 1
+dbt run -x --target $1 --full-refresh || exit 1

tox.ini

@@ -34,104 +34,50 @@ passenv =
 # Snowflake integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_snowflake]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target snowflake
-    dbt build -x --target snowflake --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target snowflake
-    dbt seed --full-refresh --target snowflake
-    dbt run -x --target snowflake --full-refresh
-
+    bash ./run_tox_tests.sh snowflake
 
 # Postgres integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_postgres]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target postgres
-    dbt build -x --target postgres --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target postgres
-    dbt seed --full-refresh --target postgres
-    dbt run -x --target postgres --full-refresh
+    bash ./run_tox_tests.sh postgres
 
 # BigQuery integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_bigquery]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target bigquery
-    dbt build -x --target bigquery --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target bigquery
-    dbt seed --full-refresh --target bigquery
-    dbt run -x --target bigquery --full-refresh
+    bash ./run_tox_tests.sh bigquery
 
 # redshift integration tests for centralized dbt testing
 # run dbt commands directly, assumes dbt is already installed in environment
 [testenv:dbt_integration_redshift]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target redshift
-    dbt build -x --target redshift --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target redshift
-    dbt seed --full-refresh --target redshift
-    dbt run -x --target redshift --full-refresh
+    bash ./run_tox_tests.sh redshift
 
+# note that duckdb is not a supported dbt target for dbt Cloud testing
 [testenv:dbt_integration_duckdb]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target duckdb
-    dbt build -x --target duckdb --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target duckdb
-    dbt seed --full-refresh --target duckdb
-    dbt run -x --target duckdb --full-refresh
+    bash ./run_tox_tests.sh duckdb
 
 [testenv:dbt_integration_databricks]
-changedir = integration_tests
 allowlist_externals = 
-    dbt
-    cd
+    bash
 skip_install = true
 commands =
-    dbt deps --target duckdb
-    dbt build -x --target duckdb --full-refresh
-
-    # test with the second project
-    cd ../integration_tests_2
-    dbt deps --target duckdb
-    dbt seed --full-refresh --target duckdb
-    dbt run -x --target duckdb --full-refresh
+    bash ./run_tox_tests.sh databricks