From a6f10dc1a38f125a81394d59241faa4f5b183c00 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:31 -0700 Subject: [PATCH 01/17] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 633cebe92..3ee7047d3 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,6 +87,14 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + with project.adapter.connection_named("__test_grants"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + yield relation + project.adapter.drop_relation(relation) @pytest.fixture(scope="class") def models(self): return { From 7b669312a49f7aa152b338dcd1c9e54193b1122d Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:51 -0700 Subject: [PATCH 02/17] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 3ee7047d3..77213bf26 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -95,6 +95,7 @@ def setup(self, project): ) yield relation project.adapter.drop_relation(relation) + @pytest.fixture(scope="class") def models(self): return { From 16573f1474dd45b4564e339fb828fea87d19d2af Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 22 Sep 2023 11:45:24 -0700 Subject: [PATCH 03/17] revert setup --- tests/functional/adapter/test_grant_access_to.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 77213bf26..633cebe92 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,15 +87,6 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - with project.adapter.connection_named("__test_grants"): - relation = project.adapter.Relation.create( - database=project.database, schema=f"{project.test_schema}_seeds" - ) - yield relation - project.adapter.drop_relation(relation) - @pytest.fixture(scope="class") def models(self): return { From 69553c46972815e3e571b3e9661838f2090b2c4d Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 7 Nov 2023 17:37:39 -0800 Subject: [PATCH 04/17] fix partitioninb --- dbt/adapters/bigquery/dataproc/batch.py | 2 +- dbt/include/bigquery/macros/materializations/table.sql | 5 ++++- tests/functional/adapter/test_python_model.py | 2 +- tox.ini | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/bigquery/dataproc/batch.py b/dbt/adapters/bigquery/dataproc/batch.py index 0dc54aa78..86c4e3b28 100644 --- a/dbt/adapters/bigquery/dataproc/batch.py +++ b/dbt/adapters/bigquery/dataproc/batch.py @@ -13,7 +13,7 @@ from dbt.adapters.bigquery.connections import DataprocBatchConfig _BATCH_RUNNING_STATES = [Batch.State.PENDING, Batch.State.RUNNING] -DEFAULT_JAR_FILE_URI = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.21.1.jar" +DEFAULT_JAR_FILE_URI = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.23.2.jar" def create_batch_request( diff --git a/dbt/include/bigquery/macros/materializations/table.sql b/dbt/include/bigquery/macros/materializations/table.sql index 16b454351..011c6bfda 100644 --- a/dbt/include/bigquery/macros/materializations/table.sql +++ b/dbt/include/bigquery/macros/materializations/table.sql @@ -110,14 +110,17 @@ df.write \ .mode("overwrite") \ .format("bigquery") \ .option("writeMethod", "indirect").option("writeDisposition", 'WRITE_TRUNCATE') \ + {%- if partition_config is not none %} {%- if partition_config.data_type | lower in ('date','timestamp','datetime') %} .option("partitionField", "{{- partition_config.field -}}") \ {%- if partition_config.granularity is not none %} .option("partitionType", "{{- partition_config.granularity -}}") \ {%- endif %} {%- endif %} + {%- endif %} {%- if raw_cluster_by is not none %} - .option("clusteredFields", "{{- raw_cluster_by|join(',') -}}") \ + .option("clusteredFields", "{{- raw_cluster_by | join(',') -}}") \ {%- endif %} + .save("{{target_relation}}") {% endmacro %} diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index b67384667..1df27d2dc 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -29,7 +29,7 @@ def models(self): return {"30_sec_python_model.py": blocks_for_thirty_sec} def test_model_times_out(self, project): - result, output = run_dbt_and_capture(["run"], expect_pass=False) + result, output = run_dbt_and_capture(["run", "--debug"], expect_pass=False) assert len(result) == 1 assert "Operation did not complete within the designated timeout of 5 seconds." in output diff --git a/tox.ini b/tox.ini index 1d5413dcf..ec9701620 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,8 @@ passenv = DD_ENV DD_SERVICE commands = - {envpython} -m pytest {posargs} -vv tests/functional -k "TestPython" --profile service_account +;TestPythonPartitionedModels + {envpython} -m pytest {posargs} -vv tests/functional -k "TestPythonModelDataprocTimeoutTest" --profile service_account deps = -rdev-requirements.txt -e. From fb11e52d45319599a9525d585a0925121c7bcd0f Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 8 Nov 2023 17:09:38 -0800 Subject: [PATCH 05/17] skip TestPythonBatchIdModels --- dbt/adapters/bigquery/dataproc/batch.py | 2 +- dbt/adapters/bigquery/python_submissions.py | 13 ++++++++++++- .../bigquery/macros/materializations/table.sql | 3 +-- tests/functional/adapter/test_python_model.py | 1 + tox.ini | 3 +-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dbt/adapters/bigquery/dataproc/batch.py b/dbt/adapters/bigquery/dataproc/batch.py index 86c4e3b28..61dc3c18b 100644 --- a/dbt/adapters/bigquery/dataproc/batch.py +++ b/dbt/adapters/bigquery/dataproc/batch.py @@ -13,7 +13,7 @@ from dbt.adapters.bigquery.connections import DataprocBatchConfig _BATCH_RUNNING_STATES = [Batch.State.PENDING, Batch.State.RUNNING] -DEFAULT_JAR_FILE_URI = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.23.2.jar" +DEFAULT_JAR_FILE_URI = "gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.13-0.34.0.jar" def create_batch_request( diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py index 8fd354eb5..0a8449c92 100644 --- a/dbt/adapters/bigquery/python_submissions.py +++ b/dbt/adapters/bigquery/python_submissions.py @@ -1,5 +1,7 @@ from typing import Dict, Union +from dbt.events import AdapterLogger + from dbt.adapters.base import PythonJobHelper from google.api_core.future.polling import POLLING_PREDICATE @@ -17,6 +19,7 @@ ) OPERATION_RETRY_TIME = 10 +logger = AdapterLogger("BigQuery") class BaseDataProcHelper(PythonJobHelper): @@ -122,10 +125,18 @@ def _get_job_client(self) -> dataproc_v1.BatchControllerClient: ) def _get_batch_id(self) -> str: - return self.parsed_model["config"].get("batch_id") + model = self.parsed_model + if "batch_id" in model["config"]: + batch_id = model["config"]["batch_id"] + else: + batch_id = model["unique_id"].replace(".", "-").replace("_", "-") + str( + int(model["created_at"]) + ) + return batch_id def _submit_dataproc_job(self) -> Batch: batch_id = self._get_batch_id() + logger.info(f"Submitting batch job with id: {batch_id}") request = create_batch_request( batch=self._configure_batch(), batch_id=batch_id, diff --git a/dbt/include/bigquery/macros/materializations/table.sql b/dbt/include/bigquery/macros/materializations/table.sql index 011c6bfda..40d4a8ea2 100644 --- a/dbt/include/bigquery/macros/materializations/table.sql +++ b/dbt/include/bigquery/macros/materializations/table.sql @@ -114,13 +114,12 @@ df.write \ {%- if partition_config.data_type | lower in ('date','timestamp','datetime') %} .option("partitionField", "{{- partition_config.field -}}") \ {%- if partition_config.granularity is not none %} - .option("partitionType", "{{- partition_config.granularity -}}") \ + .option("partitionType", "{{- partition_config.granularity|upper -}}") \ {%- endif %} {%- endif %} {%- endif %} {%- if raw_cluster_by is not none %} .option("clusteredFields", "{{- raw_cluster_by | join(',') -}}") \ {%- endif %} - .save("{{target_relation}}") {% endmacro %} diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 1df27d2dc..b9db4288e 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -216,6 +216,7 @@ def model(dbt, spark): """ +@pytest.mark.skip(reason="Currently failing as run_started_at is the same across dbt runs") class TestPythonBatchIdModels: @pytest.fixture(scope="class") def models(self): diff --git a/tox.ini b/tox.ini index ec9701620..1d5413dcf 100644 --- a/tox.ini +++ b/tox.ini @@ -48,8 +48,7 @@ passenv = DD_ENV DD_SERVICE commands = -;TestPythonPartitionedModels - {envpython} -m pytest {posargs} -vv tests/functional -k "TestPythonModelDataprocTimeoutTest" --profile service_account + {envpython} -m pytest {posargs} -vv tests/functional -k "TestPython" --profile service_account deps = -rdev-requirements.txt -e. From 31ba7ed3d47013fed2b3c374f389fd24d874c6e3 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 8 Nov 2023 17:11:39 -0800 Subject: [PATCH 06/17] add changie --- .changes/unreleased/Fixes-20231108-171128.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20231108-171128.yaml diff --git a/.changes/unreleased/Fixes-20231108-171128.yaml b/.changes/unreleased/Fixes-20231108-171128.yaml new file mode 100644 index 000000000..116ff00d2 --- /dev/null +++ b/.changes/unreleased/Fixes-20231108-171128.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix broken partition config granularity and batch_id being set to None +time: 2023-11-08T17:11:28.819877-08:00 +custom: + Author: colin-rogers-dbt + Issue: "1006" From 50ced2c86c8ff8e0112c4e32c0c44036df064535 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 8 Nov 2023 19:24:19 -0800 Subject: [PATCH 07/17] run python integration tests separately --- .github/workflows/integration.yml | 75 ++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bb0211b35..c9e32cb77 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -97,10 +97,6 @@ jobs: - 'dbt/**' - 'tests/**' - 'dev-requirements.txt' - bigquery-python: - - 'dbt/adapters/bigquery/dataproc/**' - - 'dbt/adapters/bigquery/python_submissions.py' - - 'dbt/include/bigquery/python_model/**' - name: Generate integration test matrix id: generate-matrix @@ -192,21 +188,6 @@ jobs: GCS_BUCKET: dbt-ci run: tox -- --ddtrace - # python models tests are slow so we only want to run them if we're changing them - - name: Run tox (python models) - if: needs.test-metadata.outputs.run-python-tests == 'true' - env: - BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} - BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} - BIGQUERY_TEST_NO_ACCESS_DATABASE: ${{ secrets.BIGQUERY_TEST_NO_ACCESS_DATABASE }} - DBT_TEST_USER_1: group:buildbot@dbtlabs.com - DBT_TEST_USER_2: group:engineering-core-team@dbtlabs.com - DBT_TEST_USER_3: serviceAccount:dbt-integration-test-user@dbt-test-env.iam.gserviceaccount.com - DATAPROC_REGION: us-central1 - DATAPROC_CLUSTER_NAME: dbt-test-1 - GCS_BUCKET: dbt-ci - run: tox -e python-tests -- --ddtrace - - uses: actions/upload-artifact@v3 if: always() with: @@ -225,10 +206,64 @@ jobs: name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv path: integration_results.csv + # python integration tests are slow so we only run them seperately and for a single OS / python version + test-python: + name: "test-python" + runs-on: ubuntu-latest + if: >- + ( + github.event_name != 'pull_request_target' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'ok to test') + ) + + steps: + - name: Check out the repository + if: github.event_name != 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + + # explicitly checkout the branch for the PR, + # this is necessary for the `pull_request_target` event + - name: Check out the repository (PR) + if: github.event_name == 'pull_request_target' + uses: actions/checkout@v3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: "3.8" + + - name: Install python dependencies + run: | + python -m pip install --user --upgrade pip + python -m pip install tox + python -m pip --version + tox --version + + - name: Run tox (python models) + env: + BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }} + BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }} + BIGQUERY_TEST_NO_ACCESS_DATABASE: ${{ secrets.BIGQUERY_TEST_NO_ACCESS_DATABASE }} + DBT_TEST_USER_1: group:buildbot@dbtlabs.com + DBT_TEST_USER_2: group:engineering-core-team@dbtlabs.com + DBT_TEST_USER_3: serviceAccount:dbt-integration-test-user@dbt-test-env.iam.gserviceaccount.com + DATAPROC_REGION: us-central1 + DATAPROC_CLUSTER_NAME: dbt-test-1 + GCS_BUCKET: dbt-ci + run: tox -e python-tests -- --ddtrace + require-label-comment: runs-on: ubuntu-latest - needs: test + needs: + - test + - test-python permissions: pull-requests: write From b0e4d3fe046f9375fe9861756757d3c62a95537b Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 8 Nov 2023 19:28:28 -0800 Subject: [PATCH 08/17] run python integration tests separately --- .github/workflows/integration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c9e32cb77..e67614398 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -211,6 +211,8 @@ jobs: name: "test-python" runs-on: ubuntu-latest if: >- + needs.test-metadata.outputs.matrix && + fromJSON( needs.test-metadata.outputs.matrix ).include[0] && ( github.event_name != 'pull_request_target' || github.event.pull_request.head.repo.full_name == github.repository || From 37bcc510954339de1226930da6e01edd7833e049 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 9 Nov 2023 09:14:34 -0800 Subject: [PATCH 09/17] cleanup _get_batch_id --- dbt/adapters/bigquery/python_submissions.py | 10 +++------- tests/functional/adapter/test_python_model.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py index 0a8449c92..114ebf979 100644 --- a/dbt/adapters/bigquery/python_submissions.py +++ b/dbt/adapters/bigquery/python_submissions.py @@ -126,13 +126,9 @@ def _get_job_client(self) -> dataproc_v1.BatchControllerClient: def _get_batch_id(self) -> str: model = self.parsed_model - if "batch_id" in model["config"]: - batch_id = model["config"]["batch_id"] - else: - batch_id = model["unique_id"].replace(".", "-").replace("_", "-") + str( - int(model["created_at"]) - ) - return batch_id + default_batch_id = model["unique_id"].replace(".", "-").replace("_", "-") + default_batch_id += str(int(model["created_at"])) + return model["config"].get("batch_id", default_batch_id) def _submit_dataproc_job(self) -> Batch: batch_id = self._get_batch_id() diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index b9db4288e..1e1c2775c 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -29,7 +29,7 @@ def models(self): return {"30_sec_python_model.py": blocks_for_thirty_sec} def test_model_times_out(self, project): - result, output = run_dbt_and_capture(["run", "--debug"], expect_pass=False) + result, output = run_dbt_and_capture(["run"], expect_pass=False) assert len(result) == 1 assert "Operation did not complete within the designated timeout of 5 seconds." in output From 52535459b9dd7d07c935fe0f9b27aba54aab21de Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 9 Nov 2023 09:15:22 -0800 Subject: [PATCH 10/17] add space to pipe --- dbt/include/bigquery/macros/materializations/table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/bigquery/macros/materializations/table.sql b/dbt/include/bigquery/macros/materializations/table.sql index 40d4a8ea2..68117b06a 100644 --- a/dbt/include/bigquery/macros/materializations/table.sql +++ b/dbt/include/bigquery/macros/materializations/table.sql @@ -114,7 +114,7 @@ df.write \ {%- if partition_config.data_type | lower in ('date','timestamp','datetime') %} .option("partitionField", "{{- partition_config.field -}}") \ {%- if partition_config.granularity is not none %} - .option("partitionType", "{{- partition_config.granularity|upper -}}") \ + .option("partitionType", "{{- partition_config.granularity| upper -}}") \ {%- endif %} {%- endif %} {%- endif %} From c77815a923036c3f187ac0b09865a8e00a1b33a0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 9 Nov 2023 09:21:47 -0800 Subject: [PATCH 11/17] fix integration.yml --- .github/workflows/integration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e67614398..7df6973a8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -209,6 +209,7 @@ jobs: # python integration tests are slow so we only run them seperately and for a single OS / python version test-python: name: "test-python" + needs: test-metadata runs-on: ubuntu-latest if: >- needs.test-metadata.outputs.matrix && From 2d02430686f8bf6ba89950507054f3cf9849c960 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 9 Nov 2023 09:50:33 -0800 Subject: [PATCH 12/17] add extra changie --- .changes/unreleased/Under the Hood-20231109-095012.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20231109-095012.yaml diff --git a/.changes/unreleased/Under the Hood-20231109-095012.yaml b/.changes/unreleased/Under the Hood-20231109-095012.yaml new file mode 100644 index 000000000..a93215e8f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20231109-095012.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Upgrade spark-bigquery Java deps for serverless to 2.13-0.34.0 +time: 2023-11-09T09:50:12.252774-08:00 +custom: + Author: colin-rogers-dbt + Issue: "1006" From 731cf54a51b2d9649885d86128a8074839e34862 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:31 -0700 Subject: [PATCH 13/17] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 633cebe92..3ee7047d3 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,6 +87,14 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + with project.adapter.connection_named("__test_grants"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + yield relation + project.adapter.drop_relation(relation) @pytest.fixture(scope="class") def models(self): return { From d97bb5fe74c6e2bb240dcd77a3f1adef93feac23 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:51 -0700 Subject: [PATCH 14/17] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 3ee7047d3..77213bf26 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -95,6 +95,7 @@ def setup(self, project): ) yield relation project.adapter.drop_relation(relation) + @pytest.fixture(scope="class") def models(self): return { From c584acb58026af6e33926e55b6b6ab7e47c9c86f Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 22 Sep 2023 11:45:24 -0700 Subject: [PATCH 15/17] revert setup --- tests/functional/adapter/test_grant_access_to.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 77213bf26..633cebe92 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,15 +87,6 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - with project.adapter.connection_named("__test_grants"): - relation = project.adapter.Relation.create( - database=project.database, schema=f"{project.test_schema}_seeds" - ) - yield relation - project.adapter.drop_relation(relation) - @pytest.fixture(scope="class") def models(self): return { From d76fdc2d694d42240a09b35b5f6b15df047c3dc3 Mon Sep 17 00:00:00 2001 From: Colin Date: Sat, 11 Nov 2023 15:09:26 -0800 Subject: [PATCH 16/17] replace deterministic default batch_id with uuid --- dbt/adapters/bigquery/python_submissions.py | 4 ++-- tests/functional/adapter/test_python_model.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/bigquery/python_submissions.py b/dbt/adapters/bigquery/python_submissions.py index 114ebf979..6ea5a5564 100644 --- a/dbt/adapters/bigquery/python_submissions.py +++ b/dbt/adapters/bigquery/python_submissions.py @@ -1,3 +1,4 @@ +import uuid from typing import Dict, Union from dbt.events import AdapterLogger @@ -126,8 +127,7 @@ def _get_job_client(self) -> dataproc_v1.BatchControllerClient: def _get_batch_id(self) -> str: model = self.parsed_model - default_batch_id = model["unique_id"].replace(".", "-").replace("_", "-") - default_batch_id += str(int(model["created_at"])) + default_batch_id = str(uuid.uuid4()) return model["config"].get("batch_id", default_batch_id) def _submit_dataproc_job(self) -> Batch: diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 1e1c2775c..bc5d8c91c 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -187,8 +187,6 @@ def model(dbt, spark): models: - name: python_array_batch_id description: A random table with a calculated column defined in python. - config: - batch_id: '{{ run_started_at.strftime("%Y-%m-%d-%H-%M-%S") }}-python-array' columns: - name: A description: Column A @@ -216,7 +214,6 @@ def model(dbt, spark): """ -@pytest.mark.skip(reason="Currently failing as run_started_at is the same across dbt runs") class TestPythonBatchIdModels: @pytest.fixture(scope="class") def models(self): From 3ee7e312dff0a1eccc4770f89a471312e9476599 Mon Sep 17 00:00:00 2001 From: Colin Date: Sat, 11 Nov 2023 15:10:17 -0800 Subject: [PATCH 17/17] add changie --- .changes/unreleased/Fixes-20231111-150959.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20231111-150959.yaml diff --git a/.changes/unreleased/Fixes-20231111-150959.yaml b/.changes/unreleased/Fixes-20231111-150959.yaml new file mode 100644 index 000000000..3d9f245a6 --- /dev/null +++ b/.changes/unreleased/Fixes-20231111-150959.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: replace deterministic batch_id with uuid +time: 2023-11-11T15:09:59.243797-08:00 +custom: + Author: colin-rogers-dbt + Issue: "1006"