diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7cccf8ede..52cc0acca 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.8.0b2 +current_version = 1.8.0b3 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.8.0-b3.md b/.changes/1.8.0-b3.md new file mode 100644 index 000000000..5f9d828c3 --- /dev/null +++ b/.changes/1.8.0-b3.md @@ -0,0 +1,16 @@ +## dbt-redshift 1.8.0-b3 - April 18, 2024 + +### Fixes + +- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + +### Under the Hood + +- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) + +### Contributors +- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) diff --git a/.changes/1.8.0/Fixes-20240326-123703.yaml b/.changes/1.8.0/Fixes-20240326-123703.yaml new file mode 100644 index 000000000..5d9bee694 --- /dev/null +++ b/.changes/1.8.0/Fixes-20240326-123703.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: dbt can cancel open queries upon interrupt +time: 2024-03-26T12:37:03.17481-05:00 +custom: + Author: holly-evans + Issue: "705" diff --git a/.changes/1.8.0/Security-20240416-195919.yaml b/.changes/1.8.0/Security-20240416-195919.yaml new file mode 100644 index 000000000..af8fb6f1d --- /dev/null +++ b/.changes/1.8.0/Security-20240416-195919.yaml @@ -0,0 +1,6 @@ +kind: Security +body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core +time: 2024-04-16T19:59:19.233806-05:00 +custom: + Author: McKnight-42 + PR: "768" diff --git a/.changes/unreleased/Under the Hood-20240410-182912.yaml b/.changes/1.8.0/Under the Hood-20240410-182912.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240410-182912.yaml rename to .changes/1.8.0/Under the Hood-20240410-182912.yaml diff --git a/.changes/unreleased/Features-20240404-171441.yaml b/.changes/unreleased/Features-20240404-171441.yaml new file mode 100644 index 000000000..f1ac623c0 --- /dev/null +++ b/.changes/unreleased/Features-20240404-171441.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Support TableLastModifiedMetadataBatch capability +time: 2024-04-04T17:14:41.313087-07:00 +custom: + Author: michelleark + Issue: "755" diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b08ffcd53..000000000 --- a/.flake8 +++ /dev/null @@ -1,16 +0,0 @@ -[flake8] -select = - E - W - F -ignore = - # makes Flake8 work like black - W503, - W504, - # makes Flake8 work like black - E203, - E741, - E501, -exclude = test -per-file-ignores = - */__init__.py: F401 diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 7db445d9e..ccb6d949d 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -37,7 +37,7 @@ module.exports = ({ context }) => { if (labels.includes("test macos") || testAllLabel) { include.push({ - os: "macos-latest", + os: "macos-12", adapter, "python-version": pythonVersion, }); @@ -70,7 +70,7 @@ module.exports = ({ context }) => { // additionally include runs for all adapters, on macos and windows, // but only for the default python version for (const adapter of supportedAdapters) { - for (const operatingSystem of ["windows-latest", "macos-latest"]) { + for (const operatingSystem of ["windows-latest", "macos-12"]) { include.push({ os: operatingSystem, adapter: adapter, diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9d1fe0807..ad29fef1e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -174,6 +174,33 @@ jobs: pip install bumpversion ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }} + - name: Create AWS IAM profiles + run: | + aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID + aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY + aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION + aws configure --profile $AWS_USER_PROFILE set output json + + aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID + aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY + aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_SOURCE_PROFILE set output json + + aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE + aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN + aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION + aws configure --profile $AWS_ROLE_PROFILE set output json + env: + AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user + AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} + AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }} + AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }} + AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }} + AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + - name: Run tox (redshift) if: matrix.adapter == 'redshift' env: @@ -182,6 +209,12 @@ jobs: REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} + REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }} + REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }} + REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }} + REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }} + REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }} + REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }} DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 064f4cca2..5527b568f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,6 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip --version pre-commit --version - mypy --version - name: pre-commit hooks run: pre-commit run --all-files --show-diff-on-failure @@ -174,7 +173,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-12, windows-latest] python-version: ['3.8', '3.9', '3.10', '3.11'] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d80b955c..ae249943d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,55 @@ -# For more on configuring pre-commit hooks (see https://pre-commit.com/) - -# Force all unspecified python hooks to run python 3.8 default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - id: black - alias: black-check - stages: [manual] - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - "--check" - - "--diff" -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - id: flake8 - alias: flake8-check - stages: [manual] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 - hooks: - - id: mypy - # N.B.: Mypy is... a bit fragile. - # - # By using `language: system` we run this hook in the local - # environment instead of a pre-commit isolated one. This is needed - # to ensure mypy correctly parses the project. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + additional_dependencies: [flaky] - # It may cause trouble in that it adds environmental variables out - # of our control to the mix. Unfortunately, there's nothing we can - # do about per pre-commit's author. - # See https://github.com/pre-commit/pre-commit/issues/730 for details. - args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters/.* - language: system - - id: mypy - alias: mypy-check - stages: [manual] - args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters - language: system +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + args: + - --show-error-codes + - --pretty + - --ignore-missing-imports + - --explicit-package-bases + files: ^dbt/adapters + additional_dependencies: + - types-pytz + - types-requests diff --git a/CHANGELOG.md b/CHANGELOG.md index 200815006..f45988f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-redshift 1.8.0-b3 - April 18, 2024 + +### Fixes + +- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + +### Under the Hood + +- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759)) + +### Security + +- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768)) + +### Contributors +- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705)) + + ## dbt-redshift 1.8.0-b2 - April 03, 2024 ### Features @@ -28,8 +46,6 @@ - Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743)) - - ## dbt-redshift 1.8.0-b1 - March 01, 2024 ### Features diff --git a/dbt/adapters/redshift/__version__.py b/dbt/adapters/redshift/__version__.py index 7d16c28f0..b0f82cbca 100644 --- a/dbt/adapters/redshift/__version__.py +++ b/dbt/adapters/redshift/__version__.py @@ -1 +1 @@ -version = "1.8.0b2" +version = "1.8.0b3" diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index b0fc0825d..b890127c6 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -50,7 +50,7 @@ class UserSSLMode(StrEnum): @classmethod def default(cls) -> "UserSSLMode": # default for `psycopg2`, which aligns with dbt-redshift 1.4 and provides backwards compatibility - return cls.prefer + return cls("prefer") class RedshiftSSLMode(StrEnum): @@ -60,11 +60,11 @@ class RedshiftSSLMode(StrEnum): SSL_MODE_TRANSLATION = { UserSSLMode.disable: None, - UserSSLMode.allow: RedshiftSSLMode.verify_ca, - UserSSLMode.prefer: RedshiftSSLMode.verify_ca, - UserSSLMode.require: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_ca: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_full: RedshiftSSLMode.verify_full, + UserSSLMode.allow: RedshiftSSLMode("verify-ca"), + UserSSLMode.prefer: RedshiftSSLMode("verify-ca"), + UserSSLMode.require: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_ca: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_full: RedshiftSSLMode("verify-full"), } @@ -233,27 +233,26 @@ def connect(): class RedshiftConnectionManager(SQLConnectionManager): TYPE = "redshift" - def _get_backend_pid(self): - sql = "select pg_backend_pid()" - _, cursor = self.add_query(sql) - - res = cursor.fetchone() - return res[0] - def cancel(self, connection: Connection): + pid = connection.backend_pid # type: ignore + sql = f"select pg_terminate_backend({pid})" + logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") + logger.debug(sql) + try: - pid = self._get_backend_pid() + self.add_query(sql) except redshift_connector.InterfaceError as e: if "is closed" in str(e): logger.debug(f"Connection {connection.name} was already closed") return raise - sql = f"select pg_terminate_backend({pid})" - cursor = connection.handle.cursor() - logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}") - logger.debug(sql) - cursor.execute(sql) + @classmethod + def _get_backend_pid(cls, connection): + with connection.handle.cursor() as c: + sql = "select pg_backend_pid()" + res = c.execute(sql).fetchone() + return res[0] @classmethod def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse: @@ -325,7 +324,7 @@ def exponential_backoff(attempt: int): redshift_connector.DataError, ] - return cls.retry_connection( + open_connection = cls.retry_connection( connection, connect=connect_method_factory.get_connect_method(), logger=logger, @@ -333,6 +332,8 @@ def exponential_backoff(attempt: int): retry_timeout=exponential_backoff, retryable_exceptions=retryable_exceptions, ) + open_connection.backend_pid = cls._get_backend_pid(open_connection) # type: ignore + return open_connection def execute( self, diff --git a/dbt/adapters/redshift/impl.py b/dbt/adapters/redshift/impl.py index a77601895..18faee48c 100644 --- a/dbt/adapters/redshift/impl.py +++ b/dbt/adapters/redshift/impl.py @@ -58,6 +58,7 @@ class RedshiftAdapter(SQLAdapter): { Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), + Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full), } ) diff --git a/dbt/adapters/redshift/relation_configs/dist.py b/dbt/adapters/redshift/relation_configs/dist.py index c41eda578..0104d8db4 100644 --- a/dbt/adapters/redshift/relation_configs/dist.py +++ b/dbt/adapters/redshift/relation_configs/dist.py @@ -24,7 +24,7 @@ class RedshiftDistStyle(StrEnum): @classmethod def default(cls) -> "RedshiftDistStyle": - return cls.auto + return cls("auto") @dataclass(frozen=True, eq=True, unsafe_hash=True) @@ -103,7 +103,7 @@ def parse_relation_config(cls, relation_config: RelationConfig) -> dict: config = {"diststyle": diststyle} else: - config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} + config = {"diststyle": RedshiftDistStyle.key.value, "distkey": dist} # type: ignore return config diff --git a/dbt/adapters/redshift/relation_configs/materialized_view.py b/dbt/adapters/redshift/relation_configs/materialized_view.py index 05f4b170d..f6d93754e 100644 --- a/dbt/adapters/redshift/relation_configs/materialized_view.py +++ b/dbt/adapters/redshift/relation_configs/materialized_view.py @@ -57,7 +57,7 @@ class RedshiftMaterializedViewConfig(RedshiftRelationConfigBase, RelationConfigV database_name: str query: str backup: bool = field(default=True, compare=False, hash=False) - dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle.even) + dist: RedshiftDistConfig = RedshiftDistConfig(diststyle=RedshiftDistStyle("even")) sort: RedshiftSortConfig = RedshiftSortConfig() autorefresh: bool = False diff --git a/dbt/adapters/redshift/relation_configs/sort.py b/dbt/adapters/redshift/relation_configs/sort.py index e44784c2f..91152615e 100644 --- a/dbt/adapters/redshift/relation_configs/sort.py +++ b/dbt/adapters/redshift/relation_configs/sort.py @@ -23,11 +23,11 @@ class RedshiftSortStyle(StrEnum): @classmethod def default(cls) -> "RedshiftSortStyle": - return cls.auto + return cls("auto") @classmethod def default_with_columns(cls) -> "RedshiftSortStyle": - return cls.compound + return cls("compound") @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/dev-requirements.txt b/dev-requirements.txt index 85edead99..d02863ae0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,28 +5,22 @@ git+https://github.com/dbt-labs/dbt-common.git git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core git+https://github.com/dbt-labs/dbt-postgres.git -# if version 1.x or greater -> pin to major version -# if version 0.x -> pin to minor -black>=24.3 -bumpversion~=0.6.0 -click~=8.1 +# dev +ipdb~=0.13.13 +pre-commit==3.7.0;python_version >="3.9" +pre-commit==3.5.0;python_version <"3.9" + +# test ddtrace==2.3.0 -flake8~=6.1 -flaky~=3.7 freezegun~=1.3 -ipdb~=0.13.13 -mypy==1.7.1 # patch updates have historically introduced breaking changes -pip-tools~=7.3 -pre-commit~=3.5 -pre-commit-hooks~=4.5 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 pytest-logbook~=1.2 pytest-xdist~=3.5 -pytz~=2023.3 tox~=4.11 -types-pytz~=2023.3 -types-requests~=2.31 + +# build +bumpversion~=0.6.0 twine~=4.0 wheel~=0.42 diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index b6e603581..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -namespace_packages = True diff --git a/setup.py b/setup.py index 4673657b8..dbb3913b9 100644 --- a/setup.py +++ b/setup.py @@ -66,9 +66,9 @@ def _plugin_version_trim() -> str: # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. "redshift-connector<2.0.918,>=2.0.913,!=2.0.914", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency - "dbt-core>=1.8.0a1", + "dbt-core>=1.8.0b3", # installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core - "sqlparse>=0.2.3,<0.5", + "sqlparse>=0.5.0,<0.6.0", "agate", ], zip_safe=False, diff --git a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py index 6a77d22ae..c31e9ac61 100644 --- a/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py +++ b/tests/functional/adapter/sources_freshness_tests/test_get_relation_last_modified.py @@ -1,12 +1,24 @@ import os +import pytest +from unittest import mock +from dbt.adapters.redshift.impl import RedshiftAdapter +from dbt.adapters.capability import Capability, CapabilityDict +from dbt.cli.main import dbtRunner from dbt.tests.util import run_dbt -import pytest from tests.functional.adapter.sources_freshness_tests import files -class TestGetLastRelationModified: +class SetupGetLastRelationModified: + @pytest.fixture(scope="class", autouse=True) + def set_env_vars(self, project): + os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema + yield + del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + + +class TestGetLastRelationModified(SetupGetLastRelationModified): @pytest.fixture(scope="class") def seeds(self): return { @@ -18,14 +30,6 @@ def seeds(self): def models(self): return {"schema.yml": files.SCHEMA_YML} - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - # we need the schema name for the sources section - os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] = project.test_schema - run_dbt(["seed"]) - yield - del os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] - @pytest.mark.parametrize( "source,status,expect_pass", [ @@ -34,9 +38,113 @@ def setup(self, project): ], ) def test_get_last_relation_modified(self, project, source, status, expect_pass): + run_dbt(["seed"]) + results = run_dbt( ["source", "freshness", "--select", f"source:{source}"], expect_pass=expect_pass ) assert len(results) == 1 result = results[0] assert result.status == status + + +freshness_metadata_schema_batch_yml = """ +sources: + - name: test_source + freshness: + warn_after: {count: 10, period: hour} + error_after: {count: 1, period: day} + schema: "{{ env_var('DBT_GET_LAST_RELATION_TEST_SCHEMA') }}" + tables: + - name: test_table + - name: test_table2 + - name: test_table_with_loaded_at_field + loaded_at_field: my_loaded_at_field +""" + + +class TestGetLastRelationModifiedBatch(SetupGetLastRelationModified): + @pytest.fixture(scope="class") + def custom_schema(self, project, set_env_vars): + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=os.environ["DBT_GET_LAST_RELATION_TEST_SCHEMA"] + ) + project.adapter.drop_schema(relation) + project.adapter.create_schema(relation) + + yield relation.schema + + with project.adapter.connection_named("__test"): + project.adapter.drop_schema(relation) + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": freshness_metadata_schema_batch_yml} + + def get_freshness_result_for_table(self, table_name, results): + for result in results: + if result.node.name == table_name: + return result + return None + + def test_get_last_relation_modified_batch(self, project, custom_schema): + project.run_sql( + f"create table {custom_schema}.test_table as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table2 as (select 1 as id, 'test' as name);" + ) + project.run_sql( + f"create table {custom_schema}.test_table_with_loaded_at_field as (select 1 as id, timestamp '2009-09-15 10:59:43' as my_loaded_at_field);" + ) + + runner = dbtRunner() + freshness_results_batch = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results_batch) == 3 + test_table_batch_result = self.get_freshness_result_for_table( + "test_table", freshness_results_batch + ) + test_table2_batch_result = self.get_freshness_result_for_table( + "test_table2", freshness_results_batch + ) + test_table_with_loaded_at_field_batch_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results_batch + ) + + # Remove TableLastModifiedMetadataBatch and run freshness on same input without batch strategy + capabilities_no_batch = CapabilityDict( + { + capability: support + for capability, support in RedshiftAdapter.capabilities().items() + if capability != Capability.TableLastModifiedMetadataBatch + } + ) + with mock.patch.object( + RedshiftAdapter, "capabilities", return_value=capabilities_no_batch + ): + freshness_results = runner.invoke(["source", "freshness"]).result + + assert len(freshness_results) == 3 + test_table_result = self.get_freshness_result_for_table("test_table", freshness_results) + test_table2_result = self.get_freshness_result_for_table("test_table2", freshness_results) + test_table_with_loaded_at_field_result = self.get_freshness_result_for_table( + "test_table_with_loaded_at_field", freshness_results + ) + + # assert results between batch vs non-batch freshness strategy are equivalent + assert test_table_result.status == test_table_batch_result.status + assert test_table_result.max_loaded_at == test_table_batch_result.max_loaded_at + + assert test_table2_result.status == test_table2_batch_result.status + assert test_table2_result.max_loaded_at == test_table2_batch_result.max_loaded_at + + assert ( + test_table_with_loaded_at_field_batch_result.status + == test_table_with_loaded_at_field_result.status + ) + assert ( + test_table_with_loaded_at_field_batch_result.max_loaded_at + == test_table_with_loaded_at_field_result.max_loaded_at + ) diff --git a/tests/unit/relation_configs/test_materialized_view.py b/tests/unit/relation_configs/test_materialized_view.py index 5e454fe5e..8e4f6ca3e 100644 --- a/tests/unit/relation_configs/test_materialized_view.py +++ b/tests/unit/relation_configs/test_materialized_view.py @@ -14,8 +14,8 @@ def test_redshift_materialized_view_config_handles_all_valid_bools(bool_value): query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) config_dict = config.parse_relation_config(model_node) assert isinstance(config_dict["autorefresh"], bool) @@ -33,8 +33,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: bool_value if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + bool_value if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(TypeError): config.parse_relation_config(model_node) @@ -48,8 +48,8 @@ def test_redshift_materialized_view_config_throws_expected_exception_with_invali query="select * from sometable", ) model_node = Mock() - model_node.config.extra.get = ( - lambda x, y=None: "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" + model_node.config.extra.get = lambda x, y=None: ( + "notABool" if x in ["auto_refresh", "backup"] else "someDistValue" ) with pytest.raises(ValueError): config.parse_relation_config(model_node) diff --git a/tests/unit/test_redshift_adapter.py b/tests/unit/test_redshift_adapter.py index 671e47032..0bd5f8e99 100644 --- a/tests/unit/test_redshift_adapter.py +++ b/tests/unit/test_redshift_adapter.py @@ -4,7 +4,7 @@ from unittest import mock from dbt_common.exceptions import DbtRuntimeError -from unittest.mock import Mock, call +from unittest.mock import MagicMock, call import agate import dbt @@ -67,7 +67,7 @@ def adapter(self): inject_adapter(self._adapter, RedshiftPlugin) return self._adapter - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_implicit_database_conn(self): connection = self.adapter.acquire_connection("dummy") connection.handle @@ -84,7 +84,7 @@ def test_implicit_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region_with_database_conn(self): self.config.method = "database" @@ -103,7 +103,7 @@ def test_explicit_region_with_database_conn(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_conn_without_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -129,7 +129,7 @@ def test_explicit_iam_conn_without_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_conn_timeout_30(self): self.config.credentials = self.config.credentials.replace(connect_timeout=30) connection = self.adapter.acquire_connection("dummy") @@ -147,7 +147,7 @@ def test_conn_timeout_30(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_conn_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -175,7 +175,7 @@ def test_explicit_iam_conn_with_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_iam_serverless_with_profile(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -201,7 +201,7 @@ def test_explicit_iam_serverless_with_profile(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region(self): # Successful test self.config.credentials = self.config.credentials.replace( @@ -229,7 +229,7 @@ def test_explicit_region(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_region_failure(self): # Failure test with no region self.config.credentials = self.config.credentials.replace( @@ -258,7 +258,7 @@ def test_explicit_region_failure(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_explicit_invalid_region(self): # Invalid region test self.config.credentials = self.config.credentials.replace( @@ -287,7 +287,7 @@ def test_explicit_invalid_region(self): **DEFAULT_SSL_CONFIG, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_disable(self): self.config.credentials.sslmode = "disable" connection = self.adapter.acquire_connection("dummy") @@ -306,7 +306,7 @@ def test_sslmode_disable(self): sslmode=None, ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_allow(self): self.config.credentials.sslmode = "allow" connection = self.adapter.acquire_connection("dummy") @@ -325,7 +325,7 @@ def test_sslmode_allow(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_verify_full(self): self.config.credentials.sslmode = "verify-full" connection = self.adapter.acquire_connection("dummy") @@ -344,7 +344,7 @@ def test_sslmode_verify_full(self): sslmode="verify-full", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_verify_ca(self): self.config.credentials.sslmode = "verify-ca" connection = self.adapter.acquire_connection("dummy") @@ -363,7 +363,7 @@ def test_sslmode_verify_ca(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_sslmode_prefer(self): self.config.credentials.sslmode = "prefer" connection = self.adapter.acquire_connection("dummy") @@ -382,7 +382,7 @@ def test_sslmode_prefer(self): sslmode="verify-ca", ) - @mock.patch("redshift_connector.connect", Mock()) + @mock.patch("redshift_connector.connect", MagicMock()) def test_serverless_iam_failure(self): self.config.credentials = self.config.credentials.replace( method="iam", @@ -447,6 +447,25 @@ def test_invalid_iam_no_cluster_id(self): self.assertTrue("'cluster_id' must be provided" in context.exception.msg) + @mock.patch("redshift_connector.connect", MagicMock()) + def test_connection_has_backend_pid(self): + backend_pid = 42 + + cursor = mock.MagicMock() + execute = cursor().__enter__().execute + execute().fetchone.return_value = (backend_pid,) + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + assert connection.backend_pid == backend_pid + + execute.assert_has_calls( + [ + call("select pg_backend_pid()"), + ] + ) + def test_cancel_open_connections_empty(self): self.assertEqual(len(list(self.adapter.cancel_open_connections())), 0) @@ -475,11 +494,32 @@ def test_cancel_open_connections_single(self): self.assertEqual(len(list(self.adapter.cancel_open_connections())), 1) add_query.assert_has_calls( [ - call("select pg_backend_pid()"), + call(f"select pg_terminate_backend({model.backend_pid})"), ] ) - master.handle.get_backend_pid.assert_not_called() + master.handle.backend_pid.assert_not_called() + + @mock.patch("redshift_connector.connect", MagicMock()) + def test_backend_pid_used_in_pg_terminate_backend(self): + with mock.patch.object(self.adapter.connections, "add_query") as add_query: + backend_pid = 42 + query_result = (backend_pid,) + + cursor = mock.MagicMock() + cursor().__enter__().execute().fetchone.return_value = query_result + redshift_connector.connect().cursor = cursor + + connection = self.adapter.acquire_connection("dummy") + connection.handle + + self.adapter.connections.cancel(connection) + + add_query.assert_has_calls( + [ + call(f"select pg_terminate_backend({backend_pid})"), + ] + ) def test_dbname_verification_is_case_insensitive(self): # Override adapter settings from setUp() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 3fc1d7ec6..4de03ec80 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -2,6 +2,7 @@ Note that all imports should be inside the functions to avoid import/mocking issues. """ + import string import os from unittest import mock