Merge branch 'main' into dependabot/github_actions/actions/download-a…

…rtifact-4

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.8.0b2
+current_version = 1.8.0b3
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.8.0-b3.md

@@ -0,0 +1,16 @@
+## dbt-redshift 1.8.0-b3 - April 18, 2024
+
+### Fixes
+
+- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705))
+
+### Under the Hood
+
+- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759))
+
+### Security
+
+- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768))
+
+### Contributors
+- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705))

.changes/1.8.0/Fixes-20240326-123703.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: dbt can cancel open queries upon interrupt
+time: 2024-03-26T12:37:03.17481-05:00
+custom:
+  Author: holly-evans
+  Issue: "705"

.changes/1.8.0/Security-20240416-195919.yaml

@@ -0,0 +1,6 @@
+kind: Security
+body: Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core
+time: 2024-04-16T19:59:19.233806-05:00
+custom:
+  Author: McKnight-42
+  PR: "768"

.changes/unreleased/Features-20240404-171441.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support TableLastModifiedMetadataBatch capability
+time: 2024-04-04T17:14:41.313087-07:00
+custom:
+  Author: michelleark
+  Issue: "755"

.github/scripts/integration-test-matrix.js

@@ -37,7 +37,7 @@ module.exports = ({ context }) => {
 
           if (labels.includes("test macos") || testAllLabel) {
             include.push({
-              os: "macos-latest",
+              os: "macos-12",
               adapter,
               "python-version": pythonVersion,
             });
@@ -70,7 +70,7 @@ module.exports = ({ context }) => {
   // additionally include runs for all adapters, on macos and windows,
   // but only for the default python version
   for (const adapter of supportedAdapters) {
-    for (const operatingSystem of ["windows-latest", "macos-latest"]) {
+    for (const operatingSystem of ["windows-latest", "macos-12"]) {
       include.push({
         os: operatingSystem,
         adapter: adapter,

.github/workflows/integration.yml

@@ -174,6 +174,33 @@ jobs:
           pip install bumpversion
           ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
 
+      - name: Create AWS IAM profiles
+        run: |
+          aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
+          aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY
+          aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION
+          aws configure --profile $AWS_USER_PROFILE set output json
+
+          aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID
+          aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY
+          aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION
+          aws configure --profile $AWS_SOURCE_PROFILE set output json
+
+          aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE
+          aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN
+          aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION
+          aws configure --profile $AWS_ROLE_PROFILE set output json
+        env:
+          AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
+          AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
+          AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
+          AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user
+          AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
+          AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }}
+          AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }}
+          AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }}
+          AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
+
       - name: Run tox (redshift)
         if: matrix.adapter == 'redshift'
         env:
@@ -182,6 +209,12 @@ jobs:
           REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
           REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
           REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
+          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
+          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
+          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
+          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
+          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
           DBT_TEST_USER_1: dbt_test_user_1
           DBT_TEST_USER_2: dbt_test_user_2
           DBT_TEST_USER_3: dbt_test_user_3

.github/workflows/main.yml

@@ -58,7 +58,6 @@ jobs:
           python -m pip install -r dev-requirements.txt
           python -m pip --version
           pre-commit --version
-          mypy --version
 
       - name: pre-commit hooks
         run: pre-commit run --all-files --show-diff-on-failure
@@ -174,7 +173,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-12, windows-latest]
         python-version: ['3.8', '3.9', '3.10', '3.11']
 
     steps:

.pre-commit-config.yaml

@@ -1,63 +1,55 @@
-# For more on configuring pre-commit hooks (see https://pre-commit.com/)
-
-# Force all unspecified python hooks to run python 3.8
 default_language_version:
-  python: python3
+    python: python3
 
 repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
-  hooks:
-  - id: check-yaml
-    args: [--unsafe]
-  - id: check-json
-  - id: end-of-file-fixer
-  - id: trailing-whitespace
-  - id: check-case-conflict
-- repo: https://github.com/psf/black
-  rev: 23.1.0
-  hooks:
-  - id: black
-    additional_dependencies: ['click~=8.1']
-    args:
-    - "--line-length=99"
-    - "--target-version=py38"
-  - id: black
-    alias: black-check
-    stages: [manual]
-    additional_dependencies: ['click~=8.1']
-    args:
-    - "--line-length=99"
-    - "--target-version=py38"
-    - "--check"
-    - "--diff"
-- repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
-  hooks:
-  - id: flake8
-  - id: flake8
-    alias: flake8-check
-    stages: [manual]
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.1.1
-  hooks:
-  - id: mypy
-    # N.B.: Mypy is... a bit fragile.
-    #
-    # By using `language: system` we run this hook in the local
-    # environment instead of a pre-commit isolated one.  This is needed
-    # to ensure mypy correctly parses the project.
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: check-yaml
+        args: [--unsafe]
+    -   id: check-json
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+    -   id: check-case-conflict
+
+-   repo: https://github.com/dbt-labs/pre-commit-hooks
+    rev: v0.1.0a1
+    hooks:
+    -   id: dbt-core-in-adapters-check
+
+-   repo: https://github.com/psf/black
+    rev: 24.4.0
+    hooks:
+    -   id: black
+        args:
+        -   --line-length=99
+        -   --target-version=py38
+        -   --target-version=py39
+        -   --target-version=py310
+        -   --target-version=py311
+
+-   repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+    -   id: flake8
+        exclude: tests/
+        args:
+        -   --max-line-length=99
+        -   --select=E,F,W
+        -   --ignore=E203,E501,E741,W503,W504
+        -   --per-file-ignores=*/__init__.py:F401
+        additional_dependencies: [flaky]
 
-    # It may cause trouble in that it adds environmental variables out
-    # of our control to the mix.  Unfortunately, there's nothing we can
-    # do about per pre-commit's author.
-    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
-    args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases]
-    files: ^dbt/adapters/.*
-    language: system
-  - id: mypy
-    alias: mypy-check
-    stages: [manual]
-    args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases]
-    files: ^dbt/adapters
-    language: system
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.9.0
+    hooks:
+    -   id: mypy
+        args:
+        -   --show-error-codes
+        -   --pretty
+        -   --ignore-missing-imports
+        -   --explicit-package-bases
+        files: ^dbt/adapters
+        additional_dependencies:
+        -   types-pytz
+        -   types-requests

CHANGELOG.md

@@ -5,6 +5,24 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-redshift/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-redshift 1.8.0-b3 - April 18, 2024
+
+### Fixes
+
+- dbt can cancel open queries upon interrupt ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705))
+
+### Under the Hood
+
+- Update dependabot config to cover GHA ([#759](https://github.com/dbt-labs/dbt-redshift/issues/759))
+
+### Security
+
+- Bump sqlparse to >=0.5.0, <0.6.0 to address GHSA-2m57-hf25-phgg along with dbt-core ([#768](https://github.com/dbt-labs/dbt-redshift/pull/768))
+
+### Contributors
+- [@holly-evans](https://github.com/holly-evans) ([#705](https://github.com/dbt-labs/dbt-redshift/issues/705))
+
+
 ## dbt-redshift 1.8.0-b2 - April 03, 2024
 
 ### Features
@@ -28,8 +46,6 @@
 
 - Pin `black>=24.3` in `dev-requirements.txt` ([#743](https://github.com/dbt-labs/dbt-redshift/pull/743))
 
-
-
 ## dbt-redshift 1.8.0-b1 - March 01, 2024
 
 ### Features

dbt/adapters/redshift/__version__.py

@@ -1 +1 @@
-version = "1.8.0b2"
+version = "1.8.0b3"

dbt/adapters/redshift/connections.py

@@ -50,7 +50,7 @@ class UserSSLMode(StrEnum):
     @classmethod
     def default(cls) -> "UserSSLMode":
         # default for `psycopg2`, which aligns with dbt-redshift 1.4 and provides backwards compatibility
-        return cls.prefer
+        return cls("prefer")
 
 
 class RedshiftSSLMode(StrEnum):
@@ -60,11 +60,11 @@ class RedshiftSSLMode(StrEnum):
 
 SSL_MODE_TRANSLATION = {
     UserSSLMode.disable: None,
-    UserSSLMode.allow: RedshiftSSLMode.verify_ca,
-    UserSSLMode.prefer: RedshiftSSLMode.verify_ca,
-    UserSSLMode.require: RedshiftSSLMode.verify_ca,
-    UserSSLMode.verify_ca: RedshiftSSLMode.verify_ca,
-    UserSSLMode.verify_full: RedshiftSSLMode.verify_full,
+    UserSSLMode.allow: RedshiftSSLMode("verify-ca"),
+    UserSSLMode.prefer: RedshiftSSLMode("verify-ca"),
+    UserSSLMode.require: RedshiftSSLMode("verify-ca"),
+    UserSSLMode.verify_ca: RedshiftSSLMode("verify-ca"),
+    UserSSLMode.verify_full: RedshiftSSLMode("verify-full"),
 }
 
 
@@ -233,27 +233,26 @@ def connect():
 class RedshiftConnectionManager(SQLConnectionManager):
     TYPE = "redshift"
 
-    def _get_backend_pid(self):
-        sql = "select pg_backend_pid()"
-        _, cursor = self.add_query(sql)
-
-        res = cursor.fetchone()
-        return res[0]
-
     def cancel(self, connection: Connection):
+        pid = connection.backend_pid  # type: ignore
+        sql = f"select pg_terminate_backend({pid})"
+        logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}")
+        logger.debug(sql)
+
         try:
-            pid = self._get_backend_pid()
+            self.add_query(sql)
         except redshift_connector.InterfaceError as e:
             if "is closed" in str(e):
                 logger.debug(f"Connection {connection.name} was already closed")
                 return
             raise
 
-        sql = f"select pg_terminate_backend({pid})"
-        cursor = connection.handle.cursor()
-        logger.debug(f"Cancel query on: '{connection.name}' with PID: {pid}")
-        logger.debug(sql)
-        cursor.execute(sql)
+    @classmethod
+    def _get_backend_pid(cls, connection):
+        with connection.handle.cursor() as c:
+            sql = "select pg_backend_pid()"
+            res = c.execute(sql).fetchone()
+        return res[0]
 
     @classmethod
     def get_response(cls, cursor: redshift_connector.Cursor) -> AdapterResponse:
@@ -325,14 +324,16 @@ def exponential_backoff(attempt: int):
             redshift_connector.DataError,
         ]
 
-        return cls.retry_connection(
+        open_connection = cls.retry_connection(
             connection,
             connect=connect_method_factory.get_connect_method(),
             logger=logger,
             retry_limit=credentials.retries,
             retry_timeout=exponential_backoff,
             retryable_exceptions=retryable_exceptions,
         )
+        open_connection.backend_pid = cls._get_backend_pid(open_connection)  # type: ignore
+        return open_connection
 
     def execute(
         self,

dbt/adapters/redshift/impl.py

@@ -58,6 +58,7 @@ class RedshiftAdapter(SQLAdapter):
         {
             Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full),
             Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full),
+            Capability.TableLastModifiedMetadataBatch: CapabilitySupport(support=Support.Full),
         }
     )