From 09bd592ccac42fb524b9749a9c6609b0da9ea35c Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 11:43:03 +0100 Subject: [PATCH 01/16] feat: Optional external id for arn auth --- .../jaffle_shop/run_test_workflow.sh | 2 ++ .../sources_external_tables_iam.yml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 .github/workflows/jaffle_shop/sources_external_tables_iam.yml diff --git a/.github/workflows/jaffle_shop/run_test_workflow.sh b/.github/workflows/jaffle_shop/run_test_workflow.sh index def2530f9..aadc2f4f3 100755 --- a/.github/workflows/jaffle_shop/run_test_workflow.sh +++ b/.github/workflows/jaffle_shop/run_test_workflow.sh @@ -6,6 +6,8 @@ dbt compile dbt run-operation stage_external_sources cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_id_secret.yml models/staging/sources_external_tables.yml dbt run-operation stage_external_sources --vars "ext_full_refresh: true" +cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_iam.yml models/staging/sources_external_tables.yml +dbt run-operation stage_external_sources --vars "ext_full_refresh: true" dbt seed dbt seed --full-refresh dbt run diff --git a/.github/workflows/jaffle_shop/sources_external_tables_iam.yml b/.github/workflows/jaffle_shop/sources_external_tables_iam.yml new file mode 100644 index 000000000..665eff755 --- /dev/null +++ b/.github/workflows/jaffle_shop/sources_external_tables_iam.yml @@ -0,0 +1,19 @@ +version: 2 + +sources: + - name: s3 + tables: + - name: raw_customers + external: + url: "{{ env_var('SECURE_BUCKET_PATH') }}" + object_pattern: '*raw_customers.csv' + credentials: + internal_role_arn: "{{ env_var('AWS_ACCESS_ROLE_ARN') }}" + type: '(CSV SKIP_HEADER_ROWS=true)' + columns: + - name: id + data_type: int + - name: first_name + data_type: TEXT + - name: last_name + data_type: TEXT From 1fec1df7962f0cdec945cb225c0555cd849ac799 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 11:44:04 +0100 Subject: [PATCH 02/16] add the actual change --- .../macros/dbt_external_tables/create_external_table.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql index 89452229f..cccb7da1e 100644 --- a/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql +++ b/dbt/include/firebolt/macros/dbt_external_tables/create_external_table.sql @@ -20,7 +20,10 @@ {% if external.url %} URL = '{{external.url}}' {%- endif %} {%- if credentials and credentials.internal_role_arn %} CREDENTIALS = (AWS_ROLE_ARN = '{{credentials.internal_role_arn}}' - AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}') + {%- if credentials.external_role_id %} + AWS_ROLE_EXTERNAL_ID = '{{credentials.external_role_id}}' + {%- endif -%} + ) {% elif credentials and credentials.aws_key_id %} CREDENTIALS = (AWS_KEY_ID = '{{credentials.aws_key_id}}' AWS_SECRET_KEY = '{{credentials.aws_secret_key}}') From 619e6587f952df995d45747ab16e786164a95e3a Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 11:53:29 +0100 Subject: [PATCH 03/16] changes in ci --- .github/workflows/jaffle-shop-v1.yml | 1 + .github/workflows/jaffle-shop-v2.yml | 1 + .github/workflows/nightly.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/jaffle-shop-v1.yml b/.github/workflows/jaffle-shop-v1.yml index 45022b604..568f50264 100644 --- a/.github/workflows/jaffle-shop-v1.yml +++ b/.github/workflows/jaffle-shop-v1.yml @@ -53,6 +53,7 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} working-directory: jaffle-shop run: ../dbt-firebolt/.github/workflows/jaffle_shop/run_test_workflow.sh diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index cebaeb02f..f965d3e42 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -53,6 +53,7 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} DBT_PROFILES_DIR: "../dbt-firebolt/.github/workflows/jaffle_shop" working-directory: jaffle-shop run: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 21b068f5c..f140c155b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -57,6 +57,7 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} working-directory: jaffle-shop shell: bash run: | From 2f596b7b495e5eef033402735c7a7c87ef4222c3 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 12:01:39 +0100 Subject: [PATCH 04/16] test better install --- .github/workflows/jaffle-shop-v2.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index f965d3e42..a07ff16b7 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -30,7 +30,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "dbt-firebolt/.[dev]" + python -m pip install --upgrade setuptools + python -m pip install "dbt-firebolt/.[dev]" - name: Setup database and engine id: setup From 574178ab768b0a436250f3092b301a06dfb72991 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 12:06:45 +0100 Subject: [PATCH 05/16] limit setuptools --- .github/workflows/jaffle-shop-v2.yml | 3 +-- setup.cfg | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index a07ff16b7..f965d3e42 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -30,8 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - python -m pip install "dbt-firebolt/.[dev]" + pip install "dbt-firebolt/.[dev]" - name: Setup database and engine id: setup diff --git a/setup.cfg b/setup.cfg index d7e106246..590757db4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ dev = mypy==1.4.1 pre-commit==2.15.0 pytest==7.* + setuptools<72.0.0 [black] python-version = 3.9 From 188242a418dd73b69ce82009818c92aa6ec5a47b Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 12:16:50 +0100 Subject: [PATCH 06/16] try explicit python call --- .github/workflows/jaffle-shop-v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index f965d3e42..2738a66d8 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install "dbt-firebolt/.[dev]" + python -m pip install "dbt-firebolt/.[dev]" - name: Setup database and engine id: setup From c7f18802e7c9a8b9a547a0186c483cf59e39f6e3 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 12:17:32 +0100 Subject: [PATCH 07/16] same in code check --- .github/workflows/code-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index df272117d..ac65a7569 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install ".[dev]" + python -m pip install ".[dev]" - name: Run pre-commit checks uses: pre-commit/action@v2.0.3 From 181e2bb6424fc74785aaac35f9f7b89d29c6ffb9 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:10:28 +0100 Subject: [PATCH 08/16] no cache dir --- .github/workflows/code-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index ac65a7569..e66ea5a84 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install ".[dev]" + python -m pip install ".[dev]" --no-cache-dir - name: Run pre-commit checks uses: pre-commit/action@v2.0.3 From 2f83a223f3f18259b50acbe2ef26f03f5114706f Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:13:46 +0100 Subject: [PATCH 09/16] try limit setuptools --- .github/workflows/code-check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index e66ea5a84..2e43d9b70 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -20,6 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install setuptools<56.0.0 python -m pip install ".[dev]" --no-cache-dir - name: Run pre-commit checks From 730b076a35c379bf17244a55f50645988c04cc44 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:18:26 +0100 Subject: [PATCH 10/16] fix --- .github/workflows/code-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 2e43d9b70..4c6562835 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -20,7 +20,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install setuptools<56.0.0 + python -m pip install "setuptools<56.0.0" python -m pip install ".[dev]" --no-cache-dir - name: Run pre-commit checks From 1d9d6a0e31dceb170b4e36b4805e173c20252465 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:31:21 +0100 Subject: [PATCH 11/16] try pip lower version --- .github/workflows/code-check.yml | 5 ++--- setup.cfg | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index 4c6562835..efdc7d81c 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -19,9 +19,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - python -m pip install "setuptools<56.0.0" - python -m pip install ".[dev]" --no-cache-dir + python -m pip install --upgrade "pip<=24.1.2" + python -m pip install ".[dev]" - name: Run pre-commit checks uses: pre-commit/action@v2.0.3 diff --git a/setup.cfg b/setup.cfg index 590757db4..d7e106246 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,6 @@ dev = mypy==1.4.1 pre-commit==2.15.0 pytest==7.* - setuptools<72.0.0 [black] python-version = 3.9 From e278e4398656c2a8d9c92f5153a4a7b54607c13c Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:35:44 +0100 Subject: [PATCH 12/16] fix the issue at last --- .github/workflows/code-check.yml | 1 + .github/workflows/integration-tests-v1.yml | 3 ++- .github/workflows/integration-tests-v2.yml | 3 ++- .github/workflows/jaffle-shop-v1.yml | 3 ++- .github/workflows/jaffle-shop-v2.yml | 3 ++- .github/workflows/nightly.yml | 3 ++- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index efdc7d81c..c5616b529 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -19,6 +19,7 @@ jobs: - name: Install dependencies run: | + # pip 24.2 has issues installing the Logbook dependency python -m pip install --upgrade "pip<=24.1.2" python -m pip install ".[dev]" diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml index 1d319577e..a4382106f 100644 --- a/.github/workflows/integration-tests-v1.yml +++ b/.github/workflows/integration-tests-v1.yml @@ -23,7 +23,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + # pip 24.2 has issues installing the Logbook dependency + python -m pip install --upgrade "pip<=24.1.2" pip install ".[dev]" --no-cache-dir - name: Setup database and engine diff --git a/.github/workflows/integration-tests-v2.yml b/.github/workflows/integration-tests-v2.yml index 943d4e046..88abe7b83 100644 --- a/.github/workflows/integration-tests-v2.yml +++ b/.github/workflows/integration-tests-v2.yml @@ -23,7 +23,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + # pip 24.2 has issues installing the Logbook dependency + python -m pip install --upgrade "pip<=24.1.2" pip install ".[dev]" --no-cache-dir - name: Setup database and engine diff --git a/.github/workflows/jaffle-shop-v1.yml b/.github/workflows/jaffle-shop-v1.yml index 568f50264..e54683689 100644 --- a/.github/workflows/jaffle-shop-v1.yml +++ b/.github/workflows/jaffle-shop-v1.yml @@ -29,7 +29,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + # pip 24.2 has issues installing the Logbook dependency + python -m pip install --upgrade "pip<=24.1.2" pip install "dbt-firebolt/.[dev]" - name: Setup database and engine diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index 2738a66d8..20186f102 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -29,7 +29,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + # pip 24.2 has issues installing the Logbook dependency + python -m pip install --upgrade "pip<=24.1.2" python -m pip install "dbt-firebolt/.[dev]" - name: Setup database and engine diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f140c155b..65bffa276 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,7 +33,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + # pip 24.2 has issues installing the Logbook dependency + python -m pip install --upgrade "pip<=24.1.2" pip install -e "dbt-firebolt/.[dev]" - name: Setup database and engine From 2193915c2e1fc25fab1884bb372216c18cc1bb29 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Mon, 29 Jul 2024 14:53:17 +0100 Subject: [PATCH 13/16] add changine --- .changes/unreleased/Changed-20240729-145310.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changes/unreleased/Changed-20240729-145310.yaml diff --git a/.changes/unreleased/Changed-20240729-145310.yaml b/.changes/unreleased/Changed-20240729-145310.yaml new file mode 100644 index 000000000..338a4a16c --- /dev/null +++ b/.changes/unreleased/Changed-20240729-145310.yaml @@ -0,0 +1,3 @@ +kind: Changed +body: External table authentication to inlcude optional role external id. +time: 2024-07-29T14:53:10.992711+01:00 From cf7b9bafda410e89c77c5536cc46042e9da3bdc5 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Tue, 30 Jul 2024 08:28:37 +0000 Subject: [PATCH 14/16] try without override --- .github/workflows/code-check.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml index c5616b529..ac65a7569 100644 --- a/.github/workflows/code-check.yml +++ b/.github/workflows/code-check.yml @@ -19,8 +19,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip python -m pip install ".[dev]" - name: Run pre-commit checks From baba7fac822d8d868709847b1876610a006be2d1 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Tue, 30 Jul 2024 08:30:16 +0000 Subject: [PATCH 15/16] rest of the reverts --- .github/workflows/integration-tests-v1.yml | 3 +-- .github/workflows/integration-tests-v2.yml | 3 +-- .github/workflows/jaffle-shop-v1.yml | 3 +-- .github/workflows/jaffle-shop-v2.yml | 3 +-- .github/workflows/nightly.yml | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml index a4382106f..1d319577e 100644 --- a/.github/workflows/integration-tests-v1.yml +++ b/.github/workflows/integration-tests-v1.yml @@ -23,8 +23,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip pip install ".[dev]" --no-cache-dir - name: Setup database and engine diff --git a/.github/workflows/integration-tests-v2.yml b/.github/workflows/integration-tests-v2.yml index 88abe7b83..943d4e046 100644 --- a/.github/workflows/integration-tests-v2.yml +++ b/.github/workflows/integration-tests-v2.yml @@ -23,8 +23,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip pip install ".[dev]" --no-cache-dir - name: Setup database and engine diff --git a/.github/workflows/jaffle-shop-v1.yml b/.github/workflows/jaffle-shop-v1.yml index e54683689..568f50264 100644 --- a/.github/workflows/jaffle-shop-v1.yml +++ b/.github/workflows/jaffle-shop-v1.yml @@ -29,8 +29,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip pip install "dbt-firebolt/.[dev]" - name: Setup database and engine diff --git a/.github/workflows/jaffle-shop-v2.yml b/.github/workflows/jaffle-shop-v2.yml index 20186f102..2738a66d8 100644 --- a/.github/workflows/jaffle-shop-v2.yml +++ b/.github/workflows/jaffle-shop-v2.yml @@ -29,8 +29,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip python -m pip install "dbt-firebolt/.[dev]" - name: Setup database and engine diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 65bffa276..f140c155b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -33,8 +33,7 @@ jobs: - name: Install dependencies run: | - # pip 24.2 has issues installing the Logbook dependency - python -m pip install --upgrade "pip<=24.1.2" + python -m pip install --upgrade pip pip install -e "dbt-firebolt/.[dev]" - name: Setup database and engine From 280bb28fbad0cc86ce73e884b714dbba4dc40261 Mon Sep 17 00:00:00 2001 From: ptiurin Date: Tue, 30 Jul 2024 10:08:53 +0000 Subject: [PATCH 16/16] run only on fb2.0 --- .github/workflows/jaffle-shop-v1.yml | 1 - .github/workflows/jaffle_shop/run_test_workflow.sh | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jaffle-shop-v1.yml b/.github/workflows/jaffle-shop-v1.yml index 568f50264..45022b604 100644 --- a/.github/workflows/jaffle-shop-v1.yml +++ b/.github/workflows/jaffle-shop-v1.yml @@ -53,7 +53,6 @@ jobs: SECURE_BUCKET_PATH: ${{ vars.SECURE_BUCKET_PATH }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} - AWS_ACCESS_ROLE_ARN: ${{ secrets.AWS_ACCESS_ROLE_ARN }} working-directory: jaffle-shop run: ../dbt-firebolt/.github/workflows/jaffle_shop/run_test_workflow.sh diff --git a/.github/workflows/jaffle_shop/run_test_workflow.sh b/.github/workflows/jaffle_shop/run_test_workflow.sh index aadc2f4f3..65915ae25 100755 --- a/.github/workflows/jaffle_shop/run_test_workflow.sh +++ b/.github/workflows/jaffle_shop/run_test_workflow.sh @@ -6,8 +6,11 @@ dbt compile dbt run-operation stage_external_sources cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_id_secret.yml models/staging/sources_external_tables.yml dbt run-operation stage_external_sources --vars "ext_full_refresh: true" -cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_iam.yml models/staging/sources_external_tables.yml -dbt run-operation stage_external_sources --vars "ext_full_refresh: true" +if [[ -n "$AWS_ACCESS_ROLE_ARN" ]]; then + # Can't test this on FB 1.0 + cp ../dbt-firebolt/.github/workflows/jaffle_shop/sources_external_tables_iam.yml models/staging/sources_external_tables.yml + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" +fi dbt seed dbt seed --full-refresh dbt run