From 9aaf231e6155e7e362e33c0f05cb08e808be535a Mon Sep 17 00:00:00 2001 From: u1214742 Date: Tue, 22 Jun 2021 02:54:35 -0400 Subject: [PATCH 01/11] dbt seed truncate tables --- CHANGELOG.md | 10 ++++++++++ dbt/include/spark/macros/materializations/seed.sql | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 339f7f5c4..32a0df4d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## dbt-spark 0.20.1 (June 22, 2021) + +### Features + +### Fixes +- dbt seed command fixed with expected behavior from dbt global project to [truncate table](https://spark.apache.org/docs/3.0.0-preview/sql-ref-syntax-ddl-truncate-table.html) in order remove all rows from the existing seed tables and replace values. As explained in [issue 112](https://github.com/fishtown-analytics/dbt-spark/issues/112), the current seed command in dbt-spark appends to existing seeded tables instead overwriting. + +### Contributors +- [@mv1742](https://github.com/mv1742) ([#181](https://github.com/mv1742/) + ## dbt-spark 0.20.0 (Release TBD) ### Fixes diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 536e6447b..c407dcc10 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -1,7 +1,7 @@ {% macro spark__load_csv_rows(model, agate_table) %} {% set batch_size = 1000 %} {% set column_override = model['config'].get('column_types', {}) %} - + {% set statements = [] %} {% for chunk in agate_table.rows | batch(batch_size) %} @@ -37,7 +37,9 @@ {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} - {{ adapter.drop_relation(old_relation) }} + {{ adapter.truncate_relation(old_relation) }} + {% set sql = "truncate table " ~ old_relation %} + {{ return(sql) }} {% endif %} {% set sql = create_csv_table(model, agate_table) %} {{ return(sql) }} From 04614e27dc318394bb24b3b75bb483c7e5059bfb Mon Sep 17 00:00:00 2001 From: mv1742 Date: Thu, 12 Oct 2023 12:50:36 -0400 Subject: [PATCH 02/11] Cleanup main after cutting new 1.7.latest branch (#907) * Clean up changelog on main * Bumping version to 1.8.0a1 * Code quality cleanup --- .bumpversion.cfg | 2 +- .changes/1.7.0-b1.md | 47 ----------- .changes/1.7.0-b2.md | 27 ------- .../1.7.0/Dependencies-20230424-230630.yaml | 6 -- .../1.7.0/Dependencies-20230424-230645.yaml | 6 -- .../1.7.0/Dependencies-20230501-231003.yaml | 6 -- .../1.7.0/Dependencies-20230501-231035.yaml | 6 -- .../1.7.0/Dependencies-20230510-230725.yaml | 6 -- .../1.7.0/Dependencies-20230628-121341.yaml | 6 -- .../1.7.0/Dependencies-20230803-224622.yaml | 6 -- .../1.7.0/Dependencies-20230803-224623.yaml | 6 -- .../1.7.0/Dependencies-20230803-224626.yaml | 6 -- .../1.7.0/Dependencies-20230803-224629.yaml | 6 -- .../1.7.0/Dependencies-20230804-225232.yaml | 6 -- .../1.7.0/Dependencies-20230804-225236.yaml | 6 -- .../1.7.0/Dependencies-20230804-225243.yaml | 6 -- .../1.7.0/Dependencies-20230804-225249.yaml | 6 -- .../1.7.0/Dependencies-20230807-221033.yaml | 6 -- .../1.7.0/Dependencies-20230807-221037.yaml | 6 -- .../1.7.0/Dependencies-20230809-043913.yaml | 6 -- .../1.7.0/Dependencies-20230811-221135.yaml | 6 -- .../1.7.0/Dependencies-20230814-224754.yaml | 6 -- .../1.7.0/Dependencies-20230814-224757.yaml | 6 -- .../1.7.0/Dependencies-20230816-221452.yaml | 6 -- .../1.7.0/Dependencies-20230816-221455.yaml | 6 -- .../1.7.0/Dependencies-20230825-154517.yaml | 6 -- .../1.7.0/Dependencies-20230904-221612.yaml | 6 -- .../1.7.0/Dependencies-20230911-222120.yaml | 6 -- .../1.7.0/Dependencies-20230912-222718.yaml | 6 -- .changes/1.7.0/Features-20221229-200956.yaml | 7 -- .changes/1.7.0/Features-20230707-104150.yaml | 6 -- .changes/1.7.0/Features-20230707-113337.yaml | 6 -- .changes/1.7.0/Features-20230707-114650.yaml | 6 -- .changes/1.7.0/Features-20230707-135442.yaml | 6 -- .changes/1.7.0/Features-20230817-130731.yaml | 6 -- .changes/1.7.0/Fixes-20230517-142331.yaml | 6 -- .changes/1.7.0/Fixes-20230810-014122.yaml | 6 -- .changes/1.7.0/Security-20230817-145626.yaml | 6 -- .../1.7.0/Under the Hood-20230724-165508.yaml | 6 -- .../1.7.0/Under the Hood-20230830-160616.yaml | 6 -- .../unreleased/Features-20230921-180958.yaml | 6 -- .../unreleased/Features-20231011-094718.yaml | 6 -- CHANGELOG.md | 77 ------------------- dbt/adapters/spark/__version__.py | 2 +- setup.py | 2 +- 45 files changed, 3 insertions(+), 389 deletions(-) delete mode 100644 .changes/1.7.0-b1.md delete mode 100644 .changes/1.7.0-b2.md delete mode 100644 .changes/1.7.0/Dependencies-20230424-230630.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230424-230645.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230501-231003.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230501-231035.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230510-230725.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230628-121341.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230803-224622.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230803-224623.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230803-224626.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230803-224629.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230804-225232.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230804-225236.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230804-225243.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230804-225249.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230807-221033.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230807-221037.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230809-043913.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230811-221135.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230814-224754.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230814-224757.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230816-221452.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230816-221455.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230825-154517.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230904-221612.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230911-222120.yaml delete mode 100644 .changes/1.7.0/Dependencies-20230912-222718.yaml delete mode 100644 .changes/1.7.0/Features-20221229-200956.yaml delete mode 100644 .changes/1.7.0/Features-20230707-104150.yaml delete mode 100644 .changes/1.7.0/Features-20230707-113337.yaml delete mode 100644 .changes/1.7.0/Features-20230707-114650.yaml delete mode 100644 .changes/1.7.0/Features-20230707-135442.yaml delete mode 100644 .changes/1.7.0/Features-20230817-130731.yaml delete mode 100644 .changes/1.7.0/Fixes-20230517-142331.yaml delete mode 100644 .changes/1.7.0/Fixes-20230810-014122.yaml delete mode 100644 .changes/1.7.0/Security-20230817-145626.yaml delete mode 100644 .changes/1.7.0/Under the Hood-20230724-165508.yaml delete mode 100644 .changes/1.7.0/Under the Hood-20230830-160616.yaml delete mode 100644 .changes/unreleased/Features-20230921-180958.yaml delete mode 100644 .changes/unreleased/Features-20231011-094718.yaml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index dbd792cdd..486768676 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.7.0b2 +current_version = 1.8.0a1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.7.0-b1.md b/.changes/1.7.0-b1.md deleted file mode 100644 index c2ee24afb..000000000 --- a/.changes/1.7.0-b1.md +++ /dev/null @@ -1,47 +0,0 @@ -## dbt-spark 1.7.0-b1 - August 17, 2023 - -### Features - -- Support server side parameters in thrift connection ([#387](https://github.com/dbt-labs/dbt-spark/issues/387)) -- Support server_side_parameters for Spark session connection method ([#690](https://github.com/dbt-labs/dbt-spark/issues/690)) -- Add server_side_parameters to HTTP connection method ([#824](https://github.com/dbt-labs/dbt-spark/issues/824)) -- Enforce server side parameters keys and values to be strings ([#826](https://github.com/dbt-labs/dbt-spark/issues/826)) -- Add SessionConnectionWrapper ([#829](https://github.com/dbt-labs/dbt-spark/issues/829)) - -### Fixes - -- Wrap AnalysisException into DbtRuntimeError ([#782](https://github.com/dbt-labs/dbt-spark/issues/782)) -- include tblproperties macro in adapters.sql create table ([#865](https://github.com/dbt-labs/dbt-spark/issues/865)) - -### Under the Hood - -- Update stale workflow to use centralized version ([#842](https://github.com/dbt-labs/dbt-spark/issues/842)) - -### Dependencies - -- Update tox requirement from ~=3.0 to ~=4.5 ([#741](https://github.com/dbt-labs/dbt-spark/pull/741)) -- Update pyodbc requirement from ~=4.0.30 to ~=4.0.39 ([#742](https://github.com/dbt-labs/dbt-spark/pull/742)) -- Update pre-commit requirement from ~=2.21 to ~=3.3 ([#748](https://github.com/dbt-labs/dbt-spark/pull/748)) -- Update types-requests requirement from ~=2.28 to ~=2.29 ([#749](https://github.com/dbt-labs/dbt-spark/pull/749)) -- Bump mypy from 1.2.0 to 1.3.0 ([#768](https://github.com/dbt-labs/dbt-spark/pull/768)) -- Update flake8 requirement from ~=6.0 to ~=6.1 ([#849](https://github.com/dbt-labs/dbt-spark/pull/849)) -- Update types-requests requirement from ~=2.29 to ~=2.31 ([#850](https://github.com/dbt-labs/dbt-spark/pull/850)) -- Update pytest-xdist requirement from ~=3.2 to ~=3.3 ([#851](https://github.com/dbt-labs/dbt-spark/pull/851)) -- Update pytest requirement from ~=7.3 to ~=7.4 ([#852](https://github.com/dbt-labs/dbt-spark/pull/852)) -- Update pip-tools requirement from ~=6.13 to ~=7.2 ([#856](https://github.com/dbt-labs/dbt-spark/pull/856)) -- Update black requirement from ~=23.3 to ~=23.7 ([#857](https://github.com/dbt-labs/dbt-spark/pull/857)) -- Update wheel requirement from ~=0.40 to ~=0.41 ([#858](https://github.com/dbt-labs/dbt-spark/pull/858)) -- Update mock requirement from ~=5.0 to ~=5.1 ([#859](https://github.com/dbt-labs/dbt-spark/pull/859)) -- Bump mypy from 1.3.0 to 1.4.1 ([#860](https://github.com/dbt-labs/dbt-spark/pull/860)) -- Update tox requirement from ~=4.5 to ~=4.6 ([#861](https://github.com/dbt-labs/dbt-spark/pull/861)) -- Update pip-tools requirement from ~=7.2 to ~=7.3 ([#863](https://github.com/dbt-labs/dbt-spark/pull/863)) -- Update tox requirement from ~=4.6 to ~=4.7 ([#867](https://github.com/dbt-labs/dbt-spark/pull/867)) -- Update tox requirement from ~=4.7 to ~=4.8 ([#871](https://github.com/dbt-labs/dbt-spark/pull/871)) -- Bump mypy from 1.4.1 to 1.5.0 ([#872](https://github.com/dbt-labs/dbt-spark/pull/872)) - -### Contributors -- [@Fokko](https://github.com/Fokko) ([#829](https://github.com/dbt-labs/dbt-spark/issues/829), [#782](https://github.com/dbt-labs/dbt-spark/issues/782)) -- [@Fokko,JCZuurmond](https://github.com/Fokko,JCZuurmond) ([#824](https://github.com/dbt-labs/dbt-spark/issues/824), [#826](https://github.com/dbt-labs/dbt-spark/issues/826)) -- [@alarocca-apixio](https://github.com/alarocca-apixio) ([#690](https://github.com/dbt-labs/dbt-spark/issues/690)) -- [@etheleon](https://github.com/etheleon) ([#865](https://github.com/dbt-labs/dbt-spark/issues/865)) -- [@hanna-liashchuk](https://github.com/hanna-liashchuk) ([#387](https://github.com/dbt-labs/dbt-spark/issues/387)) diff --git a/.changes/1.7.0-b2.md b/.changes/1.7.0-b2.md deleted file mode 100644 index 1f2676536..000000000 --- a/.changes/1.7.0-b2.md +++ /dev/null @@ -1,27 +0,0 @@ -## dbt-spark 1.7.0-b2 - October 02, 2023 - -### Features - -- Persist Column level comments when creating views ([#372](https://github.com/dbt-labs/dbt-spark/issues/372)) - -### Under the Hood - -- Remove dependency on hologram ([#881](https://github.com/dbt-labs/dbt-spark/issues/881)) - -### Dependencies - -- Replace sasl with pure-sasl for PyHive ([#818](https://github.com/dbt-labs/dbt-spark/pull/818)) -- Update tox requirement from ~=4.8 to ~=4.9 ([#874](https://github.com/dbt-labs/dbt-spark/pull/874)) -- Bump mypy from 1.5.0 to 1.5.1 ([#875](https://github.com/dbt-labs/dbt-spark/pull/875)) -- Update tox requirement from ~=4.9 to ~=4.10 ([#879](https://github.com/dbt-labs/dbt-spark/pull/879)) -- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#884](https://github.com/dbt-labs/dbt-spark/pull/884)) -- Update black requirement from ~=23.7 to ~=23.9 ([#886](https://github.com/dbt-labs/dbt-spark/pull/886)) -- Update tox requirement from ~=4.10 to ~=4.11 ([#887](https://github.com/dbt-labs/dbt-spark/pull/887)) - -### Security - -- Add docker image to the repo ([#876](https://github.com/dbt-labs/dbt-spark/pull/876)) - -### Contributors -- [@Fokko](https://github.com/Fokko) ([#876](https://github.com/dbt-labs/dbt-spark/pull/876)) -- [@jurasan](https://github.com/jurasan) ([#372](https://github.com/dbt-labs/dbt-spark/issues/372)) diff --git a/.changes/1.7.0/Dependencies-20230424-230630.yaml b/.changes/1.7.0/Dependencies-20230424-230630.yaml deleted file mode 100644 index 1f96daad1..000000000 --- a/.changes/1.7.0/Dependencies-20230424-230630.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=3.0 to ~=4.5" -time: 2023-04-24T23:06:30.00000Z -custom: - Author: dependabot[bot] - PR: 741 diff --git a/.changes/1.7.0/Dependencies-20230424-230645.yaml b/.changes/1.7.0/Dependencies-20230424-230645.yaml deleted file mode 100644 index 83e1bb44b..000000000 --- a/.changes/1.7.0/Dependencies-20230424-230645.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pyodbc requirement from ~=4.0.30 to ~=4.0.39" -time: 2023-04-24T23:06:45.00000Z -custom: - Author: dependabot[bot] - PR: 742 diff --git a/.changes/1.7.0/Dependencies-20230501-231003.yaml b/.changes/1.7.0/Dependencies-20230501-231003.yaml deleted file mode 100644 index b3e3a750e..000000000 --- a/.changes/1.7.0/Dependencies-20230501-231003.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit requirement from ~=2.21 to ~=3.3" -time: 2023-05-01T23:10:03.00000Z -custom: - Author: dependabot[bot] - PR: 748 diff --git a/.changes/1.7.0/Dependencies-20230501-231035.yaml b/.changes/1.7.0/Dependencies-20230501-231035.yaml deleted file mode 100644 index 7bbf98202..000000000 --- a/.changes/1.7.0/Dependencies-20230501-231035.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update types-requests requirement from ~=2.28 to ~=2.29" -time: 2023-05-01T23:10:35.00000Z -custom: - Author: dependabot[bot] - PR: 749 diff --git a/.changes/1.7.0/Dependencies-20230510-230725.yaml b/.changes/1.7.0/Dependencies-20230510-230725.yaml deleted file mode 100644 index dfd04ad3b..000000000 --- a/.changes/1.7.0/Dependencies-20230510-230725.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.2.0 to 1.3.0" -time: 2023-05-10T23:07:25.00000Z -custom: - Author: dependabot[bot] - PR: 768 diff --git a/.changes/1.7.0/Dependencies-20230628-121341.yaml b/.changes/1.7.0/Dependencies-20230628-121341.yaml deleted file mode 100644 index 5569c885d..000000000 --- a/.changes/1.7.0/Dependencies-20230628-121341.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Dependencies -body: Replace sasl with pure-sasl for PyHive -time: 2023-06-28T12:13:41.141588-07:00 -custom: - Author: colin-rogers-dbt - PR: "818" diff --git a/.changes/1.7.0/Dependencies-20230803-224622.yaml b/.changes/1.7.0/Dependencies-20230803-224622.yaml deleted file mode 100644 index 119a08e51..000000000 --- a/.changes/1.7.0/Dependencies-20230803-224622.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update flake8 requirement from ~=6.0 to ~=6.1" -time: 2023-08-03T22:46:22.00000Z -custom: - Author: dependabot[bot] - PR: 849 diff --git a/.changes/1.7.0/Dependencies-20230803-224623.yaml b/.changes/1.7.0/Dependencies-20230803-224623.yaml deleted file mode 100644 index 97ffd5550..000000000 --- a/.changes/1.7.0/Dependencies-20230803-224623.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update types-requests requirement from ~=2.29 to ~=2.31" -time: 2023-08-03T22:46:23.00000Z -custom: - Author: dependabot[bot] - PR: 850 diff --git a/.changes/1.7.0/Dependencies-20230803-224626.yaml b/.changes/1.7.0/Dependencies-20230803-224626.yaml deleted file mode 100644 index c8b9ef04a..000000000 --- a/.changes/1.7.0/Dependencies-20230803-224626.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest-xdist requirement from ~=3.2 to ~=3.3" -time: 2023-08-03T22:46:26.00000Z -custom: - Author: dependabot[bot] - PR: 851 diff --git a/.changes/1.7.0/Dependencies-20230803-224629.yaml b/.changes/1.7.0/Dependencies-20230803-224629.yaml deleted file mode 100644 index 6865c7c74..000000000 --- a/.changes/1.7.0/Dependencies-20230803-224629.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pytest requirement from ~=7.3 to ~=7.4" -time: 2023-08-03T22:46:29.00000Z -custom: - Author: dependabot[bot] - PR: 852 diff --git a/.changes/1.7.0/Dependencies-20230804-225232.yaml b/.changes/1.7.0/Dependencies-20230804-225232.yaml deleted file mode 100644 index f4a09b6b0..000000000 --- a/.changes/1.7.0/Dependencies-20230804-225232.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pip-tools requirement from ~=6.13 to ~=7.2" -time: 2023-08-04T22:52:32.00000Z -custom: - Author: dependabot[bot] - PR: 856 diff --git a/.changes/1.7.0/Dependencies-20230804-225236.yaml b/.changes/1.7.0/Dependencies-20230804-225236.yaml deleted file mode 100644 index d45b95516..000000000 --- a/.changes/1.7.0/Dependencies-20230804-225236.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.3 to ~=23.7" -time: 2023-08-04T22:52:36.00000Z -custom: - Author: dependabot[bot] - PR: 857 diff --git a/.changes/1.7.0/Dependencies-20230804-225243.yaml b/.changes/1.7.0/Dependencies-20230804-225243.yaml deleted file mode 100644 index 07b9bdb4e..000000000 --- a/.changes/1.7.0/Dependencies-20230804-225243.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update wheel requirement from ~=0.40 to ~=0.41" -time: 2023-08-04T22:52:43.00000Z -custom: - Author: dependabot[bot] - PR: 858 diff --git a/.changes/1.7.0/Dependencies-20230804-225249.yaml b/.changes/1.7.0/Dependencies-20230804-225249.yaml deleted file mode 100644 index 68b3a2485..000000000 --- a/.changes/1.7.0/Dependencies-20230804-225249.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update mock requirement from ~=5.0 to ~=5.1" -time: 2023-08-04T22:52:49.00000Z -custom: - Author: dependabot[bot] - PR: 859 diff --git a/.changes/1.7.0/Dependencies-20230807-221033.yaml b/.changes/1.7.0/Dependencies-20230807-221033.yaml deleted file mode 100644 index 94a261147..000000000 --- a/.changes/1.7.0/Dependencies-20230807-221033.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.3.0 to 1.4.1" -time: 2023-08-07T22:10:33.00000Z -custom: - Author: dependabot[bot] - PR: 860 diff --git a/.changes/1.7.0/Dependencies-20230807-221037.yaml b/.changes/1.7.0/Dependencies-20230807-221037.yaml deleted file mode 100644 index daa1e3ba0..000000000 --- a/.changes/1.7.0/Dependencies-20230807-221037.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.5 to ~=4.6" -time: 2023-08-07T22:10:37.00000Z -custom: - Author: dependabot[bot] - PR: 861 diff --git a/.changes/1.7.0/Dependencies-20230809-043913.yaml b/.changes/1.7.0/Dependencies-20230809-043913.yaml deleted file mode 100644 index 28432003d..000000000 --- a/.changes/1.7.0/Dependencies-20230809-043913.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pip-tools requirement from ~=7.2 to ~=7.3" -time: 2023-08-09T04:39:13.00000Z -custom: - Author: dependabot[bot] - PR: 863 diff --git a/.changes/1.7.0/Dependencies-20230811-221135.yaml b/.changes/1.7.0/Dependencies-20230811-221135.yaml deleted file mode 100644 index 4fd2e4f54..000000000 --- a/.changes/1.7.0/Dependencies-20230811-221135.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.6 to ~=4.7" -time: 2023-08-11T22:11:35.00000Z -custom: - Author: dependabot[bot] - PR: 867 diff --git a/.changes/1.7.0/Dependencies-20230814-224754.yaml b/.changes/1.7.0/Dependencies-20230814-224754.yaml deleted file mode 100644 index 4cd4a01d4..000000000 --- a/.changes/1.7.0/Dependencies-20230814-224754.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.7 to ~=4.8" -time: 2023-08-14T22:47:54.00000Z -custom: - Author: dependabot[bot] - PR: 871 diff --git a/.changes/1.7.0/Dependencies-20230814-224757.yaml b/.changes/1.7.0/Dependencies-20230814-224757.yaml deleted file mode 100644 index 7014382bd..000000000 --- a/.changes/1.7.0/Dependencies-20230814-224757.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.4.1 to 1.5.0" -time: 2023-08-14T22:47:57.00000Z -custom: - Author: dependabot[bot] - PR: 872 diff --git a/.changes/1.7.0/Dependencies-20230816-221452.yaml b/.changes/1.7.0/Dependencies-20230816-221452.yaml deleted file mode 100644 index 8a5d3f0ca..000000000 --- a/.changes/1.7.0/Dependencies-20230816-221452.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.8 to ~=4.9" -time: 2023-08-16T22:14:52.00000Z -custom: - Author: dependabot[bot] - PR: 874 diff --git a/.changes/1.7.0/Dependencies-20230816-221455.yaml b/.changes/1.7.0/Dependencies-20230816-221455.yaml deleted file mode 100644 index 01a28c487..000000000 --- a/.changes/1.7.0/Dependencies-20230816-221455.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Bump mypy from 1.5.0 to 1.5.1" -time: 2023-08-16T22:14:55.00000Z -custom: - Author: dependabot[bot] - PR: 875 diff --git a/.changes/1.7.0/Dependencies-20230825-154517.yaml b/.changes/1.7.0/Dependencies-20230825-154517.yaml deleted file mode 100644 index 3afae44e4..000000000 --- a/.changes/1.7.0/Dependencies-20230825-154517.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.9 to ~=4.10" -time: 2023-08-25T15:45:17.00000Z -custom: - Author: dependabot[bot] - PR: 879 diff --git a/.changes/1.7.0/Dependencies-20230904-221612.yaml b/.changes/1.7.0/Dependencies-20230904-221612.yaml deleted file mode 100644 index 50d6dbf5e..000000000 --- a/.changes/1.7.0/Dependencies-20230904-221612.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update pre-commit requirement from ~=3.3 to ~=3.4" -time: 2023-09-04T22:16:12.00000Z -custom: - Author: dependabot[bot] - PR: 884 diff --git a/.changes/1.7.0/Dependencies-20230911-222120.yaml b/.changes/1.7.0/Dependencies-20230911-222120.yaml deleted file mode 100644 index b1b2b68bf..000000000 --- a/.changes/1.7.0/Dependencies-20230911-222120.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update black requirement from ~=23.7 to ~=23.9" -time: 2023-09-11T22:21:20.00000Z -custom: - Author: dependabot[bot] - PR: 886 diff --git a/.changes/1.7.0/Dependencies-20230912-222718.yaml b/.changes/1.7.0/Dependencies-20230912-222718.yaml deleted file mode 100644 index 560c91767..000000000 --- a/.changes/1.7.0/Dependencies-20230912-222718.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: "Dependencies" -body: "Update tox requirement from ~=4.10 to ~=4.11" -time: 2023-09-12T22:27:18.00000Z -custom: - Author: dependabot[bot] - PR: 887 diff --git a/.changes/1.7.0/Features-20221229-200956.yaml b/.changes/1.7.0/Features-20221229-200956.yaml deleted file mode 100644 index 1add9bf72..000000000 --- a/.changes/1.7.0/Features-20221229-200956.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Features -body: Support server side parameters in thrift connection -time: 2022-12-29T20:09:56.457776+02:00 -custom: - Author: ' hanna-liashchuk' - Issue: "387" - PR: "577" diff --git a/.changes/1.7.0/Features-20230707-104150.yaml b/.changes/1.7.0/Features-20230707-104150.yaml deleted file mode 100644 index 183a37b45..000000000 --- a/.changes/1.7.0/Features-20230707-104150.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support server_side_parameters for Spark session connection method -time: 2023-07-07T10:41:50.01541+02:00 -custom: - Author: alarocca-apixio - Issue: "690" diff --git a/.changes/1.7.0/Features-20230707-113337.yaml b/.changes/1.7.0/Features-20230707-113337.yaml deleted file mode 100644 index de0a50fe8..000000000 --- a/.changes/1.7.0/Features-20230707-113337.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add server_side_parameters to HTTP connection method -time: 2023-07-07T11:33:37.794112+02:00 -custom: - Author: Fokko,JCZuurmond - Issue: "824" diff --git a/.changes/1.7.0/Features-20230707-114650.yaml b/.changes/1.7.0/Features-20230707-114650.yaml deleted file mode 100644 index 6f1b3d38a..000000000 --- a/.changes/1.7.0/Features-20230707-114650.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enforce server side parameters keys and values to be strings -time: 2023-07-07T11:46:50.390918+02:00 -custom: - Author: Fokko,JCZuurmond - Issue: "826" diff --git a/.changes/1.7.0/Features-20230707-135442.yaml b/.changes/1.7.0/Features-20230707-135442.yaml deleted file mode 100644 index 39b119527..000000000 --- a/.changes/1.7.0/Features-20230707-135442.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Add SessionConnectionWrapper -time: 2023-07-07T13:54:42.41341+02:00 -custom: - Author: Fokko - Issue: "829" diff --git a/.changes/1.7.0/Features-20230817-130731.yaml b/.changes/1.7.0/Features-20230817-130731.yaml deleted file mode 100644 index e88deb7bd..000000000 --- a/.changes/1.7.0/Features-20230817-130731.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Persist Column level comments when creating views -time: 2023-08-17T13:07:31.6812862Z -custom: - Author: jurasan - Issue: 372 diff --git a/.changes/1.7.0/Fixes-20230517-142331.yaml b/.changes/1.7.0/Fixes-20230517-142331.yaml deleted file mode 100644 index 9f90e48b3..000000000 --- a/.changes/1.7.0/Fixes-20230517-142331.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Wrap AnalysisException into DbtRuntimeError -time: 2023-05-17T14:23:31.263799+02:00 -custom: - Author: Fokko - Issue: "782" diff --git a/.changes/1.7.0/Fixes-20230810-014122.yaml b/.changes/1.7.0/Fixes-20230810-014122.yaml deleted file mode 100644 index fcb34237e..000000000 --- a/.changes/1.7.0/Fixes-20230810-014122.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: include tblproperties macro in adapters.sql create table -time: 2023-08-10T01:41:22.782982+08:00 -custom: - Author: etheleon - Issue: "865" diff --git a/.changes/1.7.0/Security-20230817-145626.yaml b/.changes/1.7.0/Security-20230817-145626.yaml deleted file mode 100644 index 4add88cbc..000000000 --- a/.changes/1.7.0/Security-20230817-145626.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Security -body: Add docker image to the repo -time: 2023-08-17T14:56:26.361208+02:00 -custom: - Author: Fokko - PR: "876" diff --git a/.changes/1.7.0/Under the Hood-20230724-165508.yaml b/.changes/1.7.0/Under the Hood-20230724-165508.yaml deleted file mode 100644 index 889484644..000000000 --- a/.changes/1.7.0/Under the Hood-20230724-165508.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Update stale workflow to use centralized version -time: 2023-07-24T16:55:08.096947-04:00 -custom: - Author: mikealfare - Issue: "842" diff --git a/.changes/1.7.0/Under the Hood-20230830-160616.yaml b/.changes/1.7.0/Under the Hood-20230830-160616.yaml deleted file mode 100644 index 018a94030..000000000 --- a/.changes/1.7.0/Under the Hood-20230830-160616.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Remove dependency on hologram -time: 2023-08-30T16:06:16.444881-07:00 -custom: - Author: colin-rogers-dbt - Issue: "881" diff --git a/.changes/unreleased/Features-20230921-180958.yaml b/.changes/unreleased/Features-20230921-180958.yaml deleted file mode 100644 index 66141eb5f..000000000 --- a/.changes/unreleased/Features-20230921-180958.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Support storing test failures as views -time: 2023-09-21T18:09:58.174136-04:00 -custom: - Author: mikealfare - Issue: "6914" diff --git a/.changes/unreleased/Features-20231011-094718.yaml b/.changes/unreleased/Features-20231011-094718.yaml deleted file mode 100644 index 8503a70b8..000000000 --- a/.changes/unreleased/Features-20231011-094718.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Create temporary views with 'or replace' -time: 2023-10-11T09:47:18.485764-07:00 -custom: - Author: annazizian - Issue: "350" diff --git a/CHANGELOG.md b/CHANGELOG.md index 362976a1a..902db37fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,83 +5,6 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) -## dbt-spark 1.7.0-b2 - October 02, 2023 - -### Features - -- Persist Column level comments when creating views ([#372](https://github.com/dbt-labs/dbt-spark/issues/372)) - -### Under the Hood - -- Remove dependency on hologram ([#881](https://github.com/dbt-labs/dbt-spark/issues/881)) - -### Dependencies - -- Replace sasl with pure-sasl for PyHive ([#818](https://github.com/dbt-labs/dbt-spark/pull/818)) -- Update tox requirement from ~=4.8 to ~=4.9 ([#874](https://github.com/dbt-labs/dbt-spark/pull/874)) -- Bump mypy from 1.5.0 to 1.5.1 ([#875](https://github.com/dbt-labs/dbt-spark/pull/875)) -- Update tox requirement from ~=4.9 to ~=4.10 ([#879](https://github.com/dbt-labs/dbt-spark/pull/879)) -- Update pre-commit requirement from ~=3.3 to ~=3.4 ([#884](https://github.com/dbt-labs/dbt-spark/pull/884)) -- Update black requirement from ~=23.7 to ~=23.9 ([#886](https://github.com/dbt-labs/dbt-spark/pull/886)) -- Update tox requirement from ~=4.10 to ~=4.11 ([#887](https://github.com/dbt-labs/dbt-spark/pull/887)) - -### Security - -- Add docker image to the repo ([#876](https://github.com/dbt-labs/dbt-spark/pull/876)) - -### Contributors -- [@Fokko](https://github.com/Fokko) ([#876](https://github.com/dbt-labs/dbt-spark/pull/876)) -- [@jurasan](https://github.com/jurasan) ([#372](https://github.com/dbt-labs/dbt-spark/issues/372)) - - -## dbt-spark 1.7.0-b1 - August 17, 2023 - -### Features - -- Support server side parameters in thrift connection ([#387](https://github.com/dbt-labs/dbt-spark/issues/387)) -- Support server_side_parameters for Spark session connection method ([#690](https://github.com/dbt-labs/dbt-spark/issues/690)) -- Add server_side_parameters to HTTP connection method ([#824](https://github.com/dbt-labs/dbt-spark/issues/824)) -- Enforce server side parameters keys and values to be strings ([#826](https://github.com/dbt-labs/dbt-spark/issues/826)) -- Add SessionConnectionWrapper ([#829](https://github.com/dbt-labs/dbt-spark/issues/829)) - -### Fixes - -- Wrap AnalysisException into DbtRuntimeError ([#782](https://github.com/dbt-labs/dbt-spark/issues/782)) -- include tblproperties macro in adapters.sql create table ([#865](https://github.com/dbt-labs/dbt-spark/issues/865)) - -### Under the Hood - -- Update stale workflow to use centralized version ([#842](https://github.com/dbt-labs/dbt-spark/issues/842)) - -### Dependencies - -- Update tox requirement from ~=3.0 to ~=4.5 ([#741](https://github.com/dbt-labs/dbt-spark/pull/741)) -- Update pyodbc requirement from ~=4.0.30 to ~=4.0.39 ([#742](https://github.com/dbt-labs/dbt-spark/pull/742)) -- Update pre-commit requirement from ~=2.21 to ~=3.3 ([#748](https://github.com/dbt-labs/dbt-spark/pull/748)) -- Update types-requests requirement from ~=2.28 to ~=2.29 ([#749](https://github.com/dbt-labs/dbt-spark/pull/749)) -- Bump mypy from 1.2.0 to 1.3.0 ([#768](https://github.com/dbt-labs/dbt-spark/pull/768)) -- Update flake8 requirement from ~=6.0 to ~=6.1 ([#849](https://github.com/dbt-labs/dbt-spark/pull/849)) -- Update types-requests requirement from ~=2.29 to ~=2.31 ([#850](https://github.com/dbt-labs/dbt-spark/pull/850)) -- Update pytest-xdist requirement from ~=3.2 to ~=3.3 ([#851](https://github.com/dbt-labs/dbt-spark/pull/851)) -- Update pytest requirement from ~=7.3 to ~=7.4 ([#852](https://github.com/dbt-labs/dbt-spark/pull/852)) -- Update pip-tools requirement from ~=6.13 to ~=7.2 ([#856](https://github.com/dbt-labs/dbt-spark/pull/856)) -- Update black requirement from ~=23.3 to ~=23.7 ([#857](https://github.com/dbt-labs/dbt-spark/pull/857)) -- Update wheel requirement from ~=0.40 to ~=0.41 ([#858](https://github.com/dbt-labs/dbt-spark/pull/858)) -- Update mock requirement from ~=5.0 to ~=5.1 ([#859](https://github.com/dbt-labs/dbt-spark/pull/859)) -- Bump mypy from 1.3.0 to 1.4.1 ([#860](https://github.com/dbt-labs/dbt-spark/pull/860)) -- Update tox requirement from ~=4.5 to ~=4.6 ([#861](https://github.com/dbt-labs/dbt-spark/pull/861)) -- Update pip-tools requirement from ~=7.2 to ~=7.3 ([#863](https://github.com/dbt-labs/dbt-spark/pull/863)) -- Update tox requirement from ~=4.6 to ~=4.7 ([#867](https://github.com/dbt-labs/dbt-spark/pull/867)) -- Update tox requirement from ~=4.7 to ~=4.8 ([#871](https://github.com/dbt-labs/dbt-spark/pull/871)) -- Bump mypy from 1.4.1 to 1.5.0 ([#872](https://github.com/dbt-labs/dbt-spark/pull/872)) - -### Contributors -- [@Fokko](https://github.com/Fokko) ([#829](https://github.com/dbt-labs/dbt-spark/issues/829), [#782](https://github.com/dbt-labs/dbt-spark/issues/782)) -- [@Fokko,JCZuurmond](https://github.com/Fokko,JCZuurmond) ([#824](https://github.com/dbt-labs/dbt-spark/issues/824), [#826](https://github.com/dbt-labs/dbt-spark/issues/826)) -- [@alarocca-apixio](https://github.com/alarocca-apixio) ([#690](https://github.com/dbt-labs/dbt-spark/issues/690)) -- [@etheleon](https://github.com/etheleon) ([#865](https://github.com/dbt-labs/dbt-spark/issues/865)) -- [@hanna-liashchuk](https://github.com/hanna-liashchuk) ([#387](https://github.com/dbt-labs/dbt-spark/issues/387)) - ## Previous Releases For information on prior major and minor releases, see their changelogs: - [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md) diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index 3f5d3c0b7..f15b401d1 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.7.0b2" +version = "1.8.0a1" diff --git a/setup.py b/setup.py index 088e5f87d..301b4a41f 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def _get_dbt_core_version(): package_name = "dbt-spark" -package_version = "1.7.0b2" +package_version = "1.8.0a1" dbt_core_version = _get_dbt_core_version() description = """The Apache Spark adapter plugin for dbt""" From adfc057450059c0edb1e0349ce72c6bcd89688cd Mon Sep 17 00:00:00 2001 From: mv1742 Date: Fri, 13 Oct 2023 11:31:30 -0600 Subject: [PATCH 03/11] insert overwrite instead of insert into for new seed runs changie commit Mv1742 load csv table seed (#1) * changie commit * Update Fixes-20231013-120628.yaml --- .changes/unreleased/Fixes-20231013-120032.yaml | 7 +++++++ .changes/unreleased/Fixes-20231013-120628.yaml | 7 +++++++ dbt/include/spark/macros/materializations/seed.sql | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231013-120032.yaml create mode 100644 .changes/unreleased/Fixes-20231013-120628.yaml diff --git a/.changes/unreleased/Fixes-20231013-120032.yaml b/.changes/unreleased/Fixes-20231013-120032.yaml new file mode 100644 index 000000000..f1afb0ca1 --- /dev/null +++ b/.changes/unreleased/Fixes-20231013-120032.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: '- dbt seed now removes all rows from the existing seed tables and replace values. + As explained in [issue 112](https://github.com/fishtown-analytics/dbt-spar' +time: 2023-10-13T12:00:32.866957-06:00 +custom: + Author: mv1742 + Issue: "112" diff --git a/.changes/unreleased/Fixes-20231013-120628.yaml b/.changes/unreleased/Fixes-20231013-120628.yaml new file mode 100644 index 000000000..a448944ad --- /dev/null +++ b/.changes/unreleased/Fixes-20231013-120628.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Overwrite existing rows on existing seed tables. For unmanaged databases (no location specified), the current seed command in + dbt-spark appends to existing seeded tables instead overwriting. +time: 2023-10-13T12:06:28.078483-06:00 +custom: + Author: mv1742 + Issue: "112" diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 196479cb0..d239295be 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -27,7 +27,7 @@ {% endfor %} {% set sql %} - insert into {{ this.render() }} values + insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values {% for row in chunk -%} ({%- for col_name in agate_table.column_names -%} {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%} From 31462b7ce63f996ea10c7fb93b5c95d48e7aeffc Mon Sep 17 00:00:00 2001 From: mv1742 Date: Fri, 13 Oct 2023 19:25:34 -0600 Subject: [PATCH 04/11] merge remote --- dbt/include/spark/macros/materializations/seed.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index d239295be..dd37ff8f6 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -1,11 +1,8 @@ -{% macro spark__get_binding_char() %} - {{ return('?' if target.method == 'odbc' else '%s') }} -{% endmacro %} - - {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} - {{ adapter.drop_relation(old_relation) }} + {{ adapter.truncate_relation(old_relation) }} + {% set sql = "truncate table " ~ old_relation %} + {{ return(sql) }} {% endif %} {% set sql = create_csv_table(model, agate_table) %} {{ return(sql) }} From 9d586ae58b7b9cb47e4130906f2c07206a4fd17f Mon Sep 17 00:00:00 2001 From: Matthew McKnight <91097623+McKnight-42@users.noreply.github.com> Date: Fri, 13 Oct 2023 15:48:35 -0500 Subject: [PATCH 05/11] add docs-issue workflow to dbt-spark (#913) --- .github/workflows/docs-issues.yml | 43 +++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/docs-issues.yml diff --git a/.github/workflows/docs-issues.yml b/.github/workflows/docs-issues.yml new file mode 100644 index 000000000..00a098df8 --- /dev/null +++ b/.github/workflows/docs-issues.yml @@ -0,0 +1,43 @@ +# **what?** +# Open an issue in docs.getdbt.com when a PR is labeled `user docs` + +# **why?** +# To reduce barriers for keeping docs up to date + +# **when?** +# When a PR is labeled `user docs` and is merged. Runs on pull_request_target to run off the workflow already merged, +# not the workflow that existed on the PR branch. This allows old PRs to get comments. + + +name: Open issues in docs.getdbt.com repo when a PR is labeled +run-name: "Open an issue in docs.getdbt.com for PR #${{ github.event.pull_request.number }}" + +on: + pull_request_target: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # opens new issues + pull-requests: write # comments on PRs + + +jobs: + open_issues: + # we only want to run this when the PR has been merged or the label in the labeled event is `user docs`. Otherwise it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.pull_request.merged == true) && + ((github.event.action == 'closed' && contains( github.event.pull_request.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs')) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} PR #${{ github.event.pull_request.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit From 3472886aa8653bff9c53a42e22f384e9b13d195c Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 18:51:55 -0600 Subject: [PATCH 06/11] Update seed.sql --- dbt/include/spark/macros/materializations/seed.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index dd37ff8f6..f6bed1313 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -1,3 +1,11 @@ +<<<<<<< HEAD +======= +{% macro spark__get_binding_char() %} + {{ return('?' if target.method == 'odbc' else '%s') }} +{% endmacro %} + + +>>>>>>> ab7c116 (Update seed.sql) {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} {{ adapter.truncate_relation(old_relation) }} @@ -7,7 +15,7 @@ {% set sql = create_csv_table(model, agate_table) %} {{ return(sql) }} {% endmacro %} - + {% macro spark__load_csv_rows(model, agate_table) %} @@ -74,3 +82,4 @@ {{ return(sql) }} {% endmacro %} + From 7e8f24e29f74f1169510deec39617ce040c13e36 Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 19:00:27 -0600 Subject: [PATCH 07/11] Update seed.sql --- dbt/include/spark/macros/materializations/seed.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index f6bed1313..7daf6f6f7 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -9,7 +9,8 @@ {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} {{ adapter.truncate_relation(old_relation) }} - {% set sql = "truncate table " ~ old_relation %} + {{ adapter.drop_relation(old_relation) }} + {{ return(sql) }} {% endif %} {% set sql = create_csv_table(model, agate_table) %} From c490aa55067f1a6f14ae6b69e585ba1af95eb155 Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 19:18:02 -0600 Subject: [PATCH 08/11] add truncate table function --- dbt/include/spark/macros/adapters.sql | 7 +++++++ .../macros/materializations/incremental/incremental.sql | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index bfc1f198d..461e8f14f 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -342,6 +342,13 @@ {%- endcall %} {% endmacro %} + +{% macro spark__truncate_relation(relation) -%} + {% call statement('truncate_relation', auto_begin=False) -%} + truncate {{ relation.type }} if exists {{ relation }} + {%- endcall %} +{% endmacro %} + {% macro spark__drop_relation(relation) -%} {% call statement('drop_relation', auto_begin=False) -%} drop {{ relation.type }} if exists {{ relation }} diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 10d4f3ed8..1e0531c58 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -66,6 +66,10 @@ re: python models and temporary views. Also, why do neither drop_relation or adapter.drop_relation work here?! + 'unmanaged' tables in spark need to manually delete the database + otherwise drop statement does not delete the underlying data. + TODO:add warning that this feature does not work for Unmanaged tables. + Managed tables are fine. --#} {% call statement('drop_relation') -%} drop table if exists {{ tmp_relation }} From be3a3bb342e0809f0026441029e05915c6db1283 Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 19:38:31 -0600 Subject: [PATCH 09/11] rm changelog --- .changes/unreleased/Fixes-20231013-120032.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .changes/unreleased/Fixes-20231013-120032.yaml diff --git a/.changes/unreleased/Fixes-20231013-120032.yaml b/.changes/unreleased/Fixes-20231013-120032.yaml deleted file mode 100644 index f1afb0ca1..000000000 --- a/.changes/unreleased/Fixes-20231013-120032.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Fixes -body: '- dbt seed now removes all rows from the existing seed tables and replace values. - As explained in [issue 112](https://github.com/fishtown-analytics/dbt-spar' -time: 2023-10-13T12:00:32.866957-06:00 -custom: - Author: mv1742 - Issue: "112" From 873381f145af08922fda049c1928e33fd9ecb930 Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 20:16:16 -0600 Subject: [PATCH 10/11] add seed (#3) * insert overwrite instead of insert into for new seed runs * Mv1742 load csv table seed (#1) * changie commit * Update Fixes-20231013-120628.yaml * insert overwrite instead of insert into for new seed runs changie commit Mv1742 load csv table seed (#1) * changie commit * Update Fixes-20231013-120628.yaml * Mv1742 rebase truncate (#2) * merge remote * add docs-issue workflow to dbt-spark (#913) * Update seed.sql * Update seed.sql * add truncate table function * rm changelog * merge remote --- .changes/unreleased/Fixes-20231013-120628.yaml | 7 +++++++ dbt/include/spark/macros/adapters.sql | 7 +++++++ .../macros/materializations/incremental/incremental.sql | 4 ++++ dbt/include/spark/macros/materializations/seed.sql | 5 ++++- 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231013-120628.yaml diff --git a/.changes/unreleased/Fixes-20231013-120628.yaml b/.changes/unreleased/Fixes-20231013-120628.yaml new file mode 100644 index 000000000..a448944ad --- /dev/null +++ b/.changes/unreleased/Fixes-20231013-120628.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Overwrite existing rows on existing seed tables. For unmanaged databases (no location specified), the current seed command in + dbt-spark appends to existing seeded tables instead overwriting. +time: 2023-10-13T12:06:28.078483-06:00 +custom: + Author: mv1742 + Issue: "112" diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index bfc1f198d..461e8f14f 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -342,6 +342,13 @@ {%- endcall %} {% endmacro %} + +{% macro spark__truncate_relation(relation) -%} + {% call statement('truncate_relation', auto_begin=False) -%} + truncate {{ relation.type }} if exists {{ relation }} + {%- endcall %} +{% endmacro %} + {% macro spark__drop_relation(relation) -%} {% call statement('drop_relation', auto_begin=False) -%} drop {{ relation.type }} if exists {{ relation }} diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 10d4f3ed8..1e0531c58 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -66,6 +66,10 @@ re: python models and temporary views. Also, why do neither drop_relation or adapter.drop_relation work here?! + 'unmanaged' tables in spark need to manually delete the database + otherwise drop statement does not delete the underlying data. + TODO:add warning that this feature does not work for Unmanaged tables. + Managed tables are fine. --#} {% call statement('drop_relation') -%} drop table if exists {{ tmp_relation }} diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 196479cb0..b2b1d6a44 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -5,7 +5,10 @@ {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} + {{ adapter.truncate_relation(old_relation) }} {{ adapter.drop_relation(old_relation) }} + + {{ return(sql) }} {% endif %} {% set sql = create_csv_table(model, agate_table) %} {{ return(sql) }} @@ -27,7 +30,7 @@ {% endfor %} {% set sql %} - insert into {{ this.render() }} values + insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values {% for row in chunk -%} ({%- for col_name in agate_table.column_names -%} {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%} From 48ca04b7c633a454cceb44520278a5b0138c6e3b Mon Sep 17 00:00:00 2001 From: Manrique Vargas Date: Fri, 13 Oct 2023 20:34:02 -0600 Subject: [PATCH 11/11] merge latest --- dbt/include/spark/macros/materializations/seed.sql | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dbt/include/spark/macros/materializations/seed.sql b/dbt/include/spark/macros/materializations/seed.sql index 7daf6f6f7..b2b1d6a44 100644 --- a/dbt/include/spark/macros/materializations/seed.sql +++ b/dbt/include/spark/macros/materializations/seed.sql @@ -1,11 +1,8 @@ -<<<<<<< HEAD -======= {% macro spark__get_binding_char() %} {{ return('?' if target.method == 'odbc' else '%s') }} {% endmacro %} ->>>>>>> ab7c116 (Update seed.sql) {% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %} {% if old_relation %} {{ adapter.truncate_relation(old_relation) }} @@ -16,7 +13,7 @@ {% set sql = create_csv_table(model, agate_table) %} {{ return(sql) }} {% endmacro %} - + {% macro spark__load_csv_rows(model, agate_table) %} @@ -83,4 +80,3 @@ {{ return(sql) }} {% endmacro %} -