From 0851d957d3195799da9dfc0c46b3ee0a732ddf87 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 10:45:21 -0400 Subject: [PATCH 01/18] Added new macros array_append and create_array and integration tests --- .../data/cross_db/data_array_append.csv | 2 ++ .../data/cross_db/data_create_array.csv | 2 ++ .../models/cross_db_utils/schema.yml | 12 +++++++ .../cross_db_utils/test_array_append.sql | 33 +++++++++++++++++++ .../cross_db_utils/test_create_array.sql | 12 +++++++ macros/cross_db_utils/array_append.sql | 15 +++++++++ macros/cross_db_utils/create_array.sql | 19 +++++++++++ 7 files changed, 95 insertions(+) create mode 100644 integration_tests/data/cross_db/data_array_append.csv create mode 100644 integration_tests/data/cross_db/data_create_array.csv create mode 100644 integration_tests/models/cross_db_utils/test_array_append.sql create mode 100644 integration_tests/models/cross_db_utils/test_create_array.sql create mode 100644 macros/cross_db_utils/array_append.sql create mode 100644 macros/cross_db_utils/create_array.sql diff --git a/integration_tests/data/cross_db/data_array_append.csv b/integration_tests/data/cross_db/data_array_append.csv new file mode 100644 index 00000000..8775fe12 --- /dev/null +++ b/integration_tests/data/cross_db/data_array_append.csv @@ -0,0 +1,2 @@ +array_as_string,element,result_as_string +"[1,""hello"",true,undefined]",4,"[1,""hello"",true,undefined,4]" \ No newline at end of file diff --git a/integration_tests/data/cross_db/data_create_array.csv b/integration_tests/data/cross_db/data_create_array.csv new file mode 100644 index 00000000..6a205d83 --- /dev/null +++ b/integration_tests/data/cross_db/data_create_array.csv @@ -0,0 +1,2 @@ +num_input,string_input,boolean_input,null_input,result_as_string +1,hello,true,null,"[1,""hello"",true,undefined]" \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/schema.yml b/integration_tests/models/cross_db_utils/schema.yml index e1473c9f..98382339 100644 --- a/integration_tests/models/cross_db_utils/schema.yml +++ b/integration_tests/models/cross_db_utils/schema.yml @@ -6,6 +6,12 @@ models: - dbt_utils.equality: compare_model: ref('data_any_value_expected') + - name: test_array_append + tests: + - assert_equal: + actual: actual + expected: expected + - name: test_bool_or tests: - dbt_utils.equality: @@ -16,6 +22,12 @@ models: - assert_equal: actual: actual expected: expected + + - name: test_create_array + tests: + - assert_equal: + actual: actual + expected: expected - name: test_current_timestamp tests: diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql new file mode 100644 index 00000000..757ec43e --- /dev/null +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -0,0 +1,33 @@ +with data as ( + + select + data_array_append.element, + data_array_append.result_as_string, + data_create_array.num_input, + data_create_array.string_input, + data_create_array.boolean_input, + data_create_array.null_input + from {{ ref('data_array_append') }} as data_array_append + left join {{ ref('data_create_array') }} as data_create_array + on data_array_append.array_as_string = data_create_array.result_as_string + +), + +create_array as ( + select + + {{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as array, + element, + result_as_string as expected + + from data + +) + +select + + {{ dbt_utils.array_append('array', 'element') }} as array_actual, + cast({{ dbt_utils.array_append('array', 'element') }} as string) as actual, + expected + +from create_array \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_create_array.sql new file mode 100644 index 00000000..025f4687 --- /dev/null +++ b/integration_tests/models/cross_db_utils/test_create_array.sql @@ -0,0 +1,12 @@ +with data as ( + + select * from {{ ref('data_create_array') }} + +) + +select + {{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as array_actual, + cast({{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as string) as actual, + result_as_string as expected + +from data \ No newline at end of file diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql new file mode 100644 index 00000000..50995bfd --- /dev/null +++ b/macros/cross_db_utils/array_append.sql @@ -0,0 +1,15 @@ +{% macro array_append(array, new_element) -%} + {{ return(adapter.dispatch('array_append', 'dbt_utils')(array, new_element)) }} +{%- endmacro %} + +{% macro default__array_append(array, new_element) -%} + array_append({{ array }}, {{ new_element }}) +{%- endmacro %} + +{% macro bigquery__array_append(array, new_element) -%} + array_concat({{ array }}, {{ create_array(new_element) }}) +{%- endmacro %} + +{% macro redshift__array_append(array, new_element) -%} + array_concat({{ array }}, {{ create_array(new_element) }}) +{%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/create_array.sql b/macros/cross_db_utils/create_array.sql new file mode 100644 index 00000000..3c49cba6 --- /dev/null +++ b/macros/cross_db_utils/create_array.sql @@ -0,0 +1,19 @@ +{% macro create_array(inputs) -%} + {{ return(adapter.dispatch('array', 'dbt_utils')(inputs)) }} +{%- endmacro %} + +{% macro default__array(inputs) -%} + array[ {{ inputs }} ] +{%- endmacro %} + +{% macro snowflake__array(inputs) -%} + array_construct( {{ inputs|join(' , ') }} ) +{%- endmacro %} + +{% macro redshift__array(inputs) -%} + array( {{ inputs }} ) +{%- endmacro %} + +{% macro bigquery__array(inputs) -%} + [ {{ inputs }} ] +{%- endmacro %} \ No newline at end of file From 16f12ac0a1cb719f79800d7e899802bec5e3ddbc Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 10:50:36 -0400 Subject: [PATCH 02/18] updated casting to use type_string --- integration_tests/models/cross_db_utils/test_array_append.sql | 2 +- integration_tests/models/cross_db_utils/test_create_array.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index 757ec43e..882c458e 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -27,7 +27,7 @@ create_array as ( select {{ dbt_utils.array_append('array', 'element') }} as array_actual, - cast({{ dbt_utils.array_append('array', 'element') }} as string) as actual, + cast({{ dbt_utils.array_append('array', 'element') }} as {{ dbt_utils.type_string() }}) as actual, expected from create_array \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_create_array.sql index 025f4687..79f5e09a 100644 --- a/integration_tests/models/cross_db_utils/test_create_array.sql +++ b/integration_tests/models/cross_db_utils/test_create_array.sql @@ -6,7 +6,7 @@ with data as ( select {{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as array_actual, - cast({{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as string) as actual, + cast({{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as {{ dbt_utils.type_string() }}) as actual, result_as_string as expected from data \ No newline at end of file From 299fa789edd1a184b333363c55815adc4ffe14ec Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 10:58:03 -0400 Subject: [PATCH 03/18] joined list togeter --- macros/cross_db_utils/array_append.sql | 4 ++-- macros/cross_db_utils/create_array.sql | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql index 50995bfd..bbbb5957 100644 --- a/macros/cross_db_utils/array_append.sql +++ b/macros/cross_db_utils/array_append.sql @@ -7,9 +7,9 @@ {%- endmacro %} {% macro bigquery__array_append(array, new_element) -%} - array_concat({{ array }}, {{ create_array(new_element) }}) + array_concat({{ array }}, {{ dbt_utils.create_array(new_element) }}) {%- endmacro %} {% macro redshift__array_append(array, new_element) -%} - array_concat({{ array }}, {{ create_array(new_element) }}) + array_concat({{ array }}, {{ dbt_utils.create_array(new_element) }}) {%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/create_array.sql b/macros/cross_db_utils/create_array.sql index 3c49cba6..5e42a065 100644 --- a/macros/cross_db_utils/create_array.sql +++ b/macros/cross_db_utils/create_array.sql @@ -3,7 +3,7 @@ {%- endmacro %} {% macro default__array(inputs) -%} - array[ {{ inputs }} ] + array[ {{ inputs|join(' , ') }} ] {%- endmacro %} {% macro snowflake__array(inputs) -%} @@ -11,9 +11,9 @@ {%- endmacro %} {% macro redshift__array(inputs) -%} - array( {{ inputs }} ) + array( {{ inputs|join(' , ') }} ) {%- endmacro %} {% macro bigquery__array(inputs) -%} - [ {{ inputs }} ] + [ {{ inputs|join(' , ') }} ] {%- endmacro %} \ No newline at end of file From b656aea5ec688763a48060c2a50fd45a64b82179 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 12:06:44 -0400 Subject: [PATCH 04/18] updated functionality to only allow for single data type in array to be compatible with postgres --- integration_tests/data/cross_db/data_array_append.csv | 2 +- integration_tests/data/cross_db/data_create_array.csv | 4 ++-- .../models/cross_db_utils/test_array_append.sql | 9 ++++----- .../models/cross_db_utils/test_create_array.sql | 4 ++-- macros/cross_db_utils/array_append.sql | 1 + macros/cross_db_utils/create_array.sql | 1 + 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration_tests/data/cross_db/data_array_append.csv b/integration_tests/data/cross_db/data_array_append.csv index 8775fe12..46dda688 100644 --- a/integration_tests/data/cross_db/data_array_append.csv +++ b/integration_tests/data/cross_db/data_array_append.csv @@ -1,2 +1,2 @@ array_as_string,element,result_as_string -"[1,""hello"",true,undefined]",4,"[1,""hello"",true,undefined,4]" \ No newline at end of file +"[1,2,3]",4,"[1,2,3,4]" \ No newline at end of file diff --git a/integration_tests/data/cross_db/data_create_array.csv b/integration_tests/data/cross_db/data_create_array.csv index 6a205d83..be497c39 100644 --- a/integration_tests/data/cross_db/data_create_array.csv +++ b/integration_tests/data/cross_db/data_create_array.csv @@ -1,2 +1,2 @@ -num_input,string_input,boolean_input,null_input,result_as_string -1,hello,true,null,"[1,""hello"",true,undefined]" \ No newline at end of file +num_input_1,num_input_2,num_input_3,result_as_string +1,2,3,"[1,2,3]" \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index 882c458e..f56c41e5 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -3,10 +3,9 @@ with data as ( select data_array_append.element, data_array_append.result_as_string, - data_create_array.num_input, - data_create_array.string_input, - data_create_array.boolean_input, - data_create_array.null_input + data_create_array.num_input_1, + data_create_array.num_input_2, + data_create_array.num_input_3 from {{ ref('data_array_append') }} as data_array_append left join {{ ref('data_create_array') }} as data_create_array on data_array_append.array_as_string = data_create_array.result_as_string @@ -16,7 +15,7 @@ with data as ( create_array as ( select - {{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as array, + {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array, element, result_as_string as expected diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_create_array.sql index 79f5e09a..b10edfe4 100644 --- a/integration_tests/models/cross_db_utils/test_create_array.sql +++ b/integration_tests/models/cross_db_utils/test_create_array.sql @@ -5,8 +5,8 @@ with data as ( ) select - {{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as array_actual, - cast({{ dbt_utils.create_array(['num_input', 'string_input', 'boolean_input', 'null_input']) }} as {{ dbt_utils.type_string() }}) as actual, + {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, + cast({{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as {{ dbt_utils.type_string() }}) as actual, result_as_string as expected from data \ No newline at end of file diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql index bbbb5957..ee328bf1 100644 --- a/macros/cross_db_utils/array_append.sql +++ b/macros/cross_db_utils/array_append.sql @@ -2,6 +2,7 @@ {{ return(adapter.dispatch('array_append', 'dbt_utils')(array, new_element)) }} {%- endmacro %} +{# new_element must be the same data type as elements in array to match postgres functionality #} {% macro default__array_append(array, new_element) -%} array_append({{ array }}, {{ new_element }}) {%- endmacro %} diff --git a/macros/cross_db_utils/create_array.sql b/macros/cross_db_utils/create_array.sql index 5e42a065..6211a5c4 100644 --- a/macros/cross_db_utils/create_array.sql +++ b/macros/cross_db_utils/create_array.sql @@ -2,6 +2,7 @@ {{ return(adapter.dispatch('array', 'dbt_utils')(inputs)) }} {%- endmacro %} +{# all inputs must be the same data type to match postgres functionality #} {% macro default__array(inputs) -%} array[ {{ inputs|join(' , ') }} ] {%- endmacro %} From ed205f75082da2f40f84a55c059700c2ebc651e6 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 12:35:35 -0400 Subject: [PATCH 05/18] updated test_array_append --- .../cross_db_utils/test_array_append.sql | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index f56c41e5..b1346624 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -10,23 +10,12 @@ with data as ( left join {{ ref('data_create_array') }} as data_create_array on data_array_append.array_as_string = data_create_array.result_as_string -), - -create_array as ( - select - - {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array, - element, - result_as_string as expected - - from data - ) select - {{ dbt_utils.array_append('array', 'element') }} as array_actual, - cast({{ dbt_utils.array_append('array', 'element') }} as {{ dbt_utils.type_string() }}) as actual, - expected + {{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, + cast({{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as {{ dbt_utils.type_string() }}) as actual, + result_as_string as expected -from create_array \ No newline at end of file +from data \ No newline at end of file From 3918e5376c6e7bb748d9c7cf8b26ea85983e5e80 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 15:02:22 -0400 Subject: [PATCH 06/18] replaced {} with [] for matching in postgres --- .../cross_db_utils/test_array_append.sql | 22 ++++++++++++++----- .../cross_db_utils/test_create_array.sql | 21 +++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index b1346624..4c8772f8 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -10,12 +10,22 @@ with data as ( left join {{ ref('data_create_array') }} as data_create_array on data_array_append.array_as_string = data_create_array.result_as_string -) +), -select +appended_array as ( - {{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, - cast({{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as {{ dbt_utils.type_string() }}) as actual, - result_as_string as expected + select + {{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, + cast({{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as {{ dbt_utils.type_string() }}) as actual, + result_as_string as expected + from data -from data \ No newline at end of file +) + +-- when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) +-- we need to cast the arrays to strings in order to compare them to the output in our seed file +select + array_actual, + {{ dbt_utils.replace(dbt_utils.replace('actual',"'}'","']'"),"'{'","'['") }} as actual, + expected +from appended_array \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_create_array.sql index b10edfe4..d111877c 100644 --- a/integration_tests/models/cross_db_utils/test_create_array.sql +++ b/integration_tests/models/cross_db_utils/test_create_array.sql @@ -2,11 +2,22 @@ with data as ( select * from {{ ref('data_create_array') }} +), + +create_array as ( + select + {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, + cast({{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as {{ dbt_utils.type_string() }}) as actual, + result_as_string as expected + + from data + ) +-- when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) +-- we need to cast the arrays to strings in order to compare them to the output in our seed file select - {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, - cast({{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as {{ dbt_utils.type_string() }}) as actual, - result_as_string as expected - -from data \ No newline at end of file + array_actual, + {{ dbt_utils.replace(dbt_utils.replace('actual',"'}'","']'"),"'{'","'['") }} as actual, + expected +from create_array \ No newline at end of file From 4e1ae2218b832bafcb329148712cee0f90836f07 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 15:31:16 -0400 Subject: [PATCH 07/18] fixed redshift --- macros/cross_db_utils/array_append.sql | 4 ++-- macros/cross_db_utils/create_array.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql index ee328bf1..77b84e2c 100644 --- a/macros/cross_db_utils/array_append.sql +++ b/macros/cross_db_utils/array_append.sql @@ -8,9 +8,9 @@ {%- endmacro %} {% macro bigquery__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.create_array(new_element) }}) + array_concat({{ array }}, {{ dbt_utils.create_array([new_element]) }}) {%- endmacro %} {% macro redshift__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.create_array(new_element) }}) + array_concat({{ array }}, {{ dbt_utils.create_array([new_element]) }}) {%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/create_array.sql b/macros/cross_db_utils/create_array.sql index 6211a5c4..0afde91a 100644 --- a/macros/cross_db_utils/create_array.sql +++ b/macros/cross_db_utils/create_array.sql @@ -1,4 +1,4 @@ -{% macro create_array(inputs) -%} +{% macro create_array(inputs=[]) -%} {{ return(adapter.dispatch('array', 'dbt_utils')(inputs)) }} {%- endmacro %} From dcb915c14eaf5e1dca9ace25e66653f83e06a511 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 16:39:45 -0400 Subject: [PATCH 08/18] Added cast_array_to_string macro --- .../cross_db_utils/test_array_append.sql | 3 +-- .../cross_db_utils/test_create_array.sql | 4 +--- .../cross_db_utils/cast_array_to_string.sql | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 macros/cross_db_utils/cast_array_to_string.sql diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index 4c8772f8..dd13fa46 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -16,7 +16,6 @@ appended_array as ( select {{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, - cast({{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as {{ dbt_utils.type_string() }}) as actual, result_as_string as expected from data @@ -26,6 +25,6 @@ appended_array as ( -- we need to cast the arrays to strings in order to compare them to the output in our seed file select array_actual, - {{ dbt_utils.replace(dbt_utils.replace('actual',"'}'","']'"),"'{'","'['") }} as actual, + {{ dbt_utils.cast_array_to_string('array_actual') }} as actual, expected from appended_array \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_create_array.sql index d111877c..bc5b222b 100644 --- a/integration_tests/models/cross_db_utils/test_create_array.sql +++ b/integration_tests/models/cross_db_utils/test_create_array.sql @@ -7,17 +7,15 @@ with data as ( create_array as ( select {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, - cast({{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as {{ dbt_utils.type_string() }}) as actual, result_as_string as expected from data ) --- when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) -- we need to cast the arrays to strings in order to compare them to the output in our seed file select array_actual, - {{ dbt_utils.replace(dbt_utils.replace('actual',"'}'","']'"),"'{'","'['") }} as actual, + {{ dbt_utils.cast_array_to_string('array_actual') }} as actual, expected from create_array \ No newline at end of file diff --git a/macros/cross_db_utils/cast_array_to_string.sql b/macros/cross_db_utils/cast_array_to_string.sql new file mode 100644 index 00000000..7a44f6d2 --- /dev/null +++ b/macros/cross_db_utils/cast_array_to_string.sql @@ -0,0 +1,20 @@ +{% macro cast_array_to_string(field) %} + {{ adapter.dispatch('cast_array_to_string', 'dbt_utils') (field) }} +{% endmacro %} + +{% macro default__cast_array_to_string(field) %} + cast({{ field }} as {{ dbt_utils.type_string() }}) +{% endmacro %} + +{% macro postgres__cast_array_to_string(field) %} + cast({{ dbt_utils.replace(dbt_utils.replace(field,"'}'","']'"),"'{'","'['") }} as {{ dbt_utils.type_string() }}) +{% endmacro %} + +{# redshift should use default instead of postgres #} +{% macro redshift__cast_array_to_string(field) %} + cast({{ field }} as {{ dbt_utils.type_string() }}) +{% endmacro %} + +{% macro bigquery__cast_array_to_string(field) %} + '['||(select string_agg(cast(element as string), ',') from unnest({{ field }}) element)||']' +{% endmacro %} From b47d0302bbe702e1ad8add2e452b14d525620fea Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 18 May 2022 16:53:20 -0400 Subject: [PATCH 09/18] updated postgres cast_array_to_string --- macros/cross_db_utils/cast_array_to_string.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macros/cross_db_utils/cast_array_to_string.sql b/macros/cross_db_utils/cast_array_to_string.sql index 7a44f6d2..05226a82 100644 --- a/macros/cross_db_utils/cast_array_to_string.sql +++ b/macros/cross_db_utils/cast_array_to_string.sql @@ -7,7 +7,8 @@ {% endmacro %} {% macro postgres__cast_array_to_string(field) %} - cast({{ dbt_utils.replace(dbt_utils.replace(field,"'}'","']'"),"'{'","'['") }} as {{ dbt_utils.type_string() }}) + {%- set field_as_string -%}cast({{ field }} as {{ dbt_utils.type_string() }}){%- endset -%} + {{ dbt_utils.replace(dbt_utils.replace(field_as_string,"'}'","']'"),"'{'","'['") }} {% endmacro %} {# redshift should use default instead of postgres #} From 9b2a6bfa62cc23b2b271c8cf4df96714e436cf61 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 25 May 2022 09:02:07 -0400 Subject: [PATCH 10/18] change name of create_array macro to array_construct --- ...ata_create_array.csv => data_array_construct.csv} | 0 integration_tests/models/cross_db_utils/schema.yml | 2 +- .../models/cross_db_utils/test_array_append.sql | 12 ++++++------ ...est_create_array.sql => test_array_construct.sql} | 8 ++++---- macros/cross_db_utils/array_append.sql | 4 ++-- .../{create_array.sql => array_construct.sql} | 12 ++++++------ 6 files changed, 19 insertions(+), 19 deletions(-) rename integration_tests/data/cross_db/{data_create_array.csv => data_array_construct.csv} (100%) rename integration_tests/models/cross_db_utils/{test_create_array.sql => test_array_construct.sql} (59%) rename macros/cross_db_utils/{create_array.sql => array_construct.sql} (51%) diff --git a/integration_tests/data/cross_db/data_create_array.csv b/integration_tests/data/cross_db/data_array_construct.csv similarity index 100% rename from integration_tests/data/cross_db/data_create_array.csv rename to integration_tests/data/cross_db/data_array_construct.csv diff --git a/integration_tests/models/cross_db_utils/schema.yml b/integration_tests/models/cross_db_utils/schema.yml index 98382339..f03b5c0f 100644 --- a/integration_tests/models/cross_db_utils/schema.yml +++ b/integration_tests/models/cross_db_utils/schema.yml @@ -23,7 +23,7 @@ models: actual: actual expected: expected - - name: test_create_array + - name: test_array_construct tests: - assert_equal: actual: actual diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index dd13fa46..fa3f0fae 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -3,19 +3,19 @@ with data as ( select data_array_append.element, data_array_append.result_as_string, - data_create_array.num_input_1, - data_create_array.num_input_2, - data_create_array.num_input_3 + data_array_construct.num_input_1, + data_array_construct.num_input_2, + data_array_construct.num_input_3 from {{ ref('data_array_append') }} as data_array_append - left join {{ ref('data_create_array') }} as data_create_array - on data_array_append.array_as_string = data_create_array.result_as_string + left join {{ ref('data_array_construct') }} as data_array_construct + on data_array_append.array_as_string = data_array_construct.result_as_string ), appended_array as ( select - {{ dbt_utils.array_append(dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, + {{ dbt_utils.array_append(dbt_utils.array_construct(['num_input_1', 'num_input_2', 'num_input_3']), 'element') }} as array_actual, result_as_string as expected from data diff --git a/integration_tests/models/cross_db_utils/test_create_array.sql b/integration_tests/models/cross_db_utils/test_array_construct.sql similarity index 59% rename from integration_tests/models/cross_db_utils/test_create_array.sql rename to integration_tests/models/cross_db_utils/test_array_construct.sql index bc5b222b..137562ad 100644 --- a/integration_tests/models/cross_db_utils/test_create_array.sql +++ b/integration_tests/models/cross_db_utils/test_array_construct.sql @@ -1,12 +1,12 @@ with data as ( - select * from {{ ref('data_create_array') }} + select * from {{ ref('data_array_construct') }} ), -create_array as ( +array_construct as ( select - {{ dbt_utils.create_array(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, + {{ dbt_utils.array_construct(['num_input_1', 'num_input_2', 'num_input_3']) }} as array_actual, result_as_string as expected from data @@ -18,4 +18,4 @@ select array_actual, {{ dbt_utils.cast_array_to_string('array_actual') }} as actual, expected -from create_array \ No newline at end of file +from array_construct \ No newline at end of file diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql index 77b84e2c..7f0ff056 100644 --- a/macros/cross_db_utils/array_append.sql +++ b/macros/cross_db_utils/array_append.sql @@ -8,9 +8,9 @@ {%- endmacro %} {% macro bigquery__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.create_array([new_element]) }}) + array_concat({{ array }}, {{ dbt_utils.array_construct([new_element]) }}) {%- endmacro %} {% macro redshift__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.create_array([new_element]) }}) + array_concat({{ array }}, {{ dbt_utils.array_construct([new_element]) }}) {%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/create_array.sql b/macros/cross_db_utils/array_construct.sql similarity index 51% rename from macros/cross_db_utils/create_array.sql rename to macros/cross_db_utils/array_construct.sql index 0afde91a..7e896b6b 100644 --- a/macros/cross_db_utils/create_array.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -1,20 +1,20 @@ -{% macro create_array(inputs=[]) -%} - {{ return(adapter.dispatch('array', 'dbt_utils')(inputs)) }} +{% macro array_construct(inputs=[]) -%} + {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs)) }} {%- endmacro %} {# all inputs must be the same data type to match postgres functionality #} -{% macro default__array(inputs) -%} +{% macro default__array_construct(inputs) -%} array[ {{ inputs|join(' , ') }} ] {%- endmacro %} -{% macro snowflake__array(inputs) -%} +{% macro snowflake__array_construct(inputs) -%} array_construct( {{ inputs|join(' , ') }} ) {%- endmacro %} -{% macro redshift__array(inputs) -%} +{% macro redshift__array_construct(inputs) -%} array( {{ inputs|join(' , ') }} ) {%- endmacro %} -{% macro bigquery__array(inputs) -%} +{% macro bigquery__array_construct(inputs) -%} [ {{ inputs|join(' , ') }} ] {%- endmacro %} \ No newline at end of file From 0a7fb48796ba2e06bc7ff9972828fea297e5d24a Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 25 May 2022 09:25:48 -0400 Subject: [PATCH 11/18] add array_concat macro and integration tests --- .../data/cross_db/data_array_concat.csv | 2 ++ .../data/cross_db/data_array_construct.csv | 3 +- .../models/cross_db_utils/schema.yml | 18 ++++++---- .../cross_db_utils/test_array_append.sql | 1 - .../cross_db_utils/test_array_concat.sql | 36 +++++++++++++++++++ macros/cross_db_utils/array_append.sql | 4 +-- macros/cross_db_utils/array_concat.sql | 15 ++++++++ .../cross_db_utils/cast_array_to_string.sql | 1 + 8 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 integration_tests/data/cross_db/data_array_concat.csv create mode 100644 integration_tests/models/cross_db_utils/test_array_concat.sql create mode 100644 macros/cross_db_utils/array_concat.sql diff --git a/integration_tests/data/cross_db/data_array_concat.csv b/integration_tests/data/cross_db/data_array_concat.csv new file mode 100644 index 00000000..0eda79cc --- /dev/null +++ b/integration_tests/data/cross_db/data_array_concat.csv @@ -0,0 +1,2 @@ +array_1_as_string,array_2_as_string,result_as_string +"[1,2,3]","[4,5,6]","[1,2,3,4,5,6]" \ No newline at end of file diff --git a/integration_tests/data/cross_db/data_array_construct.csv b/integration_tests/data/cross_db/data_array_construct.csv index be497c39..fe76c47b 100644 --- a/integration_tests/data/cross_db/data_array_construct.csv +++ b/integration_tests/data/cross_db/data_array_construct.csv @@ -1,2 +1,3 @@ num_input_1,num_input_2,num_input_3,result_as_string -1,2,3,"[1,2,3]" \ No newline at end of file +1,2,3,"[1,2,3]" +4,5,6,"[4,5,6]" \ No newline at end of file diff --git a/integration_tests/models/cross_db_utils/schema.yml b/integration_tests/models/cross_db_utils/schema.yml index f03b5c0f..c33b0b5e 100644 --- a/integration_tests/models/cross_db_utils/schema.yml +++ b/integration_tests/models/cross_db_utils/schema.yml @@ -11,13 +11,8 @@ models: - assert_equal: actual: actual expected: expected - - - name: test_bool_or - tests: - - dbt_utils.equality: - compare_model: ref('data_bool_or_expected') - - name: test_concat + - name: test_array_concat tests: - assert_equal: actual: actual @@ -28,6 +23,17 @@ models: - assert_equal: actual: actual expected: expected + + - name: test_bool_or + tests: + - dbt_utils.equality: + compare_model: ref('data_bool_or_expected') + + - name: test_concat + tests: + - assert_equal: + actual: actual + expected: expected - name: test_current_timestamp tests: diff --git a/integration_tests/models/cross_db_utils/test_array_append.sql b/integration_tests/models/cross_db_utils/test_array_append.sql index fa3f0fae..12193a62 100644 --- a/integration_tests/models/cross_db_utils/test_array_append.sql +++ b/integration_tests/models/cross_db_utils/test_array_append.sql @@ -21,7 +21,6 @@ appended_array as ( ) --- when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) -- we need to cast the arrays to strings in order to compare them to the output in our seed file select array_actual, diff --git a/integration_tests/models/cross_db_utils/test_array_concat.sql b/integration_tests/models/cross_db_utils/test_array_concat.sql new file mode 100644 index 00000000..907a9dff --- /dev/null +++ b/integration_tests/models/cross_db_utils/test_array_concat.sql @@ -0,0 +1,36 @@ +with data as ( + + select + data_array_concat.result_as_string, + array_1.num_input_1 as array_1_num_input_1, + array_1.num_input_2 as array_1_num_input_2, + array_1.num_input_3 as array_1_num_input_3, + array_2.num_input_1 as array_2_num_input_1, + array_2.num_input_2 as array_2_num_input_2, + array_2.num_input_3 as array_2_num_input_3 + from {{ ref('data_array_concat') }} as data_array_concat + left join {{ ref('data_array_construct') }} as array_1 + on data_array_concat.array_1_as_string = array_1.result_as_string + left join {{ ref('data_array_construct') }} as array_2 + on data_array_concat.array_2_as_string = array_2.result_as_string + +), + +concat_array as ( + + select + {{ dbt_utils.array_concat( + dbt_utils.array_construct(['array_1_num_input_1', 'array_1_num_input_2', 'array_1_num_input_3']), + dbt_utils.array_construct(['array_2_num_input_1', 'array_2_num_input_2', 'array_2_num_input_3']) + ) }} as array_actual, + result_as_string as expected + from data + +) + +-- we need to cast the arrays to strings in order to compare them to the output in our seed file +select + array_actual, + {{ dbt_utils.cast_array_to_string('array_actual') }} as actual, + expected +from concat_array \ No newline at end of file diff --git a/macros/cross_db_utils/array_append.sql b/macros/cross_db_utils/array_append.sql index 7f0ff056..04791088 100644 --- a/macros/cross_db_utils/array_append.sql +++ b/macros/cross_db_utils/array_append.sql @@ -8,9 +8,9 @@ {%- endmacro %} {% macro bigquery__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.array_construct([new_element]) }}) + {{ dbt_utils.array_concat(array, dbt_utils.array_construct([new_element])) }} {%- endmacro %} {% macro redshift__array_append(array, new_element) -%} - array_concat({{ array }}, {{ dbt_utils.array_construct([new_element]) }}) + {{ dbt_utils.array_concat(array, dbt_utils.array_construct([new_element])) }} {%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/array_concat.sql b/macros/cross_db_utils/array_concat.sql new file mode 100644 index 00000000..1b7983c2 --- /dev/null +++ b/macros/cross_db_utils/array_concat.sql @@ -0,0 +1,15 @@ +{% macro array_concat(array_1, array_2) -%} + {{ return(adapter.dispatch('array_concat', 'dbt_utils')(array_1, array_2)) }} +{%- endmacro %} + +{% macro default__array_concat(array_1, array_2) -%} + array_cat({{ array_1 }}, {{ array_2 }}) +{%- endmacro %} + +{% macro bigquery__array_concat(array_1, array_2) -%} + array_concat({{ array_1 }}, {{ array_2 }}) +{%- endmacro %} + +{% macro redshift__array_concat(array_1, array_2) -%} + array_concat({{ array_1 }}, {{ array_2 }}) +{%- endmacro %} \ No newline at end of file diff --git a/macros/cross_db_utils/cast_array_to_string.sql b/macros/cross_db_utils/cast_array_to_string.sql index 05226a82..9c7fefef 100644 --- a/macros/cross_db_utils/cast_array_to_string.sql +++ b/macros/cross_db_utils/cast_array_to_string.sql @@ -6,6 +6,7 @@ cast({{ field }} as {{ dbt_utils.type_string() }}) {% endmacro %} +{# when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) #} {% macro postgres__cast_array_to_string(field) %} {%- set field_as_string -%}cast({{ field }} as {{ dbt_utils.type_string() }}){%- endset -%} {{ dbt_utils.replace(dbt_utils.replace(field_as_string,"'}'","']'"),"'{'","'['") }} From 668a914995029270751883cf9d0a8c93631ccbf1 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 12:45:22 -0400 Subject: [PATCH 12/18] updated readme and added empty array_contruct to intergation tests --- README.md | 56 ++++++++++++++++++- .../cross_db_utils/test_array_construct.sql | 6 ++ .../cross_db_utils/cast_array_to_string.sql | 24 ++++---- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 97660630..3123ad09 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ For compatibility details between versions of dbt-core and dbt-utils, [see this - [last_day](#last_day-source) - [width_bucket](#width_bucket-source) - [listagg](#listagg-source) + - [array_construct](#array_construct-source) + - [array_append](#array_append-source) + - [array_concat](#array_concat-source) + - [cast_array_to_string](#cast_array_to_string-source) - [Jinja Helpers](#jinja-helpers) - [pretty_time](#pretty_time-source) @@ -991,7 +995,7 @@ Boolean values are replaced with the strings 'true'|'false' | 2017-03-01 | processing | size | S | | 2017-03-01 | processing | color | red | -**Args**: +**Args:** - `relation`: The [Relation](https://docs.getdbt.com/docs/writing-code-in-dbt/class-reference/#relation) to unpivot. - `cast_to`: The data type to cast the unpivoted values to, default is varchar - `exclude`: A list of columns to exclude from the unpivot operation but keep in the resulting table. @@ -1055,7 +1059,7 @@ This macro calculates the difference between two dates. #### split_part ([source](macros/cross_db_utils/split_part.sql)) This macro splits a string of text using the supplied delimiter and returns the supplied part number (1-indexed). -**Args**: +**Args:** - `string_text` (required): Text to be split into parts. - `delimiter_text` (required): Text representing the delimiter to split by. - `part_number` (required): Requested part of the split (1-based). If the value is negative, the parts are counted backward from the end of the string. @@ -1114,7 +1118,7 @@ When an expression falls outside the range, the function returns: #### listagg ([source](macros/cross_db_utils/listagg.sql)) This macro returns the concatenated input values from a group of rows separated by a specified deliminator. -**Args**: +**Args:** - `measure` (required): The expression (typically a column name) that determines the values to be concatenated. To only include distinct values add keyword DISTINCT to beginning of expression (example: 'DISTINCT column_to_agg'). - `delimiter_text` (required): Text representing the delimiter to separate concatenated values by. - `order_by_clause` (optional): An expression (typically a column name) that determines the order of the concatenated values. @@ -1127,6 +1131,52 @@ Note: If there are instances of `delimiter_text` within your `measure`, you cann {{ dbt_utils.listagg(measure='column_to_agg', delimiter_text="','", order_by_clause="order by order_by_column", limit_num=10) }} ``` +#### array_construct ([source](macros/cross_db_utils/array_construct.sql)) +This macro returns an array constructed from a set of inputs. + +**Args:** +- `inputs` (optional): The list of array contents. If not provided, this macro will create an empty array. All inputs must be the *same data type* in order to match Postgres functionality and *not null* to match Bigquery functionality. + +**Usage:** +``` +{{ dbt_utils.array_construct(['column_1', 'column_2', 'column_3']) }} +``` + +#### array_append ([source](macros/cross_db_utils/array_append.sql)) +This macro appends an element to the end of an array and returns the appended array. + +**Args:** +- `array` (required): The array to append to. +- `new_element` (required): The element to be appended. This element must *match the data type of the existing elements* in the array in order to match Postgres functionality and *not null* to match Bigquery functionality. + +**Usage:** +``` +{{ dbt_utils.array_append('array_column', 'element_column') }} +``` + +#### array_concat ([source](macros/cross_db_utils/array_concat.sql)) +This macro returns the concatenation of two arrays. + +**Args:** +- `array_1` (required): The array to append to. +- `array_2` (required): The array to be appended to `array_1`. This array must match the data type of `array_1` in order to match Postgres functionality. + +**Usage:** +``` +{{ dbt_utils.array_concat('array_column_1', 'array_column_2') }} +``` + +#### cast_array_to_string ([source](macros/cross_db_utils/cast_array_to_string.sql)) +This macro converts an array to a single string value and returns the resulting string. + +**Args:** +- `array` (required): The array to convert to a string. + +**Usage:** +``` +{{ dbt_utils.cast_array_to_string('array_column') }} +``` + --- ### Jinja Helpers #### pretty_time ([source](macros/jinja_helpers/pretty_time.sql)) diff --git a/integration_tests/models/cross_db_utils/test_array_construct.sql b/integration_tests/models/cross_db_utils/test_array_construct.sql index 137562ad..52dbf4be 100644 --- a/integration_tests/models/cross_db_utils/test_array_construct.sql +++ b/integration_tests/models/cross_db_utils/test_array_construct.sql @@ -11,6 +11,12 @@ array_construct as ( from data + union all + + select + {{ dbt_utils.array_construct() }} as array_actual, + '[]' as expected + ) -- we need to cast the arrays to strings in order to compare them to the output in our seed file diff --git a/macros/cross_db_utils/cast_array_to_string.sql b/macros/cross_db_utils/cast_array_to_string.sql index 9c7fefef..c88d6ef0 100644 --- a/macros/cross_db_utils/cast_array_to_string.sql +++ b/macros/cross_db_utils/cast_array_to_string.sql @@ -1,22 +1,22 @@ -{% macro cast_array_to_string(field) %} - {{ adapter.dispatch('cast_array_to_string', 'dbt_utils') (field) }} +{% macro cast_array_to_string(array) %} + {{ adapter.dispatch('cast_array_to_string', 'dbt_utils') (array) }} {% endmacro %} -{% macro default__cast_array_to_string(field) %} - cast({{ field }} as {{ dbt_utils.type_string() }}) +{% macro default__cast_array_to_string(array) %} + cast({{ array }} as {{ dbt_utils.type_string() }}) {% endmacro %} {# when casting as array to string, postgres uses {} (ex: {1,2,3}) while other dbs use [] (ex: [1,2,3]) #} -{% macro postgres__cast_array_to_string(field) %} - {%- set field_as_string -%}cast({{ field }} as {{ dbt_utils.type_string() }}){%- endset -%} - {{ dbt_utils.replace(dbt_utils.replace(field_as_string,"'}'","']'"),"'{'","'['") }} +{% macro postgres__cast_array_to_string(array) %} + {%- set array_as_string -%}cast({{ array }} as {{ dbt_utils.type_string() }}){%- endset -%} + {{ dbt_utils.replace(dbt_utils.replace(array_as_string,"'}'","']'"),"'{'","'['") }} {% endmacro %} {# redshift should use default instead of postgres #} -{% macro redshift__cast_array_to_string(field) %} - cast({{ field }} as {{ dbt_utils.type_string() }}) +{% macro redshift__cast_array_to_string(array) %} + cast({{ array }} as {{ dbt_utils.type_string() }}) {% endmacro %} -{% macro bigquery__cast_array_to_string(field) %} - '['||(select string_agg(cast(element as string), ',') from unnest({{ field }}) element)||']' -{% endmacro %} +{% macro bigquery__cast_array_to_string(array) %} + '['||(select string_agg(cast(element as string), ',') from unnest({{ array }}) element)||']' +{% endmacro %} \ No newline at end of file From 9c3d02a8554fb8a86fc873b0f4001035496f7a57 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 13:01:09 -0400 Subject: [PATCH 13/18] added default array data_type for postgres func --- README.md | 2 ++ macros/cross_db_utils/array_construct.sql | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3123ad09..9221ebe1 100644 --- a/README.md +++ b/README.md @@ -1136,10 +1136,12 @@ This macro returns an array constructed from a set of inputs. **Args:** - `inputs` (optional): The list of array contents. If not provided, this macro will create an empty array. All inputs must be the *same data type* in order to match Postgres functionality and *not null* to match Bigquery functionality. +- `data_type` (optional): Specifies the data type of the constructed array. This is only relevant when creating an empty array (will otherwise use the data type of the inputs). If `inputs` are `data_type` are both not provided, this macro will create an empty array of type varchar. **Usage:** ``` {{ dbt_utils.array_construct(['column_1', 'column_2', 'column_3']) }} +{{ dbt_utils.array_construct([],'integer') }} ``` #### array_append ([source](macros/cross_db_utils/array_append.sql)) diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index 7e896b6b..b39873c2 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -1,10 +1,14 @@ -{% macro array_construct(inputs=[]) -%} +{% macro array_construct(inputs = [], data_type = 'varchar') -%} {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs)) }} {%- endmacro %} {# all inputs must be the same data type to match postgres functionality #} {% macro default__array_construct(inputs) -%} + {% if input|length > 0 %} array[ {{ inputs|join(' , ') }} ] + {% else %} + array[]::{{data_type}}[] + {% endif %} {%- endmacro %} {% macro snowflake__array_construct(inputs) -%} From 014c524d75f6e026cda4fd42a942864ab5037ed4 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 13:04:08 -0400 Subject: [PATCH 14/18] added data_type arg --- macros/cross_db_utils/array_construct.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index b39873c2..aa58ac4d 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -3,7 +3,7 @@ {%- endmacro %} {# all inputs must be the same data type to match postgres functionality #} -{% macro default__array_construct(inputs) -%} +{% macro default__array_construct(inputs, data_type) -%} {% if input|length > 0 %} array[ {{ inputs|join(' , ') }} ] {% else %} @@ -11,14 +11,14 @@ {% endif %} {%- endmacro %} -{% macro snowflake__array_construct(inputs) -%} +{% macro snowflake__array_construct(inputs, data_type) -%} array_construct( {{ inputs|join(' , ') }} ) {%- endmacro %} -{% macro redshift__array_construct(inputs) -%} +{% macro redshift__array_construct(inputs, data_type) -%} array( {{ inputs|join(' , ') }} ) {%- endmacro %} -{% macro bigquery__array_construct(inputs) -%} +{% macro bigquery__array_construct(inputs, data_type) -%} [ {{ inputs|join(' , ') }} ] {%- endmacro %} \ No newline at end of file From 2bd3df1b771cd059bf78b0a29d52b8c6fa5d41ba Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 14:38:37 -0400 Subject: [PATCH 15/18] postgres fix --- macros/cross_db_utils/array_construct.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index aa58ac4d..809f4e4a 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -1,5 +1,5 @@ {% macro array_construct(inputs = [], data_type = 'varchar') -%} - {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs)) }} + {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs, data_type)) }} {%- endmacro %} {# all inputs must be the same data type to match postgres functionality #} From d76654d17decedb86a42bf20f82ecf9e0e77112b Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 14:52:53 -0400 Subject: [PATCH 16/18] fix variable name --- macros/cross_db_utils/array_construct.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index 809f4e4a..d7e801f9 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -4,7 +4,7 @@ {# all inputs must be the same data type to match postgres functionality #} {% macro default__array_construct(inputs, data_type) -%} - {% if input|length > 0 %} + {% if inputs|length > 0 %} array[ {{ inputs|join(' , ') }} ] {% else %} array[]::{{data_type}}[] From b6cdf9ec74867f1644061fded013d9eff917b1c1 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 14:59:42 -0400 Subject: [PATCH 17/18] changed default array data type to integer --- README.md | 2 +- macros/cross_db_utils/array_construct.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9221ebe1..2656da09 100644 --- a/README.md +++ b/README.md @@ -1136,7 +1136,7 @@ This macro returns an array constructed from a set of inputs. **Args:** - `inputs` (optional): The list of array contents. If not provided, this macro will create an empty array. All inputs must be the *same data type* in order to match Postgres functionality and *not null* to match Bigquery functionality. -- `data_type` (optional): Specifies the data type of the constructed array. This is only relevant when creating an empty array (will otherwise use the data type of the inputs). If `inputs` are `data_type` are both not provided, this macro will create an empty array of type varchar. +- `data_type` (optional): Specifies the data type of the constructed array. This is only relevant when creating an empty array (will otherwise use the data type of the inputs). If `inputs` are `data_type` are both not provided, this macro will create an empty array of type integer. **Usage:** ``` diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index d7e801f9..0c323ab4 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -1,4 +1,4 @@ -{% macro array_construct(inputs = [], data_type = 'varchar') -%} +{% macro array_construct(inputs = [], data_type = 'integer') -%} {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs, data_type)) }} {%- endmacro %} From 0085420684142a19b0a02518b2c814a81f1a5c1c Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 14 Jun 2022 15:13:20 -0400 Subject: [PATCH 18/18] updated default data_type to use api.Column.translate_type --- macros/cross_db_utils/array_construct.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/cross_db_utils/array_construct.sql b/macros/cross_db_utils/array_construct.sql index 0c323ab4..63c6fe3b 100644 --- a/macros/cross_db_utils/array_construct.sql +++ b/macros/cross_db_utils/array_construct.sql @@ -1,4 +1,4 @@ -{% macro array_construct(inputs = [], data_type = 'integer') -%} +{% macro array_construct(inputs = [], data_type = api.Column.translate_type('integer')) -%} {{ return(adapter.dispatch('array_construct', 'dbt_utils')(inputs, data_type)) }} {%- endmacro %}