From 6077e0c3e171b1aaac5d28eece8a974bb4b588f5 Mon Sep 17 00:00:00 2001 From: Pat Kearns Date: Thu, 18 Aug 2022 14:41:08 +1000 Subject: [PATCH 1/2] add extra fields --- macros/create_tests_table_if_not_exists.sql | 15 ++++++ macros/upload_tests.sql | 55 +++++++++++++++++++-- models/dim_dbt__tests.sql | 5 ++ models/staging/stg_dbt__tests.sql | 5 ++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/macros/create_tests_table_if_not_exists.sql b/macros/create_tests_table_if_not_exists.sql index d6044af5..a989cb18 100644 --- a/macros/create_tests_table_if_not_exists.sql +++ b/macros/create_tests_table_if_not_exists.sql @@ -22,7 +22,12 @@ node_id STRING, run_started_at TIMESTAMP, name STRING, + short_name STRING, + test_type STRING, + test_severity_config STRING, depends_on_nodes STRING, + model_refs STRING, + source_refs STRING, package_name STRING, test_path STRING, tags STRING @@ -36,7 +41,12 @@ node_id STRING, run_started_at TIMESTAMP_TZ, name STRING, + short_name STRING, + test_type STRING, + test_severity_config STRING, depends_on_nodes ARRAY, + model_refs STRING, + source_refs STRING, package_name STRING, test_path STRING, tags ARRAY @@ -49,7 +59,12 @@ node_id STRING, run_started_at TIMESTAMP, name STRING, + short_name STRING, + test_type STRING, + test_severity_config STRING, depends_on_nodes STRING, + model_refs STRING, + source_refs STRING, package_name STRING, test_path STRING, tags STRING diff --git a/macros/upload_tests.sql b/macros/upload_tests.sql index a62e213f..20d7a443 100644 --- a/macros/upload_tests.sql +++ b/macros/upload_tests.sql @@ -12,18 +12,43 @@ {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(5)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, - {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }} + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }} from values {% for test in tests -%} + + {%- if test.test_metadata is defined -%} + {%- set test_name = test.test_metadata.name -%} + {%- set test_type = 'generic' -%} + + {%- if test_name == 'relationships' -%} + {%- set column_name = test.test_metadata.kwargs.field ~ ',' ~ test.test_metadata.kwargs.column_name -%} + {%- else -%} + {%- set column_name = test.test_metadata.kwargs.column_name -%} + {%- endif -%} + {%- elif test.name is defined -%} + {%- set test_name = test.name -%} + {%- set test_type = 'singular' -%} + {%- endif %} + ( '{{ invocation_id }}', {# command_invocation_id #} '{{ test.unique_id }}', {# node_id #} '{{ run_started_at }}', {# run_started_at #} '{{ test.name }}', {# name #} + '{{ test_name }}', {# short_name #} + '{{ test_type }}', {# test_type #} + '{{ test.config.severity }}', {# test_severity_config #} '{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ process_refs(test.refs) }}', {# model_refs #} + '{{ process_refs(test.sources, is_src=true) }}', {# source_refs #} '{{ test.package_name }}', {# package_name #} '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} '{{ tojson(test.tags) }}' {# tags #} @@ -31,7 +56,6 @@ {%- if not loop.last %},{%- endif %} {%- endfor %} {% endset %} - {{ dbt_artifacts.insert_into_metadata_table( database_name=src_dbt_tests.database, schema_name=src_dbt_tests.schema, @@ -41,3 +65,28 @@ }} {% endif %} {% endmacro -%} + +/* + return a comma delimited string of the models or sources were related to the test. + e.g. dim_customers,fct_orders + behaviour changes slightly with the is_src flag because: + - models come through as [['model'], ['model_b']] + - srcs come through as [['source','table'], ['source_b','table_b']] +*/ +{% macro process_refs( ref_list, is_src=false ) %} + {% set refs = [] %} + + {% if ref_list is defined and ref_list|length > 0 %} + {% for ref in ref_list %} + {% if is_src %} + {{ refs.append(ref|join('.')) }} + {% else %} + {{ refs.append(ref[0]) }} + {% endif %} + {% endfor %} + + {{ return(refs|join(',')) }} + {% else %} + {{ return('') }} + {% endif %} +{% endmacro %} diff --git a/models/dim_dbt__tests.sql b/models/dim_dbt__tests.sql index c56504e9..84836d27 100644 --- a/models/dim_dbt__tests.sql +++ b/models/dim_dbt__tests.sql @@ -13,7 +13,12 @@ tests as ( node_id, run_started_at, name, + short_name, + test_type, + test_severity_config, depends_on_nodes, + model_refs, + source_refs, package_name, test_path, tags diff --git a/models/staging/stg_dbt__tests.sql b/models/staging/stg_dbt__tests.sql index a69b4088..6c62adb0 100644 --- a/models/staging/stg_dbt__tests.sql +++ b/models/staging/stg_dbt__tests.sql @@ -13,7 +13,12 @@ enhanced as ( node_id, run_started_at, name, + short_name, + test_type, + test_severity_config, depends_on_nodes, + model_refs, + source_refs, package_name, test_path, tags From 69383297703777bb3d8f4200239c042760ff2809 Mon Sep 17 00:00:00 2001 From: Pat Kearns Date: Thu, 18 Aug 2022 14:51:45 +1000 Subject: [PATCH 2/2] add docs --- models/dim_dbt__tests.yml | 10 ++++++++++ models/docs.md | 32 +++++++++++++++++++++++++++++++ models/staging/stg_dbt__tests.yml | 10 ++++++++++ 3 files changed, 52 insertions(+) diff --git a/models/dim_dbt__tests.yml b/models/dim_dbt__tests.yml index 5a94af15..ef5a6722 100644 --- a/models/dim_dbt__tests.yml +++ b/models/dim_dbt__tests.yml @@ -8,6 +8,8 @@ models: description: '{{ doc("command_invocation_id") }}' - name: depends_on_nodes description: '{{ doc("depends_on_nodes") }}' + - name: model_refs + description: '{{ doc("model_refs") }}' - name: name description: '{{ doc("name") }}' - name: node_id @@ -16,9 +18,17 @@ models: description: '{{ doc("package_name") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' + - name: short_name + description: '{{ doc("short_name") }}' + - name: source_refs + description: '{{ doc("source_refs") }}' - name: tags description: '{{ doc("tags") }}' - name: test_execution_id description: '{{ doc("test_execution_id") }}' - name: test_path description: '{{ doc("test_path") }}' + - name: test_severity_config + description: '{{ doc("test_severity_config") }}' + - name: test_type + description: '{{ doc("test_type") }}' diff --git a/models/docs.md b/models/docs.md index e02e065f..3efc8777 100644 --- a/models/docs.md +++ b/models/docs.md @@ -16,6 +16,12 @@ Name of the node. {% enddocs %} +{% docs short_name %} + +Short Name for a test, for example 'unique'. + +{% enddocs %} + {% docs schema %} Configured schema for the node. @@ -352,3 +358,29 @@ Number of rows affected by the node's last run. Total time spent executing the node's last run (seconds). {% enddocs %} + + + +{% docs model_refs %} + +A comma delimited string of the models related to this node, as defined in the execution graph. + +{% enddocs %} + +{% docs model_refs %} + +A comma delimited string of the sources related to this node, as defined in the execution graph. + +{% enddocs %} + +{% docs test_severity_config %} + +Configuration for test severity. For example 'ERROR' or 'WARN'. + +{% enddocs %} + +{% docs test_type %} + +Type of test. For example 'singular' or 'generic' + +{% enddocs %} diff --git a/models/staging/stg_dbt__tests.yml b/models/staging/stg_dbt__tests.yml index bf7be7fa..dec8bba0 100644 --- a/models/staging/stg_dbt__tests.yml +++ b/models/staging/stg_dbt__tests.yml @@ -8,6 +8,8 @@ models: description: '{{ doc("command_invocation_id") }}' - name: depends_on_nodes description: '{{ doc("depends_on_nodes") }}' + - name: model_refs + description: '{{ doc("model_refs") }}' - name: name description: '{{ doc("name") }}' - name: node_id @@ -16,9 +18,17 @@ models: description: '{{ doc("package_name") }}' - name: run_started_at description: '{{ doc("run_started_at") }}' + - name: short_name + description: '{{ doc("short_name") }}' + - name: source_refs + description: '{{ doc("source_refs") }}' - name: tags description: '{{ doc("tags") }}' - name: test_execution_id description: '{{ doc("test_execution_id") }}' - name: test_path description: '{{ doc("test_path") }}' + - name: test_severity_config + description: '{{ doc("test_severity_config") }}' + - name: test_type + description: '{{ doc("test_type") }}'