Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/tests add fields #177

Closed
wants to merge 2 commits into from
Closed

Feature/tests add fields #177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6077e0c
add extra fields
patkearns10 Aug 18, 2022
6938329
add docs
patkearns10 Aug 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions macros/create_tests_table_if_not_exists.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@
node_id STRING,
run_started_at TIMESTAMP,
name STRING,
short_name STRING,
test_type STRING,
test_severity_config STRING,
depends_on_nodes STRING,
model_refs STRING,
source_refs STRING,
package_name STRING,
test_path STRING,
tags STRING
Expand All @@ -36,7 +41,12 @@
node_id STRING,
run_started_at TIMESTAMP_TZ,
name STRING,
short_name STRING,
test_type STRING,
test_severity_config STRING,
depends_on_nodes ARRAY,
model_refs STRING,
source_refs STRING,
package_name STRING,
test_path STRING,
tags ARRAY
Expand All @@ -49,7 +59,12 @@
node_id STRING,
run_started_at TIMESTAMP,
name STRING,
short_name STRING,
test_type STRING,
test_severity_config STRING,
depends_on_nodes STRING,
model_refs STRING,
source_refs STRING,
package_name STRING,
test_path STRING,
tags STRING
Expand Down
55 changes: 52 additions & 3 deletions macros/upload_tests.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,50 @@
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(5)) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }}
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(8)) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }},
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }},
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }}
from values
{% for test in tests -%}

{%- if test.test_metadata is defined -%}
{%- set test_name = test.test_metadata.name -%}
{%- set test_type = 'generic' -%}

{%- if test_name == 'relationships' -%}
{%- set column_name = test.test_metadata.kwargs.field ~ ',' ~ test.test_metadata.kwargs.column_name -%}
{%- else -%}
{%- set column_name = test.test_metadata.kwargs.column_name -%}
{%- endif -%}
{%- elif test.name is defined -%}
{%- set test_name = test.name -%}
{%- set test_type = 'singular' -%}
{%- endif %}

(
'{{ invocation_id }}', {# command_invocation_id #}
'{{ test.unique_id }}', {# node_id #}
'{{ run_started_at }}', {# run_started_at #}
'{{ test.name }}', {# name #}
'{{ test_name }}', {# short_name #}
'{{ test_type }}', {# test_type #}
'{{ test.config.severity }}', {# test_severity_config #}
'{{ tojson(test.depends_on.nodes) }}', {# depends_on_nodes #}
'{{ process_refs(test.refs) }}', {# model_refs #}
'{{ process_refs(test.sources, is_src=true) }}', {# source_refs #}
'{{ test.package_name }}', {# package_name #}
'{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #}
'{{ tojson(test.tags) }}' {# tags #}
)
{%- if not loop.last %},{%- endif %}
{%- endfor %}
{% endset %}

{{ dbt_artifacts.insert_into_metadata_table(
database_name=src_dbt_tests.database,
schema_name=src_dbt_tests.schema,
Expand All @@ -41,3 +65,28 @@
}}
{% endif %}
{% endmacro -%}

/*
return a comma delimited string of the models or sources were related to the test.
e.g. dim_customers,fct_orders
behaviour changes slightly with the is_src flag because:
- models come through as [['model'], ['model_b']]
- srcs come through as [['source','table'], ['source_b','table_b']]
*/
{% macro process_refs( ref_list, is_src=false ) %}
{% set refs = [] %}

{% if ref_list is defined and ref_list|length > 0 %}
{% for ref in ref_list %}
{% if is_src %}
{{ refs.append(ref|join('.')) }}
{% else %}
{{ refs.append(ref[0]) }}
{% endif %}
{% endfor %}

{{ return(refs|join(',')) }}
{% else %}
{{ return('') }}
{% endif %}
{% endmacro %}
5 changes: 5 additions & 0 deletions models/dim_dbt__tests.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@ tests as (
node_id,
run_started_at,
name,
short_name,
test_type,
test_severity_config,
depends_on_nodes,
model_refs,
source_refs,
package_name,
test_path,
tags
Expand Down
10 changes: 10 additions & 0 deletions models/dim_dbt__tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ models:
description: '{{ doc("command_invocation_id") }}'
- name: depends_on_nodes
description: '{{ doc("depends_on_nodes") }}'
- name: model_refs
description: '{{ doc("model_refs") }}'
- name: name
description: '{{ doc("name") }}'
- name: node_id
Expand All @@ -16,9 +18,17 @@ models:
description: '{{ doc("package_name") }}'
- name: run_started_at
description: '{{ doc("run_started_at") }}'
- name: short_name
description: '{{ doc("short_name") }}'
- name: source_refs
description: '{{ doc("source_refs") }}'
- name: tags
description: '{{ doc("tags") }}'
- name: test_execution_id
description: '{{ doc("test_execution_id") }}'
- name: test_path
description: '{{ doc("test_path") }}'
- name: test_severity_config
description: '{{ doc("test_severity_config") }}'
- name: test_type
description: '{{ doc("test_type") }}'
32 changes: 32 additions & 0 deletions models/docs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ Name of the node.

{% enddocs %}

{% docs short_name %}

Short Name for a test, for example 'unique'.

{% enddocs %}

{% docs schema %}

Configured schema for the node.
Expand Down Expand Up @@ -352,3 +358,29 @@ Number of rows affected by the node's last run.
Total time spent executing the node's last run (seconds).

{% enddocs %}



{% docs model_refs %}

A comma delimited string of the models related to this node, as defined in the execution graph.

{% enddocs %}

{% docs model_refs %}

A comma delimited string of the sources related to this node, as defined in the execution graph.

{% enddocs %}

{% docs test_severity_config %}

Configuration for test severity. For example 'ERROR' or 'WARN'.

{% enddocs %}

{% docs test_type %}

Type of test. For example 'singular' or 'generic'

{% enddocs %}
5 changes: 5 additions & 0 deletions models/staging/stg_dbt__tests.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@ enhanced as (
node_id,
run_started_at,
name,
short_name,
test_type,
test_severity_config,
depends_on_nodes,
model_refs,
source_refs,
package_name,
test_path,
tags
Expand Down
10 changes: 10 additions & 0 deletions models/staging/stg_dbt__tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ models:
description: '{{ doc("command_invocation_id") }}'
- name: depends_on_nodes
description: '{{ doc("depends_on_nodes") }}'
- name: model_refs
description: '{{ doc("model_refs") }}'
- name: name
description: '{{ doc("name") }}'
- name: node_id
Expand All @@ -16,9 +18,17 @@ models:
description: '{{ doc("package_name") }}'
- name: run_started_at
description: '{{ doc("run_started_at") }}'
- name: short_name
description: '{{ doc("short_name") }}'
- name: source_refs
description: '{{ doc("source_refs") }}'
- name: tags
description: '{{ doc("tags") }}'
- name: test_execution_id
description: '{{ doc("test_execution_id") }}'
- name: test_path
description: '{{ doc("test_path") }}'
- name: test_severity_config
description: '{{ doc("test_severity_config") }}'
- name: test_type
description: '{{ doc("test_type") }}'