From 803397bb9c16e27a1c3eabb23b1a9d0e1da4fb8f Mon Sep 17 00:00:00 2001 From: jonhopper-dataengineers Date: Sun, 21 Jul 2024 12:32:23 +1200 Subject: [PATCH] Added in the ability to set the profile targets for the materializations `stages`, `file_format`, `tables` --- CHANGELOG.md | 5 ++ macros/file_formats/stage_file_formats.sql | 39 +++++----- .../snowflake__get_source_build_plan.sql | 3 +- macros/source_tables/stage_table_sources.sql | 74 ++++++++++--------- macros/stages/stage_stages.sql | 32 ++++---- 5 files changed, 82 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7352bab..8dd5295 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # dbt_dataengineers_materializations Changelog +## 0.2.8.1 - Profile Targets + +* Added in the ability to set the profile targets for the materializations `stages`, `file_format`, `tables` + - tables + ## 0.2.8 - Secrets, Network Rules & UDF's Addition of the materializations: diff --git a/macros/file_formats/stage_file_formats.sql b/macros/file_formats/stage_file_formats.sql index 933a013..4a813bf 100644 --- a/macros/file_formats/stage_file_formats.sql +++ b/macros/file_formats/stage_file_formats.sql @@ -1,26 +1,27 @@ -{% macro stage_file_formats(enabled_targets) %} - {% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %} - {% if target.name in enabled_targets %} - {% set items_to_stage = [] %} - - {% set nodes = graph.nodes.values() if graph.nodes else [] %} - {% for node in nodes %} - {% if node.config.materialized == 'file_format' %} - {% do items_to_stage.append(node) %} +{% macro stage_file_formats(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %} + {% if target.profile_name in enabled_profiles %} + {% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %} + {% if target.name in enabled_targets %} + {% set items_to_stage = [] %} + + {% set nodes = graph.nodes.values() if graph.nodes else [] %} + {% for node in nodes %} + {% if node.config.materialized == 'file_format' %} + {% do items_to_stage.append(node) %} + {% endif %} + {% endfor %} + + {% do log('file formats to create: ' ~ items_to_stage|length, info = true) %} + + {# Initial run to cater for #} + {% if items_to_stage|length > 0 %} + {% do dbt_dataengineers_materializations.stage_file_format_plans(items_to_stage) %} {% endif %} - {% endfor %} - - {% do log('file formats to create: ' ~ items_to_stage|length, info = true) %} - - {# Initial run to cater for #} - {% if items_to_stage|length > 0 %} - {% do dbt_dataengineers_materializations.stage_file_format_plans(items_to_stage) %} + {% else %} + {% do log('file formats to create: Not enabled', info = true) %} {% endif %} - {% else %} - {% do log('file formats to create: Not enabled', info = true) %} {% endif %} {% endif %} - {% endmacro %} diff --git a/macros/source_tables/snowflake/snowflake__get_source_build_plan.sql b/macros/source_tables/snowflake/snowflake__get_source_build_plan.sql index c8b385b..f7d6dab 100644 --- a/macros/source_tables/snowflake/snowflake__get_source_build_plan.sql +++ b/macros/source_tables/snowflake/snowflake__get_source_build_plan.sql @@ -23,7 +23,6 @@ {%- set stream_name = dbt_dataengineers_materializations.snowflake_get_stream_name(identifier) -%} {%- set stream_relation = api.Relation.create(schema=schema, identifier=stream_name) -%} - {% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_schema(target_relation)) %} {# determine if we need to replace a view with this table #} {% if current_relation_exists_as_view %} {% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_view(current_relation)) %} @@ -46,6 +45,8 @@ {% if current_relation_exists_as_table %} {% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_stream_statement(stream_relation)) %} {% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_table(current_relation)) %} + {% else %} + {% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_schema(target_relation)) %} {% endif %} {% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_or_replace_table(target_relation, source_node)) %} {% elif auto_maintained %} diff --git a/macros/source_tables/stage_table_sources.sql b/macros/source_tables/stage_table_sources.sql index 0abe147..65275c2 100644 --- a/macros/source_tables/stage_table_sources.sql +++ b/macros/source_tables/stage_table_sources.sql @@ -1,46 +1,48 @@ -{% macro stage_table_sources(enabled_targets) %} - {% if target.name in enabled_targets %} - {% if flags.WHICH == 'run' %} - {% set sources_to_stage_auto_maintained = [] %} - {% set externals_tables_to_stage_auto_maintained = [] %} - {% set sources_to_stage_no_maintenance = [] %} - {% set externals_tables_to_stage_no_maintenance = [] %} +{% macro stage_table_sources(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %} + {% if target.profile_name in enabled_profiles %} + {% if target.name in enabled_targets %} + {% if flags.WHICH == 'run' %} + {% set sources_to_stage_auto_maintained = [] %} + {% set externals_tables_to_stage_auto_maintained = [] %} + {% set sources_to_stage_no_maintenance = [] %} + {% set externals_tables_to_stage_no_maintenance = [] %} - {% set source_nodes = graph.sources.values() if graph.sources else [] %} - {% for node in source_nodes %} - {% if node.external %} - {% if node.external.auto_create_table %} - {% if node.external.auto_maintained %} - {% if node.external.location %} - {% do externals_tables_to_stage_auto_maintained.append(node) %} + {% set source_nodes = graph.sources.values() if graph.sources else [] %} + {% for node in source_nodes %} + {% if node.external %} + {% if node.external.auto_create_table %} + {% if node.external.auto_maintained %} + {% if node.external.location %} + {% do externals_tables_to_stage_auto_maintained.append(node) %} + {% else %} + {% do sources_to_stage_auto_maintained.append(node) %} + {% endif %} {% else %} - {% do sources_to_stage_auto_maintained.append(node) %} - {% endif %} - {% else %} - {% if node.external.location %} - {% do externals_tables_to_stage_no_maintenance.append(node) %} - {% else %} - {% do sources_to_stage_no_maintenance.append(node) %} + {% if node.external.location %} + {% do externals_tables_to_stage_no_maintenance.append(node) %} + {% else %} + {% do sources_to_stage_no_maintenance.append(node) %} + {% endif %} {% endif %} {% endif %} {% endif %} - {% endif %} - {% endfor %} - {# Initial run to cater for #} - {% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %} - {% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %} - {% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %} - {% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%} - {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %} + {% endfor %} + {# Initial run to cater for #} + {% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %} + {% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %} + {% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %} + {% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%} + {% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %} + {% endif %} + {% else %} + {% do log('tables creation not enabled for ' ~ target.name, info = true) %} {% endif %} - {% else %} - {% do log('tables creation not enabled for ' ~ target.name, info = true) %} {% endif %} {% endmacro %} diff --git a/macros/stages/stage_stages.sql b/macros/stages/stage_stages.sql index e995503..986ee62 100644 --- a/macros/stages/stage_stages.sql +++ b/macros/stages/stage_stages.sql @@ -1,21 +1,23 @@ -{% macro stage_stages(enabled_targets) %} - {% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %} - {% if target.name in enabled_targets %} - {% set stages_to_stage = [] %} +{% macro stage_stages(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %} + {% if target.profile_name in enabled_profiles %} + {% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %} + {% if target.name in enabled_targets %} + {% set stages_to_stage = [] %} - {% set nodes = graph.nodes.values() if graph.nodes else [] %} - {% for node in nodes %} - {% if node.config.materialized == 'stage' %} - {% do stages_to_stage.append(node) %} - {% endif %} - {% endfor %} + {% set nodes = graph.nodes.values() if graph.nodes else [] %} + {% for node in nodes %} + {% if node.config.materialized == 'stage' %} + {% do stages_to_stage.append(node) %} + {% endif %} + {% endfor %} - {% do log('stages to create: ' ~ stages_to_stage|length, info = true) %} + {% do log('stages to create: ' ~ stages_to_stage|length, info = true) %} - {# Initial run to cater for #} - {% do dbt_dataengineers_materializations.stage_stages_plans(stages_to_stage) %} - {% else %} - {% do log('stages to create: Not enabled', info = true) %} + {# Initial run to cater for #} + {% do dbt_dataengineers_materializations.stage_stages_plans(stages_to_stage) %} + {% else %} + {% do log('stages to create: Not enabled', info = true) %} + {% endif %} {% endif %} {% endif %} {% endmacro %}