Skip to content

Commit

Permalink
Added in the ability to set the profile targets for the materializati…
Browse files Browse the repository at this point in the history
…ons `stages`, `file_format`, `tables`
  • Loading branch information
jonhopper-dataengineers committed Jul 21, 2024
1 parent bad1b33 commit 803397b
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 71 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# dbt_dataengineers_materializations Changelog

## 0.2.8.1 - Profile Targets

* Added in the ability to set the profile targets for the materializations `stages`, `file_format`, `tables`
- tables

## 0.2.8 - Secrets, Network Rules & UDF's

Addition of the materializations:
Expand Down
39 changes: 20 additions & 19 deletions macros/file_formats/stage_file_formats.sql
Original file line number Diff line number Diff line change
@@ -1,26 +1,27 @@
{% macro stage_file_formats(enabled_targets) %}
{% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %}
{% if target.name in enabled_targets %}
{% set items_to_stage = [] %}

{% set nodes = graph.nodes.values() if graph.nodes else [] %}
{% for node in nodes %}
{% if node.config.materialized == 'file_format' %}
{% do items_to_stage.append(node) %}
{% macro stage_file_formats(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %}
{% if target.profile_name in enabled_profiles %}
{% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %}
{% if target.name in enabled_targets %}
{% set items_to_stage = [] %}

{% set nodes = graph.nodes.values() if graph.nodes else [] %}
{% for node in nodes %}
{% if node.config.materialized == 'file_format' %}
{% do items_to_stage.append(node) %}
{% endif %}
{% endfor %}

{% do log('file formats to create: ' ~ items_to_stage|length, info = true) %}

{# Initial run to cater for #}
{% if items_to_stage|length > 0 %}
{% do dbt_dataengineers_materializations.stage_file_format_plans(items_to_stage) %}
{% endif %}
{% endfor %}

{% do log('file formats to create: ' ~ items_to_stage|length, info = true) %}

{# Initial run to cater for #}
{% if items_to_stage|length > 0 %}
{% do dbt_dataengineers_materializations.stage_file_format_plans(items_to_stage) %}
{% else %}
{% do log('file formats to create: Not enabled', info = true) %}
{% endif %}
{% else %}
{% do log('file formats to create: Not enabled', info = true) %}
{% endif %}
{% endif %}

{% endmacro %}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
{%- set stream_name = dbt_dataengineers_materializations.snowflake_get_stream_name(identifier) -%}
{%- set stream_relation = api.Relation.create(schema=schema, identifier=stream_name) -%}

{% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_schema(target_relation)) %}
{# determine if we need to replace a view with this table #}
{% if current_relation_exists_as_view %}
{% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_view(current_relation)) %}
Expand All @@ -46,6 +45,8 @@
{% if current_relation_exists_as_table %}
{% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_stream_statement(stream_relation)) %}
{% do build_plan.append(dbt_dataengineers_materializations.snowflake_drop_table(current_relation)) %}
{% else %}
{% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_schema(target_relation)) %}
{% endif %}
{% do build_plan.append(dbt_dataengineers_materializations.snowflake_create_or_replace_table(target_relation, source_node)) %}
{% elif auto_maintained %}
Expand Down
74 changes: 38 additions & 36 deletions macros/source_tables/stage_table_sources.sql
Original file line number Diff line number Diff line change
@@ -1,46 +1,48 @@

{% macro stage_table_sources(enabled_targets) %}
{% if target.name in enabled_targets %}
{% if flags.WHICH == 'run' %}
{% set sources_to_stage_auto_maintained = [] %}
{% set externals_tables_to_stage_auto_maintained = [] %}
{% set sources_to_stage_no_maintenance = [] %}
{% set externals_tables_to_stage_no_maintenance = [] %}
{% macro stage_table_sources(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %}
{% if target.profile_name in enabled_profiles %}
{% if target.name in enabled_targets %}
{% if flags.WHICH == 'run' %}
{% set sources_to_stage_auto_maintained = [] %}
{% set externals_tables_to_stage_auto_maintained = [] %}
{% set sources_to_stage_no_maintenance = [] %}
{% set externals_tables_to_stage_no_maintenance = [] %}

{% set source_nodes = graph.sources.values() if graph.sources else [] %}
{% for node in source_nodes %}
{% if node.external %}
{% if node.external.auto_create_table %}
{% if node.external.auto_maintained %}
{% if node.external.location %}
{% do externals_tables_to_stage_auto_maintained.append(node) %}
{% set source_nodes = graph.sources.values() if graph.sources else [] %}
{% for node in source_nodes %}
{% if node.external %}
{% if node.external.auto_create_table %}
{% if node.external.auto_maintained %}
{% if node.external.location %}
{% do externals_tables_to_stage_auto_maintained.append(node) %}
{% else %}
{% do sources_to_stage_auto_maintained.append(node) %}
{% endif %}
{% else %}
{% do sources_to_stage_auto_maintained.append(node) %}
{% endif %}
{% else %}
{% if node.external.location %}
{% do externals_tables_to_stage_no_maintenance.append(node) %}
{% else %}
{% do sources_to_stage_no_maintenance.append(node) %}
{% if node.external.location %}
{% do externals_tables_to_stage_no_maintenance.append(node) %}
{% else %}
{% do sources_to_stage_no_maintenance.append(node) %}
{% endif %}
{% endif %}
{% endif %}
{% endif %}
{% endif %}
{% endfor %}
{# Initial run to cater for #}
{% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %}
{% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %}
{% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %}
{% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %}
{% endfor %}
{# Initial run to cater for #}
{% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %}
{% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %}
{% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %}
{% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %}
{% endif %}
{% else %}
{% do log('tables creation not enabled for ' ~ target.name, info = true) %}
{% endif %}
{% else %}
{% do log('tables creation not enabled for ' ~ target.name, info = true) %}
{% endif %}
{% endmacro %}

Expand Down
32 changes: 17 additions & 15 deletions macros/stages/stage_stages.sql
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
{% macro stage_stages(enabled_targets) %}
{% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %}
{% if target.name in enabled_targets %}
{% set stages_to_stage = [] %}
{% macro stage_stages(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %}
{% if target.profile_name in enabled_profiles %}
{% if flags.WHICH == 'run' or flags.WHICH == 'run-operation' %}
{% if target.name in enabled_targets %}
{% set stages_to_stage = [] %}

{% set nodes = graph.nodes.values() if graph.nodes else [] %}
{% for node in nodes %}
{% if node.config.materialized == 'stage' %}
{% do stages_to_stage.append(node) %}
{% endif %}
{% endfor %}
{% set nodes = graph.nodes.values() if graph.nodes else [] %}
{% for node in nodes %}
{% if node.config.materialized == 'stage' %}
{% do stages_to_stage.append(node) %}
{% endif %}
{% endfor %}

{% do log('stages to create: ' ~ stages_to_stage|length, info = true) %}
{% do log('stages to create: ' ~ stages_to_stage|length, info = true) %}

{# Initial run to cater for #}
{% do dbt_dataengineers_materializations.stage_stages_plans(stages_to_stage) %}
{% else %}
{% do log('stages to create: Not enabled', info = true) %}
{# Initial run to cater for #}
{% do dbt_dataengineers_materializations.stage_stages_plans(stages_to_stage) %}
{% else %}
{% do log('stages to create: Not enabled', info = true) %}
{% endif %}
{% endif %}
{% endif %}
{% endmacro %}
Expand Down

0 comments on commit 803397b

Please sign in to comment.