Skip to content

Commit

Permalink
updated when messages are displayed based on execute and run
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonhopper-dataengineers
jonhopper-dataengineers committed Dec 30, 2024
1 parent fec1b24 commit a78527e
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 38 deletions.
74 changes: 38 additions & 36 deletions macros/source_tables/stage_table_sources.sql
View file
Original file line number Diff line number Diff line change
@@ -1,47 +1,49 @@

{% macro stage_table_sources(enabled_targets=[target.name], enabled_profiles=[target.profile_name]) %}
{% if target.profile_name in enabled_profiles %}
{% if target.name in enabled_targets %}
{% if flags.WHICH == 'run' %}
{% set sources_to_stage_auto_maintained = [] %}
{% set externals_tables_to_stage_auto_maintained = [] %}
{% set sources_to_stage_no_maintenance = [] %}
{% set externals_tables_to_stage_no_maintenance = [] %}
{% if execute %}
{% if flags.WHICH == 'run' %}
{% if target.profile_name in enabled_profiles %}
{% if target.name in enabled_targets %}
{% set sources_to_stage_auto_maintained = [] %}
{% set externals_tables_to_stage_auto_maintained = [] %}
{% set sources_to_stage_no_maintenance = [] %}
{% set externals_tables_to_stage_no_maintenance = [] %}

{% set source_nodes = graph.sources.values() if graph.sources else [] %}
{% for node in source_nodes %}
{% if node.external %}
{% if node.external.auto_create_table %}
{% if node.external.auto_maintained %}
{% if node.external.location %}
{% do externals_tables_to_stage_auto_maintained.append(node) %}
{% set source_nodes = graph.sources.values() if graph.sources else [] %}
{% for node in source_nodes %}
{% if node.external %}
{% if node.external.auto_create_table %}
{% if node.external.auto_maintained %}
{% if node.external.location %}
{% do externals_tables_to_stage_auto_maintained.append(node) %}
{% else %}
{% do sources_to_stage_auto_maintained.append(node) %}
{% endif %}
{% else %}
{% do sources_to_stage_auto_maintained.append(node) %}
{% endif %}
{% else %}
{% if node.external.location %}
{% do externals_tables_to_stage_no_maintenance.append(node) %}
{% else %}
{% do sources_to_stage_no_maintenance.append(node) %}
{% if node.external.location %}
{% do externals_tables_to_stage_no_maintenance.append(node) %}
{% else %}
{% do sources_to_stage_no_maintenance.append(node) %}
{% endif %}
{% endif %}
{% endif %}
{% endif %}
{% endif %}
{% endfor %}
{# Initial run to cater for #}
{% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %}
{% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %}
{% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %}
{% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %}
{% endfor %}
{# Initial run to cater for #}
{% do log('===> ' ~ sources_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, true, 'internal', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_auto_maintained, false, 'internal', true) %}
{% do log('===> ' ~ sources_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(sources_to_stage_no_maintenance, true, 'internal', false) %}
{% do log('===> ' ~ externals_tables_to_stage_auto_maintained|length ~ ' Tables to be maintained by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, true, 'external', true) %}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_auto_maintained, false, 'external', true) %}
{% do log('===> ' ~ externals_tables_to_stage_no_maintenance|length ~ ' Tables only to be created by dbt (with external source) <===', info = true) -%}
{% do dbt_dataengineers_materializations.stage_table_sources_plans(externals_tables_to_stage_no_maintenance, true, 'external', false) %}
{% else %}
{% do log('tables creation not enabled for ' ~ target.name, info = true) %}
{% endif %}
{% endif %}
{% else %}
{% do log('tables creation not enabled for ' ~ target.name, info = true) %}
{% endif %}
{% endif %}
{% endmacro %}
Expand Down
4 changes: 2 additions & 2 deletions macros/tasks/enable_task_dependants.sql
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{% macro enable_task_dependants(root_task, enabled_targets) %}
{{ log("START - Enabling task and dependant tasks for " ~ root_task , info=True) }}
{%- if execute -%}
{% if target.name in enabled_targets -%}
{% if flags.WHICH == 'run' %}
{{ log("START - Enabling task and dependant tasks for " ~ root_task , info=True) }}
{% set nodes = graph.nodes.values() if graph.nodes else [] %}
{% set matching_nodes = nodes
| selectattr("name", "equalto", root_task | lower)
Expand All @@ -15,8 +15,8 @@
{% do dbt_dataengineers_materializations.snowflake_resume_task_with_dependants_statement(task_relation) %}
{% endcall %}
{% endfor %}
{{ log("END - Enabling task and dependant tasks for " ~ root_task , info=True) }}
{% endif %}
{% endif %}
{% endif %}
{{ log("END - Enabling task and dependant tasks for " ~ root_task , info=True) }}
{% endmacro %}

0 comments on commit a78527e

Please sign in to comment.