Use an Artifact ID rather than a composite OR statment in several pla…

…ces. (#81) * Use artifact_run_id instead of an OR statement
brooklyn-data · Feb 2, 2022 · f124f53 · f124f53
1 parent bad83e8
commit f124f53
Show file tree

Hide file tree

Showing 27 changed files with 51 additions and 32 deletions.
diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql
@@ -22,9 +22,7 @@ model_executions as (
 -- Get the most recent comile run
 latest_compile as (
 
-    select
-        command_invocation_id,
-        dbt_cloud_run_id
+    select artifact_run_id
     from run_results
     where execution_command = 'run'
     order by artifact_generated_at desc
@@ -39,8 +37,7 @@ latest_models as (
     from models
     -- In a local deploy, the command id is sufficient, but not in cloud - that requires the cloud run id to achieve a match.
     inner join latest_compile
-        on models.command_invocation_id = latest_compile.command_invocation_id
-            or models.dbt_cloud_run_id = latest_compile.dbt_cloud_run_id
+        on models.artifact_run_id = latest_compile.artifact_run_id
 
 ),
 

diff --git a/models/fct_dbt__critical_path.sql b/models/fct_dbt__critical_path.sql
@@ -15,9 +15,7 @@ latest_executions as (
 latest_id as (
     -- Find the latest full, incremental execution
 
-    select
-        any_value(command_invocation_id) as command_invocation_id,
-        any_value(dbt_cloud_run_id) as dbt_cloud_run_id
+    select any_value(artifact_run_id) as artifact_run_id
     from latest_executions
 
 ),
@@ -31,8 +29,7 @@ latest_models as (
         models.model_materialization
     from latest_id
     left join models on
-        latest_id.command_invocation_id = models.command_invocation_id
-        or latest_id.dbt_cloud_run_id = models.dbt_cloud_run_id
+        latest_id.artifact_run_id = models.artifact_run_id
 
 
 ),

diff --git a/models/fct_dbt__latest_full_model_executions.sql b/models/fct_dbt__latest_full_model_executions.sql
@@ -28,8 +28,7 @@ joined as (
         model_executions.*
     from latest_full
     left join model_executions on
-        model_executions.command_invocation_id = latest_full.command_invocation_id
-        or model_executions.dbt_cloud_run_id = latest_full.dbt_cloud_run_id
+        model_executions.artifact_run_id = latest_full.artifact_run_id
 
 ),
 
@@ -39,6 +38,7 @@ fields as (
         artifact_generated_at,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         compile_started_at,
         query_completed_at,
         total_node_runtime,

diff --git a/models/incremental/dim_dbt__exposures.sql b/models/incremental/dim_dbt__exposures.sql
@@ -25,8 +25,7 @@ dbt_exposures_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_exposures.command_invocation_id = run_results.command_invocation_id
-        or dbt_exposures.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_exposures.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -41,6 +40,7 @@ fields as (
         t.manifest_exposure_id,
         t.command_invocation_id,
         t.dbt_cloud_run_id,
+        t.artifact_run_id,
         t.artifact_generated_at,
         t.node_id,
         t.name,

diff --git a/models/incremental/dim_dbt__models.sql b/models/incremental/dim_dbt__models.sql
@@ -20,8 +20,7 @@ dbt_models_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_models.command_invocation_id = run_results.command_invocation_id
-        or dbt_models.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_models.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -36,6 +35,7 @@ fields as (
         manifest_model_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         model_database,

diff --git a/models/incremental/dim_dbt__seeds.sql b/models/incremental/dim_dbt__seeds.sql
@@ -20,8 +20,7 @@ dbt_seeds_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_seeds.command_invocation_id = run_results.command_invocation_id
-        or dbt_seeds.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_seeds.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -36,6 +35,7 @@ fields as (
         manifest_seed_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         seed_database,

diff --git a/models/incremental/dim_dbt__snapshots.sql b/models/incremental/dim_dbt__snapshots.sql
@@ -20,8 +20,7 @@ dbt_snapshots_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_snapshots.command_invocation_id = run_results.command_invocation_id
-        or dbt_snapshots.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_snapshots.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -36,6 +35,7 @@ fields as (
         manifest_snapshot_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         snapshot_database,

diff --git a/models/incremental/dim_dbt__sources.sql b/models/incremental/dim_dbt__sources.sql
@@ -20,8 +20,7 @@ dbt_sources_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_sources.command_invocation_id = run_results.command_invocation_id
-        or dbt_sources.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_sources.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -36,6 +35,7 @@ fields as (
         manifest_source_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         name,

diff --git a/models/incremental/dim_dbt__tests.sql b/models/incremental/dim_dbt__tests.sql
@@ -20,8 +20,7 @@ dbt_tests_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        dbt_tests.command_invocation_id = run_results.command_invocation_id
-        or dbt_tests.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        dbt_tests.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -36,6 +35,7 @@ fields as (
         manifest_test_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         name,

diff --git a/models/incremental/fct_dbt__model_executions.sql b/models/incremental/fct_dbt__model_executions.sql
@@ -28,8 +28,7 @@ model_executions_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        model_executions.command_invocation_id = run_results.command_invocation_id
-        or model_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        model_executions.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -61,6 +60,7 @@ fields as (
         model_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/incremental/fct_dbt__run_results.sql b/models/incremental/fct_dbt__run_results.sql
@@ -27,6 +27,7 @@ fields as (
         artifact_generated_at,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         dbt_version,
         elapsed_time,
         execution_command,

diff --git a/models/incremental/fct_dbt__seed_executions.sql b/models/incremental/fct_dbt__seed_executions.sql
@@ -28,8 +28,7 @@ seed_executions_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        seed_executions.command_invocation_id = run_results.command_invocation_id
-        or seed_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        seed_executions.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -60,6 +59,7 @@ fields as (
         seed_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/incremental/fct_dbt__snapshot_executions.sql b/models/incremental/fct_dbt__snapshot_executions.sql
@@ -28,8 +28,7 @@ snapshot_executions_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        snapshot_executions.command_invocation_id = run_results.command_invocation_id
-        or snapshot_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        snapshot_executions.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -60,6 +59,7 @@ fields as (
         snapshot_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/incremental/fct_dbt__test_executions.sql b/models/incremental/fct_dbt__test_executions.sql
@@ -21,8 +21,7 @@ test_executions_incremental as (
     -- Inner join with run results to enforce consistency and avoid race conditions.
     -- https://github.com/brooklyn-data/dbt_artifacts/issues/75
     inner join run_results on
-        test_executions.command_invocation_id = run_results.command_invocation_id
-        or test_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id
+        test_executions.artifact_run_id = run_results.artifact_run_id
 
     {% if is_incremental() %}
         -- this filter will only be applied on an incremental run
@@ -37,6 +36,7 @@ fields as (
         test_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/incremental/int_dbt__model_executions.sql b/models/incremental/int_dbt__model_executions.sql
@@ -26,6 +26,7 @@ fields as (
         model_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/staging/stg_dbt__artifacts.sql b/models/staging/stg_dbt__artifacts.sql
@@ -36,6 +36,8 @@ artifacts as (
     select
         command_invocation_id,
         dbt_cloud_run_id,
+        -- This ID provides a reliable ID, regardless of whether running in a local or cloud environment.
+        sha2_hex(coalesce(dbt_cloud_run_id::string, command_invocation_id::string), 256) as artifact_run_id,
         generated_at,
         path,
         artifact_type,

diff --git a/models/staging/stg_dbt__exposures.sql b/models/staging/stg_dbt__exposures.sql
@@ -18,6 +18,7 @@ flatten as (
     select
         manifests.command_invocation_id,
         manifests.dbt_cloud_run_id,
+        manifests.artifact_run_id,
         manifests.generated_at as artifact_generated_at,
         node.key as node_id,
         node.value:name::string as name,
@@ -38,6 +39,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_exposure_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         name,

diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql
@@ -26,6 +26,7 @@ fields as (
     select
         dbt_run.command_invocation_id,
         dbt_run.dbt_cloud_run_id,
+        dbt_run.artifact_run_id,
         dbt_run.generated_at as artifact_generated_at,
         coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh,
         result.value:unique_id::string as node_id,
@@ -54,6 +55,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql
@@ -18,6 +18,7 @@ flatten as (
     select
         manifests.command_invocation_id,
         manifests.dbt_cloud_run_id,
+        manifests.artifact_run_id,
         manifests.generated_at as artifact_generated_at,
         node.key as node_id,
         node.value:database::string as model_database,
@@ -40,6 +41,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_model_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         model_database,

diff --git a/models/staging/stg_dbt__run_results.sql b/models/staging/stg_dbt__run_results.sql
@@ -27,6 +27,7 @@ fields as (
         generated_at as artifact_generated_at,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         data:metadata:dbt_version::string as dbt_version,
         data:metadata:env as env,
         data:elapsed_time::float as elapsed_time,

diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql
@@ -26,6 +26,7 @@ fields as (
     select
         dbt_run.command_invocation_id,
         dbt_run.dbt_cloud_run_id,
+        dbt_run.artifact_run_id,
         dbt_run.generated_at as artifact_generated_at,
         coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh,
         result.value:unique_id::string as node_id,
@@ -55,6 +56,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql
@@ -18,6 +18,7 @@ flatten as (
     select
         manifests.command_invocation_id,
         manifests.dbt_cloud_run_id,
+        manifests.artifact_run_id,
         manifests.generated_at as artifact_generated_at,
         node.key as node_id,
         node.value:database::string as seed_database,
@@ -39,6 +40,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_seed_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         seed_database,

diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql
@@ -26,6 +26,7 @@ fields as (
     select
         dbt_run.command_invocation_id,
         dbt_run.dbt_cloud_run_id,
+        dbt_run.artifact_run_id,
         dbt_run.generated_at as artifact_generated_at,
         coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh,
         result.value:unique_id::string as node_id,
@@ -55,6 +56,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         was_full_refresh,
         node_id,

diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql
@@ -18,6 +18,7 @@ flatten as (
     select
         manifests.command_invocation_id,
         manifests.dbt_cloud_run_id,
+        manifests.artifact_run_id,
         manifests.generated_at as artifact_generated_at,
         node.key as node_id,
         node.value:database::string as snapshot_database,
@@ -39,6 +40,7 @@ surrogate_key as (
         {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_snapshot_id,
         command_invocation_id,
         dbt_cloud_run_id,
+        artifact_run_id,
         artifact_generated_at,
         node_id,
         snapshot_database,