Merge pull request #109 from Opetushallitus/OK-602__pub-kerroksen-kor…

…jauksia Ok 602 pub kerroksen korjauksia
Opetushallitus · Feb 5, 2025 · 0ad6996 · 0ad6996
2 parents 7656d0a + 88677b7
commit 0ad6996
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 7 deletions.
diff --git a/dbt-container/run.sh b/dbt-container/run.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set +e
 
 echo "Running Ovara DBT script..."
 
@@ -22,13 +22,23 @@ aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET on
 dbt seed -s raw_taulut --target=prod
 dbt run-operation create_raw_tables --target=prod
 
+is_error="0"
+
 if [[ -z "$1" ]]; then
   echo "Running DBT without any extra paramaters"
-  dbt build --target=prod
+  if dbt build --target=prod ; then
+  	is_error="0"
+  else
+    is_error="1"
+  fi
   echo "Finished running DBT"
 else
   echo "Running DBT with extra paramaters: $1"
-  dbt build --target=prod "$1"
+  if dbt build --target=prod "$1"; then
+  	is_error="0"
+  else
+    is_error="1"
+  fi
   echo "Finished running DBT"
 fi
 
@@ -54,4 +64,9 @@ aws s3 cp ./logs s3://$DBT_LOGS_BUCKET/$CURRENT_TIME --recursive --include 'logs
 echo "Merkitään DynamoDB:hen että prosessi ei ole enää ajossa"
 aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET onKaynnissa='false' WHERE prosessi='dbt-scheduled-task' RETURNING ALL NEW *"
 
+if [ $is_error -eq "1" ]; then
+	echo "Error: Ajossa tapahtui virhe"
+	exit 1
+fi
+
 exit 0
diff --git a/dbt/models/dw/yleiskayttoiset/dw_onr_henkilo.sql b/dbt/models/dw/yleiskayttoiset/dw_onr_henkilo.sql
@@ -1,4 +1,35 @@
-{%- set stage_model = ref('stg_onr_henkilo') -%}
-{%- set key_columns_list = ['henkilo_oid'] -%}
+{{
+  config(
+    materialized = 'incremental',
+    incremental_strategy = 'merge',
+    unique_key = 'henkilo_oid',
+    merge_exclude_columns = [
+            'dw_metadata_dbt_copied_at',
+            'dw_metadata_file_row_number',
+            'dw_metadata_filename',
+            'dw_metadata_source_timestamp_at',
+            'dw_metadata_dw_stored_at'
+            ],
+    indexes = [
+        {'columns': ['henkilo_oid']},
+        {'columns': ['muokattu']}
+    ]
+    )
+}}
 
-{{ generate_dw_model_muokattu(stage_model, key_columns_list) }}
+with raw as (
+    select * from {{ ref('stg_onr_henkilo') }}
+    {% if is_incremental() %}
+      where muokattu > (select coalesce(max(t.muokattu),date('1900-01-01'))  from {{this}} as t )
+    {% endif %}
+),
+
+final as (
+    select
+        *,
+        current_timestamp as dw_metadata_dw_stored_at
+    from raw
+)
+
+
+select * from final
diff --git a/dbt/models/stg/yleiskayttoiset/stg_onr_henkilo.sql b/dbt/models/stg/yleiskayttoiset/stg_onr_henkilo.sql
@@ -2,7 +2,8 @@
   config(
     materialized = 'table',
     indexes = [
-        {'columns': ['henkilo_oid']}
+        {'columns': ['henkilo_oid']},
+        {'columns': ['muokattu']}
     ]
     )
 }}
@@ -36,6 +37,8 @@ final as (
         (data ->> 'sukupuoli')::int as sukupuoli,
         (data ->> 'turvakielto')::boolean = 't' as turvakielto,
         (data ->> 'yksiloityvtj')::boolean = 't' as yksiloityvtj,
+        (data ->> 'created')::timestamptz as luotu,
+        (data ->> 'updated')::timestamptz as muokattu,
         {{ metadata_columns() }}
     from source
 )