diff --git a/docs/guide/advanced/custom_libs.mdx b/docs/guide/advanced/custom_libs.mdx index c7dafe25..bc199a46 100644 --- a/docs/guide/advanced/custom_libs.mdx +++ b/docs/guide/advanced/custom_libs.mdx @@ -215,12 +215,12 @@ The following guide will explain how to do so, step by step. ``` We can look at the compiled tests query located in the file - `sdftarget/dbg/preprocessed/sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql`. + `sdftarget/dbg/preprocessed/tests_workspace/pub/test_raw_inapp_events.sql`. Notice our custom logic in the compiled query: ``` run shell - cd tmp/tests && cat sdftarget/dbg/preprocessed/sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql + cd tmp/tests && cat sdftarget/dbg/preprocessed/tests_workspace/pub/test_raw_inapp_events.sql ``` diff --git a/docs/guide/advanced/logging.mdx b/docs/guide/advanced/logging.mdx index d9e8eb1a..cbf4824d 100644 --- a/docs/guide/advanced/logging.mdx +++ b/docs/guide/advanced/logging.mdx @@ -32,16 +32,16 @@ The following guide will go through the different logging options available in S 
 
-{"_ts":"2024-11-22T20:47:24.392467943+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:105","ev":"sdf"}
-{"_ts":"2024-11-22T20:47:24.392507897+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:106","ev":"sdf","set":"job_id","val_s":"cec26154-10f1-4e7b-a673-1f82a8c870e4"}
-{"_ts":"2024-11-22T20:47:24.392514991+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:107","ev":"sdf","set":"args","val_as":["compile","moms_flower_shop.raw.raw_inapp_events","--log-level","info","--show","none"]}
-{"_ts":"2024-11-22T20:47:24.393764443+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:117","ev":"initialize"}
-{"_ts":"2024-11-22T20:47:24.393922949+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:301","ev":"initialize.load_workspace"}
-{"_ts":"2024-11-22T20:47:24.411549278+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:381","ev":"initialize.load_workspace","st_dur_ms":17}
-{"_ts":"2024-11-22T20:47:24.411563645+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:383","ev":"initialize.load_dependencies"}
-{"_ts":"2024-11-22T20:47:24.411576088+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:515","ev":"initialize.load_dependencies","st_dur_ms":0}
-{"_ts":"2024-11-22T20:47:24.411585346+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:517","ev":"initialize.load_prelude"}
-{"_ts":"2024-11-22T20:47:24.414352270+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:553","ev":"initialize.load_prelude","st_dur_ms":2}
+{"_ts":"2024-11-25T19:27:41.544039825+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:105","ev":"sdf"}
+{"_ts":"2024-11-25T19:27:41.544101961+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:106","ev":"sdf","set":"job_id","val_s":"1de199b0-e6cf-4b5d-b411-e8da7b471841"}
+{"_ts":"2024-11-25T19:27:41.544109706+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/main.rs:107","ev":"sdf","set":"args","val_as":["compile","moms_flower_shop.raw.raw_inapp_events","--log-level","info","--show","none"]}
+{"_ts":"2024-11-25T19:27:41.545433248+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:119","ev":"initialize"}
+{"_ts":"2024-11-25T19:27:41.545581124+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:298","ev":"initialize.load_workspace"}
+{"_ts":"2024-11-25T19:27:41.562529100+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:367","ev":"initialize.load_workspace","st_dur_ms":16}
+{"_ts":"2024-11-25T19:27:41.562542926+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:369","ev":"initialize.load_dependencies"}
+{"_ts":"2024-11-25T19:27:41.562555489+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:500","ev":"initialize.load_dependencies","st_dur_ms":0}
+{"_ts":"2024-11-25T19:27:41.562564597+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:502","ev":"initialize.load_prelude"}
+{"_ts":"2024-11-25T19:27:41.565143694+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:538","ev":"initialize.load_prelude","st_dur_ms":2}
@@ -63,38 +63,58 @@ The following guide will go through the different logging options available in S 
 
-{"_ts":"2024-11-22T20:47:25.914801726+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:521","ev":"cmd.prepare.read_makefile","set":"verdict","val_s":"full-rebuild"}
-{"_ts":"2024-11-22T20:47:25.914808759+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:574","ev":"cmd.prepare.read_makefile","set":"analyze_with_sources","val_b":false}
-{"_ts":"2024-11-22T20:47:25.914815973+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:579","ev":"cmd.prepare.read_makefile","message":"Exceeds limit; use log-level trace","set":"resolved_table_names"}
-{"_ts":"2024-11-22T20:47:25.914824769+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:585","ev":"cmd.prepare.read_makefile","message":"Exceeds limit; use log-level trace","set":"seen_these_files"}
-{"_ts":"2024-11-22T20:47:25.914829979+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:591","ev":"cmd.prepare.read_makefile","message":"Exceeds limit; use log-level trace","set":"sources"}
-{"_ts":"2024-11-22T20:47:25.914835058+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:601","ev":"cmd.prepare.read_makefile","st_dur_ms":0}
-
-{"_ts":"2024-11-22T20:47:25.922594312+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:1896","ev":"cmd.schedule.dag"}
-{"_ts":"2024-11-22T20:47:25.922607596+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:1906","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"dag.initial"}
-{"_ts":"2024-11-22T20:47:25.922615752+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:1918","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"dag.cleaned"}
-{"_ts":"2024-11-22T20:47:25.922624368+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:1924","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"missing_defs"}
-{"_ts":"2024-11-22T20:47:25.922641440+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2002","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"dag.resolved_targets"}
-{"_ts":"2024-11-22T20:47:25.922653753+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2031","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"dag.targeted"}
-{"_ts":"2024-11-22T20:47:25.922684660+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2072","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"dag.acyclic"}
-{"_ts":"2024-11-22T20:47:25.922694649+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2078","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"cycles"}
-{"_ts":"2024-11-22T20:47:25.922700460+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2084","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"cut_points"}
-{"_ts":"2024-11-22T20:47:25.922709216+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2090","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"order.initial"}
-{"_ts":"2024-11-22T20:47:25.922745865+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2590","ev":"cmd.schedule.dag","message":"Exceeds limit; use log-level trace","set":"order.without_providers"}
-{"_ts":"2024-11-22T20:47:25.922753849+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2616","ev":"cmd.schedule.dag","st_dur_ms":0}
-
-{"_ts":"2024-11-22T20:47:25.922767144+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2623","ev":"cmd.do"}
-{"_ts":"2024-11-22T20:47:25.922774668+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:2649","ev":"cmd.do","message":"Exceeds limit; use log-level trace","set":"order.without_providers_or_cached"}
-{"_ts":"2024-11-22T20:47:25.922785749+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2666","ev":"cmd.do.sources"}
-
-{"_ts":"2024-11-22T20:47:25.923464474+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:3926","ev":"cmd.do.derived.wave_1.mat","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events"}
-{"_ts":"2024-11-22T20:47:25.924190229+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/describe.rs:255","ev":"cmd.do.derived.wave_1.meta_analysis","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events"}
-{"_ts":"2024-11-22T20:47:25.924238689+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/describe.rs:296","ev":"cmd.do.derived.wave_1.meta_analysis","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events","st_dur_ms":0}
-{"_ts":"2024-11-22T20:47:25.924266170+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:4054","ev":"cmd.do.derived.wave_1.mat","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_purpose":"model","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events","st_code":"succeeded","st_done":"executed","st_dur_ms":0}
-
-{"_ts":"2024-11-22T20:47:25.924947752+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:3202","ev":"cmd.do.gen_makefile"}
-{"_ts":"2024-11-22T20:47:25.925160560+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/utils/dep_cache.rs:2277","ev":"written_makefile","message":"Exceeds limit; use log-level trace","set":"makefile"}
-{"_ts":"2024-11-22T20:47:25.925259967+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:3216","ev":"cmd.do.gen_makefile","st_dur_ms":0}
+{"_ts":"2024-11-25T19:27:43.315973549+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:577","ev":"initialize.prep_env.resolve_env.build_env","st_dur_ms":554}
+{"_ts":"2024-11-25T19:27:43.316038690+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_test at sdftarget/dbg/.lib/sdf_test/macros/test.jinja"}
+{"_ts":"2024-11-25T19:27:43.316313443+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_utils at sdftarget/dbg/.lib/sdf_utils/macros/date_spine.jinja"}
+{"_ts":"2024-11-25T19:27:43.316389876+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_utils at sdftarget/dbg/.lib/sdf_utils/macros/generate_integer_values.jinja"}
+{"_ts":"2024-11-25T19:27:43.316429730+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_utils at sdftarget/dbg/.lib/sdf_utils/macros/generate_surrogate_key.jinja"}
+{"_ts":"2024-11-25T19:27:43.316468202+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_utils at sdftarget/dbg/.lib/sdf_utils/macros/group_by.jinja"}
+{"_ts":"2024-11-25T19:27:43.316505101+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_utils at sdftarget/dbg/.lib/sdf_utils/macros/generate_date_values.jinja"}
+{"_ts":"2024-11-25T19:27:43.316541610+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/context/env.rs:907","message":"Reading jinja file: sdf_mat at sdftarget/dbg/.lib/sdf_mat/macros/dialect/trino/materialize.jinja"}
+{"_ts":"2024-11-25T19:27:43.316909266+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/utils/initializing.rs:776","ev":"initialize.prep_env.resolve_env","st_dur_ms":555}
+
+{"_ts":"2024-11-25T19:27:43.317546747+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:518","ev":"cmd.prepare.read_makefile","set":"verdict","val_s":"full-rebuild"}
+{"_ts":"2024-11-25T19:27:43.317561134+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:574","ev":"cmd.prepare.read_makefile","set":"analyze_with_sources","val_b":false}
+{"_ts":"2024-11-25T19:27:43.317571884+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:579","ev":"cmd.prepare.read_makefile","set":"resolved_table_names","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.317578717+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:585","ev":"cmd.prepare.read_makefile","set":"seen_these_files","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.317584838+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:591","ev":"cmd.prepare.read_makefile","set":"sources","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.317590519+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:602","ev":"cmd.prepare.read_makefile","st_dur_ms":0}
+
+{"_ts":"2024-11-25T19:27:43.324007415+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:1605","ev":"cmd.schedule.prep"}
+{"_ts":"2024-11-25T19:27:43.324037220+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:1627","ev":"cmd.schedule","set":"deps","val_as":{"moms_flower_shop.raw.raw_inapp_events":[]}}
+{"_ts":"2024-11-25T19:27:43.324068819+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:1707","ev":"cmd.schedule.prep","st_dur_ms":0}
+{"_ts":"2024-11-25T19:27:43.324077596+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:1804","ev":"cmd.schedule","set":"resolved_tables","val_as":{}}
+{"_ts":"2024-11-25T19:27:43.324083397+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:1810","ev":"cmd.schedule","set":"sources_defined_by_providers","val_as":[]}
+{"_ts":"2024-11-25T19:27:43.324093245+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/build.rs:1856","ev":"cmd.schedule","set":"to_be_downloaded","val_as":[]}
+{"_ts":"2024-11-25T19:27:43.324130795+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2034","ev":"cmd.schedule.dag"}
+{"_ts":"2024-11-25T19:27:43.324140914+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2036","ev":"cmd.schedule.dag","set":"dag.deps","val_s":"{\"moms_flower_shop.raw.raw_inapp_events\":[]}"}
+{"_ts":"2024-11-25T19:27:43.324155872+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2050","ev":"cmd.schedule.dag","set":"dag.initial","val_s":"{\"moms_flower_shop.raw.raw_inapp_events\":[]}"}
+{"_ts":"2024-11-25T19:27:43.324165139+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2062","ev":"cmd.schedule.dag","set":"dag.cleaned","val_s":"{\"moms_flower_shop.raw.raw_inapp_events\":[]}"}
+{"_ts":"2024-11-25T19:27:43.324171421+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2068","ev":"cmd.schedule.dag","set":"missing_defs","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.324185818+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2146","ev":"cmd.schedule.dag","set":"dag.resolved_targets","val_s":"[\"moms_flower_shop.raw.raw_inapp_events\"]"}
+{"_ts":"2024-11-25T19:27:43.324196668+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2174","ev":"cmd.schedule.dag","set":"dag.targeted","val_s":"{\"moms_flower_shop.raw.raw_inapp_events\":[]}"}
+{"_ts":"2024-11-25T19:27:43.324227846+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2233","ev":"cmd.schedule.dag","set":"dag.acyclic","val_s":"{\"moms_flower_shop.raw.raw_inapp_events\":[]}"}
+{"_ts":"2024-11-25T19:27:43.324234018+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2239","ev":"cmd.schedule.dag","set":"cycles","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.324239999+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2245","ev":"cmd.schedule.dag","set":"cut_points","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.324247263+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2251","ev":"cmd.schedule.dag","set":"order.initial","val_s":"[\"moms_flower_shop.raw.raw_inapp_events\"]"}
+{"_ts":"2024-11-25T19:27:43.324291275+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2520","ev":"cmd.schedule.dag","set":"showable_root_tables","val_s":"[]"}
+{"_ts":"2024-11-25T19:27:43.324310691+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2809","ev":"cmd.schedule.dag","set":"order.without_providers","val_s":"[\"moms_flower_shop.raw.raw_inapp_events\"]"}
+{"_ts":"2024-11-25T19:27:43.324321842+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2835","ev":"cmd.schedule.dag","st_dur_ms":0}
+
+{"_ts":"2024-11-25T19:27:43.324334946+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2842","ev":"cmd.do"}
+{"_ts":"2024-11-25T19:27:43.324350996+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/subcommands/build.rs:2868","ev":"cmd.do","set":"order.without_providers_or_cached","val_s":"[\"moms_flower_shop.raw.raw_inapp_events\"]"}
+{"_ts":"2024-11-25T19:27:43.324360704+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:2885","ev":"cmd.do.sources"}
+
+{"_ts":"2024-11-25T19:27:43.324896452+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:4314","ev":"cmd.do.derived.wave_1.mat","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events"}
+{"_ts":"2024-11-25T19:27:43.325682059+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/describe.rs:262","ev":"cmd.do.derived.wave_1.meta_analysis","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events"}
+{"_ts":"2024-11-25T19:27:43.325727454+00:00","_ll":"DEBUG","_fn":"crates/sdf-cli/src/subcommands/describe.rs:307","ev":"cmd.do.derived.wave_1.meta_analysis","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events","st_dur_ms":0}
+{"_ts":"2024-11-25T19:27:43.325746490+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:4438","ev":"cmd.do.derived.wave_1.mat","ev_tb":"moms_flower_shop.raw.raw_inapp_events","ev_tb_catalog":"moms_flower_shop","ev_tb_purpose":"model","ev_tb_schema":"raw","ev_tb_table":"raw_inapp_events","st_code":"succeeded","st_done":"executed","st_dur_ms":0}
+
+{"_ts":"2024-11-25T19:27:43.325791063+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:4298","ev":"cmd.do.derived","st_dur_ms":0}
+{"_ts":"2024-11-25T19:27:43.326141992+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/utils/dep_cache.rs:1888","message":"Saving table cache: moms_flower_shop.raw.raw_inapp_events, sdftarget/dbg/table/moms_flower_shop/raw/raw_inapp_events.sdf.yml"}
+{"_ts":"2024-11-25T19:27:43.326429736+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:3520","ev":"cmd.do.gen_makefile"}
+{"_ts":"2024-11-25T19:27:43.326648344+00:00","_ll":"TRACE","_fn":"crates/sdf-cli/src/utils/dep_cache.rs:2298","ev":"written_makefile","set":"makefile","val_m":{"contents":{"cli_vars":"99914b932bd37a50b983c5e7c90ae93b","credentials":"99914b932bd37a50b983c5e7c90ae93b","env_vars":"d751713988987e9331980363e24189ce"},"prelude":[{"file":"workspace.sdf.yml","mtime":"2024-11-25T19:27:41.251134683Z"},{"file":"metadata/analytics/dim_marketing_campaigns.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/analytics/agg_installs_and_campaigns.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_customers.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_inapp_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/staging/inapp_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/staging/app_installs.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_marketing_campaign_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_addresses.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"classifications/column_classifiers.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"classifications/table_classifiers.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/analytics/dim_marketing_campaigns.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/analytics/agg_installs_and_campaigns.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_customers.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_inapp_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/staging/inapp_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/staging/app_installs.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_marketing_campaign_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"metadata/raw/raw_addresses.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"sdftarget/dbg/.lib/sdf_test/workspace.sdf.yml","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/workspace.sdf.yml","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_mat/workspace.sdf.yml","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_test/macros/test.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/macros/date_spine.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/macros/generate_integer_values.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/macros/generate_surrogate_key.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/macros/group_by.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_utils/macros/generate_date_values.jinja","mtime":"2024-11-25T19:27:42.755134237Z"},{"file":"sdftarget/dbg/.lib/sdf_mat/macros/dialect/trino/materialize.jinja","mtime":"2024-11-25T19:27:42.755134237Z"}],"deps":[{"target":"moms_flower_shop.raw.raw_inapp_events","deps":[]}],"source_files":{"values":{"moms_flower_shop.raw.raw_inapp_events":[{"file":"metadata/raw/raw_inapp_events.sdf.yml","mtime":"2024-11-25T19:27:41.247134684Z"},{"file":"models/raw/raw_inapp_events.sql","mtime":"2024-11-25T19:27:41.251134683Z"},{"file":"seeds/parquet/inapp_events.parquet","mtime":"2024-11-25T19:27:41.251134683Z"}]}},"incomplete_source_files":{"values":[]},"target_status":{"values":{"moms_flower_shop.raw.raw_inapp_events":{"purpose":"model","status":"succeeded","draft":false}}}}}
+{"_ts":"2024-11-25T19:27:43.326811502+00:00","_ll":"INFO","_fn":"crates/sdf-cli/src/subcommands/build.rs:3534","ev":"cmd.do.gen_makefile","st_dur_ms":0}
@@ -152,7 +172,7 @@ The following guide will go through the different logging options available in S 
 
 Working set 12 model files, 1 seed file, 19 .sdf files
-   Finished 1 model [1 succeeded] in 0.757 secs
+   Finished 1 model [1 succeeded] in 0.824 secs
 
 Schema moms_flower_shop.pub.compile_info
 ┌───────────────┬────────────────┬────────────┬─────────────┐
@@ -168,7 +188,6 @@ Schema moms_flower_shop.pub.compile_info
 │ st_dur_ms     ┆ bigint         ┆            ┆             │
 │ ev_fn         ┆ varchar        ┆            ┆             │
 │ st_code       ┆ varchar        ┆            ┆             │
-│ message       ┆ varchar        ┆            ┆             │
 │ ev_tb         ┆ varchar        ┆            ┆             │
 │ ev_tb_catalog ┆ varchar        ┆            ┆             │
 │ ev_tb_schema  ┆ varchar        ┆            ┆             │
@@ -194,24 +213,24 @@ Schema moms_flower_shop.pub.compile_info
 
 Working set 13 model files, 1 seed file, 19 .sdf files
     Running moms_flower_shop.pub.query (./sdftarget/dbg/query.sql)
-   Finished 2 models [2 succeeded] in 0.769 secs
+   Finished 2 models [2 succeeded] in 0.887 secs
 
 Table moms_flower_shop.pub.query
 ┌─────────────────────────────────────┬──────────────────────────────┬────────────────┐
 │ timestamp                           ┆ event                        ┆ duration_in_ms │
 ╞═════════════════════════════════════╪══════════════════════════════╪════════════════╡
-│ 2024-11-22T20:47:27.149645189+00:00 ┆ sdf                          ┆                │
-│ 2024-11-22T20:47:27.149808024+00:00 ┆ sdf                          ┆                │
-│ 2024-11-22T20:47:27.149965287+00:00 ┆ sdf                          ┆                │
-│ 2024-11-22T20:47:27.151149136+00:00 ┆ initialize                   ┆                │
-│ 2024-11-22T20:47:27.151351104+00:00 ┆ initialize.load_workspace    ┆                │
-│ 2024-11-22T20:47:27.169119334+00:00 ┆ initialize.load_workspace    ┆ 17             │
-│ 2024-11-22T20:47:27.169220724+00:00 ┆ initialize.load_dependencies ┆                │
-│ 2024-11-22T20:47:27.169304149+00:00 ┆ initialize.load_dependencies ┆ 0              │
-│ 2024-11-22T20:47:27.169395239+00:00 ┆ initialize.load_prelude      ┆                │
-│ 2024-11-22T20:47:27.172282799+00:00 ┆ initialize.load_prelude      ┆ 2              │
+│ 2024-11-25T19:27:44.773528974+00:00 ┆ sdf                          ┆                │
+│ 2024-11-25T19:27:44.773713910+00:00 ┆ sdf                          ┆                │
+│ 2024-11-25T19:27:44.773878878+00:00 ┆ sdf                          ┆                │
+│ 2024-11-25T19:27:44.775022104+00:00 ┆ initialize                   ┆                │
+│ 2024-11-25T19:27:44.775225524+00:00 ┆ initialize.load_workspace    ┆                │
+│ 2024-11-25T19:27:44.792045513+00:00 ┆ initialize.load_workspace    ┆ 16             │
+│ 2024-11-25T19:27:44.792145780+00:00 ┆ initialize.load_dependencies ┆                │
+│ 2024-11-25T19:27:44.792229156+00:00 ┆ initialize.load_dependencies ┆ 0              │
+│ 2024-11-25T19:27:44.792342999+00:00 ┆ initialize.load_prelude      ┆                │
+│ 2024-11-25T19:27:44.794951533+00:00 ┆ initialize.load_prelude      ┆ 2              │
 └─────────────────────────────────────┴──────────────────────────────┴────────────────┘
-62 rows, showing only 10 rows.
+60 rows, showing only 10 rows.
   Run with --limit 0 to show all rows.
 
 
diff --git a/docs/guide/data-quality/checks.mdx b/docs/guide/data-quality/checks.mdx
index 03ddae67..7e73be01 100644
--- a/docs/guide/data-quality/checks.mdx
+++ b/docs/guide/data-quality/checks.mdx
@@ -36,7 +36,7 @@ Let's write and modify a check, starting with one of the sdf samples.
         
         📚 Read the Docs to Get Started -> https://docs.sdf.com/
         
-   Finished new in 0.224 secs
+   Finished new in 0.273 secs
@@ -78,7 +78,7 @@ Working set 1 model file, 1 .sdf file   Compiling hello.pub.main (./models/main.sql) Working set 1 check file, 1 .sdf file    Checking hello.pub.code_check (./checks/code_check.sql) -   Finished 1 model [1 succeeded], 1 check [1 passed] in 1.295 secs +   Finished 1 model [1 succeeded], 1 check [1 passed] in 1.419 secs [Pass] Check hello.pub.code_check @@ -122,7 +122,7 @@ Working set 1 model file, 1 .sdf file   Compiling hello.pub.main (./models/main.sql) Working set 1 check file, 1 .sdf file    Checking hello.pub.code_check (./checks/code_check.sql) -   Finished 1 model [1 succeeded], 1 check [1 failed] in 1.283 secs, for details see below. +   Finished 1 model [1 succeeded], 1 check [1 failed] in 1.375 secs, for details see below. [Fail] Check hello.pub.code_check ┌────────────┬─────────────┬─────────────┐ @@ -133,7 +133,7 @@ Working set 1 check file, 1 .sdf file 1 rows. ------- -Summary 1 model [1 succeeded], 1 check [1 failed] in 1.283 secs. +Summary 1 model [1 succeeded], 1 check [1 failed] in 1.375 secs. ------- @@ -195,7 +195,7 @@ Working set 2 model files, 1 .sdf file   Compiling hello.pub.sink (./models/sink.sql) Working set 1 check file, 1 .sdf file    Checking hello.pub.code_check (./checks/code_check.sql) -   Finished 2 models [2 succeeded], 1 check [1 passed] in 1.294 secs +   Finished 2 models [2 succeeded], 1 check [1 passed] in 1.429 secs [Pass] Check hello.pub.code_check diff --git a/docs/guide/data-quality/reports.mdx b/docs/guide/data-quality/reports.mdx index aac0b676..3a630a50 100644 --- a/docs/guide/data-quality/reports.mdx +++ b/docs/guide/data-quality/reports.mdx @@ -50,7 +50,7 @@ Let's write and modify a report, starting with one of the sdf samples.                  📚 Read the Docs to Get Started -> https://docs.sdf.com/          -   Finished new in 0.216 secs +   Finished new in 0.299 secs @@ -111,10 +111,9 @@ Let's write and modify a report, starting with one of the sdf samples. 
 
 Working set 12 model files, 3 .sdf files
+  Compiling payment.public.organizations (./ddls/payment/public/organizations.sql)
   Compiling payment.public.invoices (./ddls/payment/public/invoices.sql)
   Compiling payment.public.users (./ddls/payment/public/users.sql)
-  Compiling payment.public.organizations (./ddls/payment/public/organizations.sql)
-  Compiling transformations.external.invoice_stats (./models/external/invoice_stats.sql)
   Compiling transformations.internal.mau_per_org (./models/internal/mau_per_org.sql)
   Compiling transformations.external.org_invoice_stats (./models/external/org_invoice_stats.sql)
   Compiling transformations.internal.users_per_domain (./models/internal/users_per_domain.sql)
@@ -123,18 +122,19 @@ Working set 12 model files, 3 .sdf files
   Compiling transformations.internal.most_frequent_payer (./models/internal/most_frequent_payer.sql)
   Compiling transformations.internal.users_per_org (./models/internal/users_per_org.sql)
   Compiling transformations.internal.total_revenue_per_org (./models/internal/total_revenue_per_org.sql)
+  Compiling transformations.external.invoice_stats (./models/external/invoice_stats.sql)
 Working set 1 report file, 1 .sdf file
   Reporting sdf.reports.tables_with_pii (./reports/tables_with_pii.sql)
-   Finished 12 models [12 succeeded], 1 report [1 succeeded] in 1.849 secs
+   Finished 12 models [12 succeeded], 1 report [1 succeeded] in 1.929 secs
 
-Report sdf.reports.tables_with_pii
+Report sdf.reports.TABLES_WITH_PII
 ┌──────────────────────────────────────────────┬─────────────────────────┬───────────┐
 │ table_id                                     ┆ description             ┆ dialect   │
 ╞══════════════════════════════════════════════╪═════════════════════════╪═══════════╡
 │ TRANSFORMATIONS.INTERNAL.MOST_FREQUENT_PAYER ┆                         ┆ snowflake │
 │ payment.public.USERS                         ┆ DDL for the users table ┆ snowflake │
-│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆                         ┆ snowflake │
 │ TRANSFORMATIONS.INTERNAL.USERS_PER_DOMAIN    ┆                         ┆ snowflake │
+│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆                         ┆ snowflake │
 └──────────────────────────────────────────────┴─────────────────────────┴───────────┘
 4 rows.
 
@@ -167,16 +167,16 @@ Report sdf.reports.tables_with_pii
 Working set 12 model files, 3 .sdf files
 Working set 1 report file, 1 .sdf file
   Reporting sdf.reports.tables_with_pii (./reports/tables_with_pii.sql)
-   Finished 12 models [12 reused], 1 report [1 succeeded] in 1.583 secs
+   Finished 12 models [12 reused], 1 report [1 succeeded] in 1.698 secs
 
-Report sdf.reports.tables_with_pii
+Report sdf.reports.TABLES_WITH_PII
 ┌──────────────────────────────────────────────┬──────────────┬─────────────────────────┬───────────┐
 │ table_id                                     ┆ column_name  ┆ description             ┆ dialect   │
 ╞══════════════════════════════════════════════╪══════════════╪═════════════════════════╪═══════════╡
-│ payment.public.USERS                         ┆ NAME         ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.INTERNAL.MOST_FREQUENT_PAYER ┆ EMAIL        ┆                         ┆ snowflake │
-│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ NAME         ┆                         ┆ snowflake │
+│ payment.public.USERS                         ┆ NAME         ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.INTERNAL.USERS_PER_DOMAIN    ┆ EMAIL_DOMAIN ┆                         ┆ snowflake │
+│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ NAME         ┆                         ┆ snowflake │
 │ payment.public.USERS                         ┆ EMAIL        ┆ DDL for the users table ┆ snowflake │
 │ payment.public.USERS                         ┆ PHONE        ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ EMAIL        ┆                         ┆ snowflake │
@@ -208,23 +208,23 @@ Report sdf.reports.tables_with_pii
 Working set 12 model files, 3 .sdf files
 Working set 2 report files, 1 .sdf file
   Reporting sdf.reports.pii_datatypes (./reports/pii_datatypes.sql)
-   Finished 12 models [12 reused], 2 reports [1 succeeded, 1 reused] in 1.582 secs
+   Finished 12 models [12 reused], 2 reports [1 succeeded, 1 reused] in 1.757 secs
 
-Report sdf.reports.tables_with_pii
+Report sdf.reports.TABLES_WITH_PII
 ┌──────────────────────────────────────────────┬──────────────┬─────────────────────────┬───────────┐
 │ table_id                                     ┆ column_name  ┆ description             ┆ dialect   │
 ╞══════════════════════════════════════════════╪══════════════╪═════════════════════════╪═══════════╡
 │ payment.public.USERS                         ┆ EMAIL        ┆ DDL for the users table ┆ snowflake │
 │ payment.public.USERS                         ┆ PHONE        ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ EMAIL        ┆                         ┆ snowflake │
-│ payment.public.USERS                         ┆ NAME         ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.INTERNAL.MOST_FREQUENT_PAYER ┆ EMAIL        ┆                         ┆ snowflake │
-│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ NAME         ┆                         ┆ snowflake │
+│ payment.public.USERS                         ┆ NAME         ┆ DDL for the users table ┆ snowflake │
 │ TRANSFORMATIONS.INTERNAL.USERS_PER_DOMAIN    ┆ EMAIL_DOMAIN ┆                         ┆ snowflake │
+│ TRANSFORMATIONS.EXTERNAL.INVOICE_STATS       ┆ NAME         ┆                         ┆ snowflake │
 └──────────────────────────────────────────────┴──────────────┴─────────────────────────┴───────────┘
 7 rows.
 
-Report sdf.reports.pii_datatypes
+Report sdf.reports.PII_DATATYPES
 ┌──────────┬───────────┐
 │ datatype ┆ frequency │
 ╞══════════╪═══════════╡
diff --git a/docs/guide/data-quality/tests.mdx b/docs/guide/data-quality/tests.mdx
index d726a6cf..219efd74 100644
--- a/docs/guide/data-quality/tests.mdx
+++ b/docs/guide/data-quality/tests.mdx
@@ -26,7 +26,7 @@ When `sdf test` runs this query, it will check if any data is returned and fail
 Working set 1 model file, 1 test file, 1 .sdf file
     Running hello.pub.main (./models/main.sql)
     Testing hello.pub.test_main (./sdftarget/dbg/tests/hello/pub/test_main.sql)
-   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.767 secs
+   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.858 secs
 [Pass] Test hello.pub.test_main
 
 
@@ -180,7 +180,7 @@ table:
 Working set 1 model file, 1 test file, 2 .sdf files
     Running tests_workspace.pub.raw_inapp_events (./models/raw_inapp_events.sql)
     Testing tests_workspace.pub.test_raw_inapp_events (./sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql)
-   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.790 secs
+   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.946 secs
 [Pass] Test tests_workspace.pub.test_raw_inapp_events
 
 
@@ -189,7 +189,7 @@ Working set 1 model file, 1 test file, 2
   
   
     This step is not mandatory, but sometimes it can be helpful to view the compiled test query.
-    To access the query, navigate to the file located in `sdftarget/dbg/preprocessed/sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql`.
+    To access the query, navigate to the file located in `sdftarget/dbg/preprocessed/tests_workspace/pub/test_raw_inapp_events.sql`.
 
     The file will hold the compiled tests query and will look like this:
 

@@ -347,7 +347,7 @@ to run custom testing macros.
 
 
     Cleaned sdftarget
-   Finished clean in 0.214 secs
+   Finished clean in 0.286 secs
 .
 ├── custom_tests
 ├── models
@@ -431,7 +431,7 @@ to run custom testing macros.
 Working set 1 model file, 1 test file, 2 .sdf files
     Running tests_workspace.pub.raw_inapp_events (./models/raw_inapp_events.sql)
     Testing tests_workspace.pub.test_raw_inapp_events (./sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql)
-   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.777 secs
+   Finished 1 model [1 succeeded], 1 test [1 passed] in 0.869 secs
 [Pass] Test tests_workspace.pub.test_raw_inapp_events
 
 
@@ -439,7 +439,7 @@ Working set 1 model file, 1 test file, 2
 

 
         We can look at the compiled tests query located in the file 
-        `sdftarget/dbg/preprocessed/sdftarget/dbg/tests/tests_workspace/pub/test_raw_inapp_events.sql`.
+        `sdftarget/dbg/preprocessed/tests_workspace/pub/test_raw_inapp_events.sql`.
         Notice our custom logic in the compiled query:
 
 

diff --git a/docs/guide/setup/io.mdx b/docs/guide/setup/io.mdx
index b5f09273..449b0d22 100644
--- a/docs/guide/setup/io.mdx
+++ b/docs/guide/setup/io.mdx
@@ -93,7 +93,7 @@ Let's try these options. Start by creating a new SDF workspace with a sample pro
         
         📚 Read the Docs to Get Started -> https://docs.sdf.com/
         
-   Finished new in 0.219 secs
+   Finished new in 0.271 secs
@@ -124,7 +124,7 @@ Working set 4 model files, 1 .sdf file   Compiling lineage.pub.middle (./models/middle.sql)   Compiling lineage.pub.knis (./models/knis.sql)   Compiling lineage.pub.sink (./models/sink.sql) -   Finished 4 models [4 succeeded] in 0.766 secs +   Finished 4 models [4 succeeded] in 0.849 secs diff --git a/docs/guide/setup/workspaces.mdx b/docs/guide/setup/workspaces.mdx index 2f78abe8..9db6fcb8 100644 --- a/docs/guide/setup/workspaces.mdx +++ b/docs/guide/setup/workspaces.mdx @@ -306,6 +306,7 @@ Here's an example of a full-fledged table definition generated in the `sdftarget table:   name: moms_flower_shop.analytics.agg_installs_and_campaigns   dialect: trino +  casing-policy: preserve   materialization: view   purpose: model   dependencies: @@ -317,11 +318,13 @@ table:       modify:       - moms_flower_shop.staging.app_installs_v2.install_time   - name: campaign_name +    description: The campaign name associated with the campaign_id     datatype: varchar     lineage:       copy:       - moms_flower_shop.staging.app_installs_v2.campaign_name   - name: platform +    description: iOS or Android     datatype: varchar     lineage:       copy: diff --git a/docs/integrations/aws/S3/s3-example.mdx b/docs/integrations/aws/S3/s3-example.mdx index 7a08c448..7e8e30d0 100644 --- a/docs/integrations/aws/S3/s3-example.mdx +++ b/docs/integrations/aws/S3/s3-example.mdx @@ -57,7 +57,7 @@ integration with SDF to analyze world population data. 
 
 Working set 1 .sdf file
-   Finished run in 0.743 secs
+   Finished run in 0.826 secs
@@ -116,7 +116,7 @@ Working set 2 model files, 1 .sdf file     Running hello_world.pub.un_pop_data (./remote/un_pop_data.sql) Downloading hello_world.pub.un_pop_data  (s3://sdfdatasets/hello-world/world_population_full.csv)     Running hello_world.pub.q1 (./remote/q1.sql) -   Finished 2 models [2 succeeded] in 1.670 secs +   Finished 2 models [2 succeeded] in 1.461 secs Table hello_world.pub.un_pop_data ┌───────┬───────────┬───────────────────┬───────────────┬─────────────────┬───────┬──────┬──────────────────────┬──────────────────────────────────┬────────────┐ diff --git a/docs/integrations/bigquery/basic-materialization.mdx b/docs/integrations/bigquery/basic-materialization.mdx index 91e8e5f4..81219825 100644 --- a/docs/integrations/bigquery/basic-materialization.mdx +++ b/docs/integrations/bigquery/basic-materialization.mdx @@ -112,12 +112,12 @@ Before beginning, we'll need to make sure we have the following: 
 
 Working set 2 model files, 1 .sdf file
-Downloading "integration-test-sdf".sdf_ecommerce_example.orders (schema)
 Downloading "integration-test-sdf".sdf_ecommerce_example.customers (schema)
+Downloading "integration-test-sdf".sdf_ecommerce_example.orders (schema)
 Downloading "integration-test-sdf".sdf_ecommerce_example.products (schema)
   Compiling integration-test-sdf.sdf_ecommerce_example.aggregate_orders (./models/aggregate_orders.sql)
   Compiling integration-test-sdf.sdf_ecommerce_example.customers_over_100 (./models/customers_over_100.sql)
-   Finished 5 models [2 succeeded, 3 downloaded] in 1.674 secs
+   Finished 5 models [5 succeeded] in 1.901 secs
@@ -156,12 +156,12 @@ Downloading "integration-test-sdf".sdf_ecommerce_example. 
 
 Working set 2 model files, 1 .sdf file
-Downloading "integration-test-sdf".sdf_ecommerce_example.customers (schema)
-Downloading "integration-test-sdf".sdf_ecommerce_example.products (schema)
 Downloading "integration-test-sdf".sdf_ecommerce_example.orders (schema)
+Downloading "integration-test-sdf".sdf_ecommerce_example.products (schema)
+Downloading "integration-test-sdf".sdf_ecommerce_example.customers (schema)
     Running integration-test-sdf.sdf_ecommerce_example.aggregate_orders (./models/aggregate_orders.sql)
     Running integration-test-sdf.sdf_ecommerce_example.customers_over_100 (./models/customers_over_100.sql)
-   Finished 5 models [2 succeeded, 3 downloaded] in 3.114 secs
+   Finished 5 models [5 succeeded] in 3.597 secs
@@ -183,7 +183,17 @@ Downloading "integration-test-sdf".sdf_ecommerce_example. Downloading "integration-test-sdf".sdf_ecommerce_example.customers (schema)     Running integration-test-sdf.sdf_ecommerce_example.aggregate_orders (./models/aggregate_orders.sql)     Running integration-test-sdf.sdf_ecommerce_example.customers_over_100 (./models/customers_over_100.sql) -   Finished 5 models [2 succeeded, 3 downloaded] in 6.843 secs +   Finished 5 models [5 succeeded] in 9.424 secs + +Table "integration-test-sdf".sdf_ecommerce_example.products +┌────────────┬──────────────┬────────┬─────────────┐ +│ product_id ┆ product_name ┆ price  ┆ category    │ +╞════════════╪══════════════╪════════╪═════════════╡ +│ 1          ┆ Laptop       ┆ 999.99 ┆ Electronics │ +│ 2          ┆ Headphones   ┆ 199.99 ┆ Electronics │ +│ 3          ┆ Coffee Maker ┆ 89.99  ┆ Appliances  │ +└────────────┴──────────────┴────────┴─────────────┘ +3 rows. Table "integration-test-sdf".sdf_ecommerce_example.orders ┌──────────┬─────────────┬────────────┬─────────────────────┬──────────┐ @@ -205,16 +215,6 @@ Table "integration-test-sdf".sdf_ecommerce_example.custom └─────────────┴───────────────┴───────────────────────┴─────────────────────┘ 3 rows. -Table "integration-test-sdf".sdf_ecommerce_example.products -┌────────────┬──────────────┬────────┬─────────────┐ -│ product_id ┆ product_name ┆ price  ┆ category    │ -╞════════════╪══════════════╪════════╪═════════════╡ -│ 1          ┆ Laptop       ┆ 999.99 ┆ Electronics │ -│ 2          ┆ Headphones   ┆ 199.99 ┆ Electronics │ -│ 3          ┆ Coffee Maker ┆ 89.99  ┆ Appliances  │ -└────────────┴──────────────┴────────┴─────────────┘ -3 rows. - Table "integration-test-sdf".sdf_ecommerce_example.aggregate_orders ┌─────────────┬───────────────┬──────────────┬─────────────┐ │ customer_id ┆ customer_name ┆ total_orders ┆ total_spent │ diff --git a/docs/integrations/dagster/getting-started.mdx b/docs/integrations/dagster/getting-started.mdx index 81093eb0..bc3a30a9 100644 --- a/docs/integrations/dagster/getting-started.mdx +++ b/docs/integrations/dagster/getting-started.mdx @@ -41,7 +41,7 @@ The following guide assumes you already have an active SDF workspace. If not, yo 
 
-sdf 0.5.16
+sdf 0.10.5-p
diff --git a/docs/integrations/dbt/migrating.mdx b/docs/integrations/dbt/migrating.mdx index 7287ba30..377629d7 100644 --- a/docs/integrations/dbt/migrating.mdx +++ b/docs/integrations/dbt/migrating.mdx @@ -153,7 +153,7 @@ workspace: 
 
 Working set 3 .sdf files
-   Finished 3 models [3 succeeded] in 0.742 secs
+   Finished 3 models [3 succeeded] in 0.863 secs
 
 Schema jaffle_shop.raw.raw_customers
 ┌─────────────┬───────────┬────────────┬─────────────┐
@@ -413,10 +413,10 @@ tmp/jaffle_shop/
 
 
 Working set 3 model files, 3 .sdf files
-  Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
+  Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
-   Finished 6 models [6 succeeded] in 0.770 secs
+   Finished 6 models [6 succeeded] in 0.888 secs
 
 
 

@@ -436,10 +436,10 @@ Working set 3 model files, 3 .sdf files
 
 
 Working set 3 model files, 2 .sdf files
-    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
-    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
     Running jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
-   Finished 6 models [6 succeeded] in 0.780 secs
+    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
+    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
+   Finished 6 models [6 succeeded] in 0.918 secs
 
 Table jaffle_shop.staging.stg_customers
 ┌─────────────┬────────────┬───────────┐
@@ -459,24 +459,6 @@ Table jaffle_shop.staging.stg_customers
 100 rows, showing only 10 rows.
   Run with --limit 0 to show all rows.
 
-Table jaffle_shop.staging.stg_orders
-┌──────────┬─────────────┬────────────┬───────────┐
-│ order_id ┆ customer_id ┆ order_date ┆ status    │
-╞══════════╪═════════════╪════════════╪═══════════╡
-│ 1        ┆ 1           ┆ 2018-01-01 ┆ returned  │
-│ 2        ┆ 3           ┆ 2018-01-02 ┆ completed │
-│ 3        ┆ 94          ┆ 2018-01-04 ┆ completed │
-│ 4        ┆ 50          ┆ 2018-01-05 ┆ completed │
-│ 5        ┆ 64          ┆ 2018-01-05 ┆ completed │
-│ 6        ┆ 54          ┆ 2018-01-07 ┆ completed │
-│ 7        ┆ 88          ┆ 2018-01-09 ┆ completed │
-│ 8        ┆ 2           ┆ 2018-01-11 ┆ returned  │
-│ 9        ┆ 53          ┆ 2018-01-12 ┆ completed │
-│ 10       ┆ 7           ┆ 2018-01-14 ┆ completed │
-└──────────┴─────────────┴────────────┴───────────┘
-99 rows, showing only 10 rows.
-  Run with --limit 0 to show all rows.
-
 Table jaffle_shop.staging.stg_payments
 ┌────────────┬──────────┬────────────────┬────────┐
 │ payment_id ┆ order_id ┆ payment_method ┆ amount │
@@ -495,6 +477,24 @@ Table jaffle_shop.staging.stg_payments
 113 rows, showing only 10 rows.
   Run with --limit 0 to show all rows.
 
+Table jaffle_shop.staging.stg_orders
+┌──────────┬─────────────┬────────────┬───────────┐
+│ order_id ┆ customer_id ┆ order_date ┆ status    │
+╞══════════╪═════════════╪════════════╪═══════════╡
+│ 1        ┆ 1           ┆ 2018-01-01 ┆ returned  │
+│ 2        ┆ 3           ┆ 2018-01-02 ┆ completed │
+│ 3        ┆ 94          ┆ 2018-01-04 ┆ completed │
+│ 4        ┆ 50          ┆ 2018-01-05 ┆ completed │
+│ 5        ┆ 64          ┆ 2018-01-05 ┆ completed │
+│ 6        ┆ 54          ┆ 2018-01-07 ┆ completed │
+│ 7        ┆ 88          ┆ 2018-01-09 ┆ completed │
+│ 8        ┆ 2           ┆ 2018-01-11 ┆ returned  │
+│ 9        ┆ 53          ┆ 2018-01-12 ┆ completed │
+│ 10       ┆ 7           ┆ 2018-01-14 ┆ completed │
+└──────────┴─────────────┴────────────┴───────────┘
+99 rows, showing only 10 rows.
+  Run with --limit 0 to show all rows.
+
@@ -640,11 +640,11 @@ Table jaffle_shop.staging.stg_payments 
 
 Working set 4 model files, 3 .sdf files
+  Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
   Compiling jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
-  Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-   Finished 7 models [7 succeeded] in 0.784 secs
+   Finished 7 models [7 succeeded] in 0.895 secs
@@ -660,10 +660,10 @@ Working set 4 model files, 3 .sdf files 
 
 Working set 4 model files, 2 .sdf files
-    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
+    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-   Finished 5 models [5 succeeded] in 0.804 secs
+   Finished 5 models [5 succeeded] in 0.935 secs
 
 Table jaffle_shop.analytics.orders
 ┌──────────┬─────────────┬────────────┬───────────┬────────────────────┬───────────────┬──────────────────────┬──────────────────┬────────┐
@@ -725,7 +725,7 @@ Working set 4 model files, 4 .sdf files
   Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
   Compiling jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-   Finished 7 models [7 succeeded] in 0.781 secs
+   Finished 7 models [7 succeeded] in 0.899 secs
@@ -857,16 +857,16 @@ tmp/jaffle_shop/ 
 
 Working set 4 model files, 3 test files, 7 .sdf files
+    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
     Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
-    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
+    Testing jaffle_shop.staging.test_stg_orders (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_orders.sql)
     Testing jaffle_shop.staging.test_stg_payments (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_payments.sql)
     Testing jaffle_shop.staging.test_stg_customers (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_customers.sql)
-    Testing jaffle_shop.staging.test_stg_orders (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_orders.sql)
-   Finished 6 models [6 succeeded], 3 tests [3 passed] in 0.844 secs
-[Pass] Test jaffle_shop.staging.test_stg_customers
+   Finished 6 models [6 succeeded], 3 tests [3 passed] in 0.942 secs
 [Pass] Test jaffle_shop.staging.test_stg_payments
 [Pass] Test jaffle_shop.staging.test_stg_orders
+[Pass] Test jaffle_shop.staging.test_stg_customers
@@ -918,12 +918,12 @@ tmp/jaffle_shop_complete/ 
 
 Working set 5 model files, 10 .sdf files
-  Compiling jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
   Compiling jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
   Compiling jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
+  Compiling jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
   Compiling jaffle_shop.analytics.customers (./models/analytics/customers.sql)
   Compiling jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-   Finished 8 models [8 succeeded] in 0.802 secs
+   Finished 8 models [8 succeeded] in 0.915 secs
@@ -939,12 +939,12 @@ Working set 5 model files, 10 .sdf files 
 
 Working set 5 model files, 10 .sdf files
+    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
     Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
-    Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.analytics.customers (./models/analytics/customers.sql)
     Running jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-   Finished 8 models [8 succeeded] in 0.842 secs
+   Finished 8 models [8 succeeded] in 0.951 secs
@@ -960,20 +960,20 @@ Working set 5 model files, 10 .sdf files 
 
 Working set 5 model files, 5 test files, 10 .sdf files
-    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
     Running jaffle_shop.staging.stg_orders (./models/staging/stg_orders.sql)
     Running jaffle_shop.staging.stg_payments (./models/staging/stg_payments.sql)
-    Testing jaffle_shop.staging.test_stg_customers (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_customers.sql)
+    Running jaffle_shop.staging.stg_customers (./models/staging/stg_customers.sql)
+    Testing jaffle_shop.staging.test_stg_orders (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_orders.sql)
     Testing jaffle_shop.staging.test_stg_payments (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_payments.sql)
+    Testing jaffle_shop.staging.test_stg_customers (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_customers.sql)
     Running jaffle_shop.analytics.customers (./models/analytics/customers.sql)
     Running jaffle_shop.analytics.orders (./models/analytics/orders.sql)
-    Testing jaffle_shop.staging.test_stg_orders (./sdftarget/dbg/tests/jaffle_shop/staging/test_stg_orders.sql)
-    Testing jaffle_shop.analytics.test_customers (./sdftarget/dbg/tests/jaffle_shop/analytics/test_customers.sql)
     Testing jaffle_shop.analytics.test_orders (./sdftarget/dbg/tests/jaffle_shop/analytics/test_orders.sql)
-   Finished 8 models [8 succeeded], 5 tests [5 passed] in 0.941 secs
-[Pass] Test jaffle_shop.staging.test_stg_customers
-[Pass] Test jaffle_shop.staging.test_stg_payments
+    Testing jaffle_shop.analytics.test_customers (./sdftarget/dbg/tests/jaffle_shop/analytics/test_customers.sql)
+   Finished 8 models [8 succeeded], 5 tests [5 passed] in 1.043 secs
 [Pass] Test jaffle_shop.staging.test_stg_orders
+[Pass] Test jaffle_shop.staging.test_stg_payments
+[Pass] Test jaffle_shop.staging.test_stg_customers
 [Pass] Test jaffle_shop.analytics.test_customers
 [Pass] Test jaffle_shop.analytics.test_orders
 
diff --git a/docs/integrations/snowflake/basic-materialization.mdx b/docs/integrations/snowflake/basic-materialization.mdx
index b8bd972e..347673c0 100644
--- a/docs/integrations/snowflake/basic-materialization.mdx
+++ b/docs/integrations/snowflake/basic-materialization.mdx
@@ -46,7 +46,7 @@ When installing the Cybersyn dataset, make sure to grant the Snowflake role you'
 Working set 3 model files, 1 .sdf file
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.OPENALEX_FUNDERS_INDEX (schema)
   Compiling sdf_snowflake.cybersyn_tech_innovation.funder_aggregates (./models/sdf_snowflake/cybersyn_tech_innovation/funder_aggregates.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 1.424 secs
+   Finished 2 models [2 succeeded] in 1.765 secs
 
 Schema sdf_snowflake.cybersyn_tech_innovation.FUNDER_AGGREGATES
 ┌──────────────────────┬────────────────┬────────────┬──────────────────────────────────────────────────────────────────┐
@@ -172,7 +172,7 @@ Schema sdf_snowflake.cybersyn_tech_innovation.FUNDER_AGGREGATES
 Working set 4 model files, 1 .sdf file
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_REPOS (schema)
   Compiling sdf_snowflake.staging.latest_repo_name (./models/sdf_snowflake/staging/latest_repo_name.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 1.514 secs
+   Finished 2 models [2 succeeded] in 1.665 secs
 
 Schema sdf_snowflake.staging.LATEST_REPO_NAME
 ┌─────────────┬────────────────┬────────────┬──────────────────────────────────────┐
@@ -216,23 +216,23 @@ Schema sdf_snowflake.staging.LATEST_REPO_NAME
 Working set 4 model files, 1 .sdf file
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_REPOS (schema)
     Running sdf_snowflake.staging.latest_repo_name (./models/sdf_snowflake/staging/latest_repo_name.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 4.696 secs
+   Finished 2 models [2 succeeded] in 5.150 secs
 
 Table sdf_snowflake.staging.LATEST_REPO_NAME
-┌───────────────────────────────────────────────┬───────────┐
-│ REPO_NAME                                     ┆ REPO_ID   │
-╞═══════════════════════════════════════════════╪═══════════╡
-│ 0-ANNONYMOUS-0/Flight-Price-Prediction        ┆ 892133700 │
-│ 0-Blu/SideStore                               ┆ 891173583 │
-│ 0-Nox/python-projects                         ┆ 891938782 │
-│ 0-bcda-0/AIT-React-Art                        ┆ 891938121 │
-│ 0-bcda-0/AIT-React-Investment_Calculator      ┆ 891929104 │
-│ 0-ft/lexi                                     ┆ 892311030 │
-│ 00-Piter-00/biblioteca-login-cadastro-simples ┆ 892191828 │
-│ 00-theFOOL/PatientAmbulation                  ┆ 891922546 │
-│ 000-KunalPal/kp-stack                         ┆ 892208958 │
-│ 000-beb/000-beb                               ┆ 892291204 │
-└───────────────────────────────────────────────┴───────────┘
+┌───────────────────────────┬───────────┐
+│ REPO_NAME                 ┆ REPO_ID   │
+╞═══════════════════════════╪═══════════╡
+│ 0-Artha-0/0-Artha-0       ┆ 893439527 │
+│ 0-Artha-0/FloraGuide      ┆ 893550378 │
+│ 0-Crypto-Bits-0/Viktor    ┆ 893292867 │
+│ 0-Godus-0/CMPSC457HW3     ┆ 893518756 │
+│ 0-Godus-0/CMPSC457HW3POST ┆ 893554016 │
+│ 0-Godus-0/MAX             ┆ 893602492 │
+│ 0-Inacio-0/TimeMixer      ┆ 893147904 │
+│ 0-Octet-1/LMS             ┆ 893524410 │
+│ 0-SEVN/IS-201             ┆ 893282694 │
+│ 0-SEVN/IS-201.1           ┆ 893333580 │
+└───────────────────────────┴───────────┘
 10 rows.
 
 
@@ -282,7 +282,7 @@ Table sdf_snowflake.staging.LATEST_REPO_NAME
 Working set 5 model files, 1 .sdf file
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_EVENTS (schema)
   Compiling sdf_snowflake.staging.push_events (./models/sdf_snowflake/staging/push_events.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 1.449 secs
+   Finished 2 models [2 succeeded] in 1.751 secs
 
 Schema sdf_snowflake.staging.PUSH_EVENTS
 ┌─────────────┬────────────────┬────────────┬────────────────────────────────────────────────────────────────────────────────────────────────┐
@@ -348,20 +348,20 @@ Schema sdf_snowflake.staging.PUSH_EVENTS
 
 
 Working set 6 model files, 1 .sdf file
-Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_STARS (schema)
-Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_EVENTS (schema)
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.OPENALEX_FUNDERS_INDEX (schema)
-Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.USPTO_PATENT_INDEX (schema)
-Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.USPTO_CONTRIBUTOR_INDEX (schema)
+Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_EVENTS (schema)
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.USPTO_PATENT_CONTRIBUTOR_RELATIONSHIPS (schema)
+Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.USPTO_PATENT_INDEX (schema)
+Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_STARS (schema)
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_REPOS (schema)
-  Compiling sdf_snowflake.staging.push_events (./models/sdf_snowflake/staging/push_events.sql)
+Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.USPTO_CONTRIBUTOR_INDEX (schema)
   Compiling sdf_snowflake.staging.latest_repo_name (./models/sdf_snowflake/staging/latest_repo_name.sql)
-  Compiling sdf_snowflake.cybersyn_tech_innovation.funder_aggregates (./models/sdf_snowflake/cybersyn_tech_innovation/funder_aggregates.sql)
-  Compiling sdf_snowflake.cybersyn_tech_innovation.all_nvidia_patents (./models/sdf_snowflake/cybersyn_tech_innovation/all_nvidia_patents.sql)
   Compiling sdf_snowflake.cybersyn_tech_innovation.most_starred_repos (./models/sdf_snowflake/cybersyn_tech_innovation/most_starred_repos.sql)
+  Compiling sdf_snowflake.cybersyn_tech_innovation.all_nvidia_patents (./models/sdf_snowflake/cybersyn_tech_innovation/all_nvidia_patents.sql)
+  Compiling sdf_snowflake.cybersyn_tech_innovation.funder_aggregates (./models/sdf_snowflake/cybersyn_tech_innovation/funder_aggregates.sql)
+  Compiling sdf_snowflake.staging.push_events (./models/sdf_snowflake/staging/push_events.sql)
   Compiling sdf_snowflake.cybersyn_tech_innovation.events_by_repo (./models/sdf_snowflake/cybersyn_tech_innovation/events_by_repo.sql)
-   Finished 13 models [6 succeeded, 7 downloaded] in 1.820 secs
+   Finished 13 models [13 succeeded] in 1.908 secs
 
 
 

diff --git a/docs/integrations/snowflake/incremental-materialization.mdx b/docs/integrations/snowflake/incremental-materialization.mdx
index 3e089002..2218db48 100644
--- a/docs/integrations/snowflake/incremental-materialization.mdx
+++ b/docs/integrations/snowflake/incremental-materialization.mdx
@@ -50,7 +50,7 @@ When installing the Cybersyn dataset, make sure to grant the Snowflake role you'
 Working set 3 model files, 1 .sdf file
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.OPENALEX_FUNDERS_INDEX (schema)
   Compiling sdf_snowflake.cybersyn_tech_innovation.funder_aggregates (./models/sdf_snowflake/cybersyn_tech_innovation/funder_aggregates.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 1.605 secs
+   Finished 2 models [2 succeeded] in 2.474 secs
 
 Schema sdf_snowflake.cybersyn_tech_innovation.FUNDER_AGGREGATES
 ┌──────────────────────┬────────────────┬────────────┬──────────────────────────────────────────────────────────────────┐
@@ -137,7 +137,7 @@ Working set 4 model files, 1 .sdf file
 Downloading SDF_SNOWFLAKE.STAGING.DATAFUSION_PUSH_EVENTS (exists_remotely)
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_EVENTS (schema)
   Compiling sdf_snowflake.staging.datafusion_push_events (./models/sdf_snowflake/staging/datafusion_push_events.sql)
-   Finished 2 models [1 succeeded, 1 downloaded] in 2.049 secs
+   Finished 2 models [2 succeeded] in 2.275 secs
 
 Schema sdf_snowflake.staging.DATAFUSION_PUSH_EVENTS
 ┌────────────────────────────────┬────────────────┬────────────┬────────────────────────────────────────────────────────────────────────────────────────────────┐
@@ -350,16 +350,16 @@ Downloading SDF_SNOWFLAKE.CYBERSYN_TECH_INNOVATION.TOP_DATAFUSION_CONTRIBUT
 Downloading TECH__INNOVATION_ESSENTIALS.CYBERSYN.GITHUB_EVENTS (schema)
   Compiling sdf_snowflake.staging.datafusion_push_events (./models/sdf_snowflake/staging/datafusion_push_events.sql)
   Compiling sdf_snowflake.cybersyn_tech_innovation.top_datafusion_contributors (./models/sdf_snowflake/cybersyn_tech_innovation/top_datafusion_contributors.sql)
-   Finished 3 models [2 succeeded, 1 downloaded] in 1.993 secs
+   Finished 3 models [3 succeeded] in 2.362 secs
 
 Schema sdf_snowflake.cybersyn_tech_innovation.TOP_DATAFUSION_CONTRIBUTORS
-┌─────────────────────┬────────────────┬────────────┬─────────────┐
-│ column_name         ┆ data_type      ┆ classifier ┆ description │
-╞═════════════════════╪════════════════╪════════════╪═════════════╡
-│ ACTOR_ID            ┆ decimal(38, 0) ┆            ┆             │
-│ ACTOR_DISPLAY_LOGIN ┆ varchar        ┆            ┆             │
-│ CONTRIBUTION_COUNT  ┆ decimal(38, 0) ┆            ┆             │
-└─────────────────────┴────────────────┴────────────┴─────────────┘
+┌─────────────────────┬────────────────┬────────────┬──────────────────────────────────────────────────────────────┐
+│ column_name         ┆ data_type      ┆ classifier ┆ description                                                  │
+╞═════════════════════╪════════════════╪════════════╪══════════════════════════════════════════════════════════════╡
+│ ACTOR_ID            ┆ decimal(38, 0) ┆            ┆ Unique identifier of the user or actor involved in the event │
+│ ACTOR_DISPLAY_LOGIN ┆ varchar        ┆            ┆ Display name of the user or actor involved in the event      │
+│ CONTRIBUTION_COUNT  ┆ decimal(38, 0) ┆            ┆                                                              │
+└─────────────────────┴────────────────┴────────────┴──────────────────────────────────────────────────────────────┘
diff --git a/docs/introduction/getting-started.mdx b/docs/introduction/getting-started.mdx index 09e2fea8..5c9f7f99 100644 --- a/docs/introduction/getting-started.mdx +++ b/docs/introduction/getting-started.mdx @@ -31,6 +31,8 @@ Commands:   new      Create a new sdf workspace   clean    Remove artifacts that sdf has generated in the past   compile  Compile models +  lint     Lint models +  format   Format models   run      Run models   test     Test your models   stats    Statistics for your data @@ -79,7 +81,7 @@ After running the command, you will see the following output:                  📚 Read the Docs to Get Started -> https://docs.sdf.com/          -   Finished new in 0.219 secs +   Finished new in 0.267 secs @@ -167,7 +169,7 @@ Let's see just how easy it is to set up SDF and run your first query. Working set 1 model file, 1 .sdf file   Compiling hello.pub.main (./models/main.sql) -   Finished 1 model [1 succeeded] in 0.731 secs +   Finished 1 model [1 succeeded] in 0.866 secs Schema hello.pub.main ┌─────────────┬──────────────────┬────────────┬─────────────┐ @@ -204,7 +206,7 @@ Schema hello.pub.main Working set 2 model files, 1 .sdf file   Compiling hello.pub.main (./models/main.sql)   Compiling hello.pub.main2 (./models/main2.sql) -   Finished 2 models [2 succeeded] in 0.747 secs +   Finished 2 models [2 succeeded] in 1.430 secs hello.pub.main2.message │ │ copy @@ -229,7 +231,7 @@ hello.pub.main2.message Working set 2 model files, 1 .sdf file     Running hello.pub.main (./models/main.sql)     Running hello.pub.main2 (./models/main2.sql) -   Finished 2 models [2 succeeded] in 0.759 secs +   Finished 2 models [2 succeeded] in 1.392 secs Table hello.pub.main ┌──────────────┐ diff --git a/docs/introduction/install.mdx b/docs/introduction/install.mdx index 94deda24..0d50cd9b 100644 --- a/docs/introduction/install.mdx +++ b/docs/introduction/install.mdx @@ -34,6 +34,8 @@ Commands:   new      Create a new sdf workspace   clean    Remove artifacts that sdf has generated in the past   compile  Compile models +  lint     Lint models +  format   Format models   run      Run models   test     Test your models   stats    Statistics for your data diff --git a/docs/linter/overview.mdx b/docs/linter/overview.mdx index eb92bd4c..fcbfd6da 100644 --- a/docs/linter/overview.mdx +++ b/docs/linter/overview.mdx @@ -105,7 +105,7 @@ If you specify your own YML linter configuration, you will **turn off** the defa ### Rules Reference For the full list of rules and configuration options, please visit our -[lint rules reference page](/linter/reference) +[lint rules reference page](/reference/lint-rules) | Type | Configuration | SDF Err | SQLFluff Err | Auto-Fix | Default diff --git a/docs/reference/error-codes.mdx b/docs/reference/error-codes.mdx index 2b7adae1..892d13c1 100644 --- a/docs/reference/error-codes.mdx +++ b/docs/reference/error-codes.mdx @@ -62,6 +62,8 @@ These codes are grouped into ranges. Error codes pertain **only** to SQL stateme | 1043 | BuildError | | 1044 | UnimplementedFunction | | 1045 | NoTableFoundForPrefix | +| 1046 | AmbiguousSourceSchema | +| 1047 | UnsupportedLogicalPlanForLocalExecution | | 1500 | MacroUnsupportedValueType | | 1501 | JinjaError | | 1502 | MacroSyntaxError | @@ -72,6 +74,7 @@ These codes are grouped into ranges. Error codes pertain **only** to SQL stateme | 2001 | UnsupportedLocalExecutionDialect | | 3000 | SltParse | | 3001 | SltLimits | +| 3002 | SltConfig | | 8998 | InvalidDialect | | 8999 | RuntimeError | | 8997 | InvalidUserInput | diff --git a/docs/reference/sdf-cli.mdx b/docs/reference/sdf-cli.mdx index 9d0fac7b..f9556b45 100644 --- a/docs/reference/sdf-cli.mdx +++ b/docs/reference/sdf-cli.mdx @@ -13,6 +13,8 @@ This document contains the help content for the `sdf` command-line program. * [`sdf new`↴](#sdf-new) * [`sdf clean`↴](#sdf-clean) * [`sdf compile`↴](#sdf-compile) +* [`sdf lint`↴](#sdf-lint) +* [`sdf format`↴](#sdf-format) * [`sdf run`↴](#sdf-run) * [`sdf test`↴](#sdf-test) * [`sdf stats`↴](#sdf-stats) @@ -58,6 +60,8 @@ SDF: A fast SQL compiler, local development framework, and in-memory analytical * `new` — Create a new sdf workspace * `clean` — Remove artifacts that sdf has generated in the past * `compile` — Compile models +* `lint` — Lint models +* `format` — Format models * `run` — Run models * `test` — Test your models * `stats` — Statistics for your data @@ -204,10 +208,168 @@ Available warnings: - `capitalization-aliases=upper|lower|pascal|snake|camel|consistent`: Set capitalization style for aliases. - `type-implicit-conversions`: Warn about implicit type conversions. - `references-consistent`: Warn about consistent references. +- `inconsistent-schema`: Warn about schema inconsistency between local and remote schema. Default: on. +All warnings are by default turned off, unless specified. +## `sdf lint` + +Lint models + +**Usage:** `sdf lint [OPTIONS] [TARGETS]...` + +###### **Arguments:** + +* ` TARGETS ` — Lint only the given source dirs, and files with .sql extension [default: all model .sql files in workspace] + +###### **Options:** + +* `-e`, `--environment ENVIRONMENT ` — Use this environment +* `-s`, `--show SHOW ` — Display messages [default: progress if TARGETS is empty, all otherwise] + + Possible values: `all`, `progress`, `result`, `none` + +* `-q`, `--query QUERY ` — Supply a .sql file or provide a sql snippet on the cmd line, e.g. 'select * from t' +* `--stage STAGE ` — Run the following stages [default: all stages] + + Possible values: `preprocess`, `parse`, `lint`, `resolve`, `classify`, `execute` + +* `--cache CACHE ` — Controls cache use + + Default value: `read-write` + + Possible values: `read-write`, `write-only`, `read-only`, `none` + +* `--save SAVE ` — Controls which assets to save [default: none] + + Possible values: `info-schema`, `assembly`, `table-deps` + +* `--targets-only` — Processes only specified targets assuming that all the non-target dependencies already exist + + Default value: `false` +* `--vars VARS ` — Supply var bindings as a yml file or provide them as string e.g. '(key: value)' +* `--env-vars ENV_VARS ` — Supply env var bindings as a yml file or provide them as string e.g. '(key: value)' +* `--downstream` — Execute cmd not only the given targets but also for all its downstream artifacts + + Default value: `false` +* `--format FORMAT ` — Show error tables in this format + + Default value: `table` + + Possible values: `table`, `csv`, `tsv`, `json`, `nd-json`, `yml` + +* `--limit LIMIT ` — Limiting number of shown rows. Run with --limit 0 to remove limit +* `--prefer-local` — When set will minimize remote DB accesses and will rely on local schema information in .sdf.yml files + + Default value: `false` +* `--fix` — Fix the linting issues + + Default value: `false` +* `-w`, `--warnings WARNINGS ` — +Specify the warnings to apply. Example: `-w warning1 -w warning2`. Later options override former ones. + Legend: (*) This warning can be automatically fixed by sdf. + +Available Warnings +- `all`: Turn on all warnings. +- `none`: Turn off all warnings. +- `error`: Treat warnings as errors. + +Capitalization Settings +- `capitalization-keywords=upper|lower|pascal|snake|camel|consistent`: Set capitalization style for keywords. Default: consistent (*) +- `capitalization-literals=upper|lower|pascal|snake|camel|consistent`: Set capitalization style for literals. Default: consistent (*) +- `capitalization-types=upper|lower|pascal|snake|camel|consistent`: Set capitalization style for types. Default: consistent (*) +- `capitalization-functions=upper|lower|pascal|snake|camel|consistent`: Set capitalization style for functions. Default: consistent (*) + +Convention and Reference Settings: +- `convention-blocked-words= word `: Specify blocked words. +- `convention-terminator`: Warn about terminator conventions. (*) +- `references-keywords`: Warn about keyword references. +- `references-special-chars= char `: Warn about special character references. +- `references-quoting`: Warn about quoting references. (*) +- `references-qualification`: Warn about qualification references. +- `references-columns`: Warn about ambiguous column references. + +Structure-Specific Warnings: +- `structure-else-null`: Warn about ELSE NULL in structures. (*) +- `structure-simple-case`: Warn about simple CASE structures. (*) +- `structure-unused-cte`: Warn about unused CTEs. (*) +- `structure-nested-case`: Warn about nested CASE structures. +- `structure-distinct`: Warn about DISTINCT usage. (*) +- `structure-subquery`: Warn about subquery structures. +- `structure-join-condition-order`: Warn about join condition order. +- `structure-column-order`: Warn about column order in structures. + +More warnings available as part of sdf compile ... + + + + +## `sdf format` + +Format models + +**Usage:** `sdf format [OPTIONS] [TARGETS]...` + +###### **Arguments:** + +* ` TARGETS ` — Format the given .sql files or directories [default: all .sql files from the workspace/environment] + +###### **Options:** + +* `-e`, `--environment ENVIRONMENT ` — Use this environment +* `-s`, `--show SHOW ` — Display messages [default: progress if TARGETS is empty, all otherwise] + + Possible values: `all`, `progress`, `result`, `none` + +* `-q`, `--query QUERY ` — Supply a .sql file or provide a sql snippet on the cmd line, e.g. 'select * from t' +* `--stage STAGE ` — Run the following stages [default: all stages] + + Possible values: `preprocess`, `parse`, `lint`, `resolve`, `classify`, `execute` + +* `--cache CACHE ` — Controls cache use + + Default value: `read-write` + + Possible values: `read-write`, `write-only`, `read-only`, `none` + +* `--save SAVE ` — Controls which assets to save [default: none] + + Possible values: `info-schema`, `assembly`, `table-deps` + +* `--targets-only` — Processes only specified targets assuming that all the non-target dependencies already exist + + Default value: `false` +* `--vars VARS ` — Supply var bindings as a yml file or provide them as string e.g. '(key: value)' +* `--env-vars ENV_VARS ` — Supply env var bindings as a yml file or provide them as string e.g. '(key: value)' +* `--downstream` — Execute cmd not only the given targets but also for all its downstream artifacts + + Default value: `false` +* `--format FORMAT ` — Show error tables in this format + + Default value: `table` + + Possible values: `table`, `csv`, `tsv`, `json`, `nd-json`, `yml` + +* `--limit LIMIT ` — Limiting number of shown rows. Run with --limit 0 to remove limit +* `--prefer-local` — When set will minimize remote DB accesses and will rely on local schema information in .sdf.yml files + + Default value: `false` +* `-l`, `--layout LAYOUT ` — +Specify the layout options. Example: `-l layout1 -l layout2` + +Available layout options: +* `indent= number `: Set the number of spaces to indent. +* `commas=leading|trailing`: Set the comma layout. +* `line-length= number `: Set the line length. + +* `--check` + + Default value: `false` + + + ## `sdf run` Run models diff --git a/docs/reference/sdf-yml.mdx b/docs/reference/sdf-yml.mdx index 50b48992..055a209e 100644 --- a/docs/reference/sdf-yml.mdx +++ b/docs/reference/sdf-yml.mdx @@ -24,6 +24,7 @@ title: "SDF YML Schema" `name:` | [`String`](#nested-element-string)| The name of the table (syntax: [[catalog.]schema].table)

Note: this field is typed as a [QualifiedName] for serialization. In almost all cases you should use the [Self::fqn()] method to get the fully qualified [TableName] instead of accessing this field directly.| `description:` | `string?`| A description of this table| `dialect:` | [`Dialect?`](#enum-dialect)| The dialect of this table, defaults to undefinedtrinoundefined| +`casing-policy:` | [`CasingPolicy?`](#enum-casingpolicy)| Case normalization policy for names specified in this table| `materialization:` | [`Materialization?`](#enum-materialization)| The table-type of this table (new version)| `warehouse:` | `string?`| The warehouse where this table is computed| `purpose:` | [`TablePurpose?`](#enum-tablepurpose)| Specify what kind of table or view this is| @@ -151,6 +152,7 @@ title: "SDF YML Schema" `vars:` | `object`| A map of named values for setting SQL variables from your environment Ex. -dt: dt, used in SQL as @dt, and in Jinja as | `source-locations:` | `array`| Workspace defined by these set of files| `preprocessor:` | [`PreprocessorType?`](#enum-preprocessortype)| Experimental: This project has jinja| +`dbt:` | [`DbtConfig?`](#nested-element-dbtconfig)| Configuration for dbt integration| ## Enum: `IncludeType` Value | Description| | :--- | :------- | @@ -174,27 +176,26 @@ Value | Description| `schema-table-name` | Table name is inferred from the file name without the extension. Schema is inferred from the directory name in which the file resides. Catalog is inferred from the `defaults` configuration. SDF assumes that a referenced table will reside in the directory/file with the corresponding schema/table name. As a result, only the required subset of files is parsed. This option is also fast, and it allows the schema to vary based on the directory name, but it still requires catalog to use the default values| `catalog-schema-table-name` | Table name is inferred from the file name without the extension. Schema is inferred from the directory name in which the file resides. Catalog is inferred from the directory name in which the schema directory resides. Defaults are not used. SDF assumes that a referenced table will reside in the catalog/schema/table file with the corresponding schema/catalog/table names. As a result, only the required subset of files is parsed. This option is also fast, and it provides the most flexibility as catalog, schema and table names can all vary| ## Enum: `Dialect` -Value | Description| -| :--- | :------- | -`trino` | undefined| -`snowflake` | Snowflake dialect with unnormalized column names - -When this dialect is specified, column names will be automatically converted to uppercase. Use this dialect if - *all* column names in your database are case-normalized to uppercase, and - you want the column names in your YAML files to be case-insensitive. - -If your table has lower-case or mixed-case column names, then this dialect will not work -- you *must* use [Dialect::Snowflake] instead. - -Note that the difference between this dialect and [Dialect::Snowflake] is limited to non-SQL sources *only* (e.g. `.sdf.yml` files, parquet/CSV files). For SQL files, these two dialects are identical. - -NOTE: the serialized form of this dialect is `snowflake`, for backward compatibility's sake:| -`snowflake_normalized` | Snowflake dialect with semantically accurate column names - -When this dialect is specified, column names will be left as-is. - -Users should be encouraged to use this dialect, as it is more accurate, with less potential for surprises. +Supported dialects in YAML files -Note that the difference between this dialect and [Dialect::Snowflake] is limited to non-SQL sources *only* (e.g. `.sdf.yml` files, parquet/CSV files). For SQL files, these two dialects are identical. +Note: this [Dialect] type is *only* meant for serializing to and from `.sdf.yml` files. For internal use, you should almost always use the semantic dialect type ([types::Dialect]) instead. -NOTE: the serialized form of this dialect is `snowflake_normalized`, for backward compatibility's sake:| +Note specifically that the lack of a `.to_string()` method (i.e. [!Display]) on this type is intentional -- you must first convert this type to a [types::Dialect], by using either [Dialect::to_semantic_dialect] or [types::Dialect::from], before you can convert it to a string.. +Values | +| :--- | +`snowflake` +`trino` +`bigquery` +`redshift` +`spark-lp` +`sdf` +`databricks` +## Enum: `CasingPolicy` +Values | +| :--- | +`preserve` +`to-upper` +`to-lower` ## Enum: `PreprocessorType` Values | | :--- | @@ -371,6 +372,7 @@ Values | | :---- | :--- | :------- | `environment:` | [`String?`](#nested-element-string)| The default environment (can only be set on the level of the workspace)| `dialect:` | [`Dialect?`](#enum-dialect)| The dialect of this environment. If not set, defaults to trino| +`casing-policy:` | [`CasingPolicy?`](#enum-casingpolicy)| Case normalization policy for names| `preprocessor:` | [`PreprocessorType?`](#enum-preprocessortype)| The preprocessor for this environment. If not set, defaults to local| `catalog:` | [`String?`](#nested-element-string)| Defines a default catalog. If not set, defaults to the (catalog/workspace) name in an outer scope| `schema:` | [`String?`](#nested-element-string)| Defines a default schema, If not set, defaults to the schema name in an outer scope, if not set, defaults to 'pub'| @@ -460,6 +462,7 @@ Values | `profile:` | [`String?`](#nested-element-string)| The dbt profile to use| `target:` | [`String?`](#nested-element-string)| The dbt target for the profile| `auto-parse:` | `boolean`| Automatically Run Parse in-between commands (default: true)| +`disable-introspection:` | `boolean`| Disable introspection (default: false)| ### Nested element: `IncrementalOptions` |Field | Type | Description| | :---- | :--- | :------- | diff --git a/docs/reference/snowflake/window_functions.mdx b/docs/reference/snowflake/window_functions.mdx index 09e483bf..3b417b62 100644 --- a/docs/reference/snowflake/window_functions.mdx +++ b/docs/reference/snowflake/window_functions.mdx @@ -46,6 +46,7 @@ Returns the first value of a column *Supported Signatures* ```sql function first_value($1) returns $1 +function first_value($1, integer) returns $1 ``` [🔗 Official Documentation](https://docs.snowflake.com/en/sql-reference/functions/first_value) @@ -66,6 +67,7 @@ Returns the last value of a column *Supported Signatures* ```sql function last_value($1) returns $1 +function last_value($1, integer) returns $1 ``` [🔗 Official Documentation](https://docs.snowflake.com/en/sql-reference/functions/last_value) diff --git a/docs/tutorials/creating-a-model.mdx b/docs/tutorials/creating-a-model.mdx index f89618ea..97025ec0 100644 --- a/docs/tutorials/creating-a-model.mdx +++ b/docs/tutorials/creating-a-model.mdx @@ -102,18 +102,18 @@ Init for run commands: 
 
 Working set 11 model files, 22 .sdf files
-  Compiling moms_flower_shop.raw.raw_addresses (./models/raw/raw_addresses.sql)
   Compiling moms_flower_shop.raw.raw_customers (./models/raw/raw_customers.sql)
-  Compiling moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
   Compiling moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql)
-  Compiling moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
+  Compiling moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
+  Compiling moms_flower_shop.raw.raw_addresses (./models/raw/raw_addresses.sql)
   Compiling moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)
+  Compiling moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
   Compiling moms_flower_shop.staging.app_installs (./models/staging/app_installs.sql)
   Compiling moms_flower_shop.staging.app_installs_v2 (./models/staging/app_installs_v2.sql)
   Compiling moms_flower_shop.analytics.agg_installs_and_campaigns (./models/analytics/agg_installs_and_campaigns.sql)
   Compiling moms_flower_shop.staging.customers (./models/staging/customers.sql)
   Compiling moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)
-   Finished 11 models [11 succeeded] in 0.810 secs
+   Finished 11 models [11 succeeded] in 0.915 secs
@@ -138,14 +138,14 @@ Working set 11 model files, 22 .sdf files 
 
 Working set 12 model files, 15 .sdf files
-    Running moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
     Running moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql)
+    Running moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
     Running moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)
     Running moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
     Running moms_flower_shop.staging.app_installs (./models/staging/app_installs.sql)
     Running moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)
     Running moms_flower_shop.analytics.dim_marketing_campaigns (./models/analytics/dim_marketing_campaigns.sql)
-   Finished 7 models [7 succeeded] in 0.874 secs
+   Finished 7 models [7 succeeded] in 0.964 secs
 
 Table moms_flower_shop.analytics.dim_marketing_campaigns
 ┌─────────────┬─────────────────────────────────┬────────────────────┬───────────────────────────────┬────────────────────────┐
@@ -159,8 +159,8 @@ Table moms_flower_shop.analytics.dim_marketing_campaigns
 │ 17          ┆ google_search_17_a400715565     ┆ 9                  ┆ 245.21392390373887            ┆ 1                      │
 │ 49          ┆ friends_referrals_49_16b0b159fa ┆ 9                  ┆ 288.469560173737              ┆ 2                      │
 │ 32          ┆ google_search_32_893397b50d     ┆ 8                  ┆ 369.19360069982207            ┆ 2                      │
+│ 4           ┆ friends_referrals_4_c554b77114  ┆ 8                  ┆ 347.85997737922236            ┆ 2                      │
 │ 103         ┆ instagram_ads_103_cd5421fbd9    ┆ 8                  ┆ 223.14798839074024            ┆ 1                      │
-│ 60          ┆ facebook_ads_60_c107c6b147      ┆ 8                  ┆ 345.1581886309037             ┆ 2                      │
 └─────────────┴─────────────────────────────────┴────────────────────┴───────────────────────────────┴────────────────────────┘
 114 rows, showing only 10 rows.
   Run with --limit 0 to show all rows.
diff --git a/docs/tutorials/debugging.mdx b/docs/tutorials/debugging.mdx
index b4a2fec1..c7b14923 100644
--- a/docs/tutorials/debugging.mdx
+++ b/docs/tutorials/debugging.mdx
@@ -153,7 +153,7 @@ moms_flower_shop.staging.customers.campaign_id
 Working set 12 model files, 15 .sdf files
   Compiling moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)
   Compiling moms_flower_shop.analytics.dim_marketing_campaigns (./models/analytics/dim_marketing_campaigns.sql)
-   Finished 11 models [2 succeeded, 9 reused] in 0.769 secs
+   Finished 11 models [2 succeeded, 9 reused] in 0.863 secs
@@ -206,12 +206,12 @@ moms_flower_shop.staging.stg_installs_per_campaign.total_num_installs Working set 12 model files, 15 .sdf files     Running moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)     Running moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql) -    Running moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)     Running moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql) +    Running moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)     Running moms_flower_shop.staging.app_installs_v2 (./models/staging/app_installs_v2.sql)     Running moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)     Running moms_flower_shop.analytics.dim_marketing_campaigns (./models/analytics/dim_marketing_campaigns.sql) -   Finished 7 models [7 succeeded] in 0.905 secs +   Finished 7 models [7 succeeded] in 0.985 secs Table moms_flower_shop.analytics.dim_marketing_campaigns ┌─────────────┬─────────────────────────────────┬────────────────────┬───────────────────────────────┬────────────────────────┐ diff --git a/docs/tutorials/deprecating-a-model.mdx b/docs/tutorials/deprecating-a-model.mdx index 254fc584..e81ea7d8 100644 --- a/docs/tutorials/deprecating-a-model.mdx +++ b/docs/tutorials/deprecating-a-model.mdx @@ -135,10 +135,10 @@ Schema moms_flower_shop.staging.app_installs ┌───────────────┬───────────┬────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ │ column_name   ┆ data_type ┆ classifier ┆ description                                                                                                                                  │ ╞═══════════════╪═══════════╪════════════╪══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ -│ event_id      ┆ bigint    ┆            ┆ A unique identifier of the marketing campaign event                                                                                          │ -│ customer_id   ┆ bigint    ┆            ┆                                                                                                                                              │ +│ event_id      ┆ bigint    ┆            ┆                                                                                                                                              │ +│ customer_id   ┆ bigint    ┆            ┆ The identifier of the customer that performed the event                                                                                      │ │ install_time  ┆ timestamp ┆            ┆                                                                                                                                              │ -│ platform      ┆ varchar   ┆            ┆                                                                                                                                              │ +│ platform      ┆ varchar   ┆            ┆ iOS or Android                                                                                                                               │ │ campaign_id   ┆ bigint    ┆            ┆ The identifier of the campaign that is associated with the event                                                                             │ │ campaign_name ┆ varchar   ┆            ┆ The campaign name associated with the campaign_id                                                                                            │ │ campaign_type ┆ varchar   ┆            ┆ A substring of the campaign name contain the campaign type.  Supported types - instagram_ads, friends_referrals, facebook_ads, google_search │ diff --git a/docs/tutorials/enriching-your-warehouse.mdx b/docs/tutorials/enriching-your-warehouse.mdx index 88739bf5..84de3987 100644 --- a/docs/tutorials/enriching-your-warehouse.mdx +++ b/docs/tutorials/enriching-your-warehouse.mdx @@ -132,15 +132,15 @@ Init for run commands: Schema moms_flower_shop.analytics.dim_marketing_campaigns -┌───────────────────────────────┬───────────┬─────────────┬─────────────┐ -│ column_name                   ┆ data_type ┆ classifier  ┆ description │ -╞═══════════════════════════════╪═══════════╪═════════════╪═════════════╡ -│ campaign_id                   ┆ bigint    ┆             ┆             │ -│ campaign_name                 ┆ varchar   ┆             ┆             │ -│ total_num_installs            ┆ bigint    ┆ EVENT.inapp ┆             │ -│ avg_customer_acquisition_cost ┆ double    ┆ EVENT.inapp ┆             │ -│ install_duration_ratio        ┆ bigint    ┆ EVENT.inapp ┆             │ -└───────────────────────────────┴───────────┴─────────────┴─────────────┘ +┌───────────────────────────────┬───────────┬─────────────┬──────────────────────────────────────────────────────────────────┐ +│ column_name                   ┆ data_type ┆ classifier  ┆ description                                                      │ +╞═══════════════════════════════╪═══════════╪═════════════╪══════════════════════════════════════════════════════════════════╡ +│ campaign_id                   ┆ bigint    ┆             ┆ The identifier of the campaign that is associated with the event │ +│ campaign_name                 ┆ varchar   ┆             ┆ The campaign name associated with the campaign_id                │ +│ total_num_installs            ┆ bigint    ┆ EVENT.inapp ┆                                                                  │ +│ avg_customer_acquisition_cost ┆ double    ┆ EVENT.inapp ┆                                                                  │ +│ install_duration_ratio        ┆ bigint    ┆ EVENT.inapp ┆                                                                  │ +└───────────────────────────────┴───────────┴─────────────┴──────────────────────────────────────────────────────────────────┘ @@ -162,10 +162,10 @@ Schema moms_flower_shop.staging.app_installs ┌───────────────┬───────────┬─────────────────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ │ column_name   ┆ data_type ┆ classifier                  ┆ description                                                                                                                                  │ ╞═══════════════╪═══════════╪═════════════════════════════╪══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ -│ event_id      ┆ bigint    ┆ EVENT.inapp EVENT.marketing ┆ A unique identifier of the marketing campaign event                                                                                          │ -│ customer_id   ┆ bigint    ┆                             ┆                                                                                                                                              │ +│ event_id      ┆ bigint    ┆ EVENT.inapp EVENT.marketing ┆                                                                                                                                              │ +│ customer_id   ┆ bigint    ┆                             ┆ The identifier of the customer that performed the event                                                                                      │ │ install_time  ┆ timestamp ┆                             ┆                                                                                                                                              │ -│ platform      ┆ varchar   ┆                             ┆                                                                                                                                              │ +│ platform      ┆ varchar   ┆                             ┆ iOS or Android                                                                                                                               │ │ campaign_id   ┆ bigint    ┆                             ┆ The identifier of the campaign that is associated with the event                                                                             │ │ campaign_name ┆ varchar   ┆                             ┆ The campaign name associated with the campaign_id                                                                                            │ │ campaign_type ┆ varchar   ┆                             ┆ A substring of the campaign name contain the campaign type.  Supported types - instagram_ads, friends_referrals, facebook_ads, google_search │ @@ -267,22 +267,23 @@ to monitor your warehouse's health and compliance. Schema moms_flower_shop.staging.customers -┌───────────────┬───────────┬─────────────┬────────────────────────────────────────────────────────────┐ -│ column_name   ┆ data_type ┆ classifier  ┆ description                                                │ -╞═══════════════╪═══════════╪═════════════╪════════════════════════════════════════════════════════════╡ -│ customer_id   ┆ bigint    ┆             ┆ A unique identifier of a mom's floewr shop customer        │ -│ first_name    ┆ varchar   ┆ PII.name    ┆ The first name of the customer                             │ -│ last_name     ┆ varchar   ┆ PII.name    ┆ The last name of the customer                              │ -│ full_name     ┆ varchar   ┆ PII.name    ┆                                                            │ -│ email         ┆ varchar   ┆ PII.email   ┆ The emial of the customer                                  │ -│ gender        ┆ varchar   ┆ PII.gender  ┆ The gender of the customer                                 │ -│ campaign_id   ┆ bigint    ┆             ┆                                                            │ -│ campaign_name ┆ varchar   ┆             ┆                                                            │ -│ campaign_type ┆ varchar   ┆             ┆                                                            │ -│ address_id    ┆ bigint    ┆ PII.address ┆ The address_id of the address associated with the customer │ -│ full_address  ┆ varchar   ┆             ┆ The full address associated with the address_id            │ -│ state         ┆ varchar   ┆             ┆ The address US state associated with the address_id        │ -└───────────────┴───────────┴─────────────┴────────────────────────────────────────────────────────────┘ +┌───────────────┬───────────┬─────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ column_name   ┆ data_type ┆ classifier  ┆ description                                                                                                                                  │ +╞═══════════════╪═══════════╪═════════════╪══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╡ +│ customer_id   ┆ bigint    ┆             ┆ A unique identifier of a mom's floewr shop customer                                                                                          │ +│ first_name    ┆ varchar   ┆ PII.name    ┆ The first name of the customer                                                                                                               │ +│ last_name     ┆ varchar   ┆ PII.name    ┆ The last name of the customer                                                                                                                │ +│ full_name     ┆ varchar   ┆ PII.name    ┆                                                                                                                                              │ +│ email         ┆ varchar   ┆ PII.email   ┆ The emial of the customer                                                                                                                    │ +│ gender        ┆ varchar   ┆ PII.gender  ┆ The gender of the customer                                                                                                                   │ +│ campaign_id   ┆ bigint    ┆             ┆ The identifier of the campaign that is associated with the event                                                                             │ +│ campaign_name ┆ varchar   ┆             ┆ The campaign name associated with the campaign_id                                                                                            │ +│ campaign_type ┆ varchar   ┆             ┆ A substring of the campaign name contain the campaign type.  Supported types - instagram_ads, friends_referrals, facebook_ads, google_search │ +│               ┆           ┆             ┆                                                                                                                                              │ +│ address_id    ┆ bigint    ┆ PII.address ┆ The address_id of the address associated with the customer                                                                                   │ +│ full_address  ┆ varchar   ┆             ┆ The full address associated with the address_id                                                                                              │ +│ state         ┆ varchar   ┆             ┆ The address US state associated with the address_id                                                                                          │ +└───────────────┴───────────┴─────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ @@ -375,15 +376,15 @@ Schema moms_flower_shop.raw.raw_addresses Schema moms_flower_shop.analytics.dim_marketing_campaigns        [RETENTION.infinity] -┌───────────────────────────────┬───────────┬─────────────┬─────────────┐ -│ column_name                   ┆ data_type ┆ classifier  ┆ description │ -╞═══════════════════════════════╪═══════════╪═════════════╪═════════════╡ -│ campaign_id                   ┆ bigint    ┆             ┆             │ -│ campaign_name                 ┆ varchar   ┆             ┆             │ -│ total_num_installs            ┆ bigint    ┆ EVENT.inapp ┆             │ -│ avg_customer_acquisition_cost ┆ double    ┆ EVENT.inapp ┆             │ -│ install_duration_ratio        ┆ bigint    ┆ EVENT.inapp ┆             │ -└───────────────────────────────┴───────────┴─────────────┴─────────────┘ +┌───────────────────────────────┬───────────┬─────────────┬──────────────────────────────────────────────────────────────────┐ +│ column_name                   ┆ data_type ┆ classifier  ┆ description                                                      │ +╞═══════════════════════════════╪═══════════╪═════════════╪══════════════════════════════════════════════════════════════════╡ +│ campaign_id                   ┆ bigint    ┆             ┆ The identifier of the campaign that is associated with the event │ +│ campaign_name                 ┆ varchar   ┆             ┆ The campaign name associated with the campaign_id                │ +│ total_num_installs            ┆ bigint    ┆ EVENT.inapp ┆                                                                  │ +│ avg_customer_acquisition_cost ┆ double    ┆ EVENT.inapp ┆                                                                  │ +│ install_duration_ratio        ┆ bigint    ┆ EVENT.inapp ┆                                                                  │ +└───────────────────────────────┴───────────┴─────────────┴──────────────────────────────────────────────────────────────────┘ diff --git a/docs/tutorials/script_test.mdx b/docs/tutorials/script_test.mdx index d9b7f8e1..ba973a7c 100644 --- a/docs/tutorials/script_test.mdx +++ b/docs/tutorials/script_test.mdx @@ -11,18 +11,18 @@ Not an error: Working set 12 model files, 24 .sdf files     Running moms_flower_shop.raw.raw_customers (./models/raw/raw_customers.sql) -    Running moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql)     Running moms_flower_shop.raw.raw_addresses (./models/raw/raw_addresses.sql)     Running moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql) -    Running moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql) +    Running moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql)     Running moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql) +    Running moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)     Running moms_flower_shop.staging.app_installs (./models/staging/app_installs.sql)     Running moms_flower_shop.staging.app_installs_v2 (./models/staging/app_installs_v2.sql) +    Running moms_flower_shop.staging.customers (./models/staging/customers.sql)     Running moms_flower_shop.analytics.agg_installs_and_campaigns (./models/analytics/agg_installs_and_campaigns.sql)     Running moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql) -    Running moms_flower_shop.staging.customers (./models/staging/customers.sql)     Running moms_flower_shop.analytics.dim_marketing_campaigns (./models/analytics/dim_marketing_campaigns.sql) -   Finished 12 models [12 succeeded] in 0.946 secs +   Finished 12 models [12 succeeded] in 1.038 secs @@ -33,15 +33,15 @@ Working set 12 model files, 24 .sdf files Schema moms_flower_shop.analytics.dim_marketing_campaigns -┌───────────────────────────────┬───────────┬────────────┬─────────────┐ -│ column_name                   ┆ data_type ┆ classifier ┆ description │ -╞═══════════════════════════════╪═══════════╪════════════╪═════════════╡ -│ campaign_id                   ┆ bigint    ┆            ┆             │ -│ campaign_name                 ┆ varchar   ┆            ┆             │ -│ total_num_installs            ┆ bigint    ┆            ┆             │ -│ avg_customer_acquisition_cost ┆ double    ┆            ┆             │ -│ install_duration_ratio        ┆ bigint    ┆            ┆             │ -└───────────────────────────────┴───────────┴────────────┴─────────────┘ +┌───────────────────────────────┬───────────┬────────────┬──────────────────────────────────────────────────────────────────┐ +│ column_name                   ┆ data_type ┆ classifier ┆ description                                                      │ +╞═══════════════════════════════╪═══════════╪════════════╪══════════════════════════════════════════════════════════════════╡ +│ campaign_id                   ┆ bigint    ┆            ┆ The identifier of the campaign that is associated with the event │ +│ campaign_name                 ┆ varchar   ┆            ┆ The campaign name associated with the campaign_id                │ +│ total_num_installs            ┆ bigint    ┆            ┆                                                                  │ +│ avg_customer_acquisition_cost ┆ double    ┆            ┆                                                                  │ +│ install_duration_ratio        ┆ bigint    ┆            ┆                                                                  │ +└───────────────────────────────┴───────────┴────────────┴──────────────────────────────────────────────────────────────────┘ diff --git a/docs/tutorials/tutorials-intro.mdx b/docs/tutorials/tutorials-intro.mdx index 11e02e48..3492b45f 100644 --- a/docs/tutorials/tutorials-intro.mdx +++ b/docs/tutorials/tutorials-intro.mdx @@ -225,14 +225,14 @@ Table moms_flower_shop.raw.raw_customers 
 
 Working set 11 model files, 22 .sdf files
-  Compiling moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
   Compiling moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)
+  Compiling moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
   Compiling moms_flower_shop.staging.app_installs (./models/staging/app_installs.sql)
   Compiling moms_flower_shop.staging.app_installs_v2 (./models/staging/app_installs_v2.sql)
   Compiling moms_flower_shop.analytics.agg_installs_and_campaigns (./models/analytics/agg_installs_and_campaigns.sql)
   Compiling moms_flower_shop.staging.customers (./models/staging/customers.sql)
   Compiling moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)
-   Finished 11 models [7 succeeded, 4 reused] in 0.851 secs
+   Finished 11 models [7 succeeded, 4 reused] in 0.960 secs
@@ -258,9 +258,9 @@ Working set 11 model files, 22 .sdf files 
 
 Working set 11 model files, 22 .sdf files
-  Compiling moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
-  Compiling moms_flower_shop.raw.raw_customers (./models/raw/raw_customers.sql)
   Compiling moms_flower_shop.raw.raw_marketing_campaign_events (./models/raw/raw_marketing_campaign_events.sql)
+  Compiling moms_flower_shop.raw.raw_customers (./models/raw/raw_customers.sql)
+  Compiling moms_flower_shop.raw.raw_inapp_events (./models/raw/raw_inapp_events.sql)
   Compiling moms_flower_shop.raw.raw_addresses (./models/raw/raw_addresses.sql)
   Compiling moms_flower_shop.staging.marketing_campaigns (./models/staging/marketing_campaigns.sql)
   Compiling moms_flower_shop.staging.inapp_events (./models/staging/inapp_events.sql)
@@ -269,7 +269,7 @@ Working set 11 model files, 22 .sdf files
   Compiling moms_flower_shop.analytics.agg_installs_and_campaigns (./models/analytics/agg_installs_and_campaigns.sql)
   Compiling moms_flower_shop.staging.customers (./models/staging/customers.sql)
   Compiling moms_flower_shop.staging.stg_installs_per_campaign (./models/staging/stg_installs_per_campaign.sql)
-   Finished 11 models [11 succeeded] in 0.877 secs
+   Finished 11 models [11 succeeded] in 0.937 secs