From 896c03344886f304e15f8f307c861bdb5da34140 Mon Sep 17 00:00:00 2001 From: Renee Li Date: Tue, 21 May 2024 16:31:09 -0400 Subject: [PATCH 01/15] initial --- dbt_project.yml | 2 +- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 4 ++-- .../int_zendesk__agent_work_time_business_hours.sql | 6 +++--- ...int_zendesk__requester_wait_time_business_hours.sql | 10 +++++----- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index a18ac5f2..68e4b1cc 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.16.0' +version: '0.16.1' config-version: 2 diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 7a390072..c8352954 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: zendesk_integration_tests_50 + schema: zendesk_integration_tests_51 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: zendesk_integration_tests_50 + schema: zendesk_integration_tests_51 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: zendesk_integration_tests_50 + schema: zendesk_integration_tests_51 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: zendesk_integration_tests_50 + schema: zendesk_integration_tests_51 threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: zendesk_integration_tests_50 + schema: zendesk_integration_tests_51 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 53532764..4ee4edfc 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,12 +1,12 @@ config-version: 2 name: 'zendesk_integration_tests' -version: '0.16.0' +version: '0.16.1' profile: 'integration_tests' vars: - zendesk_schema: zendesk_integration_tests_50 + zendesk_schema: zendesk_integration_tests_51 zendesk_source: zendesk_organization_identifier: "organization_data" zendesk_schedule_identifier: "schedule_data" diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql index 5abad813..21a777fe 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql @@ -75,7 +75,7 @@ with agent_work_time_filtered_statuses as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_full_solved_time as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks @@ -129,8 +129,8 @@ with agent_work_time_filtered_statuses as ( and weekly_period_agent_work_time.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date) ), intercepted_periods_with_running_total as ( diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql index 53b09dca..bfa24ba6 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql @@ -75,7 +75,7 @@ with requester_wait_time_filtered_statuses as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_full_solved_time as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks @@ -129,9 +129,9 @@ with requester_wait_time_filtered_statuses as ( and weekly_period_requester_wait_time.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) - + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date) + ), intercepted_periods_with_running_total as ( select @@ -172,7 +172,7 @@ with requester_wait_time_filtered_statuses as ( from intercepted_periods_agent_with_breach_flag ), requester_wait_business_breach as ( - + select *, {{ fivetran_utils.timestamp_add( From f3e89588731ff6deb3827d1be19a4295f1992602 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 23 Aug 2024 11:10:26 -0500 Subject: [PATCH 02/15] bug/missing-sla-policies --- ...int_zendesk__agent_work_time_business_hours.sql | 12 ++++++++---- ...zendesk__requester_wait_time_business_hours.sql | 14 +++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql index 5abad813..84489d0a 100644 --- a/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql +++ b/models/sla_policy/agent_work_time/int_zendesk__agent_work_time_business_hours.sql @@ -120,11 +120,15 @@ with agent_work_time_filtered_statuses as ( weekly_period_agent_work_time.week_number, weekly_period_agent_work_time.ticket_week_start_time_minute, weekly_period_agent_work_time.ticket_week_end_time_minute, - schedule.start_time_utc as schedule_start_time, + coalesce(schedule.start_time_utc, 0) as schedule_start_time, schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time_minute, schedule.end_time_utc) - greatest(weekly_period_agent_work_time.ticket_week_start_time_minute, schedule.start_time_utc) as scheduled_minutes + coalesce( + least(ticket_week_end_time_minute, schedule.end_time_utc) + - greatest(weekly_period_agent_work_time.ticket_week_start_time_minute, schedule.start_time_utc), + 0) as scheduled_minutes from weekly_period_agent_work_time - join schedule on ticket_week_start_time_minute <= schedule.end_time_utc + left join schedule + on ticket_week_start_time_minute <= schedule.end_time_utc and ticket_week_end_time_minute >= schedule.start_time_utc and weekly_period_agent_work_time.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule @@ -152,7 +156,7 @@ with agent_work_time_filtered_statuses as ( lag(target - running_total_scheduled_minutes) over (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) as lag_check, case when (target - running_total_scheduled_minutes) = 0 then true - when (target - running_total_scheduled_minutes) < 0 + when (target - running_total_scheduled_minutes) < 0 and (lag(target - running_total_scheduled_minutes) over (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) > 0 diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql index 53b09dca..01a24139 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql @@ -120,18 +120,22 @@ with requester_wait_time_filtered_statuses as ( weekly_period_requester_wait_time.week_number, weekly_period_requester_wait_time.ticket_week_start_time_minute, weekly_period_requester_wait_time.ticket_week_end_time_minute, - schedule.start_time_utc as schedule_start_time, + coalesce(schedule.start_time_utc, 0) as schedule_start_time, schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time_minute, schedule.end_time_utc) - greatest(weekly_period_requester_wait_time.ticket_week_start_time_minute, schedule.start_time_utc) as scheduled_minutes + coalesce( + least(ticket_week_end_time_minute, schedule.end_time_utc) + - greatest(weekly_period_requester_wait_time.ticket_week_start_time_minute, schedule.start_time_utc), + 0) as scheduled_minutes from weekly_period_requester_wait_time - join schedule on ticket_week_start_time_minute <= schedule.end_time_utc + left join schedule + on ticket_week_start_time_minute <= schedule.end_time_utc and ticket_week_end_time_minute >= schedule.start_time_utc and weekly_period_requester_wait_time.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time_minute', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) - + ), intercepted_periods_with_running_total as ( select @@ -152,7 +156,7 @@ with requester_wait_time_filtered_statuses as ( lag(target - running_total_scheduled_minutes) over (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) as lag_check, case when (target - running_total_scheduled_minutes) = 0 then true - when (target - running_total_scheduled_minutes) < 0 + when (target - running_total_scheduled_minutes) < 0 and (lag(target - running_total_scheduled_minutes) over (partition by ticket_id, sla_applied_at order by valid_starting_at, week_number, schedule_end_time) > 0 From 1eb68619cd79adb799f60e0be202bb940a243faa Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:25:51 -0500 Subject: [PATCH 03/15] update changelog and add integrity test --- .buildkite/pipeline.yml | 2 +- CHANGELOG.md | 10 ++++++ integration_tests/ci/sample.profiles.yml | 10 +++--- integration_tests/dbt_project.yml | 2 +- .../tests/integrity/sla_metrics_parity.sql | 36 +++++++++++++++++++ ...sk__requester_wait_time_business_hours.sql | 2 +- 6 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 integration_tests/tests/integrity/sla_metrics_parity.sql diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 985bfc37..281f1d42 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -58,7 +58,7 @@ steps: commands: | bash .buildkite/scripts/run_models.sh redshift - - label: ":bricks: Run Tests - Databricks" + - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" plugins: - docker#v3.13.0: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e56c6d2..21ab51cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# dbt_zendesk v0.16.1 + +## Bug Fixes +- Fixed an issue in the `zendesk__sla_policies` model where tickets that were opened and solved outside of scheduled hours were not being reported. + - Resolved by adjusting the join logic in `int_zendesk__agent_work_time_business_hours` and `int_zendesk__requester_wait_time_business_hours`. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164), [#156](https://github.com/fivetran/dbt_zendesk/pull/156)) + +## Under the hood +- Added integrity validation within integration tests to ensure `zendesk__sla_policies` and `zendesk__ticket_metrics` models produce consistent time results. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164)) +- Reduced the weeks looking ahead from 208 to 52 for performance. ([#156](https://github.com/fivetran/dbt_zendesk/pull/156)) + # dbt_zendesk v0.16.0 ## 🚨 Minor Upgrade 🚨 Although this update is not a breaking change, it will likely impact the output of the `zendesk__sla_policies` and `zendesk__sla_metrics` models. [PR #154](https://github.com/fivetran/dbt_zendesk/pull/154) includes the following changes: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index c8352954..7a390072 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: zendesk_integration_tests_51 + schema: zendesk_integration_tests_50 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: zendesk_integration_tests_51 + schema: zendesk_integration_tests_50 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: zendesk_integration_tests_51 + schema: zendesk_integration_tests_50 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: zendesk_integration_tests_51 + schema: zendesk_integration_tests_50 threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: zendesk_integration_tests_51 + schema: zendesk_integration_tests_50 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4ee4edfc..8d27ea87 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ version: '0.16.1' profile: 'integration_tests' vars: - zendesk_schema: zendesk_integration_tests_51 + zendesk_schema: zendesk_integration_tests_50 zendesk_source: zendesk_organization_identifier: "organization_data" zendesk_schedule_identifier: "schedule_data" diff --git a/integration_tests/tests/integrity/sla_metrics_parity.sql b/integration_tests/tests/integrity/sla_metrics_parity.sql new file mode 100644 index 00000000..850a48b0 --- /dev/null +++ b/integration_tests/tests/integrity/sla_metrics_parity.sql @@ -0,0 +1,36 @@ +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +/* +This test is to ensure the sla_elapsed_time from zendesk__sla_policies matches the corresponding time in zendesk__ticket_metrics. +*/ + +with dev_slas as ( + select * + from {{ target.schema }}_zendesk_dev.zendesk__sla_policies + where in_business_hours + +), dev_metrics as ( + select * + from {{ target.schema }}_zendesk_dev.zendesk__ticket_metrics + +), dev_compare as ( + select + dev_slas.ticket_id, + dev_slas.metric, + cast(dev_slas.sla_elapsed_time as {{ dbt.type_int() }}) as time_from_slas, + case when metric = 'agent_work_time' then dev_metrics.agent_work_time_in_business_minutes + when metric = 'requester_wait_time' then dev_metrics.requester_wait_time_in_business_minutes + when metric = 'first_reply_time' then dev_metrics.first_reply_time_business_minutes + end as time_from_metrics + from dev_slas + left join dev_metrics + on dev_metrics.ticket_id = dev_slas.ticket_id +) + +select * +from dev_compare +where abs(time_from_slas - time_from_metrics) >= 5 +and ticket_id not in {{ var('fivetran_integrity_sla_metric_parity_exclusion_tickets',()) }} \ No newline at end of file diff --git a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql index d449a11e..1b8656f9 100644 --- a/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql +++ b/models/sla_policy/requester_wait_time/int_zendesk__requester_wait_time_business_hours.sql @@ -176,7 +176,7 @@ with requester_wait_time_filtered_statuses as ( from intercepted_periods_agent_with_breach_flag ), requester_wait_business_breach as ( - + select *, {{ fivetran_utils.timestamp_add( From 4f682afca73e818d7cdd57b230a4ad6b63079be9 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:18:43 -0500 Subject: [PATCH 04/15] update test --- .../tests/integrity/sla_metrics_parity.sql | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/integration_tests/tests/integrity/sla_metrics_parity.sql b/integration_tests/tests/integrity/sla_metrics_parity.sql index 850a48b0..fc73b512 100644 --- a/integration_tests/tests/integrity/sla_metrics_parity.sql +++ b/integration_tests/tests/integrity/sla_metrics_parity.sql @@ -7,30 +7,30 @@ This test is to ensure the sla_elapsed_time from zendesk__sla_policies matches the corresponding time in zendesk__ticket_metrics. */ -with dev_slas as ( +with slas as ( select * from {{ target.schema }}_zendesk_dev.zendesk__sla_policies where in_business_hours -), dev_metrics as ( +), metrics as ( select * from {{ target.schema }}_zendesk_dev.zendesk__ticket_metrics -), dev_compare as ( +), comparison as ( select - dev_slas.ticket_id, - dev_slas.metric, - cast(dev_slas.sla_elapsed_time as {{ dbt.type_int() }}) as time_from_slas, - case when metric = 'agent_work_time' then dev_metrics.agent_work_time_in_business_minutes - when metric = 'requester_wait_time' then dev_metrics.requester_wait_time_in_business_minutes - when metric = 'first_reply_time' then dev_metrics.first_reply_time_business_minutes + slas.ticket_id, + slas.metric, + cast(slas.sla_elapsed_time as {{ dbt.type_int() }}) as time_from_slas, + case when slas.metric = 'agent_work_time' then metrics.agent_work_time_in_business_minutes + when slas.metric = 'requester_wait_time' then metrics.requester_wait_time_in_business_minutes + when slas.metric = 'first_reply_time' then metrics.first_reply_time_business_minutes end as time_from_metrics - from dev_slas - left join dev_metrics - on dev_metrics.ticket_id = dev_slas.ticket_id + from slas + left join metrics + on metrics.ticket_id = slas.ticket_id ) select * -from dev_compare +from comparison where abs(time_from_slas - time_from_metrics) >= 5 -and ticket_id not in {{ var('fivetran_integrity_sla_metric_parity_exclusion_tickets',()) }} \ No newline at end of file +and ticket_id not in {{ var('fivetran_integrity_sla_metric_parity_exclusion_tickets', ()) }} \ No newline at end of file From 0bf1d07d1c6fef94aff04f4a6ad16cafa1796ef0 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 27 Aug 2024 20:59:29 -0500 Subject: [PATCH 05/15] update changelog, readme and tests --- CHANGELOG.md | 2 +- README.md | 2 +- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- .../consistency/consistency_sla_policies.sql | 62 ++++++++++++------- .../consistency_sla_policy_count.sql | 2 + .../tests/integrity/sla_metrics_parity.sql | 28 ++++----- 7 files changed, 61 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ab51cd..45f7e5af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# dbt_zendesk v0.16.1 +# dbt_zendesk v0.17.0 ## Bug Fixes - Fixed an issue in the `zendesk__sla_policies` model where tickets that were opened and solved outside of scheduled hours were not being reported. diff --git a/README.md b/README.md index 8c0b9fdf..384a6c8d 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Include the following zendesk package version in your `packages.yml` file: ```yml packages: - package: fivetran/zendesk - version: [">=0.16.0", "<0.17.0"] + version: [">=0.17.0", "<0.18.0"] ``` > **Note**: Do not include the Zendesk Support source package. The Zendesk Support transform package already has a dependency on the source in its own `packages.yml` file. diff --git a/dbt_project.yml b/dbt_project.yml index 68e4b1cc..1439368e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'zendesk' -version: '0.16.1' +version: '0.17.0' config-version: 2 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8d27ea87..649995d9 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'zendesk_integration_tests' -version: '0.16.1' +version: '0.17.0' profile: 'integration_tests' diff --git a/integration_tests/tests/consistency/consistency_sla_policies.sql b/integration_tests/tests/consistency/consistency_sla_policies.sql index 6cab6333..8e27d00a 100644 --- a/integration_tests/tests/consistency/consistency_sla_policies.sql +++ b/integration_tests/tests/consistency/consistency_sla_policies.sql @@ -5,11 +5,16 @@ ) }} with prod as ( - select + select ticket_id, - metric, + sla_policy_name, + metric, sla_applied_at, - sla_elapsed_time, + target, + in_business_hours, + sla_breach_at, + round(sla_elapsed_time, -1) as sla_elapsed_time, --round to the nearest tens + is_active_sla, is_sla_breach from {{ target.schema }}_zendesk_prod.zendesk__sla_policies ), @@ -17,31 +22,46 @@ with prod as ( dev as ( select ticket_id, - metric, + sla_policy_name, + metric, sla_applied_at, - sla_elapsed_time, + target, + in_business_hours, + sla_breach_at, + round(sla_elapsed_time, -1) as sla_elapsed_time, --round to the nearest tens + is_active_sla, is_sla_breach from {{ target.schema }}_zendesk_dev.zendesk__sla_policies ), +prod_not_in_dev as ( + -- rows from prod not found in dev + select * from prod + except distinct + select * from dev +), + +dev_not_in_prod as ( + -- rows from dev not found in prod + select * from dev + except distinct + select * from prod +), + final as ( - select - prod.ticket_id, - prod.metric, - prod.sla_applied_at, - prod.sla_elapsed_time as prod_sla_elapsed_time, - dev.sla_elapsed_time as dev_sla_elapsed_time, - prod.is_sla_breach as prod_is_sla_breach, - dev.is_sla_breach as dev_is_sla_breach - from prod - full outer join dev - on dev.ticket_id = prod.ticket_id - and dev.metric = prod.metric - and dev.sla_applied_at = prod.sla_applied_at + select + *, + 'from prod' as source + from prod_not_in_dev + + union all -- union since we only care if rows are produced + + select + *, + 'from dev' as source + from dev_not_in_prod ) select * from final -where (abs(prod_sla_elapsed_time - dev_sla_elapsed_time) >= 5 - or prod_is_sla_breach != dev_is_sla_breach) - {{ "and prod.ticket_id not in " ~ var('fivetran_consistency_sla_policies_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policies_exclusion_tickets',[]) }} \ No newline at end of file +{{ "where ticket_id not in " ~ var('fivetran_consistency_sla_policies_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policies_exclusion_tickets',[]) }} \ No newline at end of file diff --git a/integration_tests/tests/consistency/consistency_sla_policy_count.sql b/integration_tests/tests/consistency/consistency_sla_policy_count.sql index 65e31d7f..d5b9e2ab 100644 --- a/integration_tests/tests/consistency/consistency_sla_policy_count.sql +++ b/integration_tests/tests/consistency/consistency_sla_policy_count.sql @@ -9,6 +9,7 @@ with prod as ( 1 as join_key, count(*) as total_slas from {{ target.schema }}_zendesk_prod.zendesk__sla_policies + where prod.ticket_id not in {{ var('fivetran_consistency_sla_policy_count_exclusion_tickets',()) }} group by 1 ), @@ -17,6 +18,7 @@ dev as ( 1 as join_key, count(*) as total_slas from {{ target.schema }}_zendesk_dev.zendesk__sla_policies + where dev.ticket_id not in {{ var('fivetran_consistency_sla_policy_count_exclusion_tickets',()) }} group by 1 ), diff --git a/integration_tests/tests/integrity/sla_metrics_parity.sql b/integration_tests/tests/integrity/sla_metrics_parity.sql index fc73b512..76f20a1d 100644 --- a/integration_tests/tests/integrity/sla_metrics_parity.sql +++ b/integration_tests/tests/integrity/sla_metrics_parity.sql @@ -7,30 +7,30 @@ This test is to ensure the sla_elapsed_time from zendesk__sla_policies matches the corresponding time in zendesk__ticket_metrics. */ -with slas as ( +with dev_slas as ( select * from {{ target.schema }}_zendesk_dev.zendesk__sla_policies where in_business_hours -), metrics as ( +), dev_metrics as ( select * from {{ target.schema }}_zendesk_dev.zendesk__ticket_metrics -), comparison as ( +), dev_compare as ( select - slas.ticket_id, - slas.metric, - cast(slas.sla_elapsed_time as {{ dbt.type_int() }}) as time_from_slas, - case when slas.metric = 'agent_work_time' then metrics.agent_work_time_in_business_minutes - when slas.metric = 'requester_wait_time' then metrics.requester_wait_time_in_business_minutes - when slas.metric = 'first_reply_time' then metrics.first_reply_time_business_minutes + dev_slas.ticket_id, + dev_slas.metric, + cast(dev_slas.sla_elapsed_time as {{ dbt.type_int() }}) as time_from_slas, + case when metric = 'agent_work_time' then dev_metrics.agent_work_time_in_business_minutes + when metric = 'requester_wait_time' then dev_metrics.requester_wait_time_in_business_minutes + when metric = 'first_reply_time' then dev_metrics.first_reply_time_business_minutes end as time_from_metrics - from slas - left join metrics - on metrics.ticket_id = slas.ticket_id + from dev_slas + left join dev_metrics + on dev_metrics.ticket_id = dev_slas.ticket_id ) select * -from comparison +from dev_compare where abs(time_from_slas - time_from_metrics) >= 5 -and ticket_id not in {{ var('fivetran_integrity_sla_metric_parity_exclusion_tickets', ()) }} \ No newline at end of file +{{ "and ticket_id not in " ~ var('fivetran_integrity_sla_metric_parity_exclusion_tickets',[]) ~ "" if var('fivetran_integrity_sla_metric_parity_exclusion_tickets',[]) }} \ No newline at end of file From 2768caf8005c73d37b8a657cb871dcc6d091b07d Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 27 Aug 2024 21:01:46 -0500 Subject: [PATCH 06/15] update test --- .../consistency/consistency_sla_policy_count.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integration_tests/tests/consistency/consistency_sla_policy_count.sql b/integration_tests/tests/consistency/consistency_sla_policy_count.sql index d5b9e2ab..8937ac38 100644 --- a/integration_tests/tests/consistency/consistency_sla_policy_count.sql +++ b/integration_tests/tests/consistency/consistency_sla_policy_count.sql @@ -6,32 +6,32 @@ with prod as ( select - 1 as join_key, + ticket_id, count(*) as total_slas from {{ target.schema }}_zendesk_prod.zendesk__sla_policies - where prod.ticket_id not in {{ var('fivetran_consistency_sla_policy_count_exclusion_tickets',()) }} + {{ "where ticket_id not in " ~ var('fivetran_consistency_sla_policy_count_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policy_count_exclusion_tickets',[]) }} group by 1 ), dev as ( select - 1 as join_key, + ticket_id, count(*) as total_slas from {{ target.schema }}_zendesk_dev.zendesk__sla_policies - where dev.ticket_id not in {{ var('fivetran_consistency_sla_policy_count_exclusion_tickets',()) }} + {{ "where ticket_id not in " ~ var('fivetran_consistency_sla_policy_count_exclusion_tickets',[]) ~ "" if var('fivetran_consistency_sla_policy_count_exclusion_tickets',[]) }} group by 1 ), final as ( select - prod.join_key, + prod.ticket_id, prod.total_slas as prod_sla_total, dev.total_slas as dev_sla_total from prod full outer join dev - on dev.join_key = prod.join_key + on dev.ticket_id = prod.ticket_id ) select * from final -where prod_sla_total != dev_sla_total +where prod_sla_total != dev_sla_total \ No newline at end of file From 98fd26baa5d5c913ec16c2714f88870325713775 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 28 Aug 2024 22:26:27 -0500 Subject: [PATCH 07/15] bug/intercepted-period-joins --- ...int_zendesk__ticket_work_time_business.sql | 22 +++++++++---------- ...desk__ticket_first_reply_time_business.sql | 19 ++++++++-------- ..._ticket_first_resolution_time_business.sql | 22 +++++++++---------- ...__ticket_full_resolution_time_business.sql | 8 +++---- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql index 01c0ae18..b897cfe6 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql @@ -99,14 +99,14 @@ with ticket_historical_status as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(weekly_periods.ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule on - ticket_week_start_time <= schedule.end_time_utc + left join schedule + on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date) ), business_minutes as ( @@ -133,12 +133,12 @@ with ticket_historical_status as ( select ticket_id, - sum(agent_wait_time_in_minutes) as agent_wait_time_in_business_minutes, - sum(requester_wait_time_in_minutes) as requester_wait_time_in_business_minutes, - sum(solve_time_in_minutes) as solve_time_in_business_minutes, - sum(agent_work_time_in_minutes) as agent_work_time_in_business_minutes, - sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes, - sum(new_status_duration_minutes) as new_status_duration_in_business_minutes, - sum(open_status_duration_minutes) as open_status_duration_in_business_minutes + coalesce(sum(agent_wait_time_in_minutes), 0) as agent_wait_time_in_business_minutes, + coalesce(sum(requester_wait_time_in_minutes), 0) as requester_wait_time_in_business_minutes, + coalesce(sum(solve_time_in_minutes), 0) as solve_time_in_business_minutes, + coalesce(sum(agent_work_time_in_minutes), 0) as agent_work_time_in_business_minutes, + coalesce(sum(on_hold_time_in_minutes), 0) as on_hold_time_in_business_minutes, + coalesce(sum(new_status_duration_minutes), 0) as new_status_duration_in_business_minutes, + coalesce(sum(open_status_duration_minutes), 0) as open_status_duration_in_business_minutes from business_minutes group by 1 diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index 8ab35ea6..355101d1 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -83,15 +83,16 @@ with ticket_reply_times as ( ), intercepted_periods as ( select ticket_id, - week_number, - weekly_periods.schedule_id, - ticket_week_start_time, - ticket_week_end_time, - schedule.start_time_utc as schedule_start_time, - schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes + week_number, + weekly_periods.schedule_id, + ticket_week_start_time, + ticket_week_end_time, + schedule.start_time_utc as schedule_start_time, + schedule.end_time_utc as schedule_end_time, + least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule on ticket_week_start_time <= schedule.end_time_utc + left join schedule + on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule @@ -102,6 +103,6 @@ with ticket_reply_times as ( ) select ticket_id, - sum(scheduled_minutes) as first_reply_time_business_minutes + coalesce(sum(scheduled_minutes), 0) as first_reply_time_business_minutes from intercepted_periods group by 1 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 90a3fb13..63397bb3 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -73,26 +73,26 @@ with ticket_resolution_times_calendar as ( ), intercepted_periods as ( select ticket_id, - week_number, - weekly_periods.schedule_id, - ticket_week_start_time, - ticket_week_end_time, - schedule.start_time_utc as schedule_start_time, - schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes + week_number, + weekly_periods.schedule_id, + ticket_week_start_time, + ticket_week_end_time, + schedule.start_time_utc as schedule_start_time, + schedule.end_time_utc as schedule_end_time, + least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule on ticket_week_start_time <= schedule.end_time_utc + left join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date) ) select ticket_id, - sum(scheduled_minutes) as first_resolution_business_minutes + coalesce(sum(scheduled_minutes), 0) as first_resolution_business_minutes from intercepted_periods group by 1 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql index 414ab654..2e370100 100644 --- a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql @@ -80,18 +80,18 @@ with ticket_resolution_times_calendar as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule on ticket_week_start_time <= schedule.end_time_utc + left join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }}) - and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as date) > cast(schedule.valid_from as date) + and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as date) < cast(schedule.valid_until as date) ) select ticket_id, - sum(scheduled_minutes) as full_resolution_business_minutes + coalesce(sum(scheduled_minutes), 0) as full_resolution_business_minutes from intercepted_periods group by 1 \ No newline at end of file From fea4e7e7c0d4071ca07d6fa826a159e49276f8e0 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Wed, 28 Aug 2024 23:37:04 -0500 Subject: [PATCH 08/15] adjustmnt --- .../int_zendesk__ticket_work_time_business.sql | 16 ++++++++-------- ...zendesk__ticket_first_reply_time_business.sql | 4 ++-- ...sk__ticket_first_resolution_time_business.sql | 4 ++-- ...esk__ticket_full_resolution_time_business.sql | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql index b897cfe6..dddfb9af 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql @@ -99,7 +99,7 @@ with ticket_historical_status as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(weekly_periods.ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - left join schedule + join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id @@ -133,12 +133,12 @@ with ticket_historical_status as ( select ticket_id, - coalesce(sum(agent_wait_time_in_minutes), 0) as agent_wait_time_in_business_minutes, - coalesce(sum(requester_wait_time_in_minutes), 0) as requester_wait_time_in_business_minutes, - coalesce(sum(solve_time_in_minutes), 0) as solve_time_in_business_minutes, - coalesce(sum(agent_work_time_in_minutes), 0) as agent_work_time_in_business_minutes, - coalesce(sum(on_hold_time_in_minutes), 0) as on_hold_time_in_business_minutes, - coalesce(sum(new_status_duration_minutes), 0) as new_status_duration_in_business_minutes, - coalesce(sum(open_status_duration_minutes), 0) as open_status_duration_in_business_minutes + sum(agent_wait_time_in_minutes) as agent_wait_time_in_business_minutes, + sum(requester_wait_time_in_minutes) as requester_wait_time_in_business_minutes, + sum(solve_time_in_minutes) as solve_time_in_business_minutes, + sum(agent_work_time_in_minutes) as agent_work_time_in_business_minutes, + sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes, + sum(new_status_duration_minutes) as new_status_duration_in_business_minutes, + sum(open_status_duration_minutes) as open_status_duration_in_business_minutes from business_minutes group by 1 diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index 355101d1..20ab156d 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -91,7 +91,7 @@ with ticket_reply_times as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - left join schedule + join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id @@ -103,6 +103,6 @@ with ticket_reply_times as ( ) select ticket_id, - coalesce(sum(scheduled_minutes), 0) as first_reply_time_business_minutes + sum(scheduled_minutes) as first_reply_time_business_minutes from intercepted_periods group by 1 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 63397bb3..316dc727 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -81,7 +81,7 @@ with ticket_resolution_times_calendar as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - left join schedule on ticket_week_start_time <= schedule.end_time_utc + join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule @@ -93,6 +93,6 @@ with ticket_resolution_times_calendar as ( select ticket_id, - coalesce(sum(scheduled_minutes), 0) as first_resolution_business_minutes + sum(scheduled_minutes) as first_resolution_business_minutes from intercepted_periods group by 1 \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql index 2e370100..628c00b2 100644 --- a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql @@ -80,7 +80,7 @@ with ticket_resolution_times_calendar as ( schedule.end_time_utc as schedule_end_time, least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - left join schedule on ticket_week_start_time <= schedule.end_time_utc + join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule @@ -92,6 +92,6 @@ with ticket_resolution_times_calendar as ( select ticket_id, - coalesce(sum(scheduled_minutes), 0) as full_resolution_business_minutes + sum(scheduled_minutes) as full_resolution_business_minutes from intercepted_periods group by 1 \ No newline at end of file From 0eee4046a02b9d27afc8e0888d9488661f2b0d5a Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:52:01 -0500 Subject: [PATCH 09/15] update weeks --- .../int_zendesk__ticket_first_reply_time_business.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index 20ab156d..8343ce9b 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -57,7 +57,7 @@ with ticket_reply_times as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_first_reply as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks From 73daccb50f5a8943b604b3982766de936712afd5 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:52:49 -0500 Subject: [PATCH 10/15] update weeks --- ...desk__ticket_first_reply_time_business.sql | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql index 8343ce9b..f148ca06 100644 --- a/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql +++ b/models/reply_times/int_zendesk__ticket_first_reply_time_business.sql @@ -83,16 +83,15 @@ with ticket_reply_times as ( ), intercepted_periods as ( select ticket_id, - week_number, - weekly_periods.schedule_id, - ticket_week_start_time, - ticket_week_end_time, - schedule.start_time_utc as schedule_start_time, - schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes + week_number, + weekly_periods.schedule_id, + ticket_week_start_time, + ticket_week_end_time, + schedule.start_time_utc as schedule_start_time, + schedule.end_time_utc as schedule_end_time, + least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule - on ticket_week_start_time <= schedule.end_time_utc + join schedule on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule @@ -103,6 +102,6 @@ with ticket_reply_times as ( ) select ticket_id, - sum(scheduled_minutes) as first_reply_time_business_minutes + sum(scheduled_minutes) as first_reply_time_business_minutes from intercepted_periods group by 1 \ No newline at end of file From 7ac92f5a11418f65feff8eda1ee2a6c23bab65ff Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:56:11 -0500 Subject: [PATCH 11/15] add integrity test --- .../tests/integrity/metrics_count_match.sql | 24 +++++++++++++++++++ ..._ticket_first_resolution_time_business.sql | 20 +++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 integration_tests/tests/integrity/metrics_count_match.sql diff --git a/integration_tests/tests/integrity/metrics_count_match.sql b/integration_tests/tests/integrity/metrics_count_match.sql new file mode 100644 index 00000000..715176c4 --- /dev/null +++ b/integration_tests/tests/integrity/metrics_count_match.sql @@ -0,0 +1,24 @@ + +{{ config( + tags="fivetran_validations", + enabled=var('fivetran_validation_tests_enabled', false) +) }} + +-- check that all the tickets are accounted for in the metrics +with stg_count as ( + select + count(*) as stg_ticket_count + from {{ ref('stg_zendesk__ticket') }} +), + +metric_count as ( + select + count(*) as metric_ticket_count + from source + from {{ ref('zendesk__ticket_metrics') }} +) + +select * +from stg_count +join metric_count + on stg_ticket_count != metric_ticket_count \ No newline at end of file diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 316dc727..5d95d6b5 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -72,16 +72,18 @@ with ticket_resolution_times_calendar as ( ), intercepted_periods as ( - select ticket_id, - week_number, - weekly_periods.schedule_id, - ticket_week_start_time, - ticket_week_end_time, - schedule.start_time_utc as schedule_start_time, - schedule.end_time_utc as schedule_end_time, - least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes + select + ticket_id, + week_number, + weekly_periods.schedule_id, + ticket_week_start_time, + ticket_week_end_time, + schedule.start_time_utc as schedule_start_time, + schedule.end_time_utc as schedule_end_time, + least(ticket_week_end_time, schedule.end_time_utc) - greatest(ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes from weekly_periods - join schedule on ticket_week_start_time <= schedule.end_time_utc + join schedule + on ticket_week_start_time <= schedule.end_time_utc and ticket_week_end_time >= schedule.start_time_utc and weekly_periods.schedule_id = schedule.schedule_id -- this chooses the Daylight Savings Time or Standard Time version of the schedule From 4831ab95961bd253cf42f44da326cf37eac915c8 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:57:51 -0500 Subject: [PATCH 12/15] update weeks --- .../agent_work_time/int_zendesk__ticket_work_time_business.sql | 2 +- .../int_zendesk__ticket_first_resolution_time_business.sql | 2 +- .../int_zendesk__ticket_full_resolution_time_business.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql index dddfb9af..7725165b 100644 --- a/models/agent_work_time/int_zendesk__ticket_work_time_business.sql +++ b/models/agent_work_time/int_zendesk__ticket_work_time_business.sql @@ -63,7 +63,7 @@ with ticket_historical_status as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_full_solved_time as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks diff --git a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql index 5d95d6b5..f61e69b7 100644 --- a/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_first_resolution_time_business.sql @@ -46,7 +46,7 @@ with ticket_resolution_times_calendar as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_first_resolution_time as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks diff --git a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql index 628c00b2..fe4df912 100644 --- a/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql +++ b/models/resolution_times/int_zendesk__ticket_full_resolution_time_business.sql @@ -45,7 +45,7 @@ with ticket_resolution_times_calendar as ( ), weeks as ( - {{ dbt_utils.generate_series(208) }} + {{ dbt_utils.generate_series(52) }} ), weeks_cross_ticket_full_resolution_time as ( -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks From a8b4190286cfd9c9fd86e24d5d4e609ab9214c0a Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:28:05 -0500 Subject: [PATCH 13/15] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f7e5af..ac527d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## Bug Fixes - Fixed an issue in the `zendesk__sla_policies` model where tickets that were opened and solved outside of scheduled hours were not being reported. - - Resolved by adjusting the join logic in `int_zendesk__agent_work_time_business_hours` and `int_zendesk__requester_wait_time_business_hours`. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164), [#156](https://github.com/fivetran/dbt_zendesk/pull/156)) + - Resolved by adjusting the join logic in models `int_zendesk__agent_work_time_business_hours` and `int_zendesk__requester_wait_time_business_hours`. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164), [#156](https://github.com/fivetran/dbt_zendesk/pull/156)) +- Fixed as issue in the `zendesk__ticket_metrics` model where certain tickets had miscalculated metrics. + - Resolved by adjusting the join logic in models `int_zendesk__ticket_work_time_business`, `int_zendesk__ticket_first_resolution_time_business`, and `int_zendesk__ticket_full_resolution_time_business`. ([#167](https://github.com/fivetran/dbt_zendesk/pull/167)) ## Under the hood - Added integrity validation within integration tests to ensure `zendesk__sla_policies` and `zendesk__ticket_metrics` models produce consistent time results. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164)) From 2a202afcf1f34dce83910f549a9088bee4413007 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:28:17 -0500 Subject: [PATCH 14/15] update changelog --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac527d37..25508c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,10 @@ - Resolved by adjusting the join logic in models `int_zendesk__ticket_work_time_business`, `int_zendesk__ticket_first_resolution_time_business`, and `int_zendesk__ticket_full_resolution_time_business`. ([#167](https://github.com/fivetran/dbt_zendesk/pull/167)) ## Under the hood -- Added integrity validation within integration tests to ensure `zendesk__sla_policies` and `zendesk__ticket_metrics` models produce consistent time results. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164)) -- Reduced the weeks looking ahead from 208 to 52 for performance. ([#156](https://github.com/fivetran/dbt_zendesk/pull/156)) +- Added integrity validations: + - Test to ensure `zendesk__sla_policies` and `zendesk__ticket_metrics` models produce consistent time results. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164)) + - Test to ensure `zendesk__ticket_metrics` contains all the tickets found in `stg_zendesk__ticket`. +- Reduced the weeks looking ahead from 208 to 52 for performance. ([#156](https://github.com/fivetran/dbt_zendesk/pull/156), [#167](https://github.com/fivetran/dbt_zendesk/pull/167)) # dbt_zendesk v0.16.0 ## 🚨 Minor Upgrade 🚨 From 0c382e5b750c5360d07f540d7e45bad64f90af9c Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:56:57 -0500 Subject: [PATCH 15/15] update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26a654fa..3bd3d308 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - `zendesk__ticket_field_history` ## Bug Fixes -- Fixed an issue in the `zendesk__sla_policies` model where tickets that were opened and solved outside of scheduled hours were not being reported. +- Fixed an issue in the `zendesk__sla_policies` model where tickets that were opened and solved outside of scheduled hours were not being reported, specifically for the metrics `requester_wait_time` and `agent_work_time`. - Resolved by adjusting the join logic in models `int_zendesk__agent_work_time_business_hours` and `int_zendesk__requester_wait_time_business_hours`. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164), [#156](https://github.com/fivetran/dbt_zendesk/pull/156)) - Fixed as issue in the `zendesk__ticket_metrics` model where certain tickets had miscalculated metrics. - Resolved by adjusting the join logic in models `int_zendesk__ticket_work_time_business`, `int_zendesk__ticket_first_resolution_time_business`, and `int_zendesk__ticket_full_resolution_time_business`. ([#167](https://github.com/fivetran/dbt_zendesk/pull/167)) @@ -17,7 +17,7 @@ - Test to ensure `zendesk__sla_policies` and `zendesk__ticket_metrics` models produce consistent time results. ([#164](https://github.com/fivetran/dbt_zendesk/pull/164)) - Test to ensure `zendesk__ticket_metrics` contains all the tickets found in `stg_zendesk__ticket`. - Modified the `consistency_sla_policy_count` validation test to group by `ticket_id` for more accurate testing. ([#165](https://github.com/fivetran/dbt_zendesk/pull/165)) -- Reduced the weeks looking ahead from 208 to 52 for performance. ([#156](https://github.com/fivetran/dbt_zendesk/pull/156), [#167](https://github.com/fivetran/dbt_zendesk/pull/167)) +- Reduced the weeks looking ahead from 208 to 52 to improve performance, as tracking ticket SLAs beyond one year was unnecessary. ([#156](https://github.com/fivetran/dbt_zendesk/pull/156), [#167](https://github.com/fivetran/dbt_zendesk/pull/167)) - Updated seed files to reflect a real world ticket field history update scenario. ([#165](https://github.com/fivetran/dbt_zendesk/pull/165)) # dbt_zendesk v0.16.0