From b5e9ee53ab9235d7e3efcc6179f8f68bd3e3c892 Mon Sep 17 00:00:00 2001 From: Rory Sawyer Date: Thu, 3 Aug 2023 15:09:43 -0400 Subject: [PATCH 1/3] feat: use dbt models for problem datasets Converts `fact_problem_responses` and `fact_learner_problem_summary` from virtual to physical datasets, relying on models created by dbt. --- .../fact_learner_problem_summary.yaml | 4 +- .../fact_problem_responses.yaml | 4 +- .../queries/fact_learner_problem_summary.sql | 37 ------------------- .../queries/fact_problem_responses.sql | 36 ------------------ .../queries/hints_per_success.sql | 6 +-- 5 files changed, 5 insertions(+), 82 deletions(-) delete mode 100644 tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql delete mode 100644 tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql diff --git a/tutoraspects/templates/openedx-assets/fact_learner_problem_summary.yaml b/tutoraspects/templates/openedx-assets/fact_learner_problem_summary.yaml index 1220066d1..733d73149 100644 --- a/tutoraspects/templates/openedx-assets/fact_learner_problem_summary.yaml +++ b/tutoraspects/templates/openedx-assets/fact_learner_problem_summary.yaml @@ -127,8 +127,8 @@ warning_text: null offset: 0 params: null - schema: null - sql: "{% include 'openedx-assets/queries/fact_learner_problem_summary.sql' %}" + schema: {{ DBT_PROFILE_TARGET_DATABASE }} + sql: null table_name: fact_learner_problem_summary template_params: null uuid: 9362354c-1541-43c2-b769-da9818236298 diff --git a/tutoraspects/templates/openedx-assets/fact_problem_responses.yaml b/tutoraspects/templates/openedx-assets/fact_problem_responses.yaml index e2eb38ee5..200d5c218 100644 --- a/tutoraspects/templates/openedx-assets/fact_problem_responses.yaml +++ b/tutoraspects/templates/openedx-assets/fact_problem_responses.yaml @@ -127,8 +127,8 @@ warning_text: null offset: 0 params: null - schema: null - sql: "{% include 'openedx-assets/queries/fact_problem_responses.sql' %}" + schema: {{ DBT_PROFILE_TARGET_DATABASE }} + sql: null table_name: fact_problem_responses template_params: null uuid: 511dd5f1-3925-4e9e-ad8a-a227b5680047 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql deleted file mode 100644 index 45b8a621b..000000000 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ /dev/null @@ -1,37 +0,0 @@ -with course_problems as ( - {% include 'openedx-assets/queries/dim_course_problems.sql' %} -), summary as ( - select - org, - course_key, - problem_id, - actor_id, - success, - attempts, - num_hints_displayed, - num_answers_displayed - from {{ DBT_PROFILE_TARGET_DATABASE }}.learner_problem_summary - {% raw -%} - {% if filter_values('org') != [] %} - where - org in {{ filter_values('org') | where_in }} - {% endif %} - {%- endraw %} -) - -select - summary.org as org, - courses.course_name as course_name, - splitByString('+', courses.course_key)[-1] as run_name, - blocks.block_name as problem_name, - summary.actor_id as actor_id, - summary.success as success, - summary.attempts as attempts, - summary.num_hints_displayed as num_hints_displayed, - summary.num_answers_displayed as num_answers_displayed -from - summary - join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses - on summary.course_key = courses.course_key - join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks - on summary.problem_id = blocks.location diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql deleted file mode 100644 index f2363895d..000000000 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ /dev/null @@ -1,36 +0,0 @@ -with responses as ( - select - emission_time, - org, - course_key, - problem_id, - actor_id, - responses, - success, - attempts - from - {{ DBT_PROFILE_TARGET_DATABASE }}.problem_responses - {% raw -%} - {% if filter_values('org') != [] %} - where - org in {{ filter_values('org') | where_in }} - {% endif %} - {%- endraw %} -) - -select - responses.emission_time as emission_time, - responses.org as org, - courses.course_name as course_name, - splitByString('+', courses.course_key)[-1] as run_name, - blocks.block_name as problem_name, - responses.actor_id as actor_id, - responses.responses as responses, - responses.success as success, - responses.attempts as attempts -from - responses - join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses - on responses.course_key = courses.course_key - join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_block_names blocks - on responses.problem_id = blocks.location diff --git a/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql b/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql index e9f215d81..73a66a388 100644 --- a/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql +++ b/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql @@ -1,7 +1,3 @@ -with summary as ( - {% include 'openedx-assets/queries/fact_learner_problem_summary.sql' %} -) - select org, course_name, @@ -10,7 +6,7 @@ select actor_id, sum(num_hints_displayed) + sum(num_answers_displayed) as total_hints from - summary + {{ DBT_PROFILE_TARGET_DATABASE }}.fact_learner_problem_summary where success = 1 group by org, From f92e1620517c34d12de473e2f0f46da7adb294b3 Mon Sep 17 00:00:00 2001 From: Rory Sawyer Date: Fri, 4 Aug 2023 11:30:57 -0400 Subject: [PATCH 2/3] chore: bump aspects-dbt version to v1.3 --- tutoraspects/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 42f046e41..da842abcc 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -213,7 +213,7 @@ # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v1.2"), + ("DBT_BRANCH", "v1.3"), # Path to the dbt project inside the repository ("DBT_REPOSITORY_PATH", "aspects-dbt/aspects"), # This is a pip compliant list of Python packages to install to run dbt From 0abf8f68fd280cdc646aa38334f3e44caa4d5304 Mon Sep 17 00:00:00 2001 From: Rory Sawyer Date: Mon, 7 Aug 2023 11:07:01 -0400 Subject: [PATCH 3/3] chore: bump aspects-dbt version --- tutoraspects/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index da842abcc..4a64e7949 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -213,7 +213,7 @@ # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v1.3"), + ("DBT_BRANCH", "v1.3.1"), # Path to the dbt project inside the repository ("DBT_REPOSITORY_PATH", "aspects-dbt/aspects"), # This is a pip compliant list of Python packages to install to run dbt