diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index a1aeed495..a6a226401 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -74,6 +74,8 @@ jobs: tutor local do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor local do import-assets + - name: Performance metrics + run: tutor local do performance-metrics - name: Tutor stop run: tutor local stop @@ -136,6 +138,8 @@ jobs: tutor dev do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor dev do import-assets + - name: Performance metrics + run: tutor dev do performance-metrics - name: Tutor stop run: tutor dev stop @@ -218,6 +222,8 @@ jobs: tutor k8s do dump-data-to-clickhouse --options "--object course_overviews" make extract_translations tutor k8s do import-assets + - name: Performance metrics + run: tutor k8s do performance-metrics - name: Check failure logs if: failure() run: | diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py index af26fce26..24ef35519 100644 --- a/tutoraspects/asset_command_helpers.py +++ b/tutoraspects/asset_command_helpers.py @@ -157,12 +157,11 @@ class ChartAsset(Asset): path = "charts" omitted_vars = [ - "query_context", "params.dashboards", "params.datasource", "params.slice_id", ] - raw_vars = ["sqlExpression"] + raw_vars = ["sqlExpression", "query_context"] class DashboardAsset(Asset): diff --git a/tutoraspects/commands_v0.py b/tutoraspects/commands_v0.py index cbbcf2cde..bb58e9d87 100644 --- a/tutoraspects/commands_v0.py +++ b/tutoraspects/commands_v0.py @@ -107,6 +107,23 @@ def alembic(context, command) -> None: runner.run_job("aspects", command) +# Ex: "tutor local do performance-metrics " +@click.command(context_settings={"ignore_unknown_options": True}) +@click.pass_obj +def performance_metrics(context) -> None: + """ + Job to measure performance metrics of charts and its queries in Superset and ClickHouse. + """ + config = tutor_config.load(context.root) + runner = context.job_runner(config) + + command = """echo 'Performance...' && + python /app/pythonpath/performance_metrics.py && + echo 'Done!'; + """ + runner.run_job("superset", command) + + # Ex: "tutor local do import_assets " @click.command(context_settings={"ignore_unknown_options": True}) @click.pass_obj diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py index c9d799da5..0bc6ab6d5 100644 --- a/tutoraspects/commands_v1.py +++ b/tutoraspects/commands_v1.py @@ -130,6 +130,22 @@ def import_assets() -> list[tuple[str, str]]: ] +# Ex: "tutor local do performance-metrics " +@click.command(context_settings={"ignore_unknown_options": True}) +def performance_metrics() -> list[tuple[str, str]]: + """ + Job to measure performance metrics of charts and its queries in Superset and ClickHouse. + """ + return [ + ( + "superset", + "echo 'Performance...' && " + "python /app/pythonpath/performance_metrics.py &&" + "echo 'Done!';", + ), + ] + + # Ex: "tutor local do dump_data_to_clickhouse " @click.command(context_settings={"ignore_unknown_options": True}) @click.option( @@ -311,6 +327,7 @@ def check_superset_assets(): dump_data_to_clickhouse, transform_tracking_logs, import_assets, + performance_metrics, ) COMMANDS = (aspects,) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py index 218c5f3c6..5f0698517 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/openedx_sso_security_manager.py @@ -61,14 +61,18 @@ def get_preferences(self, username): openedx_apis = current_app.config["OPENEDX_API_URLS"] url = openedx_apis["get_preference"].format(username=username) oauth_remote = self.oauth_remotes.get("openedxsso") - response = oauth_remote.get(url, token=self.get_oauth_token()).json() - locale_preference = response.get("pref-lang", "en").replace("-", "_") + locale_preference = "en" + try: + response = oauth_remote.get(url, token=self.get_oauth_token()).json() + locale_preference = response.get("pref-lang", "en").replace("-", "_") + except Exception as e: + return locale_preference if locale_preference not in current_app.config["DASHBOARD_LOCALES"]: - log.warning( - f"Language {locale_preference} is not supported by Superset" - ) - locale_preference = "en" + log.warning( + f"Language {locale_preference} is not supported by Superset" + ) + return locale_preference return locale_preference diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py new file mode 100644 index 000000000..7d3746b22 --- /dev/null +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -0,0 +1,154 @@ +from superset.app import create_app + +app = create_app() +app.app_context().push() + + +import json +import logging +import time +import uuid +from datetime import datetime +from unittest.mock import patch + +import sqlparse +from flask import g +from superset import security_manager +from superset.charts.data.commands.get_data_command import ChartDataCommand +from superset.charts.schemas import ChartDataQueryContextSchema +from superset.extensions import db +from superset.models.dashboard import Dashboard +from superset.models.slice import Slice + +logger = logging.getLogger("performance_metrics") + +ASPECTS_VERSION = "{{ASPECTS_VERSION}}" +UUID = str(uuid.uuid4())[0:6] +RUN_ID = f"aspects-{ASPECTS_VERSION}-{UUID}" + +report_format = "{i}. {slice}\n" "Superset time: {superset_time} (s).\n" + +query_format = ( + "Query duration: {query_duration_ms} (s).\n" + "Result rows: {result_rows}\n" + "Memory Usage (MB): {memory_usage_mb}\n" + "Row count (superset) {rowcount:}\n" + "Filters: {filters}\n\n" +) + + +def performance_metrics(): + """Measure the performance of the dashboard.""" + # Mock the client name to identify the queries in the clickhouse system.query_log table by + # by the http_user_agent field. + with patch("clickhouse_connect.common.build_client_name") as mock_build_client_name: + mock_build_client_name.return_value = RUN_ID + embedable_dashboards = {{SUPERSET_EMBEDDABLE_DASHBOARDS}} + dashboards = ( + db.session.query(Dashboard) + .filter(Dashboard.slug.in_(embedable_dashboards)) + .all() + ) + report = [] + for dashboard in dashboards: + logger.info(f"Dashboard: {dashboard.slug}") + for slice in dashboard.slices: + result = measure_chart(slice) + for query in result["queries"]: + # Remove the data from the query to avoid memory issues on large datasets. + query.pop("data") + report.append(result) + return report + + +def measure_chart(slice, extra_filters=[]): + """ + Measure the performance of a chart and return the results. + """ + logger.info(f"Fetching slice data: {slice}") + query_context = json.loads(slice.query_context) + query_context.update( + { + "result_format": "json", + "result_type": "full", + "force": True, + "datasource": { + "type": "table", + "id": slice.datasource_id, + }, + } + ) + + if extra_filters: + query_context["filters"].extend(extra_filters) + + g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}") + query_context = ChartDataQueryContextSchema().load(query_context) + command = ChartDataCommand(query_context) + + start_time = datetime.now() + result = command.run() + end_time = datetime.now() + + result["time_elapsed"] = (end_time - start_time).total_seconds() + result["slice"] = slice + return result + + +def get_query_log_from_clickhouse(report): + """ + Get the query log from clickhouse and print the results. + """ + chart_uuid = "bb13bb31-c797-4ed3-a7f9-7825cc6dc482" + + slice = db.session.query(Slice).filter(Slice.uuid == chart_uuid).one() + + query_context = json.loads(slice.query_context) + query_context["queries"][0]["filters"].append( + {"col": "http_user_agent", "op": "==", "val": RUN_ID} + ) + slice.query_context = json.dumps(query_context) + + result = measure_chart(slice) + + clickhouse_queries = {} + for query in result["queries"]: + for row in query["data"]: + parsed_sql = str(sqlparse.parse(row.pop("query"))[0]) + clickhouse_queries[parsed_sql] = row + + # Sort report by slowest queries + report = sorted(report, key=lambda x: x["time_elapsed"], reverse=True) + + report_str = f"\nSuperset Reports: {RUN_ID}\n\n" + for i, result in enumerate(report): + report_str+=( + report_format.format( + i=(i + 1), slice=result["slice"], superset_time=result["time_elapsed"] + ) + ) + for i, query in enumerate(result["queries"]): + parsed_sql = ( + str(sqlparse.parse(query["query"])[0]).replace(";", "") + + "\n FORMAT Native" + ) + clickhouse_report = clickhouse_queries.get(parsed_sql, {}) + report_str+=( + query_format.format( + query_duration_ms=clickhouse_report.get("query_duration_ms") / 1000, + memory_usage_mb=clickhouse_report.get("memory_usage_mb"), + result_rows=clickhouse_report.get("result_rows"), + rowcount=query["rowcount"], + filters=query["applied_filters"], + ) + ) + logger.info(report_str) + + +if __name__ == "__main__": + logger.info(f"Running performance metrics. RUN ID: {RUN_ID}") + report = performance_metrics() + # Clickhouse query log takes some seconds to log queries. + logger.info("Waiting for clickhouse log...") + time.sleep(10) + get_query_log_from_clickhouse(report) diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml index 74e2acff5..679a0c2a6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml @@ -6,12 +6,16 @@ dataset_uuid: d777bd95-2110-46db-a1c4-8358be81a85a description: The distribution of grades for a course, out of 100%. Grades are grouped in ranges of 10%. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - grade_bucket metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Students +query_context: '{"datasource":{"id":236,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["grade_bucket"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"236__table","viz_type":"dist_bar","slice_id":298,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["grade_bucket"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Students","y_axis_bounds":[null,null],"x_axis_label":"Course Grade (out of 100%)","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Course Grade Distribution uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml index 9e302b841..0695c2526 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml @@ -19,13 +19,17 @@ params: operatorId: EQUALS sqlExpression: null subject: enrollment_status + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: enrollment_status_date color_picker: a: 1 b: 135 g: 122 r: 0 extra_form_data: {} - granularity_sqla: enrollment_status_date header_font_size: 0.4 metric: count rolling_type: None @@ -34,9 +38,12 @@ params: subheader_font_size: 0.15 time_format: smart_date time_grain_sqla: P1D - time_range: No filter viz_type: big_number + x_axis: enrollment_status_date y_axis_format: SMART_NUMBER +query_context: '{"datasource":{"id":188,"type":"table"},"force":false,"queries":[{"filters":[{"col":"enrollment_status","op":"==","val":"registered"},{"col":"enrollment_status_date","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"enrollment_status_date","label":"enrollment_status_date","expressionType":"SQL"}],"metrics":["count"],"annotation_layers":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[{"operation":"pivot","options":{"index":["enrollment_status_date"],"columns":[],"aggregates":{"count":{"operator":"mean"}},"drop_missing_columns":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"188__table","viz_type":"big_number","slice_id":344,"x_axis":"enrollment_status_date","time_grain_sqla":"P1D","metric":"count","adhoc_filters":[{"clause":"WHERE","comparator":"registered","expressionType":"SIMPLE","filterOptionName":"filter_hcnm4t7piq6_hfbtt65nqqs","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"enrollment_status"},{"clause":"WHERE","subject":"enrollment_status_date","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"show_trend_line":true,"start_y_axis_at_zero":true,"color_picker":{"a":1,"b":135,"g":122,"r":0},"header_font_size":0.4,"subheader_font_size":0.15,"y_axis_format":"SMART_NUMBER","time_format":"smart_date","rolling_type":"None","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml index c488832d5..da88796b5 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml @@ -5,9 +5,13 @@ certified_by: null dataset_uuid: 39d1e786-c0c8-4c56-81c8-56fb0df88001 description: null params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time extra_form_data: {} - granularity_sqla: emission_time header_font_size: 0.4 metric: aggregate: COUNT_DISTINCT @@ -36,9 +40,11 @@ params: sqlExpression: null subheader_font_size: 0.15 time_format: smart_date - time_range: No filter viz_type: big_number_total y_axis_format: SMART_NUMBER +query_context: '{"datasource":{"id":242,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":445,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_5y4uvwa13v4_f12i3twecs6","sqlExpression":null}],"annotation_layers":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"242__table","viz_type":"big_number_total","slice_id":395,"metric":{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":445,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_5y4uvwa13v4_f12i3twecs6","sqlExpression":null},"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"header_font_size":0.4,"subheader_font_size":0.15,"y_axis_format":"SMART_NUMBER","time_format":"smart_date","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml index 35b0cad7c..51841b393 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml @@ -38,6 +38,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":209,"type":"table"},"force":false,"queries":[{"filters":[{"col":"success","op":"==","val":"true"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["attempts"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"209__table","viz_type":"dist_bar","slice_id":331,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","comparator":"true","expressionType":"SIMPLE","filterOptionName":"filter_0fpmws3t1h6a_md2ud9xse7m","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"success"}],"groupby":["attempts"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Number + Of Attempts To Find Correct Answer","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml index 7de1ed6ef..ba11490e6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml @@ -29,6 +29,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":191,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["total_hints"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"191__table","viz_type":"dist_bar","slice_id":307,"metrics":["students"],"adhoc_filters":[],"groupby":["total_hints"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Hints + / Answer Displayed Before Correct Answer Chosen","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Hints Per Correct Answer uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml index aa0aac2ea..7e32dd219 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml @@ -7,12 +7,16 @@ description: 'This chart shows the number of students who scored within a certai percentage of points for this problem. For problems that are pass/fail it will only show the lowest and highest percentage ranges. ' params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - grade_bucket metrics: @@ -30,6 +34,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Students +query_context: '{"datasource":{"id":215,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["grade_bucket"],"metrics":["students"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"215__table","viz_type":"dist_bar","slice_id":382,"metrics":["students"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["grade_bucket"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Students","y_axis_bounds":[null,null],"x_axis_label":"Percentage + Grade","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml index 4c8430535..1a3702359 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml @@ -7,12 +7,16 @@ description: This chart shows how often an answer or combination of answers (for is selected by learners. Some problems allow learners to submit a response more than once, this chart will include all of the responses in that case. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - responses metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Responses +query_context: '{"datasource":{"id":225,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["responses"],"metrics":["total_responses"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"225__table","viz_type":"dist_bar","slice_id":352,"metrics":["total_responses"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["responses"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Responses","y_axis_bounds":[null,null],"x_axis_label":"Answers + Chosen","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml index a8063f0b9..36c794a15 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Events_Per_Day.yaml @@ -7,14 +7,18 @@ description: A count of the number of enrollments and un-enrollments per day. Le can enroll and unenroll multiple times, in this chart each individual enrollment and unenrollment will be counted. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time annotation_layers: [] color_scheme: supersetColors comparison_type: values extra_form_data: {} forecastInterval: 0.8 forecastPeriods: 10 - granularity_sqla: emission_time groupby: - expressionType: SQL label: Enrollment Status @@ -32,13 +36,17 @@ params: row_limit: 10000 show_empty_columns: true show_legend: false - stack: true + sort_series_type: sum + stack: null time_grain_sqla: P1D - time_range: No filter tooltipTimeFormat: smart_date truncate_metric: true viz_type: echarts_timeseries_bar xAxisLabelRotation: 45 + x_axis: emission_time + x_axis_sort_asc: true + x_axis_sort_series: name + x_axis_sort_series_ascending: true x_axis_time_format: '%Y-%m-%d' x_axis_title_margin: 15 y_axis_bounds: @@ -48,6 +56,8 @@ params: y_axis_title: Number Of Events y_axis_title_margin: 15 y_axis_title_position: Left +query_context: |- + {% raw %}{"datasource":{"id":224,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_time","label":"emission_time","expressionType":"SQL"},{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"metrics":["count"],"orderby":[["count",false]],"annotation_layers":[],"row_limit":10000,"series_columns":[{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_time"],"columns":["Enrollment Status"],"aggregates":{"count":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"rename","options":{"columns":{"count":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"224__table","viz_type":"echarts_timeseries_bar","slice_id":332,"x_axis":"emission_time","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":["count"],"groupby":[{"expressionType":"SQL","label":"Enrollment Status","sqlExpression":"{{translate_column('enrollment_status')}}"}],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No filter","expressionType":"SIMPLE"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":15,"y_axis_title":"Number Of Events","y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","stack":null,"only_total":true,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","xAxisLabelRotation":45,"y_axis_format":"SMART_NUMBER","y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}{% endraw %} slice_name: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml index 058e24c5e..cd34a4bb7 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml @@ -18,6 +18,11 @@ params: operatorId: EQUALS sqlExpression: null subject: enrollment_status + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time color_scheme: supersetColors date_format: smart_date extra_form_data: {} @@ -40,6 +45,8 @@ params: show_legend: true sort_by_metric: true viz_type: pie +query_context: |- + {% raw %}{"datasource":{"id":224,"type":"table"},"force":false,"queries":[{"filters":[{"col":"enrollment_status","op":"==","val":"registered"},{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"expressionType":"SQL","label":"Enrollment Mode","sqlExpression":"{{translate_column('enrollment_mode')}}"}],"metrics":["count"],"orderby":[["count",false]],"annotation_layers":[],"row_limit":100,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"224__table","viz_type":"pie","slice_id":411,"groupby":[{"expressionType":"SQL","label":"Enrollment Mode","sqlExpression":"{{translate_column('enrollment_mode')}}"}],"metric":"count","adhoc_filters":[{"clause":"WHERE","comparator":"registered","expressionType":"SIMPLE","filterOptionName":"filter_itrvc9c0jxb_g4qd4b18na4","isExtra":false,"isNew":false,"operator":"==","operatorId":"EQUALS","sqlExpression":null,"subject":"enrollment_status"},{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No filter","expressionType":"SIMPLE"}],"row_limit":100,"sort_by_metric":true,"color_scheme":"supersetColors","show_labels_threshold":5,"show_legend":true,"legendType":"scroll","legendOrientation":"top","label_type":"key","number_format":"SMART_NUMBER","date_format":"smart_date","show_labels":true,"labels_outside":true,"outerRadius":70,"innerRadius":30,"extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}{% endraw %} slice_name: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml index 76d1bdd0a..3382ff2cf 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml @@ -10,7 +10,6 @@ params: color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: null groupby: - num_posts metrics: @@ -19,7 +18,6 @@ params: rich_tooltip: true row_limit: 10000 show_legend: true - time_range: No filter timeseries_limit_metric: aggregate: null column: null @@ -37,6 +35,8 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of users +query_context: '{"datasource":{"id":235,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["num_posts"],"metrics":["users"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"series_limit_metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"num_posts","optionName":"metric_tdazpzouidq_76qbq0kf86v","sqlExpression":"num_posts"},"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"235__table","viz_type":"dist_bar","slice_id":310,"metrics":["users"],"adhoc_filters":[],"groupby":["num_posts"],"columns":[],"row_limit":10000,"timeseries_limit_metric":{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":false,"label":"num_posts","optionName":"metric_tdazpzouidq_76qbq0kf86v","sqlExpression":"num_posts"},"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of users","y_axis_bounds":[null,null],"x_axis_label":"Posts per user","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Posts per user uuid: bc191ce7-f39d-48db-86a9-d19949f4211d version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml new file mode 100644 index 000000000..32af356d8 --- /dev/null +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Query_performance.yaml @@ -0,0 +1,43 @@ +_file_name: Query_performance.yaml +cache_timeout: null +certification_details: null +certified_by: null +dataset_uuid: 53099fd2-6739-45e5-9a93-fcb9fa70119f +description: null +params: + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: event_time + all_columns: [] + color_pn: true + extra_form_data: {} + groupby: + - http_user_agent + - query_duration_ms + - result_rows + - event_time + - memory_usage_mb + - tables + - query + order_by_cols: [] + order_desc: true + percent_metrics: [] + query_mode: aggregate + row_limit: 1000 + server_page_length: 10 + show_cell_bars: true + table_timestamp_format: smart_date + temporal_columns_lookup: + event_time: true + time_grain_sqla: P1D + viz_type: table +query_context: '{"datasource":{"id":266,"type":"table"},"force":false,"queries":[{"filters":[{"col":"event_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":["http_user_agent","query_duration_ms","result_rows",{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"event_time","label":"event_time","expressionType":"SQL"},"memory_usage_mb","tables","query"],"metrics":[],"orderby":[],"annotation_layers":[],"row_limit":1000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[]}],"form_data":{"datasource":"266__table","viz_type":"table","slice_id":422,"query_mode":"aggregate","groupby":["http_user_agent","query_duration_ms","result_rows","event_time","memory_usage_mb","tables","query"],"time_grain_sqla":"P1D","temporal_columns_lookup":{"event_time":true},"all_columns":[],"percent_metrics":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No + filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"event_time"}],"order_by_cols":[],"row_limit":1000,"server_page_length":10,"order_desc":true,"table_timestamp_format":"smart_date","show_cell_bars":true,"color_pn":true,"extra_form_data":{},"dashboards":[],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' +slice_name: Query performance +uuid: bb13bb31-c797-4ed3-a7f9-7825cc6dc482 +version: 1.0.0 +viz_type: table diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml index e56e2dd64..d16ddf9ea 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml @@ -55,6 +55,9 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Students +query_context: '{"datasource":{"id":222,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["problem_name_with_location","success"],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":301,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":true,"isNew":false,"label":"Learners","optionName":"metric_g889221mh0r_lf203igqxgg","sqlExpression":null}],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"222__table","viz_type":"dist_bar","slice_id":339,"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":301,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":true,"isNew":false,"label":"Learners","optionName":"metric_g889221mh0r_lf203igqxgg","sqlExpression":null}],"adhoc_filters":[],"groupby":["problem_name_with_location"],"columns":["success"],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":false,"rich_tooltip":true,"bar_stacked":true,"order_bars":false,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Students","show_controls":false,"y_axis_bounds":[null,null],"x_axis_label":"Question + Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml index 3a2fadb8b..0c26e6069 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml @@ -6,12 +6,16 @@ dataset_uuid: a96a4b13-a429-442d-83ca-5d6f94010909 description: |- This chart helps understand how many learners are using the video's transcripts or closed captions. If the video has not transcripts or captions the chart will be empty. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - video_name_with_location metrics: @@ -29,6 +33,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of Users / Uses +query_context: '{"datasource":{"id":220,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["video_name_with_location"],"metrics":["total_transcript_usage","unique_transcript_users"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"220__table","viz_type":"dist_bar","slice_id":410,"metrics":["total_transcript_usage","unique_transcript_users"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["video_name_with_location"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of Users / Uses","y_axis_bounds":[null,null],"x_axis_label":"Video Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml index 727bb7760..87d2df9d2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml @@ -6,13 +6,17 @@ dataset_uuid: c2c391b3-3403-4f05-bc0b-3de53bd366ec description: This chart shows which parts of a video are most watched, and which are most re-watched. Each segment represents 5 seconds of the video. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: started_at bar_stacked: true bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: started_at groupby: - segment_start metrics: @@ -31,6 +35,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number Of Viewers / Views +query_context: '{"datasource":{"id":183,"type":"table"},"force":false,"queries":[{"filters":[{"col":"started_at","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["segment_start"],"metrics":["unique_viewers","repeat_views"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"183__table","viz_type":"dist_bar","slice_id":371,"metrics":["unique_viewers","repeat_views"],"adhoc_filters":[{"clause":"WHERE","subject":"started_at","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["segment_start"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"bar_stacked":true,"order_bars":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + Of Viewers / Views","y_axis_bounds":[null,null],"x_axis_label":"Seconds Of Video","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml index 82da00741..c847530c1 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml @@ -7,12 +7,16 @@ description: This chart shows how many unique learners have watched each video, how many repeat views each has gotten. If a video has never been played it will not appear in this chart. params: - adhoc_filters: [] + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: emission_time bottom_margin: auto color_scheme: supersetColors columns: [] extra_form_data: {} - granularity_sqla: emission_time groupby: - video_name_with_location metrics: @@ -30,6 +34,10 @@ params: - null y_axis_format: SMART_NUMBER y_axis_label: Number of Views / Viewers +query_context: '{"datasource":{"id":214,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No + filter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["video_name_with_location"],"metrics":["total_plays","unique_watchers"],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"214__table","viz_type":"dist_bar","slice_id":359,"metrics":["total_plays","unique_watchers"],"adhoc_filters":[{"clause":"WHERE","subject":"emission_time","operator":"TEMPORAL_RANGE","comparator":"No + filter","expressionType":"SIMPLE"}],"groupby":["video_name_with_location"],"columns":[],"row_limit":10000,"order_desc":true,"color_scheme":"supersetColors","show_legend":true,"rich_tooltip":true,"y_axis_format":"SMART_NUMBER","y_axis_label":"Number + of Views / Viewers","y_axis_bounds":[null,null],"x_axis_label":"Video Title","bottom_margin":"auto","x_ticks_layout":"auto","extra_form_data":{},"dashboards":[148],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}' slice_name: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml index b74a7d7f2..06885d666 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml @@ -307,8 +307,7 @@ metadata: datasetUuid: 417b2035-8fa1-4c60-a405-4b1947c3c966 type: NATIVE_FILTER refresh_frequency: 0 - shared_label_colors: - count: '#454E7C' + shared_label_colors: {} timed_refresh_immune_slices: [] position: CHART-AZZnl_lpMv: diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml new file mode 100644 index 000000000..6ed95e148 --- /dev/null +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml @@ -0,0 +1,125 @@ +_file_name: query_log.yaml +cache_timeout: null +columns: +- advanced_data_type: null + column_name: event_time + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: true + python_date_format: null + type: DateTime + verbose_name: Event Time +- advanced_data_type: null + column_name: query + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Query +- advanced_data_type: null + column_name: tables + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Array(LowCardinality(String)) + verbose_name: null +- advanced_data_type: null + column_name: http_user_agent + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: LowCardinality(String) + verbose_name: null +- advanced_data_type: null + column_name: memory_usage_mb + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: null +- advanced_data_type: null + column_name: query_duration_ms + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: null +- advanced_data_type: null + column_name: result_rows + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: null +database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 +default_endpoint: null +description: null +extra: null +fetch_values_predicate: null +filter_select_enabled: true +main_dttm_col: null +metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: + warning_markdown: '' + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null +normalize_columns: false +offset: 0 +params: null +schema: main +sql: | + SELECT + query_duration_ms, + result_rows, + memory_usage / (1024 * 1024 ) AS memory_usage_mb, + event_time, + tables, + query, + http_user_agent + FROM system.query_log + WHERE (has(databases, '{{ASPECTS_XAPI_DATABASE}}') OR has(databases, '{{ASPECTS_EVENT_SINK_DATABASE}}') OR has(databases, '{{DBT_PROFILE_TARGET_DATABASE}}')) AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish') + ORDER BY event_time DESC +table_name: query_log +template_params: null +uuid: 53099fd2-6739-45e5-9a93-fcb9fa70119f +version: 1.0.0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql index 92ae63466..6341e83c1 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql @@ -3,10 +3,11 @@ with grades as ( from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades where grade_type = 'course' {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + {% elif filter_values('course_name') != [] %} and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }} {% else %} - and 0=1 + and 1=0 {% endif %} {% endraw %} {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index bf4468a0c..19d27b027 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -134,7 +134,9 @@ SELECT FROM summary WHERE {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index e55e91418..692f06beb 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -118,7 +118,9 @@ SELECT FROM summary where {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index e401148dd..d9be4306d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -5,7 +5,8 @@ with grades as ( grade_type = 'problem' {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + {% elif filter_values('problem_name_with_location') != [] %} and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} {% else %} and 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 0962f17a9..e320b21db 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -25,7 +25,9 @@ from problem_responses where {% raw %} - {% if filter_values('problem_name_with_location') != [] %} + {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('problem_name_with_location') != [] %} problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index bba2968c7..3059cc779 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -3,10 +3,12 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage where {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} - 0=1 + 1=0 {% endif %} {% endraw %} {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index 4bdc18c9b..9609b93f5 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -3,7 +3,9 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays where {% raw %} - {% if filter_values('course_name') != [] %} + {% if get_filters('course_name', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('course_name') != [] %} course_name in {{ filter_values('course_name') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index bb64b1301..6b9dcfb6a 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -75,7 +75,9 @@ select from enriched_segments where {% raw %} - {% if filter_values('video_name_with_location') != [] %} + {% if get_filters('video_name_with_location', remove_filter=True) == [] %} + 1=1 + {% elif filter_values('video_name_with_location') != [] %} video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} {% else %} 1=0 diff --git a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql index 530da31f6..4a450f40d 100644 --- a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql +++ b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql @@ -7,6 +7,6 @@ SELECT FROM system.query_log WHERE - TYPE = 'QueryFinish' + type = 'QueryFinish' ORDER BY query_duration_ms DESC;