diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py index a76504cb8..5c53a2f27 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py @@ -19,7 +19,7 @@ SECURITY_FILTERS = [ { - "schema": "{{ASPECTS_XAPI_DATABASE}}", + "schema": VIRTUAL_TABLE_SCHEMA, "table_name": "{{ASPECTS_XAPI_TABLE}}", "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}", "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}", @@ -91,7 +91,7 @@ "filter_type": "Regular" }, { - "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}", + "schema": VIRTUAL_TABLE_SCHEMA, "table_name": "course_names", "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}", "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}", @@ -99,7 +99,7 @@ "filter_type": "Regular" }, { - "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}", + "schema": VIRTUAL_TABLE_SCHEMA, "table_name": "course_overviews", "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}", "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}", @@ -107,7 +107,7 @@ "filter_type": "Regular" }, { - "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}", + "schema": VIRTUAL_TABLE_SCHEMA, "table_name": "course_blocks", "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}", "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}", diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml index 175fb4a09..6d0992b6b 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml @@ -191,7 +191,7 @@ normalize_columns: false offset: 0 params: null schema: superset -sql: '' +sql: 'select * from superset.ab_user' table_name: ab_user template_params: null uuid: 304a5d6d-c589-483a-8e8d-3fadbccef648 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml index bf1c66db1..1d00523cd 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml @@ -127,8 +127,8 @@ metrics: warning_text: null offset: 0 params: null -schema: "{{ ASPECTS_EVENT_SINK_DATABASE }}" -sql: null +schema: main +sql: "select * from {{ASPECTS_EVENT_SINK_DATABASE}}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}" table_name: "{{ ASPECTS_EVENT_SINK_NODES_TABLE }}" template_params: null uuid: 1b73d066-fd6c-411d-a99d-fc585f9474b1 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml index 5c709a76a..3b9f9f91a 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml @@ -69,8 +69,8 @@ metrics: normalize_columns: true offset: 0 params: null -schema: '{{ ASPECTS_EVENT_SINK_DATABASE }}' -sql: null +schema: main +sql: "select * from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names" table_name: course_names template_params: null uuid: 41278a97-d0ff-4645-9514-d79f80d275df diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml index 257bd0f9b..fa0e20904 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml @@ -177,8 +177,8 @@ metrics: normalize_columns: true offset: 0 params: null -schema: '{{ ASPECTS_EVENT_SINK_DATABASE }}' -sql: null +schema: main +sql: "select * from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_overviews" table_name: course_overviews template_params: null uuid: 46183302-6fa6-41a3-b6a7-79ff6c1c8402 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml index 7cb4d02ac..3136aaf27 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml @@ -2,31 +2,31 @@ _file_name: dashboards.yaml cache_timeout: null columns: - advanced_data_type: null - column_name: created_on + column_name: dashboard_title description: null expression: '' extra: {} filterable: true groupby: true is_active: true - is_dttm: true + is_dttm: false python_date_format: null - type: TIMESTAMP WITHOUT TIME ZONE - verbose_name: Created On + type: VARCHAR(500) + verbose_name: null - advanced_data_type: null - column_name: changed_on + column_name: uuid description: null expression: '' extra: {} filterable: true groupby: true is_active: true - is_dttm: true + is_dttm: null python_date_format: null - type: TIMESTAMP WITHOUT TIME ZONE - verbose_name: Changed On + type: BINARY(16) + verbose_name: null - advanced_data_type: null - column_name: dashboard_title + column_name: published description: null expression: '' extra: {} @@ -35,8 +35,8 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(500) - verbose_name: Dashboard Title + type: TINYINT(1) + verbose_name: null - advanced_data_type: null column_name: slug description: null @@ -48,9 +48,33 @@ columns: is_dttm: false python_date_format: null type: VARCHAR(255) - verbose_name: Slug + verbose_name: null - advanced_data_type: null - column_name: published + column_name: created_on + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: true + python_date_format: null + type: DATETIME + verbose_name: null +- advanced_data_type: null + column_name: changed_on + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: true + python_date_format: null + type: DATETIME + verbose_name: null +- advanced_data_type: null + column_name: json_metadata description: null expression: '' extra: {} @@ -59,10 +83,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: BOOLEAN - verbose_name: Published + type: MEDIUMTEXT + verbose_name: null - advanced_data_type: null - column_name: uuid + column_name: position_json description: null expression: '' extra: {} @@ -71,8 +95,8 @@ columns: is_active: true is_dttm: false python_date_format: null - type: UUID - verbose_name: UUID + type: MEDIUMTEXT + verbose_name: null - advanced_data_type: null column_name: created_by_fk description: null @@ -84,7 +108,7 @@ columns: is_dttm: false python_date_format: null type: INTEGER - verbose_name: Created By FK + verbose_name: null - advanced_data_type: null column_name: changed_by_fk description: null @@ -96,7 +120,7 @@ columns: is_dttm: false python_date_format: null type: INTEGER - verbose_name: Changed By FK + verbose_name: null - advanced_data_type: null column_name: id description: null @@ -108,9 +132,9 @@ columns: is_dttm: false python_date_format: null type: INTEGER - verbose_name: ID + verbose_name: null - advanced_data_type: null - column_name: json_metadata + column_name: certification_details description: null expression: '' extra: {} @@ -120,9 +144,9 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: JSON Metadata + verbose_name: null - advanced_data_type: null - column_name: certification_details + column_name: certified_by description: null expression: '' extra: {} @@ -132,9 +156,9 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: Certification Details + verbose_name: null - advanced_data_type: null - column_name: certified_by + column_name: description description: null expression: '' extra: {} @@ -144,9 +168,9 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: Certified By + verbose_name: null - advanced_data_type: null - column_name: position_json + column_name: css description: null expression: '' extra: {} @@ -156,23 +180,23 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: Position JSON + verbose_name: null - advanced_data_type: null - column_name: description + column_name: is_managed_externally description: null - expression: '' + expression: null extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: TEXT - verbose_name: Description + type: TINYINT(1) + verbose_name: null - advanced_data_type: null - column_name: css + column_name: external_url description: null - expression: '' + expression: null extra: {} filterable: true groupby: true @@ -180,7 +204,7 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: CSS + verbose_name: null database_uuid: 06cd566f-e22a-4a79-b0e0-4c947de93ee0 default_endpoint: null description: null @@ -193,16 +217,17 @@ metrics: d3format: null description: null expression: COUNT(*) - extra: {} + extra: + warning_markdown: '' metric_name: count metric_type: count verbose_name: COUNT(*) warning_text: null -normalize_columns: false +normalize_columns: true offset: 0 params: null schema: superset -sql: null +sql: 'select * from ' table_name: dashboards template_params: null uuid: 3dee777a-3dd4-4fcb-a749-4d13651b3444 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml index 12eae71d1..5c7b031d8 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml @@ -129,7 +129,7 @@ metrics: normalize_columns: false offset: 0 params: null -schema: event_sink +schema: main sql: |- {% filter indent(width=2) %}{% include 'openedx-assets/queries/indexed_events.sql' %}{% endfilter %} table_name: indexed_events diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slices.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slices.yaml index d7f10752e..53a57da57 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slices.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slices.yaml @@ -38,7 +38,7 @@ columns: type: TIMESTAMP WITHOUT TIME ZONE verbose_name: Changed On - advanced_data_type: null - column_name: schema_perm + column_name: last_saved_by_fk description: null expression: '' extra: {} @@ -47,10 +47,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(1000) - verbose_name: Schema Perm + type: INTEGER + verbose_name: Last Saved By Fk - advanced_data_type: null - column_name: datasource_name + column_name: created_by_fk description: null expression: '' extra: {} @@ -59,10 +59,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(2000) - verbose_name: Datasource Name + type: INTEGER + verbose_name: Created By Fk - advanced_data_type: null - column_name: perm + column_name: changed_by_fk description: null expression: '' extra: {} @@ -71,10 +71,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(2000) - verbose_name: Perm + type: INTEGER + verbose_name: Changed By Fk - advanced_data_type: null - column_name: datasource_type + column_name: schema_perm description: null expression: '' extra: {} @@ -83,10 +83,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(200) - verbose_name: Datasource Type + type: VARCHAR(1000) + verbose_name: Schema Perm - advanced_data_type: null - column_name: slice_name + column_name: datasource_name description: null expression: '' extra: {} @@ -95,10 +95,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(250) - verbose_name: Slice Name + type: VARCHAR(2000) + verbose_name: Datasource Name - advanced_data_type: null - column_name: viz_type + column_name: datasource_type description: null expression: '' extra: {} @@ -107,10 +107,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: VARCHAR(250) - verbose_name: Viz Type + type: VARCHAR(200) + verbose_name: Datasource Type - advanced_data_type: null - column_name: uuid + column_name: slice_name description: null expression: '' extra: {} @@ -119,10 +119,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: UUID - verbose_name: UUID + type: VARCHAR(250) + verbose_name: Slice Name - advanced_data_type: null - column_name: last_saved_by_fk + column_name: viz_type description: null expression: '' extra: {} @@ -131,10 +131,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: INTEGER - verbose_name: Last Saved By Fk + type: VARCHAR(250) + verbose_name: Viz Type - advanced_data_type: null - column_name: created_by_fk + column_name: certified_by description: null expression: '' extra: {} @@ -143,10 +143,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: INTEGER - verbose_name: Created By Fk + type: TEXT + verbose_name: Certified By - advanced_data_type: null - column_name: changed_by_fk + column_name: certification_details description: null expression: '' extra: {} @@ -155,10 +155,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: INTEGER - verbose_name: Changed By Fk + type: TEXT + verbose_name: Certification Details - advanced_data_type: null - column_name: datasource_id + column_name: cache_timeout description: null expression: '' extra: {} @@ -168,9 +168,9 @@ columns: is_dttm: false python_date_format: null type: INTEGER - verbose_name: Datasource ID + verbose_name: Cache Timeout - advanced_data_type: null - column_name: cache_timeout + column_name: query_context description: null expression: '' extra: {} @@ -179,10 +179,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: INTEGER - verbose_name: Cache Timeout + type: TEXT + verbose_name: Query Context - advanced_data_type: null - column_name: id + column_name: datasource_id description: null expression: '' extra: {} @@ -192,9 +192,9 @@ columns: is_dttm: false python_date_format: null type: INTEGER - verbose_name: ID + verbose_name: Datasource ID - advanced_data_type: null - column_name: certification_details + column_name: perm description: null expression: '' extra: {} @@ -203,10 +203,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: TEXT - verbose_name: Certification Details + type: VARCHAR(2000) + verbose_name: Perm - advanced_data_type: null - column_name: certified_by + column_name: uuid description: null expression: '' extra: {} @@ -215,10 +215,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: TEXT - verbose_name: Certified By + type: UUID + verbose_name: UUID - advanced_data_type: null - column_name: query_context + column_name: id description: null expression: '' extra: {} @@ -227,10 +227,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: TEXT - verbose_name: Query Context + type: INTEGER + verbose_name: ID - advanced_data_type: null - column_name: params + column_name: description description: null expression: '' extra: {} @@ -240,9 +240,9 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: Parameters + verbose_name: Description - advanced_data_type: null - column_name: description + column_name: params description: null expression: '' extra: {} @@ -252,7 +252,7 @@ columns: is_dttm: false python_date_format: null type: TEXT - verbose_name: Description + verbose_name: Parameters database_uuid: 06cd566f-e22a-4a79-b0e0-4c947de93ee0 default_endpoint: null description: null @@ -270,11 +270,11 @@ metrics: metric_type: count verbose_name: COUNT(*) warning_text: null -normalize_columns: false +normalize_columns: true offset: 0 params: null schema: superset -sql: null +sql: select * from superset.slices table_name: slices template_params: null uuid: aaf1ae12-73c1-4b88-b07d-7a8988333018 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml index 39b52fbf3..c39ba8b7d 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml @@ -81,7 +81,7 @@ metrics: normalize_columns: false offset: 0 params: null -schema: xapi +schema: main sql: |- {% filter indent(width=2) %}{% include 'openedx-assets/queries/slowest_clickhouse_queries.sql' %}{% endfilter %} table_name: slowest_clickhouse_queries diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml index b57a3df2b..340bdbb20 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml @@ -190,7 +190,7 @@ metrics: normalize_columns: false offset: 0 params: null -schema: superset +schema: main sql: |- {% filter indent(width=2) %}{% include 'openedx-assets/queries/superset_action_log.sql' %}{% endfilter %} table_name: superset_action_log diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/user_pii.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/user_pii.yaml index a25b13cb5..651d520b2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/user_pii.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/user_pii.yaml @@ -5,8 +5,8 @@ description: null default_endpoint: null offset: 0 cache_timeout: null -schema: event_sink -sql: null +schema: main +sql: 'select * from event_sink.user_pii' params: null template_params: null filter_select_enabled: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/xapi_events_all_parsed.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/xapi_events_all_parsed.yaml index 7066d9d5b..82b11c119 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/xapi_events_all_parsed.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/xapi_events_all_parsed.yaml @@ -183,8 +183,8 @@ metrics: normalize_columns: true offset: 0 params: null -schema: '{{ASPECTS_XAPI_DATABASE}}' -sql: '' +schema: main +sql: 'SELECT * FROM {{ASPECTS_XAPI_DATABASE}}.xapi_events_all_parsed' table_name: xapi_events_all_parsed template_params: null uuid: 2a2498dc-03ce-41a0-b798-d84f808f7da6 diff --git a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql index c04825297..530da31f6 100644 --- a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql +++ b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql @@ -2,7 +2,7 @@ SELECT event_time, query_duration_ms / 1000 AS duration_secs, read_rows, - memory_usage 1024 AS memory_usage_kb, + memory_usage / 1024 AS memory_usage_kb, query FROM system.query_log