feat: turn datasets into virtual

openedx · Mar 13, 2024 · d4432b5 · d4432b5
1 parent 7c7e26c
commit d4432b5
Show file tree

Hide file tree

Showing 13 changed files with 135 additions and 110 deletions.
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py
@@ -19,7 +19,7 @@
 
 SECURITY_FILTERS = [
     {
-        "schema": "{{ASPECTS_XAPI_DATABASE}}",
+        "schema": VIRTUAL_TABLE_SCHEMA,
         "table_name": "{{ASPECTS_XAPI_TABLE}}",
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
@@ -91,23 +91,23 @@
         "filter_type": "Regular"
     },
     {
-        "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}",
+        "schema": VIRTUAL_TABLE_SCHEMA,
         "table_name": "course_names",
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
         "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
         "filter_type": "Regular"
     },
     {
-        "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}",
+        "schema": VIRTUAL_TABLE_SCHEMA,
         "table_name": "course_overviews",
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
         "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
         "filter_type": "Regular"
     },
     {
-        "schema": "{{ASPECTS_EVENT_SINK_DATABASE}}",
+        "schema": VIRTUAL_TABLE_SCHEMA,
         "table_name": "course_blocks",
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",

diff --git a/...ects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml b/...ects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/ab_user.yaml
@@ -191,7 +191,7 @@ normalize_columns: false
 offset: 0
 params: null
 schema: superset
-sql: ''
+sql: 'select * from superset.ab_user'
 table_name: ab_user
 template_params: null
 uuid: 304a5d6d-c589-483a-8e8d-3fadbccef648

diff --git a/...emplates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml b/...emplates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_blocks.yaml
@@ -127,8 +127,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: "{{ ASPECTS_EVENT_SINK_DATABASE }}"
-sql: null
+schema: main
+sql: "select * from {{ASPECTS_EVENT_SINK_DATABASE}}.{{ ASPECTS_EVENT_SINK_NODES_TABLE }}"
 table_name: "{{ ASPECTS_EVENT_SINK_NODES_TABLE }}"
 template_params: null
 uuid: 1b73d066-fd6c-411d-a99d-fc585f9474b1

diff --git a/...templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml b/...templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml
@@ -69,8 +69,8 @@ metrics:
 normalize_columns: true
 offset: 0
 params: null
-schema: '{{ ASPECTS_EVENT_SINK_DATABASE }}'
-sql: null
+schema: main
+sql: "select * from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names"
 table_name: course_names
 template_params: null
 uuid: 41278a97-d0ff-4645-9514-d79f80d275df

diff --git a/...lates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml b/...lates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_overviews.yaml
@@ -177,8 +177,8 @@ metrics:
 normalize_columns: true
 offset: 0
 params: null
-schema: '{{ ASPECTS_EVENT_SINK_DATABASE }}'
-sql: null
+schema: main
+sql: "select * from {{ ASPECTS_EVENT_SINK_DATABASE }}.course_overviews"
 table_name: course_overviews
 template_params: null
 uuid: 46183302-6fa6-41a3-b6a7-79ff6c1c8402

diff --git a/...s/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml b/...s/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml
@@ -2,31 +2,31 @@ _file_name: dashboards.yaml
 cache_timeout: null
 columns:
 - advanced_data_type: null
-  column_name: created_on
+  column_name: dashboard_title
   description: null
   expression: ''
   extra: {}
   filterable: true
   groupby: true
   is_active: true
-  is_dttm: true
+  is_dttm: false
   python_date_format: null
-  type: TIMESTAMP WITHOUT TIME ZONE
-  verbose_name: Created On
+  type: VARCHAR(500)
+  verbose_name: null
 - advanced_data_type: null
-  column_name: changed_on
+  column_name: uuid
   description: null
   expression: ''
   extra: {}
   filterable: true
   groupby: true
   is_active: true
-  is_dttm: true
+  is_dttm: null
   python_date_format: null
-  type: TIMESTAMP WITHOUT TIME ZONE
-  verbose_name: Changed On
+  type: BINARY(16)
+  verbose_name: null
 - advanced_data_type: null
-  column_name: dashboard_title
+  column_name: published
   description: null
   expression: ''
   extra: {}
@@ -35,8 +35,8 @@ columns:
   is_active: true
   is_dttm: false
   python_date_format: null
-  type: VARCHAR(500)
-  verbose_name: Dashboard Title
+  type: TINYINT(1)
+  verbose_name: null
 - advanced_data_type: null
   column_name: slug
   description: null
@@ -48,9 +48,33 @@ columns:
   is_dttm: false
   python_date_format: null
   type: VARCHAR(255)
-  verbose_name: Slug
+  verbose_name: null
 - advanced_data_type: null
-  column_name: published
+  column_name: created_on
+  description: null
+  expression: ''
+  extra: {}
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: true
+  python_date_format: null
+  type: DATETIME
+  verbose_name: null
+- advanced_data_type: null
+  column_name: changed_on
+  description: null
+  expression: ''
+  extra: {}
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: true
+  python_date_format: null
+  type: DATETIME
+  verbose_name: null
+- advanced_data_type: null
+  column_name: json_metadata
   description: null
   expression: ''
   extra: {}
@@ -59,10 +83,10 @@ columns:
   is_active: true
   is_dttm: false
   python_date_format: null
-  type: BOOLEAN
-  verbose_name: Published
+  type: MEDIUMTEXT
+  verbose_name: null
 - advanced_data_type: null
-  column_name: uuid
+  column_name: position_json
   description: null
   expression: ''
   extra: {}
@@ -71,8 +95,8 @@ columns:
   is_active: true
   is_dttm: false
   python_date_format: null
-  type: UUID
-  verbose_name: UUID
+  type: MEDIUMTEXT
+  verbose_name: null
 - advanced_data_type: null
   column_name: created_by_fk
   description: null
@@ -84,7 +108,7 @@ columns:
   is_dttm: false
   python_date_format: null
   type: INTEGER
-  verbose_name: Created By FK
+  verbose_name: null
 - advanced_data_type: null
   column_name: changed_by_fk
   description: null
@@ -96,7 +120,7 @@ columns:
   is_dttm: false
   python_date_format: null
   type: INTEGER
-  verbose_name: Changed By FK
+  verbose_name: null
 - advanced_data_type: null
   column_name: id
   description: null
@@ -108,9 +132,9 @@ columns:
   is_dttm: false
   python_date_format: null
   type: INTEGER
-  verbose_name: ID
+  verbose_name: null
 - advanced_data_type: null
-  column_name: json_metadata
+  column_name: certification_details
   description: null
   expression: ''
   extra: {}
@@ -120,9 +144,9 @@ columns:
   is_dttm: false
   python_date_format: null
   type: TEXT
-  verbose_name: JSON Metadata
+  verbose_name: null
 - advanced_data_type: null
-  column_name: certification_details
+  column_name: certified_by
   description: null
   expression: ''
   extra: {}
@@ -132,9 +156,9 @@ columns:
   is_dttm: false
   python_date_format: null
   type: TEXT
-  verbose_name: Certification Details
+  verbose_name: null
 - advanced_data_type: null
-  column_name: certified_by
+  column_name: description
   description: null
   expression: ''
   extra: {}
@@ -144,9 +168,9 @@ columns:
   is_dttm: false
   python_date_format: null
   type: TEXT
-  verbose_name: Certified By
+  verbose_name: null
 - advanced_data_type: null
-  column_name: position_json
+  column_name: css
   description: null
   expression: ''
   extra: {}
@@ -156,31 +180,31 @@ columns:
   is_dttm: false
   python_date_format: null
   type: TEXT
-  verbose_name: Position JSON
+  verbose_name: null
 - advanced_data_type: null
-  column_name: description
+  column_name: is_managed_externally
   description: null
-  expression: ''
+  expression: null
   extra: {}
   filterable: true
   groupby: true
   is_active: true
   is_dttm: false
   python_date_format: null
-  type: TEXT
-  verbose_name: Description
+  type: TINYINT(1)
+  verbose_name: null
 - advanced_data_type: null
-  column_name: css
+  column_name: external_url
   description: null
-  expression: ''
+  expression: null
   extra: {}
   filterable: true
   groupby: true
   is_active: true
   is_dttm: false
   python_date_format: null
   type: TEXT
-  verbose_name: CSS
+  verbose_name: null
 database_uuid: 06cd566f-e22a-4a79-b0e0-4c947de93ee0
 default_endpoint: null
 description: null
@@ -193,16 +217,17 @@ metrics:
   d3format: null
   description: null
   expression: COUNT(*)
-  extra: {}
+  extra:
+    warning_markdown: ''
   metric_name: count
   metric_type: count
   verbose_name: COUNT(*)
   warning_text: null
-normalize_columns: false
+normalize_columns: true
 offset: 0
 params: null
 schema: superset
-sql: null
+sql: 'select * from '
 table_name: dashboards
 template_params: null
 uuid: 3dee777a-3dd4-4fcb-a749-4d13651b3444

diff --git a/...mplates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml b/...mplates/aspects/build/aspects-superset/openedx-assets/assets/datasets/indexed_events.yaml
@@ -129,7 +129,7 @@ metrics:
 normalize_columns: false
 offset: 0
 params: null
-schema: event_sink
+schema: main
 sql: |-
   {% filter indent(width=2) %}{% include 'openedx-assets/queries/indexed_events.sql' %}{% endfilter %}
 table_name: indexed_events