From c84c912a3bf97838967df9739da70dd08fa69d9b Mon Sep 17 00:00:00 2001 From: FahadKhalid210 Date: Mon, 29 Apr 2024 16:27:45 +0500 Subject: [PATCH] upgrade clickhouse --- ...155841_fahad.khalid_fahad_clickhouse_30.md | 1 + .../0012_modify_course_enrollments.sql | 23 +++++++++++++++++++ .../0013_modify_course_block_completion.sql | 15 ++++++++++++ .../cairn/build/cairn-clickhouse/Dockerfile | 6 ++--- .../build/cairn-superset/cairn/bootstrap.py | 16 +++++++++---- 5 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20240429_155841_fahad.khalid_fahad_clickhouse_30.md create mode 100644 tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0012_modify_course_enrollments.sql create mode 100644 tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0013_modify_course_block_completion.sql diff --git a/changelog.d/20240429_155841_fahad.khalid_fahad_clickhouse_30.md b/changelog.d/20240429_155841_fahad.khalid_fahad_clickhouse_30.md new file mode 100644 index 0000000..40caf76 --- /dev/null +++ b/changelog.d/20240429_155841_fahad.khalid_fahad_clickhouse_30.md @@ -0,0 +1 @@ +- 💥[Feature] Upgrade Clickhouse version to `24.1.8.22` and fix query issues due to deprecation of Live views. (by @Fahadkhalid210) diff --git a/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0012_modify_course_enrollments.sql b/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0012_modify_course_enrollments.sql new file mode 100644 index 0000000..2552c17 --- /dev/null +++ b/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0012_modify_course_enrollments.sql @@ -0,0 +1,23 @@ +RENAME TABLE _openedx_course_enrollments TO openedx_course_enrollments; +RENAME TABLE _openedx_user_profiles TO openedx_user_profiles; +RENAME TABLE _openedx_users TO openedx_users; + +DROP TABLE course_enrollments; +CREATE VIEW course_enrollments AS +SELECT + openedx_course_enrollments.course_id AS course_id, + openedx_course_enrollments.created AS enrollment_created, + openedx_course_enrollments.is_active AS enrollment_is_active, + openedx_course_enrollments.mode AS enrollment_mode, + openedx_course_enrollments.user_id AS user_id, + openedx_users.username AS username, + openedx_users.email AS user_email, + openedx_user_profiles.year_of_birth AS user_year_of_birth, + openedx_user_profiles.gender AS user_gender, + openedx_user_profiles.level_of_education AS user_level_of_education, + openedx_user_profiles.city AS user_city, + openedx_user_profiles.state AS user_state, + openedx_user_profiles.country AS user_country +FROM openedx_course_enrollments +INNER JOIN openedx_user_profiles ON openedx_course_enrollments.user_id = openedx_user_profiles.user_id +INNER JOIN openedx_users ON openedx_course_enrollments.user_id = openedx_users.id; \ No newline at end of file diff --git a/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0013_modify_course_block_completion.sql b/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0013_modify_course_block_completion.sql new file mode 100644 index 0000000..523530b --- /dev/null +++ b/tutorcairn/templates/cairn/apps/clickhouse/migrations.d/0013_modify_course_block_completion.sql @@ -0,0 +1,15 @@ +RENAME TABLE _openedx_block_completion TO openedx_block_completion; + +DROP TABLE course_block_completion; + +CREATE VIEW course_block_completion AS +SELECT + openedx_block_completion.course_key AS course_id, + openedx_block_completion.block_key AS block_key, + openedx_block_completion.user_id AS user_id, + openedx_block_completion.completion AS completion, + course_blocks.position as position, + course_blocks.display_name as display_name, + course_blocks.full_name as full_name +FROM openedx_block_completion +INNER JOIN course_blocks ON openedx_block_completion.block_key = course_blocks.block_key; diff --git a/tutorcairn/templates/cairn/build/cairn-clickhouse/Dockerfile b/tutorcairn/templates/cairn/build/cairn-clickhouse/Dockerfile index 32af310..bb24cfc 100644 --- a/tutorcairn/templates/cairn/build/cairn-clickhouse/Dockerfile +++ b/tutorcairn/templates/cairn/build/cairn-clickhouse/Dockerfile @@ -1,11 +1,11 @@ -# https://hub.docker.com/r/yandex/clickhouse-server/tags -FROM docker.io/yandex/clickhouse-server:22.1.3.7 +# https://hub.docker.com/r/clickhouse/clickhouse-server/tags +FROM docker.io/clickhouse/clickhouse-server:24.1.8.22 # The clickhouse repo is currently unavailable in some parts of the world. If we don't # remove this repo here then `apt update` will fail. Check if the problem is resolved with: # curl https://repo.yandex.ru/clickhouse/deb/stable/ # The above command should be a 200, and not a 404. -RUN rm /etc/apt/sources.list.d/clickhouse.list +# RUN rm /etc/apt/sources.list.d/clickhouse.list RUN apt update && apt install -y python3 COPY ./scripts /scripts RUN chmod a+x /scripts/* diff --git a/tutorcairn/templates/cairn/build/cairn-superset/cairn/bootstrap.py b/tutorcairn/templates/cairn/build/cairn-superset/cairn/bootstrap.py index 25ba62f..f88fab7 100644 --- a/tutorcairn/templates/cairn/build/cairn-superset/cairn/bootstrap.py +++ b/tutorcairn/templates/cairn/build/cairn-superset/cairn/bootstrap.py @@ -117,11 +117,17 @@ def grant_clickhouse_row_based_access(clickhouse_username, course_ids=None): for table in make_clickhouse_query("SHOW TABLES").split("\n"): if not table.startswith("_"): make_clickhouse_query( - f"""GRANT SELECT ON {table} TO '{clickhouse_username}';""" - ) - make_clickhouse_query( - f"""CREATE ROW POLICY OR REPLACE '{clickhouse_username}' ON {table} AS RESTRICTIVE FOR SELECT USING {condition} TO '{clickhouse_username}';""" - ) + f"""GRANT SELECT ON {table} TO '{clickhouse_username}';""" + ) + + if table in ["openedx_users", "openedx_user_profiles", "openedx_block_completion"]: + make_clickhouse_query( + f"""CREATE ROW POLICY OR REPLACE '{clickhouse_username}' ON {table} AS RESTRICTIVE FOR SELECT USING 1 TO '{clickhouse_username}';""" + ) + else: + make_clickhouse_query( + f"""CREATE ROW POLICY OR REPLACE '{clickhouse_username}' ON {table} AS RESTRICTIVE FOR SELECT USING {condition} TO '{clickhouse_username}';""" + ) def make_clickhouse_query(query):