feat: use custom docker images for services

.github/workflows/release.yaml

@@ -0,0 +1,70 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  bumpversion:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.tag_version.outputs.new_version }}
+      previous_tag: ${{ steps.tag_version.outputs.previous_tag }}
+      bump_commit_sha: ${{ steps.bumpversion.outputs.commit_hash }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get next version
+        id: tag_version
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          default_bump: false
+          default_prerelease_bump: false
+          dry_run: true
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Create bumpversion
+        if: steps.tag_version.outputs.new_version
+        run: |
+          pip install bump2version
+          bump2version --new-version ${{ steps.tag_version.outputs.new_version }} setup.cfg
+      - name: Update Changelog
+        if: steps.tag_version.outputs.new_version
+        uses: stefanzweifel/[email protected]
+        with:
+          latest-version: ${{ steps.tag_version.outputs.new_tag }}
+          release-notes: ${{ steps.tag_version.outputs.changelog }}
+      - name: Commit bumpversion
+        id: bumpversion
+        if: steps.tag_version.outputs.new_version
+        uses: stefanzweifel/[email protected]
+        with:
+          branch: ${{ github.ref }}
+          commit_message: "chore: preparing release ${{ steps.tag_version.outputs.new_version }}"
+          file_pattern: CHANGELOG.md setup.cfg tutoraspects/*
+  release:
+    needs: bumpversion
+    if: needs.bumpversion.outputs.version
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag_version.outputs.new_tag }}
+      changelog: ${{ steps.tag_version.outputs.changelog }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Create tag
+        id: tag_version
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          commit_sha: ${{ needs.bumpversion.outputs.bump_commit_sha }}
+          default_bump: false
+          default_prerelease_bump: false
+      - name: Create a GitHub release
+        if: steps.tag_version.outputs.new_tag
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.tag_version.outputs.new_tag }}
+          name: Release ${{ steps.tag_version.outputs.new_tag }}
+          body: ${{ steps.tag_version.outputs.changelog }}

CHANGELOG.md

@@ -0,0 +1,6 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

setup.cfg

@@ -0,0 +1,4 @@
+[bumpversion]
+current_version = 0.0.0
+commit = False
+tag = False

tutoraspects/patches/local-docker-compose-services

@@ -81,7 +81,7 @@ superset-worker-beat:
 vector:
     image: {{ DOCKER_IMAGE_VECTOR }}
     volumes:
-        - ../../aspects/data/vector:/var/lib/vector
+        - ../../data/vector:/var/lib/vector
         - ../plugins/aspects/apps/vector/local.toml:/etc/vector/vector.toml:ro
         {% if ASPECTS_DOCKER_HOST_SOCK_PATH %}- {{ ASPECTS_DOCKER_HOST_SOCK_PATH }}:/var/run/docker.sock:ro{% endif %}
     environment:

tutoraspects/plugin.py

@@ -31,10 +31,10 @@
         ("RUN_CLICKHOUSE", True),
         ("RUN_RALPH", True),
         ("RUN_SUPERSET", True),
-        ("DOCKER_IMAGE_ASPECTS", "python:3.8"),
+        ("DOCKER_IMAGE_ASPECTS", "edunext/aspects:{{ ASPECTS_VERSION }}"),
         ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:23.3"),
         ("DOCKER_IMAGE_RALPH", "fundocker/ralph:3.8.0"),
-        ("DOCKER_IMAGE_SUPERSET", "apache/superset:2.0.1"),
+        ("DOCKER_IMAGE_SUPERSET", "edunext/aspects-superset:{{ ASPECTS_VERSION }}"),
         ("DOCKER_IMAGE_VECTOR", "timberio/vector:0.30.0-alpine"),
         (
             "OPENEDX_EXTRA_PIP_REQUIREMENTS",
@@ -112,6 +112,7 @@
         ("SUPERSET_DB_PORT", "{{ MYSQL_PORT }}"),
         ("SUPERSET_DB_NAME", "superset"),
         ("SUPERSET_DB_USERNAME", "superset"),
+        ("SUPERSET_EXTRA_REQUIREMENTS", []),
         ("SUPERSET_OAUTH2_ACCESS_TOKEN_PATH", "/oauth2/access_token/"),
         ("SUPERSET_OAUTH2_AUTHORIZE_PATH", "/oauth2/authorize/"),
         (
@@ -160,7 +161,7 @@
         ("DBT_REPOSITORY_PATH", "aspects-dbt/aspects"),
         # This is a pip compliant list of Python packages to install to run dbt
         # make sure packages with versions are enclosed in double quotes
-        ("DBT_PACKAGES", '"dbt-core==1.4.0" "dbt-clickhouse==1.4.1"'),
+        ("EXTRA_DBT_PACKAGES", []),
         # If set, DDL/table operations will be executed with the `ON CLUSTER` clause
         # using this cluster. This has not been tested with Aspects and is unlikely to
         # work.
@@ -275,32 +276,10 @@
 # ("<service>", ("<path>", "<to>", "<script>", "<template>"))
 MY_INIT_TASKS: list[tuple[str, tuple[str, ...], int]] = [
     ("mysql", ("aspects", "jobs", "init", "superset", "init-mysql.sh"), 92),
-    ("superset", ("aspects", "jobs", "init", "superset", "init-superset.sh"), 93),
-    ("lms", ("aspects", "jobs", "init", "superset", "init-openedx.sh"), 94),
-    ("clickhouse", ("aspects", "jobs", "init", "clickhouse", "init-clickhouse.sh"), 95),
-    (
-        "clickhouse",
-        (
-            "aspects",
-            "jobs",
-            "init",
-            "clickhouse",
-            "aspects_init_schemas_tables_users.sh",
-        ),
-        96,
-    ),
-    ("aspects", ("aspects", "jobs", "init", "dbt", "init-dbt.sh"), 97),
-    (
-        "superset",
-        ("aspects", "jobs", "init", "superset", "superset-init-security.sh"),
-        99,
-    ),
-    ("lms", ("aspects", "jobs", "init", "lms", "configure-aspects-lms.sh"), 100),
-    (
-        "superset",
-        ("aspects", "jobs", "init", "superset", "superset-api-dashboard.sh"),
-        101,
-    ),
+    ("clickhouse", ("aspects", "jobs", "init", "clickhouse", "init-clickhouse.sh"), 93),
+    ("superset", ("aspects", "jobs", "init", "superset", "init-superset.sh"), 94),
+    ("lms", ("aspects", "jobs", "init", "superset", "init-openedx.sh"), 95),
+    ("aspects", ("aspects", "jobs", "init", "dbt", "init-dbt.sh"), 96),
 ]
 
 # For each task added to MY_INIT_TASKS, we load the task template
@@ -324,15 +303,18 @@
 #     ("<tutor_image_name>", ("path", "to", "build", "dir"), "<docker_image_tag>", "<build_args>")
 hooks.Filters.IMAGES_BUILD.add_items(
     [
-        # To build `myimage` with `tutor images build myimage`,
-        # you would add a Dockerfile to templates/aspects/build/myimage,
-        # and then write:
-        ### (
-        ###     "myimage",
-        ###     ("plugins", "aspects", "build", "myimage"),
-        ###     "docker.io/myimage:{{ ASPECTS_VERSION }}",
-        ###     (),
-        ### ),
+        (
+            "aspects-superset",
+            ("plugins", "aspects", "build", "aspects-superset"),
+            "{{DOCKER_IMAGE_SUPERSET}}",
+            (),
+        ),
+        (
+            "aspects",
+            ("plugins", "aspects", "build", "aspects"),
+            "{{DOCKER_IMAGE_ASPECTS}}",
+            (),
+        ),
     ]
 )
 
@@ -341,11 +323,14 @@
 #     ("<tutor_image_name>", "<docker_image_tag>")
 hooks.Filters.IMAGES_PULL.add_items(
     [
-        # To pull `myimage` with `tutor images pull myimage`, you would write:
-        ### (
-        ###     "myimage",
-        ###     "docker.io/myimage:{{ ASPECTS_VERSION }}",
-        ### ),
+        (
+            "aspects-superset",
+            "{{DOCKER_IMAGE_SUPERSET}}",
+        ),
+        (
+            "aspects",
+            "{{DOCKER_IMAGE_ASPECTS}}",
+        ),
     ]
 )
 
@@ -354,11 +339,14 @@
 #     ("<tutor_image_name>", "<docker_image_tag>")
 hooks.Filters.IMAGES_PUSH.add_items(
     [
-        # To push `myimage` with `tutor images push myimage`, you would write:
-        ### (
-        ###     "myimage",
-        ###     "docker.io/myimage:{{ ASPECTS_VERSION }}",
-        ### ),
+        (
+            "aspects-superset",
+            "{{DOCKER_IMAGE_SUPERSET}}",
+        ),
+        (
+            "aspects",
+            "{{DOCKER_IMAGE_ASPECTS}}",
+        ),
     ]
 )
 

tutoraspects/templates/aspects/apps/scripts/aspects/clickhouse-demo-xapi-data.sh

@@ -4,8 +4,6 @@ python3 -m venv virtualenv
 . virtualenv/bin/activate
 
 echo "Loading demo xAPI data..."
-pip install git+https://github.com/openedx/[email protected]#egg=xapi-db-load==0.2
-pip install pandas  # clickhouse_connect is missing this
 xapi-db-load --backend ralph_clickhouse \
     --num_batches $1 \
     --batch_size $2 \

tutoraspects/templates/aspects/apps/scripts/aspects/dbt.sh

@@ -4,7 +4,8 @@ python3 -m venv virtualenv
 . virtualenv/bin/activate
 
 echo "Installing dbt packages..."
-pip install {{ DBT_PACKAGES }}
+
+pip install -r requirements.txt
 
 echo "Installing aspects-dbt"
 git clone -b {{ DBT_BRANCH }} {{ DBT_REPOSITORY }}

tutoraspects/templates/aspects/apps/scripts/aspects/requirements.txt

@@ -0,0 +1 @@
+{% for requirement in EXTRA_DBT_PACKAGES %}{{ requirement }}{% endfor %}

tutoraspects/templates/aspects/apps/superset/docker/requirements-local.txt

@@ -1,4 +1 @@
-authlib  # OAuth2
-mysqlclient
-clickhouse-connect>0.5,<0.6
-sentry-sdk[flask]
+{% for requirement in SUPERSET_EXTRA_REQUIREMENTS %}{{ requirement }}{% endfor %}

tutoraspects/templates/aspects/jobs/init/superset/superset-init-security.sh → tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py

@@ -1,23 +1,13 @@
-#!/usr/bin/env bash
-set -e
+from superset.app import create_app
 
-#
-# Always install local overrides first
-#
-/usr/bin/env bash /app/docker/docker-bootstrap.sh
+app = create_app()
+app.app_context().push()
 
-# Set up a Row-Level Security filter to enforce course-based access restrictions.
-# Note: there are no cli commands or REST API endpoints to help us with this,
-# so we have to pipe python code directly into the superset shell. Yuck!
-superset shell <<EOF
-from superset.connectors.sqla.models import (
-    RowLevelSecurityFilter,
-    RLSFilterRoles,
-    SqlaTable,
-)
-from superset.utils.core import RowLevelSecurityFilterType
+from superset.connectors.sqla.models import (RLSFilterRoles,
+                                             RowLevelSecurityFilter, SqlaTable)
 from superset.extensions import security_manager
 from superset.migrations.shared.security_converge import Role
+from superset.utils.core import RowLevelSecurityFilterType
 
 session = security_manager.get_session()
 
@@ -43,6 +33,7 @@
     ).filter(
         SqlaTable.table_name == table_name
     ).first()
+    print(table)
     assert table, f"{schema}.{table_name} table doesn't exist yet?"
     # See if the Row Level Security Filter already exists
     rlsf = (
@@ -90,7 +81,3 @@
         session.commit()
 
 print("Successfully create row-level security filters.")
-
-EOF
-# The blank line above EOF is critical -- don't remove it.
-# And we can't have any indented blank lines for some reason, with code piped into the superset shell

tutoraspects/templates/aspects/build/aspects-superset/Dockerfile

@@ -0,0 +1,18 @@
+# Superset image with additional database drivers
+# https://hub.docker.com/r/apache/superset
+# https://github.com/apache/superset/releases
+# https://github.com/apache/superset/blob/master/Dockerfile
+# https://superset.apache.org/docs/databases/installing-database-drivers
+FROM apache/superset:2.0.1
+
+USER root
+
+COPY ./requirements.txt /app/requirements.txt
+
+RUN \
+    {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt \{% endif %}
+    apt-get update && apt-get install -y zip
+
+RUN \
+    {% if is_buildkit_enabled() %}--mount=type=cache,target=/root/.cache/pip,sharing=shared \{% endif %}
+    pip install -r /app/requirements.txt

tutoraspects/templates/aspects/build/aspects-superset/requirements.txt

@@ -0,0 +1,3 @@
+authlib  # OAuth2
+clickhouse-connect>0.5,<0.6
+sentry-sdk[flask]

tutoraspects/templates/aspects/build/aspects/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.8
+
+RUN pip install dbt-core==1.4.0 dbt-clickhouse==1.4.1
+
+RUN pip install git+https://github.com/openedx/[email protected]#egg=xapi-db-load==0.2