feat: use custom aspects image

tutoraspects/plugin.py

@@ -31,7 +31,7 @@
         ("RUN_CLICKHOUSE", True),
         ("RUN_RALPH", True),
         ("RUN_SUPERSET", True),
-        ("DOCKER_IMAGE_ASPECTS", "python:3.8"),
+        ("DOCKER_IMAGE_ASPECTS", "edunext/aspects:{{ ASPECTS_VERSION }}"),
         ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:23.3"),
         ("DOCKER_IMAGE_RALPH", "fundocker/ralph:3.8.0"),
         ("DOCKER_IMAGE_SUPERSET", "edunext/aspects-superset:{{ ASPECTS_VERSION }}"),
@@ -161,7 +161,7 @@
         ("DBT_REPOSITORY_PATH", "aspects-dbt/aspects"),
         # This is a pip compliant list of Python packages to install to run dbt
         # make sure packages with versions are enclosed in double quotes
-        ("DBT_PACKAGES", '"dbt-core==1.4.0" "dbt-clickhouse==1.4.1"'),
+        ("EXTRA_DBT_PACKAGES", []),
         # If set, DDL/table operations will be executed with the `ON CLUSTER` clause
         # using this cluster. This has not been tested with Aspects and is unlikely to
         # work.
@@ -309,6 +309,12 @@
             "{{DOCKER_IMAGE_SUPERSET}}",
             (),
         ),
+        (
+            "aspects",
+            ("plugins", "aspects", "build", "aspects"),
+            "{{DOCKER_IMAGE_ASPECTS}}",
+            (),
+        ),
     ]
 )
 
@@ -321,6 +327,10 @@
             "aspects-superset",
             "{{DOCKER_IMAGE_SUPERSET}}",
         ),
+        (
+            "aspects",
+            "{{DOCKER_IMAGE_ASPECTS}}",
+        ),
     ]
 )
 

tutoraspects/templates/aspects/apps/scripts/aspects/clickhouse-demo-xapi-data.sh

@@ -4,8 +4,6 @@ python3 -m venv virtualenv
 . virtualenv/bin/activate
 
 echo "Loading demo xAPI data..."
-pip install git+https://github.com/openedx/[email protected]#egg=xapi-db-load==0.2
-pip install pandas  # clickhouse_connect is missing this
 xapi-db-load --backend ralph_clickhouse \
     --num_batches $1 \
     --batch_size $2 \

tutoraspects/templates/aspects/apps/scripts/aspects/dbt.sh

@@ -4,7 +4,8 @@ python3 -m venv virtualenv
 . virtualenv/bin/activate
 
 echo "Installing dbt packages..."
-pip install {{ DBT_PACKAGES }}
+
+pip install -r requirements.txt
 
 echo "Installing aspects-dbt"
 git clone -b {{ DBT_BRANCH }} {{ DBT_REPOSITORY }}

tutoraspects/templates/aspects/apps/scripts/aspects/requirements.txt

@@ -0,0 +1 @@
+{% for requirement in EXTRA_DBT_PACKAGES %}{{ requirement }}{% endfor %}

tutoraspects/templates/aspects/build/aspects/Dockerfile

@@ -1,12 +1,5 @@
-# Superset image with additional database drivers
-# https://hub.docker.com/r/apache/superset
-# https://github.com/apache/superset/releases
-# https://github.com/apache/superset/blob/master/Dockerfile
-# https://superset.apache.org/docs/databases/installing-database-drivers
-FROM python3.8
+FROM python:3.8
 
-USER root
+RUN pip install dbt-core==1.4.0 dbt-clickhouse==1.4.1 pandas
 
-#COPY ./requirements.txt /app/requirements.txt
-
-RUN echo "Hi"
+RUN pip install git+https://github.com/openedx/[email protected]#egg=xapi-db-load==0.2

tutoraspects/templates/aspects/jobs/init/dbt/init-dbt.sh

@@ -4,7 +4,8 @@ python3 -m venv virtualenv
 . virtualenv/bin/activate
 
 echo "Installing dbt packages..."
-pip install {{ DBT_PACKAGES }}
+
+pip install -r /app/aspects/scripts/aspects/requirements.txt
 
 echo "Installing aspects-dbt"
 git clone -b {{ DBT_BRANCH }} {{ DBT_REPOSITORY }}