From 5352ffb29a004e990d097b7fa9c6931c93d2bbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 25 Oct 2021 19:32:13 +0200 Subject: [PATCH] feat: move all forum-related code to a dedicated plugin Forum is an optional feature, and as such it deserves its own plugin. Starting from Maple, users will be able to install the forum from https://github.com/overhangio/tutor-forum/ Close #450. --- CHANGELOG-nightly.md | 1 + CHANGELOG.md | 2 + Makefile | 2 +- bin/main.py | 1 + docs/configuration.rst | 11 +---- docs/plugins/api.rst | 2 +- requirements/plugins.txt | 1 + tutor/commands/images.py | 2 +- tutor/config.py | 1 - tutor/jobs.py | 2 +- .../apps/openedx/config/lms.env.json | 2 - .../apps/openedx/settings/lms/development.py | 1 - tutor/templates/build/forum/Dockerfile | 48 ------------------- .../build/forum/bin/docker-entrypoint.sh | 16 ------- tutor/templates/config.yml | 4 -- tutor/templates/k8s/deployments.yml | 39 --------------- tutor/templates/k8s/jobs.yml | 25 ---------- tutor/templates/k8s/services.yml | 14 ------ tutor/templates/local/docker-compose.jobs.yml | 10 ---- tutor/templates/local/docker-compose.yml | 16 ------- 20 files changed, 10 insertions(+), 190 deletions(-) delete mode 100644 tutor/templates/build/forum/Dockerfile delete mode 100755 tutor/templates/build/forum/bin/docker-entrypoint.sh diff --git a/CHANGELOG-nightly.md b/CHANGELOG-nightly.md index 8e6ffe15d5d..e7c11217742 100644 --- a/CHANGELOG-nightly.md +++ b/CHANGELOG-nightly.md @@ -2,6 +2,7 @@ Note: Breaking changes between versions are indicated by "💥". +- 💥[Improvement] Move the Open edX forum to a [dedicated plugin](https://github.com/overhangio/tutor-forum/) (#450). - 💥[Improvement] Run all services as unprivileged containers, for better security. This has multiple consequences: - The "openedx-dev" image is now built with `tutor dev dc build lms`. - The "smtp" service now runs the "devture/exim-relay" Docker image, which is unprivileged. Also, the default SMTP port is now 8025. diff --git a/CHANGELOG.md b/CHANGELOG.md index ac08a332db4..e18b3484c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +## v12.1.5 (2021-10-25) + - 💥[Improvement] Change the `settheme` command such that, by default, a custom theme is assigned to the LMS and the CMS, both in production and development mode. ## v12.1.4 (2021-10-11) diff --git a/Makefile b/Makefile index 9ddd9c355c9..e2b854e3fc9 100644 --- a/Makefile +++ b/Makefile @@ -98,7 +98,7 @@ ci-test-bundle: ## Run basic tests on bundle yes "" | ./dist/tutor config save --interactive ./dist/tutor config save ./dist/tutor plugins list - ./dist/tutor plugins enable android discovery ecommerce license mfe minio notes webui xqueue + ./dist/tutor plugins enable android discovery ecommerce forum license mfe minio notes webui xqueue ./dist/tutor plugins list ./dist/tutor license --help diff --git a/bin/main.py b/bin/main.py index e259de2840f..4c294418522 100755 --- a/bin/main.py +++ b/bin/main.py @@ -6,6 +6,7 @@ "android", "discovery", "ecommerce", + "forum", "license", "mfe", "minio", diff --git a/docs/configuration.rst b/docs/configuration.rst index 16aafee2e90..1a972e1abf6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -42,7 +42,6 @@ Individual service activation - ``RUN_LMS`` (default: ``true``) - ``RUN_CMS`` (default: ``true``) -- ``RUN_FORUM`` (default: ``true``) - ``RUN_ELASTICSEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) @@ -61,9 +60,8 @@ Custom images ************* - ``DOCKER_IMAGE_OPENEDX`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}"``) -- ``DOCKER_IMAGE_FORUM`` (default: ``"{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}"``) -These configuration parameters define which image to run for each service. By default, the docker image tag matches the Tutor version it was built with. +Theis configuration parameter defines the name of the Docker image to run for the lms and cms containers. By default, the Docker image tag matches the Tutor version it was built with. Custom registry *************** @@ -165,13 +163,6 @@ SMTP Note that the SMTP server shipped with Tutor by default does not implement TLS. With external servers, only one of SSL or TLS should be enabled, at most. -Forum -***** - -- ``RUN_FORUM`` (default: ``true``) -- ``FORUM_HOST`` (default: ``"forum"``) -- ``FORUM_MONGODB_DATABASE`` (default: ``"cs_comments_service"``) - SSL/TLS certificates for HTTPS access ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/plugins/api.rst b/docs/plugins/api.rst index cc85a9d4718..d989e95a6b2 100644 --- a/docs/plugins/api.rst +++ b/docs/plugins/api.rst @@ -83,7 +83,7 @@ Example:: During initialisation, "myservice1" and "myservice2" will be run in sequence with the commands defined in the templates ``myplugin/hooks/myservice1/init`` and ``myplugin/hooks/myservice2/init``. -To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job, forum-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. +To initialise a "foo" service, Tutor runs the "foo-job" service that is found in the ``env/local/docker-compose.jobs.yml`` file. By default, Tutor comes with a few services in this file: mysql-job, lms-job, cms-job. If your plugin requires running custom services during initialisation, you will need to add them to the ``docker-compose.jobs.yml`` template. To do so, just use the "local-docker-compose-jobs-services" patch. In Kubernetes, the approach is the same, except that jobs are implemented as actual job objects in the ``k8s/jobs.yml`` template. To add your own services there, your plugin should implement the "k8s-jobs" patch. diff --git a/requirements/plugins.txt b/requirements/plugins.txt index da66ffdfe5e..85053a377d6 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -2,6 +2,7 @@ tutor-android>=12.0.0,<13.0.0 tutor-discovery>=12.0.0,<13.0.0 tutor-ecommerce>=12.0.0,<13.0.0 +tutor-forum>=12.0.0,<13.0.0 tutor-license>=12.0.0,<13.0.0 tutor-mfe>=12.0.0,<13.0.0 tutor-minio>=12.0.0,<13.0.0 diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 5982f4cd767..4a0fdd34c96 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -10,7 +10,7 @@ from ..types import Config from .context import Context -BASE_IMAGE_NAMES = ["openedx", "forum", "permissions"] +BASE_IMAGE_NAMES = ["openedx", "permissions"] VENDOR_IMAGES = [ "caddy", "elasticsearch", diff --git a/tutor/config.py b/tutor/config.py index 7770c826f9e..3511ede27a1 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -165,7 +165,6 @@ def upgrade_obsolete(config: Config) -> None: for name in [ "ACTIVATE_LMS", "ACTIVATE_CMS", - "ACTIVATE_FORUM", "ACTIVATE_ELASTICSEARCH", "ACTIVATE_MONGODB", "ACTIVATE_MYSQL", diff --git a/tutor/jobs.py b/tutor/jobs.py index 0c16ab20f7c..2df90d8044e 100644 --- a/tutor/jobs.py +++ b/tutor/jobs.py @@ -49,7 +49,7 @@ def initialise(runner: BaseJobRunner, limit_to: Optional[str] = None) -> None: runner.run_job_from_template( service, plugin_name, "hooks", service, "pre-init" ) - for service in ["lms", "cms", "forum"]: + for service in ["lms", "cms"]: if limit_to is None or limit_to == service: fmt.echo_info("Initialising {}...".format(service)) runner.run_job_from_template(service, "hooks", service, "init") diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json index 3b730799143..e39533a2a12 100644 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ b/tutor/templates/apps/openedx/config/lms.env.json @@ -35,8 +35,6 @@ "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", "ALTERNATE_WORKER_QUEUES": "cms", - "COMMENTS_SERVICE_URL": "http://{{ FORUM_HOST }}:4567", - "COMMENTS_SERVICE_KEY": "forumapikey", "ENABLE_COMPREHENSIVE_THEMING": true, "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], "STATIC_ROOT_BASE": "/openedx/staticfiles", diff --git a/tutor/templates/apps/openedx/settings/lms/development.py b/tutor/templates/apps/openedx/settings/lms/development.py index 09c92a493fb..10388901b84 100644 --- a/tutor/templates/apps/openedx/settings/lms/development.py +++ b/tutor/templates/apps/openedx/settings/lms/development.py @@ -18,7 +18,6 @@ LOGIN_REDIRECT_WHITELIST.append(CMS_BASE) FEATURES['ENABLE_COURSEWARE_MICROFRONTEND'] = False -COMMENTS_SERVICE_URL = "http://{{ FORUM_HOST }}:4567" LOGGING["loggers"]["oauth2_provider"] = { "handlers": ["console"], diff --git a/tutor/templates/build/forum/Dockerfile b/tutor/templates/build/forum/Dockerfile deleted file mode 100644 index cdb059e51bc..00000000000 --- a/tutor/templates/build/forum/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM docker.io/ruby:2.5.7-slim-stretch -MAINTAINER Overhang.io - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ - apt upgrade -y && \ - apt install -y git wget autoconf bison build-essential libssl-dev libyaml-dev libreadline6-dev zlib1g-dev libncurses5-dev libffi-dev libgdbm-dev - -# Install dockerize to wait for mongodb/elasticsearch availability -ARG DOCKERIZE_VERSION=v0.6.1 -RUN wget -O /tmp/dockerize.tar.gz https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && tar -C /usr/local/bin -xzvf /tmp/dockerize.tar.gz \ - && rm /tmp/dockerize.tar.gz - -# Create unprivileged "app" user -RUN useradd --home-dir /app --create-home --shell /bin/bash --uid 1000 app - -# Copy custom scripts -COPY ./bin /app/bin -RUN chmod a+x /app/bin/* -ENV PATH :${PATH} - -# From then on, run as unprivileged app user -USER app - -# Install rake and bundler -ENV PATH "/app/bin:/app/.gem/ruby/2.5.0/bin:$PATH" -RUN gem install --user-install bundler --version 1.17.3 -RUN gem install --user-install rake --version 13.0.1 - -# Install forum -RUN git clone https://github.com/edx/cs_comments_service.git --branch {{ OPENEDX_COMMON_VERSION }} --depth 1 /app/cs_comments_service -WORKDIR /app/cs_comments_service -RUN bundle install --deployment - -ENTRYPOINT ["docker-entrypoint.sh"] - -ENV SINATRA_ENV staging -ENV NEW_RELIC_ENABLE false -ENV API_KEY forumapikey -ENV SEARCH_SERVER "http://elasticsearch:9200" -ENV MONGODB_AUTH "" -ENV MONGOID_AUTH_MECH "" -ENV MONGODB_HOST "mongodb" -ENV MONGODB_PORT "27017" -ENV MONGODB_DATABASE "cs_comments_service" -EXPOSE 4567 -CMD ./bin/unicorn -c config/unicorn_tcp.rb -I '.' diff --git a/tutor/templates/build/forum/bin/docker-entrypoint.sh b/tutor/templates/build/forum/bin/docker-entrypoint.sh deleted file mode 100755 index 4f5ac7c6cfd..00000000000 --- a/tutor/templates/build/forum/bin/docker-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -export MONGOHQ_URL="mongodb://$MONGODB_AUTH$MONGODB_HOST:$MONGODB_PORT/$MONGODB_DATABASE" -# the search server variable was renamed after the upgrade to elasticsearch 7 -export SEARCH_SERVER_ES7="$SEARCH_SERVER" - -# make sure that there is an actual authentication mechanism in place, if necessary -if [ -n "$MONGODB_AUTH" ] -then - export MONGOID_AUTH_MECH=":scram" -fi - -echo "Waiting for mongodb/elasticsearch..." -dockerize -wait tcp://$MONGODB_HOST:$MONGODB_PORT -wait $SEARCH_SERVER -wait-retry-interval 5s -timeout 600s - -exec "$@" diff --git a/tutor/templates/config.yml b/tutor/templates/config.yml index eb28c0a4741..acd48823b3b 100644 --- a/tutor/templates/config.yml +++ b/tutor/templates/config.yml @@ -11,7 +11,6 @@ LMS_HOST: "www.myopenedx.com" # The following are default values RUN_LMS: true RUN_CMS: true -RUN_FORUM: true RUN_ELASTICSEARCH: true RUN_MONGODB: true RUN_MYSQL: true @@ -29,7 +28,6 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev" DOCKER_IMAGE_CADDY: "{{ DOCKER_REGISTRY }}caddy:2.3.0" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" -DOCKER_IMAGE_FORUM: "{{ DOCKER_REGISTRY }}overhangio/openedx-forum:{{ TUTOR_VERSION }}" DOCKER_IMAGE_MONGODB: "{{ DOCKER_REGISTRY }}mongo:4.2.17" DOCKER_IMAGE_MYSQL: "{{ DOCKER_REGISTRY }}mysql:5.7.35" DOCKER_IMAGE_ELASTICSEARCH: "{{ DOCKER_REGISTRY }}elasticsearch:7.10.1" @@ -44,8 +42,6 @@ ELASTICSEARCH_SCHEME: "http" ELASTICSEARCH_HEAP_SIZE: 1g ENABLE_HTTPS: false ENABLE_WEB_PROXY: true -FORUM_HOST: "forum" -FORUM_MONGODB_DATABASE: "cs_comments_service" JWT_COMMON_AUDIENCE: "openedx" JWT_COMMON_ISSUER: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}/oauth2" JWT_COMMON_SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index ddfa4f1eeec..4295ef413f3 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -130,45 +130,6 @@ spec: configMap: name: openedx-config {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: forum - labels: - app.kubernetes.io/name: forum -spec: - selector: - matchLabels: - app.kubernetes.io/name: forum - template: - metadata: - labels: - app.kubernetes.io/name: forum - spec: - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - ports: - - containerPort: 4567 - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" - securityContext: - allowPrivilegeEscalation: false -{% endif %} {% if RUN_LMS %} --- apiVersion: apps/v1 diff --git a/tutor/templates/k8s/jobs.yml b/tutor/templates/k8s/jobs.yml index 96a4d3ad06b..d514d01997f 100644 --- a/tutor/templates/k8s/jobs.yml +++ b/tutor/templates/k8s/jobs.yml @@ -77,30 +77,5 @@ spec: containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: forum-job - labels: - app.kubernetes.io/component: job -spec: - template: - spec: - restartPolicy: Never - containers: - - name: forum - image: {{ DOCKER_IMAGE_FORUM }} - env: - - name: SEARCH_SERVER - value: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - - name: MONGODB_AUTH - value: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - - name: MONGODB_HOST - value: "{{ MONGODB_HOST }}" - - name: MONGODB_PORT - value: "{{ MONGODB_PORT }}" - - name: MONGODB_DATABASE - value: "{{ FORUM_MONGODB_DATABASE }}" {{ patch("k8s-jobs") }} diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 68e78ec81fd..b768098f8dc 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -28,20 +28,6 @@ spec: selector: app.kubernetes.io/name: cms {% endif %} -{% if RUN_FORUM %} ---- -apiVersion: v1 -kind: Service -metadata: - name: forum -spec: - type: NodePort - ports: - - port: 4567 - protocol: TCP - selector: - app.kubernetes.io/name: forum -{% endif %} {% if RUN_LMS %} --- apiVersion: v1 diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 26b123788df..880660f0e43 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -27,14 +27,4 @@ services: - ../apps/openedx/config/:/openedx/config/:ro depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} - forum-job: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index e79294a4e59..163a70e69b1 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -97,21 +97,6 @@ services: HOSTNAME: "{{ LMS_HOST }}" {% endif %} - ############# Forum - - {% if RUN_FORUM %} - forum: - image: {{ DOCKER_IMAGE_FORUM }} - environment: - SEARCH_SERVER: "{{ ELASTICSEARCH_SCHEME }}://{{ ELASTICSEARCH_HOST }}:{{ ELASTICSEARCH_PORT }}" - MONGODB_AUTH: "{% if MONGODB_USERNAME and MONGODB_PASSWORD %}{{ MONGODB_USERNAME}}:{{ MONGODB_PASSWORD }}@{% endif %}" - MONGODB_HOST: "{{ MONGODB_HOST }}" - MONGODB_PORT: "{{ MONGODB_PORT }}" - MONGODB_DATABASE: "{{ FORUM_MONGODB_DATABASE }}" - restart: unless-stopped - depends_on: {{ [("elasticsearch", RUN_ELASTICSEARCH), ("mongodb", RUN_MONGODB)]|list_if }} - {% endif %} - ############# LMS and CMS {% if RUN_LMS %} @@ -132,7 +117,6 @@ services: - lms-permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} - {% if RUN_FORUM %}- forum{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %}