From 998dcfd7bcdf75ec7c1e7d7761b52cf788aa35d6 Mon Sep 17 00:00:00 2001 From: Emad Ehsanrad Date: Mon, 10 Jul 2023 20:34:59 +0330 Subject: [PATCH 01/96] chore: version updated --- tutor/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/__about__.py b/tutor/__about__.py index e05ea243688..af3394f2628 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "15.3.7" +__version__ = "15.3.8" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From b62fb25d1325cfb731b4f036289e4788e8ae3924 Mon Sep 17 00:00:00 2001 From: Emad Ehsanrad Date: Mon, 10 Jul 2023 20:36:38 +0330 Subject: [PATCH 02/96] fix: load kube config from file --- tutor/commands/k8s.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index ce8c0c41b8e..b761fd0b895 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -27,7 +27,9 @@ def __init__(self) -> None: # pylint: disable=import-outside-toplevel from kubernetes import client, config - if os.path.exists(config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION): + if os.path.exists( + os.path.expanduser(config.kube_config.KUBE_CONFIG_DEFAULT_LOCATION) + ): # found the kubeconfig file, let's load it! config.load_kube_config() elif ( From d16be77b0bc1ec9e16987b360c2623a51d020da2 Mon Sep 17 00:00:00 2001 From: Emad Ehsanrad Date: Mon, 10 Jul 2023 20:37:06 +0330 Subject: [PATCH 03/96] fix: backoffLimit and ttl value increased --- tutor/commands/k8s.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index b761fd0b895..231b18a4d58 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -113,8 +113,8 @@ def run_task(self, service: str, command: str) -> int: else: container_args = shell_command + [command] job["spec"]["template"]["spec"]["containers"][0]["args"] = container_args - job["spec"]["backoffLimit"] = 1 - job["spec"]["ttlSecondsAfterFinished"] = 3600 + job["spec"]["backoffLimit"] = 5 + job["spec"]["ttlSecondsAfterFinished"] = 72000 # 24 hours with open( tutor_env.pathjoin(self.root, "k8s", "jobs.yml"), "w", encoding="utf-8" From 19dcbbf15615c970a190180865b8f2b21fde3d34 Mon Sep 17 00:00:00 2001 From: Emad Ehsanrad Date: Mon, 10 Jul 2023 20:37:18 +0330 Subject: [PATCH 04/96] chore: kind label added --- tutor/templates/kustomization.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tutor/templates/kustomization.yml b/tutor/templates/kustomization.yml index f29fea1e818..4ba22a05545 100644 --- a/tutor/templates/kustomization.yml +++ b/tutor/templates/kustomization.yml @@ -33,36 +33,42 @@ configMapGenerator: options: labels: app.kubernetes.io/name: caddy + app.kubernetes.io/kind: ConfigMap - name: openedx-settings-lms files:{% for file in "apps/openedx/settings/lms"|walk_templates %} - {{ file }}{% endfor %} options: labels: app.kubernetes.io/name: openedx + app.kubernetes.io/kind: ConfigMap - name: openedx-settings-cms files:{% for file in "apps/openedx/settings/cms"|walk_templates %} - {{ file }}{% endfor %} options: labels: app.kubernetes.io/name: openedx + app.kubernetes.io/kind: ConfigMap - name: openedx-config files:{% for file in "apps/openedx/config"|walk_templates %} - {{ file }}{% endfor %} options: labels: app.kubernetes.io/name: openedx + app.kubernetes.io/kind: ConfigMap - name: openedx-uwsgi-config files: - apps/openedx/uwsgi.ini options: labels: app.kubernetes.io/name: openedx + app.kubernetes.io/kind: ConfigMap - name: redis-config files: - apps/redis/redis.conf options: labels: app.kubernetes.io/name: redis + app.kubernetes.io/kind: ConfigMap {{ patch("kustomization-configmapgenerator") }} {%- if patch("k8s-override") or patch("kustomization-patches-strategic-merge") %} From 39a206b69963cdd769c29af6ec9a8114d9d764c6 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Wed, 11 Oct 2023 10:35:03 +0330 Subject: [PATCH 05/96] fix: build error caused by removed py2neo package On Oct. 10, py2neo package was abruptly removed from pypi, GitHub, and the py2neo website now displays just a super funny meme: https://py2neo.org/ --- tutor/templates/build/openedx/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 698f6311f45..a8da6b2a559 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -69,6 +69,9 @@ RUN apt update && apt install -y software-properties-common libmysqlclient-dev l # https://pypi.org/project/wheel/ RUN pip install setuptools==65.5.1 pip==22.3.1. wheel==0.38.4 +# Install missing py2neo package that was abruptly trimmed from pypi +RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz + # Install base requirements COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt RUN pip install -r /tmp/base.txt From 3c5ca4a2785c76b84720472911349c597076fa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 27 Apr 2023 16:55:42 +0200 Subject: [PATCH 06/96] feat: faster builds with registry cache Automatically pull Docker build cache from remote registry. This considerably improves build performances, as discovered here: https://github.com/overhangio/test-docker-build/ --- .../20230427_165520_regis_build_mount.md | 1 + tutor/commands/images.py | 39 +++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 changelog.d/20230427_165520_regis_build_mount.md diff --git a/changelog.d/20230427_165520_regis_build_mount.md b/changelog.d/20230427_165520_regis_build_mount.md new file mode 100644 index 00000000000..fb29b8cf85c --- /dev/null +++ b/changelog.d/20230427_165520_regis_build_mount.md @@ -0,0 +1 @@ +- [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 06d2a2bb67a..c300ff80616 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -6,7 +6,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, hooks, images +from tutor import exceptions, hooks, images, utils from tutor.commands.context import Context from tutor.core.hooks import Filter from tutor.types import Config @@ -68,14 +68,21 @@ def images_command() -> None: pass -@click.command( - short_help="Build docker images", - help="Build the docker images necessary for an Open edX platform.", -) +@click.command() @click.argument("image_names", metavar="image", nargs=-1) @click.option( "--no-cache", is_flag=True, help="Do not use cache when building the image" ) +@click.option( + "--no-registry-cache", + is_flag=True, + help="Do not use registry cache when building the image", +) +@click.option( + "--cache-to-registry", + is_flag=True, + help="Push the build cache to the remote registry. You should only enable this option if you have push rights to the remote registry.", +) @click.option( "-a", "--build-arg", @@ -105,11 +112,19 @@ def build( context: Context, image_names: list[str], no_cache: bool, + no_registry_cache: bool, + cache_to_registry: bool, build_args: list[str], add_hosts: list[str], target: str, docker_args: list[str], ) -> None: + """ + Build docker images + + Build the docker images necessary for an Open edX platform. By default, the remote + registry cache will be used for better performance. + """ config = tutor_config.load(context.root) command_args = [] if no_cache: @@ -120,15 +135,25 @@ def build( command_args += ["--add-host", add_host] if target: command_args += ["--target", target] + if utils.is_buildkit_enabled(): + # Export image to docker. + command_args.append("--output=type=image") if docker_args: command_args += docker_args for image in image_names: for _name, path, tag, custom_args in find_images_to_build(config, image): + image_build_args = [*command_args, *custom_args] + if not no_registry_cache: + # Use registry cache + image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") + if cache_to_registry: + image_build_args.append( + f"--cache-to=type=registry,mode=max,ref={tag}-cache" + ) images.build( tutor_env.pathjoin(context.root, *path), tag, - *command_args, - *custom_args, + *image_build_args, ) From bc478bb62d49d00ef38ef4fa1d3c62c9d4abf5fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 12 Apr 2023 10:35:00 +0200 Subject: [PATCH 07/96] feat: upgrade to Palm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Among other changes: ORA2 file uploads were stored in a folder named "SET-ME-PLEASE (ex. bucket-name)" (sigh). With this change, the folder should be automatically renamed to "openedxuploads". This issue has been occuring since June 2019... (sigh²) Close #707 --- changelog.d/20230412_100608_regis_palm.md | 2 ++ docs/configuration.rst | 28 ++++++++++++------- docs/dev.rst | 2 +- docs/faq.rst | 2 +- docs/quickstart.rst | 2 +- docs/reference/indexes.rst | 10 +++---- docs/tutorials/arm64.rst | 23 ++------------- requirements/plugins.txt | 22 +++++++-------- tests/commands/test_images.py | 2 +- tests/test_env.py | 2 +- tutor/__about__.py | 2 +- tutor/commands/dev.py | 12 ++------ tutor/commands/local.py | 15 ++-------- tutor/commands/upgrade/__init__.py | 12 ++++++-- tutor/commands/upgrade/common.py | 8 ++++++ tutor/commands/upgrade/k8s.py | 20 +++++++++++++ tutor/commands/upgrade/local.py | 15 ++++++++++ tutor/env.py | 8 ++++-- .../openedx/settings/partials/common_all.py | 11 ++++++-- .../openedx/settings/partials/common_lms.py | 5 ---- tutor/templates/build/openedx/Dockerfile | 16 +++++------ tutor/templates/build/openedx/revisions.yml | 2 +- tutor/templates/config/defaults.yml | 14 +++++----- tutor/utils.py | 13 +++++++++ 24 files changed, 145 insertions(+), 103 deletions(-) create mode 100644 changelog.d/20230412_100608_regis_palm.md diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md new file mode 100644 index 00000000000..381ba296636 --- /dev/null +++ b/changelog.d/20230412_100608_regis_palm.md @@ -0,0 +1,2 @@ +- 💥[Feature] Upgrade to Palm. (by @regisb) + - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. diff --git a/docs/configuration.rst b/docs/configuration.rst index 254e751cbec..b36f4930adf 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -67,11 +67,13 @@ This configuration parameter defines the name of the Docker image to run for the This configuration paramater defines the name of the Docker image to run the development version of the lms and cms containers. By default, the Docker image tag matches the Tutor version it was built with. +.. https://hub.docker.com/r/devture/exim-relay/tags + - ``DOCKER_IMAGE_CADDY`` (default: ``"docker.io/caddy:2.6.2"``) This configuration paramater defines which Caddy Docker image to use. -- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.10.1"``) +- ``DOCKER_IMAGE_ELASTICSEARCH`` (default: ``"docker.io/elasticsearch:7.17.9"``) This configuration parameter defines which Elasticsearch Docker image to use. @@ -79,15 +81,21 @@ This configuration parameter defines which Elasticsearch Docker image to use. This configuration parameter defines which MongoDB Docker image to use. -- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:5.7.35"``) +.. https://hub.docker.com/_/mysql/tags?page=1&name=8.0 + +- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.0.33"``) This configuration parameter defines which MySQL Docker image to use. -- ``DOCKER_IMAGE_REDIS`` (default: ``"docker.io/redis:6.2.6"``) +.. https://hub.docker.com/_/redis/tags + +- ``DOCKER_IMAGE_REDIS`` (default: ``"docker.io/redis:7.0.11"``) This configuration parameter defines which Redis Docker image to use. -- ``DOCKER_IMAGE_SMTP`` (default: ``"docker.io/devture/exim-relay:4.95-r0-2``) +.. https://hub.docker.com/r/devture/exim-relay/tags + +- ``DOCKER_IMAGE_SMTP`` (default: ``"docker.io/devture/exim-relay:4.96-r1-0``) This configuration parameter defines which Simple Mail Transfer Protocol (SMTP) Docker image to use. @@ -130,7 +138,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/olive.4"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.master"``) This defines the default version that will be pulled from all Open edX git repositories. @@ -150,7 +158,7 @@ These two configuration parameters define which Redis database to use for Open e .. _openedx_extra_pip_requirements: -- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``["openedx-scorm-xblock>=15.0.0,<16.0.0"]``) +- ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` (default: ``["openedx-scorm-xblock>=16.0.0,<17.0.0"]``) This defines extra pip packages that are going to be installed for Open edX. @@ -404,14 +412,14 @@ If you don't create your fork from this tag, you *will* have important compatibi - Do not try to run a fork from an older (pre-Olive) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. -- Do not try to run a fork from the open-release/olive.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/olive.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/olive.master branch. +- Do not try to run a fork from the open-release/palm.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/palm.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/palm.master branch. .. _i18n: Adding custom translations ~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. +If you are not running Open edX in English (``LANGUAGE_CODE`` default: ``"en"``), chances are that some strings will not be properly translated. In most cases, this is because not enough contributors have helped translate Open edX into your language. It happens! With Tutor, available translated languages include those that come bundled with `edx-platform `__ as well as those from `openedx-i18n `__. Tutor offers a relatively simple mechanism to add custom translations to the openedx Docker image. You should create a folder that corresponds to your language code in the "build/openedx/locale" folder of the Tutor environment. This folder should contain a "LC_MESSAGES" folder. For instance:: @@ -432,9 +440,9 @@ Then, add a "django.po" file there that will contain your custom translations:: .. warning:: Don't forget to specify the file ``Content-Type`` when adding message strings with non-ASCII characters; otherwise a ``UnicodeDecodeError`` will be raised during compilation. -The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. +The "String to translate" part should match *exactly* the string that you would like to translate. You cannot make it up! The best way to find this string is to copy-paste it from the `upstream django.po file for the English language `__. -If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. +If you cannot find the string to translate in this file, then it means that you are trying to translate a string that is used in some piece of javascript code. Those strings are stored in a different file named "djangojs.po". You can check it out `in the edx-platform repo as well `__. Your custom javascript strings should also be stored in a "djangojs.po" file that should be placed in the same directory. To recap, here is an example. To translate a few strings in French, both from django.po and djangojs.po, we would have the following file hierarchy:: diff --git a/docs/dev.rst b/docs/dev.rst index 13d75fa0bda..5ee10c01a5e 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -128,7 +128,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +- The edx-platform `development requirements `__ are installed. If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: diff --git a/docs/faq.rst b/docs/faq.rst index 045eaa550b1..a2fc8d12faf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -38,7 +38,7 @@ The `devstack `_ is meant for development o Is Tutor officially supported by edX? ------------------------------------- -Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. +Yes: as of the Open edX Maple release (December 9th 2021), Tutor is the only officially supported installation method for Open edX: see the `official installation instructions `__. What features are missing from Tutor? ------------------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5bb1a63b325..cad6dd20773 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,7 +22,7 @@ Yes :) This is what happens when you run ``tutor local launch``: 2. Configuration files are generated from templates. 3. Docker images are downloaded. 4. Docker containers are provisioned. -5. A full, production-ready Open edX platform (`Olive `__ release) is run with docker-compose. +5. A full, production-ready Open edX platform (`Palm `__ release) is run with docker-compose. The whole procedure should require less than 10 minutes, on a server with good bandwidth. Note that your host environment will not be affected in any way, since everything runs inside docker containers. Root access is not even necessary. diff --git a/docs/reference/indexes.rst b/docs/reference/indexes.rst index a1eb7de6f2f..db06fcfadb6 100644 --- a/docs/reference/indexes.rst +++ b/docs/reference/indexes.rst @@ -7,10 +7,10 @@ Plugin indexes are a great way to have your plugins discovered by other users. P Index file paths ================ -A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Olive" release: +A plugin index is a yaml-formatted file. It can be stored on the web or on your computer. In both cases, the index file location must end with "/plugins.yml". For instance, the following are valid index locations if you run the Open edX "Palm" release: -- https://overhang.io/tutor/main/olive/plugins.yml -- ``/path/to/your/local/index/olive/plugins.yml`` +- https://overhang.io/tutor/main/palm/plugins.yml +- ``/path/to/your/local/index/palm/plugins.yml`` To add either indexes, run the ``tutor plugins index add`` command without the suffix. For instance:: @@ -106,9 +106,9 @@ Manage plugins in development Plugin developers and maintainers often want to install local versions of their plugins. They usually achieve this with ``pip install -e /path/to/tutor-plugin``. We can improve that workflow by creating an index for local plugins:: # Create the plugin index directory - mkdir -p ~/localindex/olive/ + mkdir -p ~/localindex/palm/ # Edit the index - vim ~/localindex/olive/plugins.yml + vim ~/localindex/palm/plugins.yml Add the following to the index:: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index a48c6194d11..0ddc177a714 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -7,7 +7,6 @@ Tutor can be used on ARM64 systems, although no official ARM64 docker images are .. note:: There are generally two ways to run Tutor on an ARM system - using emulation (via qemu or Rosetta 2) to run x86_64 images or running native ARM images. Since emulation can be noticeably slower (typically 20-100% slower depending on the emulation method), this tutorial aims to use native images where possible. - Building the images ------------------- @@ -27,30 +26,14 @@ Then, build the "openedx" and "permissions" images:: tutor images build openedx permissions +.. TODO we don't want this instruction anymore If you want to use Tutor as an Open edX development environment, you should also build the development images:: - tutor dev dc build lms -Change the database server --------------------------- - -The version of MySQL that Open edX uses by default (5.7) does not support the ARM architecture. You need to tell Tutor to use MySQL 8.0, which does support the ARM architecture and which has been supported by Open edX since the "Nutmeg" release. - -Configure Tutor to use MySQL 8:: - - tutor config save --set DOCKER_IMAGE_MYSQL=docker.io/mysql:8.0 - -(If you need to run an older release of Open edX on ARM64, you can try using `mariadb:10.4` although it's not officially supported nor recommended for production.) - -Finish setup and start Tutor ----------------------------- - From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: - tutor local start -d - tutor local do init + tutor local launch Or for a development environment:: - tutor dev start -d - tutor dev do init + tutor dev launch diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 7958312d7b4..13e4d0c8006 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,11 +1,11 @@ -# change version ranges when upgrading from olive -tutor-android>=15.0.0,<16.0.0 -tutor-discovery>=15.0.0,<16.0.0 -tutor-ecommerce>=15.0.0,<16.0.0 -tutor-forum>=15.0.0,<16.0.0 -tutor-license>=15.0.0,<16.0.0 -tutor-mfe>=15.0.0,<16.0.0 -tutor-minio>=15.0.0,<16.0.0 -tutor-notes>=15.0.0,<16.0.0 -tutor-webui>=15.0.0,<16.0.0 -tutor-xqueue>=15.0.0,<16.0.0 +# change version ranges when upgrading from palm +tutor-android>=16.0.0,<17.0.0 +tutor-discovery>=16.0.0,<17.0.0 +tutor-ecommerce>=16.0.0,<17.0.0 +tutor-forum>=16.0.0,<17.0.0 +tutor-license>=16.0.0,<17.0.0 +tutor-mfe>=16.0.0,<17.0.0 +tutor-minio>=16.0.0,<17.0.0 +tutor-notes>=16.0.0,<17.0.0 +tutor-webui>=16.0.0,<17.0.0 +tutor-xqueue>=16.0.0,<17.0.0 diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 99e74e941b0..20abdd2bd1e 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -49,7 +49,7 @@ def test_images_pull_all_vendor_images(self, image_pull: Mock) -> None: self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) # Note: we should update this tag whenever the mysql image is updated - image_pull.assert_called_once_with("docker.io/mysql:5.7.35") + image_pull.assert_called_once_with("docker.io/mysql:8.0.33") def test_images_printtag_image(self) -> None: result = self.invoke(["images", "printtag", "openedx"]) diff --git a/tests/test_env.py b/tests/test_env.py index e79d8910ca8..9dd299c0120 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -259,7 +259,7 @@ def test_current_version_in_latest_env(self) -> None: ) as f: f.write(__version__) self.assertEqual(__version__, env.current_version(root)) - self.assertEqual("olive", env.get_env_release(root)) + self.assertEqual("palm", env.get_env_release(root)) self.assertIsNone(env.should_upgrade_from_release(root)) self.assertTrue(env.is_up_to_date(root)) diff --git a/tutor/__about__.py b/tutor/__about__.py index af3394f2628..45ab4966d55 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "15.3.8" +__version__ = "16.0.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 0fcb58df3d0..124c6a9728f 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -4,7 +4,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, hooks +from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils from tutor.commands import compose @@ -73,15 +73,7 @@ def launch( mounts: tuple[list[compose.MountParam.MountType]], ) -> None: compose.mount_tmp_volumes(mounts, context.obj) - try: - utils.check_macos_docker_memory() - except exceptions.TutorError as e: - fmt.echo_alert( - f"""Could not verify sufficient RAM allocation in Docker: - {e} -Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: - https://docs.tutor.overhang.io/install.html""" - ) + utils.warn_macos_docker_memory() click.echo(fmt.title("Interactive platform configuration")) config = tutor_config.load_minimal(context.obj.root) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index e391b9ccfb8..9a91392c069 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -6,7 +6,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, hooks +from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils from tutor.commands import compose @@ -77,18 +77,7 @@ def launch( pullimages: bool, ) -> None: compose.mount_tmp_volumes(mounts, context.obj) - try: - utils.check_macos_docker_memory() - except exceptions.TutorError as e: - fmt.echo_alert( - f"""Could not verify sufficient RAM allocation in Docker: - - {e} - -Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: - - https://docs.tutor.overhang.io/install.html""" - ) + utils.warn_macos_docker_memory() run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) if run_upgrade_from_release is not None: diff --git a/tutor/commands/upgrade/__init__.py b/tutor/commands/upgrade/__init__.py index cf484d10b68..9c1d2034249 100644 --- a/tutor/commands/upgrade/__init__.py +++ b/tutor/commands/upgrade/__init__.py @@ -1,2 +1,10 @@ -# Note: don't forget to change this when we upgrade from olive -OPENEDX_RELEASE_NAMES = ["ironwood", "juniper", "koa", "lilac", "maple", "nutmeg"] +# Note: don't forget to change this when we upgrade from palm +OPENEDX_RELEASE_NAMES = [ + "ironwood", + "juniper", + "koa", + "lilac", + "maple", + "nutmeg", + "olive", +] diff --git a/tutor/commands/upgrade/common.py b/tutor/commands/upgrade/common.py index 85ef5cc6679..0b50f3eeb79 100644 --- a/tutor/commands/upgrade/common.py +++ b/tutor/commands/upgrade/common.py @@ -39,3 +39,11 @@ def upgrade_from_nutmeg(context: click.Context, config: Config) -> None: context.obj.job_runner(config).run_task( "lms", "./manage.py lms compute_grades -v1 --all_courses" ) + + +PALM_RENAME_ORA2_FOLDER_COMMAND = """ +if stat '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' 2> /dev/null; then + echo "Renaming ora2 folder..." + mv '/openedx/data/ora2/SET-ME-PLEASE (ex. bucket-name)' /openedx/data/ora2/openedxuploads +fi +""" diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 7d9d94a0709..df83eade168 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -38,6 +38,10 @@ def upgrade_from(context: click.Context, from_release: str) -> None: common_upgrade.upgrade_from_nutmeg(context, config) running_release = "olive" + if running_release == "olive": + upgrade_from_olive(context.obj, config) + running_release = "palm" + def upgrade_from_ironwood(config: Config) -> None: if not config["RUN_MONGODB"]: @@ -173,3 +177,19 @@ def upgrade_from_maple(context: Context, config: Config) -> None: k8s.kubectl_exec( config, "cms", ["sh", "-e", "-c", "./manage.py cms simulate_publish"] ) + + +def upgrade_from_olive(context: Context, config: Config) -> None: + # Note that we need to exec because the ora2 folder is not bind-mounted in the job + # services. + k8s.kubectl_apply( + context.root, + "--selector", + "app.kubernetes.io/name=lms", + ) + k8s.wait_for_deployment_ready(config, "lms") + k8s.kubectl_exec( + config, + "lms", + ["sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], + ) diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/local.py index 7c7832f7c52..80b0092be25 100644 --- a/tutor/commands/upgrade/local.py +++ b/tutor/commands/upgrade/local.py @@ -39,6 +39,10 @@ def upgrade_from(context: click.Context, from_release: str) -> None: common_upgrade.upgrade_from_nutmeg(context, config) running_release = "olive" + if running_release == "olive": + upgrade_from_olive(context) + running_release = "palm" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) @@ -146,6 +150,17 @@ def upgrade_from_maple(context: click.Context, config: Config) -> None: ) +def upgrade_from_olive(context: click.Context) -> None: + # Note that we need to exec because the ora2 folder is not bind-mounted in the job + # services. + context.invoke(compose.start, detach=True, services=["lms"]) + context.invoke( + compose.execute, + args=["lms", "sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], + ) + context.invoke(compose.stop) + + def upgrade_mongodb( context: click.Context, config: Config, diff --git a/tutor/env.py b/tutor/env.py index f0375a43b70..e24e90e5f88 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -161,7 +161,7 @@ def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: try: patches.append(self.render_str(patch)) except exceptions.TutorError: - fmt.echo_error(f"Error rendering patch '{name}': {patch}") + fmt.echo_error(f"Error rendering patch '{name}':\n{patch}") raise rendered = separator.join(patches) if rendered: @@ -169,7 +169,10 @@ def patch(self, name: str, separator: str = "\n", suffix: str = "") -> str: return rendered def render_str(self, text: str) -> str: - template = self.environment.from_string(text) + try: + template = self.environment.from_string(text) + except jinja2.exceptions.TemplateSyntaxError as e: + raise exceptions.TutorError(f"Template syntax error: {e.args[0]}") return self.__render(template) def render_template(self, template_name: str) -> t.Union[str, bytes]: @@ -450,6 +453,7 @@ def get_release(version: str) -> str: "13": "maple", "14": "nutmeg", "15": "olive", + "16": "palm", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index c7a46692bd9..46a563df663 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -118,8 +118,10 @@ }, } +# ORA2 ORA2_FILEUPLOAD_BACKEND = "filesystem" ORA2_FILEUPLOAD_ROOT = "/openedx/data/ora2" +FILE_UPLOAD_STORAGE_BUCKET_NAME = "openedxuploads" ORA2_FILEUPLOAD_CACHE_NAME = "ora2-storage" # Change syslog-based loggers which don't work inside docker containers @@ -135,18 +137,21 @@ "formatter": "standard", } LOGGING["loggers"]["tracking"]["handlers"] = ["console", "local", "tracking"] + # Silence some loggers (note: we must attempt to get rid of these when upgrading from one release to the next) +LOGGING["loggers"]["blockstore.apps.bundles.storage"] = {"handlers": ["console"], "level": "WARNING"} +# These warnings are visible in simple commands and init tasks import warnings from django.utils.deprecation import RemovedInDjango40Warning, RemovedInDjango41Warning warnings.filterwarnings("ignore", category=RemovedInDjango40Warning) warnings.filterwarnings("ignore", category=RemovedInDjango41Warning) -warnings.filterwarnings("ignore", category=DeprecationWarning, module="lms.djangoapps.course_wiki.plugins.markdownedx.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="wiki.plugins.links.wiki_plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="boto.plugin") warnings.filterwarnings("ignore", category=DeprecationWarning, module="botocore.vendored.requests.packages.urllib3._collections") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="storages.backends.s3boto") -warnings.filterwarnings("ignore", category=DeprecationWarning, module="openedx.core.types.admin") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="pkg_resources") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="fs") +warnings.filterwarnings("ignore", category=DeprecationWarning, module="fs.opener") SILENCED_SYSTEM_CHECKS = ["2_0.W001", "fields.W903"] # Email diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 5461d65aa6e..c1ec4af0601 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -36,11 +36,6 @@ "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "staticfiles_lms", } -CACHES["ora2-storage"] = { - "KEY_PREFIX": "ora2-storage", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", -} # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index a8da6b2a559..c48198589c2 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -15,9 +15,11 @@ RUN apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git -ARG PYTHON_VERSION=3.8.12 +# https://github.com/pyenv/pyenv/releases +# https://www.python.org/downloads/ +ARG PYTHON_VERSION=3.8.15 ENV PYENV_ROOT /opt/pyenv -RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.2.2 --depth 1 +RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv @@ -38,12 +40,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Fix broken Circuit Schematic Builder problem template -# https://github.com/openedx/edx-platform/pull/31365 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/20b93b8b01276edadddfbbb67f15714fddd81c31.patch | git am {%- endif %} -{# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ | git am #} +{# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} ###### Download extra locales to /openedx/locale/contrib/locale @@ -67,7 +66,7 @@ RUN apt update && apt install -y software-properties-common libmysqlclient-dev l # https://pypi.org/project/setuptools/ # https://pypi.org/project/pip/ # https://pypi.org/project/wheel/ -RUN pip install setuptools==65.5.1 pip==22.3.1. wheel==0.38.4 +RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install missing py2neo package that was abruptly trimmed from pypi RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz @@ -101,6 +100,7 @@ ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} # Install nodeenv with the version provided by edx-platform # https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt +# https://github.com/pyenv/pyenv/releases RUN pip install nodeenv==1.7.0 RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt @@ -227,7 +227,7 @@ USER app RUN pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN pip install ipdb==0.13.9 ipython==8.7.0 +RUN pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index ecd9dcea730..34724a9707f 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: olive +EDX_PLATFORM_REVISION: palm diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 48de1266c9e..e00f3c2dc81 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -12,13 +12,13 @@ DOCKER_COMPOSE_VERSION: "3.7" DOCKER_REGISTRY: "docker.io/" DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION }}" DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.3" -DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.10.1" +DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" +DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.2.24" -DOCKER_IMAGE_MYSQL: "docker.io/mysql:5.7.35" +DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" -DOCKER_IMAGE_REDIS: "docker.io/redis:6.2.6" -DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.95-r0-2" +DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" +DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" EDX_PLATFORM_REPOSITORY: "https://github.com/openedx/edx-platform.git" EDX_PLATFORM_VERSION: "{{ OPENEDX_COMMON_VERSION }}" ELASTICSEARCH_HOST: "elasticsearch" @@ -51,9 +51,9 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/olive.4" +OPENEDX_COMMON_VERSION: "open-release/palm.master" OPENEDX_EXTRA_PIP_REQUIREMENTS: - - "openedx-scorm-xblock>=15.0.0,<16.0.0" + - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" MYSQL_PORT: 3306 MYSQL_ROOT_USERNAME: "root" diff --git a/tutor/utils.py b/tutor/utils.py index 15291cd0812..64929588521 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -261,6 +261,19 @@ def check_output(*command: str) -> bytes: raise exceptions.TutorError(f"Command failed: {literal_command}") from e +def warn_macos_docker_memory() -> None: + try: + check_macos_docker_memory() + except exceptions.TutorError as e: + fmt.echo_alert( + f"""Could not verify sufficient RAM allocation in Docker: + + {e} + +Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: + https://docs.tutor.overhang.io/install.html""" + ) + def check_macos_docker_memory() -> None: """ Try to check that the RAM allocated to the Docker VM on macOS is at least 4 GB. From 9149be23af40ddae0f3c4c9d39f0464f3bddbd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 May 2023 09:09:17 +0200 Subject: [PATCH 08/96] feat: `importdemocourse --repo-dir=...` option This allows us to run: tutor local do importdemocourse --repo=https://github.com/openedx/openedx-test-course --version=open-release/palm.master --repo-dir=test-course/course --- changelog.d/20230502_090803_regis_palm.md | 2 ++ tutor/commands/jobs.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20230502_090803_regis_palm.md diff --git a/changelog.d/20230502_090803_regis_palm.md b/changelog.d/20230502_090803_regis_palm.md new file mode 100644 index 00000000000..3a1160685b7 --- /dev/null +++ b/changelog.d/20230502_090803_regis_palm.md @@ -0,0 +1,2 @@ +- [Feature] Add the `do importdemocourse --repo-dir=...` option, to import courses from subdirectories of git repositories. This allows us to import the openedx-test-course in Palm with: `tutor local do importdemocourse --repo=https://github.com/openedx/openedx-test-course --version=o +pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 22de90c09e0..8e694fc89f7 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -146,19 +146,26 @@ def create_user_template( show_default=True, help="Git repository that contains the course to be imported", ) +@click.option( + "-d", + "--repo-dir", + default="", + show_default=True, + help="Git relative subdirectory to import data from", +) @click.option( "-v", "--version", help="Git branch, tag or sha1 identifier. If unspecified, will default to the value of the OPENEDX_COMMON_VERSION setting.", ) def importdemocourse( - repo: str, version: t.Optional[str] + repo: str, repo_dir: str, version: t.Optional[str] ) -> t.Iterable[tuple[str, str]]: version = version or "{{ OPENEDX_COMMON_VERSION }}" template = f""" # Import demo course git clone {repo} --branch {version} --depth 1 /tmp/course -python ./manage.py cms import ../data /tmp/course +python ./manage.py cms import ../data /tmp/course/{repo_dir} # Re-index courses ./manage.py cms reindex_course --all --setup""" From 078f664c72f2b9c9be3b214f2e4197de56a78cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 17 Apr 2023 18:07:53 +0200 Subject: [PATCH 09/96] feat: hide TOS link during registration Registering a user was causing a 400 error because the LMS expected the TOS checkbox to be checked, but it's not displayed in the frontend. So we just disable it. Close https://github.com/openedx/wg-build-test-release/issues/262 --- changelog.d/20230412_100608_regis_palm.md | 1 + tutor/templates/apps/openedx/settings/partials/common_lms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 381ba296636..63fb682a814 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -1,2 +1,3 @@ - 💥[Feature] Upgrade to Palm. (by @regisb) - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. + - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. (by @regisb) diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index c1ec4af0601..e3b86492e05 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -4,7 +4,7 @@ LOGIN_REDIRECT_WHITELIST = ["{{ CMS_HOST }}"] # Better layout of honor code/tos links during registration -REGISTRATION_EXTRA_FIELDS["terms_of_service"] = "required" +REGISTRATION_EXTRA_FIELDS["terms_of_service"] = "hidden" REGISTRATION_EXTRA_FIELDS["honor_code"] = "hidden" # Fix media files paths From 11542ebc1c3403a978b66c9720747c994db395dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 10:13:46 +0200 Subject: [PATCH 10/96] depr: halt compatibility with python 3.7 --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 4 +-- Dockerfile | 2 +- Makefile | 1 - changelog.d/20230412_100608_regis_palm.md | 1 + docs/conf.py | 12 +++------ requirements/base.txt | 9 +------ requirements/dev.in | 2 +- requirements/dev.txt | 30 +++-------------------- requirements/docs.txt | 24 +++--------------- setup.py | 4 +-- tutor/commands/jobs.py | 5 ++-- tutor/core/hooks/filters.py | 13 +++------- tutor/utils.py | 17 ++----------- 14 files changed, 29 insertions(+), 97 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a95bf47d4b7..52ebe050b7b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: # https://github.com/actions/setup-python uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: 3.8 cache: 'pip' cache-dependency-path: requirements/dev.txt - name: Upgrade pip and setuptools diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bb858820e2a..b7940a41a85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,11 +14,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: 3.8 cache: 'pip' cache-dependency-path: requirements/dev.txt - name: Upgrade pip - run: python -m pip install --upgrade pip setuptools==44.0.0 + run: python -m pip install --upgrade pip setuptools - name: Install dependencies run: pip install -r requirements/dev.txt - name: Static code analysis diff --git a/Dockerfile b/Dockerfile index a77d35eb9fe..ce3b10536ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ # Because this image is still experimental, and we are not quite sure if it's going to # be very useful, we do not provide any usage documentation. -FROM docker.io/python:3.7-slim-stretch +FROM docker.io/python:3.8-slim-stretch # As per https://github.com/docker/compose/issues/3918 COPY --from=library/docker:19.03 /usr/local/bin/docker /usr/bin/docker diff --git a/Makefile b/Makefile index 9dbdeb7b34a..b1755a5dc0f 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,6 @@ bootstrap-dev-plugins: bootstrap-dev ## Install dev requirements and all support pull-base-images: # Manually pull base images docker image pull docker.io/ubuntu:20.04 - docker image pull docker.io/python:3.7-alpine ci-info: ## Print info about environment python --version diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 63fb682a814..6af204de407 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -1,3 +1,4 @@ - 💥[Feature] Upgrade to Palm. (by @regisb) - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. (by @regisb) + - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. (by @regisb) diff --git a/docs/conf.py b/docs/conf.py index 92f9a545758..38fb30767bf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,7 @@ # For the life of me I can't get the docs to compile in nitpicky mode without these # ignore statements. You are most welcome to try and remove them. # To make matters worse, some ignores are only required for some versions of Python, -# from 3.7 to 3.10... +# from 3.8 to 3.10... nitpick_ignore = [ # Sphinx does not handle ParamSpec arguments ("py:class", "T.args"), @@ -48,8 +48,6 @@ ("py:class", "t.Callable"), ("py:class", "t.Iterator"), ("py:class", "t.Optional"), - # python 3.7 - ("py:class", "Concatenate"), # python 3.10 ("py:class", "NoneType"), ("py:class", "click.core.Command"), @@ -57,8 +55,6 @@ # Resolve type aliases here # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#confval-autodoc_type_aliases autodoc_type_aliases: dict[str, str] = { - "T1": "tutor.core.hooks.filters.T1", - "L": "tutor.core.hooks.filters.L", # python 3.10 "T": "tutor.core.hooks.actions.T", "T2": "tutor.core.hooks.filters.T2", @@ -132,14 +128,12 @@ def youtube( return [ docutils.nodes.raw( "", - """ + f""" """.format( - video_id=video_id - ), +""", format="html", ) ] diff --git a/requirements/base.txt b/requirements/base.txt index b4947ede021..fa904772bb2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile requirements/base.in @@ -20,8 +20,6 @@ google-auth==2.19.1 # via kubernetes idna==3.4 # via requests -importlib-metadata==6.6.0 - # via click jinja2==3.1.2 # via -r requirements/base.in kubernetes==26.1.0 @@ -63,12 +61,9 @@ six==1.16.0 # python-dateutil tomli==2.0.1 # via mypy -typed-ast==1.5.4 - # via mypy typing-extensions==4.6.3 # via # -r requirements/base.in - # importlib-metadata # mypy urllib3==1.26.16 # via @@ -77,8 +72,6 @@ urllib3==1.26.16 # requests websocket-client==1.5.2 # via kubernetes -zipp==3.15.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/dev.in b/requirements/dev.in index 4c1100d5be5..8dc0c3d1f82 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -9,7 +9,7 @@ twine # doc requirement is lagging behind # https://github.com/readthedocs/sphinx_rtd_theme/issues/1323 -docutils<0.18 +docutils<0.19 # Types packages types-docutils diff --git a/requirements/dev.txt b/requirements/dev.txt index ce33e734c1f..6e1addc4849 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile requirements/dev.in @@ -48,7 +48,7 @@ cryptography==41.0.1 # via secretstorage dill==0.3.6 # via pylint -docutils==0.17.1 +docutils==0.18.1 # via # -r requirements/dev.in # readme-renderer @@ -62,16 +62,9 @@ idna==3.4 # requests importlib-metadata==6.6.0 # via - # -r requirements/base.txt - # attrs - # build - # click # keyring - # pyinstaller # twine -importlib-resources==5.12.0 - # via keyring -isort==5.11.5 +isort==5.12.0 # via pylint jaraco-classes==3.2.3 # via keyring @@ -206,12 +199,6 @@ tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/dev.in -typed-ast==1.5.4 - # via - # -r requirements/base.txt - # astroid - # black - # mypy types-docutils==0.20.0.1 # via -r requirements/dev.in types-pyyaml==6.0.12.10 @@ -222,13 +209,7 @@ typing-extensions==4.6.3 # via # -r requirements/base.txt # astroid - # black - # importlib-metadata - # markdown-it-py # mypy - # platformdirs - # pylint - # rich urllib3==1.26.16 # via # -r requirements/base.txt @@ -247,10 +228,7 @@ wheel==0.40.0 wrapt==1.15.0 # via astroid zipp==3.15.0 - # via - # -r requirements/base.txt - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 5fcdbc15e90..016cb3d244f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile requirements/docs.in @@ -42,11 +42,6 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.6.0 - # via - # -r requirements/base.txt - # click - # sphinx jinja2==3.1.2 # via # -r requirements/base.txt @@ -86,8 +81,6 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # kubernetes -pytz==2023.3 - # via babel pyyaml==6.0 # via # -r requirements/base.txt @@ -114,7 +107,7 @@ six==1.16.0 # python-dateutil snowballstemmer==2.2.0 # via sphinx -sphinx==5.3.0 +sphinx==6.2.1 # via # -r requirements/docs.in # sphinx-click @@ -124,11 +117,11 @@ sphinx-click==4.4.0 # via -r requirements/docs.in sphinx-rtd-theme==1.2.1 # via -r requirements/docs.in -sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jquery==4.1 # via sphinx-rtd-theme @@ -142,14 +135,9 @@ tomli==2.0.1 # via # -r requirements/base.txt # mypy -typed-ast==1.5.4 - # via - # -r requirements/base.txt - # mypy typing-extensions==4.6.3 # via # -r requirements/base.txt - # importlib-metadata # mypy urllib3==1.26.16 # via @@ -161,10 +149,6 @@ websocket-client==1.5.2 # via # -r requirements/base.txt # kubernetes -zipp==3.15.0 - # via - # -r requirements/base.txt - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index ff7ee31b157..8a5c4707bf5 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ def is_requirement(line: str) -> bool: long_description_content_type="text/x-rst", packages=find_packages(exclude=["tests*"]), include_package_data=True, - python_requires=">=3.7", + python_requires=">=3.8", install_requires=load_requirements("base.in"), extras_require={ "full": load_requirements("plugins.txt"), @@ -68,10 +68,10 @@ def is_requirement(line: str) -> bool: "License :: OSI Approved :: GNU Affero General Public License v3", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], test_suite="tests", ) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 8e694fc89f7..a6648c7c634 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -4,13 +4,14 @@ from __future__ import annotations import functools +import shlex import typing as t import click from typing_extensions import ParamSpec from tutor import config as tutor_config -from tutor import env, fmt, hooks, utils +from tutor import env, fmt, hooks from tutor.hooks import priorities @@ -260,7 +261,7 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: """ command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}" if args: - command += " " + utils._shlex_join(*args) # pylint: disable=protected-access + command += " " + shlex.join(args) # pylint: disable=protected-access yield ("lms", command) diff --git a/tutor/core/hooks/filters.py b/tutor/core/hooks/filters.py index c513baee5d7..9e9b70e81d7 100644 --- a/tutor/core/hooks/filters.py +++ b/tutor/core/hooks/filters.py @@ -225,15 +225,10 @@ def add_items( my_filter.add_item("item2") """ - # Unfortunately we have to type-ignore this line. If not, mypy complains with: - # - # Argument 1 has incompatible type "Callable[[Arg(List[E], 'values'), **T2], List[E]]"; expected "Callable[[List[E], **T2], List[E]]" - # This is likely because "callback" has named arguments: "values". Consider marking them positional-only - # - # But we are unable to mark arguments positional-only (by adding / after values arg) in Python 3.7. - # Get rid of this statement after Python 3.7 EOL. - @self.add(priority=priority) # type: ignore - def callback(values: list[L], *_args: T2.args, **_kwargs: T2.kwargs) -> list[L]: + @self.add(priority=priority) + def callback( + values: list[L], /, *_args: T2.args, **_kwargs: T2.kwargs + ) -> list[L]: return values + items def iterate( diff --git a/tutor/utils.py b/tutor/utils.py index 64929588521..6d62777035d 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -216,7 +216,7 @@ def is_a_tty() -> bool: def execute(*command: str) -> int: - click.echo(fmt.command(_shlex_join(*command))) + click.echo(fmt.command(shlex.join(command))) return execute_silent(*command) @@ -239,21 +239,8 @@ def execute_silent(*command: str) -> int: return result -def _shlex_join(*split_command: str) -> str: - """ - Return a shell-escaped string from *split_command. - - TODO: REMOVE THIS FUNCTION AFTER 2023-06-27. - This function is a shim for the ``shlex.join`` standard library function, - which becomes available in Python 3.8 The end-of-life date for Python 3.7 - is in Jan 2023 (https://endoflife.date/python). After that point, it - would be good to delete this function and just use Py3.8's ``shlex.join``. - """ - return " ".join(shlex.quote(arg) for arg in split_command) - - def check_output(*command: str) -> bytes: - literal_command = _shlex_join(*command) + literal_command = shlex.join(command) click.echo(fmt.command(literal_command)) try: return subprocess.check_output(command) From 5f39775fda6238496374da8c41d8ec679934ac52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 10:36:45 +0200 Subject: [PATCH 11/96] depr: drop support for `docker-compose` Instead, the compose plugin must be installed. We deprecate docker-compose because v1 is not supported starting from the end of June 2023. See "evolution of compose": https://docs.docker.com/compose/compose-v2/ --- changelog.d/20230412_100608_regis_palm.md | 5 +++-- tutor/utils.py | 24 +---------------------- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 6af204de407..13a2011573b 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -1,4 +1,5 @@ - 💥[Feature] Upgrade to Palm. (by @regisb) - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. - - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. (by @regisb) - - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. (by @regisb) + - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. + - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. + - 💥[Deprecation] Drop support for `docker-compose`, also known as Compose V1. The `docker compose` (no hyphen) plugin must be installed. diff --git a/tutor/utils.py b/tutor/utils.py index 6d62777035d..545ce38b204 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,30 +173,8 @@ def docker(*command: str) -> int: return execute("docker", *command) -@lru_cache(maxsize=None) -def _docker_compose_command() -> Tuple[str, ...]: - """ - A helper function to determine which program to call when running docker compose - """ - if os.environ.get("TUTOR_USE_COMPOSE_SUBCOMMAND") is not None: - return ("docker", "compose") - if shutil.which("docker-compose") is not None: - return ("docker-compose",) - if shutil.which("docker") is not None: - if ( - subprocess.run( - ["docker", "compose"], capture_output=True, check=False - ).returncode - == 0 - ): - return ("docker", "compose") - raise exceptions.TutorError( - "docker-compose is not installed. Please follow instructions from https://docs.docker.com/compose/install/" - ) - - def docker_compose(*command: str) -> int: - return execute(*_docker_compose_command(), *command) + return execute("docker", "compose", *command) def kubectl(*command: str) -> int: From 7161729860ffb63f6d5edfcbc535863e94704c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sat, 25 Mar 2023 20:58:15 -0400 Subject: [PATCH 12/96] fix: remove useless "privileged: false" statements These values are by default anyway. --- tutor/templates/local/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 36e03686384..dafdc08cfd5 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -10,7 +10,6 @@ services: command: mongod --nojournal --storageEngine wiredTiger restart: unless-stopped user: "999:999" - privileged: false volumes: - ../../data/mongodb:/data/db depends_on: @@ -29,7 +28,6 @@ services: command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci restart: unless-stopped user: "999:999" - privileged: false volumes: - ../../data/mysql:/var/lib/mysql environment: From b56d42d8da70894fcdfa6106375c861dc76ca3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sat, 25 Mar 2023 20:07:37 -0400 Subject: [PATCH 13/96] depr: RUN_LMS, RUN_CMS settings These tutor settings are mostly useless and make templates much more difficult to work with. --- changelog.d/20230325_205654_regis_permissions.md | 1 + docs/configuration.rst | 2 -- tutor/commands/compose.py | 5 +---- tutor/templates/config/defaults.yml | 2 -- tutor/templates/k8s/deployments.yml | 4 ---- tutor/templates/k8s/services.yml | 4 ---- tutor/templates/local/docker-compose.yml | 8 -------- 7 files changed, 2 insertions(+), 24 deletions(-) create mode 100644 changelog.d/20230325_205654_regis_permissions.md diff --git a/changelog.d/20230325_205654_regis_permissions.md b/changelog.d/20230325_205654_regis_permissions.md new file mode 100644 index 00000000000..08841ade868 --- /dev/null +++ b/changelog.d/20230325_205654_regis_permissions.md @@ -0,0 +1 @@ +- 💥[Improvement] Deprecate the `RUN_LMS` and `RUN_CMS` tutor settings, which should be mostly unused. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index b36f4930adf..244133eff51 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -40,8 +40,6 @@ With an up-to-date environment, Tutor is ready to launch an Open edX platform an Individual service activation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``RUN_LMS`` (default: ``true``) -- ``RUN_CMS`` (default: ``true``) - ``RUN_ELASTICSEARCH`` (default: ``true``) - ``RUN_MONGODB`` (default: ``true``) - ``RUN_MYSQL`` (default: ``true``) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index af561a82223..44f515853a8 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -306,10 +306,7 @@ def restart(context: BaseComposeContext, services: list[str]) -> None: else: for service in services: if service == "openedx": - if config["RUN_LMS"]: - command += ["lms", "lms-worker"] - if config["RUN_CMS"]: - command += ["cms", "cms-worker"] + command += ["lms", "lms-worker", "cms", "cms-worker"] else: command.append(service) context.job_runner(config).docker_compose(*command) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index e00f3c2dc81..da341ed16f4 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -64,9 +64,7 @@ REDIS_HOST: "redis" REDIS_PORT: 6379 REDIS_USERNAME: "" REDIS_PASSWORD: "" -RUN_CMS: true RUN_ELASTICSEARCH: true -RUN_LMS: true RUN_MONGODB: true RUN_MYSQL: true RUN_REDIS: true diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 7571ae51966..30c73b8afa0 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -57,7 +57,6 @@ spec: persistentVolumeClaim: claimName: caddy {%- endif %} -{% if RUN_CMS %} --- apiVersion: apps/v1 kind: Deployment @@ -167,8 +166,6 @@ spec: - name: config configMap: name: openedx-config -{% endif %} -{% if RUN_LMS %} --- apiVersion: apps/v1 kind: Deployment @@ -278,7 +275,6 @@ spec: - name: config configMap: name: openedx-config -{% endif %} {% if RUN_ELASTICSEARCH %} --- apiVersion: apps/v1 diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index 180032bad43..aaaa9b67d2d 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -34,7 +34,6 @@ spec: selector: app.kubernetes.io/name: caddy {% endif %} -{% if RUN_CMS %} --- apiVersion: v1 kind: Service @@ -49,8 +48,6 @@ spec: protocol: TCP selector: app.kubernetes.io/name: cms -{% endif %} -{% if RUN_LMS %} --- apiVersion: v1 kind: Service @@ -65,7 +62,6 @@ spec: protocol: TCP selector: app.kubernetes.io/name: lms -{% endif %} {% if RUN_ELASTICSEARCH %} --- apiVersion: v1 diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index dafdc08cfd5..fa7eb536b28 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -97,7 +97,6 @@ services: ############# LMS and CMS - {% if RUN_LMS %} lms: image: {{ DOCKER_IMAGE_OPENEDX }} environment: @@ -129,7 +128,6 @@ services: - ../../data/openedx-media:/openedx/media {% endif %} - {% if RUN_CMS %} cms: image: {{ DOCKER_IMAGE_OPENEDX }} environment: @@ -151,7 +149,6 @@ services: {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} - {% if RUN_LMS %}- lms{% endif %} {{ patch("local-docker-compose-cms-dependencies")|indent(6) }} cms-permissions: image: {{ DOCKER_IMAGE_PERMISSIONS }} @@ -160,11 +157,9 @@ services: volumes: - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media - {% endif %} ############# LMS and CMS workers - {% if RUN_LMS %} lms-worker: image: {{ DOCKER_IMAGE_OPENEDX }} environment: @@ -180,9 +175,7 @@ services: - ../../data/openedx-media:/openedx/media depends_on: - lms - {% endif %} - {% if RUN_CMS %} cms-worker: image: {{ DOCKER_IMAGE_OPENEDX }} environment: @@ -198,6 +191,5 @@ services: - ../../data/openedx-media:/openedx/media depends_on: - cms - {% endif %} {{ patch("local-docker-compose-services")|indent(2) }} From 486f9455dd86a6cf5ec6f3e0af02a8f697ff980a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Sat, 25 Mar 2023 21:17:07 -0400 Subject: [PATCH 14/96] feat: simplify docker-compose permissions It was useless to create a *-permissions job for every application. Instead, we create a single "permissions" service. It can be extended via the "docker-compose-permissions-command" patch. --- .../20230325_211520_regis_permissions.md | 1 + docs/reference/patches.rst | 19 +++++ tutor/templates/apps/permissions/setowners.sh | 8 +++ tutor/templates/dev/docker-compose.yml | 8 +-- tutor/templates/local/docker-compose.yml | 71 +++++++------------ 5 files changed, 58 insertions(+), 49 deletions(-) create mode 100644 changelog.d/20230325_211520_regis_permissions.md create mode 100644 tutor/templates/apps/permissions/setowners.sh diff --git a/changelog.d/20230325_211520_regis_permissions.md b/changelog.d/20230325_211520_regis_permissions.md new file mode 100644 index 00000000000..f9e37e92dad --- /dev/null +++ b/changelog.d/20230325_211520_regis_permissions.md @@ -0,0 +1 @@ +- [Improvement] Greatly simplify ownership of bind-mounted volumes with docker-compose. Instead of running one service per application, we run just a single "permissions" service. This change should be backward-compatible. (by @regisb) diff --git a/docs/reference/patches.rst b/docs/reference/patches.rst index 4cc86ca5af1..e5817860941 100644 --- a/docs/reference/patches.rst +++ b/docs/reference/patches.rst @@ -203,6 +203,25 @@ File: ``local/docker-compose.jobs.yml`` File: ``local/docker-compose.yml`` +.. patch:: local-docker-compose-permissions-command + +``local-docker-compose-permissions-command`` +============================================ + +File: ``apps/permissions/setowners.sh`` + +Add commands to this script to set ownership of bind-mounted docker-compose volumes at runtime. See :patch:`local-docker-compose-permissions-volumes`. + + +.. patch:: local-docker-compose-permissions-volumes + +``local-docker-compose-permissions-volumes`` +============================================ + +File: ``local/docker-compose.yml`` + +Add bind-mounted volumes to this patch to set their owners properly. See :patch:`local-docker-compose-permissions-command`. + .. patch:: local-docker-compose-prod-services ``local-docker-compose-prod-services`` diff --git a/tutor/templates/apps/permissions/setowners.sh b/tutor/templates/apps/permissions/setowners.sh new file mode 100644 index 00000000000..d4044f9067a --- /dev/null +++ b/tutor/templates/apps/permissions/setowners.sh @@ -0,0 +1,8 @@ +#! /bin/sh +setowner $OPENEDX_USER_ID /mounts/lms /mounts/cms /mounts/openedx +{% if RUN_ELASTICSEARCH %}setowner 1000 /mounts/elasticsearch{% endif %} +{% if RUN_MONGODB %}setowner 999 /mounts/mongodb{% endif %} +{% if RUN_MYSQL %}setowner 999 /mounts/mysql{% endif %} +{% if RUN_REDIS %}setowner 1000 /mounts/redis{% endif %} + +{{ patch("local-docker-compose-permissions-command") }} diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 58b27aa8329..96d045ea382 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -22,11 +22,9 @@ x-openedx-service: - ../build/openedx/requirements:/openedx/requirements services: - lms-permissions: - command: ["{{ HOST_USER_ID }}", "/openedx/data", "/openedx/media"] - - cms-permissions: - command: ["{{ HOST_USER_ID }}", "/openedx/data", "/openedx/media"] + permissions: + environment: + OPENEDX_USER_ID: "{{ HOST_USER_ID }}" lms: <<: *openedx-service diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index fa7eb536b28..b30afb18d33 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -1,6 +1,27 @@ version: "{{ DOCKER_COMPOSE_VERSION }}" services: + # Set bind-mounted folder ownership + permissions: + image: {{ DOCKER_IMAGE_PERMISSIONS }} + restart: on-failure + entrypoint: [] + command: ["sh", "/usr/local/bin/setowners.sh"] + environment: + OPENEDX_USER_ID: "1000" + volumes: + # Command script + - ../apps/permissions/setowners.sh:/usr/local/bin/setowners.sh:ro + # Bind-mounted volumes to set ownership + - ../../data/lms:/mounts/lms + - ../../data/cms:/mounts/cms + - ../../data/openedx-media:/mounts/openedx + {% if RUN_MONGODB %}- ../../data/mongodb:/mounts/mongodb{% endif %} + {% if RUN_MYSQL %}- ../../data/mysql:/mounts/mysql{% endif %} + {% if RUN_ELASTICSEARCH %}- ../../data/elasticsearch:/mounts/elasticsearch{% endif %} + {% if RUN_REDIS %}- ../../data/redis:/mounts/redis{% endif %} + {{ patch("local-docker-compose-permissions-volumes")|indent(6) }} + ############# External services {% if RUN_MONGODB %} @@ -13,13 +34,7 @@ services: volumes: - ../../data/mongodb:/data/db depends_on: - - mongodb-permissions - mongodb-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["999", "/data/db"] - restart: on-failure - volumes: - - ../../data/mongodb:/data/db + - permissions {% endif %} {% if RUN_MYSQL %} @@ -32,12 +47,6 @@ services: - ../../data/mysql:/var/lib/mysql environment: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" - mysql-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["999", "/var/lib/mysql"] - restart: on-failure - volumes: - - ../../data/mysql:/var/lib/mysql {% endif %} {% if RUN_ELASTICSEARCH %} @@ -57,13 +66,7 @@ services: volumes: - ../../data/elasticsearch:/usr/share/elasticsearch/data depends_on: - - elasticsearch-permissions - elasticsearch-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["1000", "/usr/share/elasticsearch/data"] - restart: on-failure - volumes: - - ../../data/elasticsearch:/usr/share/elasticsearch/data + - permissions {% endif %} {% if RUN_REDIS %} @@ -77,13 +80,7 @@ services: command: redis-server /openedx/redis/config/redis.conf restart: unless-stopped depends_on: - - redis-permissions - redis-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["1000", "/openedx/redis/data"] - restart: on-failure - volumes: - - ../../data/redis:/openedx/redis/data + - permissions {% endif %} {% if RUN_SMTP %} @@ -112,21 +109,13 @@ services: - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media depends_on: - - lms-permissions + - permissions {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} {{ patch("local-docker-compose-lms-dependencies")|indent(6) }} - lms-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["1000", "/openedx/data", "/openedx/media"] - restart: on-failure - volumes: - - ../../data/lms:/openedx/data - - ../../data/openedx-media:/openedx/media - {% endif %} cms: image: {{ DOCKER_IMAGE_OPENEDX }} @@ -143,20 +132,14 @@ services: - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media depends_on: - - cms-permissions + - permissions + - lms {% if RUN_MYSQL %}- mysql{% endif %} {% if RUN_ELASTICSEARCH %}- elasticsearch{% endif %} {% if RUN_MONGODB %}- mongodb{% endif %} {% if RUN_REDIS %}- redis{% endif %} {% if RUN_SMTP %}- smtp{% endif %} {{ patch("local-docker-compose-cms-dependencies")|indent(6) }} - cms-permissions: - image: {{ DOCKER_IMAGE_PERMISSIONS }} - command: ["1000", "/openedx/data", "/openedx/media"] - restart: on-failure - volumes: - - ../../data/cms:/openedx/data - - ../../data/openedx-media:/openedx/media ############# LMS and CMS workers From 9afe25bfa58a516098b7a30945c131953953f83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 27 Apr 2023 12:20:50 +0200 Subject: [PATCH 15/96] feat: `config save --append/--remove KEY=VAL` options This paves the way for persisting bind-mounts across runs, since this setting will be a list. --- .../20230427_121619_regis_config_append.md | 1 + docs/configuration.rst | 11 +-- tests/commands/test_config.py | 40 ++++++++++ tutor/commands/config.py | 74 +++++++++++++++++-- 4 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 changelog.d/20230427_121619_regis_config_append.md diff --git a/changelog.d/20230427_121619_regis_config_append.md b/changelog.d/20230427_121619_regis_config_append.md new file mode 100644 index 00000000000..6d507918915 --- /dev/null +++ b/changelog.d/20230427_121619_regis_config_append.md @@ -0,0 +1 @@ +- [Feature] Add a `config save -a/--append -A/--remove` options to conveniently append and remove values to/from list entries. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 244133eff51..02e5ad96e24 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -359,19 +359,12 @@ Installing extra xblocks and requirements Would you like to include custom xblocks, or extra requirements to your Open edX platform? Additional requirements can be added to the ``OPENEDX_EXTRA_PIP_REQUIREMENTS`` parameter in the :ref:`config file ` or to the ``env/build/openedx/requirements/private.txt`` file. The difference between them, is that ``private.txt`` file, even though it could be used for both, :ref:`should be used for installing extra xblocks or requirements from private repositories `. For instance, to include the `polling xblock from Opencraft `_: -- add the following to the ``config.yml``:: + tutor config save --append OPENEDX_EXTRA_PIP_REQUIREMENTS=git+https://github.com/open-craft/xblock-poll.git - OPENEDX_EXTRA_PIP_REQUIREMENTS: - - "git+https://github.com/open-craft/xblock-poll.git" - -.. warning:: - Specifying extra requirements through ``config.yml`` overwrites :ref:`the default extra requirements`. You might need to add them to the list if your configuration depends on them. - -- or add the dependency to ``private.txt``:: +Alternatively, add the dependency to ``private.txt``:: echo "git+https://github.com/open-craft/xblock-poll.git" >> "$(tutor config printroot)/env/build/openedx/requirements/private.txt" - Then, the ``openedx`` docker image must be rebuilt:: tutor images build openedx diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index 314d8fe5b13..f3448978371 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -1,5 +1,6 @@ import unittest +from tutor import config as tutor_config from tests.helpers import temporary_root from .base import TestCommandMixin @@ -59,6 +60,45 @@ def test_config_printvalue(self) -> None: self.assertEqual(0, result.exit_code) self.assertTrue(result.output) + def test_config_append(self) -> None: + with temporary_root() as root: + self.invoke_in_root( + root, ["config", "save", "--append=TEST=value"], catch_exceptions=False + ) + config1 = tutor_config.load(root) + self.invoke_in_root( + root, ["config", "save", "--append=TEST=value"], catch_exceptions=False + ) + config2 = tutor_config.load(root) + self.invoke_in_root( + root, ["config", "save", "--remove=TEST=value"], catch_exceptions=False + ) + config3 = tutor_config.load(root) + # Value is appended + self.assertEqual(["value"], config1["TEST"]) + # Value is not appended a second time + self.assertEqual(["value"], config2["TEST"]) + # Value is removed + self.assertEqual([], config3["TEST"]) + + def test_config_append_with_existing_default(self) -> None: + with temporary_root() as root: + self.invoke_in_root( + root, + [ + "config", + "save", + "--append=OPENEDX_EXTRA_PIP_REQUIREMENTS=my-package==1.0.0", + ], + catch_exceptions=False, + ) + config = tutor_config.load(root) + assert isinstance(config["OPENEDX_EXTRA_PIP_REQUIREMENTS"], list) + self.assertEqual(2, len(config["OPENEDX_EXTRA_PIP_REQUIREMENTS"])) + self.assertEqual( + "my-package==1.0.0", config["OPENEDX_EXTRA_PIP_REQUIREMENTS"][1] + ) + class PatchesTests(unittest.TestCase, TestCommandMixin): def test_config_patches_list(self) -> None: diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 19d7ce828ec..72d1d2bc6a2 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -34,9 +34,8 @@ def shell_complete( for key, _value in self._shell_complete_config_items(ctx, incomplete) ] - @staticmethod def _shell_complete_config_items( - ctx: click.Context, incomplete: str + self, ctx: click.Context, incomplete: str ) -> list[tuple[str, ConfigValue]]: # Here we want to auto-complete the name of the config key. For that we need to # figure out the list of enabled plugins, and for that we need the project root. @@ -48,9 +47,16 @@ def _shell_complete_config_items( ).get("root", "") config = tutor_config.load_full(root) return [ - (key, value) for key, value in config.items() if key.startswith(incomplete) + (key, value) + for key, value in self._candidate_config_items(config) + if key.startswith(incomplete) ] + def _candidate_config_items( + self, config: Config + ) -> t.Iterable[tuple[str, ConfigValue]]: + yield from config.items() + class ConfigKeyValParamType(ConfigKeyParamType): """ @@ -91,6 +97,19 @@ def shell_complete( return [] +class ConfigListKeyValParamType(ConfigKeyValParamType): + """ + Same as the parent class, but for keys of type `list`. + """ + + def _candidate_config_items( + self, config: Config + ) -> t.Iterable[tuple[str, ConfigValue]]: + for key, val in config.items(): + if isinstance(val, list): + yield key, val + + @click.command(help="Create and save configuration interactively") @click.option("-i", "--interactive", is_flag=True, help="Run interactively") @click.option( @@ -102,6 +121,24 @@ def shell_complete( metavar="KEY=VAL", help="Set a configuration value (can be used multiple times)", ) +@click.option( + "-a", + "--append", + "append_vars", + type=ConfigListKeyValParamType(), + multiple=True, + metavar="KEY=VAL", + help="Append an item to a configuration value of type list. The value will only be added it it is not already present. (can be used multiple times)", +) +@click.option( + "-A", + "--remove", + "remove_vars", + type=ConfigListKeyValParamType(), + multiple=True, + metavar="KEY=VAL", + help="Remove an item from a configuration value of type list (can be used multiple times)", +) @click.option( "-U", "--unset", @@ -117,16 +154,43 @@ def shell_complete( def save( context: Context, interactive: bool, - set_vars: Config, + set_vars: tuple[str, t.Any], + append_vars: tuple[str, t.Any], + remove_vars: tuple[str, t.Any], unset_vars: list[str], env_only: bool, ) -> None: config = tutor_config.load_minimal(context.root) + config_full = tutor_config.load_full(context.root) if interactive: interactive_config.ask_questions(config) if set_vars: - for key, value in dict(set_vars).items(): + for key, value in set_vars: config[key] = env.render_unknown(config, value) + if append_vars: + for key, value in append_vars: + if key not in config: + config[key] = config_full.get(key, []) + values = config[key] + if not isinstance(values, list): + raise exceptions.TutorError( + f"Could not append value to '{key}': current setting is of type '{values.__class__.__name__}', expected list." + ) + if not isinstance(value, str): + raise exceptions.TutorError( + f"Could not append value to '{key}': appended value is of type '{value.__class__.__name__}', expected str." + ) + if value not in values: + values.append(value) + if remove_vars: + for key, value in remove_vars: + values = config.get(key, []) + if not isinstance(values, list): + raise exceptions.TutorError( + f"Could not remove value from '{key}': current setting is of type '{values.__class__.__name__}', expected list." + ) + while value in values: + values.remove(value) for key in unset_vars: config.pop(key, None) if not env_only: From 548d20cd4bde829e2ab0fa66090d83eedec9ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 3 Apr 2023 20:44:15 +0200 Subject: [PATCH 16/96] feat: leverage `RUN --mount` for faster image building We make use of the Docker build cache to install python and nodejs requirements faster in the case of repeated builds. This feature is only possible for users of BuildKit, so we detect whether `docker buildx` is available at runtime. We do not make use of `COPY --link` because the `--link` option is incompatible with `--chown=app:app`: https://github.com/docker/buildx/issues/1408 For reference, see: https://www.docker.com/blog/dockerfiles-now-support-multiple-build-contexts/ https://docs.docker.com/engine/reference/commandline/buildx_build/#build-context --- .../20230427_154822_regis_build_mount.md | 1 + tests/commands/test_images.py | 27 ++++-- tutor/commands/images.py | 14 ++- tutor/env.py | 1 + tutor/images.py | 7 +- tutor/templates/build/openedx/Dockerfile | 88 ++++++++++++------- tutor/utils.py | 19 ++++ 7 files changed, 111 insertions(+), 46 deletions(-) create mode 100644 changelog.d/20230427_154822_regis_build_mount.md diff --git a/changelog.d/20230427_154822_regis_build_mount.md b/changelog.d/20230427_154822_regis_build_mount.md new file mode 100644 index 00000000000..dbc5e908de1 --- /dev/null +++ b/changelog.d/20230427_154822_regis_build_mount.md @@ -0,0 +1 @@ +- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 20abdd2bd1e..622be9b8aa7 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -1,7 +1,7 @@ from unittest.mock import Mock, patch from tests.helpers import PluginsTestCase, temporary_root -from tutor import images, plugins +from tutor import images, plugins, utils from tutor.__about__ import __version__ from tutor.commands.images import ImageNotFoundError @@ -128,16 +128,29 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "service1", ] with temporary_root() as root: - self.invoke_in_root(root, ["config", "save"]) - result = self.invoke_in_root(root, build_args) + utils.is_buildkit_enabled.cache_clear() + with patch.object(utils, "is_buildkit_enabled", return_value=False): + self.invoke_in_root(root, ["config", "save"]) + result = self.invoke_in_root(root, build_args) self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) image_build.assert_called() self.assertIn("service1:1.0.0", image_build.call_args[0]) - for arg in image_build.call_args[0][2:]: - # The only extra args are `--build-arg` - if arg != "--build-arg": - self.assertIn(arg, build_args) + self.assertEqual( + [ + "service1:1.0.0", + "--no-cache", + "--build-arg", + "myarg=value", + "--add-host", + "host", + "--target", + "target", + "docker_args", + "--cache-from=type=registry,ref=service1:1.0.0-cache", + ], + list(image_build.call_args[0][1:]) + ) def test_images_push(self) -> None: result = self.invoke(["images", "push"]) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index c300ff80616..1dfce27b8f5 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -83,6 +83,14 @@ def images_command() -> None: is_flag=True, help="Push the build cache to the remote registry. You should only enable this option if you have push rights to the remote registry.", ) +@click.option( + "--output", + "docker_output", + # Export image to docker. This is necessary to make the image available to docker-compose. + # The `--load` option is a shorthand for `--output=type=docker`. + default="type=docker", + help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.", +) @click.option( "-a", "--build-arg", @@ -114,6 +122,7 @@ def build( no_cache: bool, no_registry_cache: bool, cache_to_registry: bool, + docker_output: str, build_args: list[str], add_hosts: list[str], target: str, @@ -135,9 +144,8 @@ def build( command_args += ["--add-host", add_host] if target: command_args += ["--target", target] - if utils.is_buildkit_enabled(): - # Export image to docker. - command_args.append("--output=type=image") + if utils.is_buildkit_enabled() and docker_output: + command_args.append(f"--output={docker_output}") if docker_args: command_args += docker_args for image in image_names: diff --git a/tutor/env.py b/tutor/env.py index e24e90e5f88..2460ec8a70e 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -54,6 +54,7 @@ def _prepare_environment() -> None: ("HOST_USER_ID", utils.get_user_id()), ("TUTOR_APP", __app__.replace("-", "_")), ("TUTOR_VERSION", __version__), + ("is_buildkit_enabled", utils.is_buildkit_enabled), ], ) diff --git a/tutor/images.py b/tutor/images.py index d0637fe4903..0d1e80bd578 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -9,9 +9,10 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - command = hooks.Filters.DOCKER_BUILD_COMMAND.apply( - ["build", "-t", tag, *args, path] - ) + build_command = ["build", "-t", tag, *args, path] + if utils.is_buildkit_enabled(): + build_command.insert(0, "buildx") + command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) utils.docker(*command) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index c48198589c2..86006a8751d 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,9 +1,12 @@ +{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %} ###### Minimal image with base system requirements for most stages FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \ + apt update && \ apt install -y build-essential curl git language-pack-en ENV LC_ALL en_US.UTF-8 {{ patch("openedx-dockerfile-minimal") }} @@ -11,16 +14,23 @@ ENV LC_ALL en_US.UTF-8 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv FROM minimal as python # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN apt update && \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ apt install -y libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev python-openssl git -# https://github.com/pyenv/pyenv/releases + +# Install pyenv # https://www.python.org/downloads/ +# https://github.com/pyenv/pyenv/releases ARG PYTHON_VERSION=3.8.15 ENV PYENV_ROOT /opt/pyenv RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1 + +# Install Python RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION + +# Create virtualenv RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv ###### Checkout edx-platform code @@ -45,6 +55,12 @@ RUN git config --global user.email "tutor@overhang.io" \ {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} +##### Empty layer with just the repo at the root. +# This is useful when overriding the build context with a host repo: +# docker build --build-context edx-platform=/path/to/edx-platform +FROM scratch as edx-platform +COPY --from=code /openedx/edx-platform / + ###### Download extra locales to /openedx/locale/contrib/locale FROM minimal as locales ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }} @@ -59,39 +75,42 @@ RUN cd /tmp \ FROM python as python-requirements ENV PATH /openedx/venv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ +ENV XDG_CACHE_HOME /openedx/.cache -RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev # Install the right version of pip/setuptools -# https://pypi.org/project/setuptools/ -# https://pypi.org/project/pip/ -# https://pypi.org/project/wheel/ -RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # https://pypi.org/project/setuptools/ + # https://pypi.org/project/pip/ + # https://pypi.org/project/wheel/ + setuptools==67.6.1 pip==23.0.1. wheel==0.40.0 # Install missing py2neo package that was abruptly trimmed from pypi RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # Install base requirements -COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt -RUN pip install -r /tmp/base.txt +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ + --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt -# Install django-redis for using redis as a django cache -# https://pypi.org/project/django-redis/ -RUN pip install django-redis==5.2.0 - -# Install uwsgi -# https://pypi.org/project/uWSGI/ -RUN pip install uwsgi==2.0.21 +# Install extra requirements +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \ + # Use redis as a django cache https://pypi.org/project/django-redis/ + django-redis==5.2.0 \ + # uwsgi server https://pypi.org/project/uWSGI/ + uwsgi==2.0.21 {{ patch("openedx-dockerfile-post-python-requirements") }} # Install private requirements: this is useful for installing custom xblocks. COPY ./requirements/ /openedx/requirements -RUN cd /openedx/requirements/ \ +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \ && touch ./private.txt \ && pip install -r ./private.txt -{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}' +{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}' {% endfor %} ###### Install nodejs with nodeenv in /openedx/nodeenv @@ -106,18 +125,18 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} -COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json -COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json WORKDIR /openedx/edx-platform -RUN npm clean-install --verbose --registry=$NPM_REGISTRY +RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ + --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ + --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY ###### Production image with system and python requirements FROM minimal as production # Install system requirements -RUN apt update && \ - apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \ - rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \ + && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind # From then on, run as unprivileged "app" user # Note that this must always be different from root (APP_USER_ID=0) @@ -127,14 +146,17 @@ RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags -COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize -COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform +COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize +COPY --chown=app:app --from=edx-platform / /openedx/edx-platform COPY --chown=app:app --from=locales /openedx/locale /openedx/locale COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules +COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules + +# Symlink node_modules such that we can bind-mount the edx-platform repository +RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ @@ -218,16 +240,16 @@ FROM production as development # Install useful system requirements (as root) USER root -RUN apt update && \ - apt install -y vim iputils-ping dnsutils telnet \ - && rm -rf /var/lib/apt/lists/* +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \ + apt install -y vim iputils-ping dnsutils telnet USER app # Install dev python requirements -RUN pip install -r requirements/edx/development.txt +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt # https://pypi.org/project/ipdb/ # https://pypi.org/project/ipython -RUN pip install ipdb==0.13.13 ipython==8.12.0 +RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0 # Add ipdb as default PYTHONBREAKPOINT ENV PYTHONBREAKPOINT=ipdb.set_trace diff --git a/tutor/utils.py b/tutor/utils.py index 545ce38b204..06c0e3d0b98 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -173,6 +173,25 @@ def docker(*command: str) -> int: return execute("docker", *command) +@lru_cache(maxsize=None) +def is_buildkit_enabled() -> bool: + """ + A helper function to determine whether we can run `docker buildx` with BuildKit. + """ + # First, we respect the DOCKER_BUILDKIT environment variable + enabled_by_env = { + "1": True, + "0": False, + }.get(os.environ.get("DOCKER_BUILDKIT", "")) + if enabled_by_env is not None: + return enabled_by_env + try: + subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True) + return True + except subprocess.CalledProcessError: + return False + + def docker_compose(*command: str) -> int: return execute("docker", "compose", *command) From a518981fd4bb76be06547e19e167be761d190c0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 27 Apr 2023 20:25:20 +0200 Subject: [PATCH 17/96] feat: persistent bind-mounts This is an important change, where we get remove the previous `--mount` option, and instead opt for persistent bind-mounts. Persistent bind mounts have several advantages: - They make it easier to remember which folders need to be bind-mounted. - Code is *much* less clunky, as we no longer need to generate temporary docker-compose files. - They allow us to bind-mount host directories *at build time* using the buildx `--build-context` option. - The transition from development to production becomes much easier, as images will automatically be built using the host repo. The only drawback is that persistent bind-mounts are slightly less portable: when a config.yml file is moved to a different folder, many things will break if the repo is not checked out in the same path. For instance, this is how to start working on a local fork of edx-platform: tutor config save --append MOUNTS=/path/to/edx-platform And that's all there is to it. No, this fork will be used whenever we run: tutor images build openedx tutor local start tutor dev start This change is made possible by huge improvements in the build time performance. These improvements make it convenient to re-build Docker images often. Related issues: https://github.com/openedx/wg-developer-experience/issues/71 https://github.com/openedx/wg-developer-experience/issues/66 https://github.com/openedx/wg-developer-experience/issues/166 --- .../20230427_165520_regis_build_mount.md | 2 + docs/dev.rst | 100 ++++----- docs/tutorials/arm64.rst | 1 + tests/commands/test_compose.py | 88 -------- tests/commands/test_images.py | 2 +- tests/test_bindmount.py | 65 ++++++ tutor/bindmount.py | 71 ++++++ tutor/bindmounts.py | 61 ------ tutor/commands/compose.py | 203 +----------------- tutor/commands/dev.py | 21 -- tutor/commands/images.py | 52 ++++- tutor/commands/local.py | 22 -- tutor/hooks/catalog.py | 55 ++--- tutor/images.py | 2 +- tutor/templates/config/defaults.yml | 1 + tutor/templates/local/docker-compose.jobs.yml | 6 + tutor/templates/local/docker-compose.yml | 32 ++- 17 files changed, 310 insertions(+), 474 deletions(-) delete mode 100644 tests/commands/test_compose.py create mode 100644 tests/test_bindmount.py create mode 100644 tutor/bindmount.py delete mode 100644 tutor/bindmounts.py diff --git a/changelog.d/20230427_165520_regis_build_mount.md b/changelog.d/20230427_165520_regis_build_mount.md index fb29b8cf85c..36fc658b9e4 100644 --- a/changelog.d/20230427_165520_regis_build_mount.md +++ b/changelog.d/20230427_165520_regis_build_mount.md @@ -1 +1,3 @@ - [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) +- [Feature] Automatically mount host folders *at build time*. This is a really important feature, as it allows us to transparently build images using local forks of remote repositories. (by @regisb) +- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts. (by @regisb) diff --git a/docs/dev.rst b/docs/dev.rst index 5ee10c01a5e..684d806bd96 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -12,31 +12,25 @@ First-time setup Firstly, either :ref:`install Tutor ` (for development against the named releases of Open edX) or :ref:`install Tutor Nightly ` (for development against Open edX's master branches). +Then, optionally, tell Tutor to use a local fork of edx-platform. In that case you will need to rebuild the "openedx" Docker image:: + + tutor config save --append MOUNTS=./edx-platform + tutor images build openedx + Then, run one of the following in order to launch the developer platform setup process:: # To use the edx-platform repository that is built into the image, run: tutor dev launch - # To bind-mount and run a local clone of edx-platform, replace - # './edx-platform' with the path to the local clone and run: - tutor dev launch --mount=./edx-platform - This will perform several tasks. It will: * stop any existing locally-running Tutor containers, - * disable HTTPS, - * set ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), - * prompt for a platform details (with suitable defaults), - * build an ``openedx-dev`` image, which is based ``openedx`` production image but is `specialized for developer usage`_, - * start LMS, CMS, supporting services, and any plugged-in services, - * ensure databases are created and migrated, and - * run service initialization scripts, such as service user creation and Waffle configuration. Additionally, when a local clone of edx-platform is bind-mounted, it will: @@ -55,10 +49,13 @@ Now, use the ``tutor dev ...`` command-line interface to manage the development .. note:: - Wherever the ``[--mount=./edx-platform]`` option is present, either: + If you've added your edx-platform to the ``MOUNTS`` setting, you can remove at any time by running:: + + tutor config save --remove MOUNTS=./edx-platform - * omit it when running of the edx-platform repository built into the image, or - * substitute it with ``--mount=``. + At any time, check your configuration by running:: + + tutor config printvalue MOUNTS Read more about bind-mounts :ref:`below `. @@ -74,17 +71,17 @@ Starting the platform back up Once first-time setup has been performed with ``launch``, the platform can be started going forward with the lighter-weight ``start -d`` command, which brings up containers *detached* (that is: in the background), but does not perform any initialization tasks:: - tutor dev start -d [--mount=./edx-platform] + tutor dev start -d Or, to start with platform with containers *attached* (that is: in the foreground, the current terminal), omit the ``-d`` flag:: - tutor dev start [--mount=./edx-platform] + tutor dev start When running containers attached, stop the platform with ``Ctrl+c``, or switch to detached mode using ``Ctrl+z``. Finally, the platform can also be started back up with ``launch``. It will take longer than ``start``, but it will ensure that config is applied, databases are provisioned & migrated, plugins are fully initialized, and (if applicable) the bind-mounted edx-platform is set up. Notably, ``launch`` is idempotent, so it is always safe to run it again without risk to data. Including the ``--pullimages`` flag will also ensure that container images are up-to-date:: - tutor dev launch [--mount=./edx-platform] --pullimages + tutor dev launch --pullimages Debugging with breakpoints -------------------------- @@ -92,32 +89,32 @@ Debugging with breakpoints To debug a local edx-platform repository, add a `python breakpoint `__ with ``breakpoint()`` anywhere in the code. Then, attach to the applicable service's container by running ``start`` (without ``-d``) followed by the service's name:: # Debugging LMS: - tutor dev start [--mount=./edx-platform] lms + tutor dev start lms # Or, debugging CMS: - tutor dev start [--mount=./edx-platform] cms + tutor dev start cms Running arbitrary commands -------------------------- To run any command inside one of the containers, run ``tutor dev run [OPTIONS] SERVICE [COMMAND] [ARGS]...``. For instance, to open a bash shell in the LMS or CMS containers:: - tutor dev run [--mount=./edx-platform] lms bash - tutor dev run [--mount=./edx-platform] cms bash + tutor dev run lms bash + tutor dev run cms bash To open a python shell in the LMS or CMS, run:: - tutor dev run [--mount=./edx-platform] lms ./manage.py lms shell - tutor dev run [--mount=./edx-platform] cms ./manage.py cms shell + tutor dev run lms ./manage.py lms shell + tutor dev run cms ./manage.py cms shell You can then import edx-platform and django modules and execute python code. To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor:: - tutor dev run [--mount=./edx-platform] lms openedx-assets build --env=dev + tutor dev run lms openedx-assets build --env=dev -.. _specialized for developer usage: +.. _specialized for developer usage: Rebuilding the openedx-dev image -------------------------------- @@ -143,35 +140,42 @@ Sharing directories with containers It may sometimes be convenient to mount container directories on the host, for instance: for editing and debugging. Tutor provides different solutions to this problem. -.. _mount_option: +.. _persistent_mounts: + +Persistent bind-mounted volumes with ``MOUNTS`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``MOUNTS`` is a Tutor setting to bind-mount host directories both at build time and run time: + +- At build time: plugins can automatically add certain directories listed in this setting to the `Docker build context `__. This makes it possible to transparently build a Docker image using a locally checked-out repository. +- At run time: host directories will be bind-mounted in running containers, using either an automatic or a manual configuration. + +After some values have been added to the ``MOUNTS`` setting, all ``tutor dev`` and ``tutor local`` commands will make use of these bind-mount volumes. -Bind-mount volumes with ``--mount`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Values added to ``MOUNTS`` can take one of two forms. The first is explicit:: -The ``launch``, ``run``, ``init`` and ``start`` subcommands of ``tutor dev`` and ``tutor local`` support the ``-m/--mount`` option (see :option:`tutor dev start -m`) which can take two different forms. The first is explicit:: + tutor config save --append MOUNTS=lms:/path/to/edx-platform:/openedx/edx-platform - tutor dev start --mount=lms:/path/to/edx-platform:/openedx/edx-platform lms +The second is implicit:: -And the second is implicit:: + tutor config save --append MOUNTS=/path/to/edx-platform - tutor dev start --mount=/path/to/edx-platform lms +With the explicit form, the setting means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container at run time". -With the explicit form, the ``--mount`` option means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container". +If you use the explicit format, you will quickly realise that you usually want to bind-mount folders in multiple containers at a time. For instance, you will want to bind-mount the edx-platform repository in the "cms" container, but also the "lms-worker" and "cms-worker" containers. To do that, write instead:: -If you use the explicit format, you will quickly realise that you usually want to bind-mount folders in multiple containers at a time. For instance, you will want to bind-mount the edx-platform repository in the "cms" container. To do that, write instead:: + # each service is added to a coma-separated list + tutor config save --append MOUNTS=lms,cms,lms-worker,cms-worker:/path/to/edx-platform:/openedx/edx-platform - tutor dev start --mount=lms,cms:/path/to/edx-platform:/openedx/edx-platform lms +This command line is a bit cumbersome. In addition, with this explicit form, the edx-platform repository will *not* be added to the build context at build time. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``MOUNTS`` setting. For instance, the following implicit form can be used instead of the explicit form above:: -This command line can become cumbersome and inconvenient to work with. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``--mount`` option. For instance, the following commands are equivalent:: + tutor config save --append MOUNTS=/path/to/edx-platform - # Explicit form - tutor dev start --mount=lms,lms-worker,lms-job,cms,cms-worker,cms-job:/path/to/edx-platform:/openedx/edx-platform lms - # Implicit form - tutor dev start --mount=/path/to/edx-platform lms +With this implicit form, the edx-platform repo will be bind-mounted in the containers at run time, just like with the explicit form. But in addition, the edx-platform will also automatically be added to the Docker image at build time. -So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``--mount=~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: +So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``--append MOUNTS=~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: - tutor dev start --mount=lms:~/venvs/edx-platform:/openedx/venv lms + tutor config save --append MOUNTS=lms:~/venvs/edx-platform:/openedx/venv .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`. @@ -182,16 +186,16 @@ Sometimes, you may want to modify some of the files inside a container for which tutor dev copyfrom lms /openedx/venv ~ -Then, bind-mount that folder back in the container with the ``--mount`` option (described :ref:`above `):: +Then, bind-mount that folder back in the container with the ``MOUNTS`` setting (described :ref:`above `):: - tutor dev start --mount lms:~/venv:/openedx/venv lms + tutor config save --append MOUNTS=lms:~/venv:/openedx/venv -You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. +You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your "lms" container. Manual bind-mount to any directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. warning:: Manually bind-mounting volumes with the ``--volume`` option makes it difficult to simultaneously bind-mount to multiple containers. Also, the ``--volume`` options are not compatible with ``start`` commands. For an alternative, see the :ref:`mount option `. +.. warning:: Manually bind-mounting volumes with the ``--volume`` option makes it difficult to simultaneously bind-mount to multiple containers. Also, the ``--volume`` options are not compatible with ``start`` commands. For an alternative, see the :ref:`persistent mounts `. The above solution may not work for you if you already have an existing directory, outside of the "volumes/" directory, which you would like mounted in one of your containers. For instance, you may want to mount your copy of the `edx-platform `__ repository. In such cases, you can simply use the ``-v/--volume`` `Docker option `__:: @@ -200,7 +204,7 @@ The above solution may not work for you if you already have an existing director Override docker-compose volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The above solutions require that you explicitly pass the ``-m/--mount`` options to every ``run``, ``start`` or ``init`` command, which may be inconvenient. To address these issues, you can create a ``docker-compose.override.yml`` file that will specify custom volumes to be used with all ``dev`` commands:: +Adding items to the ``MOUNTS`` setting effectively adds new bind-mount volumes to the ``docker-compose.yml`` files. But you might want to have more control over your volumes, such as adding read-only options, or customising other fields of the different services. To address these issues, you can create a ``docker-compose.override.yml`` file that will specify custom volumes to be used with all ``dev`` commands:: vim "$(tutor config printroot)/env/dev/docker-compose.override.yml" @@ -221,7 +225,7 @@ You are then free to bind-mount any directory to any container. For instance, to volumes: - /path/to/edx-platform:/openedx/edx-platform -This override file will be loaded when running any ``tutor dev ..`` command. The edx-platform repo mounted at the specified path will be automatically mounted inside all LMS and CMS containers. With this file, you should no longer specify the ``-m/--mount`` option from the command line. +This override file will be loaded when running any ``tutor dev ..`` command. The edx-platform repo mounted at the specified path will be automatically mounted inside all LMS and CMS containers. .. note:: The ``tutor local`` commands load the ``docker-compose.override.yml`` file from the ``$(tutor config printroot)/env/local/docker-compose.override.yml`` directory. One-time jobs from initialisation commands load the ``local/docker-compose.jobs.override.yml`` and ``dev/docker-compose.jobs.override.yml``. diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 0ddc177a714..63c8a92944a 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -27,6 +27,7 @@ Then, build the "openedx" and "permissions" images:: tutor images build openedx permissions .. TODO we don't want this instruction anymore + If you want to use Tutor as an Open edX development environment, you should also build the development images:: tutor dev dc build lms diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py deleted file mode 100644 index 9cccfb94e91..00000000000 --- a/tests/commands/test_compose.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import typing as t -import unittest -from io import StringIO -from unittest.mock import patch - -from click.exceptions import ClickException - -from tutor import hooks -from tutor.commands import compose -from tutor.commands.local import LocalContext - - -class ComposeTests(unittest.TestCase): - maxDiff = None # Ensure we can see long diffs of YAML files. - - def test_mount_option_parsing(self) -> None: - param = compose.MountParam() - - self.assertEqual( - [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], - param("lms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms,cms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms, cms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms,lms-worker:/path/to/edx-platform:/openedx/edx-platform"), - ) - with self.assertRaises(ClickException): - param("lms,:/path/to/edx-platform:/openedx/edx-platform") - - @patch("sys.stdout", new_callable=StringIO) - def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: - """ - Ensure that docker-compose.tmp.yml is correctly generated. - """ - param = compose.MountParam() - mount_args = ( - # Auto-mounting of edx-platform to lms* and cms* - param.convert_implicit_form("/path/to/edx-platform"), - # Manual mounting of some other folder to mfe and lms - param.convert_explicit_form( - "mfe,lms:/path/to/something-else:/openedx/something-else" - ), - ) - # Mount volumes - compose.mount_tmp_volumes(mount_args, LocalContext("")) - - compose_file: dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) - actual_services: dict[str, t.Any] = compose_file["services"] - expected_services: dict[str, t.Any] = { - "cms": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "cms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "lms": { - "volumes": [ - "/path/to/edx-platform:/openedx/edx-platform", - "/path/to/something-else:/openedx/something-else", - ] - }, - "lms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "mfe": {"volumes": ["/path/to/something-else:/openedx/something-else"]}, - } - self.assertEqual(actual_services, expected_services) - - compose_jobs_file = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) - actual_jobs_services = compose_jobs_file["services"] - expected_jobs_services: dict[str, t.Any] = { - "cms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "lms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - } - self.assertEqual(actual_jobs_services, expected_jobs_services) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 622be9b8aa7..f3e0132b617 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -149,7 +149,7 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "docker_args", "--cache-from=type=registry,ref=service1:1.0.0-cache", ], - list(image_build.call_args[0][1:]) + list(image_build.call_args[0][1:]), ) def test_images_push(self) -> None: diff --git a/tests/test_bindmount.py b/tests/test_bindmount.py new file mode 100644 index 00000000000..4e656a1c26e --- /dev/null +++ b/tests/test_bindmount.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import unittest + +from tutor import bindmount + + +class BindmountTests(unittest.TestCase): + def test_parse_explicit(self) -> None: + self.assertEqual( + [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], + bindmount.parse_explicit_mount( + "lms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms,cms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms, cms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms,lms-worker:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], + bindmount.parse_explicit_mount( + "lms,:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + + def test_parse_implicit(self) -> None: + # Import module to make sure filter is created + # pylint: disable=import-outside-toplevel,unused-import + import tutor.commands.compose + + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-job", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms-job", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_implicit_mount("/path/to/edx-platform"), + ) diff --git a/tutor/bindmount.py b/tutor/bindmount.py new file mode 100644 index 00000000000..0f2d2069b49 --- /dev/null +++ b/tutor/bindmount.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from functools import lru_cache +import os +import re +import typing as t + +from tutor import hooks + + +def iter_mounts(user_mounts: list[str], name: str) -> t.Iterable[str]: + """ + Iterate on the bind-mounts that are available to any given compose service. The list + of bind-mounts is parsed from `user_mounts` and we yield only those for service + `name`. + + Calling this function multiple times makes repeated calls to the parsing functions, + but that's OK because their result is cached. + """ + for user_mount in user_mounts: + for service, host_path, container_path in parse_mount(user_mount): + if service == name: + yield f"{host_path}:{container_path}" + + +def parse_mount(value: str) -> list[tuple[str, str, str]]: + """ + Parser for mount arguments of the form "service1[,service2,...]:/host/path:/container/path". + + Returns a list of (service, host_path, container_path) tuples. + """ + mounts = parse_explicit_mount(value) or parse_implicit_mount(value) + return mounts + + +@lru_cache(maxsize=None) +def parse_explicit_mount(value: str) -> list[tuple[str, str, str]]: + """ + Argument is of the form "containers:/host/path:/container/path". + """ + # Note that this syntax does not allow us to include colon ':' characters in paths + match = re.match( + r"(?P[a-zA-Z0-9-_, ]+):(?P[^:]+):(?P[^:]+)", + value, + ) + if not match: + return [] + + mounts: list[tuple[str, str, str]] = [] + services: list[str] = [service.strip() for service in match["services"].split(",")] + host_path = os.path.abspath(os.path.expanduser(match["host_path"])) + host_path = host_path.replace(os.path.sep, "/") + container_path = match["container_path"] + for service in services: + if service: + mounts.append((service, host_path, container_path)) + return mounts + + +@lru_cache(maxsize=None) +def parse_implicit_mount(value: str) -> list[tuple[str, str, str]]: + """ + Argument is of the form "/host/path" + """ + mounts: list[tuple[str, str, str]] = [] + host_path = os.path.abspath(os.path.expanduser(value)) + for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( + os.path.basename(host_path) + ): + mounts.append((service, host_path, container_path)) + return mounts diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py deleted file mode 100644 index d80735812cf..00000000000 --- a/tutor/bindmounts.py +++ /dev/null @@ -1,61 +0,0 @@ -import os - -from tutor.exceptions import TutorError -from tutor.tasks import BaseComposeTaskRunner -from tutor.utils import get_user_id - - -def create( - runner: BaseComposeTaskRunner, - service: str, - path: str, -) -> str: - volumes_root_path = get_root_path(runner.root) - volume_name = get_name(path) - container_volumes_root_path = "/tmp/volumes" - command = """rm -rf {volumes_path}/{volume_name} -cp -r {src_path} {volumes_path}/{volume_name} -chown -R {user_id} {volumes_path}/{volume_name}""".format( - volumes_path=container_volumes_root_path, - volume_name=volume_name, - src_path=path, - user_id=get_user_id(), - ) - - # Create volumes root dir if it does not exist. Otherwise it is created with root owner and might not be writable - # in the container, e.g: in the dev containers. - if not os.path.exists(volumes_root_path): - os.makedirs(volumes_root_path) - - runner.docker_compose( - "run", - "--rm", - "--no-deps", - "--user=0", - "--volume", - f"{volumes_root_path}:{container_volumes_root_path}", - service, - "sh", - "-e", - "-c", - command, - ) - return os.path.join(volumes_root_path, volume_name) - - -def get_path(root: str, container_bind_path: str) -> str: - bind_basename = get_name(container_bind_path) - return os.path.join(get_root_path(root), bind_basename) - - -def get_name(container_bind_path: str) -> str: - # We rstrip slashes, otherwise os.path.basename returns an empty string - # We don't use basename here as it will not work on Windows - name = container_bind_path.rstrip("/").split("/")[-1] - if not name: - raise TutorError("Mounting a container root folder is not supported") - return name - - -def get_root_path(root: str) -> str: - return os.path.join(root, "volumes") diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 44f515853a8..53112d0eb7c 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,17 +1,12 @@ from __future__ import annotations import os -import re -import typing as t -from copy import deepcopy import click -from click.shell_completion import CompletionItem -from typing_extensions import TypeAlias from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks, serialize, utils +from tutor import bindmount, hooks, utils from tutor.commands import jobs from tutor.commands.context import BaseTaskContext from tutor.core.hooks import Filter # pylint: disable=unused-import @@ -19,8 +14,6 @@ from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -COMPOSE_FILTER_TYPE: TypeAlias = "Filter[dict[str, t.Any], []]" - class ComposeTaskRunner(BaseComposeTaskRunner): def __init__(self, root: str, config: Config): @@ -47,47 +40,6 @@ def docker_compose(self, *command: str) -> int: *args, "--project-name", self.project_name, *command ) - def update_docker_compose_tmp( - self, - compose_tmp_filter: COMPOSE_FILTER_TYPE, - compose_jobs_tmp_filter: COMPOSE_FILTER_TYPE, - docker_compose_tmp_path: str, - docker_compose_jobs_tmp_path: str, - ) -> None: - """ - Update the contents of the docker-compose.tmp.yml and - docker-compose.jobs.tmp.yml files, which are generated at runtime. - """ - compose_base: dict[str, t.Any] = { - "version": "{{ DOCKER_COMPOSE_VERSION }}", - "services": {}, - } - - # 1. Apply compose_tmp filter - # 2. Render the resulting dict - # 3. Serialize to yaml - # 4. Save to disk - docker_compose_tmp: str = serialize.dumps( - tutor_env.render_unknown( - self.config, compose_tmp_filter.apply(deepcopy(compose_base)) - ) - ) - tutor_env.write_to( - docker_compose_tmp, - docker_compose_tmp_path, - ) - - # Same thing but with tmp jobs - docker_compose_jobs_tmp: str = serialize.dumps( - tutor_env.render_unknown( - self.config, compose_jobs_tmp_filter.apply(deepcopy(compose_base)) - ) - ) - tutor_env.write_to( - docker_compose_jobs_tmp, - docker_compose_jobs_tmp_path, - ) - def run_task(self, service: str, command: str) -> int: """ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the @@ -113,148 +65,22 @@ def run_task(self, service: str, command: str) -> int: class BaseComposeContext(BaseTaskContext): - COMPOSE_TMP_FILTER: COMPOSE_FILTER_TYPE = NotImplemented - COMPOSE_JOBS_TMP_FILTER: COMPOSE_FILTER_TYPE = NotImplemented - def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError -class MountParam(click.ParamType): - """ - Parser for --mount arguments of the form "service1[,service2,...]:/host/path:/container/path". - """ - - name = "mount" - MountType = t.Tuple[str, str, str] - # Note that this syntax does not allow us to include colon ':' characters in paths - PARAM_REGEXP = ( - r"(?P[a-zA-Z0-9-_, ]+):(?P[^:]+):(?P[^:]+)" - ) - - def convert( - self, - value: str, - param: t.Optional["click.Parameter"], - ctx: t.Optional[click.Context], - ) -> list["MountType"]: - mounts = self.convert_explicit_form(value) or self.convert_implicit_form(value) - return mounts - - def convert_explicit_form(self, value: str) -> list["MountParam.MountType"]: - """ - Argument is of the form "containers:/host/path:/container/path". - """ - match = re.match(self.PARAM_REGEXP, value) - if not match: - return [] - - mounts: list["MountParam.MountType"] = [] - services: list[str] = [ - service.strip() for service in match["services"].split(",") - ] - host_path = os.path.abspath(os.path.expanduser(match["host_path"])) - host_path = host_path.replace(os.path.sep, "/") - container_path = match["container_path"] - for service in services: - if not service: - self.fail(f"incorrect services syntax: '{match['services']}'") - mounts.append((service, host_path, container_path)) - return mounts - - def convert_implicit_form(self, value: str) -> list["MountParam.MountType"]: - """ - Argument is of the form "/host/path" - """ - mounts: list["MountParam.MountType"] = [] - host_path = os.path.abspath(os.path.expanduser(value)) - for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( - os.path.basename(host_path) - ): - mounts.append((service, host_path, container_path)) - if not mounts: - raise self.fail(f"no mount found for {value}") - return mounts - - def shell_complete( - self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> list[CompletionItem]: - """ - Mount argument completion works only for the single path (implicit) form. The - reason is that colons break words in bash completion: - http://tiswww.case.edu/php/chet/bash/FAQ (E13) - Thus, we do not even attempt to auto-complete mount arguments that include - colons: such arguments will not even reach this method. - """ - return [CompletionItem(incomplete, type="file")] - - -mount_option = click.option( - "-m", - "--mount", - "mounts", - help="""Bind-mount a folder from the host in the right containers. This option can take two different forms. The first one is explicit: 'service1[,service2...]:/host/path:/container/path'. The other is implicit: '/host/path'. Arguments passed in the implicit form will be parsed by plugins to define the right folders to bind-mount from the host.""", - type=MountParam(), - multiple=True, -) - - -def mount_tmp_volumes( - all_mounts: tuple[list[MountParam.MountType], ...], - context: BaseComposeContext, -) -> None: - for mounts in all_mounts: - for service, host_path, container_path in mounts: - mount_tmp_volume(service, host_path, container_path, context) - - -def mount_tmp_volume( - service: str, - host_path: str, - container_path: str, - context: BaseComposeContext, -) -> None: - """ - Append user-defined bind-mounted volumes to the docker-compose.tmp file(s). - - The service/host path/container path values are appended to the docker-compose - files by mean of two filters. Each dev/local environment is then responsible for - generating the files based on the output of these filters. - - Bind-mounts that are associated to "*-job" services will be added to the - docker-compose jobs file. - """ - fmt.echo_info(f"Bind-mount: {host_path} -> {container_path} in {service}") - compose_tmp_filter: COMPOSE_FILTER_TYPE = ( - context.COMPOSE_JOBS_TMP_FILTER - if service.endswith("-job") - else context.COMPOSE_TMP_FILTER - ) - - @compose_tmp_filter.add() - def _add_mounts_to_docker_compose_tmp( - docker_compose: dict[str, t.Any], - ) -> dict[str, t.Any]: - services = docker_compose.setdefault("services", {}) - services.setdefault(service, {"volumes": []}) - services[service]["volumes"].append(f"{host_path}:{container_path}") - return docker_compose - - @click.command( short_help="Run all or a selection of services.", help="Run all or a selection of services. Docker images will be rebuilt where necessary.", ) @click.option("--skip-build", is_flag=True, help="Skip image building") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") -@mount_option @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def start( context: BaseComposeContext, skip_build: bool, detach: bool, - mounts: tuple[list[MountParam.MountType]], services: list[str], ) -> None: command = ["up", "--remove-orphans"] @@ -264,7 +90,6 @@ def start( command.append("-d") # Start services - mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) context.job_runner(config).docker_compose(*command, *services) @@ -313,21 +138,11 @@ def restart(context: BaseComposeContext, services: list[str]) -> None: @jobs.do_group -@mount_option -@click.pass_obj -def do(context: BaseComposeContext, mounts: tuple[list[MountParam.MountType]]) -> None: +def do() -> None: """ Run a custom job in the right container(s). """ - @hooks.Actions.DO_JOB.add() - def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: - """ - We add this logic to an action callback because we do not want to trigger it - whenever we run `tutor local do --help`. - """ - mount_tmp_volumes(mounts, context) - @click.command( short_help="Run a command in a new container", @@ -338,18 +153,16 @@ def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: ), context_settings={"ignore_unknown_options": True}, ) -@mount_option @click.argument("args", nargs=-1, required=True) @click.pass_context def run( context: click.Context, - mounts: tuple[list[MountParam.MountType]], args: list[str], ) -> None: extra_args = ["--rm"] if not utils.is_a_tty(): extra_args.append("-T") - context.invoke(dc_command, mounts=mounts, command="run", args=[*extra_args, *args]) + context.invoke(dc_command, command="run", args=[*extra_args, *args]) @click.command( @@ -446,17 +259,14 @@ def status(context: click.Context) -> None: context_settings={"ignore_unknown_options": True}, name="dc", ) -@mount_option @click.argument("command") @click.argument("args", nargs=-1) @click.pass_obj def dc_command( context: BaseComposeContext, - mounts: tuple[list[MountParam.MountType]], command: str, args: list[str], ) -> None: - mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) context.job_runner(config).docker_compose(command, *args) @@ -466,8 +276,8 @@ def _mount_edx_platform( volumes: list[tuple[str, str]], name: str ) -> list[tuple[str, str]]: """ - When mounting edx-platform with `--mount=/path/to/edx-platform`, bind-mount the host - repo in the lms/cms containers. + When mounting edx-platform with `tutor config save --append MOUNTS=/path/to/edx-platform`, + bind-mount the host repo in the lms/cms containers. """ if name == "edx-platform": path = "/openedx/edx-platform" @@ -482,6 +292,9 @@ def _mount_edx_platform( return volumes +hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) + + def add_commands(command_group: click.Group) -> None: command_group.add_command(start) command_group.add_command(stop) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 124c6a9728f..f5f5df286b0 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -18,39 +18,21 @@ def __init__(self, root: str, config: Config): """ super().__init__(root, config) self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str) - docker_compose_tmp_path = tutor_env.pathjoin( - self.root, "dev", "docker-compose.tmp.yml" - ) - docker_compose_jobs_tmp_path = tutor_env.pathjoin( - self.root, "dev", "docker-compose.jobs.tmp.yml" - ) self.docker_compose_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"), - docker_compose_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"), ] self.docker_compose_job_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"), - docker_compose_jobs_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"), ] - # Update docker-compose.tmp files - self.update_docker_compose_tmp( - hooks.Filters.COMPOSE_DEV_TMP, - hooks.Filters.COMPOSE_DEV_JOBS_TMP, - docker_compose_tmp_path, - docker_compose_jobs_tmp_path, - ) class DevContext(compose.BaseComposeContext): - COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_DEV_TMP - COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_DEV_JOBS_TMP - def job_runner(self, config: Config) -> DevTaskRunner: return DevTaskRunner(self.root, config) @@ -64,15 +46,12 @@ def dev(context: click.Context) -> None: @click.command(help="Configure and run Open edX from scratch, for development") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@compose.mount_option @click.pass_context def launch( context: click.Context, non_interactive: bool, pullimages: bool, - mounts: tuple[list[compose.MountParam.MountType]], ) -> None: - compose.mount_tmp_volumes(mounts, context.obj) utils.warn_macos_docker_memory() click.echo(fmt.title("Interactive platform configuration")) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 1dfce27b8f5..ca5e7196d36 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,12 +1,13 @@ from __future__ import annotations +import os import typing as t import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, hooks, images, utils +from tutor import exceptions, hooks, images, types, utils from tutor.commands.context import Context from tutor.core.hooks import Filter from tutor.types import Config @@ -148,16 +149,26 @@ def build( command_args.append(f"--output={docker_output}") if docker_args: command_args += docker_args + # Build context mounts + build_contexts = get_image_build_contexts(config) + for image in image_names: - for _name, path, tag, custom_args in find_images_to_build(config, image): + for name, path, tag, custom_args in find_images_to_build(config, image): image_build_args = [*command_args, *custom_args] + + # Registry cache if not no_registry_cache: - # Use registry cache image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") if cache_to_registry: image_build_args.append( f"--cache-to=type=registry,mode=max,ref={tag}-cache" ) + + # Build contexts + for host_path, stage_name in build_contexts.get(name, []): + image_build_args.append(f"--build-context={stage_name}={host_path}") + + # Build images.build( tutor_env.pathjoin(context.root, *path), tag, @@ -165,6 +176,41 @@ def build( ) +def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]]: + """ + Return all build contexts for all images. + + A build context is to bind-mount a host directory at build-time. This is useful, for + instance to build a Docker image with a local git checkout of a remote repo. + + Users configure bind-mounts with the `MOUNTS` config setting. Plugins can then + automaticall add build contexts based on these values. + """ + user_mounts = types.get_typed(config, "MOUNTS", list) + build_contexts: dict[str, list[tuple[str, str]]] = {} + for user_mount in user_mounts: + for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( + user_mount + ): + if image_name not in build_contexts: + build_contexts[image_name] = [] + build_contexts[image_name].append((user_mount, stage_name)) + return build_contexts + + +@hooks.Filters.IMAGES_BUILD_MOUNTS.add() +def _mount_edx_platform( + volumes: list[tuple[str, str]], path: str +) -> list[tuple[str, str]]: + """ + Automatically add an edx-platform repo from the host to the build context whenever + it is added to the `MOUNTS` setting. + """ + if os.path.basename(path) == "edx-platform": + volumes.append(("openedx", "edx-platform")) + return volumes + + @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 9a91392c069..9ead815d845 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -23,38 +23,19 @@ def __init__(self, root: str, config: Config): """ super().__init__(root, config) self.project_name = get_typed(self.config, "LOCAL_PROJECT_NAME", str) - docker_compose_tmp_path = tutor_env.pathjoin( - self.root, "local", "docker-compose.tmp.yml" - ) - docker_compose_jobs_tmp_path = tutor_env.pathjoin( - self.root, "local", "docker-compose.jobs.tmp.yml" - ) self.docker_compose_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.yml"), tutor_env.pathjoin(self.root, "local", "docker-compose.prod.yml"), - docker_compose_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"), ] self.docker_compose_job_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"), - docker_compose_jobs_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"), ] - # Update docker-compose.tmp files - self.update_docker_compose_tmp( - hooks.Filters.COMPOSE_LOCAL_TMP, - hooks.Filters.COMPOSE_LOCAL_JOBS_TMP, - docker_compose_tmp_path, - docker_compose_jobs_tmp_path, - ) - # pylint: disable=too-few-public-methods class LocalContext(compose.BaseComposeContext): - COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_TMP - COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP - def job_runner(self, config: Config) -> LocalTaskRunner: return LocalTaskRunner(self.root, config) @@ -66,17 +47,14 @@ def local(context: click.Context) -> None: @click.command(help="Configure and run Open edX from scratch") -@compose.mount_option @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @click.pass_context def launch( context: click.Context, - mounts: tuple[list[compose.MountParam.MountType]], non_interactive: bool, pullimages: bool, ) -> None: - compose.mount_tmp_volumes(mounts, context.obj) utils.warn_macos_docker_memory() run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 9db9d7037fe..a21fd515ac0 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -227,33 +227,20 @@ def your_filter_callback(some_data): "commands:pre-init" ) - #: Same as :py:data:`COMPOSE_LOCAL_JOBS_TMP` but for the development environment. - COMPOSE_DEV_JOBS_TMP: Filter[Config, []] = filters.get("compose:dev-jobs:tmp") - - #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for the development environment. - COMPOSE_DEV_TMP: Filter[Config, []] = filters.get("compose:dev:tmp") - - #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for jobs - COMPOSE_LOCAL_JOBS_TMP: Filter[Config, []] = filters.get("compose:local-jobs:tmp") - - #: Contents of the (local|dev)/docker-compose.tmp.yml files that will be generated at - #: runtime. This is used for instance to bind-mount folders from the host (see - #: :py:data:`COMPOSE_MOUNTS`) - #: - #: :parameter dict[str, ...] docker_compose_tmp: values which will be serialized to local/docker-compose.tmp.yml. - #: Keys and values will be rendered before saving, such that you may include ``{{ ... }}`` statements. - COMPOSE_LOCAL_TMP: Filter[Config, []] = filters.get("compose:local:tmp") - #: List of folders to bind-mount in docker-compose containers, either in ``tutor local`` or ``tutor dev``. #: - #: Many ``tutor local`` and ``tutor dev`` commands support ``--mounts`` options - #: that allow plugins to define custom behaviour at runtime. For instance - #: ``--mount=/path/to/edx-platform`` would cause this host folder to be - #: bind-mounted in different containers (lms, lms-worker, cms, cms-worker) at the + #: This filter is for processing values of the ``MOUNTS`` setting such as:: + #: + #: tutor config save --append MOUNTS=/path/to/edx-platform + #: + #: In this example, this host folder would be bind-mounted in different containers + #: (lms, lms-worker, cms, cms-worker, lms-job, cms-job) at the #: /openedx/edx-platform location. Plugin developers may implement this filter to #: define custom behaviour when mounting folders that relate to their plugins. For - #: instance, the ecommerce plugin may process the ``--mount=/path/to/ecommerce`` - #: option. + #: instance, the ecommerce plugin may process the ``/path/to/ecommerce`` value. + #: + #: To also bind-mount these folder at build time, implement also the + #: :py:data:`IMAGES_BUILD_MOUNTS` filter. #: #: :parameter list[tuple[str, str]] mounts: each item is a ``(service, path)`` #: tuple, where ``service`` is the name of the docker-compose service and ``path`` is @@ -262,7 +249,7 @@ def your_filter_callback(some_data): #: the ``path`` because it will fail on Windows. #: :parameter str name: basename of the host-mounted folder. In the example above, #: this is "edx-platform". When implementing this filter you should check this name to - #: conditionnally add mounts. + #: conditionally add mounts. COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get("compose:mounts") #: Declare new default configuration settings that don't necessarily have to be saved in the user @@ -402,6 +389,26 @@ def your_filter_callback(some_data): list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] ] = filters.get("images:build") + #: List of host directories to be automatically bind-mounted in Docker images at + #: build time. For instance, this is useful to build Docker images using a custom + #: repository on the host. + #: + #: This filter works similarly to the :py:data:`COMPOSE_MOUNTS` filter, with a few differences. + #: + #: :parameter list[tuple[str, str]] mounts: each item is a pair of ``(name, value)`` + #: used to generate a build context at build time. See the corresponding `Docker + #: documentation `__. + #: The following option will be added to the ``docker buildx build`` command: + #: ``--build-context={name}={value}``. If the Dockerfile contains a "name" stage, then + #: that stage will be replaced by the corresponding directory on the host. + #: :parameter str name: full path to the host-mounted folder. As opposed to + #: :py:data:`COMPOSE_MOUNTS`, this is not just the basename, but the full path. When + #: implementing this filter you should check this path (for instance: with + #: ``os.path.basename(path)``) to conditionally add mounts. + IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get( + "images:build:mounts" + ) + #: List of images to be pulled when we run ``tutor images pull ...``. #: #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. diff --git a/tutor/images.py b/tutor/images.py index 0d1e80bd578..dc640d00ee9 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -9,7 +9,7 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - build_command = ["build", "-t", tag, *args, path] + build_command = ["build", f"--tag={tag}", *args, path] if utils.is_buildkit_enabled(): build_command.insert(0, "buildx") command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index da341ed16f4..642d38aef90 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -43,6 +43,7 @@ MONGODB_USERNAME: "" MONGODB_PASSWORD: "" MONGODB_REPLICA_SET: "" MONGODB_USE_SSL: false +MOUNTS: [] OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" OPENEDX_CACHE_REDIS_DB: 1 diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 37e116c3286..c70fa23f594 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -27,6 +27,9 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "lms-job") %} + - {{ mount }} + {%- endfor %} depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }} cms-job: @@ -38,6 +41,9 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "cms-job") %} + - {{ mount }} + {%- endfor %} depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("elasticsearch", RUN_ELASTICSEARCH), ("redis", RUN_REDIS)]|list_if }} {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index b30afb18d33..2dc6b72cfcb 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -24,7 +24,7 @@ services: ############# External services - {% if RUN_MONGODB %} + {% if RUN_MONGODB -%} mongodb: image: {{ DOCKER_IMAGE_MONGODB }} # Use WiredTiger in all environments, just like at edx.org @@ -35,9 +35,9 @@ services: - ../../data/mongodb:/data/db depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_MYSQL %} + {% if RUN_MYSQL -%} mysql: image: {{ DOCKER_IMAGE_MYSQL }} command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci @@ -47,9 +47,9 @@ services: - ../../data/mysql:/var/lib/mysql environment: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" - {% endif %} + {%- endif %} - {% if RUN_ELASTICSEARCH %} + {% if RUN_ELASTICSEARCH -%} elasticsearch: image: {{ DOCKER_IMAGE_ELASTICSEARCH }} environment: @@ -67,9 +67,9 @@ services: - ../../data/elasticsearch:/usr/share/elasticsearch/data depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_REDIS %} + {% if RUN_REDIS -%} redis: image: {{ DOCKER_IMAGE_REDIS }} working_dir: /openedx/redis/data @@ -81,16 +81,16 @@ services: restart: unless-stopped depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_SMTP %} + {% if RUN_SMTP -%} smtp: image: {{ DOCKER_IMAGE_SMTP }} restart: unless-stopped user: "100:101" environment: HOSTNAME: "{{ LMS_HOST }}" - {% endif %} + {%- endif %} ############# LMS and CMS @@ -108,6 +108,9 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/edx-platform/uwsgi.ini:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "lms") %} + - {{ mount }} + {%- endfor %} depends_on: - permissions {% if RUN_MYSQL %}- mysql{% endif %} @@ -131,6 +134,9 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/edx-platform/uwsgi.ini:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "cms") %} + - {{ mount }} + {%- endfor %} depends_on: - permissions - lms @@ -156,6 +162,9 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "lms-worker") %} + - {{ mount }} + {%- endfor %} depends_on: - lms @@ -172,6 +181,9 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "cms-worker") %} + - {{ mount }} + {%- endfor %} depends_on: - cms From c184a918485ab40d349efdad485f60860023f7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 12:02:04 +0200 Subject: [PATCH 18/96] docs: improve all available values from `ENV_TEMPLATE_VARIABLES` --- tutor/hooks/catalog.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index a21fd515ac0..8f540078544 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -367,6 +367,16 @@ def your_filter_callback(some_data): #: List of extra variables to be included in all templates. #: + #: Out of the box, this filter will include all configuration settings, but also the following: + #: + #: - `HOST_USER_ID`: the numerical ID of the user on the host. + #: - `TUTOR_APP`: the app name ("tutor" by default), used to determine the dev/local project names. + #: - `TUTOR_VERSION`: the current version of Tutor. + #: - `is_buildkit_enabled`: a boolean function that indicates whether BuildKit is available on the host. + #: - `iter_values_named`: a function to iterate on variables that start or end with a given string. + #: - `iter_mounts`: a function that yields compose-compatible bind-mounts for any given service. + #: - `patch`: a function to incorporate extra content into a template. + #: #: :parameter filters: list of (name, value) tuples. ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = filters.get( "env:templates:variables" From ae14fa592e44261e73c42daefc1f3c6466f32db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 17:11:14 +0200 Subject: [PATCH 19/96] depr: templated hooks Templated hooks we almost completely useless, so we get rid of them. This allows us to get rid entirely of hook names and hook indexes, which makes the whole implementation much simpler. Hook removal (with `clear_all`) is achieved thanks to weak references. --- changelog.d/20230412_100608_regis_palm.md | 6 + docs/reference/api/hooks/actions.rst | 3 - docs/reference/api/hooks/filters.rst | 3 - tests/commands/test_config.py | 2 +- tests/core/hooks/test_actions.py | 22 ++-- tests/core/hooks/test_filters.py | 44 +++---- tests/test_plugins_v0.py | 2 +- tutor/bindmount.py | 2 +- tutor/commands/compose.py | 3 +- tutor/commands/jobs.py | 8 +- tutor/config.py | 2 +- tutor/core/hooks/__init__.py | 12 +- tutor/core/hooks/actions.py | 135 +++---------------- tutor/core/hooks/contexts.py | 16 --- tutor/core/hooks/filters.py | 152 +++------------------- tutor/env.py | 2 +- tutor/hooks/catalog.py | 127 +++++++----------- tutor/plugins/__init__.py | 17 ++- tutor/plugins/v0.py | 5 +- tutor/plugins/v1.py | 30 ++--- tutor/utils.py | 1 + 21 files changed, 175 insertions(+), 419 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 13a2011573b..1641eebff2f 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -3,3 +3,9 @@ - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. - 💥[Deprecation] Drop support for `docker-compose`, also known as Compose V1. The `docker compose` (no hyphen) plugin must be installed. + - 💥[Refactor] We simplify the hooks API by getting rid of the `ContextTemplate`, `FilterTemplate` and `ActionTemplate` classes. As a consequences, the following changes occur: + - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. + - Removed the `ENV_PATCH` filter, which was for internal use only anyway. + - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) + - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence: + - The syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. diff --git a/docs/reference/api/hooks/actions.rst b/docs/reference/api/hooks/actions.rst index b9f94bb870b..bf73ddbd4d4 100644 --- a/docs/reference/api/hooks/actions.rst +++ b/docs/reference/api/hooks/actions.rst @@ -9,9 +9,6 @@ Actions are one of the two types of hooks (the other being :ref:`filters`) that .. autoclass:: tutor.core.hooks.Action :members: -.. autoclass:: tutor.core.hooks.ActionTemplate - :members: - .. The following are only to ensure that the docs build without warnings .. class:: tutor.core.hooks.actions.T .. class:: tutor.types.Config diff --git a/docs/reference/api/hooks/filters.rst b/docs/reference/api/hooks/filters.rst index f14a582c847..81ec4433740 100644 --- a/docs/reference/api/hooks/filters.rst +++ b/docs/reference/api/hooks/filters.rst @@ -9,9 +9,6 @@ Filters are one of the two types of hooks (the other being :ref:`actions`) that .. autoclass:: tutor.core.hooks.Filter :members: -.. autoclass:: tutor.core.hooks.FilterTemplate - :members: - .. The following are only to ensure that the docs build without warnings .. class:: tutor.core.hooks.filters.T1 .. class:: tutor.core.hooks.filters.T2 diff --git a/tests/commands/test_config.py b/tests/commands/test_config.py index f3448978371..5c9dcea307a 100644 --- a/tests/commands/test_config.py +++ b/tests/commands/test_config.py @@ -1,7 +1,7 @@ import unittest -from tutor import config as tutor_config from tests.helpers import temporary_root +from tutor import config as tutor_config from .base import TestCommandMixin diff --git a/tests/core/hooks/test_actions.py b/tests/core/hooks/test_actions.py index 60654a6a58e..e0c884b9a3e 100644 --- a/tests/core/hooks/test_actions.py +++ b/tests/core/hooks/test_actions.py @@ -8,16 +8,12 @@ class PluginActionsTests(unittest.TestCase): def setUp(self) -> None: self.side_effect_int = 0 - def tearDown(self) -> None: - super().tearDown() - actions.clear_all(context="tests") - def run(self, result: t.Any = None) -> t.Any: with contexts.enter("tests"): return super().run(result=result) def test_do(self) -> None: - action: actions.Action[int] = actions.get("test-action") + action: actions.Action[int] = actions.Action() @action.add() def _test_action_1(increment: int) -> None: @@ -31,29 +27,33 @@ def _test_action_2(increment: int) -> None: self.assertEqual(3, self.side_effect_int) def test_priority(self) -> None: - @actions.add("test-action", priority=2) + action: actions.Action[[]] = actions.Action() + + @action.add(priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @actions.add("test-action", priority=1) + @action.add(priority=1) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed before action 1 self.side_effect_int = 4 - actions.do("test-action") + action.do() self.assertEqual(6, self.side_effect_int) def test_equal_priority(self) -> None: - @actions.add("test-action", priority=2) + action: actions.Action[[]] = actions.Action() + + @action.add(priority=2) def _test_action_1() -> None: self.side_effect_int += 4 - @actions.add("test-action", priority=2) + @action.add(priority=2) def _test_action_2() -> None: self.side_effect_int = self.side_effect_int // 2 # Action 2 must be performed after action 1 self.side_effect_int = 4 - actions.do("test-action") + action.do() self.assertEqual(4, self.side_effect_int) diff --git a/tests/core/hooks/test_filters.py b/tests/core/hooks/test_filters.py index 4d7b8217a56..3443e9158dd 100644 --- a/tests/core/hooks/test_filters.py +++ b/tests/core/hooks/test_filters.py @@ -7,32 +7,32 @@ class PluginFiltersTests(unittest.TestCase): - def tearDown(self) -> None: - super().tearDown() - filters.clear_all(context="tests") - def run(self, result: t.Any = None) -> t.Any: with contexts.enter("tests"): return super().run(result=result) def test_add(self) -> None: - @filters.add("tests:count-sheeps") + filtre: filters.Filter[int, []] = filters.Filter() + + @filtre.add() def filter1(value: int) -> int: return value + 1 - value = filters.apply("tests:count-sheeps", 0) + value = filtre.apply(0) self.assertEqual(1, value) def test_add_items(self) -> None: - @filters.add("tests:add-sheeps") + filtre: filters.Filter[list[int], []] = filters.Filter() + + @filtre.add() def filter1(sheeps: list[int]) -> list[int]: return sheeps + [0] - filters.add_item("tests:add-sheeps", 1) - filters.add_item("tests:add-sheeps", 2) - filters.add_items("tests:add-sheeps", [3, 4]) + filtre.add_item(1) + filtre.add_item(2) + filtre.add_items([3, 4]) - sheeps: list[int] = filters.apply("tests:add-sheeps", []) + sheeps: list[int] = filtre.apply([]) self.assertEqual([0, 1, 2, 3, 4], sheeps) def test_filter_callbacks(self) -> None: @@ -42,20 +42,20 @@ def test_filter_callbacks(self) -> None: self.assertEqual(1, callback.apply(0)) def test_filter_context(self) -> None: + filtre: filters.Filter[list[int], []] = filters.Filter() with contexts.enter("testcontext"): - filters.add_item("test:sheeps", 1) - filters.add_item("test:sheeps", 2) + filtre.add_item(1) + filtre.add_item(2) - self.assertEqual([1, 2], filters.apply("test:sheeps", [])) - self.assertEqual( - [1], filters.apply_from_context("testcontext", "test:sheeps", []) - ) + self.assertEqual([1, 2], filtre.apply([])) + self.assertEqual([1], filtre.apply_from_context("testcontext", [])) def test_clear_context(self) -> None: + filtre: filters.Filter[list[int], []] = filters.Filter() with contexts.enter("testcontext"): - filters.add_item("test:sheeps", 1) - filters.add_item("test:sheeps", 2) + filtre.add_item(1) + filtre.add_item(2) - self.assertEqual([1, 2], filters.apply("test:sheeps", [])) - filters.clear("test:sheeps", context="testcontext") - self.assertEqual([2], filters.apply("test:sheeps", [])) + self.assertEqual([1, 2], filtre.apply([])) + filtre.clear(context="testcontext") + self.assertEqual([2], filtre.apply([])) diff --git a/tests/test_plugins_v0.py b/tests/test_plugins_v0.py index d3430874ff4..248cf800254 100644 --- a/tests/test_plugins_v0.py +++ b/tests/test_plugins_v0.py @@ -92,7 +92,7 @@ def test_patches(self) -> None: def test_plugin_without_patches(self) -> None: plugins_v0.DictPlugin({"name": "plugin1"}) - plugins.load("plugin1") + plugins.load_all(["plugin1"]) patches = list(plugins.iter_patches("patch1")) self.assertEqual([], patches) diff --git a/tutor/bindmount.py b/tutor/bindmount.py index 0f2d2069b49..fab7dae22c3 100644 --- a/tutor/bindmount.py +++ b/tutor/bindmount.py @@ -1,9 +1,9 @@ from __future__ import annotations -from functools import lru_cache import os import re import typing as t +from functools import lru_cache from tutor import hooks diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 53112d0eb7c..c7a4f6d6e57 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -4,9 +4,10 @@ import click +from tutor import bindmount from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import bindmount, hooks, utils +from tutor import hooks, utils from tutor.commands import jobs from tutor.commands.context import BaseTaskContext from tutor.core.hooks import Filter # pylint: disable=unused-import diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index a6648c7c634..0412fc1478b 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -37,11 +37,11 @@ def _add_core_init_tasks() -> None: The context is important, because it allows us to select the init scripts based on the --limit argument. """ - with hooks.Contexts.APP("mysql").enter(): + with hooks.Contexts.app("mysql").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) - with hooks.Contexts.APP("lms").enter(): + with hooks.Contexts.app("lms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ( "lms", @@ -54,7 +54,7 @@ def _add_core_init_tasks() -> None: hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("lms", env.read_core_template_file("jobs", "init", "lms.sh")) ) - with hooks.Contexts.APP("cms").enter(): + with hooks.Contexts.app("cms").enter(): hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("cms", env.read_core_template_file("jobs", "init", "cms.sh")) ) @@ -64,7 +64,7 @@ def _add_core_init_tasks() -> None: @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") def initialise(limit: t.Optional[str]) -> t.Iterator[tuple[str, str]]: fmt.echo_info("Initialising all services...") - filter_context = hooks.Contexts.APP(limit).name if limit else None + filter_context = hooks.Contexts.app(limit).name if limit else None # Deprecated pre-init tasks for service, path in hooks.Filters.COMMANDS_PRE_INIT.iterate_from_context( diff --git a/tutor/config.py b/tutor/config.py index 3459a825b6c..a5e1ecab15a 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -304,7 +304,7 @@ def _remove_plugin_config_overrides_on_unload( # Find the configuration entries that were overridden by the plugin and # remove them from the current config for key, _value in hooks.Filters.CONFIG_OVERRIDES.iterate_from_context( - hooks.Contexts.APP(plugin).name + hooks.Contexts.app(plugin).name ): value = config.pop(key, None) value = env.render_unknown(config, value) diff --git a/tutor/core/hooks/__init__.py b/tutor/core/hooks/__init__.py index 1609444b51a..fa1451a5bbc 100644 --- a/tutor/core/hooks/__init__.py +++ b/tutor/core/hooks/__init__.py @@ -1,15 +1,13 @@ import typing as t -from .actions import Action, ActionTemplate -from .actions import clear_all as _clear_all_actions -from .contexts import Context, ContextTemplate -from .filters import Filter, FilterTemplate -from .filters import clear_all as _clear_all_filters +from .actions import Action +from .contexts import Context +from .filters import Filter def clear_all(context: t.Optional[str] = None) -> None: """ Clear both actions and filters. """ - _clear_all_actions(context=context) - _clear_all_filters(context=context) + Action.clear_all(context=context) + Filter.clear_all(context=context) diff --git a/tutor/core/hooks/actions.py b/tutor/core/hooks/actions.py index e6050d9fdc5..924c5ae6245 100644 --- a/tutor/core/hooks/actions.py +++ b/tutor/core/hooks/actions.py @@ -5,9 +5,11 @@ import sys import typing as t +from weakref import WeakSet from typing_extensions import ParamSpec + from . import priorities from .contexts import Contextualized @@ -44,32 +46,25 @@ class Action(t.Generic[T]): This is the typical action lifecycle: - 1. Create an action with method :py:meth:`get`. - 2. Add callbacks with method :py:meth:`add`. - 3. Call the action callbacks with method :py:meth:`do`. + 1. Create an action with ``Action()``. + 2. Add callbacks with :py:meth:`add`. + 3. Call the action callbacks with :py:meth:`do`. - The ``P`` type parameter of the Action class corresponds to the expected signature of + The ``T`` type parameter of the Action class corresponds to the expected signature of the action callbacks. For instance, ``Action[[str, int]]`` means that the action callbacks are expected to take two arguments: one string and one integer. - This strong typing makes it easier for plugin developers to quickly check whether they are adding and calling action callbacks correctly. + This strong typing makes it easier for plugin developers to quickly check whether + they are adding and calling action callbacks correctly. """ - INDEX: dict[str, "Action[t.Any]"] = {} + # Keep a weak reference to all created filters. This allows us to clear them when + # necessary. + INSTANCES: WeakSet[Action[t.Any]] = WeakSet() - def __init__(self, name: str) -> None: - self.name = name + def __init__(self) -> None: self.callbacks: list[ActionCallback[T]] = [] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.name}')" - - @classmethod - def get(cls, name: str) -> "Action[t.Any]": - """ - Get an existing action with the given name from the index, or create one. - """ - return cls.INDEX.setdefault(name, cls(name)) + self.INSTANCES.add(self) def add( self, priority: t.Optional[int] = None @@ -144,7 +139,7 @@ def do_from_context( ) except: sys.stderr.write( - f"Error applying action '{self.name}': func={callback.func} contexts={callback.contexts}'\n" + f"Error applying action: func={callback.func} contexts={callback.contexts}'\n" ) raise @@ -168,98 +163,10 @@ def clear(self, context: t.Optional[str] = None) -> None: if not callback.is_in_context(context) ] - -class ActionTemplate(t.Generic[T]): - """ - Action templates are for actions for which the name needs to be formatted - before the action can be applied. - - Action templates can generate different :py:class:`Action` objects for which the - name matches a certain template. - - Templated actions must be formatted with ``(*args)`` before being applied. For example:: - - action_template = ActionTemplate("namespace:{0}") - - # Return the action named "namespace:name" - my_action = action_template("name") - - @my_action.add() - def my_callback(): - ... - - my_action.do() - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Action[T]: - name = self.template.format(*args, **kwargs) - action: Action[T] = Action.get(name) - return action - - -# Syntactic sugar -get = Action.get - - -def get_template(name: str) -> ActionTemplate[t.Any]: - """ - Create an action with a template name. - """ - return ActionTemplate(name) - - -def add( - name: str, priority: t.Optional[int] = None -) -> t.Callable[[ActionCallbackFunc[T]], ActionCallbackFunc[T]]: - """ - Decorator to add a callback action associated to a name. - """ - return get(name).add(priority=priority) - - -def do( - name: str, - *args: T.args, - **kwargs: T.kwargs, -) -> None: - """ - Run action callbacks associated to a name/context. - """ - action: Action[T] = Action.get(name) - action.do(*args, **kwargs) - - -def do_from_context( - context: str, - name: str, - *args: T.args, - **kwargs: T.kwargs, -) -> None: - """ - Same as :py:func:`do` but only run the callbacks that were created in a given context. - """ - action: Action[T] = Action.get(name) - action.do_from_context(context, *args, **kwargs) - - -def clear_all(context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given context. - - This will call :py:func:`clear` with all action names. - """ - for name in Action.INDEX: - clear(name, context=context) - - -def clear(name: str, context: t.Optional[str] = None) -> None: - """ - Clear any previously defined action with the given name and context. - """ - Action.get(name).clear(context=context) + @classmethod + def clear_all(cls, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined action with the given context. + """ + for action in cls.INSTANCES: + action.clear(context) diff --git a/tutor/core/hooks/contexts.py b/tutor/core/hooks/contexts.py index 1dc48a82ba9..940d2bac6c4 100644 --- a/tutor/core/hooks/contexts.py +++ b/tutor/core/hooks/contexts.py @@ -43,22 +43,6 @@ def enter(self) -> t.Iterator[None]: Context.CURRENT.pop() -class ContextTemplate: - """ - Context templates are for filters for which the name needs to be formatted - before the filter can be applied. - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Context: - return Context(self.template.format(*args, **kwargs)) - - class Contextualized: """ This is a simple class to store the current context in hooks. diff --git a/tutor/core/hooks/filters.py b/tutor/core/hooks/filters.py index 9e9b70e81d7..ef42325dbd7 100644 --- a/tutor/core/hooks/filters.py +++ b/tutor/core/hooks/filters.py @@ -5,6 +5,7 @@ import sys import typing as t +from weakref import WeakSet from typing_extensions import Concatenate, ParamSpec @@ -43,16 +44,16 @@ class Filter(t.Generic[T1, T2]): This is the typical filter lifecycle: - 1. Create an action with method :py:meth:`get`. - 2. Add callbacks with method :py:meth:`add`. + 1. Create a filter with ``Filter()``. + 2. Add callbacks with :py:meth:`add`. 3. Call the filter callbacks with method :py:meth:`apply`. The result of each callback is passed as the first argument to the next one. Thus, the type of the first argument must match the callback return type. - The `T` and `P` type parameters of the Filter class correspond to the expected - signature of the filter callbacks. `T` is the type of the first argument (and thus - the return value type as well) and `P` is the signature of the other arguments. + The ``T1`` and ``T2`` type parameters of the Filter class correspond to the expected + signature of the filter callbacks. ``T1`` is the type of the first argument (and thus + the return value type as well) and ``T2`` is the signature of the other arguments. For instance, `Filter[str, [int]]` means that the filter callbacks are expected to take two arguments: one string and one integer. Each callback must then return a @@ -62,21 +63,13 @@ class Filter(t.Generic[T1, T2]): they are adding and calling filter callbacks correctly. """ - INDEX: dict[str, "Filter[t.Any, t.Any]"] = {} + # Keep a weak reference to all created filters. This allows us to clear them when + # necessary. + INSTANCES: WeakSet[Filter[t.Any, t.Any]] = WeakSet() - def __init__(self, name: str) -> None: - self.name = name + def __init__(self) -> None: self.callbacks: list[FilterCallback[T1, T2]] = [] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.name}')" - - @classmethod - def get(cls, name: str) -> "Filter[t.Any, t.Any]": - """ - Get an existing action with the given name from the index, or create one. - """ - return cls.INDEX.setdefault(name, cls(name)) + self.INSTANCES.add(self) def add( self, priority: t.Optional[int] = None @@ -156,7 +149,7 @@ def apply_from_context( ) except: sys.stderr.write( - f"Error applying filter '{self.name}': func={callback.func} contexts={callback.contexts}'\n" + f"Error applying filter: func={callback.func} contexts={callback.contexts}'\n" ) raise return value @@ -171,6 +164,14 @@ def clear(self, context: t.Optional[str] = None) -> None: if not callback.is_in_context(context) ] + @classmethod + def clear_all(cls, context: t.Optional[str] = None) -> None: + """ + Clear any previously defined filter with the given context. + """ + for filtre in cls.INSTANCES: + filtre.clear(context) + # The methods below are specific to filters which take lists as first arguments def add_item( self: "Filter[list[L], T2]", item: L, priority: t.Optional[int] = None @@ -205,8 +206,8 @@ def add_items( ``add_item`` multiple times on the same filter, then you probably want to use a single call to ``add_items`` instead. - :param name: filter name. :param list[object] items: items that will be appended to the resulting list. + :param int priority: optional priority. Usage:: @@ -261,114 +262,3 @@ def iterate_from_context( Same as :py:func:`Filter.iterate` but apply only callbacks from a given context. """ yield from self.apply_from_context(context, [], *args, **kwargs) - - -class FilterTemplate(t.Generic[T1, T2]): - """ - Filter templates are for filters for which the name needs to be formatted - before the filter can be applied. - - Similar to :py:class:`tutor.core.hooks.ActionTemplate`, filter templates are used to generate - :py:class:`Filter` objects for which the name matches a certain template. - - Templated filters must be formatted with ``(*args)`` before being applied. For example:: - - filter_template = FilterTemplate("namespace:{0}") - named_filter = filter_template("name") - - @named_filter.add() - def my_callback(x: int) -> int: - ... - - named_filter.apply(42) - """ - - def __init__(self, name: str): - self.template = name - - def __repr__(self) -> str: - return f"{self.__class__.__name__}('{self.template}')" - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> Filter[T1, T2]: - return get(self.template.format(*args, **kwargs)) - - -# Syntactic sugar -get = Filter.get - - -def get_template(name: str) -> FilterTemplate[t.Any, t.Any]: - """ - Create a filter with a template name. - """ - return FilterTemplate(name) - - -def add( - name: str, priority: t.Optional[int] = None -) -> t.Callable[[FilterCallbackFunc[T1, T2]], FilterCallbackFunc[T1, T2]]: - """ - Decorator for functions that will be applied to a single named filter. - """ - return Filter.get(name).add(priority=priority) - - -def add_item(name: str, item: T1, priority: t.Optional[int] = None) -> None: - """ - Convenience function to add a single item to a filter that returns a list of items. - """ - get(name).add_item(item, priority=priority) - - -def add_items(name: str, items: list[T1], priority: t.Optional[int] = None) -> None: - """ - Convenience decorator to add multiple item to a filter that returns a list of items. - """ - get(name).add_items(items, priority=priority) - - -def iterate(name: str, *args: t.Any, **kwargs: t.Any) -> t.Iterator[T1]: - """ - Convenient function to iterate over the results of a filter result list. - """ - yield from iterate_from_context(None, name, *args, **kwargs) - - -def iterate_from_context( - context: t.Optional[str], name: str, *args: t.Any, **kwargs: t.Any -) -> t.Iterator[T1]: - yield from Filter.get(name).iterate_from_context(context, *args, **kwargs) - - -def apply(name: str, value: T1, *args: t.Any, **kwargs: t.Any) -> T1: - """ - Apply all declared filters to a single value, passing along the additional arguments. - """ - return apply_from_context(None, name, value, *args, **kwargs) - - -def apply_from_context( - context: t.Optional[str], name: str, value: T1, *args: T2.args, **kwargs: T2.kwargs -) -> T1: - """ - Same as :py:func:`apply` but only run the callbacks that were created in a given context. - """ - filtre: Filter[T1, T2] = Filter.get(name) - return filtre.apply_from_context(context, value, *args, **kwargs) - - -def clear_all(context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given context. - """ - for name in Filter.INDEX: - clear(name, context=context) - - -def clear(name: str, context: t.Optional[str] = None) -> None: - """ - Clear any previously defined filter with the given name and context. - """ - filtre = Filter.INDEX.get(name) - if filtre: - filtre.clear(context=context) diff --git a/tutor/env.py b/tutor/env.py index 2460ec8a70e..980bb773490 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -526,7 +526,7 @@ def _delete_plugin_templates(plugin: str, root: str, _config: Config) -> None: Delete plugin env files on unload. """ targets = hooks.Filters.ENV_TEMPLATE_TARGETS.iterate_from_context( - hooks.Contexts.APP(plugin).name + hooks.Contexts.app(plugin).name ) for src, dst in targets: path = pathjoin(root, dst.replace("/", os.sep), src.replace("/", os.sep)) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 8f540078544..23cb78dd365 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -11,16 +11,7 @@ import click -from tutor.core.hooks import ( - Action, - ActionTemplate, - Context, - ContextTemplate, - Filter, - FilterTemplate, - actions, - filters, -) +from tutor.core.hooks import Action, Context, Filter from tutor.types import Config __all__ = ["Actions", "Filters", "Contexts"] @@ -64,9 +55,7 @@ def run_this_on_start(root, config, name): #: :parameter str root: project root. #: :parameter dict config: project configuration. #: :parameter str name: docker-compose project name. - COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = actions.get( - "compose:project:started" - ) + COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = Action() #: Called whenever the core project is ready to run. This action is called as soon #: as possible. This is the right time to discover plugins, for instance. In @@ -81,14 +70,14 @@ def run_this_on_start(root, config, name): #: developers probably don't have to implement this action themselves. #: #: This action does not have any parameter. - CORE_READY: Action[[]] = actions.get("core:ready") + CORE_READY: Action[[]] = Action() #: Called just before triggering the job tasks of any ``... do `` command. #: #: :parameter str job: job name. #: :parameter args: job positional arguments. #: :parameter kwargs: job named arguments. - DO_JOB: Action[[str, Any]] = actions.get("do:job") + DO_JOB: Action[[str, Any]] = Action() #: Triggered when a single plugin needs to be loaded. Only plugins that have previously been #: discovered can be loaded (see :py:data:`CORE_READY`). @@ -100,14 +89,14 @@ def run_this_on_start(root, config, name): #: Most plugin developers will not have to implement this action themselves, unless #: they want to perform a specific action at the moment the plugin is enabled. #: - #: This action does not have any parameter. - PLUGIN_LOADED: ActionTemplate[[]] = actions.get_template("plugins:loaded:{0}") + #: :parameter str plugin: plugin name. + PLUGIN_LOADED: Action[[str]] = Action() #: Triggered after all plugins have been loaded. At this point the list of loaded #: plugins may be obtained from the :py:data:`Filters.PLUGINS_LOADED` filter. #: #: This action does not have any parameter. - PLUGINS_LOADED: Action[[]] = actions.get("plugins:loaded") + PLUGINS_LOADED: Action[[]] = Action() #: Triggered when a single plugin is unloaded. Only plugins that have previously been #: loaded can be unloaded (see :py:data:`PLUGIN_LOADED`). @@ -120,12 +109,12 @@ def run_this_on_start(root, config, name): #: :parameter str plugin: plugin name. #: :parameter str root: absolute path to the project root. #: :parameter config: full project configuration - PLUGIN_UNLOADED: Action[str, str, Config] = actions.get("plugins:unloaded") + PLUGIN_UNLOADED: Action[str, str, Config] = Action() #: Called as soon as we have access to the Tutor project root. #: #: :parameter str root: absolute path to the project root. - PROJECT_ROOT_READY: Action[str] = actions.get("project:root:ready") + PROJECT_ROOT_READY: Action[str] = Action() class Filters: @@ -178,7 +167,7 @@ def your_filter_callback(some_data): #: #: :parameter list commands: commands are instances of ``click.Command``. They will #: all be added as subcommands of the main ``tutor`` command. - CLI_COMMANDS: Filter[list[click.Command], []] = filters.get("cli:commands") + CLI_COMMANDS: Filter[list[click.Command], []] = Filter() #: List of `do ...` commands. #: @@ -188,7 +177,7 @@ def your_filter_callback(some_data): #: in the "service" container, both in local, dev and k8s mode. CLI_DO_COMMANDS: Filter[ list[Callable[[Any], Iterable[tuple[str, str]]]], [] - ] = filters.get("cli:commands:do") + ] = Filter() #: List of initialization tasks (scripts) to be run in the `init` job. This job #: includes all database migrations, setting up, etc. To run some tasks before or @@ -197,9 +186,7 @@ def your_filter_callback(some_data): #: :parameter list[tuple[str, str]] tasks: list of ``(service, task)`` tuples. Each #: task is essentially a bash script to be run in the "service" container. Scripts #: may contain Jinja markup, similar to templates. - CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = filters.get( - "cli:commands:do:init" - ) + CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = Filter() #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead. #: @@ -212,9 +199,7 @@ def your_filter_callback(some_data): #: - ``path`` is a tuple that corresponds to a template relative path. #: Example: ``("myplugin", "hooks", "myservice", "pre-init")`` (see :py:data:`IMAGES_BUILD`). #: The command to execute will be read from that template, after it is rendered. - COMMANDS_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = filters.get( - "commands:init" - ) + COMMANDS_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead with a lower priority score. #: @@ -223,9 +208,7 @@ def your_filter_callback(some_data): #: #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` #: tasks. (see :py:data:`COMMANDS_INIT`). - COMMANDS_PRE_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = filters.get( - "commands:pre-init" - ) + COMMANDS_PRE_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() #: List of folders to bind-mount in docker-compose containers, either in ``tutor local`` or ``tutor dev``. #: @@ -250,7 +233,7 @@ def your_filter_callback(some_data): #: :parameter str name: basename of the host-mounted folder. In the example above, #: this is "edx-platform". When implementing this filter you should check this name to #: conditionally add mounts. - COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get("compose:mounts") + COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() #: Declare new default configuration settings that don't necessarily have to be saved in the user #: ``config.yml`` file. Default settings may be overridden with ``tutor config save --set=...``, in which @@ -258,44 +241,34 @@ def your_filter_callback(some_data): #: #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All #: new entries must be prefixed with the plugin name in all-caps. - CONFIG_DEFAULTS: Filter[list[tuple[str, Any]], []] = filters.get("config:defaults") + CONFIG_DEFAULTS: Filter[list[tuple[str, Any]], []] = Filter() #: Modify existing settings, either from Tutor core or from other plugins. Beware not to override any #: important setting, such as passwords! Overridden setting values will be printed to stdout when the plugin #: is disabled, such that users have a chance to back them up. #: #: :parameter list[tuple[str, ...]] items: list of (name, value) settings. - CONFIG_OVERRIDES: Filter[list[tuple[str, Any]], []] = filters.get( - "config:overrides" - ) + CONFIG_OVERRIDES: Filter[list[tuple[str, Any]], []] = Filter() #: Declare unique configuration settings that must be saved in the user ``config.yml`` file. This is where #: you should declare passwords and randomly-generated values that are different from one environment to the next. #: #: :parameter list[tuple[str, ...]] items: list of (name, value) new settings. All #: names must be prefixed with the plugin name in all-caps. - CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = filters.get("config:unique") + CONFIG_UNIQUE: Filter[list[tuple[str, Any]], []] = Filter() #: Use this filter to modify the ``docker build`` command. For instance, to replace #: the ``build`` subcommand by ``buildx build``. #: #: :parameter list[str] command: the full build command, including options and #: arguments. Note that these arguments do not include the leading ``docker`` command. - DOCKER_BUILD_COMMAND: Filter[list[str], []] = filters.get("docker:build:command") + DOCKER_BUILD_COMMAND: Filter[list[str], []] = Filter() - #: List of patches that should be inserted in a given location of the templates. The - #: filter name must be formatted with the patch name. - #: This filter is not so convenient and plugin developers will probably - #: prefer :py:data:`ENV_PATCHES`. - #: - #: :parameter list[str] patches: each item is the unrendered patch content. - ENV_PATCH: FilterTemplate[list[str], []] = filters.get_template("env:patches:{0}") - - #: List of patches that should be inserted in a given location of the templates. This is very similar to :py:data:`ENV_PATCH`, except that the patch is added as a ``(name, content)`` tuple. + #: List of patches that should be inserted in a given location of the templates. #: #: :parameter list[tuple[str, str]] patches: pairs of (name, content) tuples. Use this #: filter to modify the Tutor templates. - ENV_PATCHES: Filter[list[tuple[str, str]], []] = filters.get("env:patches") + ENV_PATCHES: Filter[list[tuple[str, str]], []] = Filter() #: List of template path patterns to be ignored when rendering templates to the project root. By default, we ignore: #: @@ -306,13 +279,13 @@ def your_filter_callback(some_data): #: Ignored patterns are overridden by include patterns; see :py:data:`ENV_PATTERNS_INCLUDE`. #: #: :parameter list[str] patterns: list of regular expression patterns. E.g: ``r"(.*/)?ignored_file_name(/.*)?"``. - ENV_PATTERNS_IGNORE: Filter[list[str], []] = filters.get("env:patterns:ignore") + ENV_PATTERNS_IGNORE: Filter[list[str], []] = Filter() #: List of template path patterns to be included when rendering templates to the project root. #: Patterns from this list will take priority over the patterns from :py:data:`ENV_PATTERNS_IGNORE`. #: #: :parameter list[str] patterns: list of regular expression patterns. See :py:data:`ENV_PATTERNS_IGNORE`. - ENV_PATTERNS_INCLUDE: Filter[list[str], []] = filters.get("env:patterns:include") + ENV_PATTERNS_INCLUDE: Filter[list[str], []] = Filter() #: List of `Jinja2 filters `__ that will be #: available in templates. Jinja2 filters are basically functions that can be used @@ -343,16 +316,14 @@ def your_filter_callback(some_data): #: #: :parameter filters: list of (name, function) tuples. The function signature #: should correspond to its usage in templates. - ENV_TEMPLATE_FILTERS: Filter[ - list[tuple[str, Callable[..., Any]]], [] - ] = filters.get("env:templates:filters") + ENV_TEMPLATE_FILTERS: Filter[list[tuple[str, Callable[..., Any]]], []] = Filter() #: List of all template root folders. #: #: :parameter list[str] templates_root: absolute paths to folders which contain templates. #: The templates in these folders will then be accessible by the environment #: renderer using paths that are relative to their template root. - ENV_TEMPLATE_ROOTS: Filter[list[str], []] = filters.get("env:templates:roots") + ENV_TEMPLATE_ROOTS: Filter[list[str], []] = Filter() #: List of template source/destination targets. #: @@ -361,9 +332,7 @@ def your_filter_callback(some_data): #: is a path relative to the environment root. For instance: adding ``("c/d", #: "a/b")`` to the filter will cause all files from "c/d" to be rendered to the ``a/b/c/d`` #: subfolder. - ENV_TEMPLATE_TARGETS: Filter[list[tuple[str, str]], []] = filters.get( - "env:templates:targets" - ) + ENV_TEMPLATE_TARGETS: Filter[list[tuple[str, str]], []] = Filter() #: List of extra variables to be included in all templates. #: @@ -378,9 +347,7 @@ def your_filter_callback(some_data): #: - `patch`: a function to incorporate extra content into a template. #: #: :parameter filters: list of (name, value) tuples. - ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = filters.get( - "env:templates:variables" - ) + ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = Filter() #: List of images to be built when we run ``tutor images build ...``. #: @@ -397,7 +364,7 @@ def your_filter_callback(some_data): #: :parameter Config config: user configuration. IMAGES_BUILD: Filter[ list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] - ] = filters.get("images:build") + ] = Filter() #: List of host directories to be automatically bind-mounted in Docker images at #: build time. For instance, this is useful to build Docker images using a custom @@ -415,9 +382,7 @@ def your_filter_callback(some_data): #: :py:data:`COMPOSE_MOUNTS`, this is not just the basename, but the full path. When #: implementing this filter you should check this path (for instance: with #: ``os.path.basename(path)``) to conditionally add mounts. - IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get( - "images:build:mounts" - ) + IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() #: List of images to be pulled when we run ``tutor images pull ...``. #: @@ -426,11 +391,11 @@ def your_filter_callback(some_data): #: - ``name`` is the name of the image, as in ``tutor images pull myimage``. #: - ``tag`` is the Docker tag that will be applied to the image. (see :py:data:`IMAGES_BUILD`). #: :parameter Config config: user configuration. - IMAGES_PULL: Filter[list[tuple[str, str]], [Config]] = filters.get("images:pull") + IMAGES_PULL: Filter[list[tuple[str, str]], [Config]] = Filter() #: List of images to be pushed when we run ``tutor images push ...``. #: Parameters are the same as for :py:data:`IMAGES_PULL`. - IMAGES_PUSH: Filter[list[tuple[str, str]], [Config]] = filters.get("images:push") + IMAGES_PUSH: Filter[list[tuple[str, str]], [Config]] = Filter() #: List of plugin indexes that are loaded when we run `tutor plugins update`. By #: default, the plugin indexes are stored in the user configuration. This filter makes @@ -438,13 +403,13 @@ def your_filter_callback(some_data): #: #: :parameter list[str] indexes: list of index URLs. Remember that entries further #: in the list have priority. - PLUGIN_INDEXES: Filter[list[str], []] = filters.get("plugins:indexes:entries") + PLUGIN_INDEXES: Filter[list[str], []] = Filter() #: Filter to modify the url of a plugin index url. This is convenient to alias #: plugin indexes with a simple name, such as "main" or "contrib". #: #: :parameter str url: value passed to the `index add/remove` commands. - PLUGIN_INDEX_URL: Filter[str, []] = filters.get("plugins:indexes:url") + PLUGIN_INDEX_URL: Filter[str, []] = Filter() #: When installing an entry from a plugin index, the plugin data from the index will #: go through this filter before it is passed along to `pip install`. Thus, this is a @@ -453,17 +418,13 @@ def your_filter_callback(some_data): #: #: :parameter dict[str, str] plugin: the dict entry from the plugin index. It #: includes an additional "index" key which contains the plugin index URL. - PLUGIN_INDEX_ENTRY_TO_INSTALL: Filter[dict[str, str], []] = filters.get( - "plugins:indexes:entries:install" - ) + PLUGIN_INDEX_ENTRY_TO_INSTALL: Filter[dict[str, str], []] = Filter() #: Information about each installed plugin, including its version. #: Keep this information to a single line for easier parsing by 3rd-party scripts. #: #: :param list[tuple[str, str]] versions: each pair is a ``(plugin, info)`` tuple. - PLUGINS_INFO: Filter[list[tuple[str, str]], []] = filters.get( - "plugins:installed:versions" - ) + PLUGINS_INFO: Filter[list[tuple[str, str]], []] = Filter() #: List of installed plugins. In order to be added to this list, a plugin must first #: be discovered (see :py:data:`Actions.CORE_READY`). @@ -471,13 +432,13 @@ def your_filter_callback(some_data): #: :param list[str] plugins: plugin developers probably don't have to implement this #: filter themselves, but they can apply it to check for the presence of other #: plugins. - PLUGINS_INSTALLED: Filter[list[str], []] = filters.get("plugins:installed") + PLUGINS_INSTALLED: Filter[list[str], []] = Filter() #: List of loaded plugins. #: #: :param list[str] plugins: plugin developers probably don't have to modify this #: filter themselves, but they can apply it to check whether other plugins are enabled. - PLUGINS_LOADED: Filter[list[str], []] = filters.get("plugins:loaded") + PLUGINS_LOADED: Filter[list[str], []] = Filter() class Contexts: @@ -497,10 +458,16 @@ class Contexts: hooks.Filters.MY_FILTER.apply_from_context(hooks.Contexts.SOME_CONTEXT.name) """ - #: We enter this context whenever we create hooks for a specific application or : - #: plugin. For instance, plugin "myplugin" will be enabled within the "app:myplugin" - #: context. - APP = ContextTemplate("app:{0}") + #: Dictionary of name/contexts. Each value is a context that we enter whenever we + #: create hooks for a specific application or : : plugin. For instance, plugin + #: "myplugin" will be enabled within the "app:myplugin" : context. + APP: dict[str, Context] = {} + + @classmethod + def app(cls, name: str) -> Context: + if name not in cls.APP: + cls.APP[name] = Context(f"app:{name}") + return cls.APP[name] #: Plugins will be installed and enabled within this context. PLUGINS = Context("plugins") diff --git a/tutor/plugins/__init__.py b/tutor/plugins/__init__.py index b8e55649f0d..c5e88d20b58 100644 --- a/tutor/plugins/__init__.py +++ b/tutor/plugins/__init__.py @@ -12,19 +12,24 @@ # Import modules to trigger hook creation from . import v0, v1 +# Cache of plugin patches, for efficiency +ENV_PATCHES_DICT: dict[str, list[str]] = {} + @hooks.Actions.PLUGINS_LOADED.add() def _convert_plugin_patches() -> None: """ Some patches are added as (name, content) tuples with the ENV_PATCHES - filter. We convert these patches to add them to ENV_PATCH. This makes it + filter. We convert these patches to add them to ENV_PATCHES_DICT. This makes it easier for end-user to declare patches, and it's more performant. This action is run after plugins have been loaded. """ + ENV_PATCHES_DICT.clear() patches: t.Iterable[tuple[str, str]] = hooks.Filters.ENV_PATCHES.iterate() for name, content in patches: - hooks.Filters.ENV_PATCH(name).add_item(content) + ENV_PATCHES_DICT.setdefault(name, []) + ENV_PATCHES_DICT[name].append(content) def is_installed(name: str) -> bool: @@ -89,8 +94,8 @@ def load(name: str) -> None: if not is_installed(name): raise exceptions.TutorError(f"plugin '{name}' is not installed.") with hooks.Contexts.PLUGINS.enter(): - with hooks.Contexts.APP(name).enter(): - hooks.Actions.PLUGIN_LOADED(name).do() + with hooks.Contexts.app(name).enter(): + hooks.Actions.PLUGIN_LOADED.do(name) hooks.Filters.PLUGINS_LOADED.add_item(name) @@ -109,14 +114,14 @@ def iter_patches(name: str) -> t.Iterator[str]: """ Yields: patch (str) """ - yield from hooks.Filters.ENV_PATCH(name).iterate() + yield from ENV_PATCHES_DICT.get(name, []) def unload(plugin: str) -> None: """ Remove all filters and actions associated to a given plugin. """ - hooks.clear_all(context=hooks.Contexts.APP(plugin).name) + hooks.clear_all(context=hooks.Contexts.app(plugin).name) @hooks.Actions.PLUGIN_UNLOADED.add(priority=50) diff --git a/tutor/plugins/v0.py b/tutor/plugins/v0.py index 74c40c0eb68..16e67704e0b 100644 --- a/tutor/plugins/v0.py +++ b/tutor/plugins/v0.py @@ -60,7 +60,10 @@ def _discover(self) -> None: hooks.Filters.PLUGINS_INFO.add_item((self.name, self._version() or "")) # Create actions and filters on load - hooks.Actions.PLUGIN_LOADED(self.name).add()(self.__load) + @hooks.Actions.PLUGIN_LOADED.add() + def _load_plugin(name: str) -> None: + if name == self.name: + self.__load() def __load(self) -> None: """ diff --git a/tutor/plugins/v1.py b/tutor/plugins/v1.py index ec0f9acd5a6..cf24bf0f001 100644 --- a/tutor/plugins/v1.py +++ b/tutor/plugins/v1.py @@ -43,16 +43,17 @@ def discover_module(path: str) -> None: hooks.Filters.PLUGINS_INFO.add_item((name, path)) # Import module on enable - load_plugin_action = hooks.Actions.PLUGIN_LOADED(name) - - @load_plugin_action.add() - def load() -> None: - # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly - spec = importlib.util.spec_from_file_location("tutor.plugin.v1.{name}", path) - if spec is None or spec.loader is None: - raise ValueError("Plugin could not be found: {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + @hooks.Actions.PLUGIN_LOADED.add() + def load(plugin_name: str) -> None: + if name == plugin_name: + # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly + spec = importlib.util.spec_from_file_location( + "tutor.plugin.v1.{name}", path + ) + if spec is None or spec.loader is None: + raise ValueError("Plugin could not be found: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) def discover_package(entrypoint: pkg_resources.EntryPoint) -> None: @@ -70,8 +71,7 @@ def discover_package(entrypoint: pkg_resources.EntryPoint) -> None: hooks.Filters.PLUGINS_INFO.add_item((name, entrypoint.dist.version)) # Import module on enable - load_plugin_action = hooks.Actions.PLUGIN_LOADED(name) - - @load_plugin_action.add() - def load() -> None: - entrypoint.load() + @hooks.Actions.PLUGIN_LOADED.add() + def load(plugin_name: str) -> None: + if name == plugin_name: + entrypoint.load() diff --git a/tutor/utils.py b/tutor/utils.py index 06c0e3d0b98..59adee43778 100644 --- a/tutor/utils.py +++ b/tutor/utils.py @@ -258,6 +258,7 @@ def warn_macos_docker_memory() -> None: https://docs.tutor.overhang.io/install.html""" ) + def check_macos_docker_memory() -> None: """ Try to check that the RAM allocated to the Docker VM on macOS is at least 4 GB. From cdf46cbe6a358494c6d9041ce74d3a0162e04b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 20:00:04 +0200 Subject: [PATCH 20/96] depr: remove obsolete task actions --- changelog.d/20230412_100608_regis_palm.md | 1 + tutor/commands/jobs.py | 19 ------------------- tutor/hooks/catalog.py | 22 ---------------------- 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 1641eebff2f..5d383821766 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -9,3 +9,4 @@ - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence: - The syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. + - 💥[Deprecation] The obsolete filters `COMMANDS_PRE_INIT` and `COMMANDS_INIT` have been removed. Plugin developers should instead use `CLI_DO_INIT_TASKS` (with suitable priorities). diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 0412fc1478b..a95bf075c19 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -66,31 +66,12 @@ def initialise(limit: t.Optional[str]) -> t.Iterator[tuple[str, str]]: fmt.echo_info("Initialising all services...") filter_context = hooks.Contexts.app(limit).name if limit else None - # Deprecated pre-init tasks - for service, path in hooks.Filters.COMMANDS_PRE_INIT.iterate_from_context( - filter_context - ): - fmt.echo_alert( - f"Running deprecated pre-init task: {'/'.join(path)}. Init tasks should no longer be added to the COMMANDS_PRE_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead, with a high priority." - ) - yield service, env.read_template_file(*path) - - # Init tasks for service, task in hooks.Filters.CLI_DO_INIT_TASKS.iterate_from_context( filter_context ): fmt.echo_info(f"Running init task in {service}") yield service, task - # Deprecated init tasks - for service, path in hooks.Filters.COMMANDS_INIT.iterate_from_context( - filter_context - ): - fmt.echo_alert( - f"Running deprecated init task: {'/'.join(path)}. Init tasks should no longer be added to the COMMANDS_INIT filter. Plugin developers should use the CLI_DO_INIT_TASKS filter instead." - ) - yield service, env.read_template_file(*path) - fmt.echo_info("All services initialised.") diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 23cb78dd365..2a1e55f506a 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -188,28 +188,6 @@ def your_filter_callback(some_data): #: may contain Jinja markup, similar to templates. CLI_DO_INIT_TASKS: Filter[list[tuple[str, str]], []] = Filter() - #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead. - #: - #: List of commands to be executed during initialization. These commands typically - #: include database migrations, setting feature flags, etc. - #: - #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` tasks. - #: - #: - ``service`` is the name of the container in which the task will be executed. - #: - ``path`` is a tuple that corresponds to a template relative path. - #: Example: ``("myplugin", "hooks", "myservice", "pre-init")`` (see :py:data:`IMAGES_BUILD`). - #: The command to execute will be read from that template, after it is rendered. - COMMANDS_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() - - #: DEPRECATED use :py:data:`CLI_DO_INIT_TASKS` instead with a lower priority score. - #: - #: List of commands to be executed prior to initialization. These commands are run even - #: before the mysql databases are created and the migrations are applied. - #: - #: :parameter list[tuple[str, tuple[str, ...]]] tasks: list of ``(service, path)`` - #: tasks. (see :py:data:`COMMANDS_INIT`). - COMMANDS_PRE_INIT: Filter[list[tuple[str, tuple[str, ...]]], []] = Filter() - #: List of folders to bind-mount in docker-compose containers, either in ``tutor local`` or ``tutor dev``. #: #: This filter is for processing values of the ``MOUNTS`` setting such as:: From 69a85dddf53c83c73f2565766d11a48c574be4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Apr 2023 21:14:22 +0200 Subject: [PATCH 21/96] feat: `images build openedx-dev` We no longer run `docker-compose up --build`. Instead, users are encouraged to build the "openedx-dev" Docker image. --- changelog.d/20230412_100608_regis_palm.md | 6 +- docs/dev.rst | 9 +- docs/tutorials/arm64.rst | 5 +- tutor/commands/compose.py | 146 +++++++++++++++++- tutor/commands/dev.py | 51 +----- tutor/commands/images.py | 15 ++ tutor/commands/local.py | 120 +------------- .../commands/upgrade/{local.py => compose.py} | 0 tutor/core/hooks/actions.py | 1 - tutor/hooks/catalog.py | 9 ++ tutor/templates/dev/docker-compose.yml | 6 - 11 files changed, 178 insertions(+), 190 deletions(-) rename tutor/commands/upgrade/{local.py => compose.py} (100%) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 5d383821766..3d6e6c5c20b 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -7,6 +7,8 @@ - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. - Removed the `ENV_PATCH` filter, which was for internal use only anyway. - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) - - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence: - - The syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. + - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence, the syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. - 💥[Deprecation] The obsolete filters `COMMANDS_PRE_INIT` and `COMMANDS_INIT` have been removed. Plugin developers should instead use `CLI_DO_INIT_TASKS` (with suitable priorities). + - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: + - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. + - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. diff --git a/docs/dev.rst b/docs/dev.rst index 684d806bd96..a4297119417 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -12,14 +12,13 @@ First-time setup Firstly, either :ref:`install Tutor ` (for development against the named releases of Open edX) or :ref:`install Tutor Nightly ` (for development against Open edX's master branches). -Then, optionally, tell Tutor to use a local fork of edx-platform. In that case you will need to rebuild the "openedx" Docker image:: +Then, optionally, tell Tutor to use a local fork of edx-platform.:: tutor config save --append MOUNTS=./edx-platform - tutor images build openedx -Then, run one of the following in order to launch the developer platform setup process:: +Then, build the "openedx" Docker image for development and launch the developer platfornm setup process:: - # To use the edx-platform repository that is built into the image, run: + tutor images build openedx-dev tutor dev launch This will perform several tasks. It will: @@ -130,7 +129,7 @@ The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: - tutor dev dc build lms + tutor images build openedx-dev .. _bind_mounts: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 63c8a92944a..60d73a8965b 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -26,10 +26,9 @@ Then, build the "openedx" and "permissions" images:: tutor images build openedx permissions -.. TODO we don't want this instruction anymore +If you want to use Tutor as an Open edX development environment, you should also build the development image:: -If you want to use Tutor as an Open edX development environment, you should also build the development images:: - tutor dev dc build lms + tutor images build openedx-dev From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index c7a4f6d6e57..ab0bff1377e 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,15 +1,21 @@ from __future__ import annotations import os +import typing as t import click from tutor import bindmount from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import hooks, utils +from tutor import fmt, hooks +from tutor import interactive as interactive_config +from tutor import utils from tutor.commands import jobs +from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseTaskContext +from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES +from tutor.commands.upgrade.compose import upgrade_from from tutor.core.hooks import Filter # pylint: disable=unused-import from tutor.exceptions import TutorError from tutor.tasks import BaseComposeTaskRunner @@ -70,22 +76,143 @@ def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError +@click.command(help="Configure and run Open edX from scratch") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def launch( + context: click.Context, + non_interactive: bool, + pullimages: bool, +) -> None: + utils.warn_macos_docker_memory() + interactive_upgrade(context, not non_interactive) + + click.echo(fmt.title("Stopping any existing platform")) + context.invoke(stop) + + if pullimages: + click.echo(fmt.title("Docker image updates")) + context.invoke(dc_command, command="pull") + + click.echo(fmt.title("Starting the platform in detached mode")) + context.invoke(start, detach=True) + + click.echo(fmt.title("Database creation and migrations")) + context.invoke(do.commands["init"]) + + config = tutor_config.load(context.obj.root) + project_name = context.obj.job_runner(config).project_name + + # Print the urls of the user-facing apps + public_app_hosts = "" + for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(project_name): + public_app_host = tutor_env.render_str( + config, "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://" + host + ) + public_app_hosts += f" {public_app_host}\n" + if public_app_hosts: + fmt.echo_info( + f"""The platform is now running and can be accessed at the following urls: + +{public_app_hosts}""" + ) + + +def interactive_upgrade(context: click.Context, interactive: bool) -> None: + """ + Piece of code that is only used in launch. + """ + run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) + if run_upgrade_from_release is not None: + click.echo(fmt.title("Upgrading from an older release")) + if interactive: + to_release = tutor_env.get_current_open_edx_release_name() + question = f"""You are about to upgrade your Open edX platform from {run_upgrade_from_release.capitalize()} to {to_release.capitalize()} + +It is strongly recommended to make a backup before upgrading. To do so, run: + + tutor local stop # or 'tutor dev stop' in development + sudo rsync -avr "$(tutor config printroot)"/ /tmp/tutor-backup/ + +In case of problem, to restore your backup you will then have to run: sudo rsync -avr /tmp/tutor-backup/ "$(tutor config printroot)"/ + +Are you sure you want to continue?""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + context.invoke( + upgrade, + from_release=run_upgrade_from_release, + ) + + click.echo(fmt.title("Interactive platform configuration")) + config = tutor_config.load_minimal(context.obj.root) + if interactive: + interactive_config.ask_questions(config) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) + + if run_upgrade_from_release and interactive: + question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. + +If you run custom Docker images, you must rebuild them now by running the following command in a different shell: + + tutor images build all # list your custom images here + +See the documentation for more information: + + https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release + +Press enter when you are ready to continue""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + + +@click.command( + short_help="Perform release-specific upgrade tasks", + help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", +) +@click.option( + "--from", + "from_release", + type=click.Choice(OPENEDX_RELEASE_NAMES), +) +@click.pass_context +def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: + fmt.echo_alert( + "This command only performs a partial upgrade of your Open edX platform. " + "To perform a full upgrade, you should run `tutor local launch` (or `tutor dev launch` " + "in development)." + ) + if from_release is None: + from_release = tutor_env.get_env_release(context.obj.root) + if from_release is None: + fmt.echo_info("Your environment is already up-to-date") + else: + upgrade_from(context, from_release) + # We update the environment to update the version + context.invoke(config_save_command) + + @click.command( short_help="Run all or a selection of services.", help="Run all or a selection of services. Docker images will be rebuilt where necessary.", ) -@click.option("--skip-build", is_flag=True, help="Skip image building") +@click.option("--build", is_flag=True, help="Build images on start") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def start( context: BaseComposeContext, - skip_build: bool, + build: bool, detach: bool, services: list[str], ) -> None: command = ["up", "--remove-orphans"] - if not skip_build: + if build: command.append("--build") if detach: command.append("-d") @@ -293,10 +420,21 @@ def _mount_edx_platform( return volumes +def _edx_platform_public_hosts(hosts: list[str], project_name: str) -> list[str]: + edx_platform_hosts = ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] + if project_name == "dev": + edx_platform_hosts[0] += ":8000" + edx_platform_hosts[1] += ":8001" + hosts += edx_platform_hosts + return hosts + + hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) def add_commands(command_group: click.Group) -> None: + command_group.add_command(launch) + command_group.add_command(upgrade) command_group.add_command(start) command_group.add_command(stop) command_group.add_command(restart) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index f5f5df286b0..a60d7e72e41 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -2,11 +2,8 @@ import click -from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks -from tutor import interactive as interactive_config -from tutor import utils +from tutor import hooks from tutor.commands import compose from tutor.types import Config, get_typed @@ -43,51 +40,6 @@ def dev(context: click.Context) -> None: context.obj = DevContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch, for development") -@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@click.pass_context -def launch( - context: click.Context, - non_interactive: bool, - pullimages: bool, -) -> None: - utils.warn_macos_docker_memory() - - click.echo(fmt.title("Interactive platform configuration")) - config = tutor_config.load_minimal(context.obj.root) - if not non_interactive: - interactive_config.ask_questions(config, run_for_prod=False) - tutor_config.save_config_file(context.obj.root, config) - config = tutor_config.load_full(context.obj.root) - tutor_env.save(context.obj.root, config) - - click.echo(fmt.title("Stopping any existing platform")) - context.invoke(compose.stop) - - if pullimages: - click.echo(fmt.title("Docker image updates")) - context.invoke(compose.dc_command, command="pull") - - click.echo(fmt.title("Starting the platform in detached mode")) - context.invoke(compose.start, detach=True) - - click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.do.commands["init"]) - - fmt.echo_info( - """The Open edX platform is now running in detached mode -Your Open edX platform is ready and can be accessed at the following urls: - {http}://{lms_host}:8000 - {http}://{cms_host}:8001 - """.format( - http="https" if config["ENABLE_HTTPS"] else "http", - lms_host=config["LMS_HOST"], - cms_host=config["CMS_HOST"], - ) - ) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: """ @@ -99,5 +51,4 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") -dev.add_command(launch) compose.add_commands(dev) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ca5e7196d36..ba31debb600 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -34,6 +34,20 @@ def _add_core_images_to_build( for image in BASE_IMAGE_NAMES: tag = images.get_tag(config, image) build_images.append((image, ("build", image), tag, ())) + + # Build openedx-dev image + build_images.append( + ( + "openedx-dev", + ("build", "openedx"), + images.get_tag(config, "openedx-dev"), + ( + "--target=development", + f"--build-arg=APP_USER_ID={utils.get_user_id() or 1000}", + ), + ) + ) + return build_images @@ -208,6 +222,7 @@ def _mount_edx_platform( """ if os.path.basename(path) == "edx-platform": volumes.append(("openedx", "edx-platform")) + volumes.append(("openedx-dev", "edx-platform")) return volumes diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 9ead815d845..2b2781d5070 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -1,18 +1,10 @@ from __future__ import annotations -import typing as t - import click -from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks -from tutor import interactive as interactive_config -from tutor import utils +from tutor import hooks from tutor.commands import compose -from tutor.commands.config import save as config_save_command -from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES -from tutor.commands.upgrade.local import upgrade_from from tutor.types import Config, get_typed @@ -46,114 +38,6 @@ def local(context: click.Context) -> None: context.obj = LocalContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch") -@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") -@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@click.pass_context -def launch( - context: click.Context, - non_interactive: bool, - pullimages: bool, -) -> None: - utils.warn_macos_docker_memory() - - run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) - if run_upgrade_from_release is not None: - click.echo(fmt.title("Upgrading from an older release")) - if not non_interactive: - to_release = tutor_env.get_current_open_edx_release_name() - question = f"""You are about to upgrade your Open edX platform from {run_upgrade_from_release.capitalize()} to {to_release.capitalize()} - -It is strongly recommended to make a backup before upgrading. To do so, run: - - tutor local stop - sudo rsync -avr "$(tutor config printroot)"/ /tmp/tutor-backup/ - -In case of problem, to restore your backup you will then have to run: sudo rsync -avr /tmp/tutor-backup/ "$(tutor config printroot)"/ - -Are you sure you want to continue?""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) - context.invoke( - upgrade, - from_release=run_upgrade_from_release, - ) - - click.echo(fmt.title("Interactive platform configuration")) - config = tutor_config.load_minimal(context.obj.root) - if not non_interactive: - interactive_config.ask_questions(config) - tutor_config.save_config_file(context.obj.root, config) - config = tutor_config.load_full(context.obj.root) - tutor_env.save(context.obj.root, config) - - if run_upgrade_from_release and not non_interactive: - question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. - -If you run custom Docker images, you must rebuild them now by running the following command in a different shell: - - tutor images build all # list your custom images here - -See the documentation for more information: - - https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release - -Press enter when you are ready to continue""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) - - click.echo(fmt.title("Stopping any existing platform")) - context.invoke(compose.stop) - if pullimages: - click.echo(fmt.title("Docker image updates")) - context.invoke(compose.dc_command, command="pull") - click.echo(fmt.title("Starting the platform in detached mode")) - context.invoke(compose.start, detach=True) - click.echo(fmt.title("Database creation and migrations")) - context.invoke(compose.do.commands["init"]) - - config = tutor_config.load(context.obj.root) - fmt.echo_info( - """The Open edX platform is now running in detached mode -Your Open edX platform is ready and can be accessed at the following urls: - - {http}://{lms_host} - {http}://{cms_host} - """.format( - http="https" if config["ENABLE_HTTPS"] else "http", - lms_host=config["LMS_HOST"], - cms_host=config["CMS_HOST"], - ) - ) - - -@click.command( - short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `launch`.", -) -@click.option( - "--from", - "from_release", - type=click.Choice(OPENEDX_RELEASE_NAMES), -) -@click.pass_context -def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: - fmt.echo_alert( - "This command only performs a partial upgrade of your Open edX platform. " - "To perform a full upgrade, you should run `tutor local launch`." - ) - if from_release is None: - from_release = tutor_env.get_env_release(context.obj.root) - if from_release is None: - fmt.echo_info("Your environment is already up-to-date") - else: - upgrade_from(context, from_release) - # We update the environment to update the version - context.invoke(config_save_command) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: """ @@ -165,6 +49,4 @@ def _stop_on_dev_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") -local.add_command(launch) -local.add_command(upgrade) compose.add_commands(local) diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/compose.py similarity index 100% rename from tutor/commands/upgrade/local.py rename to tutor/commands/upgrade/compose.py diff --git a/tutor/core/hooks/actions.py b/tutor/core/hooks/actions.py index 924c5ae6245..e081244c1dc 100644 --- a/tutor/core/hooks/actions.py +++ b/tutor/core/hooks/actions.py @@ -9,7 +9,6 @@ from typing_extensions import ParamSpec - from . import priorities from .contexts import Contextualized diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 2a1e55f506a..32a77d4b0ef 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -163,6 +163,15 @@ def your_filter_callback(some_data): :py:class:`tutor.core.hooks.Filter` API. """ + #: Hostnames of user-facing applications. + #: + #: So far this filter is only used to inform the user of application urls after they have run ``launch``. + #: + #: :parameter list[str] hostnames: items from this list are templates that will be + #: rendered by the environment. + #: :parameter str project_name: compose project name, such as "local" or "dev". + APP_PUBLIC_HOSTS: Filter[list[str], [str]] = Filter() + #: List of command line interface (CLI) commands. #: #: :parameter list commands: commands are instances of ``click.Command``. They will diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 96d045ea382..4b8cd38c9bd 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -3,12 +3,6 @@ version: "{{ DOCKER_COMPOSE_VERSION }}" x-openedx-service: &openedx-service image: {{ DOCKER_IMAGE_OPENEDX_DEV }} - build: - context: ../build/openedx/ - target: development - args: - # Note that we never build the openedx-dev image with root user ID, as it would simply fail. - APP_USER_ID: "{{ HOST_USER_ID or 1000 }}" stdin_open: true tty: true volumes: From c018888df90410a42d52c089c606ca32c84b7680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 May 2023 10:03:26 +0200 Subject: [PATCH 22/96] refactor: simplify image tag management --- tests/test_images.py | 14 -------------- tutor/commands/images.py | 40 ++++++++++++++++++++++------------------ tutor/images.py | 6 ------ 3 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 tests/test_images.py diff --git a/tests/test_images.py b/tests/test_images.py deleted file mode 100644 index 549406de222..00000000000 --- a/tests/test_images.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest - -from tutor import images -from tutor.types import Config - - -class ImagesTests(unittest.TestCase): - def test_get_tag(self) -> None: - config: Config = { - "DOCKER_IMAGE_OPENEDX": "registry/openedx", - "DOCKER_IMAGE_OPENEDX_DEV": "registry/openedxdev", - } - self.assertEqual("registry/openedx", images.get_tag(config, "openedx")) - self.assertEqual("registry/openedxdev", images.get_tag(config, "openedx-dev")) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ba31debb600..1d7b8d65208 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -12,14 +12,9 @@ from tutor.core.hooks import Filter from tutor.types import Config -BASE_IMAGE_NAMES = ["openedx", "permissions"] -VENDOR_IMAGES = [ - "caddy", - "elasticsearch", - "mongodb", - "mysql", - "redis", - "smtp", +BASE_IMAGE_NAMES = [ + ("openedx", "DOCKER_IMAGE_OPENEDX"), + ("permissions", "DOCKER_IMAGE_PERMISSIONS"), ] @@ -31,16 +26,17 @@ def _add_core_images_to_build( """ Add base images to the list of Docker images to build on `tutor build all`. """ - for image in BASE_IMAGE_NAMES: - tag = images.get_tag(config, image) - build_images.append((image, ("build", image), tag, ())) + for image, tag in BASE_IMAGE_NAMES: + build_images.append( + (image, ("build", image), tutor_config.get_typed(config, tag, str), ()) + ) # Build openedx-dev image build_images.append( ( "openedx-dev", ("build", "openedx"), - images.get_tag(config, "openedx-dev"), + tutor_config.get_typed(config, "DOCKER_IMAGE_OPENEDX_DEV", str), ( "--target=development", f"--build-arg=APP_USER_ID={utils.get_user_id() or 1000}", @@ -58,11 +54,19 @@ def _add_images_to_pull( """ Add base and vendor images to the list of Docker images to pull on `tutor pull all`. """ - for image in VENDOR_IMAGES: + vendor_images = [ + ("caddy", "DOCKER_IMAGE_CADDY"), + ("elasticsearch", "DOCKER_IMAGE_ELASTICSEARCH"), + ("mongodb", "DOCKER_IMAGE_MONGODB"), + ("mysql", "DOCKER_IMAGE_MYSQL"), + ("redis", "DOCKER_IMAGE_REDIS"), + ("smtp", "DOCKER_IMAGE_SMTP"), + ] + for image, tag_name in vendor_images: if config.get(f"RUN_{image.upper()}", True): - remote_images.append((image, images.get_tag(config, image))) - for image in BASE_IMAGE_NAMES: - remote_images.append((image, images.get_tag(config, image))) + remote_images.append((image, tutor_config.get_typed(config, tag_name, str))) + for image, tag in BASE_IMAGE_NAMES: + remote_images.append((image, tutor_config.get_typed(config, tag, str))) return remote_images @@ -73,8 +77,8 @@ def _add_core_images_to_push( """ Add base images to the list of Docker images to push on `tutor push all`. """ - for image in BASE_IMAGE_NAMES: - remote_images.append((image, images.get_tag(config, image))) + for image, tag in BASE_IMAGE_NAMES: + remote_images.append((image, tutor_config.get_typed(config, tag, str))) return remote_images diff --git a/tutor/images.py b/tutor/images.py index dc640d00ee9..26f80b1824e 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -1,10 +1,4 @@ from tutor import fmt, hooks, utils -from tutor.types import Config, get_typed - - -def get_tag(config: Config, name: str) -> str: - key = "DOCKER_IMAGE_" + name.upper().replace("-", "_") - return get_typed(config, key, str) def build(path: str, tag: str, *args: str) -> None: From f5906198fa8c2a1e2f29d7bd0b53addacfc1c698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 2 May 2023 10:49:19 +0200 Subject: [PATCH 23/96] feat: auto build "openedx-dev" on "dev launch" To achieve that, we introduce a new IMAGES_BUILD_REQUIRED filter. --- changelog.d/20230412_100608_regis_palm.md | 1 + docs/dev.rst | 11 +-- docs/tutorials/arm64.rst | 2 +- tutor/commands/compose.py | 86 +++++++++++++++-------- tutor/commands/dev.py | 13 ++++ tutor/commands/images.py | 20 ++++-- tutor/commands/local.py | 2 + tutor/hooks/catalog.py | 19 +++-- 8 files changed, 106 insertions(+), 48 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 3d6e6c5c20b..9e3c8d268c5 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -12,3 +12,4 @@ - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. + - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. diff --git a/docs/dev.rst b/docs/dev.rst index a4297119417..c22c10a41bb 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -16,18 +16,19 @@ Then, optionally, tell Tutor to use a local fork of edx-platform.:: tutor config save --append MOUNTS=./edx-platform -Then, build the "openedx" Docker image for development and launch the developer platfornm setup process:: +Then, launch the developer platform setup process:: tutor images build openedx-dev tutor dev launch This will perform several tasks. It will: +* build the "openedx-dev" Docker image, which is based on the "openedx" production image but is `specialized for developer usage`_ (eventually with your fork), * stop any existing locally-running Tutor containers, * disable HTTPS, * set ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), * prompt for a platform details (with suitable defaults), -* build an ``openedx-dev`` image, which is based ``openedx`` production image but is `specialized for developer usage`_, +* build an ``openedx-dev`` image, * start LMS, CMS, supporting services, and any plugged-in services, * ensure databases are created and migrated, and * run service initialization scripts, such as service user creation and Waffle configuration. @@ -121,9 +122,7 @@ Rebuilding the openedx-dev image The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by ``tutor local ...`` to run LMS and CMS. However, it has a few differences to make it more convenient for developers: - The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). - - Additional Python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. - - The edx-platform `development requirements `__ are installed. @@ -131,6 +130,10 @@ If you are using a custom ``openedx`` image, then you will need to rebuild ``ope tutor images build openedx-dev +Alternatively, the image will be automatically rebuilt every time you run:: + + tutor dev launch + .. _bind_mounts: diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 60d73a8965b..23beed570fc 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -28,7 +28,7 @@ Then, build the "openedx" and "permissions" images:: If you want to use Tutor as an Open edX development environment, you should also build the development image:: - tutor images build openedx-dev + tutor images build openedx-dev # this will be automatically done by `tutor dev launch` From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index ab0bff1377e..288725f5bc3 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -11,7 +11,7 @@ from tutor import fmt, hooks from tutor import interactive as interactive_config from tutor import utils -from tutor.commands import jobs +from tutor.commands import images, jobs from tutor.commands.config import save as config_save_command from tutor.commands.context import BaseTaskContext from tutor.commands.upgrade import OPENEDX_RELEASE_NAMES @@ -72,6 +72,8 @@ def run_task(self, service: str, command: str) -> int: class BaseComposeContext(BaseTaskContext): + NAME: t.Literal["local", "dev"] + def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError @@ -79,14 +81,29 @@ def job_runner(self, config: Config) -> ComposeTaskRunner: @click.command(help="Configure and run Open edX from scratch") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.option("--skip-build", is_flag=True, help="Skip building Docker images") @click.pass_context def launch( context: click.Context, non_interactive: bool, pullimages: bool, + skip_build: bool, ) -> None: + context_name = context.obj.NAME + run_for_prod = context_name != "dev" + utils.warn_macos_docker_memory() - interactive_upgrade(context, not non_interactive) + interactive_upgrade(context, not non_interactive, run_for_prod) + interactive_configuration(context, not non_interactive, run_for_prod) + + config = tutor_config.load(context.obj.root) + + if not skip_build: + click.echo(fmt.title("Building Docker images")) + images_to_build = hooks.Filters.IMAGES_BUILD_REQUIRED.apply([], context_name) + if not images_to_build: + fmt.echo_info("No image to build") + context.invoke(images.build, image_names=images_to_build) click.echo(fmt.title("Stopping any existing platform")) context.invoke(stop) @@ -101,12 +118,9 @@ def launch( click.echo(fmt.title("Database creation and migrations")) context.invoke(do.commands["init"]) - config = tutor_config.load(context.obj.root) - project_name = context.obj.job_runner(config).project_name - # Print the urls of the user-facing apps public_app_hosts = "" - for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(project_name): + for host in hooks.Filters.APP_PUBLIC_HOSTS.iterate(context_name): public_app_host = tutor_env.render_str( config, "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://" + host ) @@ -119,7 +133,9 @@ def launch( ) -def interactive_upgrade(context: click.Context, interactive: bool) -> None: +def interactive_upgrade( + context: click.Context, interactive: bool, run_for_prod: bool +) -> None: """ Piece of code that is only used in launch. """ @@ -146,29 +162,37 @@ def interactive_upgrade(context: click.Context, interactive: bool) -> None: from_release=run_upgrade_from_release, ) - click.echo(fmt.title("Interactive platform configuration")) - config = tutor_config.load_minimal(context.obj.root) - if interactive: - interactive_config.ask_questions(config) - tutor_config.save_config_file(context.obj.root, config) - config = tutor_config.load_full(context.obj.root) - tutor_env.save(context.obj.root, config) + # Update env and configuration + interactive_configuration(context, interactive, run_for_prod) - if run_upgrade_from_release and interactive: - question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. + # Post upgrade + if run_upgrade_from_release and interactive: + question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. -If you run custom Docker images, you must rebuild them now by running the following command in a different shell: + If you run custom Docker images, you must rebuild them now by running the following command in a different shell: - tutor images build all # list your custom images here + tutor images build all # list your custom images here -See the documentation for more information: + See the documentation for more information: - https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release + https://docs.tutor.overhang.io/install.html#upgrading-to-a-new-open-edx-release -Press enter when you are ready to continue""" - click.confirm( - fmt.question(question), default=True, abort=True, prompt_suffix=" " - ) + Press enter when you are ready to continue""" + click.confirm( + fmt.question(question), default=True, abort=True, prompt_suffix=" " + ) + + +def interactive_configuration( + context: click.Context, interactive: bool, run_for_prod: bool +) -> None: + click.echo(fmt.title("Interactive platform configuration")) + config = tutor_config.load_minimal(context.obj.root) + if interactive: + interactive_config.ask_questions(config, run_for_prod=run_for_prod) + tutor_config.save_config_file(context.obj.root, config) + config = tutor_config.load_full(context.obj.root) + tutor_env.save(context.obj.root, config) @click.command( @@ -420,12 +444,14 @@ def _mount_edx_platform( return volumes -def _edx_platform_public_hosts(hosts: list[str], project_name: str) -> list[str]: - edx_platform_hosts = ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] - if project_name == "dev": - edx_platform_hosts[0] += ":8000" - edx_platform_hosts[1] += ":8001" - hosts += edx_platform_hosts +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _edx_platform_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"] + else: + hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] return hosts diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index a60d7e72e41..659e303dabb 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,5 +1,7 @@ from __future__ import annotations +import typing as t + import click from tutor import env as tutor_env @@ -30,6 +32,8 @@ def __init__(self, root: str, config: Config): class DevContext(compose.BaseComposeContext): + NAME = "dev" + def job_runner(self, config: Config) -> DevTaskRunner: return DevTaskRunner(self.root, config) @@ -51,4 +55,13 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: runner.docker_compose("stop") +@hooks.Filters.IMAGES_BUILD_REQUIRED.add() +def _build_openedx_dev_on_launch( + image_names: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + image_names.append("openedx-dev") + return image_names + + compose.add_commands(dev) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 1d7b8d65208..9c74c65087f 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -20,22 +20,27 @@ @hooks.Filters.IMAGES_BUILD.add() def _add_core_images_to_build( - build_images: list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], + build_images: list[tuple[str, t.Union[str, tuple[str, ...]], str, tuple[str, ...]]], config: Config, -) -> list[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: +) -> list[tuple[str, t.Union[str, tuple[str, ...]], str, tuple[str, ...]]]: """ Add base images to the list of Docker images to build on `tutor build all`. """ for image, tag in BASE_IMAGE_NAMES: build_images.append( - (image, ("build", image), tutor_config.get_typed(config, tag, str), ()) + ( + image, + os.path.join("build", image), + tutor_config.get_typed(config, tag, str), + (), + ) ) # Build openedx-dev image build_images.append( ( "openedx-dev", - ("build", "openedx"), + os.path.join("build", "openedx"), tutor_config.get_typed(config, "DOCKER_IMAGE_OPENEDX_DEV", str), ( "--target=development", @@ -188,7 +193,7 @@ def build( # Build images.build( - tutor_env.pathjoin(context.root, *path), + tutor_env.pathjoin(context.root, path), tag, *image_build_args, ) @@ -262,7 +267,7 @@ def printtag(context: Context, image_names: list[str]) -> None: def find_images_to_build( config: Config, image: str -) -> t.Iterator[tuple[str, tuple[str, ...], str, tuple[str, ...]]]: +) -> t.Iterator[tuple[str, str, str, tuple[str, ...]]]: """ Iterate over all images to build. @@ -272,10 +277,11 @@ def find_images_to_build( """ found = False for name, path, tag, args in hooks.Filters.IMAGES_BUILD.iterate(config): + relative_path = path if isinstance(path, str) else os.path.join(*path) if image in [name, "all"]: found = True tag = tutor_env.render_str(config, tag) - yield (name, path, tag, args) + yield (name, relative_path, tag, args) if not found: raise ImageNotFoundError(image) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 2b2781d5070..320edaab796 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -28,6 +28,8 @@ def __init__(self, root: str, config: Config): # pylint: disable=too-few-public-methods class LocalContext(compose.BaseComposeContext): + NAME = "local" + def job_runner(self, config: Config) -> LocalTaskRunner: return LocalTaskRunner(self.root, config) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 32a77d4b0ef..d59fdd0832f 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -7,7 +7,7 @@ # The Tutor plugin system is licensed under the terms of the Apache 2.0 license. __license__ = "Apache 2.0" -from typing import Any, Callable, Iterable +from typing import Any, Callable, Iterable, Literal, Union import click @@ -169,8 +169,8 @@ def your_filter_callback(some_data): #: #: :parameter list[str] hostnames: items from this list are templates that will be #: rendered by the environment. - #: :parameter str project_name: compose project name, such as "local" or "dev". - APP_PUBLIC_HOSTS: Filter[list[str], [str]] = Filter() + #: :parameter str context_name: either "local" or "dev", depending on the calling context. + APP_PUBLIC_HOSTS: Filter[list[str], [Literal["local", "dev"]]] = Filter() #: List of command line interface (CLI) commands. #: @@ -341,18 +341,25 @@ def your_filter_callback(some_data): #: :parameter list[tuple[str, tuple[str, ...], str, tuple[str, ...]]] tasks: list of ``(name, path, tag, args)`` tuples. #: #: - ``name`` is the name of the image, as in ``tutor images build myimage``. - #: - ``path`` is the relative path to the folder that contains the Dockerfile. + #: - ``path`` is the relative path to the folder that contains the Dockerfile. This can be either a string or a tuple of strings. #: For instance ``("myplugin", "build", "myservice")`` indicates that the template will be read from - #: ``myplugin/build/myservice/Dockerfile`` + #: ``myplugin/build/myservice/Dockerfile``. This argument value would be equivalent to "myplugin/build/myservice". #: - ``tag`` is the Docker tag that will be applied to the image. It will be #: rendered at runtime with the user configuration. Thus, the image tag could #: be ``"{{ DOCKER_REGISTRY }}/myimage:{{ TUTOR_VERSION }}"``. #: - ``args`` is a list of arguments that will be passed to ``docker build ...``. #: :parameter Config config: user configuration. IMAGES_BUILD: Filter[ - list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] + list[tuple[str, Union[str, tuple[str, ...]], str, tuple[str, ...]]], [Config] ] = Filter() + #: List of image names which must be built prior to launching the platform. These + #: images will be built on launch, in "dev" and "local" mode (but not in Kubernetes). + #: + #: :parameter list[str] names: list of image names. + #: :parameter str context_name: either "local" or "dev", depending on the calling context. + IMAGES_BUILD_REQUIRED: Filter[list[str], [Literal["local", "dev"]]] = Filter() + #: List of host directories to be automatically bind-mounted in Docker images at #: build time. For instance, this is useful to build Docker images using a custom #: repository on the host. From 7d8154a7d401fcc609684ff1878bcd9a80f63259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 4 May 2023 11:39:02 +0200 Subject: [PATCH 24/96] feat: auto-complete image names in `images build/pull/...` --- changelog.d/20230412_100608_regis_palm.md | 1 + tutor/commands/images.py | 62 ++++++++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 9e3c8d268c5..5bf40a1a7e1 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -13,3 +13,4 @@ - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. + - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 9c74c65087f..1c25a30b1d6 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -7,7 +7,7 @@ from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, hooks, images, types, utils +from tutor import exceptions, fmt, hooks, images, types, utils from tutor.commands.context import Context from tutor.core.hooks import Filter from tutor.types import Config @@ -87,13 +87,60 @@ def _add_core_images_to_push( return remote_images +class ImageNameParam(click.ParamType): + """ + Convenient auto-completion of image names. + """ + + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> list[click.shell_completion.CompletionItem]: + # Hackish way to get the project root and config + root = getattr( + getattr(getattr(ctx, "parent", None), "parent", None), "params", {} + ).get("root", "") + config = tutor_config.load_full(root) + + results = [] + for name in self.iter_image_names(config): + if name.startswith(incomplete): + results.append(click.shell_completion.CompletionItem(name)) + return results + + def iter_image_names(self, config: Config) -> t.Iterable["str"]: + raise NotImplementedError + + +class BuildImageNameParam(ImageNameParam): + def iter_image_names(self, config: Config) -> t.Iterable["str"]: + for name, _path, _tag, _args in hooks.Filters.IMAGES_BUILD.iterate(config): + yield name + + +class PullImageNameParam(ImageNameParam): + def iter_image_names(self, config: Config) -> t.Iterable["str"]: + for name, _tag in hooks.Filters.IMAGES_PULL.iterate(config): + yield name + + +class PushImageNameParam(ImageNameParam): + def iter_image_names(self, config: Config) -> t.Iterable["str"]: + for name, _tag in hooks.Filters.IMAGES_PUSH.iterate(config): + yield name + + @click.group(name="images", short_help="Manage docker images") def images_command() -> None: pass @click.command() -@click.argument("image_names", metavar="image", nargs=-1) +@click.argument( + "image_names", + metavar="image", + nargs=-1, + type=BuildImageNameParam(), +) @click.option( "--no-cache", is_flag=True, help="Do not use cache when building the image" ) @@ -207,7 +254,7 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] instance to build a Docker image with a local git checkout of a remote repo. Users configure bind-mounts with the `MOUNTS` config setting. Plugins can then - automaticall add build contexts based on these values. + automatically add build contexts based on these values. """ user_mounts = types.get_typed(config, "MOUNTS", list) build_contexts: dict[str, list[tuple[str, str]]] = {} @@ -215,6 +262,9 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( user_mount ): + fmt.echo_info( + f"Adding {user_mount} to the build context '{stage_name}' of image '{image_name}'" + ) if image_name not in build_contexts: build_contexts[image_name] = [] build_contexts[image_name].append((user_mount, stage_name)) @@ -236,7 +286,7 @@ def _mount_edx_platform( @click.command(short_help="Pull images from the Docker registry") -@click.argument("image_names", metavar="image", nargs=-1) +@click.argument("image_names", metavar="image", type=PullImageNameParam(), nargs=-1) @click.pass_obj def pull(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) @@ -246,7 +296,7 @@ def pull(context: Context, image_names: list[str]) -> None: @click.command(short_help="Push images to the Docker registry") -@click.argument("image_names", metavar="image", nargs=-1) +@click.argument("image_names", metavar="image", type=PushImageNameParam(), nargs=-1) @click.pass_obj def push(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) @@ -256,7 +306,7 @@ def push(context: Context, image_names: list[str]) -> None: @click.command(short_help="Print tag associated to a Docker image") -@click.argument("image_names", metavar="image", nargs=-1) +@click.argument("image_names", metavar="image", type=BuildImageNameParam(), nargs=-1) @click.pass_obj def printtag(context: Context, image_names: list[str]) -> None: config = tutor_config.load_full(context.root) From dac0b382324c240483c5911d7a07882091c8f66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 15 May 2023 22:06:25 +0200 Subject: [PATCH 25/96] docs: bump minimal required docker/compose versions --- changelog.d/20230412_100608_regis_palm.md | 1 + docs/install.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index 5bf40a1a7e1..d53b1b6ef71 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -14,3 +14,4 @@ - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. + - [Deprecation] For local installations, Docker v20.10.15 and Compose v2.0.0 are now the minimum required versions. diff --git a/docs/install.rst b/docs/install.rst index 5059b33e2e8..a9f47cfc4ec 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,8 +12,8 @@ Requirements * Architecture: support for ARM64 is a work-in-progress. See `this issue `__. * Required software: - - `Docker `__: v18.06.0+ - - `Docker Compose `__: v1.22.0+ + - `Docker `__: v20.10.15+ + - `Docker Compose `__: v2.0.0+ .. warning:: Do not attempt to simply run ``apt-get install docker docker-compose`` on older Ubuntu platforms, such as 16.04 (Xenial), as you will get older versions of these utilities. From 1f979d58c38ce88f9162014166fbd5cef21ef9bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 4 May 2023 14:06:28 +0200 Subject: [PATCH 26/96] feat: separate mounts command Manual configuration via the `MOUNTS` setting was inconvenient. We (re)introduce a new(ish) `tutor mounts` command. Old timers will perhaps remember that we used to have a `tutor bindmount` command. Well, it's back! But better and different. --- .../20230427_165520_regis_build_mount.md | 2 +- docs/dev.rst | 74 +++++++--- tests/commands/test_plugins.py | 1 - tutor/bindmount.py | 11 +- tutor/commands/cli.py | 12 +- tutor/commands/compose.py | 2 +- tutor/commands/config.py | 56 +++---- tutor/commands/images.py | 31 ++-- tutor/commands/mounts.py | 137 ++++++++++++++++++ tutor/commands/params.py | 29 ++++ tutor/hooks/catalog.py | 2 +- 11 files changed, 276 insertions(+), 81 deletions(-) create mode 100644 tutor/commands/mounts.py create mode 100644 tutor/commands/params.py diff --git a/changelog.d/20230427_165520_regis_build_mount.md b/changelog.d/20230427_165520_regis_build_mount.md index 36fc658b9e4..88f883dd4b2 100644 --- a/changelog.d/20230427_165520_regis_build_mount.md +++ b/changelog.d/20230427_165520_regis_build_mount.md @@ -1,3 +1,3 @@ - [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) - [Feature] Automatically mount host folders *at build time*. This is a really important feature, as it allows us to transparently build images using local forks of remote repositories. (by @regisb) -- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts. (by @regisb) +- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts, which are managed by the `tutor mounts` commands. (by @regisb) diff --git a/docs/dev.rst b/docs/dev.rst index c22c10a41bb..d6d2851ed0b 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -14,7 +14,7 @@ Firstly, either :ref:`install Tutor ` (for development against the name Then, optionally, tell Tutor to use a local fork of edx-platform.:: - tutor config save --append MOUNTS=./edx-platform + tutor mounts add ./edx-platform Then, launch the developer platform setup process:: @@ -49,13 +49,13 @@ Now, use the ``tutor dev ...`` command-line interface to manage the development .. note:: - If you've added your edx-platform to the ``MOUNTS`` setting, you can remove at any time by running:: + If you've added your edx-platform to the bind-mounted folders, you can remove at any time by running:: - tutor config save --remove MOUNTS=./edx-platform + tutor mounts remove ./edx-platform At any time, check your configuration by running:: - tutor config printvalue MOUNTS + tutor mounts list Read more about bind-mounts :ref:`below `. @@ -144,40 +144,74 @@ It may sometimes be convenient to mount container directories on the host, for i .. _persistent_mounts: -Persistent bind-mounted volumes with ``MOUNTS`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Persistent bind-mounted volumes with ``tutor mounts`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``MOUNTS`` is a Tutor setting to bind-mount host directories both at build time and run time: +``tutor mounts`` is a set of Tutor command to manage bind-mounted host directories. Directories are mounted `both` at build time and run time: -- At build time: plugins can automatically add certain directories listed in this setting to the `Docker build context `__. This makes it possible to transparently build a Docker image using a locally checked-out repository. +- At build time: some of the host directories will be added the `Docker build context `__. This makes it possible to transparently build a Docker image using a locally checked-out repository. - At run time: host directories will be bind-mounted in running containers, using either an automatic or a manual configuration. -After some values have been added to the ``MOUNTS`` setting, all ``tutor dev`` and ``tutor local`` commands will make use of these bind-mount volumes. -Values added to ``MOUNTS`` can take one of two forms. The first is explicit:: +After some directories have been added with ``tutor mounts add``, all ``tutor dev`` and ``tutor local`` commands will make use of these bind-mount volumes. - tutor config save --append MOUNTS=lms:/path/to/edx-platform:/openedx/edx-platform +Values passed to ``tutor mounts add ...`` can take one of two forms. The first is explicit:: + + tutor mounts add lms:/path/to/edx-platform:/openedx/edx-platform The second is implicit:: - tutor config save --append MOUNTS=/path/to/edx-platform + tutor mounts add /path/to/edx-platform -With the explicit form, the setting means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container at run time". +With the explicit form, the value means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container at run time". If you use the explicit format, you will quickly realise that you usually want to bind-mount folders in multiple containers at a time. For instance, you will want to bind-mount the edx-platform repository in the "cms" container, but also the "lms-worker" and "cms-worker" containers. To do that, write instead:: # each service is added to a coma-separated list - tutor config save --append MOUNTS=lms,cms,lms-worker,cms-worker:/path/to/edx-platform:/openedx/edx-platform + tutor mounts add lms,cms,lms-worker,cms-worker:/path/to/edx-platform:/openedx/edx-platform -This command line is a bit cumbersome. In addition, with this explicit form, the edx-platform repository will *not* be added to the build context at build time. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``MOUNTS`` setting. For instance, the following implicit form can be used instead of the explicit form above:: +This command line is a bit cumbersome. In addition, with this explicit form, the edx-platform repository will *not* be added to the build context at build time. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``tutor mounts add`` command. For instance, the following implicit form can be used instead of the explicit form above:: - tutor config save --append MOUNTS=/path/to/edx-platform + tutor mounts add /path/to/edx-platform With this implicit form, the edx-platform repo will be bind-mounted in the containers at run time, just like with the explicit form. But in addition, the edx-platform will also automatically be added to the Docker image at build time. -So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``--append MOUNTS=~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: - - tutor config save --append MOUNTS=lms:~/venvs/edx-platform:/openedx/venv +To check whether you have used the correct syntax, you should run ``tutor mounts list``. This command will indicate whether your folders will be bind-mounted at build time, run time, or both. For instance:: + + $ tutor mounts add /path/to/edx-platform + $ tutor mounts list + - name: /home/data/regis/projets/overhang/repos/edx/edx-platform + build_mounts: + - image: openedx + context: edx-platform + - image: openedx-dev + context: edx-platform + compose_mounts: + - service: lms + container_path: /openedx/edx-platform + - service: cms + container_path: /openedx/edx-platform + - service: lms-worker + container_path: /openedx/edx-platform + - service: cms-worker + container_path: /openedx/edx-platform + - service: lms-job + container_path: /openedx/edx-platform + - service: cms-job + container_path: /openedx/edx-platform + +So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``mounts add ~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: + + tutor mounts add lms:~/venvs/edx-platform:/openedx/venv + +Verify the configuration with the ``list`` command:: + + $ tutor mounts list + - name: lms:~/venvs/edx-platform:/openedx/venv + build_mounts: [] + compose_mounts: + - service: lms + container_path: /openedx/venv .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`. @@ -190,7 +224,7 @@ Sometimes, you may want to modify some of the files inside a container for which Then, bind-mount that folder back in the container with the ``MOUNTS`` setting (described :ref:`above `):: - tutor config save --append MOUNTS=lms:~/venv:/openedx/venv + tutor mounts add lms:~/venv:/openedx/venv You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your "lms" container. diff --git a/tests/commands/test_plugins.py b/tests/commands/test_plugins.py index 34fd6a45068..bcec79b8fe1 100644 --- a/tests/commands/test_plugins.py +++ b/tests/commands/test_plugins.py @@ -1,4 +1,3 @@ -import typing as t import unittest from unittest.mock import Mock, patch diff --git a/tutor/bindmount.py b/tutor/bindmount.py index fab7dae22c3..003840a8014 100644 --- a/tutor/bindmount.py +++ b/tutor/bindmount.py @@ -5,7 +5,11 @@ import typing as t from functools import lru_cache -from tutor import hooks +from tutor import hooks, types + + +def get_mounts(config: types.Config) -> list[str]: + return types.get_typed(config, "MOUNTS", list) def iter_mounts(user_mounts: list[str], name: str) -> t.Iterable[str]: @@ -25,7 +29,8 @@ def iter_mounts(user_mounts: list[str], name: str) -> t.Iterable[str]: def parse_mount(value: str) -> list[tuple[str, str, str]]: """ - Parser for mount arguments of the form "service1[,service2,...]:/host/path:/container/path". + Parser for mount arguments of the form + "service1[,service2,...]:/host/path:/container/path" (explicit) or "/host/path". Returns a list of (service, host_path, container_path) tuples. """ @@ -60,7 +65,7 @@ def parse_explicit_mount(value: str) -> list[tuple[str, str, str]]: @lru_cache(maxsize=None) def parse_implicit_mount(value: str) -> list[tuple[str, str, str]]: """ - Argument is of the form "/host/path" + Argument is of the form "/path/to/host/directory" """ mounts: list[tuple[str, str, str]] = [] host_path = os.path.abspath(os.path.expanduser(value)) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index dcd894989ef..c8e552a2c9c 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -14,6 +14,7 @@ from tutor.commands.images import images_command from tutor.commands.k8s import k8s from tutor.commands.local import local +from tutor.commands.mounts import mounts_command from tutor.commands.plugins import plugins_command @@ -129,7 +130,16 @@ def help_command(context: click.Context) -> None: hooks.Filters.CLI_COMMANDS.add_items( - [images_command, config_command, local, dev, k8s, help_command, plugins_command] + [ + config_command, + dev, + help_command, + images_command, + k8s, + local, + mounts_command, + plugins_command, + ] ) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 288725f5bc3..8c980507840 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -428,7 +428,7 @@ def _mount_edx_platform( volumes: list[tuple[str, str]], name: str ) -> list[tuple[str, str]]: """ - When mounting edx-platform with `tutor config save --append MOUNTS=/path/to/edx-platform`, + When mounting edx-platform with `tutor mounts add /path/to/edx-platform`, bind-mount the host repo in the lms/cms containers. """ if name == "edx-platform": diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 72d1d2bc6a2..c1c1b9fcdc4 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -6,12 +6,13 @@ import click import click.shell_completion -from .. import config as tutor_config -from .. import env, exceptions, fmt -from .. import interactive as interactive_config -from .. import serialize -from ..types import Config, ConfigValue -from .context import Context +from tutor import config as tutor_config +from tutor import env, exceptions, fmt +from tutor import interactive as interactive_config +from tutor import serialize +from tutor.commands.context import Context +from tutor.commands.params import ConfigLoaderParam +from tutor.types import ConfigValue @click.group( @@ -23,7 +24,7 @@ def config_command() -> None: pass -class ConfigKeyParamType(click.ParamType): +class ConfigKeyParamType(ConfigLoaderParam): name = "configkey" def shell_complete( @@ -31,31 +32,20 @@ def shell_complete( ) -> list[click.shell_completion.CompletionItem]: return [ click.shell_completion.CompletionItem(key) - for key, _value in self._shell_complete_config_items(ctx, incomplete) + for key, _value in self._shell_complete_config_items(incomplete) ] def _shell_complete_config_items( - self, ctx: click.Context, incomplete: str + self, incomplete: str ) -> list[tuple[str, ConfigValue]]: - # Here we want to auto-complete the name of the config key. For that we need to - # figure out the list of enabled plugins, and for that we need the project root. - # The project root would ordinarily be stored in ctx.obj.root, but during - # auto-completion we don't have access to our custom Tutor context. So we resort - # to a dirty hack, which is to examine the grandparent context. - root = getattr( - getattr(getattr(ctx, "parent", None), "parent", None), "params", {} - ).get("root", "") - config = tutor_config.load_full(root) return [ (key, value) - for key, value in self._candidate_config_items(config) + for key, value in self._candidate_config_items() if key.startswith(incomplete) ] - def _candidate_config_items( - self, config: Config - ) -> t.Iterable[tuple[str, ConfigValue]]: - yield from config.items() + def _candidate_config_items(self) -> t.Iterable[tuple[str, ConfigValue]]: + yield from self.config.items() class ConfigKeyValParamType(ConfigKeyParamType): @@ -82,16 +72,14 @@ def shell_complete( # further auto-complete later. return [ click.shell_completion.CompletionItem(f"'{key}='") - for key, value in self._shell_complete_config_items(ctx, incomplete) + for key, value in self._shell_complete_config_items(incomplete) ] if incomplete.endswith("="): # raise ValueError(f"incomplete: <{incomplete}>") # Auto-complete with '=' return [ click.shell_completion.CompletionItem(f"{key}={json.dumps(value)}") - for key, value in self._shell_complete_config_items( - ctx, incomplete[:-1] - ) + for key, value in self._shell_complete_config_items(incomplete[:-1]) ] # Else, don't bother return [] @@ -102,10 +90,8 @@ class ConfigListKeyValParamType(ConfigKeyValParamType): Same as the parent class, but for keys of type `list`. """ - def _candidate_config_items( - self, config: Config - ) -> t.Iterable[tuple[str, ConfigValue]]: - for key, val in config.items(): + def _candidate_config_items(self) -> t.Iterable[tuple[str, ConfigValue]]: + for key, val in self.config.items(): if isinstance(val, list): yield key, val @@ -154,9 +140,9 @@ def _candidate_config_items( def save( context: Context, interactive: bool, - set_vars: tuple[str, t.Any], - append_vars: tuple[str, t.Any], - remove_vars: tuple[str, t.Any], + set_vars: list[tuple[str, t.Any]], + append_vars: list[tuple[str, t.Any]], + remove_vars: list[tuple[str, t.Any]], unset_vars: list[str], env_only: bool, ) -> None: @@ -235,5 +221,5 @@ def patches_list(context: Context) -> None: config_command.add_command(save) config_command.add_command(printroot) config_command.add_command(printvalue) -config_command.add_command(patches_command) patches_command.add_command(patches_list) +config_command.add_command(patches_command) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 1c25a30b1d6..0108fed4050 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -5,10 +5,12 @@ import click +from tutor import bindmount from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, fmt, hooks, images, types, utils +from tutor import exceptions, fmt, hooks, images, utils from tutor.commands.context import Context +from tutor.commands.params import ConfigLoaderParam from tutor.core.hooks import Filter from tutor.types import Config @@ -87,7 +89,7 @@ def _add_core_images_to_push( return remote_images -class ImageNameParam(click.ParamType): +class ImageNameParam(ConfigLoaderParam): """ Convenient auto-completion of image names. """ @@ -95,37 +97,31 @@ class ImageNameParam(click.ParamType): def shell_complete( self, ctx: click.Context, param: click.Parameter, incomplete: str ) -> list[click.shell_completion.CompletionItem]: - # Hackish way to get the project root and config - root = getattr( - getattr(getattr(ctx, "parent", None), "parent", None), "params", {} - ).get("root", "") - config = tutor_config.load_full(root) - results = [] - for name in self.iter_image_names(config): + for name in self.iter_image_names(): if name.startswith(incomplete): results.append(click.shell_completion.CompletionItem(name)) return results - def iter_image_names(self, config: Config) -> t.Iterable["str"]: + def iter_image_names(self) -> t.Iterable["str"]: raise NotImplementedError class BuildImageNameParam(ImageNameParam): - def iter_image_names(self, config: Config) -> t.Iterable["str"]: - for name, _path, _tag, _args in hooks.Filters.IMAGES_BUILD.iterate(config): + def iter_image_names(self) -> t.Iterable["str"]: + for name, _path, _tag, _args in hooks.Filters.IMAGES_BUILD.iterate(self.config): yield name class PullImageNameParam(ImageNameParam): - def iter_image_names(self, config: Config) -> t.Iterable["str"]: - for name, _tag in hooks.Filters.IMAGES_PULL.iterate(config): + def iter_image_names(self) -> t.Iterable["str"]: + for name, _tag in hooks.Filters.IMAGES_PULL.iterate(self.config): yield name class PushImageNameParam(ImageNameParam): - def iter_image_names(self, config: Config) -> t.Iterable["str"]: - for name, _tag in hooks.Filters.IMAGES_PUSH.iterate(config): + def iter_image_names(self) -> t.Iterable["str"]: + for name, _tag in hooks.Filters.IMAGES_PUSH.iterate(self.config): yield name @@ -256,9 +252,8 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] Users configure bind-mounts with the `MOUNTS` config setting. Plugins can then automatically add build contexts based on these values. """ - user_mounts = types.get_typed(config, "MOUNTS", list) build_contexts: dict[str, list[tuple[str, str]]] = {} - for user_mount in user_mounts: + for user_mount in bindmount.get_mounts(config): for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( user_mount ): diff --git a/tutor/commands/mounts.py b/tutor/commands/mounts.py new file mode 100644 index 00000000000..547b095e0c4 --- /dev/null +++ b/tutor/commands/mounts.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import os + +import click +import yaml + +from tutor import bindmount, exceptions, fmt, hooks +from tutor.commands.context import Context +from tutor.commands.params import ConfigLoaderParam +from tutor import config as tutor_config +from tutor.commands.config import save as config_save + + +class MountParamType(ConfigLoaderParam): + name = "mount" + + def shell_complete( + self, ctx: click.Context, param: click.Parameter, incomplete: str + ) -> list[click.shell_completion.CompletionItem]: + mounts = bindmount.get_mounts(self.config) + return [ + click.shell_completion.CompletionItem(mount) + for mount in mounts + if mount.startswith(incomplete) + ] + + +@click.group(name="mounts") +def mounts_command() -> None: + """ + Manage host bind-mounts + + Bind-mounted folders are used both in image building, development (`dev` commands) + and `local` deployments. + """ + + +@click.command(name="list") +@click.pass_obj +def mounts_list(context: Context) -> None: + """ + List bind-mounted folders + + Entries will be fetched from the `MOUNTS` project setting. + """ + config = tutor_config.load(context.root) + mounts = [] + for mount_name in bindmount.get_mounts(config): + build_mounts = [ + {"image": image_name, "context": stage_name} + for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( + mount_name + ) + ] + compose_mounts = [ + { + "service": service, + "container_path": container_path, + } + for service, _host_path, container_path in bindmount.parse_mount(mount_name) + ] + mounts.append( + { + "name": mount_name, + "build_mounts": build_mounts, + "compose_mounts": compose_mounts, + } + ) + fmt.echo(yaml.dump(mounts, default_flow_style=False, sort_keys=False)) + + +@click.command(name="add") +@click.argument("mounts", metavar="mount", type=click.Path(), nargs=-1) +@click.pass_context +def mounts_add(context: click.Context, mounts: list[str]) -> None: + """ + Add a bind-mounted folder + + The bind-mounted folder will be added to the project configuration, in the ``MOUNTS`` + setting. + + Values passed to this command can take one of two forms. The first is explicit:: + + tutor mounts add myservice:/host/path:/container/path + + The second is implicit:: + + tutor mounts add /host/path + + With the explicit form, the value means "bind-mount the host folder /host/path to + /container/path in the "myservice" container at run time". + + With the implicit form, plugins are in charge of automatically detecting in which + containers and locations the /host/path folder should be bind-mounted. In this case, + folders can be bind-mounted at build-time -- which cannot be achieved with the + explicit form. + """ + new_mounts = [] + for mount in mounts: + if not bindmount.parse_explicit_mount(mount): + # Path is implicit: check that this path is valid + # (we don't try to validate explicit mounts) + mount = os.path.abspath(os.path.expanduser(mount)) + if not os.path.exists(mount): + raise exceptions.TutorError(f"Path {mount} does not exist on the host") + new_mounts.append(mount) + fmt.echo_info(f"Adding bind-mount: {mount}") + + context.invoke(config_save, append_vars=[("MOUNTS", mount) for mount in new_mounts]) + + +@click.command(name="remove") +@click.argument("mounts", metavar="mount", type=MountParamType(), nargs=-1) +@click.pass_context +def mounts_remove(context: click.Context, mounts: list[str]) -> None: + """ + Remove a bind-mounted folder + + The bind-mounted folder will be removed from the ``MOUNTS`` project setting. + """ + removed_mounts = [] + for mount in mounts: + if not bindmount.parse_explicit_mount(mount): + # Path is implicit: expand it + mount = os.path.abspath(os.path.expanduser(mount)) + removed_mounts.append(mount) + fmt.echo_info(f"Removing bind-mount: {mount}") + + context.invoke( + config_save, remove_vars=[("MOUNTS", mount) for mount in removed_mounts] + ) + + +mounts_command.add_command(mounts_list) +mounts_command.add_command(mounts_add) +mounts_command.add_command(mounts_remove) diff --git a/tutor/commands/params.py b/tutor/commands/params.py new file mode 100644 index 00000000000..a9e0e6be228 --- /dev/null +++ b/tutor/commands/params.py @@ -0,0 +1,29 @@ +import typing as t + +import click + +from tutor import config as tutor_config +from tutor import hooks +from tutor.types import Config + + +class ConfigLoaderParam(click.ParamType): + """ + Convenient param child class that automatically loads the user configuration on auto-complete. + """ + + def __init__(self) -> None: + self.root = None + self._config: t.Optional[Config] = None + + @hooks.Actions.PROJECT_ROOT_READY.add() + def _on_root_ready(root: str) -> None: + self.root = root + + @property + def config(self) -> Config: + if self.root is None: + return {} + if self._config is None: + self._config = tutor_config.load_full(self.root) + return self._config diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index d59fdd0832f..b3b80a94c6c 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -201,7 +201,7 @@ def your_filter_callback(some_data): #: #: This filter is for processing values of the ``MOUNTS`` setting such as:: #: - #: tutor config save --append MOUNTS=/path/to/edx-platform + #: tutor mounts add /path/to/edx-platform #: #: In this example, this host folder would be bind-mounted in different containers #: (lms, lms-worker, cms, cms-worker, lms-job, cms-job) at the From 7fed4a97a5b032c19d943cd757e36f65e94f7f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 16 May 2023 09:17:37 +0200 Subject: [PATCH 27/96] fix: format output of `config printvalue` as yaml --- changelog.d/20230412_100608_regis_palm.md | 1 + tests/test_serialize.py | 14 ++++++++++++++ tutor/commands/config.py | 4 ++-- tutor/commands/mounts.py | 7 ++++--- tutor/serialize.py | 23 ++++++++++++++++++++--- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index d53b1b6ef71..d4701286e2d 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -15,3 +15,4 @@ - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. - [Deprecation] For local installations, Docker v20.10.15 and Compose v2.0.0 are now the minimum required versions. + - [Bugfix] Make `tutor config printvalue ...` print actual yaml-formatted values, such as "true" and "null" diff --git a/tests/test_serialize.py b/tests/test_serialize.py index f0bd9fde7d7..874675854b5 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -41,3 +41,17 @@ def test_parse_key_value(self) -> None: "x=key1:\n subkey: value\nkey2:\n subkey: value" ), ) + + def test_str_format(self) -> None: + self.assertEqual("true", serialize.str_format(True)) + self.assertEqual("false", serialize.str_format(False)) + self.assertEqual("null", serialize.str_format(None)) + self.assertEqual("éü©", serialize.str_format("éü©")) + self.assertEqual("""[1, 'abcd']""", serialize.str_format([1, "abcd"])) + + def test_load_str_format(self) -> None: + self.assertEqual(True, serialize.load(serialize.str_format(True))) + self.assertEqual(False, serialize.load(serialize.str_format(False))) + self.assertEqual(None, serialize.load(serialize.str_format(None))) + self.assertEqual("éü©", serialize.load(serialize.str_format("éü©"))) + self.assertEqual([1, "abcd"], serialize.load(serialize.str_format([1, "abcd"]))) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index c1c1b9fcdc4..abcdc031a7f 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -199,10 +199,10 @@ def printroot(context: Context) -> None: def printvalue(context: Context, key: str) -> None: config = tutor_config.load(context.root) try: - # Note that this will incorrectly print None values - fmt.echo(str(config[key])) + value = config[key] except KeyError as e: raise exceptions.TutorError(f"Missing configuration value: {key}") from e + fmt.echo(serialize.str_format(value)) @click.group(name="patches", help="Commands related to patches in configurations") diff --git a/tutor/commands/mounts.py b/tutor/commands/mounts.py index 547b095e0c4..c97075521b4 100644 --- a/tutor/commands/mounts.py +++ b/tutor/commands/mounts.py @@ -5,11 +5,12 @@ import click import yaml -from tutor import bindmount, exceptions, fmt, hooks -from tutor.commands.context import Context -from tutor.commands.params import ConfigLoaderParam +from tutor import bindmount from tutor import config as tutor_config +from tutor import exceptions, fmt, hooks from tutor.commands.config import save as config_save +from tutor.commands.context import Context +from tutor.commands.params import ConfigLoaderParam class MountParamType(ConfigLoaderParam): diff --git a/tutor/serialize.py b/tutor/serialize.py index 0833749b2f6..0f4ae6cf1f2 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -18,19 +18,36 @@ def load_all(stream: str) -> t.Iterator[t.Any]: def dump_all(documents: t.Sequence[t.Any], fileobj: TextIOWrapper) -> None: - yaml.safe_dump_all(documents, stream=fileobj, default_flow_style=False) + yaml.safe_dump_all( + documents, stream=fileobj, default_flow_style=False, allow_unicode=True + ) def dump(content: t.Any, fileobj: TextIOWrapper) -> None: - yaml.dump(content, stream=fileobj, default_flow_style=False) + yaml.dump(content, stream=fileobj, default_flow_style=False, allow_unicode=True) def dumps(content: t.Any) -> str: - result = yaml.dump(content, default_flow_style=False) + result = yaml.dump(content, default_flow_style=False, allow_unicode=True) assert isinstance(result, str) return result +def str_format(content: t.Any) -> str: + """ + Convert a value to str. + + This is almost like json, but more convenient for printing to the standard output. + """ + if content is True: + return "true" + if content is False: + return "false" + if content is None: + return "null" + return str(content) + + def parse(v: t.Union[str, t.IO[str]]) -> t.Any: """ Parse a yaml-formatted string. From cefaf6c21d304bbaf4d4a93cbace807bdc707b45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 30 May 2023 16:01:17 +0200 Subject: [PATCH 28/96] feat: upgrade mongodb to 4.4 This is for https://github.com/openedx/wg-build-test-release/issues/288 Note that we also upgrade mongodb from 4.0 to 4.2, because somehow this hasn't been done in olive. --- changelog.d/20230412_100608_regis_palm.md | 1 + docs/configuration.rst | 2 +- tutor/commands/upgrade/compose.py | 24 ++++---- tutor/commands/upgrade/k8s.py | 67 +++++++++-------------- tutor/templates/config/defaults.yml | 2 +- 5 files changed, 40 insertions(+), 56 deletions(-) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md index d4701286e2d..aff63c34301 100644 --- a/changelog.d/20230412_100608_regis_palm.md +++ b/changelog.d/20230412_100608_regis_palm.md @@ -16,3 +16,4 @@ - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. - [Deprecation] For local installations, Docker v20.10.15 and Compose v2.0.0 are now the minimum required versions. - [Bugfix] Make `tutor config printvalue ...` print actual yaml-formatted values, such as "true" and "null" + - 💥[Improvement] MongoDb was upgraded to 4.4. diff --git a/docs/configuration.rst b/docs/configuration.rst index 02e5ad96e24..4426c5b0614 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -75,7 +75,7 @@ This configuration paramater defines which Caddy Docker image to use. This configuration parameter defines which Elasticsearch Docker image to use. -- ``DOCKER_IMAGE_MONGODB`` (default: ``"docker.io/mongo:4.2.24"``) +- ``DOCKER_IMAGE_MONGODB`` (default: ``"docker.io/mongo:4.4.22"``) This configuration parameter defines which MongoDB Docker image to use. diff --git a/tutor/commands/upgrade/compose.py b/tutor/commands/upgrade/compose.py index 80b0092be25..ea0eaaf9dd4 100644 --- a/tutor/commands/upgrade/compose.py +++ b/tutor/commands/upgrade/compose.py @@ -40,7 +40,7 @@ def upgrade_from(context: click.Context, from_release: str) -> None: running_release = "olive" if running_release == "olive": - upgrade_from_olive(context) + upgrade_from_olive(context, config) running_release = "palm" @@ -51,18 +51,8 @@ def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Stopping any existing platform")) context.invoke(compose.stop) - if not config["RUN_MONGODB"]: - fmt.echo_info( - "You are not running MongoDB (RUN_MONGODB=false). It is your " - "responsibility to upgrade your MongoDb instance to v3.6. There is " - "nothing left to do to upgrade from Ironwood to Juniper." - ) - return - upgrade_mongodb(context, config, "3.4", "3.4") - context.invoke(compose.stop) upgrade_mongodb(context, config, "3.6", "3.6") - context.invoke(compose.stop) def upgrade_from_juniper(context: click.Context, config: Config) -> None: @@ -150,7 +140,7 @@ def upgrade_from_maple(context: click.Context, config: Config) -> None: ) -def upgrade_from_olive(context: click.Context) -> None: +def upgrade_from_olive(context: click.Context, config: Config) -> None: # Note that we need to exec because the ora2 folder is not bind-mounted in the job # services. context.invoke(compose.start, detach=True, services=["lms"]) @@ -158,7 +148,8 @@ def upgrade_from_olive(context: click.Context) -> None: compose.execute, args=["lms", "sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], ) - context.invoke(compose.stop) + upgrade_mongodb(context, config, "4.2.17", "4.2") + upgrade_mongodb(context, config, "4.4.22", "4.4") def upgrade_mongodb( @@ -167,6 +158,13 @@ def upgrade_mongodb( to_docker_version: str, to_compatibility_version: str, ) -> None: + if not config["RUN_MONGODB"]: + fmt.echo_info( + f"You are not running MongoDB (RUN_MONGODB=false). It is your " + f"responsibility to upgrade your MongoDb instance to {to_docker_version}." + ) + return + click.echo(fmt.title(f"Upgrading MongoDb to v{to_docker_version}")) # Note that the DOCKER_IMAGE_MONGODB value is never saved, because we only save the # environment, not the configuration. diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index df83eade168..b84ac4f300f 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -44,30 +44,8 @@ def upgrade_from(context: click.Context, from_release: str) -> None: def upgrade_from_ironwood(config: Config) -> None: - if not config["RUN_MONGODB"]: - fmt.echo_info( - "You are not running MongoDB (RUN_MONGODB=false). It is your " - "responsibility to upgrade your MongoDb instance to v3.6. There is " - "nothing left to do to upgrade from Ironwood." - ) - return - message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Ironwood, you should upgrade -your MongoDb cluster from v3.2 to v3.6. You should run something similar to: - - # Upgrade from v3.2 to v3.4 - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.4.24 - tutor k8s start - tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.4" })' - - # Upgrade from v3.4 to v3.6 - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MONGODB=mongo:3.6.18 - tutor k8s start - tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "3.6" })' - - tutor config save --unset DOCKER_IMAGE_MONGODB""" - fmt.echo_info(message) + upgrade_mongodb(config, "3.4.24", "3.4") + upgrade_mongodb(config, "3.6.18", "3.6") def upgrade_from_juniper(config: Config) -> None: @@ -91,23 +69,7 @@ def upgrade_from_juniper(config: Config) -> None: def upgrade_from_koa(config: Config) -> None: - if not config["RUN_MONGODB"]: - fmt.echo_info( - "You are not running MongoDB (RUN_MONGODB=false). It is your " - "responsibility to upgrade your MongoDb instance to v4.0. There is " - "nothing left to do to upgrade to Lilac from Koa." - ) - return - message = """Automatic release upgrade is unsupported in Kubernetes. To upgrade from Koa to Lilac, you should upgrade -your MongoDb cluster from v3.6 to v4.0. You should run something similar to: - - tutor k8s stop - tutor config save --set DOCKER_IMAGE_MONGODB=mongo:4.0.25 - tutor k8s start - tutor k8s exec mongodb mongo --eval 'db.adminCommand({ setFeatureCompatibilityVersion: "4.0" })' - tutor config save --unset DOCKER_IMAGE_MONGODB - """ - fmt.echo_info(message) + upgrade_mongodb(config, "4.0.25", "4.0") def upgrade_from_lilac(config: Config) -> None: @@ -193,3 +155,26 @@ def upgrade_from_olive(context: Context, config: Config) -> None: "lms", ["sh", "-e", "-c", common_upgrade.PALM_RENAME_ORA2_FOLDER_COMMAND], ) + upgrade_mongodb(config, "4.2.17", "4.2") + upgrade_mongodb(config, "4.4.22", "4.4") + + +def upgrade_mongodb( + config: Config, to_docker_version: str, to_compatibility_version: str +) -> None: + if not config["RUN_MONGODB"]: + fmt.echo_info( + "You are not running MongoDB (RUN_MONGODB=false). It is your " + "responsibility to upgrade your MongoDb instance to {to_docker_version}." + ) + return + message = f"""Automatic release upgrade is unsupported in Kubernetes. You should manually upgrade +your MongoDb cluster to {to_docker_version} by running something similar to: + + tutor k8s stop + tutor config save --set DOCKER_IMAGE_MONGODB=mongo:{to_docker_version} + tutor k8s start + tutor k8s exec mongodb mongo --eval 'db.adminCommand({{ setFeatureCompatibilityVersion: "{to_compatibility_version}" }})' + tutor config save --unset DOCKER_IMAGE_MONGODB + """ + fmt.echo_info(message) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 642d38aef90..afc24673071 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -14,7 +14,7 @@ DOCKER_IMAGE_OPENEDX: "{{ DOCKER_REGISTRY }}overhangio/openedx:{{ TUTOR_VERSION DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" -DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.2.24" +DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.22" DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" From f8eba056f6e194ee488582f6acee6894a69513c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 30 May 2023 16:17:07 +0200 Subject: [PATCH 29/96] fix: double config prompting during upgrade --- tutor/commands/compose.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 8c980507840..8e66c6d46ff 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -93,6 +93,8 @@ def launch( run_for_prod = context_name != "dev" utils.warn_macos_docker_memory() + + # Upgrade has to run before configuration interactive_upgrade(context, not non_interactive, run_for_prod) interactive_configuration(context, not non_interactive, run_for_prod) @@ -163,10 +165,11 @@ def interactive_upgrade( ) # Update env and configuration - interactive_configuration(context, interactive, run_for_prod) + # Don't run in interactive mode, otherwise users gets prompted twice. + interactive_configuration(context, False, run_for_prod) # Post upgrade - if run_upgrade_from_release and interactive: + if interactive: question = f"""Your platform is being upgraded from {run_upgrade_from_release.capitalize()}. If you run custom Docker images, you must rebuild them now by running the following command in a different shell: From 8e7fdd6d02334bb1d0da0658bbd4d517ca65bb73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 14 Jun 2023 19:40:31 +0200 Subject: [PATCH 30/96] v16.0.0 --- CHANGELOG.md | 31 +++++++++++++++++++ .../20230325_205654_regis_permissions.md | 1 - .../20230325_211520_regis_permissions.md | 1 - changelog.d/20230412_100608_regis_palm.md | 19 ------------ .../20230427_121619_regis_config_append.md | 1 - .../20230427_154822_regis_build_mount.md | 1 - .../20230427_165520_regis_build_mount.md | 3 -- changelog.d/20230502_090803_regis_palm.md | 2 -- docs/configuration.rst | 2 +- tutor/templates/config/defaults.yml | 2 +- 10 files changed, 33 insertions(+), 30 deletions(-) delete mode 100644 changelog.d/20230325_205654_regis_permissions.md delete mode 100644 changelog.d/20230325_211520_regis_permissions.md delete mode 100644 changelog.d/20230412_100608_regis_palm.md delete mode 100644 changelog.d/20230427_121619_regis_config_append.md delete mode 100644 changelog.d/20230427_154822_regis_build_mount.md delete mode 100644 changelog.d/20230427_165520_regis_build_mount.md delete mode 100644 changelog.d/20230502_090803_regis_palm.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c9581780fe3..751d448ae54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,37 @@ instructions, because git commits are used to generate release notes: + +## v16.0.0 (2023-06-14) +- 💥[Feature] Upgrade to Palm. (by @regisb) + - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. + - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. + - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. + - 💥[Deprecation] Drop support for `docker-compose`, also known as Compose V1. The `docker compose` (no hyphen) plugin must be installed. + - 💥[Refactor] We simplify the hooks API by getting rid of the `ContextTemplate`, `FilterTemplate` and `ActionTemplate` classes. As a consequences, the following changes occur: + - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. + - Removed the `ENV_PATCH` filter, which was for internal use only anyway. + - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) + - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence, the syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. + - 💥[Deprecation] The obsolete filters `COMMANDS_PRE_INIT` and `COMMANDS_INIT` have been removed. Plugin developers should instead use `CLI_DO_INIT_TASKS` (with suitable priorities). + - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: + - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. + - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. + - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. + - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. + - [Deprecation] For local installations, Docker v20.10.15 and Compose v2.0.0 are now the minimum required versions. + - [Bugfix] Make `tutor config printvalue ...` print actual yaml-formatted values, such as "true" and "null" + - 💥[Improvement] MongoDb was upgraded to 4.4. +- 💥[Improvement] Deprecate the `RUN_LMS` and `RUN_CMS` tutor settings, which should be mostly unused. (by @regisb) +- [Improvement] Greatly simplify ownership of bind-mounted volumes with docker-compose. Instead of running one service per application, we run just a single "permissions" service. This change should be backward-compatible. (by @regisb) +- [Feature] Add a `config save -a/--append -A/--remove` options to conveniently append and remove values to/from list entries. (by @regisb) +- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb) +- [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) +- [Feature] Automatically mount host folders *at build time*. This is a really important feature, as it allows us to transparently build images using local forks of remote repositories. (by @regisb) +- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts, which are managed by the `tutor mounts` commands. (by @regisb) +- [Feature] Add the `do importdemocourse --repo-dir=...` option, to import courses from subdirectories of git repositories. This allows us to import the openedx-test-course in Palm with: `tutor local do importdemocourse --repo=https://github.com/openedx/openedx-test-course --version=o +pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) + ## v15.3.7 (2023-06-13) diff --git a/changelog.d/20230325_205654_regis_permissions.md b/changelog.d/20230325_205654_regis_permissions.md deleted file mode 100644 index 08841ade868..00000000000 --- a/changelog.d/20230325_205654_regis_permissions.md +++ /dev/null @@ -1 +0,0 @@ -- 💥[Improvement] Deprecate the `RUN_LMS` and `RUN_CMS` tutor settings, which should be mostly unused. (by @regisb) diff --git a/changelog.d/20230325_211520_regis_permissions.md b/changelog.d/20230325_211520_regis_permissions.md deleted file mode 100644 index f9e37e92dad..00000000000 --- a/changelog.d/20230325_211520_regis_permissions.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Greatly simplify ownership of bind-mounted volumes with docker-compose. Instead of running one service per application, we run just a single "permissions" service. This change should be backward-compatible. (by @regisb) diff --git a/changelog.d/20230412_100608_regis_palm.md b/changelog.d/20230412_100608_regis_palm.md deleted file mode 100644 index aff63c34301..00000000000 --- a/changelog.d/20230412_100608_regis_palm.md +++ /dev/null @@ -1,19 +0,0 @@ -- 💥[Feature] Upgrade to Palm. (by @regisb) - - [Bugfix] Rename ORA2 file upload folder from "SET-ME-PLEASE (ex. bucket-name)" to "openedxuploads". This has the effect of moving the corresponding folder from the `/data/lms/ora2` directory. MinIO users were not affected by this bug. - - 💥[Improvement] During registration, the honor code and terms of service links are no longer visible by default. For most platforms, these links did not work anyway. - - 💥[Deprecation] Halt support for Python 3.7. The binary release of Tutor is also no longer compatible with macOS 10. - - 💥[Deprecation] Drop support for `docker-compose`, also known as Compose V1. The `docker compose` (no hyphen) plugin must be installed. - - 💥[Refactor] We simplify the hooks API by getting rid of the `ContextTemplate`, `FilterTemplate` and `ActionTemplate` classes. As a consequences, the following changes occur: - - `APP` was previously a ContextTemplate, and is now a dictionary of contexts indexed by name. Developers who implemented this context should replace `Contexts.APP(...)` by `Contexts.app(...)`. - - Removed the `ENV_PATCH` filter, which was for internal use only anyway. - - The `PLUGIN_LOADED` ActionTemplate is now an Action which takes a single argument. (the plugin name) - - 💥[Refactor] We refactored the hooks API further by removing the static hook indexes and the hooks names. As a consequence, the syntactic sugar functions from the "filters" and "actions" modules were all removed: `get`, `add*`, `iterate*`, `apply*`, `do*`, etc. - - 💥[Deprecation] The obsolete filters `COMMANDS_PRE_INIT` and `COMMANDS_INIT` have been removed. Plugin developers should instead use `CLI_DO_INIT_TASKS` (with suitable priorities). - - 💥[Feature] The "openedx" Docker image is no longer built with docker-compose in development on `tutor dev start`. This used to be the case to make sure that it was always up-to-date, but it introduced a discrepancy in how images were build (`docker compose build` vs `docker build`). As a consequence: - - The "openedx" Docker image in development can be built with `tutor images build openedx-dev`. - - The `tutor dev/local start --skip-build` option is removed. It is replaced by opt-in `--build`. - - [Improvement] The `IMAGES_BUILD` filter now supports relative paths as strings, and not just as tuple of strings. - - [Improvement] Auto-complete the image names in the `images build/pull/push/printtag` commands. - - [Deprecation] For local installations, Docker v20.10.15 and Compose v2.0.0 are now the minimum required versions. - - [Bugfix] Make `tutor config printvalue ...` print actual yaml-formatted values, such as "true" and "null" - - 💥[Improvement] MongoDb was upgraded to 4.4. diff --git a/changelog.d/20230427_121619_regis_config_append.md b/changelog.d/20230427_121619_regis_config_append.md deleted file mode 100644 index 6d507918915..00000000000 --- a/changelog.d/20230427_121619_regis_config_append.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] Add a `config save -a/--append -A/--remove` options to conveniently append and remove values to/from list entries. (by @regisb) diff --git a/changelog.d/20230427_154822_regis_build_mount.md b/changelog.d/20230427_154822_regis_build_mount.md deleted file mode 100644 index dbc5e908de1..00000000000 --- a/changelog.d/20230427_154822_regis_build_mount.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb) diff --git a/changelog.d/20230427_165520_regis_build_mount.md b/changelog.d/20230427_165520_regis_build_mount.md deleted file mode 100644 index 88f883dd4b2..00000000000 --- a/changelog.d/20230427_165520_regis_build_mount.md +++ /dev/null @@ -1,3 +0,0 @@ -- [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) -- [Feature] Automatically mount host folders *at build time*. This is a really important feature, as it allows us to transparently build images using local forks of remote repositories. (by @regisb) -- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts, which are managed by the `tutor mounts` commands. (by @regisb) diff --git a/changelog.d/20230502_090803_regis_palm.md b/changelog.d/20230502_090803_regis_palm.md deleted file mode 100644 index 3a1160685b7..00000000000 --- a/changelog.d/20230502_090803_regis_palm.md +++ /dev/null @@ -1,2 +0,0 @@ -- [Feature] Add the `do importdemocourse --repo-dir=...` option, to import courses from subdirectories of git repositories. This allows us to import the openedx-test-course in Palm with: `tutor local do importdemocourse --repo=https://github.com/openedx/openedx-test-course --version=o -pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 4426c5b0614..ca4b978f1a9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.master"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.1"``) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index afc24673071..29996e4ce49 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -52,7 +52,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/palm.master" +OPENEDX_COMMON_VERSION: "open-release/palm.1" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" From bb78030807244d824ad11c6bccddc7f2cc1acd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 15 Jun 2023 01:34:05 +0200 Subject: [PATCH 31/96] docs: olive -> palm mentions --- docs/configuration.rst | 2 +- docs/install.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index ca4b978f1a9..c636939e458 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -401,7 +401,7 @@ Note that your edx-platform version must be a fork of the latest release **tag** If you don't create your fork from this tag, you *will* have important compatibility issues with other services. In particular: -- Do not try to run a fork from an older (pre-Olive) version of edx-platform: this will simply not work. +- Do not try to run a fork from an older (pre-Palm) version of edx-platform: this will simply not work. - Do not try to run a fork from the edx-platform master branch: there is a 99% probability that it will fail. - Do not try to run a fork from the open-release/palm.master branch: Tutor will attempt to apply security and bug fix patches that might already be included in the open-release/palm.master but which were not yet applied to the latest release tag. Patch application will thus fail if you base your fork from the open-release/palm.master branch. diff --git a/docs/install.rst b/docs/install.rst index a9f47cfc4ec..bb52cf003a2 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -121,11 +121,11 @@ Major Open edX releases are published twice a year, in June and December, by the 4. Test the new release in a sandboxed environment. 5. If you are running edx-platform, or some other repository from a custom branch, then you should rebase (and test) your changes on top of the latest release tag (see :ref:`edx_platform_fork`). -The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Nutmeg to Olive and rebuild some Docker images, run:: +The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``launch`` command (see above). The single difference is that if the ``launch`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``launch``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Olive to Palm and rebuild some Docker images, run:: tutor config save tutor images build all # list the images that should be rebuilt here - tutor local upgrade --from=nutmeg + tutor local upgrade --from=olive tutor local launch .. _autocomplete: From 08a45d8255e24230051f767416981d49fee1d8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 16 Jun 2023 11:38:59 +0200 Subject: [PATCH 32/96] fix: load kube config from file Close #860 --- changelog.d/20230616_113533_regis.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/20230616_113533_regis.md diff --git a/changelog.d/20230616_113533_regis.md b/changelog.d/20230616_113533_regis.md new file mode 100644 index 00000000000..82374054039 --- /dev/null +++ b/changelog.d/20230616_113533_regis.md @@ -0,0 +1 @@ +- [Bugfix] Fix loading default Kubernetes config. (by @regisb) From 60aff3b8ea27c7c855b35dd4513057db54b2167f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 16 Jun 2023 11:45:17 +0200 Subject: [PATCH 33/96] v16.0.1 --- CHANGELOG.md | 5 +++++ changelog.d/20230616_113533_regis.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230616_113533_regis.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 751d448ae54..589513af335 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.0.1 (2023-06-16) + +- [Bugfix] Fix loading default Kubernetes config. (by @regisb) + ## v16.0.0 (2023-06-14) - 💥[Feature] Upgrade to Palm. (by @regisb) diff --git a/changelog.d/20230616_113533_regis.md b/changelog.d/20230616_113533_regis.md deleted file mode 100644 index 82374054039..00000000000 --- a/changelog.d/20230616_113533_regis.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Fix loading default Kubernetes config. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 45ab4966d55..e6f268fef80 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.0" +__version__ = "16.0.1" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 314ca6ccaa7e66488cafae912a4660481cea56ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 22 Jun 2023 12:33:57 +0200 Subject: [PATCH 34/96] fix: mysql deployment on k8s The `--ignore-db-dir` option is no longer supported on MySQL 8. See: https://dev.mysql.com/doc/refman/8.0/en/upgrade-prerequisites.html This option was causing the mysql container to fail. --- changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md | 1 + tutor/templates/k8s/deployments.yml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md diff --git a/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md b/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md new file mode 100644 index 00000000000..40c94221312 --- /dev/null +++ b/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md @@ -0,0 +1 @@ +- [Bugfix] On Kubernetes, fix mysql deployment by removing the `-ignore-db-dir` option, which no longer exists on MySQL 8. (by @regisb) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 30c73b8afa0..ee017246ed6 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -392,9 +392,7 @@ spec: containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} - # Note the ignore-db-dir: this is because ext4 volumes are created with a lost+found directory in them, which causes mysql - # initialisation to fail - args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci", "--ignore-db-dir=lost+found"] + args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"] env: - name: MYSQL_ROOT_PASSWORD value: "{{ MYSQL_ROOT_PASSWORD }}" From ec8d0ff7e01559b811a132cca80224ce59f79cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 22 Jun 2023 13:00:17 +0200 Subject: [PATCH 35/96] v16.0.2 --- CHANGELOG.md | 5 +++++ changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 589513af335..c37c90358be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.0.2 (2023-06-22) + +- [Bugfix] On Kubernetes, fix mysql deployment by removing the `--ignore-db-dir` option, which no longer exists on MySQL 8. (by @regisb) + ## v16.0.1 (2023-06-16) diff --git a/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md b/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md deleted file mode 100644 index 40c94221312..00000000000 --- a/changelog.d/20230622_123257_regis_k8s_fix_mysql8_start.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] On Kubernetes, fix mysql deployment by removing the `-ignore-db-dir` option, which no longer exists on MySQL 8. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index e6f268fef80..40bcca6ab48 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.1" +__version__ = "16.0.2" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From b585ec8d237269e6fe9a859c7381e70ca259e443 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 15 Jun 2023 12:06:56 -0700 Subject: [PATCH 36/96] docs: update ARM tutorial --- docs/tutorials/arm64.rst | 37 +++++-------------------------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/docs/tutorials/arm64.rst b/docs/tutorials/arm64.rst index 23beed570fc..2d04f7ae352 100644 --- a/docs/tutorials/arm64.rst +++ b/docs/tutorials/arm64.rst @@ -3,37 +3,10 @@ Running Tutor on ARM-based systems ================================== -Tutor can be used on ARM64 systems, although no official ARM64 docker images are available. If you want to get started quickly, there is `an unofficial community-maintained ARM64 plugin `_ which will set the required settings for you and which includes unofficial docker images. If you prefer not to use an unofficial plugin, you can follow this tutorial. +Tutor can be used on ARM64 systems, and official ARM64 docker images are available starting from Tutor v16. -.. note:: There are generally two ways to run Tutor on an ARM system - using emulation (via qemu or Rosetta 2) to run x86_64 images or running native ARM images. Since emulation can be noticeably slower (typically 20-100% slower depending on the emulation method), this tutorial aims to use native images where possible. +For older versions of Tutor (v14 or v15), there are several options: -Building the images -------------------- - -Although there are no official ARM64 images, Tutor makes it easy to build the images yourself. - -Start by :ref:`installing ` Tutor and its dependencies (e.g. Docker) onto your system. - -.. note:: For Open edX developers, if you want to use the :ref:`nightly ` version of Tutor to "run master", install Tutor using git and check out the ``nightly`` branch of Tutor at this point. See the :ref:`nightly documentation ` for details. - -Next, configure Tutor:: - - tutor config save --interactive - -Go through the configuration process, answering each question. - -Then, build the "openedx" and "permissions" images:: - - tutor images build openedx permissions - -If you want to use Tutor as an Open edX development environment, you should also build the development image:: - - tutor images build openedx-dev # this will be automatically done by `tutor dev launch` - -From this point on, use Tutor as normal. For example, start Open edX and run migrations with:: - - tutor local launch - -Or for a development environment:: - - tutor dev launch +* Use emulation (via qemu or Rosetta 2) to run x86_64 images. Just make sure your installation of Docker supports emulation and use Tutor as normal. This may be 20%-100% slower than native images, depending on the emulation method. +* Use the `unofficial community-maintained ARM64 plugin `_ which will set the required settings for you and which includes unofficial docker images. +* Build your own ARM64 images, e.g. using ``tutor images build openedx permissions`` and/or ``tutor images build openedx-dev`` before launching the LMS. From 3a2d004fd00273f47c35fe8a80dbe4300a51f3b7 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Wed, 7 Jun 2023 15:05:16 -0400 Subject: [PATCH 37/96] docs: Tweak upgrade instructions --- docs/install.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/install.rst b/docs/install.rst index bb52cf003a2..0c39f1e2ebe 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -87,7 +87,9 @@ Tutor can be launched on Amazon Web Services very quickly with the `official Tut Upgrading --------- -To upgrade Open edX or benefit from the latest features and bug fixes, you should simply upgrade Tutor. Start by upgrading the "tutor" package and its dependencies:: +To upgrade your Open edX site or benefit from the latest features and bug fixes, you should simply upgrade Tutor. Start by backing up your data and reading the `release notes `_ for the current release. + +Next, upgrade the "tutor" package and its dependencies:: pip install --upgrade "tutor[full]" From 1314b5e80165b90acf43cce3532c76f5917f4e59 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 14 Jul 2023 10:01:43 -0400 Subject: [PATCH 38/96] fix: set default theme by simply deleting SiteTheme objects `tutor ... do settheme default` is meant to revert to the default theme. However, in its current implementation, it creates SiteTheme objects pointing to a theme named "default", which doesn't exist, resulting in errors like: Theme dirs: [Path('/openedx/themes')]] Traceback (most recent call last): File "/openedx/edx-platform/openedx/core/djangoapps/theming/helpers.py", line 204, in get_current_theme themes_base_dir=get_theme_base_dir(site_theme.theme_dir_name), File "/openedx/edx-platform/openedx/core/djangoapps/theming/helpers.py", line 242, in get_theme_base_dir raise ValueError( ValueError: Theme 'default' not found in any of the following themes dirs, This works from the perspective of the user, because a missing theme is treated as the default theme. However, the errors are unneccesary & confusing. By simply deleting & not recreating SiteTheme objects instead, we are able to revert to the default theme while keeping the logs clear of theming errors. --- changelog.d/20230714_100627_kyle.md | 1 + tutor/commands/jobs.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20230714_100627_kyle.md diff --git a/changelog.d/20230714_100627_kyle.md b/changelog.d/20230714_100627_kyle.md new file mode 100644 index 00000000000..97a1722ef90 --- /dev/null +++ b/changelog.d/20230714_100627_kyle.md @@ -0,0 +1 @@ +- [Bugfix] Improve `tutor ... do settheme default` so that it reverts to the default theme rather than trying to switch to a nonexistent theme named "default". This will clear up some error noise from LMS/CMS logs. (by @kdmccormick) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index a95bf075c19..eb356834a9d 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -216,7 +216,8 @@ def assign_theme(name, domain): site.name = domain[:name_max_length] site.save() site.themes.all().delete() - site.themes.create(theme_dir_name=name) + if name != 'default': + site.themes.create(theme_dir_name=name) """ domain_names = domain_names or [ "{{ LMS_HOST }}", From 771ea67881d8d256e23f2975e24fafe9b3f41e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 21 Jul 2023 12:11:36 +0200 Subject: [PATCH 39/96] chore: upgrade cryptography requirement Apply security update: https://github.com/overhangio/tutor/security/dependabot/14 --- requirements/base.txt | 2 +- requirements/dev.txt | 2 +- requirements/docs.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index fa904772bb2..ef14424fbd6 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile requirements/base.in +# pip-compile --config=pyproject.toml requirements/base.in # appdirs==1.4.4 # via -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 6e1addc4849..6689b858508 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile requirements/dev.in +# pip-compile --config=pyproject.toml requirements/dev.in # altgraph==0.17.3 # via pyinstaller diff --git a/requirements/docs.txt b/requirements/docs.txt index 016cb3d244f..cc0ab3dd156 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile requirements/docs.in +# pip-compile --config=pyproject.toml requirements/docs.in # alabaster==0.7.13 # via sphinx From 76ce13a4c9d372c13b9f675aa64c4b73b0f63e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 21 Jul 2023 12:45:12 +0200 Subject: [PATCH 40/96] chore: actually do upgrade cryptography req See: https://github.com/overhangio/tutor/security/dependabot/14 --- requirements/base.txt | 2 +- requirements/dev.txt | 4 ++-- requirements/docs.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index ef14424fbd6..fa904772bb2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --config=pyproject.toml requirements/base.in +# pip-compile requirements/base.in # appdirs==1.4.4 # via -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 6689b858508..316b9075dd8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --config=pyproject.toml requirements/dev.in +# pip-compile requirements/dev.in # altgraph==0.17.3 # via pyinstaller @@ -44,7 +44,7 @@ click-log==0.4.0 # via scriv coverage==7.2.7 # via -r requirements/dev.in -cryptography==41.0.1 +cryptography==41.0.2 # via secretstorage dill==0.3.6 # via pylint diff --git a/requirements/docs.txt b/requirements/docs.txt index cc0ab3dd156..016cb3d244f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --config=pyproject.toml requirements/docs.in +# pip-compile requirements/docs.in # alabaster==0.7.13 # via sphinx From 9cc0677d7298a8bd3ca53372c24cdfd924cceee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Jul 2023 21:04:26 +0200 Subject: [PATCH 41/96] security: fix unprivileged content libraries creation See: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j https://github.com/openedx/edx-platform/pull/32838 https://discuss.openedx.org/t/security-upcoming-security-release-for-edx-platform-on-2023-07-25/10769 --- changelog.d/20230728_210255_regis.md | 1 + tutor/templates/build/openedx/Dockerfile | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20230728_210255_regis.md diff --git a/changelog.d/20230728_210255_regis.md b/changelog.d/20230728_210255_regis.md new file mode 100644 index 00000000000..fc8e9854e1c --- /dev/null +++ b/changelog.d/20230728_210255_regis.md @@ -0,0 +1 @@ +- [Security] Fix content libraries creation by unprivileged users in studio (see [security advisory](https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j)). (by @regisb) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 86006a8751d..9b8ce81632d 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -50,6 +50,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform +# Security advisory: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j +# https://github.com/openedx/edx-platform/pull/32838 +RUN curl -fsSL https://github.com/openedx/edx-platform/commit/163259779297a7dccb28e1f8c3dfa4d2cbdb9655.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} From 251792765fcff20c5d30dcdb95d7df18b94d8943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 28 Jul 2023 21:56:37 +0200 Subject: [PATCH 42/96] v16.0.3 --- CHANGELOG.md | 6 ++++++ changelog.d/20230714_100627_kyle.md | 1 - changelog.d/20230728_210255_regis.md | 1 - tutor/__about__.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20230714_100627_kyle.md delete mode 100644 changelog.d/20230728_210255_regis.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c37c90358be..8c044d0ab85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ instructions, because git commits are used to generate release notes: + +## v16.0.3 (2023-07-28) + +- [Bugfix] Improve `tutor ... do settheme default` so that it reverts to the default theme rather than trying to switch to a nonexistent theme named "default". This will clear up some error noise from LMS/CMS logs. (by @kdmccormick) +- [Security] Fix content libraries creation by unprivileged users in studio (see [security advisory](https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j)). (by @regisb) + ## v16.0.2 (2023-06-22) diff --git a/changelog.d/20230714_100627_kyle.md b/changelog.d/20230714_100627_kyle.md deleted file mode 100644 index 97a1722ef90..00000000000 --- a/changelog.d/20230714_100627_kyle.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Improve `tutor ... do settheme default` so that it reverts to the default theme rather than trying to switch to a nonexistent theme named "default". This will clear up some error noise from LMS/CMS logs. (by @kdmccormick) diff --git a/changelog.d/20230728_210255_regis.md b/changelog.d/20230728_210255_regis.md deleted file mode 100644 index fc8e9854e1c..00000000000 --- a/changelog.d/20230728_210255_regis.md +++ /dev/null @@ -1 +0,0 @@ -- [Security] Fix content libraries creation by unprivileged users in studio (see [security advisory](https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j)). (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 40bcca6ab48..31820cacea4 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.2" +__version__ = "16.0.3" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 7bc14a6c385aa06b8b87e9f3460db7d28ec41b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 31 Jul 2023 09:22:04 +0200 Subject: [PATCH 43/96] chore: upgrade certifi Fix minor vulnerability: https://github.com/overhangio/tutor/security/dependabot/17 --- requirements/base.txt | 2 +- requirements/dev.txt | 2 +- requirements/docs.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index fa904772bb2..228782bb3ce 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,7 +8,7 @@ appdirs==1.4.4 # via -r requirements/base.in cachetools==5.3.1 # via google-auth -certifi==2023.5.7 +certifi==2023.7.22 # via # kubernetes # requests diff --git a/requirements/dev.txt b/requirements/dev.txt index 316b9075dd8..42829e5c087 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -22,7 +22,7 @@ cachetools==5.3.1 # via # -r requirements/base.txt # google-auth -certifi==2023.5.7 +certifi==2023.7.22 # via # -r requirements/base.txt # kubernetes diff --git a/requirements/docs.txt b/requirements/docs.txt index 016cb3d244f..cfa5d7195b5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -14,7 +14,7 @@ cachetools==5.3.1 # via # -r requirements/base.txt # google-auth -certifi==2023.5.7 +certifi==2023.7.22 # via # -r requirements/base.txt # kubernetes From f31969d0e1c5bce8f049e2b77604f5cdf2e8b076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 31 Jul 2023 15:52:13 +0200 Subject: [PATCH 44/96] docs: fix many verbatim issues in catalog --- tutor/hooks/catalog.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index b3b80a94c6c..20d3e32ec29 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -143,13 +143,13 @@ def your_filter_callback(some_data): Many filters have a list of items as the first argument. Quite often, plugin developers just want to add a new item at the end of that list. In such cases there - is no need for a callback function. Instead, you can use the `add_item` method. For + is no need for a callback function. Instead, you can use the ``add_item`` method. For instance, you can add a "hello" to the init task of the lms container by modifying the :py:data:`CLI_DO_INIT_TASKS` filter:: hooks.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) - To add multiple items at a time, use `add_items`:: + To add multiple items at a time, use ``add_items``:: hooks.CLI_DO_INIT_TASKS.add_items( ("lms", "echo 'hello from lms'"), @@ -157,7 +157,7 @@ def your_filter_callback(some_data): ) The ``echo`` commands will then be run every time the "init" tasks are run, for - instance during `tutor local launch`. + instance during ``tutor local launch``. For more information about how filters work, check out the :py:class:`tutor.core.hooks.Filter` API. @@ -178,17 +178,17 @@ def your_filter_callback(some_data): #: all be added as subcommands of the main ``tutor`` command. CLI_COMMANDS: Filter[list[click.Command], []] = Filter() - #: List of `do ...` commands. + #: List of ``do ...`` commands. #: #: :parameter list commands: see :py:data:`CLI_COMMANDS`. These commands will be - #: added as subcommands to the `local/dev/k8s do` commands. They must return a list of + #: added as subcommands to the ``local/dev/k8s do`` commands. They must return a list of #: ("service name", "service command") tuples. Each "service command" will be executed #: in the "service" container, both in local, dev and k8s mode. CLI_DO_COMMANDS: Filter[ list[Callable[[Any], Iterable[tuple[str, str]]]], [] ] = Filter() - #: List of initialization tasks (scripts) to be run in the `init` job. This job + #: List of initialization tasks (scripts) to be run in the ``init`` job. This job #: includes all database migrations, setting up, etc. To run some tasks before or #: after others, they should be assigned a different priority. #: @@ -325,13 +325,13 @@ def your_filter_callback(some_data): #: #: Out of the box, this filter will include all configuration settings, but also the following: #: - #: - `HOST_USER_ID`: the numerical ID of the user on the host. - #: - `TUTOR_APP`: the app name ("tutor" by default), used to determine the dev/local project names. - #: - `TUTOR_VERSION`: the current version of Tutor. - #: - `is_buildkit_enabled`: a boolean function that indicates whether BuildKit is available on the host. - #: - `iter_values_named`: a function to iterate on variables that start or end with a given string. - #: - `iter_mounts`: a function that yields compose-compatible bind-mounts for any given service. - #: - `patch`: a function to incorporate extra content into a template. + #: - ``HOST_USER_ID``: the numerical ID of the user on the host. + #: - ``TUTOR_APP``: the app name ("tutor" by default), used to determine the dev/local project names. + #: - ``TUTOR_VERSION``: the current version of Tutor. + #: - ``is_buildkit_enabled``: a boolean function that indicates whether BuildKit is available on the host. + #: - ``iter_values_named``: a function to iterate on variables that start or end with a given string. + #: - ``iter_mounts``: a function that yields compose-compatible bind-mounts for any given service. + #: - ``patch``: a function to incorporate extra content into a template. #: #: :parameter filters: list of (name, value) tuples. ENV_TEMPLATE_VARIABLES: Filter[list[tuple[str, Any]], []] = Filter() @@ -391,7 +391,7 @@ def your_filter_callback(some_data): #: Parameters are the same as for :py:data:`IMAGES_PULL`. IMAGES_PUSH: Filter[list[tuple[str, str]], [Config]] = Filter() - #: List of plugin indexes that are loaded when we run `tutor plugins update`. By + #: List of plugin indexes that are loaded when we run ``tutor plugins update``. By #: default, the plugin indexes are stored in the user configuration. This filter makes #: it possible to extend and modify this list with plugins. #: @@ -402,11 +402,11 @@ def your_filter_callback(some_data): #: Filter to modify the url of a plugin index url. This is convenient to alias #: plugin indexes with a simple name, such as "main" or "contrib". #: - #: :parameter str url: value passed to the `index add/remove` commands. + #: :parameter str url: value passed to the ``index add/remove`` commands. PLUGIN_INDEX_URL: Filter[str, []] = Filter() #: When installing an entry from a plugin index, the plugin data from the index will - #: go through this filter before it is passed along to `pip install`. Thus, this is a + #: go through this filter before it is passed along to ``pip install``. Thus, this is a #: good place to add custom authentication when you need to install from a private #: index. #: From 6fc4aaa1b665680c5dca4474f5f5b26f2a170c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 31 Jul 2023 15:55:17 +0200 Subject: [PATCH 45/96] fix: ignore discussion units when forum is not enabled This is a backport of a commit to the master branch: https://github.com/openedx/edx-platform/pull/32464 In particular, it will fix many issues that appear when the demo course is imported. --- changelog.d/20230731_155418_regis_fix_discussion_units.md | 1 + tutor/templates/build/openedx/Dockerfile | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20230731_155418_regis_fix_discussion_units.md diff --git a/changelog.d/20230731_155418_regis_fix_discussion_units.md b/changelog.d/20230731_155418_regis_fix_discussion_units.md new file mode 100644 index 00000000000..55ad28497e2 --- /dev/null +++ b/changelog.d/20230731_155418_regis_fix_discussion_units.md @@ -0,0 +1 @@ +- [Bugfix] Do not display discussion units when the forum is not enabled. (by @regisb) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 9b8ce81632d..7abded50525 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -53,6 +53,9 @@ RUN git config --global user.email "tutor@overhang.io" \ # Security advisory: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j # https://github.com/openedx/edx-platform/pull/32838 RUN curl -fsSL https://github.com/openedx/edx-platform/commit/163259779297a7dccb28e1f8c3dfa4d2cbdb9655.patch | git am +# Fix discussion units when forum is not enabled +# https://github.com/openedx/edx-platform/pull/32464 +RUN curl -fsSL https://github.com/openedx/edx-platform/commit/a9f66705503288c360055ab80c7c3bfb884f75fe.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} From 393cb0cd8c5f63a12f7d281d4eba437a54e18e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 23 Jun 2023 17:04:58 +0200 Subject: [PATCH 46/96] feat: add support for http/3 It was observed that waiting time was cut in half after http/3 was enabled. Plus, supporting http/3 is super easy :) Close #845 --- changelog.d/20230623_170336_regis_http3.md | 1 + docs/tutorials/proxy.rst | 1 + tutor/templates/k8s/services.yml | 5 +++++ tutor/templates/local/docker-compose.prod.yml | 6 +++++- 4 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20230623_170336_regis_http3.md diff --git a/changelog.d/20230623_170336_regis_http3.md b/changelog.d/20230623_170336_regis_http3.md new file mode 100644 index 00000000000..6f3aef411e4 --- /dev/null +++ b/changelog.d/20230623_170336_regis_http3.md @@ -0,0 +1 @@ +- [Feature] Add support for HTTP/3, which considerably improves performance for Open edX. (by @regisb and @ghassanmas) diff --git a/docs/tutorials/proxy.rst b/docs/tutorials/proxy.rst index b1bfc62f1f1..61122667b24 100644 --- a/docs/tutorials/proxy.rst +++ b/docs/tutorials/proxy.rst @@ -26,6 +26,7 @@ It is then your responsibility to configure the web proxy on the host. There are - Forward http traffic to https. - Set the following headers appropriately: ``X-Forwarded-Proto``, ``X-Forwarded-Port``. - Forward all traffic to ``localhost:81`` (or whatever port indicated by CADDY_HTTP_PORT, see above). +- If possible, add support for `HTTP/3 `__, which considerably improves performance for Open edX (see `this comment `__). .. note:: If you want to run Open edX at ``https://...`` urls (as you probably do in production) it is *crucial* that the ``ENABLE_HTTPS`` flag is set to ``true``. If not, the web services will be configured to run at ``http://...`` URLs, and all sorts of trouble will happen. Therefore, make sure to continue answering ``y`` ("yes") to the quickstart dialogue question "Activate SSL/TLS certificates for HTTPS access?". diff --git a/tutor/templates/k8s/services.yml b/tutor/templates/k8s/services.yml index aaaa9b67d2d..c34d2255d88 100644 --- a/tutor/templates/k8s/services.yml +++ b/tutor/templates/k8s/services.yml @@ -14,7 +14,12 @@ spec: name: http {%- if ENABLE_HTTPS %} - port: 443 + protocol: TCP name: https + # include support for http/3 + - port: 443 + protocol: UDP + name: http3 {%- endif %} selector: app.kubernetes.io/name: caddy diff --git a/tutor/templates/local/docker-compose.prod.yml b/tutor/templates/local/docker-compose.prod.yml index b803b41afb6..552fa44dd0d 100644 --- a/tutor/templates/local/docker-compose.prod.yml +++ b/tutor/templates/local/docker-compose.prod.yml @@ -6,7 +6,11 @@ services: restart: unless-stopped ports: - "{{ CADDY_HTTP_PORT }}:80" - {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %}- "443:443"{% endif %} + {% if ENABLE_HTTPS and ENABLE_WEB_PROXY %} + - "443:443" + # include support for http/3 + - "443:443/udp" + {% endif %} environment: default_site_port: "{% if not ENABLE_HTTPS or not ENABLE_WEB_PROXY %}:80{% endif %}" volumes: From 5583e8cf94fe7673c786b5e1905566612dbd85e9 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Thu, 3 Aug 2023 12:38:35 +0330 Subject: [PATCH 47/96] fix: remove references to the wizard edition --- README.rst | 1 - changelog.d/20230801_201804_codewithemad_remove_wizard.md | 1 + docs/faq.rst | 2 +- docs/plugins/intro.rst | 5 ++--- tutor/commands/plugins.py | 4 ++-- tutor/plugins/indexes.py | 1 - 6 files changed, 6 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20230801_201804_codewithemad_remove_wizard.md diff --git a/README.rst b/README.rst index d3c0df7997c..71cfe341367 100644 --- a/README.rst +++ b/README.rst @@ -51,7 +51,6 @@ Features * Comes with batteries included: `theming `__, `SCORM `__, `HTTPS `__, `web-based administration interface `__, `mobile app `__, `custom translations `__... * Extensible architecture with `plugins `__ * Works with `Kubernetes `__ -* Amazing premium plugins available in the `Tutor Wizard Edition `__, including `Cairn `__ the next-generation analytics solution for Open edX. * No technical skill required with the `zero-click Tutor AWS image `__ .. _readme_intro_end: diff --git a/changelog.d/20230801_201804_codewithemad_remove_wizard.md b/changelog.d/20230801_201804_codewithemad_remove_wizard.md new file mode 100644 index 00000000000..d007de980ef --- /dev/null +++ b/changelog.d/20230801_201804_codewithemad_remove_wizard.md @@ -0,0 +1 @@ + - [Improvement] Wizard references removed. (by @CodeWithEmad) \ No newline at end of file diff --git a/docs/faq.rst b/docs/faq.rst index a2fc8d12faf..b8a169f75d8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -45,7 +45,7 @@ What features are missing from Tutor? Tutor tries very hard to support all major Open edX features, notably in the form of :ref:`plugins `. If you are interested in sponsoring the development of a new plugin, please `get in touch `__! -It should be noted that the `Insights `__ stack is currently unsupported, because of its complexity, lack of support, and extensibility. To replace it, Overhang.IO developed `Cairn `__ the next-generation analytics solution for Open edX, part of the `Tutor Wizard Edition `__. You should check it out 😉 +It should be noted that the `Insights `__ stack is currently unsupported, because of its complexity, lack of support, and extensibility. To replace it, Overhang.IO developed `Cairn `__ the next-generation analytics solution for Open edX. You should check it out 😉 Are there people already running this in production? ---------------------------------------------------- diff --git a/docs/plugins/intro.rst b/docs/plugins/intro.rst index bc55b0a74ee..cb75f63660b 100644 --- a/docs/plugins/intro.rst +++ b/docs/plugins/intro.rst @@ -45,13 +45,12 @@ Many plugins are available from plugin indexes. These indexes are lists of plugi tutor plugins update tutor plugins search -More plugins can be downloaded from the "contrib" and "wizard" indexes:: +More plugins can be downloaded from the "contrib" index:: tutor plugins index add contrib - tutor plugins index add wizard tutor plugins search -The "main", "contrib" and "wizard" indexes include a curated list of plugins that are well maintained and introduce useful features to Open edX. These indexes are maintained by `Overhang.IO `__. For more information about these indexes, refer to the official `overhangio/tpi `__ repository. +The "main" and "contrib" indexes include a curated list of plugins that are well maintained and introduce useful features to Open edX. These indexes are maintained by `Overhang.IO `__. For more information about these indexes, refer to the official `overhangio/tpi `__ repository. Thanks to these indexes, it is very easy to download and upgrade plugins. For instance, to install the `notes plugin `__:: diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 0ed848e6757..e80b6fc33eb 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -382,8 +382,8 @@ def index_add(context: Context, url: str) -> None: The index URL will be appended with '{version}/plugins.yml'. The index path can be either an http(s) url or a local file path. - For official indexes, there is no need to pass a full URL. Instead, use "main", - "contrib" or "wizard". + For official indexes, there is no need to pass a full URL. Instead, use "main" or + "contrib". """ config = tutor_config.load_minimal(context.root) if indexes.add(url, config): diff --git a/tutor/plugins/indexes.py b/tutor/plugins/indexes.py index 2123b0ed035..c32d1117af3 100644 --- a/tutor/plugins/indexes.py +++ b/tutor/plugins/indexes.py @@ -31,7 +31,6 @@ def _get_index_url_from_alias(url: str) -> str: known_aliases = { "main": "https://overhang.io/tutor/main", "contrib": "https://overhang.io/tutor/contrib", - "wizard": "https://overhang.io/tutor/wizard", } return known_aliases.get(url, url) From 2121d3fdde1c71f568243bee28dde05abefda690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 3 Aug 2023 11:13:45 +0200 Subject: [PATCH 48/96] chore: upgrade cryptography in dev See: https://github.com/overhangio/tutor/security/dependabot/18 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 42829e5c087..c2400af9fac 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -44,7 +44,7 @@ click-log==0.4.0 # via scriv coverage==7.2.7 # via -r requirements/dev.in -cryptography==41.0.2 +cryptography==41.0.3 # via secretstorage dill==0.3.6 # via pylint From f66967c394c17c9b799d0bf4f6fe5548ccdfd3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 3 Aug 2023 11:19:17 +0200 Subject: [PATCH 49/96] v16.0.4 --- CHANGELOG.md | 7 +++++++ changelog.d/20230623_170336_regis_http3.md | 1 - changelog.d/20230731_155418_regis_fix_discussion_units.md | 1 - changelog.d/20230801_201804_codewithemad_remove_wizard.md | 1 - tutor/__about__.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/20230623_170336_regis_http3.md delete mode 100644 changelog.d/20230731_155418_regis_fix_discussion_units.md delete mode 100644 changelog.d/20230801_201804_codewithemad_remove_wizard.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c044d0ab85..deda386c69f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ instructions, because git commits are used to generate release notes: + +## v16.0.4 (2023-08-03) + +- [Feature] Add support for HTTP/3, which considerably improves performance for Open edX. (by @regisb and @ghassanmas) +- [Bugfix] Do not display discussion units when the forum is not enabled. (by @regisb) +- [Improvement] Remove references to the wizard edition. (by @CodeWithEmad) + ## v16.0.3 (2023-07-28) diff --git a/changelog.d/20230623_170336_regis_http3.md b/changelog.d/20230623_170336_regis_http3.md deleted file mode 100644 index 6f3aef411e4..00000000000 --- a/changelog.d/20230623_170336_regis_http3.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] Add support for HTTP/3, which considerably improves performance for Open edX. (by @regisb and @ghassanmas) diff --git a/changelog.d/20230731_155418_regis_fix_discussion_units.md b/changelog.d/20230731_155418_regis_fix_discussion_units.md deleted file mode 100644 index 55ad28497e2..00000000000 --- a/changelog.d/20230731_155418_regis_fix_discussion_units.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Do not display discussion units when the forum is not enabled. (by @regisb) diff --git a/changelog.d/20230801_201804_codewithemad_remove_wizard.md b/changelog.d/20230801_201804_codewithemad_remove_wizard.md deleted file mode 100644 index d007de980ef..00000000000 --- a/changelog.d/20230801_201804_codewithemad_remove_wizard.md +++ /dev/null @@ -1 +0,0 @@ - - [Improvement] Wizard references removed. (by @CodeWithEmad) \ No newline at end of file diff --git a/tutor/__about__.py b/tutor/__about__.py index 31820cacea4..194be8aa3b7 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.3" +__version__ = "16.0.4" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From ac6fa69f0c17c593790033bcc052063d2712ee84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 9 Aug 2023 22:41:34 +0200 Subject: [PATCH 50/96] feat: upgrade to open-release/palm.2 --- changelog.d/20230809_221139_regis.md | 1 + docs/configuration.rst | 2 +- tutor/templates/build/openedx/Dockerfile | 6 ------ tutor/templates/config/defaults.yml | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) create mode 100644 changelog.d/20230809_221139_regis.md diff --git a/changelog.d/20230809_221139_regis.md b/changelog.d/20230809_221139_regis.md new file mode 100644 index 00000000000..dce9938be2c --- /dev/null +++ b/changelog.d/20230809_221139_regis.md @@ -0,0 +1 @@ +- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index c636939e458..15543af1ab8 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.1"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.2"``) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 7abded50525..86006a8751d 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -50,12 +50,6 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Security advisory: https://github.com/openedx/edx-platform/security/advisories/GHSA-3q74-3rfh-g37j -# https://github.com/openedx/edx-platform/pull/32838 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/163259779297a7dccb28e1f8c3dfa4d2cbdb9655.patch | git am -# Fix discussion units when forum is not enabled -# https://github.com/openedx/edx-platform/pull/32464 -RUN curl -fsSL https://github.com/openedx/edx-platform/commit/a9f66705503288c360055ab80c7c3bfb884f75fe.patch | git am {%- endif %} {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 29996e4ce49..5a8399b3f69 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -52,7 +52,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/palm.1" +OPENEDX_COMMON_VERSION: "open-release/palm.2" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" From 9e770dd57dd259a69b82b634f7fc011686a0effe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 9 Aug 2023 22:42:32 +0200 Subject: [PATCH 51/96] v16.0.5 --- CHANGELOG.md | 5 +++++ changelog.d/20230809_221139_regis.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230809_221139_regis.md diff --git a/CHANGELOG.md b/CHANGELOG.md index deda386c69f..96d429ced49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.0.5 (2023-08-09) + +- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) + ## v16.0.4 (2023-08-03) diff --git a/changelog.d/20230809_221139_regis.md b/changelog.d/20230809_221139_regis.md deleted file mode 100644 index dce9938be2c..00000000000 --- a/changelog.d/20230809_221139_regis.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Upgrade the Open edX default version to open-release/palm.2. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 194be8aa3b7..24e0913309e 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.4" +__version__ = "16.0.5" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From e7838ae27f416e1685ddff1b8e330ea1ce5df7a3 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Fri, 11 Aug 2023 14:17:03 +0330 Subject: [PATCH 52/96] chore: left out "Filters" word added --- tutor/hooks/catalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 20d3e32ec29..e1eac6d71f7 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -147,11 +147,11 @@ def your_filter_callback(some_data): instance, you can add a "hello" to the init task of the lms container by modifying the :py:data:`CLI_DO_INIT_TASKS` filter:: - hooks.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) + hooks.Filters.CLI_DO_INIT_TASKS.add_item(("lms", "echo hello")) To add multiple items at a time, use ``add_items``:: - hooks.CLI_DO_INIT_TASKS.add_items( + hooks.Filters.CLI_DO_INIT_TASKS.add_items( ("lms", "echo 'hello from lms'"), ("cms", "echo 'hello from cms'"), ) From 073fbb354fb4f675532ec1db6c5353bc6600c1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 31 Jul 2023 11:13:13 +0200 Subject: [PATCH 53/96] fix: improve support of non-buildkit Docker build See comment here: https://github.com/overhangio/tutor/pull/868#issuecomment-1640429396 See also the conversation that spawned this PR: https://discuss.openedx.org/t/issue-in-tutor-palm-release-with-tuotr-dev-launch-while-installing/10629 --- ...731_110301_regis_fix_non_buildkit_build.md | 4 +++ tests/commands/test_images.py | 1 - tutor/commands/images.py | 28 ++++++++++++------- tutor/templates/build/openedx/Dockerfile | 7 +++++ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 changelog.d/20230731_110301_regis_fix_non_buildkit_build.md diff --git a/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md b/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md new file mode 100644 index 00000000000..74c9b734a9d --- /dev/null +++ b/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md @@ -0,0 +1,4 @@ +- [Improvement] Improve support of legacy non-BuildKit mode: (by @regisb) + - [Bugfix] Fix building of openedx Docker image. + - [Improvement] Remove `--cache-from` build option. + - [Improvement] Add a warning concerning the lack of support of the `--build-context` option. diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index f3e0132b617..cd0545ae9cf 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -147,7 +147,6 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "--target", "target", "docker_args", - "--cache-from=type=registry,ref=service1:1.0.0-cache", ], list(image_build.call_args[0][1:]), ) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index 0108fed4050..d21e2801cab 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -223,16 +223,27 @@ def build( image_build_args = [*command_args, *custom_args] # Registry cache - if not no_registry_cache: - image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") - if cache_to_registry: - image_build_args.append( - f"--cache-to=type=registry,mode=max,ref={tag}-cache" - ) + if utils.is_buildkit_enabled(): + if not no_registry_cache: + image_build_args.append( + f"--cache-from=type=registry,ref={tag}-cache" + ) + if cache_to_registry: + image_build_args.append( + f"--cache-to=type=registry,mode=max,ref={tag}-cache" + ) # Build contexts for host_path, stage_name in build_contexts.get(name, []): - image_build_args.append(f"--build-context={stage_name}={host_path}") + if utils.is_buildkit_enabled(): + fmt.echo_info( + f"Adding {host_path} to the build context '{stage_name}' of image '{image}'" + ) + image_build_args.append(f"--build-context={stage_name}={host_path}") + else: + fmt.echo_alert( + f"Unable to add {host_path} to the build context '{stage_name}' of image '{host_path}' because BuildKit is disabled." + ) # Build images.build( @@ -257,9 +268,6 @@ def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]] for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( user_mount ): - fmt.echo_info( - f"Adding {user_mount} to the build context '{stage_name}' of image '{image_name}'" - ) if image_name not in build_contexts: build_contexts[image_name] = [] build_contexts[image_name].append((user_mount, stage_name)) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 86006a8751d..781f48ba0e4 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -92,6 +92,9 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip, RUN pip install https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # Install base requirements +{% if not is_buildkit_enabled() %} +COPY --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt +{% endif %} RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt @@ -126,6 +129,10 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform +{% if not is_buildkit_enabled() %} +COPY --from=edx-platform /package.json /openedx/edx-platform/package.json +COPY --from=edx-platform /package-lock.json /openedx/edx-platform/package-lock.json +{% endif %} RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --no-audit --registry=$NPM_REGISTRY From 0e7374700fa4c540cbf9c426ae4017c7d26cc572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 16 Aug 2023 18:46:28 +0200 Subject: [PATCH 54/96] fix: ask whether we run as prod in `local launch` User was no longer asked whether they wanted to run on prod or not. In other words, it was not convenient to run as local.overhang.io. --- .../20230816_184458_regis_fix_local_non_prod.md | 1 + tutor/commands/compose.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20230816_184458_regis_fix_local_non_prod.md diff --git a/changelog.d/20230816_184458_regis_fix_local_non_prod.md b/changelog.d/20230816_184458_regis_fix_local_non_prod.md new file mode 100644 index 00000000000..7f80b451f83 --- /dev/null +++ b/changelog.d/20230816_184458_regis_fix_local_non_prod.md @@ -0,0 +1 @@ +- [Bugfix] Ask whether user wants to run locally during `tutor local launch`. (by @regisb) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 8e66c6d46ff..baa1ca00169 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -90,13 +90,13 @@ def launch( skip_build: bool, ) -> None: context_name = context.obj.NAME - run_for_prod = context_name != "dev" + run_for_prod = False if context_name == "dev" else None utils.warn_macos_docker_memory() # Upgrade has to run before configuration - interactive_upgrade(context, not non_interactive, run_for_prod) - interactive_configuration(context, not non_interactive, run_for_prod) + interactive_upgrade(context, not non_interactive, run_for_prod=run_for_prod) + interactive_configuration(context, not non_interactive, run_for_prod=run_for_prod) config = tutor_config.load(context.obj.root) @@ -136,7 +136,7 @@ def launch( def interactive_upgrade( - context: click.Context, interactive: bool, run_for_prod: bool + context: click.Context, interactive: bool, run_for_prod: t.Optional[bool] ) -> None: """ Piece of code that is only used in launch. @@ -187,7 +187,7 @@ def interactive_upgrade( def interactive_configuration( - context: click.Context, interactive: bool, run_for_prod: bool + context: click.Context, interactive: bool, run_for_prod: t.Optional[bool] = None ) -> None: click.echo(fmt.title("Interactive platform configuration")) config = tutor_config.load_minimal(context.obj.root) From 14f07bc04084df97261875d88d54cf71a4914f24 Mon Sep 17 00:00:00 2001 From: 0x29a Date: Fri, 14 Jul 2023 15:02:26 +0200 Subject: [PATCH 55/96] fix: race condition could cause mkdirs() to fail with "dir exists" --- CHANGELOG.md | 1 + tutor/templates/apps/openedx/settings/partials/common_cms.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96d429ced49..837f0fe2610 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -253,6 +253,7 @@ pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) - [Bugfix] Build openedx-dev Docker image even when the host user is root, for instance on Windows. (by @regisb) - [Bugfix] Patch nutmeg.1 release with [LTI 1.3 fix](https://github.com/openedx/edx-platform/pull/30716). (by @ormsbee) - [Improvement] Make it possible to override k8s resources in plugins using `k8s-override` patch. (by @foadlind) +- [Bugfix] Fix a race condition that could prevent a newly provisioned Studio container from starting due to a FileExistsError when creating logs directory. ## v14.0.2 (2022-06-27) diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index c513de5364e..567cb0caca6 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -23,7 +23,7 @@ # Create folders if necessary for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: if not os.path.exists(folder): - os.makedirs(folder) + os.makedirs(folder, exist_ok=True) {{ patch("openedx-cms-common-settings") }} From 42af604051c669d39800d2e34043c36ef94aa0d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 15 Aug 2023 06:57:31 +0200 Subject: [PATCH 56/96] fix: broken mysql after palm upgrade This fix is for a rather serious issue that affects users who upgrade from Olive to Palm. The client mysql charset and collation was incorrectly set to utf8mb4, while the server stil runs utf8mb3. Only users who run the mysql container are affected. To resolve this issue, we explicitely configure the client to use the utf8mb3 charset/collation. Important note: users who have somehow managed to upgrade from olive to Palm before may find themselves in an undefined state. They might have to fix their mysql data manually. Same thing for users who launched Palm from scratch; although, according to my preliinary tests, they should be able to downgrade their connection from utf8mb4 to utf8mb3 without issue. In addition, we upgrade to mysql 8.1.0. Among many other fixes, this avoids a server restart after the upgrade: > An in-place upgrade from MySQL 5.7 to MySQL 8.0, without a server > restart, could result in unexpected errors when executing queries on > tables. This fix eliminates the need to restart the server between the > upgrade and queries. (Bug #35410528) https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-34.html See also the 8.1.0 release notes: https://dev.mysql.com/doc/relnotes/mysql/8.1/en/news-8-1-0.html Close #887. --- changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md | 2 ++ docs/configuration.rst | 2 +- tests/commands/test_images.py | 2 +- tutor/commands/jobs.py | 2 +- tutor/templates/apps/openedx/config/partials/auth.yml | 3 +++ tutor/templates/config/defaults.yml | 2 +- tutor/templates/local/docker-compose.yml | 2 +- 7 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md diff --git a/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md b/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md new file mode 100644 index 00000000000..612e1e6de0d --- /dev/null +++ b/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md @@ -0,0 +1,2 @@ +- 💥[Bugfix] Fix mysql crash after upgrade to Palm. After an upgrade to Palm, the mysql client run by Django defaults to a utf8mb4 character set and collation, but the mysql server still runs with utf8mb3. This causes broken data during migration from Olive to Palm, and more generally when data is written to the database. To resolve this issue, we explicitely set the utf8mb3 charset and collation in the client. Users who were running Palm might have to fix their data manually. In the future we will upgrade the mysql server to utf8mb4. (by @regisb) +- [Improvement] We upgrade to MySQL 8.1.0 to avoid having to restart the server after the upgrade. diff --git a/docs/configuration.rst b/docs/configuration.rst index 15543af1ab8..a6b5e073b8b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -81,7 +81,7 @@ This configuration parameter defines which MongoDB Docker image to use. .. https://hub.docker.com/_/mysql/tags?page=1&name=8.0 -- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.0.33"``) +- ``DOCKER_IMAGE_MYSQL`` (default: ``"docker.io/mysql:8.1.0"``) This configuration parameter defines which MySQL Docker image to use. diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index cd0545ae9cf..7b0957790d9 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -49,7 +49,7 @@ def test_images_pull_all_vendor_images(self, image_pull: Mock) -> None: self.assertIsNone(result.exception) self.assertEqual(0, result.exit_code) # Note: we should update this tag whenever the mysql image is updated - image_pull.assert_called_once_with("docker.io/mysql:8.0.33") + image_pull.assert_called_once_with("docker.io/mysql:8.1.0") def test_images_printtag_image(self) -> None: result = self.invoke(["images", "printtag", "openedx"]) diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index eb356834a9d..562e78d4549 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -241,7 +241,7 @@ def sqlshell(args: list[str]) -> t.Iterable[tuple[str, str]]: Extra arguments will be passed to the `mysql` command verbatim. For instance, to show tables from the "openedx" database, run `do sqlshell openedx -e 'show tables'`. """ - command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }}" + command = "mysql --user={{ MYSQL_ROOT_USERNAME }} --password={{ MYSQL_ROOT_PASSWORD }} --host={{ MYSQL_HOST }} --port={{ MYSQL_PORT }} --default-character-set=utf8mb3" if args: command += " " + shlex.join(args) # pylint: disable=protected-access yield ("lms", command) diff --git a/tutor/templates/apps/openedx/config/partials/auth.yml b/tutor/templates/apps/openedx/config/partials/auth.yml index 3e72058013f..74d46d8b7e8 100644 --- a/tutor/templates/apps/openedx/config/partials/auth.yml +++ b/tutor/templates/apps/openedx/config/partials/auth.yml @@ -17,5 +17,8 @@ DATABASES: ATOMIC_REQUESTS: true OPTIONS: init_command: "SET sql_mode='STRICT_TRANS_TABLES'" + {%- if RUN_MYSQL %} + charset: "utf8mb3" + {%- endif %} EMAIL_HOST_USER: "{{ SMTP_USERNAME }}" EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}" diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 5a8399b3f69..16b99ed03de 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -15,7 +15,7 @@ DOCKER_IMAGE_OPENEDX_DEV: "openedx-dev:{{ TUTOR_VERSION }}" DOCKER_IMAGE_CADDY: "docker.io/caddy:2.6.4" DOCKER_IMAGE_ELASTICSEARCH: "docker.io/elasticsearch:7.17.9" DOCKER_IMAGE_MONGODB: "docker.io/mongo:4.4.22" -DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.0.33" +DOCKER_IMAGE_MYSQL: "docker.io/mysql:8.1.0" DOCKER_IMAGE_PERMISSIONS: "{{ DOCKER_REGISTRY }}overhangio/openedx-permissions:{{ TUTOR_VERSION }}" DOCKER_IMAGE_REDIS: "docker.io/redis:7.0.11" DOCKER_IMAGE_SMTP: "docker.io/devture/exim-relay:4.96-r1-0" diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 2dc6b72cfcb..a2ace7c57bd 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -40,7 +40,7 @@ services: {% if RUN_MYSQL -%} mysql: image: {{ DOCKER_IMAGE_MYSQL }} - command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci + command: mysqld --character-set-server=utf8mb3 --collation-server=utf8mb3_general_ci restart: unless-stopped user: "999:999" volumes: From e9710c5ed5ec7a18c82e641bf6d31a933a087048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 16 Aug 2023 19:12:35 +0200 Subject: [PATCH 57/96] v16.1.0 --- CHANGELOG.md | 13 ++++++++++++- .../20230731_110301_regis_fix_non_buildkit_build.md | 4 ---- ...230815_064652_regis_fix_mysql_upgrade_charset.md | 2 -- .../20230816_184458_regis_fix_local_non_prod.md | 1 - tutor/__about__.py | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/20230731_110301_regis_fix_non_buildkit_build.md delete mode 100644 changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md delete mode 100644 changelog.d/20230816_184458_regis_fix_local_non_prod.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 837f0fe2610..a6a05a9c441 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,18 @@ instructions, because git commits are used to generate release notes: + +## v16.1.0 (2023-08-16) + +- [Improvement] Improve support of legacy non-BuildKit mode: (by @regisb) + - [Bugfix] Fix building of openedx Docker image. + - [Improvement] Remove `--cache-from` build option. + - [Improvement] Add a warning concerning the lack of support of the `--build-context` option. +- 💥[Bugfix] Fix mysql crash after upgrade to Palm. After an upgrade to Palm, the mysql client run by Django defaults to a utf8mb4 character set and collation, but the mysql server still runs with utf8mb3. This causes broken data during migration from Olive to Palm, and more generally when data is written to the database. To resolve this issue, we explicitely set the utf8mb3 charset and collation in the client. Users who were running Palm might have to fix their data manually. In the future we will upgrade the mysql server to utf8mb4. (by @regisb) +- [Improvement] We upgrade to MySQL 8.1.0 to avoid having to restart the server after the upgrade. +- [Bugfix] Ask whether user wants to run locally during `tutor local launch`. (by @regisb) +- [Bugfix] Fix a race condition that could prevent a newly provisioned Studio container from starting due to a FileExistsError when creating logs directory. + ## v16.0.5 (2023-08-09) @@ -253,7 +265,6 @@ pen-release/palm.master --repo-dir=test-course/course`. (by @regisb) - [Bugfix] Build openedx-dev Docker image even when the host user is root, for instance on Windows. (by @regisb) - [Bugfix] Patch nutmeg.1 release with [LTI 1.3 fix](https://github.com/openedx/edx-platform/pull/30716). (by @ormsbee) - [Improvement] Make it possible to override k8s resources in plugins using `k8s-override` patch. (by @foadlind) -- [Bugfix] Fix a race condition that could prevent a newly provisioned Studio container from starting due to a FileExistsError when creating logs directory. ## v14.0.2 (2022-06-27) diff --git a/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md b/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md deleted file mode 100644 index 74c9b734a9d..00000000000 --- a/changelog.d/20230731_110301_regis_fix_non_buildkit_build.md +++ /dev/null @@ -1,4 +0,0 @@ -- [Improvement] Improve support of legacy non-BuildKit mode: (by @regisb) - - [Bugfix] Fix building of openedx Docker image. - - [Improvement] Remove `--cache-from` build option. - - [Improvement] Add a warning concerning the lack of support of the `--build-context` option. diff --git a/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md b/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md deleted file mode 100644 index 612e1e6de0d..00000000000 --- a/changelog.d/20230815_064652_regis_fix_mysql_upgrade_charset.md +++ /dev/null @@ -1,2 +0,0 @@ -- 💥[Bugfix] Fix mysql crash after upgrade to Palm. After an upgrade to Palm, the mysql client run by Django defaults to a utf8mb4 character set and collation, but the mysql server still runs with utf8mb3. This causes broken data during migration from Olive to Palm, and more generally when data is written to the database. To resolve this issue, we explicitely set the utf8mb3 charset and collation in the client. Users who were running Palm might have to fix their data manually. In the future we will upgrade the mysql server to utf8mb4. (by @regisb) -- [Improvement] We upgrade to MySQL 8.1.0 to avoid having to restart the server after the upgrade. diff --git a/changelog.d/20230816_184458_regis_fix_local_non_prod.md b/changelog.d/20230816_184458_regis_fix_local_non_prod.md deleted file mode 100644 index 7f80b451f83..00000000000 --- a/changelog.d/20230816_184458_regis_fix_local_non_prod.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Ask whether user wants to run locally during `tutor local launch`. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 24e0913309e..45e43fc494e 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.0.5" +__version__ = "16.1.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 728474fef38065bcaf7bbab5d45e5bf4fd3d90ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 28 Aug 2023 09:42:54 +0200 Subject: [PATCH 58/96] docs: backup with `sudo` See: https://discuss.overhang.io/t/copying-tutor-gives-permission-denied/3500 --- docs/install.rst | 2 +- docs/tutorials/datamigration.rst | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 0c39f1e2ebe..eb821010272 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -114,7 +114,7 @@ Upgrading to a new Open edX release Major Open edX releases are published twice a year, in June and December, by the Open edX `Build/Test/Release working group `__. When a new Open edX release comes out, Tutor gets a major version bump (see :ref:`versioning`). Such an upgrade typically includes multiple breaking changes. Any upgrade is final because downgrading is not supported. Thus, when upgrading your platform from one major version to the next, it is strongly recommended to do the following: 1. Read the changes listed in the `CHANGELOG.md `__ file. Breaking changes are identified by a "💥". -2. Perform a backup. On a local installation, this is typically done with:: +2. Perform a backup (see the :ref:`backup tutorial `). On a local installation, this is typically done with:: tutor local stop sudo rsync -avr "$(tutor config printroot)"/ /tmp/tutor-backup/ diff --git a/docs/tutorials/datamigration.rst b/docs/tutorials/datamigration.rst index 95bfb860658..8434eeec79e 100644 --- a/docs/tutorials/datamigration.rst +++ b/docs/tutorials/datamigration.rst @@ -1,3 +1,5 @@ +.. _backup_tutorial: + Making backups and migrating data --------------------------------- @@ -10,7 +12,7 @@ With Tutor, all data are stored in a single folder. This means that it's extreme 3. Transfer the configuration, environment, and platform data from server 1 to server 2:: - rsync -avr "$(tutor config printroot)/" username@server2:/tmp/tutor/ + sudo -avr "$(tutor config printroot)/" username@server2:/tmp/tutor/ 4. On server 2, move the data to the right location:: From d1d5ee1c96a2b10d76e932e0992afde3d43bace0 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Mon, 10 Jul 2023 08:58:24 -0400 Subject: [PATCH 59/96] docs: tutor uses `docker compose` now, not `docker-compose` --- docs/intro.rst | 4 ++-- docs/k8s.rst | 2 +- docs/local.rst | 2 +- tutor/commands/compose.py | 16 ++++++++-------- tutor/hooks/catalog.py | 2 +- tutor/templates/local/docker-compose.jobs.yml | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/intro.rst b/docs/intro.rst index bbef116f8d8..81d59e0a17b 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -51,7 +51,7 @@ Tutor simplifies the deployment of Open edX by: :width: 500px :align: center -Because Docker containers are becoming an industry-wide standard, that means that with Tutor it becomes possible to run Open edX anywhere: for now, Tutor supports deploying on a local server, with `docker-compose `_, and in a large cluster, with `Kubernetes `_. But in the future, Tutor may support other deployment platforms. +Because Docker containers are becoming an industry-wide standard, that means that with Tutor it becomes possible to run Open edX anywhere: for now, Tutor supports deploying on a local server, with `docker compose `_, and in a large cluster, with `Kubernetes `_. But in the future, Tutor may support other deployment platforms. Where can I try Open edX and Tutor? ----------------------------------- @@ -101,7 +101,7 @@ You can now take advantage of the Tutor-powered CLI (item #3) to bootstrap your tutor local launch -Under the hood, Tutor simply runs ``docker-compose`` and ``docker`` commands to launch your platform. These commands are printed in the standard output, such that you are free to replicate the same behaviour by simply copying/pasting the same commands. +Under the hood, Tutor simply runs ``docker compose`` and ``docker`` commands to launch your platform. These commands are printed in the standard output, such that you are free to replicate the same behaviour by simply copying/pasting the same commands. How do I navigate Tutor's command-line interface? ------------------------------------------------- diff --git a/docs/k8s.rst b/docs/k8s.rst index a7a80ad6934..3d934d2c5f9 100644 --- a/docs/k8s.rst +++ b/docs/k8s.rst @@ -119,7 +119,7 @@ Common tasks Executing commands inside service pods ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The Tutor and plugin documentation usually often instructions to execute some ``tutor local run ...`` commands. These commands are only valid when running Tutor locally with docker-compose, and will not work on Kubernetes. Instead, you should run ``tutor k8s exec ...`` commands. Arguments and options should be identical. +The Tutor and plugin documentation usually often instructions to execute some ``tutor local run ...`` commands. These commands are only valid when running Tutor locally with docker compose, and will not work on Kubernetes. Instead, you should run ``tutor k8s exec ...`` commands. Arguments and options should be identical. For instance, to run a Python shell in the lms container, run:: diff --git a/docs/local.rst b/docs/local.rst index 62cb431d894..7f3ebcb3075 100644 --- a/docs/local.rst +++ b/docs/local.rst @@ -6,7 +6,7 @@ Local deployment This method is for deploying Open edX locally on a single server, where docker images are orchestrated with `docker-compose `_. .. note:: - Tutor is compatible with the ``docker compose`` subcommand. However, this support is still in beta and we're not sure it will behave the same as the previous ``docker-compose`` command. So ``docker-compose`` will be preferred, unless you set an environment variable ``TUTOR_USE_COMPOSE_SUBCOMMAND`` to enforce using ``docker compose``. + As of v16.0.0, Tutor now uses the ``docker compose`` subcommand instead of the separate ``docker-compose`` command. .. _tutor_root: diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index baa1ca00169..0f19e3a5e3d 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -272,7 +272,7 @@ def reboot(context: click.Context, detach: bool, services: list[str]) -> None: @click.command( short_help="Restart some components from a running platform.", help="""Specify 'openedx' to restart the lms, cms and workers, or 'all' to -restart all services. Note that this performs a 'docker-compose restart', so new images +restart all services. Note that this performs a 'docker compose restart', so new images may not be taken into account. It is useful for reloading settings, for instance. To fully stop the platform, use the 'reboot' command.""", ) @@ -302,8 +302,8 @@ def do() -> None: @click.command( short_help="Run a command in a new container", help=( - "Run a command in a new container. This is a wrapper around `docker-compose run`. Any option or argument passed" - " to this command will be forwarded to docker-compose. Thus, you may use `-v` or `-p` to mount volumes and" + "Run a command in a new container. This is a wrapper around `docker compose run`. Any option or argument passed" + " to this command will be forwarded to docker compose. Thus, you may use `-v` or `-p` to mount volumes and" " expose ports." ), context_settings={"ignore_unknown_options": True}, @@ -368,7 +368,7 @@ def copyfrom( @click.command( short_help="Run a command in a running container", help=( - "Run a command in a running container. This is a wrapper around `docker-compose exec`. Any option or argument" + "Run a command in a running container. This is a wrapper around `docker compose exec`. Any option or argument" " passed to this command will be forwarded to docker-compose. Thus, you may use `-e` to manually define" " environment variables." ), @@ -383,7 +383,7 @@ def execute(context: click.Context, args: list[str]) -> None: @click.command( short_help="View output from containers", - help="View output from containers. This is a wrapper around `docker-compose logs`.", + help="View output from containers. This is a wrapper around `docker compose logs`.", ) @click.option("-f", "--follow", is_flag=True, help="Follow log output") @click.option("--tail", type=int, help="Number of lines to show from each container") @@ -406,10 +406,10 @@ def status(context: click.Context) -> None: @click.command( - short_help="Direct interface to docker-compose.", + short_help="Direct interface to docker compose.", help=( - "Direct interface to docker-compose. This is a wrapper around `docker-compose`. Most commands, options and" - " arguments passed to this command will be forwarded as-is to docker-compose." + "Direct interface to docker compose. This is a wrapper around `docker compose`. Most commands, options and" + " arguments passed to this command will be forwarded as-is to docker compose." ), context_settings={"ignore_unknown_options": True}, name="dc", diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index e1eac6d71f7..2f628102d6b 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -50,7 +50,7 @@ def run_this_on_start(root, config, name): For more information about how actions work, check out the :py:class:`tutor.core.hooks.Action` API. """ - #: Triggered whenever a "docker-compose start", "up" or "restart" command is executed. + #: Triggered whenever a "docker compose start", "up" or "restart" command is executed. #: #: :parameter str root: project root. #: :parameter dict config: project configuration. diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index c70fa23f594..984e085e712 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -1,8 +1,8 @@ # Tutor provides the `tutor MODE do JOB ...` CLI as a consistent way to execute jobs -# across the dev, local, and k8s modes. To support jobs in the docker-compose modes +# across the dev, local, and k8s modes. To support jobs in the docker compose modes # (dev and local), we must define a `-job` variant service in which jobs could be run. -# When `tutor local do JOB ...` is invoked, we `docker-compose run` each of JOB's +# When `tutor local do JOB ...` is invoked, we `docker compose run` each of JOB's # tasks against the appropriate `-job` services, as defined here. # When `tutor dev do JOB ...` is invoked, we do the same, but also include any # compose overrides in ../dev/docker-compose.jobs.yml. From fd4013b8e2a76501136aafa922a48c778643aec7 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Fri, 18 Aug 2023 11:55:53 -0400 Subject: [PATCH 60/96] docs: update README to reflect official ARM64 support --- docs/install.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index eb821010272..d46f6dd8516 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -9,7 +9,7 @@ Requirements ------------ * Supported OS: Tutor runs on any 64-bit, UNIX-based OS. It was also reported to work on Windows (with `WSL 2 `__). -* Architecture: support for ARM64 is a work-in-progress. See `this issue `__. +* Architecture: Both AMD64 and ARM64 are supported. * Required software: - `Docker `__: v20.10.15+ @@ -17,7 +17,6 @@ Requirements .. warning:: Do not attempt to simply run ``apt-get install docker docker-compose`` on older Ubuntu platforms, such as 16.04 (Xenial), as you will get older versions of these utilities. - * Ports 80 and 443 should be open. If other web services run on these ports, check the tutorial on :ref:`how to setup a web proxy `. * Hardware: From 9f56ee8d70d0cc0eabe28e9de3a83f6217d57075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 28 Aug 2023 09:48:16 +0200 Subject: [PATCH 61/96] docs: fix build --- docs/install.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/install.rst b/docs/install.rst index d46f6dd8516..f328ac841f7 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -17,6 +17,7 @@ Requirements .. warning:: Do not attempt to simply run ``apt-get install docker docker-compose`` on older Ubuntu platforms, such as 16.04 (Xenial), as you will get older versions of these utilities. + * Ports 80 and 443 should be open. If other web services run on these ports, check the tutorial on :ref:`how to setup a web proxy `. * Hardware: From 8e7c66a0ca7cf5438860b868b93a4faabd0083d1 Mon Sep 17 00:00:00 2001 From: Florian Haas Date: Thu, 24 Aug 2023 10:36:19 +0200 Subject: [PATCH 62/96] fix: Apply mysqld character set fix to Kubernetes deployment This is a follow-up fix to #819, where the corresponding change was added to the mysqld invocation in the "tutor local" (that is, docker-compose) deployment method, but omitted from its "tutor k8s" equivalent. --- changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md | 1 + tutor/templates/k8s/deployments.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md diff --git a/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md b/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md new file mode 100644 index 00000000000..b7320653918 --- /dev/null +++ b/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md @@ -0,0 +1 @@ +- 💥[Bugfix] Apply "fix mysql crash after upgrade to Palm" from 16.1.0 to `tutor k8s` deployments, as well. Users previously running `tutor k8s` with `RUN_MYSQL: true`, with any version between 16.0.0 and 16.1.0 including, might have to fix their data manually. For users running `tutor local`, this change has no effect, as the underlying issue was already fixed in 16.1.0. For users running `tutor k8s` with `RUN_MYSQL: false`, this change is also a no-op. (by @fghaas) diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index ee017246ed6..ff833ad1ca7 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -392,7 +392,7 @@ spec: containers: - name: mysql image: {{ DOCKER_IMAGE_MYSQL }} - args: ["mysqld", "--character-set-server=utf8", "--collation-server=utf8_general_ci"] + args: ["mysqld", "--character-set-server=utf8mb3", "--collation-server=utf8mb3_general_ci"] env: - name: MYSQL_ROOT_PASSWORD value: "{{ MYSQL_ROOT_PASSWORD }}" From fcb1d770da62b4957b0587a4a672d6515c678cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 29 Aug 2023 11:43:24 +0200 Subject: [PATCH 63/96] v16.1.1 --- CHANGELOG.md | 5 +++++ changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a6a05a9c441..08a46ac9c2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.1.1 (2023-08-29) + +- 💥[Bugfix] Apply "fix mysql crash after upgrade to Palm" from 16.1.0 to `tutor k8s` deployments, as well. Users previously running `tutor k8s` with `RUN_MYSQL: true`, with any version between 16.0.0 and 16.1.0 including, might have to fix their data manually. For users running `tutor local`, this change has no effect, as the underlying issue was already fixed in 16.1.0. For users running `tutor k8s` with `RUN_MYSQL: false`, this change is also a no-op. (by @fghaas) + ## v16.1.0 (2023-08-16) diff --git a/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md b/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md deleted file mode 100644 index b7320653918..00000000000 --- a/changelog.d/20230824_104119_fghaas_utf8mb3_k8s.md +++ /dev/null @@ -1 +0,0 @@ -- 💥[Bugfix] Apply "fix mysql crash after upgrade to Palm" from 16.1.0 to `tutor k8s` deployments, as well. Users previously running `tutor k8s` with `RUN_MYSQL: true`, with any version between 16.0.0 and 16.1.0 including, might have to fix their data manually. For users running `tutor local`, this change has no effect, as the underlying issue was already fixed in 16.1.0. For users running `tutor k8s` with `RUN_MYSQL: false`, this change is also a no-op. (by @fghaas) diff --git a/tutor/__about__.py b/tutor/__about__.py index 45e43fc494e..0066407232d 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.0" +__version__ = "16.1.1" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 67378deb387a2e3af994924db158fded3e018977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 29 Aug 2023 15:53:27 +0200 Subject: [PATCH 64/96] chore: upgrade reqs Now that sphinx_rtd support docutils>=0.19 we can drop that max version requirement. But we need to limit sphinx max version because they removed python 3.8 support before EOL. --- requirements/base.txt | 25 +++++++------ requirements/dev.in | 4 --- requirements/dev.txt | 83 +++++++++++++++++++++++-------------------- requirements/docs.in | 4 ++- requirements/docs.txt | 36 ++++++++++--------- 5 files changed, 80 insertions(+), 72 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 228782bb3ce..dff55c9f19e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements/base.in @@ -12,26 +12,28 @@ certifi==2023.7.22 # via # kubernetes # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.7 # via -r requirements/base.in -google-auth==2.19.1 +google-auth==2.22.0 # via kubernetes idna==3.4 # via requests jinja2==3.1.2 # via -r requirements/base.in -kubernetes==26.1.0 +kubernetes==27.2.0 # via -r requirements/base.in markupsafe==2.1.3 # via jinja2 -mypy==1.3.0 +mypy==1.5.1 # via -r requirements/base.in mypy-extensions==1.0.0 # via mypy oauthlib==3.2.2 - # via requests-oauthlib + # via + # kubernetes + # requests-oauthlib pyasn1==0.5.0 # via # pyasn1-modules @@ -42,7 +44,7 @@ pycryptodome==3.18.0 # via -r requirements/base.in python-dateutil==2.8.2 # via kubernetes -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/base.in # kubernetes @@ -61,7 +63,7 @@ six==1.16.0 # python-dateutil tomli==2.0.1 # via mypy -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via # -r requirements/base.in # mypy @@ -70,8 +72,5 @@ urllib3==1.26.16 # google-auth # kubernetes # requests -websocket-client==1.5.2 +websocket-client==1.6.2 # via kubernetes - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements/dev.in b/requirements/dev.in index 8dc0c3d1f82..c53771c4193 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -7,10 +7,6 @@ pyinstaller scriv twine -# doc requirement is lagging behind -# https://github.com/readthedocs/sphinx_rtd_theme/issues/1323 -docutils<0.19 - # Types packages types-docutils types-PyYAML diff --git a/requirements/dev.txt b/requirements/dev.txt index c2400af9fac..abb13c7448c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements/dev.in @@ -8,11 +8,11 @@ altgraph==0.17.3 # via pyinstaller appdirs==1.4.4 # via -r requirements/base.txt -astroid==2.15.5 +astroid==2.15.6 # via pylint attrs==23.1.0 # via scriv -black==23.3.0 +black==23.7.0 # via -r requirements/dev.in bleach==6.0.0 # via readme-renderer @@ -29,11 +29,11 @@ certifi==2023.7.22 # requests cffi==1.15.1 # via cryptography -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # -r requirements/base.txt # requests -click==8.1.3 +click==8.1.7 # via # -r requirements/base.txt # black @@ -42,17 +42,15 @@ click==8.1.3 # scriv click-log==0.4.0 # via scriv -coverage==7.2.7 +coverage==7.3.0 # via -r requirements/dev.in cryptography==41.0.3 # via secretstorage -dill==0.3.6 +dill==0.3.7 # via pylint -docutils==0.18.1 - # via - # -r requirements/dev.in - # readme-renderer -google-auth==2.19.1 +docutils==0.20.1 + # via readme-renderer +google-auth==2.22.0 # via # -r requirements/base.txt # kubernetes @@ -60,13 +58,15 @@ idna==3.4 # via # -r requirements/base.txt # requests -importlib-metadata==6.6.0 +importlib-metadata==6.8.0 # via # keyring # twine +importlib-resources==6.0.1 + # via keyring isort==5.12.0 # via pylint -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jeepney==0.8.0 # via @@ -76,13 +76,13 @@ jinja2==3.1.2 # via # -r requirements/base.txt # scriv -keyring==23.13.1 +keyring==24.2.0 # via twine -kubernetes==26.1.0 +kubernetes==27.2.0 # via -r requirements/base.txt lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 # via @@ -92,9 +92,9 @@ mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py -more-itertools==9.1.0 +more-itertools==10.1.0 # via jaraco-classes -mypy==1.3.0 +mypy==1.5.1 # via -r requirements/base.txt mypy-extensions==1.0.0 # via @@ -104,18 +104,19 @@ mypy-extensions==1.0.0 oauthlib==3.2.2 # via # -r requirements/base.txt + # kubernetes # requests-oauthlib packaging==23.1 # via # black # build -pathspec==0.11.1 +pathspec==0.11.2 # via black -pip-tools==6.13.0 +pip-tools==7.3.0 # via -r requirements/dev.in pkginfo==1.9.6 # via twine -platformdirs==3.5.1 +platformdirs==3.10.0 # via # black # pylint @@ -132,15 +133,15 @@ pycparser==2.21 # via cffi pycryptodome==3.18.0 # via -r requirements/base.txt -pygments==2.15.1 +pygments==2.16.1 # via # readme-renderer # rich -pyinstaller==5.11.0 +pyinstaller==5.13.1 # via -r requirements/dev.in -pyinstaller-hooks-contrib==2023.3 +pyinstaller-hooks-contrib==2023.7 # via pyinstaller -pylint==2.17.4 +pylint==2.17.5 # via -r requirements/dev.in pyproject-hooks==1.0.0 # via build @@ -148,11 +149,11 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # kubernetes -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/base.txt # kubernetes -readme-renderer==37.3 +readme-renderer==41.0 # via twine requests==2.31.0 # via @@ -170,7 +171,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.4.1 +rich==13.5.2 # via twine rsa==4.9 # via @@ -193,23 +194,27 @@ tomli==2.0.1 # black # build # mypy + # pip-tools # pylint # pyproject-hooks -tomlkit==0.11.8 +tomlkit==0.12.1 # via pylint twine==4.0.2 # via -r requirements/dev.in -types-docutils==0.20.0.1 +types-docutils==0.20.0.3 # via -r requirements/dev.in -types-pyyaml==6.0.12.10 +types-pyyaml==6.0.12.11 # via -r requirements/dev.in -types-setuptools==67.8.0.0 +types-setuptools==68.1.0.0 # via -r requirements/dev.in -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via # -r requirements/base.txt # astroid + # black # mypy + # pylint + # rich urllib3==1.26.16 # via # -r requirements/base.txt @@ -219,16 +224,18 @@ urllib3==1.26.16 # twine webencodings==0.5.1 # via bleach -websocket-client==1.5.2 +websocket-client==1.6.2 # via # -r requirements/base.txt # kubernetes -wheel==0.40.0 +wheel==0.41.2 # via pip-tools wrapt==1.15.0 # via astroid -zipp==3.15.0 - # via importlib-metadata +zipp==3.16.2 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.in b/requirements/docs.in index 79c02e20e35..1732d9b76f1 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -1,4 +1,6 @@ -r base.txt -sphinx +# Python 3.8 support was dropped in 7.2.0 +# https://github.com/sphinx-doc/sphinx/pull/11511 +sphinx<7.2.0 sphinx-rtd-theme sphinx-click diff --git a/requirements/docs.txt b/requirements/docs.txt index cfa5d7195b5..78d4debb081 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile requirements/docs.in @@ -19,11 +19,11 @@ certifi==2023.7.22 # -r requirements/base.txt # kubernetes # requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # -r requirements/base.txt # requests -click==8.1.3 +click==8.1.7 # via # -r requirements/base.txt # sphinx-click @@ -32,7 +32,7 @@ docutils==0.18.1 # sphinx # sphinx-click # sphinx-rtd-theme -google-auth==2.19.1 +google-auth==2.22.0 # via # -r requirements/base.txt # kubernetes @@ -42,17 +42,19 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx +importlib-metadata==6.8.0 + # via sphinx jinja2==3.1.2 # via # -r requirements/base.txt # sphinx -kubernetes==26.1.0 +kubernetes==27.2.0 # via -r requirements/base.txt markupsafe==2.1.3 # via # -r requirements/base.txt # jinja2 -mypy==1.3.0 +mypy==1.5.1 # via -r requirements/base.txt mypy-extensions==1.0.0 # via @@ -61,6 +63,7 @@ mypy-extensions==1.0.0 oauthlib==3.2.2 # via # -r requirements/base.txt + # kubernetes # requests-oauthlib packaging==23.1 # via sphinx @@ -75,13 +78,15 @@ pyasn1-modules==0.3.0 # google-auth pycryptodome==3.18.0 # via -r requirements/base.txt -pygments==2.15.1 +pygments==2.16.1 # via sphinx python-dateutil==2.8.2 # via # -r requirements/base.txt # kubernetes -pyyaml==6.0 +pytz==2023.3 + # via babel +pyyaml==6.0.1 # via # -r requirements/base.txt # kubernetes @@ -107,15 +112,15 @@ six==1.16.0 # python-dateutil snowballstemmer==2.2.0 # via sphinx -sphinx==6.2.1 +sphinx==7.1.2 # via # -r requirements/docs.in # sphinx-click # sphinx-rtd-theme # sphinxcontrib-jquery -sphinx-click==4.4.0 +sphinx-click==5.0.1 # via -r requirements/docs.in -sphinx-rtd-theme==1.2.1 +sphinx-rtd-theme==1.3.0 # via -r requirements/docs.in sphinxcontrib-applehelp==1.0.4 # via sphinx @@ -135,7 +140,7 @@ tomli==2.0.1 # via # -r requirements/base.txt # mypy -typing-extensions==4.6.3 +typing-extensions==4.7.1 # via # -r requirements/base.txt # mypy @@ -145,10 +150,9 @@ urllib3==1.26.16 # google-auth # kubernetes # requests -websocket-client==1.5.2 +websocket-client==1.6.2 # via # -r requirements/base.txt # kubernetes - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +zipp==3.16.2 + # via importlib-metadata From 5712561870fc2616055855c21f9403b16a469214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 29 Aug 2023 16:21:43 +0200 Subject: [PATCH 65/96] fix: type tests Type tests were broken following the upgrade of click. We take the opportunity to simplify the TutorCli implementation. --- tutor/commands/cli.py | 50 +++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index c8e552a2c9c..e434a05e7aa 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -32,7 +32,7 @@ def main() -> None: sys.exit(1) -class TutorCli(click.MultiCommand): +class TutorCli(click.Group): """ Dynamically load subcommands at runtime. @@ -43,26 +43,14 @@ class TutorCli(click.MultiCommand): IS_ROOT_READY = False - @classmethod - def iter_commands(cls, ctx: click.Context) -> t.Iterator[click.Command]: - """ - Return the list of subcommands (click.Command). - """ - cls.ensure_plugins_enabled(ctx) - yield from hooks.Filters.CLI_COMMANDS.iterate() - - @classmethod - def ensure_plugins_enabled(cls, ctx: click.Context) -> None: + def get_command( + self, ctx: click.Context, cmd_name: str + ) -> t.Optional[click.Command]: """ - We enable plugins as soon as possible to have access to commands. + This is run when passing a command from the CLI. E.g: tutor config ... """ - if not "root" in ctx.params: - # When generating docs, this function is called with empty args. - # That's ok, we just ignore it. - return - if not cls.IS_ROOT_READY: - hooks.Actions.PROJECT_ROOT_READY.do(ctx.params["root"]) - cls.IS_ROOT_READY = True + self.ensure_plugins_enabled(ctx) + return super().get_command(ctx, cmd_name=cmd_name) def list_commands(self, ctx: click.Context) -> list[str]: """ @@ -70,20 +58,22 @@ def list_commands(self, ctx: click.Context) -> list[str]: - shell autocompletion: tutor - print help: tutor, tutor -h """ - return sorted( - [command.name or "" for command in self.iter_commands(ctx)] - ) + self.ensure_plugins_enabled(ctx) + return super().list_commands(ctx) - def get_command( - self, ctx: click.Context, cmd_name: str - ) -> t.Optional[click.Command]: + def ensure_plugins_enabled(self, ctx: click.Context) -> None: """ - This is run when passing a command from the CLI. E.g: tutor config ... + We enable plugins as soon as possible to have access to commands. """ - for command in self.iter_commands(ctx): - if cmd_name == command.name: - return command - return None + if not "root" in ctx.params: + # When generating docs, this function is called with empty args. + # That's ok, we just ignore it. + return + if not self.IS_ROOT_READY: + hooks.Actions.PROJECT_ROOT_READY.do(ctx.params["root"]) + self.IS_ROOT_READY = True + for cmd in hooks.Filters.CLI_COMMANDS.iterate(): + self.add_command(cmd) @click.group( From 753963ccd5e848ba02ff80ecbe6a8292053780a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 5 Sep 2023 16:00:01 +0200 Subject: [PATCH 66/96] fix: render list config items Close #867. --- changelog.d/20230905_155859_regis_fix_render_config_lists.md | 1 + tests/test_env.py | 1 + tutor/env.py | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 changelog.d/20230905_155859_regis_fix_render_config_lists.md diff --git a/changelog.d/20230905_155859_regis_fix_render_config_lists.md b/changelog.d/20230905_155859_regis_fix_render_config_lists.md new file mode 100644 index 00000000000..a3aced1b800 --- /dev/null +++ b/changelog.d/20230905_155859_regis_fix_render_config_lists.md @@ -0,0 +1 @@ +- [Bugfix] Render config settings that are inside lists. (by @regisb) diff --git a/tests/test_env.py b/tests/test_env.py index 9dd299c0120..a79060991a3 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -64,6 +64,7 @@ def test_render_unknown(self) -> None: } self.assertEqual("ab", env.render_unknown(config, "{{ var1 }}b")) self.assertEqual({"x": "ac"}, env.render_unknown(config, {"x": "{{ var1 }}c"})) + self.assertEqual(["x", "ac"], env.render_unknown(config, ["x", "{{ var1 }}c"])) def test_common_domain(self) -> None: self.assertEqual( diff --git a/tutor/env.py b/tutor/env.py index 980bb773490..9ced7d5b8c7 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -376,6 +376,8 @@ def render_unknown(config: Config, value: t.Any) -> t.Any: return render_str(config, value) if isinstance(value, dict): return {k: render_unknown(config, v) for k, v in value.items()} + if isinstance(value, list): + return [render_unknown(config, v) for v in value] return value From d788ab5e02cf51ae2642fbecbee4424d0b50806b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 6 Sep 2023 10:52:02 +0200 Subject: [PATCH 67/96] fix: parse strings prefixed with "#" in `config save --set ...` Pound keys were interpreted as comments. This is annoying when we want to parse html color codes, such as in: $ tutor config save --set "INDIGO_PRIMARY_COLOR=#225522" $ tutor config printvalue INDIGO_PRIMARY_COLOR null Close #866. --- .../20230906_105044_regis_parse_pound_keys_in_settings.md | 1 + tests/test_serialize.py | 4 ++++ tutor/serialize.py | 4 ++++ 3 files changed, 9 insertions(+) create mode 100644 changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md diff --git a/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md b/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md new file mode 100644 index 00000000000..631de931680 --- /dev/null +++ b/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md @@ -0,0 +1 @@ +- [Bugfix] Correctly parse strings prefixed with pound "#" key in `tutor config save --set KEY=#value` commands. (by @regisb) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index 874675854b5..0923b15e8c3 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -41,6 +41,10 @@ def test_parse_key_value(self) -> None: "x=key1:\n subkey: value\nkey2:\n subkey: value" ), ) + self.assertEqual( + ("INDIGO_PRIMARY_COLOR", "#225522"), + serialize.parse_key_value("INDIGO_PRIMARY_COLOR=#225522"), + ) def test_str_format(self) -> None: self.assertEqual("true", serialize.str_format(True)) diff --git a/tutor/serialize.py b/tutor/serialize.py index 0f4ae6cf1f2..7d77b8349a3 100644 --- a/tutor/serialize.py +++ b/tutor/serialize.py @@ -73,4 +73,8 @@ def parse_key_value(text: str) -> t.Optional[tuple[str, t.Any]]: if not value: # Empty strings are interpreted as null values, which is incorrect. value = "''" + elif "\n" not in value and value.startswith("#"): + # Single-line string that starts with a pound # key + # We need to escape the string, otherwise pound will be interpreted as a comment. + value = f'"{value}"' return key, parse(value) From a48db2d2060f489bed69e9ef58dde63fffb038fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 7 Sep 2023 19:16:53 +0200 Subject: [PATCH 68/96] feat: add cairn to the official list of plugins This change means that cairn will be automatically installed whenever we run: pip install tutor[full] or whenever we run the tutor binary. --- requirements/plugins.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/plugins.txt b/requirements/plugins.txt index 13e4d0c8006..e67affd113c 100644 --- a/requirements/plugins.txt +++ b/requirements/plugins.txt @@ -1,5 +1,6 @@ # change version ranges when upgrading from palm tutor-android>=16.0.0,<17.0.0 +tutor-cairn>=16.0.0,<17.0.0 tutor-discovery>=16.0.0,<17.0.0 tutor-ecommerce>=16.0.0,<17.0.0 tutor-forum>=16.0.0,<17.0.0 From 2ebd9c777501997f10fbce149ebcaa616b828918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 13 Sep 2023 11:01:04 +0200 Subject: [PATCH 69/96] chore: upgrade nodeenv to fix nodejs install We upgrade nodeenv as an attempt to fix incomplete reads. From time to time we face the following error: #67 [linux/amd64 nodejs-requirements 2/4] RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt #67 0.338 * Install prebuilt node (16.14.0) .Incomplete read while readingfrom https://nodejs.org/download/release/v16.14.0/node-v16.14.0-linux-x64.tar.gz #67 204.1 . #67 204.1 Traceback (most recent call last): #67 204.1 File "/openedx/venv/bin/nodeenv", line 8, in #67 204.1 sys.exit(main()) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 1104, in main #67 204.1 create_environment(env_dir, args) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 980, in create_environment #67 204.1 install_node(env_dir, src_dir, args) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 739, in install_node #67 204.1 install_node_wrapped(env_dir, src_dir, args) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 762, in install_node_wrapped #67 204.1 download_node_src(node_url, src_dir, args) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 602, in download_node_src #67 204.1 with ctx as archive: #67 204.1 File "/opt/pyenv/versions/3.8.15/lib/python3.8/contextlib.py", line 113, in __enter__ #67 204.1 return next(self.gen) #67 204.1 File "/openedx/venv/lib/python3.8/site-packages/nodeenv.py", line 573, in tarfile_open #67 204.1 tf = tarfile.open(*args, **kwargs) #67 204.1 File "/opt/pyenv/versions/3.8.15/lib/python3.8/tarfile.py", line 1601, in open #67 204.1 saved_pos = fileobj.tell() #67 204.1 AttributeError: 'bytes' object has no attribute 'tell' This change was added to 1.8.0 as an attempt to resolve the issue: https://github.com/ekalinin/nodeenv/pull/329 We are not sure it will work every time, but it can't hurt. --- tutor/templates/build/openedx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 781f48ba0e4..6027eacaa6f 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -123,7 +123,7 @@ ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} # Install nodeenv with the version provided by edx-platform # https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt # https://github.com/pyenv/pyenv/releases -RUN pip install nodeenv==1.7.0 +RUN pip install nodeenv==1.8.0 RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt # Install nodejs requirements From 7185d8a51a57225fb5e42bc186bb2b887efc0ab7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:05:14 +0000 Subject: [PATCH 70/96] chore(deps): bump cryptography from 41.0.3 to 41.0.4 in /requirements Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: indirect ... Signed-off-by: dependabot[bot] --- requirements/dev.txt | 91 ++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 53 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index abb13c7448c..35d7546f7d5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,35 +7,35 @@ altgraph==0.17.3 # via pyinstaller appdirs==1.4.4 - # via -r requirements/base.txt + # via -r base.txt astroid==2.15.6 # via pylint attrs==23.1.0 # via scriv black==23.7.0 - # via -r requirements/dev.in + # via -r dev.in bleach==6.0.0 # via readme-renderer build==0.10.0 # via pip-tools cachetools==5.3.1 # via - # -r requirements/base.txt + # -r base.txt # google-auth certifi==2023.7.22 # via - # -r requirements/base.txt + # -r base.txt # kubernetes # requests cffi==1.15.1 # via cryptography charset-normalizer==3.2.0 # via - # -r requirements/base.txt + # -r base.txt # requests click==8.1.7 # via - # -r requirements/base.txt + # -r base.txt # black # click-log # pip-tools @@ -43,8 +43,8 @@ click==8.1.7 click-log==0.4.0 # via scriv coverage==7.3.0 - # via -r requirements/dev.in -cryptography==41.0.3 + # via -r dev.in +cryptography==41.0.4 # via secretstorage dill==0.3.7 # via pylint @@ -52,18 +52,16 @@ docutils==0.20.1 # via readme-renderer google-auth==2.22.0 # via - # -r requirements/base.txt + # -r base.txt # kubernetes idna==3.4 # via - # -r requirements/base.txt + # -r base.txt # requests importlib-metadata==6.8.0 # via # keyring # twine -importlib-resources==6.0.1 - # via keyring isort==5.12.0 # via pylint jaraco-classes==3.3.0 @@ -74,19 +72,19 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.2 # via - # -r requirements/base.txt + # -r base.txt # scriv keyring==24.2.0 # via twine kubernetes==27.2.0 - # via -r requirements/base.txt + # via -r base.txt lazy-object-proxy==1.9.0 # via astroid markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 # via - # -r requirements/base.txt + # -r base.txt # jinja2 mccabe==0.7.0 # via pylint @@ -95,15 +93,15 @@ mdurl==0.1.2 more-itertools==10.1.0 # via jaraco-classes mypy==1.5.1 - # via -r requirements/base.txt + # via -r base.txt mypy-extensions==1.0.0 # via - # -r requirements/base.txt + # -r base.txt # black # mypy oauthlib==3.2.2 # via - # -r requirements/base.txt + # -r base.txt # kubernetes # requests-oauthlib packaging==23.1 @@ -113,7 +111,7 @@ packaging==23.1 pathspec==0.11.2 # via black pip-tools==7.3.0 - # via -r requirements/dev.in + # via -r dev.in pkginfo==1.9.6 # via twine platformdirs==3.10.0 @@ -122,42 +120,42 @@ platformdirs==3.10.0 # pylint pyasn1==0.5.0 # via - # -r requirements/base.txt + # -r base.txt # pyasn1-modules # rsa pyasn1-modules==0.3.0 # via - # -r requirements/base.txt + # -r base.txt # google-auth pycparser==2.21 # via cffi pycryptodome==3.18.0 - # via -r requirements/base.txt + # via -r base.txt pygments==2.16.1 # via # readme-renderer # rich pyinstaller==5.13.1 - # via -r requirements/dev.in + # via -r dev.in pyinstaller-hooks-contrib==2023.7 # via pyinstaller pylint==2.17.5 - # via -r requirements/dev.in + # via -r dev.in pyproject-hooks==1.0.0 # via build python-dateutil==2.8.2 # via - # -r requirements/base.txt + # -r base.txt # kubernetes pyyaml==6.0.1 # via - # -r requirements/base.txt + # -r base.txt # kubernetes readme-renderer==41.0 # via twine requests==2.31.0 # via - # -r requirements/base.txt + # -r base.txt # kubernetes # requests-oauthlib # requests-toolbelt @@ -165,7 +163,7 @@ requests==2.31.0 # twine requests-oauthlib==1.3.1 # via - # -r requirements/base.txt + # -r base.txt # kubernetes requests-toolbelt==1.0.0 # via twine @@ -175,49 +173,38 @@ rich==13.5.2 # via twine rsa==4.9 # via - # -r requirements/base.txt + # -r base.txt # google-auth scriv==1.3.1 - # via -r requirements/dev.in + # via -r dev.in secretstorage==3.3.3 # via keyring six==1.16.0 # via - # -r requirements/base.txt + # -r base.txt # bleach # google-auth # kubernetes # python-dateutil tomli==2.0.1 - # via - # -r requirements/base.txt - # black - # build - # mypy - # pip-tools - # pylint - # pyproject-hooks + # via -r base.txt tomlkit==0.12.1 # via pylint twine==4.0.2 - # via -r requirements/dev.in + # via -r dev.in types-docutils==0.20.0.3 - # via -r requirements/dev.in + # via -r dev.in types-pyyaml==6.0.12.11 - # via -r requirements/dev.in + # via -r dev.in types-setuptools==68.1.0.0 - # via -r requirements/dev.in + # via -r dev.in typing-extensions==4.7.1 # via - # -r requirements/base.txt - # astroid - # black + # -r base.txt # mypy - # pylint - # rich urllib3==1.26.16 # via - # -r requirements/base.txt + # -r base.txt # google-auth # kubernetes # requests @@ -226,16 +213,14 @@ webencodings==0.5.1 # via bleach websocket-client==1.6.2 # via - # -r requirements/base.txt + # -r base.txt # kubernetes wheel==0.41.2 # via pip-tools wrapt==1.15.0 # via astroid zipp==3.16.2 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip From bc689c9ad01ec87e280b6cce94ef26ad7fb6f5c0 Mon Sep 17 00:00:00 2001 From: Pablo Thasi Date: Thu, 28 Sep 2023 09:15:52 +0100 Subject: [PATCH 71/96] Fix command to Transfer the configuration, environment, and platform data from server 1 to server 2 I added `rsync` to Transfer the configuration, environment, and platform data from server 1 to server 2 command so that we can we able to transfer data. I found that `-avr` options fit well to it. --- docs/tutorials/datamigration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/datamigration.rst b/docs/tutorials/datamigration.rst index 8434eeec79e..e7923a64635 100644 --- a/docs/tutorials/datamigration.rst +++ b/docs/tutorials/datamigration.rst @@ -12,7 +12,7 @@ With Tutor, all data are stored in a single folder. This means that it's extreme 3. Transfer the configuration, environment, and platform data from server 1 to server 2:: - sudo -avr "$(tutor config printroot)/" username@server2:/tmp/tutor/ + sudo rsync -avr "$(tutor config printroot)/" username@server2:/tmp/tutor/ 4. On server 2, move the data to the right location:: From cb66c5faa4c389a4d06d6fc243da20cbf7562ee1 Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Mon, 2 Oct 2023 10:38:07 +0330 Subject: [PATCH 72/96] feat: add CONFIG_LOADED action By utilizing CONFIG LOADED, we can now verify if PREVIEW_LMS_HOST is a subdomain of LMS_HOST and display a warning message to the user if it is not. --- ...72753_codewithemad_config_loaded_action.md | 1 + tutor/commands/config.py | 8 +++--- tutor/config.py | 27 +++++++++++++++++++ tutor/hooks/catalog.py | 7 +++++ 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 changelog.d/20230926_172753_codewithemad_config_loaded_action.md diff --git a/changelog.d/20230926_172753_codewithemad_config_loaded_action.md b/changelog.d/20230926_172753_codewithemad_config_loaded_action.md new file mode 100644 index 00000000000..588d4dae967 --- /dev/null +++ b/changelog.d/20230926_172753_codewithemad_config_loaded_action.md @@ -0,0 +1 @@ +- 💥[Feature] New action introduced: CONFIG_LOADED. This actions is called whenever the config is loaded. (by @CodeWithEmad) \ No newline at end of file diff --git a/tutor/commands/config.py b/tutor/commands/config.py index abcdc031a7f..58e18334d9e 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -114,7 +114,7 @@ def _candidate_config_items(self) -> t.Iterable[tuple[str, ConfigValue]]: type=ConfigListKeyValParamType(), multiple=True, metavar="KEY=VAL", - help="Append an item to a configuration value of type list. The value will only be added it it is not already present. (can be used multiple times)", + help="Append an item to a configuration value of type list. The value will only be added if it is not already present. (can be used multiple times)", ) @click.option( "-A", @@ -147,16 +147,18 @@ def save( env_only: bool, ) -> None: config = tutor_config.load_minimal(context.root) - config_full = tutor_config.load_full(context.root) if interactive: interactive_config.ask_questions(config) if set_vars: for key, value in set_vars: config[key] = env.render_unknown(config, value) if append_vars: + config_defaults = tutor_config.load_defaults() for key, value in append_vars: if key not in config: - config[key] = config_full.get(key, []) + config[key] = config[key] = config.get( + key, config_defaults.get(key, []) + ) values = config[key] if not isinstance(values, list): raise exceptions.TutorError( diff --git a/tutor/config.py b/tutor/config.py index a5e1ecab15a..d343394d1ea 100644 --- a/tutor/config.py +++ b/tutor/config.py @@ -1,4 +1,5 @@ from __future__ import annotations +from copy import deepcopy import os @@ -26,6 +27,15 @@ def load(root: str) -> Config: return load_full(root) +def load_defaults() -> Config: + """ + Load default configuration. + """ + config: Config = {} + update_with_defaults(config) + return config + + def load_minimal(root: str) -> Config: """ Load a minimal configuration composed of the user and the base config. @@ -51,6 +61,7 @@ def load_full(root: str) -> Config: update_with_base(config) update_with_defaults(config) render_full(config) + hooks.Actions.CONFIG_LOADED.do(deepcopy(config)) return config @@ -319,3 +330,19 @@ def _update_enabled_plugins_on_unload(_plugin: str, _root: str, config: Config) Note that this action must be performed after the plugin has been unloaded, hence the low priority. """ save_enabled_plugins(config) + + +@hooks.Actions.CONFIG_LOADED.add() +def _check_preview_lms_host(config: Config) -> None: + """ + This will check if the PREVIEW_LMS_HOST is a subdomain of LMS_HOST. + if not, prints a warning to notify the user. + """ + + lms_host = get_typed(config, "LMS_HOST", str, "") + preview_lms_host = get_typed(config, "PREVIEW_LMS_HOST", str, "") + if not preview_lms_host.endswith("." + lms_host): + fmt.echo_alert( + f'Warning: PREVIEW_LMS_HOST="{preview_lms_host}" is not a subdomain of LMS_HOST="{lms_host}". ' + "This configuration is not typically recommended and may lead to unexpected behavior." + ) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 2f628102d6b..66ed97a4ca9 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -57,6 +57,13 @@ def run_this_on_start(root, config, name): #: :parameter str name: docker-compose project name. COMPOSE_PROJECT_STARTED: Action[[str, Config, str]] = Action() + #: This action is called at the end of the tutor.config.load_full function. + #: Modifying this object will not trigger changes in the configuration. + #: For all purposes, it should be considered read-only. + #: + #: :parameter dict config: project configuration. + CONFIG_LOADED: Action[[Config]] = Action() + #: Called whenever the core project is ready to run. This action is called as soon #: as possible. This is the right time to discover plugins, for instance. In #: particular, we auto-discover the following plugins: From c0fbaa3d49a088ed57f705f718c5e9a061d48718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 2 Oct 2023 11:08:32 +0200 Subject: [PATCH 73/96] fix: file upload in open response assessments (ora2) For some reason, the ora2 cache configuration had disappeared in the upgrade to Palm. This issue was initially raised here: https://discuss.openedx.org/t/palm-2-ora-file-upload-failure/11332 Close #907. --- changelog.d/20231002_110754_regis_fix_ora2_uploads.md | 1 + tutor/templates/apps/openedx/settings/partials/common_lms.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/20231002_110754_regis_fix_ora2_uploads.md diff --git a/changelog.d/20231002_110754_regis_fix_ora2_uploads.md b/changelog.d/20231002_110754_regis_fix_ora2_uploads.md new file mode 100644 index 00000000000..d19fe246185 --- /dev/null +++ b/changelog.d/20231002_110754_regis_fix_ora2_uploads.md @@ -0,0 +1 @@ +- [Bugfix] Fix file upload in open response assessments. (by @regisb) diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index e3b86492e05..84dfd6629ab 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -36,6 +36,11 @@ "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "staticfiles_lms", } +CACHES["ora2-storage"] = { + "KEY_PREFIX": "ora2-storage", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", +} # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: From a5936a9022a6ff274141fa8910630be35c0b5b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 2 Oct 2023 11:15:51 +0200 Subject: [PATCH 74/96] v16.1.2 --- CHANGELOG.md | 8 ++++++++ .../20230905_155859_regis_fix_render_config_lists.md | 1 - .../20230906_105044_regis_parse_pound_keys_in_settings.md | 1 - .../20230926_172753_codewithemad_config_loaded_action.md | 1 - changelog.d/20231002_110754_regis_fix_ora2_uploads.md | 1 - tutor/__about__.py | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 changelog.d/20230905_155859_regis_fix_render_config_lists.md delete mode 100644 changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md delete mode 100644 changelog.d/20230926_172753_codewithemad_config_loaded_action.md delete mode 100644 changelog.d/20231002_110754_regis_fix_ora2_uploads.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a46ac9c2f..ab88a535a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,14 @@ instructions, because git commits are used to generate release notes: + +## v16.1.2 (2023-10-02) + +- [Bugfix] Render config settings that are inside lists. (by @regisb) +- [Bugfix] Correctly parse strings prefixed with pound "#" key in `tutor config save --set KEY=#value` commands. (by @regisb) +- [Feature] New action introduced: `CONFIG_LOADED`. This action is called whenever the config is loaded and makes it possible to verify the validity of configuration settings at runtime. (by @CodeWithEmad) +- [Bugfix] Fix file upload in open response assessments. (by @regisb) + ## v16.1.1 (2023-08-29) diff --git a/changelog.d/20230905_155859_regis_fix_render_config_lists.md b/changelog.d/20230905_155859_regis_fix_render_config_lists.md deleted file mode 100644 index a3aced1b800..00000000000 --- a/changelog.d/20230905_155859_regis_fix_render_config_lists.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Render config settings that are inside lists. (by @regisb) diff --git a/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md b/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md deleted file mode 100644 index 631de931680..00000000000 --- a/changelog.d/20230906_105044_regis_parse_pound_keys_in_settings.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Correctly parse strings prefixed with pound "#" key in `tutor config save --set KEY=#value` commands. (by @regisb) diff --git a/changelog.d/20230926_172753_codewithemad_config_loaded_action.md b/changelog.d/20230926_172753_codewithemad_config_loaded_action.md deleted file mode 100644 index 588d4dae967..00000000000 --- a/changelog.d/20230926_172753_codewithemad_config_loaded_action.md +++ /dev/null @@ -1 +0,0 @@ -- 💥[Feature] New action introduced: CONFIG_LOADED. This actions is called whenever the config is loaded. (by @CodeWithEmad) \ No newline at end of file diff --git a/changelog.d/20231002_110754_regis_fix_ora2_uploads.md b/changelog.d/20231002_110754_regis_fix_ora2_uploads.md deleted file mode 100644 index d19fe246185..00000000000 --- a/changelog.d/20231002_110754_regis_fix_ora2_uploads.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Fix file upload in open response assessments. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 0066407232d..bc40710ca4a 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.1" +__version__ = "16.1.2" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 4959620350b3537699e337fd6e1d381dbe5b42cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 3 Oct 2023 06:58:25 +0200 Subject: [PATCH 75/96] chore: upgrade urllib3 https://github.com/overhangio/tutor/pull/911 --- requirements/base.txt | 2 +- requirements/dev.txt | 91 +++++++++++++++++++++++++------------------ requirements/docs.txt | 6 +-- 3 files changed, 57 insertions(+), 42 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index dff55c9f19e..faf37d6a9e9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -67,7 +67,7 @@ typing-extensions==4.7.1 # via # -r requirements/base.in # mypy -urllib3==1.26.16 +urllib3==1.26.17 # via # google-auth # kubernetes diff --git a/requirements/dev.txt b/requirements/dev.txt index 35d7546f7d5..28074b82db2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,35 +7,35 @@ altgraph==0.17.3 # via pyinstaller appdirs==1.4.4 - # via -r base.txt + # via -r requirements/base.txt astroid==2.15.6 # via pylint attrs==23.1.0 # via scriv black==23.7.0 - # via -r dev.in + # via -r requirements/dev.in bleach==6.0.0 # via readme-renderer build==0.10.0 # via pip-tools cachetools==5.3.1 # via - # -r base.txt + # -r requirements/base.txt # google-auth certifi==2023.7.22 # via - # -r base.txt + # -r requirements/base.txt # kubernetes # requests cffi==1.15.1 # via cryptography charset-normalizer==3.2.0 # via - # -r base.txt + # -r requirements/base.txt # requests click==8.1.7 # via - # -r base.txt + # -r requirements/base.txt # black # click-log # pip-tools @@ -43,7 +43,7 @@ click==8.1.7 click-log==0.4.0 # via scriv coverage==7.3.0 - # via -r dev.in + # via -r requirements/dev.in cryptography==41.0.4 # via secretstorage dill==0.3.7 @@ -52,16 +52,18 @@ docutils==0.20.1 # via readme-renderer google-auth==2.22.0 # via - # -r base.txt + # -r requirements/base.txt # kubernetes idna==3.4 # via - # -r base.txt + # -r requirements/base.txt # requests importlib-metadata==6.8.0 # via # keyring # twine +importlib-resources==6.1.0 + # via keyring isort==5.12.0 # via pylint jaraco-classes==3.3.0 @@ -72,19 +74,19 @@ jeepney==0.8.0 # secretstorage jinja2==3.1.2 # via - # -r base.txt + # -r requirements/base.txt # scriv keyring==24.2.0 # via twine kubernetes==27.2.0 - # via -r base.txt + # via -r requirements/base.txt lazy-object-proxy==1.9.0 # via astroid markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 # via - # -r base.txt + # -r requirements/base.txt # jinja2 mccabe==0.7.0 # via pylint @@ -93,15 +95,15 @@ mdurl==0.1.2 more-itertools==10.1.0 # via jaraco-classes mypy==1.5.1 - # via -r base.txt + # via -r requirements/base.txt mypy-extensions==1.0.0 # via - # -r base.txt + # -r requirements/base.txt # black # mypy oauthlib==3.2.2 # via - # -r base.txt + # -r requirements/base.txt # kubernetes # requests-oauthlib packaging==23.1 @@ -111,7 +113,7 @@ packaging==23.1 pathspec==0.11.2 # via black pip-tools==7.3.0 - # via -r dev.in + # via -r requirements/dev.in pkginfo==1.9.6 # via twine platformdirs==3.10.0 @@ -120,42 +122,42 @@ platformdirs==3.10.0 # pylint pyasn1==0.5.0 # via - # -r base.txt + # -r requirements/base.txt # pyasn1-modules # rsa pyasn1-modules==0.3.0 # via - # -r base.txt + # -r requirements/base.txt # google-auth pycparser==2.21 # via cffi pycryptodome==3.18.0 - # via -r base.txt + # via -r requirements/base.txt pygments==2.16.1 # via # readme-renderer # rich pyinstaller==5.13.1 - # via -r dev.in + # via -r requirements/dev.in pyinstaller-hooks-contrib==2023.7 # via pyinstaller pylint==2.17.5 - # via -r dev.in + # via -r requirements/dev.in pyproject-hooks==1.0.0 # via build python-dateutil==2.8.2 # via - # -r base.txt + # -r requirements/base.txt # kubernetes pyyaml==6.0.1 # via - # -r base.txt + # -r requirements/base.txt # kubernetes readme-renderer==41.0 # via twine requests==2.31.0 # via - # -r base.txt + # -r requirements/base.txt # kubernetes # requests-oauthlib # requests-toolbelt @@ -163,7 +165,7 @@ requests==2.31.0 # twine requests-oauthlib==1.3.1 # via - # -r base.txt + # -r requirements/base.txt # kubernetes requests-toolbelt==1.0.0 # via twine @@ -173,38 +175,49 @@ rich==13.5.2 # via twine rsa==4.9 # via - # -r base.txt + # -r requirements/base.txt # google-auth scriv==1.3.1 - # via -r dev.in + # via -r requirements/dev.in secretstorage==3.3.3 # via keyring six==1.16.0 # via - # -r base.txt + # -r requirements/base.txt # bleach # google-auth # kubernetes # python-dateutil tomli==2.0.1 - # via -r base.txt + # via + # -r requirements/base.txt + # black + # build + # mypy + # pip-tools + # pylint + # pyproject-hooks tomlkit==0.12.1 # via pylint twine==4.0.2 - # via -r dev.in + # via -r requirements/dev.in types-docutils==0.20.0.3 - # via -r dev.in + # via -r requirements/dev.in types-pyyaml==6.0.12.11 - # via -r dev.in + # via -r requirements/dev.in types-setuptools==68.1.0.0 - # via -r dev.in + # via -r requirements/dev.in typing-extensions==4.7.1 # via - # -r base.txt + # -r requirements/base.txt + # astroid + # black # mypy -urllib3==1.26.16 + # pylint + # rich +urllib3==1.26.17 # via - # -r base.txt + # -r requirements/base.txt # google-auth # kubernetes # requests @@ -213,14 +226,16 @@ webencodings==0.5.1 # via bleach websocket-client==1.6.2 # via - # -r base.txt + # -r requirements/base.txt # kubernetes wheel==0.41.2 # via pip-tools wrapt==1.15.0 # via astroid zipp==3.16.2 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/docs.txt b/requirements/docs.txt index 78d4debb081..6f5e110d626 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -84,7 +84,7 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # kubernetes -pytz==2023.3 +pytz==2023.3.post1 # via babel pyyaml==6.0.1 # via @@ -144,7 +144,7 @@ typing-extensions==4.7.1 # via # -r requirements/base.txt # mypy -urllib3==1.26.16 +urllib3==1.26.17 # via # -r requirements/base.txt # google-auth @@ -154,5 +154,5 @@ websocket-client==1.6.2 # via # -r requirements/base.txt # kubernetes -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata From c896e30d539f08aa913bf881a953d33ec96524d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 3 Oct 2023 08:22:29 +0200 Subject: [PATCH 76/96] chore: mark compatibility with python 3.12 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8a5c4707bf5..816668b42ca 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ def is_requirement(line: str) -> bool: "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], test_suite="tests", ) From fadfeea25ec7d7ee9ef4413d05bde1d05138a771 Mon Sep 17 00:00:00 2001 From: Paulo Viadanna Date: Tue, 26 Sep 2023 17:42:56 -0300 Subject: [PATCH 77/96] fix: adds mongodb connect=False Adding connect=False to the LMS and CMS' MongoDB connection to prevent ServerSelectionError after a cluster failover. --- changelog.d/20231003_144841_paulo_mongodb_connect.md | 1 + tutor/templates/apps/openedx/settings/partials/common_all.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/20231003_144841_paulo_mongodb_connect.md diff --git a/changelog.d/20231003_144841_paulo_mongodb_connect.md b/changelog.d/20231003_144841_paulo_mongodb_connect.md new file mode 100644 index 00000000000..0f3cb5d50db --- /dev/null +++ b/changelog.d/20231003_144841_paulo_mongodb_connect.md @@ -0,0 +1 @@ +[Improvement] Adds `connect=False` to the LMS and CMS' MongoDB connection to prevent `ServerSelectionError` after a cluster failover. (by @open-craft) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index 46a563df663..b02cff42ace 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -12,6 +12,7 @@ "user": {% if MONGODB_USERNAME %}"{{ MONGODB_USERNAME }}"{% else %}None{% endif %}, "password": {% if MONGODB_PASSWORD %}"{{ MONGODB_PASSWORD }}"{% else %}None{% endif %}, # Connection/Authentication + "connect": False, "ssl": {{ MONGODB_USE_SSL }}, "authsource": "{{ MONGODB_AUTH_SOURCE }}", "replicaSet": {% if MONGODB_REPLICA_SET %}"{{ MONGODB_REPLICA_SET }}"{% else %}None{% endif %}, From dc6b39045665a7fbe9ccd6bb22323a1d9a83e00e Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 4 Oct 2023 11:49:04 -0400 Subject: [PATCH 78/96] fix: override CMS_BASE setting in Studio for dev The LMS was overriding CMS_BASE properly, but Studio (CMS) configuration was not. That meant that Studio's CMS_BASE in dev mode was using the devstack default of localhost:18010 (because this is what's defined in edx-platform). This in turn broke parts of Studio that use this value, such as the XBlock v2 API (/api/xblock/v2). This commit derives the value of the CMS_BASE Django setting from Tutor's CMS_HOST config value, in the same way that the LMS does it. --- changelog.d/20231004_114528_dave_fix_cms_base_settings.md | 1 + tutor/templates/apps/openedx/settings/cms/development.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/20231004_114528_dave_fix_cms_base_settings.md diff --git a/changelog.d/20231004_114528_dave_fix_cms_base_settings.md b/changelog.d/20231004_114528_dave_fix_cms_base_settings.md new file mode 100644 index 00000000000..c4dcce57bf8 --- /dev/null +++ b/changelog.d/20231004_114528_dave_fix_cms_base_settings.md @@ -0,0 +1 @@ +- [Bugfix] Override CMS_BASE setting in Studio for the development environment. Without this, parts of Studio will try to use the devstack default of localhost:8010 instead. (by @ormsbee) \ No newline at end of file diff --git a/tutor/templates/apps/openedx/settings/cms/development.py b/tutor/templates/apps/openedx/settings/cms/development.py index 5ba365da67a..af7af18e125 100644 --- a/tutor/templates/apps/openedx/settings/cms/development.py +++ b/tutor/templates/apps/openedx/settings/cms/development.py @@ -5,6 +5,9 @@ LMS_BASE = "{{ LMS_HOST }}:8000" LMS_ROOT_URL = "http://" + LMS_BASE +CMS_BASE = "{{ CMS_HOST }}:8001" +CMS_ROOT_URL = "http://" + CMS_BASE + # Authentication SOCIAL_AUTH_EDX_OAUTH2_KEY = "{{ CMS_OAUTH2_KEY_SSO_DEV }}" SOCIAL_AUTH_EDX_OAUTH2_PUBLIC_URL_ROOT = LMS_ROOT_URL From f8ad59d7ce2f4909bc9a15220936df69849bd8c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 10 Oct 2023 11:32:50 +0200 Subject: [PATCH 79/96] fix: build error caused by removed py2neo package On Oct. 10, py2neo package was abruptly removed from pypi, GitHub, and the py2neo website now displays just a super funny meme: https://py2neo.org/ Yes, we should get rid of that dependency, but we are still supposed to support existing users. So we install py2neo from our fork. --- changelog.d/20231010_111112_regis_fix_py2neo.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/20231010_111112_regis_fix_py2neo.md diff --git a/changelog.d/20231010_111112_regis_fix_py2neo.md b/changelog.d/20231010_111112_regis_fix_py2neo.md new file mode 100644 index 00000000000..676fb20f229 --- /dev/null +++ b/changelog.d/20231010_111112_regis_fix_py2neo.md @@ -0,0 +1 @@ +- [Bugfix] Fix build error caused by py2neo package that was abruptly pulled from pypi and GitHub. (by @regisb) From c7ff0409e8688ba1e0810b6b3f64360bab90a7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 10 Oct 2023 11:40:43 +0200 Subject: [PATCH 80/96] v16.1.3 --- CHANGELOG.md | 7 +++++++ changelog.d/20231003_144841_paulo_mongodb_connect.md | 1 - changelog.d/20231004_114528_dave_fix_cms_base_settings.md | 1 - changelog.d/20231010_111112_regis_fix_py2neo.md | 1 - tutor/__about__.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/20231003_144841_paulo_mongodb_connect.md delete mode 100644 changelog.d/20231004_114528_dave_fix_cms_base_settings.md delete mode 100644 changelog.d/20231010_111112_regis_fix_py2neo.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ab88a535a0e..ad0be3d0431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ instructions, because git commits are used to generate release notes: + +## v16.1.3 (2023-10-10) + +- [Improvement] Adds `connect=False` to the LMS and CMS' MongoDB connection to prevent `ServerSelectionError` after a cluster failover. (by @open-craft) +- [Bugfix] Override `CMS_BASE` setting in Studio for the development environment. Without this, parts of Studio will try to use the devstack default of localhost:8010 instead. (by @ormsbee) +- [Bugfix] Fix build error caused by py2neo package that was abruptly pulled from pypi and GitHub. (by @regisb) + ## v16.1.2 (2023-10-02) diff --git a/changelog.d/20231003_144841_paulo_mongodb_connect.md b/changelog.d/20231003_144841_paulo_mongodb_connect.md deleted file mode 100644 index 0f3cb5d50db..00000000000 --- a/changelog.d/20231003_144841_paulo_mongodb_connect.md +++ /dev/null @@ -1 +0,0 @@ -[Improvement] Adds `connect=False` to the LMS and CMS' MongoDB connection to prevent `ServerSelectionError` after a cluster failover. (by @open-craft) diff --git a/changelog.d/20231004_114528_dave_fix_cms_base_settings.md b/changelog.d/20231004_114528_dave_fix_cms_base_settings.md deleted file mode 100644 index c4dcce57bf8..00000000000 --- a/changelog.d/20231004_114528_dave_fix_cms_base_settings.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Override CMS_BASE setting in Studio for the development environment. Without this, parts of Studio will try to use the devstack default of localhost:8010 instead. (by @ormsbee) \ No newline at end of file diff --git a/changelog.d/20231010_111112_regis_fix_py2neo.md b/changelog.d/20231010_111112_regis_fix_py2neo.md deleted file mode 100644 index 676fb20f229..00000000000 --- a/changelog.d/20231010_111112_regis_fix_py2neo.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Fix build error caused by py2neo package that was abruptly pulled from pypi and GitHub. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index bc40710ca4a..004ee586514 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.2" +__version__ = "16.1.3" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 2ad51b961f7836d32b0ae1ab85fa8a647dcb4b7c Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Wed, 11 Oct 2023 11:32:03 +0330 Subject: [PATCH 81/96] fix: 600GB openedx-dev image on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On macOS, building the "openedx-dev" Docker image resulted in an image that required more than 600 GB of disk space. This was due to the `adduser` command which was called with a user ID of 2x10⁹ (on macOS only). This resulted in a very large /var/log/faillog file, hence the image size. Related upstream discussion: https://github.com/moby/moby/issues/5419 Close https://github.com/openedx/wg-developer-experience/issues/178 --- changelog.d/20231011_005657_codewithemad_large_dev_image.md | 1 + tutor/templates/build/openedx/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/20231011_005657_codewithemad_large_dev_image.md diff --git a/changelog.d/20231011_005657_codewithemad_large_dev_image.md b/changelog.d/20231011_005657_codewithemad_large_dev_image.md new file mode 100644 index 00000000000..efb1fd6e2a7 --- /dev/null +++ b/changelog.d/20231011_005657_codewithemad_large_dev_image.md @@ -0,0 +1 @@ +- [Improvement] No more large dev images. This was fixed by adding --no-log-init option to useradd command and reducing space usage of /var/log/faillog. (by @CodeWithEmad) \ No newline at end of file diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 6027eacaa6f..7c41bf9f9c7 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -149,7 +149,7 @@ RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,shari # Note that this must always be different from root (APP_USER_ID=0) ARG APP_USER_ID=1000 RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi -RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app +RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app USER ${APP_USER_ID} # https://hub.docker.com/r/powerman/dockerize/tags From e06e8356e99e52a87b5fbe87fd018a3c52763794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 13 Oct 2023 16:47:57 +0200 Subject: [PATCH 82/96] feat: upgrade to open-release/palm.3 --- changelog.d/20231013_164741_regis_palm_3.md | 1 + docs/configuration.rst | 2 +- tutor/templates/config/defaults.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20231013_164741_regis_palm_3.md diff --git a/changelog.d/20231013_164741_regis_palm_3.md b/changelog.d/20231013_164741_regis_palm_3.md new file mode 100644 index 00000000000..ab184b4e356 --- /dev/null +++ b/changelog.d/20231013_164741_regis_palm_3.md @@ -0,0 +1 @@ +- [Improvement] Upgrade the Open edX default version to open-release/palm.3. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index a6b5e073b8b..720b6cdae15 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.2"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.3"``) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 16b99ed03de..94f127226e1 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -52,7 +52,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/palm.2" +OPENEDX_COMMON_VERSION: "open-release/palm.3" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" From 3117c0cbce2ba51dd17ac2297ef7a8ad1ac40939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 13 Oct 2023 16:56:25 +0200 Subject: [PATCH 83/96] v16.1.4 --- CHANGELOG.md | 6 ++++++ changelog.d/20231011_005657_codewithemad_large_dev_image.md | 1 - changelog.d/20231013_164741_regis_palm_3.md | 1 - tutor/__about__.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20231011_005657_codewithemad_large_dev_image.md delete mode 100644 changelog.d/20231013_164741_regis_palm_3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ad0be3d0431..35d1a2f1320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ instructions, because git commits are used to generate release notes: + +## v16.1.4 (2023-10-13) + +- [Improvement] No more large dev images. This was fixed by adding `--no-log-init` option to useradd command and reducing space usage of `/var/log/faillog`. (by @CodeWithEmad) +- [Improvement] Upgrade the Open edX default version to open-release/palm.3. (by @regisb) + ## v16.1.3 (2023-10-10) diff --git a/changelog.d/20231011_005657_codewithemad_large_dev_image.md b/changelog.d/20231011_005657_codewithemad_large_dev_image.md deleted file mode 100644 index efb1fd6e2a7..00000000000 --- a/changelog.d/20231011_005657_codewithemad_large_dev_image.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] No more large dev images. This was fixed by adding --no-log-init option to useradd command and reducing space usage of /var/log/faillog. (by @CodeWithEmad) \ No newline at end of file diff --git a/changelog.d/20231013_164741_regis_palm_3.md b/changelog.d/20231013_164741_regis_palm_3.md deleted file mode 100644 index ab184b4e356..00000000000 --- a/changelog.d/20231013_164741_regis_palm_3.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Upgrade the Open edX default version to open-release/palm.3. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 004ee586514..b3d3260accd 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.3" +__version__ = "16.1.4" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 2fd7bce11401cc7068da58a7e63dab5f9d362c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 16 Oct 2023 15:20:56 +0200 Subject: [PATCH 84/96] wip --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 71cfe341367..af867fb556f 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ Tutor: the Docker-based Open edX distribution designed for peace of mind **Tutor** is the official Docker-based `Open edX `_ distribution, both for production and local development. The goal of Tutor is to make it easy to deploy, customise, upgrade and scale Open edX. Tutor is reliable, fast, extensible, and it is already used to deploy hundreds of Open edX platforms around the world. -Do you need professional assistance setting up or managing your Open edX platform? Overhang.IO provides online support as part of its `Long Term Support (LTS) offering `__. +Do you need professional assistance setting up or managing your Open edX platform? `Edly `__ provides online support as part of its `Open edX installation service `__. Features -------- From 9b467571388d50f29716c8e9f5db24c0a252e907 Mon Sep 17 00:00:00 2001 From: Fateme Khodayari <55655542+FatemeKhodayari@users.noreply.github.com> Date: Mon, 16 Oct 2023 20:04:28 +0330 Subject: [PATCH 85/96] fix: ora2 uploads in cms --- changelog.d/20231007_153252_fateme.khodayari97.md | 2 ++ tutor/templates/apps/openedx/settings/partials/common_all.py | 5 +++++ tutor/templates/apps/openedx/settings/partials/common_cms.py | 2 +- tutor/templates/apps/openedx/settings/partials/common_lms.py | 5 ----- 4 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/20231007_153252_fateme.khodayari97.md diff --git a/changelog.d/20231007_153252_fateme.khodayari97.md b/changelog.d/20231007_153252_fateme.khodayari97.md new file mode 100644 index 00000000000..5bd3333a0b3 --- /dev/null +++ b/changelog.d/20231007_153252_fateme.khodayari97.md @@ -0,0 +1,2 @@ +- [Bugfix] Fix ORA2 file uploads in CMS. As the cache settings are shared between CMS and LMS, the settings are moved +from `common_lms.py` to `common_all.py`. (by @FatemeKhodayari) diff --git a/tutor/templates/apps/openedx/settings/partials/common_all.py b/tutor/templates/apps/openedx/settings/partials/common_all.py index b02cff42ace..b1d5a9f82a3 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_all.py +++ b/tutor/templates/apps/openedx/settings/partials/common_all.py @@ -77,6 +77,11 @@ "BACKEND": "django_redis.cache.RedisCache", "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", }, + "ora2-storage": { + "KEY_PREFIX": "ora2-storage", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", + } } # The default Django contrib site is the one associated to the LMS domain name. 1 is diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index 567cb0caca6..1b2f48a2f91 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -21,7 +21,7 @@ FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register' # Create folders if necessary -for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE]: +for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): os.makedirs(folder, exist_ok=True) diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 84dfd6629ab..e3b86492e05 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -36,11 +36,6 @@ "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "staticfiles_lms", } -CACHES["ora2-storage"] = { - "KEY_PREFIX": "ora2-storage", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}", -} # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: From 050f04018cf122100197f108291a6fd0e3d8d9a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 17 Oct 2023 08:37:52 +0200 Subject: [PATCH 86/96] docs: overhang.io -> edly references --- README.rst | 2 +- docs/faq.rst | 8 ++++---- docs/whatnext.rst | 2 +- setup.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index af867fb556f..b7a75fde9db 100644 --- a/README.rst +++ b/README.rst @@ -81,7 +81,7 @@ Please follow the instructions from the `troubleshooting section `__ provides professional assistance as part of its `Open edX installation service `__. .. _readme_support_end: diff --git a/docs/faq.rst b/docs/faq.rst index b8a169f75d8..f4907d854c8 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -45,14 +45,14 @@ What features are missing from Tutor? Tutor tries very hard to support all major Open edX features, notably in the form of :ref:`plugins `. If you are interested in sponsoring the development of a new plugin, please `get in touch `__! -It should be noted that the `Insights `__ stack is currently unsupported, because of its complexity, lack of support, and extensibility. To replace it, Overhang.IO developed `Cairn `__ the next-generation analytics solution for Open edX. You should check it out 😉 +It should be noted that the `Insights `__ stack is currently unsupported, because of its complexity, lack of support, and extensibility. To replace it, we developed `Cairn `__ the next-generation analytics solution for Open edX. You should check it out 😉 Are there people already running this in production? ---------------------------------------------------- Yes: system administrators all around the world use Tutor to run their Open edX platforms, from single-class school teachers to renowned universities, Open edX SaaS providers, and nation-wide learning platforms. -Why should I trust software written by some random guy on the Internet? ------------------------------------------------------------------------ +Why should I trust your software? +--------------------------------- -You shouldn't :) Tutor is actively maintained by `Overhang.IO `_, a France-based company founded by `Régis Behmo `_. Régis has been working on Tutor since early 2018; he has been a contributor to the Open edX project since 2015. In particular, he has worked for 2 years at `FUN-MOOC `_, one of the top 5 largest Open edX platforms in the world. In addition, the Tutor project is a community-led project with many contributions from its :ref:`project maintainers `. +You shouldn't :) Tutor is actively maintained by `Edly `__, a US-based ed-tech company facilitating over 40 million learners worldwide through its eLearning solutions. With a credible engineering team that has won clients' hearts globally led by `Régis Behmo `__, Tutor has empowered numerous edtech ventures over the years. Additionally, Tutor is a `community-led project `__ with many contributions from its :ref:`project maintainers `. diff --git a/docs/whatnext.rst b/docs/whatnext.rst index f1856052226..2d77da5b629 100644 --- a/docs/whatnext.rst +++ b/docs/whatnext.rst @@ -38,7 +38,7 @@ Yes, Tutor comes with Kubernetes deployment support :ref:`out of the box `. Gathering insights and analytics about Open edX ----------------------------------------------- -Check out `Cairn `__, the next-generation analytics solution for Open edX. +Check out `Cairn `__, the next-generation analytics solution for Open edX. Meeting the community --------------------- diff --git a/setup.py b/setup.py index 816668b42ca..8cbee1bd1ae 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,8 @@ def is_requirement(line: str) -> bool: "Community": "https://discuss.openedx.org/tag/tutor", }, license="AGPLv3", - author="Overhang.io", - author_email="contact@overhang.io", + author="Edly", + author_email="hello@edly.io", description="The Docker-based Open edX distribution designed for peace of mind", long_description=load_readme(), long_description_content_type="text/x-rst", From a8b40d4493db22ddf3c2e573d2658b0691ef067f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 17 Oct 2023 08:55:43 +0200 Subject: [PATCH 87/96] docs: in troubleshooting section, overhang.io -> edly.io --- docs/troubleshooting.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index c329befe4f2..a9fc6d1ef8c 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -16,7 +16,7 @@ What should you do if you have a problem? 6. If despite all your efforts, you can't solve the problem by yourself, you should discuss it in the `Open edX community forum `__. Please give as many details about your problem as possible! As a rule of thumb, **people will not dedicate more time to solving your problem than you took to write your question**. You should tag your topic with "tutor" or the corresponding Tutor plugin name ("tutor-discovery", etc.) in order to notify the maintainers. 7. If you are *absolutely* positive that you are facing a technical issue with Tutor, and not with Open edX, not with your server, not your custom configuration, then, and only then, should you open an issue on `Github `__. You *must* follow the instructions from the issue template!!! If you do not follow this procedure, your Github issues will be mercilessly closed 🤯. -Do you need professional assistance with your tutor-managed Open edX platform? Overhang.IO offers online support as part of its `Long Term Support (LTS) offering `__. +Do you need professional assistance with your Open edX platform? `Edly `__ provides online support as part of its `Open edX installation service `__. .. _logging: From 9b12f4fa15655ca3493e784954025863b1d71fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Wed, 18 Oct 2023 09:24:43 +0200 Subject: [PATCH 88/96] chore: upgrade urllib3 See https://github.com/overhangio/tutor/pull/924 --- requirements/base.txt | 2 +- requirements/dev.txt | 2 +- requirements/docs.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index faf37d6a9e9..e79b5bdcc25 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -67,7 +67,7 @@ typing-extensions==4.7.1 # via # -r requirements/base.in # mypy -urllib3==1.26.17 +urllib3==1.26.18 # via # google-auth # kubernetes diff --git a/requirements/dev.txt b/requirements/dev.txt index 28074b82db2..f3c5855793a 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -215,7 +215,7 @@ typing-extensions==4.7.1 # mypy # pylint # rich -urllib3==1.26.17 +urllib3==1.26.18 # via # -r requirements/base.txt # google-auth diff --git a/requirements/docs.txt b/requirements/docs.txt index 6f5e110d626..80bb0301101 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -144,7 +144,7 @@ typing-extensions==4.7.1 # via # -r requirements/base.txt # mypy -urllib3==1.26.17 +urllib3==1.26.18 # via # -r requirements/base.txt # google-auth From 0dd7ddf5f6ecbc5112dafbc867b6a7903af13bff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 30 Oct 2023 18:26:17 +0100 Subject: [PATCH 89/96] v16.1.5 --- CHANGELOG.md | 5 +++++ changelog.d/20231007_153252_fateme.khodayari97.md | 2 -- tutor/__about__.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20231007_153252_fateme.khodayari97.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d1a2f1320..e9da88a8d28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ instructions, because git commits are used to generate release notes: + +## v16.1.5 (2023-10-30) + +- [Bugfix] Fix ORA2 file uploads in CMS. As the cache settings are shared between CMS and LMS, the settings are moved from `common_lms.py` to `common_all.py`. (by @FatemeKhodayari) + ## v16.1.4 (2023-10-13) diff --git a/changelog.d/20231007_153252_fateme.khodayari97.md b/changelog.d/20231007_153252_fateme.khodayari97.md deleted file mode 100644 index 5bd3333a0b3..00000000000 --- a/changelog.d/20231007_153252_fateme.khodayari97.md +++ /dev/null @@ -1,2 +0,0 @@ -- [Bugfix] Fix ORA2 file uploads in CMS. As the cache settings are shared between CMS and LMS, the settings are moved -from `common_lms.py` to `common_all.py`. (by @FatemeKhodayari) diff --git a/tutor/__about__.py b/tutor/__about__.py index b3d3260accd..300e83960ed 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.4" +__version__ = "16.1.5" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From b00e2600843be7cd7e3cf451534176a0365acf2b Mon Sep 17 00:00:00 2001 From: Emad Rad Date: Mon, 6 Nov 2023 20:25:41 +0330 Subject: [PATCH 90/96] feat: dev added to extras_require we can use this to install tutor development packages inside ci jobs, with one line. --- changelog.d/20231106_202213_codewithemad.md | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/20231106_202213_codewithemad.md diff --git a/changelog.d/20231106_202213_codewithemad.md b/changelog.d/20231106_202213_codewithemad.md new file mode 100644 index 00000000000..37d04ecf0da --- /dev/null +++ b/changelog.d/20231106_202213_codewithemad.md @@ -0,0 +1 @@ +- [Improvement] Install tutor development tools with `pip install tutor[dev]`. (by @CodeWithEmad) \ No newline at end of file diff --git a/setup.py b/setup.py index 8cbee1bd1ae..b1da7ff80e7 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ def is_requirement(line: str) -> bool: python_requires=">=3.8", install_requires=load_requirements("base.in"), extras_require={ + "dev": load_requirements("dev.txt"), "full": load_requirements("plugins.txt"), }, entry_points={"console_scripts": ["tutor=tutor.commands.cli:main"]}, From 0b42a6aff9a87abe5ec7d0ef25db9b2d98fdaf55 Mon Sep 17 00:00:00 2001 From: Talha Rizwan Date: Tue, 7 Nov 2023 13:27:42 +0500 Subject: [PATCH 91/96] docs: Update troubleshooting.rst with notes about docker buildkit Close https://github.com/overhangio/tutor-mfe/issues/125 --- docs/troubleshooting.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index a9fc6d1ef8c..268eaa9f187 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -168,3 +168,28 @@ This issue should only happen in development mode. Long story short, it can be s tutor dev run lms ./manage.py lms waffle_switch block_structure.invalidate_cache_on_publish on --create If you'd like to learn more, please take a look at `this Github issue `__. + +High resource consumption on ``tutor images build`` by docker +------------------------------------------------------------- + +This issue can occur when building multiple images simultaneously by Docker, issue specifically related to BuildKit. + + +Create a buildkit.toml configuration file with the following contents:: + + [worker.oci] + max-parallelism = 2 + +This configuration file limits the number of layers built concurrently to 2, which can significantly reduce resource consumption. + +Create a builder that uses this configuration:: + + docker buildx create --use --name= --driver=docker-container --config=/path/to/buildkit.toml + +Replace with a suitable name for your builder, and ensure that you specify the correct path to the buildkit.toml configuration file. + +Now build again:: + + tutor images build + +All build commands should now make use of the newly configured builder. To later revert to the default builder, run ``docker buildx use default``. From c32d6ec83811354a3f36e5ab6c35111543ef5d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Tue, 14 Nov 2023 17:27:42 +0100 Subject: [PATCH 92/96] docs: move demo server to edly url (#939) --- docs/install.rst | 6 +++--- docs/intro.rst | 8 ++++---- docs/plugins/v0/api.rst | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index f328ac841f7..df4069261c2 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -68,10 +68,10 @@ Configuring DNS records When running a server in production, it is necessary to define `DNS records `__ which will make it possible to access your Open edX platform by name in your browser. The precise procedure to create DNS records varies from one provider to the next and is beyond the scope of these docs. You should create a record of type A with a name equal to your LMS hostname (given by ``tutor config printvalue LMS_HOST``) and a value that indicates the IP address of your server. Applications other than the LMS, such as the studio, ecommerce, etc. typically reside in subdomains of the LMS. Thus, you should also create a CNAME record to point all subdomains of the LMS to the LMS_HOST. -For instance, the demo Open edX server that runs at https://demo.openedx.overhang.io has the following DNS records:: +For instance, to run an Open edX server at https://learn.mydomain.com on a server with IP address 1.1.1.1, you would need to configure the following DNS records:: - demo.openedx 1800 IN A 172.105.89.208 - *.demo.openedx 1800 IN CNAME demo.openedx.overhang.io. + learn 1800 IN A 1.1.1.1 + *.learn 1800 IN CNAME learn.mydomain.com. .. _cloud_install: diff --git a/docs/intro.rst b/docs/intro.rst index 81d59e0a17b..ba6889e7c12 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -56,17 +56,17 @@ Because Docker containers are becoming an industry-wide standard, that means tha Where can I try Open edX and Tutor? ----------------------------------- -A demo Open edX platform is available at https://demo.openedx.overhang.io. This platform was deployed using Tutor and the `Indigo theme `__. Feel free to play around with the following credentials: +A demo Open edX platform is available at https://demo.openedx.edly.io. This platform was deployed using Tutor and the `Indigo theme `__. Feel free to play around with the following credentials: * Admin user: username=admin email=admin@overhang.io password=admin * Student user: username=student email=student@overhang.io password=student -The Android mobile application for this demo platform can be downloaded at this url: https://mobile.demo.openedx.overhang.io/app.apk +The Android mobile application for this demo platform can be downloaded at this url: https://mobile.demo.openedx.edly.io/app.apk Urls: -* LMS: https://demo.openedx.overhang.io -* Studio (CMS): https://studio.demo.openedx.overhang.io +* LMS: https://demo.openedx.edly.io +* Studio (CMS): https://studio.demo.openedx.edly.io The platform is reset every day at 9:00 AM, `Paris (France) time `__, so feel free to try and break things as much as you want. diff --git a/docs/plugins/v0/api.rst b/docs/plugins/v0/api.rst index 380474ac21c..37c01eea8d9 100644 --- a/docs/plugins/v0/api.rst +++ b/docs/plugins/v0/api.rst @@ -201,7 +201,7 @@ Any user who installs the ``myplugin`` plugin can then run:: $ tutor myplugin Hello from myplugin! - My LMS host is demo.openedx.overhang.io + My LMS host is learn.myserver.com You can even define subcommands by creating `command groups `__:: From af40964ca82075e224c6716bad846b7d29167e53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 16 Nov 2023 19:04:55 +0100 Subject: [PATCH 93/96] feat: upgrade to palm.4 --- changelog.d/20231116_185835_regis_palm_4.md | 1 + docs/configuration.rst | 2 +- tutor/templates/config/defaults.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/20231116_185835_regis_palm_4.md diff --git a/changelog.d/20231116_185835_regis_palm_4.md b/changelog.d/20231116_185835_regis_palm_4.md new file mode 100644 index 00000000000..4fe97652a98 --- /dev/null +++ b/changelog.d/20231116_185835_regis_palm_4.md @@ -0,0 +1 @@ +- [Feature] Upgrade to open-release/palm.4. It is strongly recommended to upgrade to this release for as long as possible before upgrading to Quince. Otherwise, many users will be logged out after the Quince upgrade and will have to log in again -- see the Quince release notes. (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index 720b6cdae15..9375da57aa5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -136,7 +136,7 @@ Open edX customisation This defines the git repository from which you install Open edX platform code. If you run an Open edX fork with custom patches, set this to your own git repository. You may also override this configuration parameter at build time, by providing a ``--build-arg`` option. -- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.3"``) +- ``OPENEDX_COMMON_VERSION`` (default: ``"open-release/palm.4"``) This defines the default version that will be pulled from all Open edX git repositories. diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 94f127226e1..1eb1bac620b 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -52,7 +52,7 @@ OPENEDX_CMS_UWSGI_WORKERS: 2 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/palm.3" +OPENEDX_COMMON_VERSION: "open-release/palm.4" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock>=16.0.0,<17.0.0" MYSQL_HOST: "mysql" From ff6f9b9017d381ccfa468530ef4e0bce4aedba18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 16 Nov 2023 22:38:10 +0100 Subject: [PATCH 94/96] v16.1.6 --- CHANGELOG.md | 6 ++++++ changelog.d/20231106_202213_codewithemad.md | 1 - changelog.d/20231116_185835_regis_palm_4.md | 1 - tutor/__about__.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20231106_202213_codewithemad.md delete mode 100644 changelog.d/20231116_185835_regis_palm_4.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e9da88a8d28..adbc35a6630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ instructions, because git commits are used to generate release notes: + +## v16.1.6 (2023-11-16) + +- [Feature] Upgrade to open-release/palm.4. It is strongly recommended to upgrade to this release for as long as possible before upgrading to Quince. Otherwise, many users will be logged out after the Quince upgrade and will have to log in again -- see the Quince release notes. (by @regisb) +- [Improvement] Install tutor development tools with `pip install tutor[dev]`. (by @CodeWithEmad) + ## v16.1.5 (2023-10-30) diff --git a/changelog.d/20231106_202213_codewithemad.md b/changelog.d/20231106_202213_codewithemad.md deleted file mode 100644 index 37d04ecf0da..00000000000 --- a/changelog.d/20231106_202213_codewithemad.md +++ /dev/null @@ -1 +0,0 @@ -- [Improvement] Install tutor development tools with `pip install tutor[dev]`. (by @CodeWithEmad) \ No newline at end of file diff --git a/changelog.d/20231116_185835_regis_palm_4.md b/changelog.d/20231116_185835_regis_palm_4.md deleted file mode 100644 index 4fe97652a98..00000000000 --- a/changelog.d/20231116_185835_regis_palm_4.md +++ /dev/null @@ -1 +0,0 @@ -- [Feature] Upgrade to open-release/palm.4. It is strongly recommended to upgrade to this release for as long as possible before upgrading to Quince. Otherwise, many users will be logged out after the Quince upgrade and will have to log in again -- see the Quince release notes. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index 300e83960ed..c70f7403c2c 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.5" +__version__ = "16.1.6" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and From 28c9126915d3ec1b5b14a5a5d26c78e42b58a0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 17 Nov 2023 10:29:24 +0100 Subject: [PATCH 95/96] fix: missing dev.txt file in pypi package Close #943. --- MANIFEST.in | 1 + changelog.d/20231117_102830_regis.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/20231117_102830_regis.md diff --git a/MANIFEST.in b/MANIFEST.in index 9b2c42c9331..7d53c0c3bd4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include requirements/base.in include requirements/plugins.txt +include requirements/dev.txt recursive-include tutor/templates * include tutor/py.typed diff --git a/changelog.d/20231117_102830_regis.md b/changelog.d/20231117_102830_regis.md new file mode 100644 index 00000000000..4682872c159 --- /dev/null +++ b/changelog.d/20231117_102830_regis.md @@ -0,0 +1 @@ +- [Bugfix] Fix installation of tutor due to missing dev.txt file in Python package. (by @regisb) From 1f814f81cb89fcaf447dd786e40d2f66bbca92f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Fri, 17 Nov 2023 10:34:21 +0100 Subject: [PATCH 96/96] v16.1.7 --- CHANGELOG.md | 8 +++++++- changelog.d/20231117_102830_regis.md | 1 - tutor/__about__.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/20231117_102830_regis.md diff --git a/CHANGELOG.md b/CHANGELOG.md index adbc35a6630..90ab33e1bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,16 @@ instructions, because git commits are used to generate release notes: + +## v16.1.7 (2023-11-17) + +- [Feature] Upgrade to open-release/palm.4. It is strongly recommended to upgrade to this release for as long as possible before upgrading to Quince. Otherwise, many users will be logged out after the Quince upgrade and will have to log in again -- see the Quince release notes. (by @regisb) +- [Bugfix] Fix installation of tutor due to missing dev.txt file in Python package. (by @regisb) + ## v16.1.6 (2023-11-16) -- [Feature] Upgrade to open-release/palm.4. It is strongly recommended to upgrade to this release for as long as possible before upgrading to Quince. Otherwise, many users will be logged out after the Quince upgrade and will have to log in again -- see the Quince release notes. (by @regisb) +- [Feature] Upgrade to open-release/palm.4. (by @regisb) - [Improvement] Install tutor development tools with `pip install tutor[dev]`. (by @CodeWithEmad) diff --git a/changelog.d/20231117_102830_regis.md b/changelog.d/20231117_102830_regis.md deleted file mode 100644 index 4682872c159..00000000000 --- a/changelog.d/20231117_102830_regis.md +++ /dev/null @@ -1 +0,0 @@ -- [Bugfix] Fix installation of tutor due to missing dev.txt file in Python package. (by @regisb) diff --git a/tutor/__about__.py b/tutor/__about__.py index c70f7403c2c..60760ecdfa1 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "16.1.6" +__version__ = "16.1.7" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and