From 329b8c0be75f8dc5c6fe85b9279a6277d4a940a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Thu, 27 Apr 2023 20:25:20 +0200 Subject: [PATCH] feat: persistent bind-mounts This is an important change, where we get remove the previous `--mount` option, and instead opt for persistent bind-mounts. Persistent bind mounts have several advantages: - They make it easier to remember which folders need to be bind-mounted. - Code is *much* less clunky, as we no longer need to generate temporary docker-compose files. - They allow us to bind-mount host directories *at build time* using the buildx `--build-context` option. - The transition from development to production becomes much easier, as images will automatically be built using the host repo. The only drawback is that persistent bind-mounts are slightly less portable: when a config.yml file is moved to a different folder, many things will break if the repo is not checked out in the same path. For instance, this is how to start working on a local fork of edx-platform: tutor config save --append MOUNTS=/path/to/edx-platform And that's all there is to it. No, this fork will be used whenever we run: tutor images build openedx tutor local start tutor dev start This change is made possible by huge improvements in the build time performance. These improvements make it convenient to re-build Docker images often. Related issues: https://github.com/openedx/wg-developer-experience/issues/71 https://github.com/openedx/wg-developer-experience/issues/66 https://github.com/openedx/wg-developer-experience/issues/166 --- .../20230427_165520_regis_build_mount.md | 2 + docs/dev.rst | 100 ++++----- tests/commands/test_compose.py | 88 -------- tests/commands/test_images.py | 2 +- tests/test_bindmount.py | 65 ++++++ tutor/bindmount.py | 71 ++++++ tutor/bindmounts.py | 61 ------ tutor/commands/compose.py | 203 +----------------- tutor/commands/dev.py | 21 -- tutor/commands/images.py | 52 ++++- tutor/commands/local.py | 22 -- tutor/hooks/catalog.py | 55 ++--- tutor/images.py | 2 +- tutor/templates/config/defaults.yml | 1 + tutor/templates/local/docker-compose.jobs.yml | 6 + tutor/templates/local/docker-compose.yml | 32 ++- 16 files changed, 309 insertions(+), 474 deletions(-) delete mode 100644 tests/commands/test_compose.py create mode 100644 tests/test_bindmount.py create mode 100644 tutor/bindmount.py delete mode 100644 tutor/bindmounts.py diff --git a/changelog.d/20230427_165520_regis_build_mount.md b/changelog.d/20230427_165520_regis_build_mount.md index fb29b8cf85c..36fc658b9e4 100644 --- a/changelog.d/20230427_165520_regis_build_mount.md +++ b/changelog.d/20230427_165520_regis_build_mount.md @@ -1 +1,3 @@ - [Improvement] Automatically pull Docker image cache from the remote registry. Again, this will considerably improve image build-time, particularly in "cold-start" scenarios, where the images need to be built from scratch. The registry cache can be disabled with the `tutor images build --no-registry-cache` option. (by @regisb) +- [Feature] Automatically mount host folders *at build time*. This is a really important feature, as it allows us to transparently build images using local forks of remote repositories. (by @regisb) +- 💥[Deprecation] Remove the various `--mount` options. These options are replaced by persistent mounts. (by @regisb) diff --git a/docs/dev.rst b/docs/dev.rst index 5ee10c01a5e..684d806bd96 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -12,31 +12,25 @@ First-time setup Firstly, either :ref:`install Tutor ` (for development against the named releases of Open edX) or :ref:`install Tutor Nightly ` (for development against Open edX's master branches). +Then, optionally, tell Tutor to use a local fork of edx-platform. In that case you will need to rebuild the "openedx" Docker image:: + + tutor config save --append MOUNTS=./edx-platform + tutor images build openedx + Then, run one of the following in order to launch the developer platform setup process:: # To use the edx-platform repository that is built into the image, run: tutor dev launch - # To bind-mount and run a local clone of edx-platform, replace - # './edx-platform' with the path to the local clone and run: - tutor dev launch --mount=./edx-platform - This will perform several tasks. It will: * stop any existing locally-running Tutor containers, - * disable HTTPS, - * set ``LMS_HOST`` to `local.overhang.io `_ (a convenience domain that simply `points at 127.0.0.1 `_), - * prompt for a platform details (with suitable defaults), - * build an ``openedx-dev`` image, which is based ``openedx`` production image but is `specialized for developer usage`_, - * start LMS, CMS, supporting services, and any plugged-in services, - * ensure databases are created and migrated, and - * run service initialization scripts, such as service user creation and Waffle configuration. Additionally, when a local clone of edx-platform is bind-mounted, it will: @@ -55,10 +49,13 @@ Now, use the ``tutor dev ...`` command-line interface to manage the development .. note:: - Wherever the ``[--mount=./edx-platform]`` option is present, either: + If you've added your edx-platform to the ``MOUNTS`` setting, you can remove at any time by running:: + + tutor config save --remove MOUNTS=./edx-platform - * omit it when running of the edx-platform repository built into the image, or - * substitute it with ``--mount=``. + At any time, check your configuration by running:: + + tutor config printvalue MOUNTS Read more about bind-mounts :ref:`below `. @@ -74,17 +71,17 @@ Starting the platform back up Once first-time setup has been performed with ``launch``, the platform can be started going forward with the lighter-weight ``start -d`` command, which brings up containers *detached* (that is: in the background), but does not perform any initialization tasks:: - tutor dev start -d [--mount=./edx-platform] + tutor dev start -d Or, to start with platform with containers *attached* (that is: in the foreground, the current terminal), omit the ``-d`` flag:: - tutor dev start [--mount=./edx-platform] + tutor dev start When running containers attached, stop the platform with ``Ctrl+c``, or switch to detached mode using ``Ctrl+z``. Finally, the platform can also be started back up with ``launch``. It will take longer than ``start``, but it will ensure that config is applied, databases are provisioned & migrated, plugins are fully initialized, and (if applicable) the bind-mounted edx-platform is set up. Notably, ``launch`` is idempotent, so it is always safe to run it again without risk to data. Including the ``--pullimages`` flag will also ensure that container images are up-to-date:: - tutor dev launch [--mount=./edx-platform] --pullimages + tutor dev launch --pullimages Debugging with breakpoints -------------------------- @@ -92,32 +89,32 @@ Debugging with breakpoints To debug a local edx-platform repository, add a `python breakpoint `__ with ``breakpoint()`` anywhere in the code. Then, attach to the applicable service's container by running ``start`` (without ``-d``) followed by the service's name:: # Debugging LMS: - tutor dev start [--mount=./edx-platform] lms + tutor dev start lms # Or, debugging CMS: - tutor dev start [--mount=./edx-platform] cms + tutor dev start cms Running arbitrary commands -------------------------- To run any command inside one of the containers, run ``tutor dev run [OPTIONS] SERVICE [COMMAND] [ARGS]...``. For instance, to open a bash shell in the LMS or CMS containers:: - tutor dev run [--mount=./edx-platform] lms bash - tutor dev run [--mount=./edx-platform] cms bash + tutor dev run lms bash + tutor dev run cms bash To open a python shell in the LMS or CMS, run:: - tutor dev run [--mount=./edx-platform] lms ./manage.py lms shell - tutor dev run [--mount=./edx-platform] cms ./manage.py cms shell + tutor dev run lms ./manage.py lms shell + tutor dev run cms ./manage.py cms shell You can then import edx-platform and django modules and execute python code. To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor:: - tutor dev run [--mount=./edx-platform] lms openedx-assets build --env=dev + tutor dev run lms openedx-assets build --env=dev -.. _specialized for developer usage: +.. _specialized for developer usage: Rebuilding the openedx-dev image -------------------------------- @@ -143,35 +140,42 @@ Sharing directories with containers It may sometimes be convenient to mount container directories on the host, for instance: for editing and debugging. Tutor provides different solutions to this problem. -.. _mount_option: +.. _persistent_mounts: + +Persistent bind-mounted volumes with ``MOUNTS`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``MOUNTS`` is a Tutor setting to bind-mount host directories both at build time and run time: + +- At build time: plugins can automatically add certain directories listed in this setting to the `Docker build context `__. This makes it possible to transparently build a Docker image using a locally checked-out repository. +- At run time: host directories will be bind-mounted in running containers, using either an automatic or a manual configuration. + +After some values have been added to the ``MOUNTS`` setting, all ``tutor dev`` and ``tutor local`` commands will make use of these bind-mount volumes. -Bind-mount volumes with ``--mount`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Values added to ``MOUNTS`` can take one of two forms. The first is explicit:: -The ``launch``, ``run``, ``init`` and ``start`` subcommands of ``tutor dev`` and ``tutor local`` support the ``-m/--mount`` option (see :option:`tutor dev start -m`) which can take two different forms. The first is explicit:: + tutor config save --append MOUNTS=lms:/path/to/edx-platform:/openedx/edx-platform - tutor dev start --mount=lms:/path/to/edx-platform:/openedx/edx-platform lms +The second is implicit:: -And the second is implicit:: + tutor config save --append MOUNTS=/path/to/edx-platform - tutor dev start --mount=/path/to/edx-platform lms +With the explicit form, the setting means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container at run time". -With the explicit form, the ``--mount`` option means "bind-mount the host folder /path/to/edx-platform to /openedx/edx-platform in the lms container". +If you use the explicit format, you will quickly realise that you usually want to bind-mount folders in multiple containers at a time. For instance, you will want to bind-mount the edx-platform repository in the "cms" container, but also the "lms-worker" and "cms-worker" containers. To do that, write instead:: -If you use the explicit format, you will quickly realise that you usually want to bind-mount folders in multiple containers at a time. For instance, you will want to bind-mount the edx-platform repository in the "cms" container. To do that, write instead:: + # each service is added to a coma-separated list + tutor config save --append MOUNTS=lms,cms,lms-worker,cms-worker:/path/to/edx-platform:/openedx/edx-platform - tutor dev start --mount=lms,cms:/path/to/edx-platform:/openedx/edx-platform lms +This command line is a bit cumbersome. In addition, with this explicit form, the edx-platform repository will *not* be added to the build context at build time. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``MOUNTS`` setting. For instance, the following implicit form can be used instead of the explicit form above:: -This command line can become cumbersome and inconvenient to work with. But Tutor can be smart about bind-mounting folders to the right containers in the right place when you use the implicit form of the ``--mount`` option. For instance, the following commands are equivalent:: + tutor config save --append MOUNTS=/path/to/edx-platform - # Explicit form - tutor dev start --mount=lms,lms-worker,lms-job,cms,cms-worker,cms-job:/path/to/edx-platform:/openedx/edx-platform lms - # Implicit form - tutor dev start --mount=/path/to/edx-platform lms +With this implicit form, the edx-platform repo will be bind-mounted in the containers at run time, just like with the explicit form. But in addition, the edx-platform will also automatically be added to the Docker image at build time. -So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``--mount=~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: +So, when should you *not* be using the implicit form? That would be when Tutor does not know where to bind-mount your host folders. For instance, if you wanted to bind-mount your edx-platform virtual environment located in ``~/venvs/edx-platform``, you should not write ``--append MOUNTS=~/venvs/edx-platform``, because that folder would be mounted in a way that would override the edx-platform repository in the container. Instead, you should write:: - tutor dev start --mount=lms:~/venvs/edx-platform:/openedx/venv lms + tutor config save --append MOUNTS=lms:~/venvs/edx-platform:/openedx/venv .. note:: Remember to setup your edx-platform repository for development! See :ref:`edx_platform_dev_env`. @@ -182,16 +186,16 @@ Sometimes, you may want to modify some of the files inside a container for which tutor dev copyfrom lms /openedx/venv ~ -Then, bind-mount that folder back in the container with the ``--mount`` option (described :ref:`above `):: +Then, bind-mount that folder back in the container with the ``MOUNTS`` setting (described :ref:`above `):: - tutor dev start --mount lms:~/venv:/openedx/venv lms + tutor config save --append MOUNTS=lms:~/venv:/openedx/venv -You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. +You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your "lms" container. Manual bind-mount to any directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. warning:: Manually bind-mounting volumes with the ``--volume`` option makes it difficult to simultaneously bind-mount to multiple containers. Also, the ``--volume`` options are not compatible with ``start`` commands. For an alternative, see the :ref:`mount option `. +.. warning:: Manually bind-mounting volumes with the ``--volume`` option makes it difficult to simultaneously bind-mount to multiple containers. Also, the ``--volume`` options are not compatible with ``start`` commands. For an alternative, see the :ref:`persistent mounts `. The above solution may not work for you if you already have an existing directory, outside of the "volumes/" directory, which you would like mounted in one of your containers. For instance, you may want to mount your copy of the `edx-platform `__ repository. In such cases, you can simply use the ``-v/--volume`` `Docker option `__:: @@ -200,7 +204,7 @@ The above solution may not work for you if you already have an existing director Override docker-compose volumes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The above solutions require that you explicitly pass the ``-m/--mount`` options to every ``run``, ``start`` or ``init`` command, which may be inconvenient. To address these issues, you can create a ``docker-compose.override.yml`` file that will specify custom volumes to be used with all ``dev`` commands:: +Adding items to the ``MOUNTS`` setting effectively adds new bind-mount volumes to the ``docker-compose.yml`` files. But you might want to have more control over your volumes, such as adding read-only options, or customising other fields of the different services. To address these issues, you can create a ``docker-compose.override.yml`` file that will specify custom volumes to be used with all ``dev`` commands:: vim "$(tutor config printroot)/env/dev/docker-compose.override.yml" @@ -221,7 +225,7 @@ You are then free to bind-mount any directory to any container. For instance, to volumes: - /path/to/edx-platform:/openedx/edx-platform -This override file will be loaded when running any ``tutor dev ..`` command. The edx-platform repo mounted at the specified path will be automatically mounted inside all LMS and CMS containers. With this file, you should no longer specify the ``-m/--mount`` option from the command line. +This override file will be loaded when running any ``tutor dev ..`` command. The edx-platform repo mounted at the specified path will be automatically mounted inside all LMS and CMS containers. .. note:: The ``tutor local`` commands load the ``docker-compose.override.yml`` file from the ``$(tutor config printroot)/env/local/docker-compose.override.yml`` directory. One-time jobs from initialisation commands load the ``local/docker-compose.jobs.override.yml`` and ``dev/docker-compose.jobs.override.yml``. diff --git a/tests/commands/test_compose.py b/tests/commands/test_compose.py deleted file mode 100644 index 9cccfb94e91..00000000000 --- a/tests/commands/test_compose.py +++ /dev/null @@ -1,88 +0,0 @@ -from __future__ import annotations - -import typing as t -import unittest -from io import StringIO -from unittest.mock import patch - -from click.exceptions import ClickException - -from tutor import hooks -from tutor.commands import compose -from tutor.commands.local import LocalContext - - -class ComposeTests(unittest.TestCase): - maxDiff = None # Ensure we can see long diffs of YAML files. - - def test_mount_option_parsing(self) -> None: - param = compose.MountParam() - - self.assertEqual( - [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], - param("lms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms,cms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms, cms:/path/to/edx-platform:/openedx/edx-platform"), - ) - self.assertEqual( - [ - ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), - ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), - ], - param("lms,lms-worker:/path/to/edx-platform:/openedx/edx-platform"), - ) - with self.assertRaises(ClickException): - param("lms,:/path/to/edx-platform:/openedx/edx-platform") - - @patch("sys.stdout", new_callable=StringIO) - def test_compose_local_tmp_generation(self, _mock_stdout: StringIO) -> None: - """ - Ensure that docker-compose.tmp.yml is correctly generated. - """ - param = compose.MountParam() - mount_args = ( - # Auto-mounting of edx-platform to lms* and cms* - param.convert_implicit_form("/path/to/edx-platform"), - # Manual mounting of some other folder to mfe and lms - param.convert_explicit_form( - "mfe,lms:/path/to/something-else:/openedx/something-else" - ), - ) - # Mount volumes - compose.mount_tmp_volumes(mount_args, LocalContext("")) - - compose_file: dict[str, t.Any] = hooks.Filters.COMPOSE_LOCAL_TMP.apply({}) - actual_services: dict[str, t.Any] = compose_file["services"] - expected_services: dict[str, t.Any] = { - "cms": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "cms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "lms": { - "volumes": [ - "/path/to/edx-platform:/openedx/edx-platform", - "/path/to/something-else:/openedx/something-else", - ] - }, - "lms-worker": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "mfe": {"volumes": ["/path/to/something-else:/openedx/something-else"]}, - } - self.assertEqual(actual_services, expected_services) - - compose_jobs_file = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP.apply({}) - actual_jobs_services = compose_jobs_file["services"] - expected_jobs_services: dict[str, t.Any] = { - "cms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - "lms-job": {"volumes": ["/path/to/edx-platform:/openedx/edx-platform"]}, - } - self.assertEqual(actual_jobs_services, expected_jobs_services) diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py index 5c1c2933147..312f2e513af 100644 --- a/tests/commands/test_images.py +++ b/tests/commands/test_images.py @@ -149,7 +149,7 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None: "docker_args", "--cache-from=type=registry,ref=service1:1.0.0-cache", ], - list(image_build.call_args[0][1:]) + list(image_build.call_args[0][1:]), ) def test_images_push(self) -> None: diff --git a/tests/test_bindmount.py b/tests/test_bindmount.py new file mode 100644 index 00000000000..4e656a1c26e --- /dev/null +++ b/tests/test_bindmount.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import unittest + +from tutor import bindmount + + +class BindmountTests(unittest.TestCase): + def test_parse_explicit(self) -> None: + self.assertEqual( + [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], + bindmount.parse_explicit_mount( + "lms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms,cms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms, cms:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_explicit_mount( + "lms,lms-worker:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + self.assertEqual( + [("lms", "/path/to/edx-platform", "/openedx/edx-platform")], + bindmount.parse_explicit_mount( + "lms,:/path/to/edx-platform:/openedx/edx-platform" + ), + ) + + def test_parse_implicit(self) -> None: + # Import module to make sure filter is created + # pylint: disable=import-outside-toplevel,unused-import + import tutor.commands.compose + + self.assertEqual( + [ + ("lms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms-worker", "/path/to/edx-platform", "/openedx/edx-platform"), + ("lms-job", "/path/to/edx-platform", "/openedx/edx-platform"), + ("cms-job", "/path/to/edx-platform", "/openedx/edx-platform"), + ], + bindmount.parse_implicit_mount("/path/to/edx-platform"), + ) diff --git a/tutor/bindmount.py b/tutor/bindmount.py new file mode 100644 index 00000000000..0f2d2069b49 --- /dev/null +++ b/tutor/bindmount.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from functools import lru_cache +import os +import re +import typing as t + +from tutor import hooks + + +def iter_mounts(user_mounts: list[str], name: str) -> t.Iterable[str]: + """ + Iterate on the bind-mounts that are available to any given compose service. The list + of bind-mounts is parsed from `user_mounts` and we yield only those for service + `name`. + + Calling this function multiple times makes repeated calls to the parsing functions, + but that's OK because their result is cached. + """ + for user_mount in user_mounts: + for service, host_path, container_path in parse_mount(user_mount): + if service == name: + yield f"{host_path}:{container_path}" + + +def parse_mount(value: str) -> list[tuple[str, str, str]]: + """ + Parser for mount arguments of the form "service1[,service2,...]:/host/path:/container/path". + + Returns a list of (service, host_path, container_path) tuples. + """ + mounts = parse_explicit_mount(value) or parse_implicit_mount(value) + return mounts + + +@lru_cache(maxsize=None) +def parse_explicit_mount(value: str) -> list[tuple[str, str, str]]: + """ + Argument is of the form "containers:/host/path:/container/path". + """ + # Note that this syntax does not allow us to include colon ':' characters in paths + match = re.match( + r"(?P[a-zA-Z0-9-_, ]+):(?P[^:]+):(?P[^:]+)", + value, + ) + if not match: + return [] + + mounts: list[tuple[str, str, str]] = [] + services: list[str] = [service.strip() for service in match["services"].split(",")] + host_path = os.path.abspath(os.path.expanduser(match["host_path"])) + host_path = host_path.replace(os.path.sep, "/") + container_path = match["container_path"] + for service in services: + if service: + mounts.append((service, host_path, container_path)) + return mounts + + +@lru_cache(maxsize=None) +def parse_implicit_mount(value: str) -> list[tuple[str, str, str]]: + """ + Argument is of the form "/host/path" + """ + mounts: list[tuple[str, str, str]] = [] + host_path = os.path.abspath(os.path.expanduser(value)) + for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( + os.path.basename(host_path) + ): + mounts.append((service, host_path, container_path)) + return mounts diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py deleted file mode 100644 index d80735812cf..00000000000 --- a/tutor/bindmounts.py +++ /dev/null @@ -1,61 +0,0 @@ -import os - -from tutor.exceptions import TutorError -from tutor.tasks import BaseComposeTaskRunner -from tutor.utils import get_user_id - - -def create( - runner: BaseComposeTaskRunner, - service: str, - path: str, -) -> str: - volumes_root_path = get_root_path(runner.root) - volume_name = get_name(path) - container_volumes_root_path = "/tmp/volumes" - command = """rm -rf {volumes_path}/{volume_name} -cp -r {src_path} {volumes_path}/{volume_name} -chown -R {user_id} {volumes_path}/{volume_name}""".format( - volumes_path=container_volumes_root_path, - volume_name=volume_name, - src_path=path, - user_id=get_user_id(), - ) - - # Create volumes root dir if it does not exist. Otherwise it is created with root owner and might not be writable - # in the container, e.g: in the dev containers. - if not os.path.exists(volumes_root_path): - os.makedirs(volumes_root_path) - - runner.docker_compose( - "run", - "--rm", - "--no-deps", - "--user=0", - "--volume", - f"{volumes_root_path}:{container_volumes_root_path}", - service, - "sh", - "-e", - "-c", - command, - ) - return os.path.join(volumes_root_path, volume_name) - - -def get_path(root: str, container_bind_path: str) -> str: - bind_basename = get_name(container_bind_path) - return os.path.join(get_root_path(root), bind_basename) - - -def get_name(container_bind_path: str) -> str: - # We rstrip slashes, otherwise os.path.basename returns an empty string - # We don't use basename here as it will not work on Windows - name = container_bind_path.rstrip("/").split("/")[-1] - if not name: - raise TutorError("Mounting a container root folder is not supported") - return name - - -def get_root_path(root: str) -> str: - return os.path.join(root, "volumes") diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 44f515853a8..53112d0eb7c 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -1,17 +1,12 @@ from __future__ import annotations import os -import re -import typing as t -from copy import deepcopy import click -from click.shell_completion import CompletionItem -from typing_extensions import TypeAlias from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import fmt, hooks, serialize, utils +from tutor import bindmount, hooks, utils from tutor.commands import jobs from tutor.commands.context import BaseTaskContext from tutor.core.hooks import Filter # pylint: disable=unused-import @@ -19,8 +14,6 @@ from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config -COMPOSE_FILTER_TYPE: TypeAlias = "Filter[dict[str, t.Any], []]" - class ComposeTaskRunner(BaseComposeTaskRunner): def __init__(self, root: str, config: Config): @@ -47,47 +40,6 @@ def docker_compose(self, *command: str) -> int: *args, "--project-name", self.project_name, *command ) - def update_docker_compose_tmp( - self, - compose_tmp_filter: COMPOSE_FILTER_TYPE, - compose_jobs_tmp_filter: COMPOSE_FILTER_TYPE, - docker_compose_tmp_path: str, - docker_compose_jobs_tmp_path: str, - ) -> None: - """ - Update the contents of the docker-compose.tmp.yml and - docker-compose.jobs.tmp.yml files, which are generated at runtime. - """ - compose_base: dict[str, t.Any] = { - "version": "{{ DOCKER_COMPOSE_VERSION }}", - "services": {}, - } - - # 1. Apply compose_tmp filter - # 2. Render the resulting dict - # 3. Serialize to yaml - # 4. Save to disk - docker_compose_tmp: str = serialize.dumps( - tutor_env.render_unknown( - self.config, compose_tmp_filter.apply(deepcopy(compose_base)) - ) - ) - tutor_env.write_to( - docker_compose_tmp, - docker_compose_tmp_path, - ) - - # Same thing but with tmp jobs - docker_compose_jobs_tmp: str = serialize.dumps( - tutor_env.render_unknown( - self.config, compose_jobs_tmp_filter.apply(deepcopy(compose_base)) - ) - ) - tutor_env.write_to( - docker_compose_jobs_tmp, - docker_compose_jobs_tmp_path, - ) - def run_task(self, service: str, command: str) -> int: """ Run the "{{ service }}-job" service from local/docker-compose.jobs.yml with the @@ -113,148 +65,22 @@ def run_task(self, service: str, command: str) -> int: class BaseComposeContext(BaseTaskContext): - COMPOSE_TMP_FILTER: COMPOSE_FILTER_TYPE = NotImplemented - COMPOSE_JOBS_TMP_FILTER: COMPOSE_FILTER_TYPE = NotImplemented - def job_runner(self, config: Config) -> ComposeTaskRunner: raise NotImplementedError -class MountParam(click.ParamType): - """ - Parser for --mount arguments of the form "service1[,service2,...]:/host/path:/container/path". - """ - - name = "mount" - MountType = t.Tuple[str, str, str] - # Note that this syntax does not allow us to include colon ':' characters in paths - PARAM_REGEXP = ( - r"(?P[a-zA-Z0-9-_, ]+):(?P[^:]+):(?P[^:]+)" - ) - - def convert( - self, - value: str, - param: t.Optional["click.Parameter"], - ctx: t.Optional[click.Context], - ) -> list["MountType"]: - mounts = self.convert_explicit_form(value) or self.convert_implicit_form(value) - return mounts - - def convert_explicit_form(self, value: str) -> list["MountParam.MountType"]: - """ - Argument is of the form "containers:/host/path:/container/path". - """ - match = re.match(self.PARAM_REGEXP, value) - if not match: - return [] - - mounts: list["MountParam.MountType"] = [] - services: list[str] = [ - service.strip() for service in match["services"].split(",") - ] - host_path = os.path.abspath(os.path.expanduser(match["host_path"])) - host_path = host_path.replace(os.path.sep, "/") - container_path = match["container_path"] - for service in services: - if not service: - self.fail(f"incorrect services syntax: '{match['services']}'") - mounts.append((service, host_path, container_path)) - return mounts - - def convert_implicit_form(self, value: str) -> list["MountParam.MountType"]: - """ - Argument is of the form "/host/path" - """ - mounts: list["MountParam.MountType"] = [] - host_path = os.path.abspath(os.path.expanduser(value)) - for service, container_path in hooks.Filters.COMPOSE_MOUNTS.iterate( - os.path.basename(host_path) - ): - mounts.append((service, host_path, container_path)) - if not mounts: - raise self.fail(f"no mount found for {value}") - return mounts - - def shell_complete( - self, ctx: click.Context, param: click.Parameter, incomplete: str - ) -> list[CompletionItem]: - """ - Mount argument completion works only for the single path (implicit) form. The - reason is that colons break words in bash completion: - http://tiswww.case.edu/php/chet/bash/FAQ (E13) - Thus, we do not even attempt to auto-complete mount arguments that include - colons: such arguments will not even reach this method. - """ - return [CompletionItem(incomplete, type="file")] - - -mount_option = click.option( - "-m", - "--mount", - "mounts", - help="""Bind-mount a folder from the host in the right containers. This option can take two different forms. The first one is explicit: 'service1[,service2...]:/host/path:/container/path'. The other is implicit: '/host/path'. Arguments passed in the implicit form will be parsed by plugins to define the right folders to bind-mount from the host.""", - type=MountParam(), - multiple=True, -) - - -def mount_tmp_volumes( - all_mounts: tuple[list[MountParam.MountType], ...], - context: BaseComposeContext, -) -> None: - for mounts in all_mounts: - for service, host_path, container_path in mounts: - mount_tmp_volume(service, host_path, container_path, context) - - -def mount_tmp_volume( - service: str, - host_path: str, - container_path: str, - context: BaseComposeContext, -) -> None: - """ - Append user-defined bind-mounted volumes to the docker-compose.tmp file(s). - - The service/host path/container path values are appended to the docker-compose - files by mean of two filters. Each dev/local environment is then responsible for - generating the files based on the output of these filters. - - Bind-mounts that are associated to "*-job" services will be added to the - docker-compose jobs file. - """ - fmt.echo_info(f"Bind-mount: {host_path} -> {container_path} in {service}") - compose_tmp_filter: COMPOSE_FILTER_TYPE = ( - context.COMPOSE_JOBS_TMP_FILTER - if service.endswith("-job") - else context.COMPOSE_TMP_FILTER - ) - - @compose_tmp_filter.add() - def _add_mounts_to_docker_compose_tmp( - docker_compose: dict[str, t.Any], - ) -> dict[str, t.Any]: - services = docker_compose.setdefault("services", {}) - services.setdefault(service, {"volumes": []}) - services[service]["volumes"].append(f"{host_path}:{container_path}") - return docker_compose - - @click.command( short_help="Run all or a selection of services.", help="Run all or a selection of services. Docker images will be rebuilt where necessary.", ) @click.option("--skip-build", is_flag=True, help="Skip image building") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") -@mount_option @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def start( context: BaseComposeContext, skip_build: bool, detach: bool, - mounts: tuple[list[MountParam.MountType]], services: list[str], ) -> None: command = ["up", "--remove-orphans"] @@ -264,7 +90,6 @@ def start( command.append("-d") # Start services - mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) context.job_runner(config).docker_compose(*command, *services) @@ -313,21 +138,11 @@ def restart(context: BaseComposeContext, services: list[str]) -> None: @jobs.do_group -@mount_option -@click.pass_obj -def do(context: BaseComposeContext, mounts: tuple[list[MountParam.MountType]]) -> None: +def do() -> None: """ Run a custom job in the right container(s). """ - @hooks.Actions.DO_JOB.add() - def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: - """ - We add this logic to an action callback because we do not want to trigger it - whenever we run `tutor local do --help`. - """ - mount_tmp_volumes(mounts, context) - @click.command( short_help="Run a command in a new container", @@ -338,18 +153,16 @@ def _mount_tmp_volumes(_job_name: str, *_args: t.Any, **_kwargs: t.Any) -> None: ), context_settings={"ignore_unknown_options": True}, ) -@mount_option @click.argument("args", nargs=-1, required=True) @click.pass_context def run( context: click.Context, - mounts: tuple[list[MountParam.MountType]], args: list[str], ) -> None: extra_args = ["--rm"] if not utils.is_a_tty(): extra_args.append("-T") - context.invoke(dc_command, mounts=mounts, command="run", args=[*extra_args, *args]) + context.invoke(dc_command, command="run", args=[*extra_args, *args]) @click.command( @@ -446,17 +259,14 @@ def status(context: click.Context) -> None: context_settings={"ignore_unknown_options": True}, name="dc", ) -@mount_option @click.argument("command") @click.argument("args", nargs=-1) @click.pass_obj def dc_command( context: BaseComposeContext, - mounts: tuple[list[MountParam.MountType]], command: str, args: list[str], ) -> None: - mount_tmp_volumes(mounts, context) config = tutor_config.load(context.root) context.job_runner(config).docker_compose(command, *args) @@ -466,8 +276,8 @@ def _mount_edx_platform( volumes: list[tuple[str, str]], name: str ) -> list[tuple[str, str]]: """ - When mounting edx-platform with `--mount=/path/to/edx-platform`, bind-mount the host - repo in the lms/cms containers. + When mounting edx-platform with `tutor config save --append MOUNTS=/path/to/edx-platform`, + bind-mount the host repo in the lms/cms containers. """ if name == "edx-platform": path = "/openedx/edx-platform" @@ -482,6 +292,9 @@ def _mount_edx_platform( return volumes +hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) + + def add_commands(command_group: click.Group) -> None: command_group.add_command(start) command_group.add_command(stop) diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 0fcb58df3d0..fe4ed61ae81 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -18,39 +18,21 @@ def __init__(self, root: str, config: Config): """ super().__init__(root, config) self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str) - docker_compose_tmp_path = tutor_env.pathjoin( - self.root, "dev", "docker-compose.tmp.yml" - ) - docker_compose_jobs_tmp_path = tutor_env.pathjoin( - self.root, "dev", "docker-compose.jobs.tmp.yml" - ) self.docker_compose_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"), - docker_compose_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"), ] self.docker_compose_job_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"), - docker_compose_jobs_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"), tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"), ] - # Update docker-compose.tmp files - self.update_docker_compose_tmp( - hooks.Filters.COMPOSE_DEV_TMP, - hooks.Filters.COMPOSE_DEV_JOBS_TMP, - docker_compose_tmp_path, - docker_compose_jobs_tmp_path, - ) class DevContext(compose.BaseComposeContext): - COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_DEV_TMP - COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_DEV_JOBS_TMP - def job_runner(self, config: Config) -> DevTaskRunner: return DevTaskRunner(self.root, config) @@ -64,15 +46,12 @@ def dev(context: click.Context) -> None: @click.command(help="Configure and run Open edX from scratch, for development") @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") -@compose.mount_option @click.pass_context def launch( context: click.Context, non_interactive: bool, pullimages: bool, - mounts: tuple[list[compose.MountParam.MountType]], ) -> None: - compose.mount_tmp_volumes(mounts, context.obj) try: utils.check_macos_docker_memory() except exceptions.TutorError as e: diff --git a/tutor/commands/images.py b/tutor/commands/images.py index a162bee086c..09b31cece6e 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -1,12 +1,13 @@ from __future__ import annotations +import os import typing as t import click from tutor import config as tutor_config from tutor import env as tutor_env -from tutor import exceptions, hooks, images, utils +from tutor import exceptions, hooks, images, types, utils from tutor.commands.context import Context from tutor.core.hooks import Filter from tutor.types import Config @@ -141,16 +142,26 @@ def build( command_args.append("--load") if docker_args: command_args += docker_args + # Build context mounts + build_contexts = get_image_build_contexts(config) + for image in image_names: - for _name, path, tag, custom_args in find_images_to_build(config, image): + for name, path, tag, custom_args in find_images_to_build(config, image): image_build_args = [*command_args, *custom_args] + + # Registry cache if not no_registry_cache: - # Use registry cache image_build_args.append(f"--cache-from=type=registry,ref={tag}-cache") if cache_to_registry: image_build_args.append( f"--cache-to=type=registry,mode=max,ref={tag}-cache" ) + + # Build contexts + for host_path, stage_name in build_contexts.get(name, []): + image_build_args.append(f"--build-context={stage_name}={host_path}") + + # Build images.build( tutor_env.pathjoin(context.root, *path), tag, @@ -158,6 +169,41 @@ def build( ) +def get_image_build_contexts(config: Config) -> dict[str, list[tuple[str, str]]]: + """ + Return all build contexts for all images. + + A build context is to bind-mount a host directory at build-time. This is useful, for + instance to build a Docker image with a local git checkout of a remote repo. + + Users configure bind-mounts with the `MOUNTS` config setting. Plugins can then + automaticall add build contexts based on these values. + """ + user_mounts = types.get_typed(config, "MOUNTS", list) + build_contexts: dict[str, list[tuple[str, str]]] = {} + for user_mount in user_mounts: + for image_name, stage_name in hooks.Filters.IMAGES_BUILD_MOUNTS.iterate( + user_mount + ): + if image_name not in build_contexts: + build_contexts[image_name] = [] + build_contexts[image_name].append((user_mount, stage_name)) + return build_contexts + + +@hooks.Filters.IMAGES_BUILD_MOUNTS.add() +def _mount_edx_platform( + volumes: list[tuple[str, str]], path: str +) -> list[tuple[str, str]]: + """ + Automatically add an edx-platform repo from the host to the build context whenever + it is added to the `MOUNTS` setting. + """ + if os.path.basename(path) == "edx-platform": + volumes.append(("openedx", "edx-platform")) + return volumes + + @click.command(short_help="Pull images from the Docker registry") @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj diff --git a/tutor/commands/local.py b/tutor/commands/local.py index e391b9ccfb8..79ed396e2d7 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -23,38 +23,19 @@ def __init__(self, root: str, config: Config): """ super().__init__(root, config) self.project_name = get_typed(self.config, "LOCAL_PROJECT_NAME", str) - docker_compose_tmp_path = tutor_env.pathjoin( - self.root, "local", "docker-compose.tmp.yml" - ) - docker_compose_jobs_tmp_path = tutor_env.pathjoin( - self.root, "local", "docker-compose.jobs.tmp.yml" - ) self.docker_compose_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.yml"), tutor_env.pathjoin(self.root, "local", "docker-compose.prod.yml"), - docker_compose_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"), ] self.docker_compose_job_files += [ tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"), - docker_compose_jobs_tmp_path, tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"), ] - # Update docker-compose.tmp files - self.update_docker_compose_tmp( - hooks.Filters.COMPOSE_LOCAL_TMP, - hooks.Filters.COMPOSE_LOCAL_JOBS_TMP, - docker_compose_tmp_path, - docker_compose_jobs_tmp_path, - ) - # pylint: disable=too-few-public-methods class LocalContext(compose.BaseComposeContext): - COMPOSE_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_TMP - COMPOSE_JOBS_TMP_FILTER = hooks.Filters.COMPOSE_LOCAL_JOBS_TMP - def job_runner(self, config: Config) -> LocalTaskRunner: return LocalTaskRunner(self.root, config) @@ -66,17 +47,14 @@ def local(context: click.Context) -> None: @click.command(help="Configure and run Open edX from scratch") -@compose.mount_option @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @click.pass_context def launch( context: click.Context, - mounts: tuple[list[compose.MountParam.MountType]], non_interactive: bool, pullimages: bool, ) -> None: - compose.mount_tmp_volumes(mounts, context.obj) try: utils.check_macos_docker_memory() except exceptions.TutorError as e: diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 9db9d7037fe..a21fd515ac0 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -227,33 +227,20 @@ def your_filter_callback(some_data): "commands:pre-init" ) - #: Same as :py:data:`COMPOSE_LOCAL_JOBS_TMP` but for the development environment. - COMPOSE_DEV_JOBS_TMP: Filter[Config, []] = filters.get("compose:dev-jobs:tmp") - - #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for the development environment. - COMPOSE_DEV_TMP: Filter[Config, []] = filters.get("compose:dev:tmp") - - #: Same as :py:data:`COMPOSE_LOCAL_TMP` but for jobs - COMPOSE_LOCAL_JOBS_TMP: Filter[Config, []] = filters.get("compose:local-jobs:tmp") - - #: Contents of the (local|dev)/docker-compose.tmp.yml files that will be generated at - #: runtime. This is used for instance to bind-mount folders from the host (see - #: :py:data:`COMPOSE_MOUNTS`) - #: - #: :parameter dict[str, ...] docker_compose_tmp: values which will be serialized to local/docker-compose.tmp.yml. - #: Keys and values will be rendered before saving, such that you may include ``{{ ... }}`` statements. - COMPOSE_LOCAL_TMP: Filter[Config, []] = filters.get("compose:local:tmp") - #: List of folders to bind-mount in docker-compose containers, either in ``tutor local`` or ``tutor dev``. #: - #: Many ``tutor local`` and ``tutor dev`` commands support ``--mounts`` options - #: that allow plugins to define custom behaviour at runtime. For instance - #: ``--mount=/path/to/edx-platform`` would cause this host folder to be - #: bind-mounted in different containers (lms, lms-worker, cms, cms-worker) at the + #: This filter is for processing values of the ``MOUNTS`` setting such as:: + #: + #: tutor config save --append MOUNTS=/path/to/edx-platform + #: + #: In this example, this host folder would be bind-mounted in different containers + #: (lms, lms-worker, cms, cms-worker, lms-job, cms-job) at the #: /openedx/edx-platform location. Plugin developers may implement this filter to #: define custom behaviour when mounting folders that relate to their plugins. For - #: instance, the ecommerce plugin may process the ``--mount=/path/to/ecommerce`` - #: option. + #: instance, the ecommerce plugin may process the ``/path/to/ecommerce`` value. + #: + #: To also bind-mount these folder at build time, implement also the + #: :py:data:`IMAGES_BUILD_MOUNTS` filter. #: #: :parameter list[tuple[str, str]] mounts: each item is a ``(service, path)`` #: tuple, where ``service`` is the name of the docker-compose service and ``path`` is @@ -262,7 +249,7 @@ def your_filter_callback(some_data): #: the ``path`` because it will fail on Windows. #: :parameter str name: basename of the host-mounted folder. In the example above, #: this is "edx-platform". When implementing this filter you should check this name to - #: conditionnally add mounts. + #: conditionally add mounts. COMPOSE_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get("compose:mounts") #: Declare new default configuration settings that don't necessarily have to be saved in the user @@ -402,6 +389,26 @@ def your_filter_callback(some_data): list[tuple[str, tuple[str, ...], str, tuple[str, ...]]], [Config] ] = filters.get("images:build") + #: List of host directories to be automatically bind-mounted in Docker images at + #: build time. For instance, this is useful to build Docker images using a custom + #: repository on the host. + #: + #: This filter works similarly to the :py:data:`COMPOSE_MOUNTS` filter, with a few differences. + #: + #: :parameter list[tuple[str, str]] mounts: each item is a pair of ``(name, value)`` + #: used to generate a build context at build time. See the corresponding `Docker + #: documentation `__. + #: The following option will be added to the ``docker buildx build`` command: + #: ``--build-context={name}={value}``. If the Dockerfile contains a "name" stage, then + #: that stage will be replaced by the corresponding directory on the host. + #: :parameter str name: full path to the host-mounted folder. As opposed to + #: :py:data:`COMPOSE_MOUNTS`, this is not just the basename, but the full path. When + #: implementing this filter you should check this path (for instance: with + #: ``os.path.basename(path)``) to conditionally add mounts. + IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = filters.get( + "images:build:mounts" + ) + #: List of images to be pulled when we run ``tutor images pull ...``. #: #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. diff --git a/tutor/images.py b/tutor/images.py index 0d1e80bd578..dc640d00ee9 100644 --- a/tutor/images.py +++ b/tutor/images.py @@ -9,7 +9,7 @@ def get_tag(config: Config, name: str) -> str: def build(path: str, tag: str, *args: str) -> None: fmt.echo_info(f"Building image {tag}") - build_command = ["build", "-t", tag, *args, path] + build_command = ["build", f"--tag={tag}", *args, path] if utils.is_buildkit_enabled(): build_command.insert(0, "buildx") command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index d57f2673f30..9c4e823f744 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -43,6 +43,7 @@ MONGODB_USERNAME: "" MONGODB_PASSWORD: "" MONGODB_REPLICA_SET: "" MONGODB_USE_SSL: false +MOUNTS: [] OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" OPENEDX_CACHE_REDIS_DB: 1 diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index 8b87f9fd526..ad6a8ba80e0 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -14,6 +14,9 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "lms-job") %} + - {{ mount }} + {%- endfor %} depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }} cms-job: @@ -25,6 +28,9 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro + {%- for mount in iter_mounts(MOUNTS, "cms-job") %} + - {{ mount }} + {%- endfor %} depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB), ("elasticsearch", RUN_ELASTICSEARCH), ("redis", RUN_REDIS)]|list_if }} {{ patch("local-docker-compose-jobs-services")|indent(4) }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index b30afb18d33..2dc6b72cfcb 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -24,7 +24,7 @@ services: ############# External services - {% if RUN_MONGODB %} + {% if RUN_MONGODB -%} mongodb: image: {{ DOCKER_IMAGE_MONGODB }} # Use WiredTiger in all environments, just like at edx.org @@ -35,9 +35,9 @@ services: - ../../data/mongodb:/data/db depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_MYSQL %} + {% if RUN_MYSQL -%} mysql: image: {{ DOCKER_IMAGE_MYSQL }} command: mysqld --character-set-server=utf8 --collation-server=utf8_general_ci @@ -47,9 +47,9 @@ services: - ../../data/mysql:/var/lib/mysql environment: MYSQL_ROOT_PASSWORD: "{{ MYSQL_ROOT_PASSWORD }}" - {% endif %} + {%- endif %} - {% if RUN_ELASTICSEARCH %} + {% if RUN_ELASTICSEARCH -%} elasticsearch: image: {{ DOCKER_IMAGE_ELASTICSEARCH }} environment: @@ -67,9 +67,9 @@ services: - ../../data/elasticsearch:/usr/share/elasticsearch/data depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_REDIS %} + {% if RUN_REDIS -%} redis: image: {{ DOCKER_IMAGE_REDIS }} working_dir: /openedx/redis/data @@ -81,16 +81,16 @@ services: restart: unless-stopped depends_on: - permissions - {% endif %} + {%- endif %} - {% if RUN_SMTP %} + {% if RUN_SMTP -%} smtp: image: {{ DOCKER_IMAGE_SMTP }} restart: unless-stopped user: "100:101" environment: HOSTNAME: "{{ LMS_HOST }}" - {% endif %} + {%- endif %} ############# LMS and CMS @@ -108,6 +108,9 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/edx-platform/uwsgi.ini:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "lms") %} + - {{ mount }} + {%- endfor %} depends_on: - permissions {% if RUN_MYSQL %}- mysql{% endif %} @@ -131,6 +134,9 @@ services: - ../apps/openedx/uwsgi.ini:/openedx/edx-platform/uwsgi.ini:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "cms") %} + - {{ mount }} + {%- endfor %} depends_on: - permissions - lms @@ -156,6 +162,9 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/lms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "lms-worker") %} + - {{ mount }} + {%- endfor %} depends_on: - lms @@ -172,6 +181,9 @@ services: - ../apps/openedx/config:/openedx/config:ro - ../../data/cms:/openedx/data - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "cms-worker") %} + - {{ mount }} + {%- endfor %} depends_on: - cms