From b01db0f61c112f7f6e5db5ab5e0d54d40ba40b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Behmo?= Date: Mon, 11 Apr 2022 17:26:17 +0200 Subject: [PATCH] v14.0.0: upgrade to Nutmeg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 💥 [Feature] Upgrade to Nutmeg: (by @regisb) - 💥 [Feature] Persistent grades are now enabled by default. - [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). - [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running --- CHANGELOG.md | 13 ++ docs/configuration.rst | 4 +- docs/dev.rst | 17 +- docs/install.rst | 4 +- docs/plugins/intro.rst | 16 +- docs/plugins/v0/api.rst | 9 +- docs/reference/patches.rst | 12 +- requirements/plugins.txt | 24 +-- tests/commands/test_dev.py | 5 - tests/test_bindmounts.py | 36 ---- tests/test_env.py | 2 +- tutor/__about__.py | 2 +- tutor/bindmounts.py | 83 -------- tutor/commands/cli.py | 12 +- tutor/commands/compose.py | 182 +++++++++--------- tutor/commands/config.py | 29 ++- tutor/commands/dev.py | 44 +---- tutor/commands/images.py | 31 ++- tutor/commands/k8s.py | 119 ++++++++---- tutor/commands/local.py | 22 ++- tutor/commands/plugins.py | 57 +++--- tutor/commands/upgrade/k8s.py | 44 +++++ tutor/commands/upgrade/local.py | 22 +++ tutor/env.py | 1 + .../apps/openedx/config/cms.env.json | 84 -------- .../templates/apps/openedx/config/cms.env.yml | 73 +++++++ .../apps/openedx/config/lms.env.json | 97 ---------- .../templates/apps/openedx/config/lms.env.yml | 86 +++++++++ .../apps/openedx/config/partials/auth.json | 26 --- .../apps/openedx/config/partials/auth.yml | 22 +++ .../openedx/settings/partials/common_lms.py | 4 + tutor/templates/build/openedx/Dockerfile | 30 ++- tutor/templates/build/openedx/revisions.yml | 2 +- tutor/templates/config/defaults.yml | 2 +- tutor/templates/dev/docker-compose.yml | 4 - tutor/templates/hooks/lms/init | 1 + tutor/templates/k8s/deployments.yml | 4 +- tutor/templates/local/docker-compose.jobs.yml | 2 +- tutor/templates/local/docker-compose.yml | 4 +- 39 files changed, 621 insertions(+), 610 deletions(-) delete mode 100644 tests/test_bindmounts.py delete mode 100644 tutor/bindmounts.py delete mode 100644 tutor/templates/apps/openedx/config/cms.env.json create mode 100644 tutor/templates/apps/openedx/config/cms.env.yml delete mode 100644 tutor/templates/apps/openedx/config/lms.env.json create mode 100644 tutor/templates/apps/openedx/config/lms.env.yml delete mode 100644 tutor/templates/apps/openedx/config/partials/auth.json create mode 100644 tutor/templates/apps/openedx/config/partials/auth.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index ac1e78e7d89..6916f69a850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,19 @@ Every user-facing change should have an entry in this changelog. Please respect ## Unreleased +## v14.0.0 (2022-06-09) + +- 💥[Feature] Upgrade to Nutmeg: (by @regisb) + - 💥[Feature] Get rid of the `dev/local bindmount` commands which were replaced since v13.2.0 by the `--mount` option. + - 💥[Feature] Get rid of the `dev runserver` command which was replaced since v13.2.0 by `dev start`. + - [Improvement] Add `-h` option, in addition to `--help`, to print CLI help. + - 💥[Feature] Hide a course from the `/course` search page in the LMS when the course visibility is set to "none" in the Studio. (thanks @ghassanmas!) + - 💥[Improvement] The `lms.env.json` and `cms.env.json` files are moved to `lms.env.yml` and `cms.env.yml`. As a consequence, plugin developers must reformat the following patches to use YAML format, and not JSON: "common-env-features", "lms-env-features", "cms-env-features", "lms-env", "cms-env", "openedx-auth". + - 💥[Feature] Persistent grades are now enabled by default. + - [Bugfix] Remove edX references from bulk emails ([issue](https://github.com/openedx/build-test-release-wg/issues/100)). + - [Improvement] For Tutor Nightly (and only Nightly), official plugins are now installed from their nightly branches on GitHub instead of a version range on PyPI. This will allow Nightly users to install all official plugins by running ``pip install -e ".[full]"``. + - [Bugfix] Start MongoDB when running migrations, because a new data migration fails if MongoDB is not running + ## v13.3.1 (2022-06-06) - [Fix] Crashing celery workers in development (#681). (by @regisb) diff --git a/docs/configuration.rst b/docs/configuration.rst index d80328774b0..dce6331bab3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -8,7 +8,7 @@ Tutor offers plenty of possibilities for platform customisation out of the box. a. Modifying the Tutor :ref:`configuration parameters `. b. Modifying the :ref:`Open edX docker image ` that runs the Open edX platform. -This section does not cover :ref:`plugin development `. For simple changes, such as modifying the ``*.env.json`` files or the edx-platform settings, *you should not fork edx-platform or tutor*! Instead, you should create a simple :ref:`plugin for Tutor `. +This section does not cover :ref:`plugin development `. For simple changes, such as modifying the ``*.env.yml`` files or the edx-platform settings, *you should not fork edx-platform or tutor*! Instead, you should create a simple :ref:`plugin for Tutor `. .. _configuration: @@ -31,7 +31,7 @@ Or from the system environment:: export TUTOR_PARAM1=VALUE1 -Once the base configuration is created or updated, the environment is automatically re-generated. The environment is the set of all files required to manage an Open edX platform: Dockerfile, ``lms.env.json``, settings files, etc. You can view the environment files in the ``env`` folder:: +Once the base configuration is created or updated, the environment is automatically re-generated. The environment is the set of all files required to manage an Open edX platform: Dockerfile, ``lms.env.yml``, settings files, etc. You can view the environment files in the ``env`` folder:: ls "$(tutor config printroot)/env" diff --git a/docs/dev.rst b/docs/dev.rst index e599933dc7c..dcff710f48a 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -152,23 +152,8 @@ Then, bind-mount that folder back in the container with the ``--mount`` option ( You can then edit the files in ``~/venv`` on your local filesystem and see the changes live in your container. -Bind-mount from the "volumes/" directory -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. warning:: Bind-mounting volumes with the ``bindmount`` command is no longer the default, recommended way of bind-mounting volumes from the host. Instead, see the :ref:`mount option ` and the ``tutor dev/local copyfrom`` commands. - -Tutor makes it easy to create a bind-mount from an existing container. First, copy the contents of a container directory with the ``bindmount`` command. For instance, to copy the virtual environment of the "lms" container:: - - tutor dev bindmount lms /openedx/venv - -This command recursively copies the contents of the ``/opendedx/venv`` directory to ``$(tutor config printroot)/volumes/venv``. The code of any Python dependency can then be edited -- for instance, you can then add a ``import ipdb; ipdb.set_trace()`` statement for step-by-step debugging, or implement a custom feature. - -Then, bind-mount the directory back in the container with the ``--mount`` option:: - - tutor dev start --mount=lms:$(tutor config printroot)/volumes/venv:/openedx/venv lms - .. note:: - The ``bindmount`` command and the ``--mount=...`` option syntax are available both for the ``tutor local`` and ``tutor dev`` commands. + The ``--mount=...`` option syntax is available both for the ``tutor local`` and ``tutor dev`` commands. Manual bind-mount to any directory ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/install.rst b/docs/install.rst index 01253ff9f86..b9f8aa993f1 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -121,11 +121,11 @@ Major Open edX releases are published twice a year, in June and December, by the 4. Test the new release in a sandboxed environment. 5. If you are running edx-platform, or some other repository from a custom branch, then you should rebase (and test) your changes on top of the latest release tag (see :ref:`edx_platform_fork`). -The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``quickstart`` command (see above). The single difference is that if the ``quickstart`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``quickstart``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Lilac to Maple and rebuild some Docker images, run:: +The process for upgrading from one major release to the next works similarly to any other upgrade, with the ``quickstart`` command (see above). The single difference is that if the ``quickstart`` command detects that your tutor environment was generated with an older release, it will perform a few release-specific upgrade steps. These extra upgrade steps will be performed just once. But they will be ignored if you updated your local environment (for instance: with ``tutor config save``) before running ``quickstart``. This situation typically occurs if you need to re-build some Docker images (see above). In such a case, you should make use of the ``upgrade`` command. For instance, to upgrade a local installation from Maple to Nutmeg and rebuild some Docker images, run:: tutor config save tutor images build all # list the images that should be rebuilt here - tutor local upgrade --from=lilac + tutor local upgrade --from=maple tutor local quickstart .. _autocomplete: diff --git a/docs/plugins/intro.rst b/docs/plugins/intro.rst index 0de2fdb8c7c..2feb8413e6e 100644 --- a/docs/plugins/intro.rst +++ b/docs/plugins/intro.rst @@ -4,7 +4,7 @@ Introduction ============ -Tutor comes with a plugin system that allows anyone to customise the deployment of an Open edX platform very easily. The vision behind this plugin system is that users should not have to fork the Tutor repository to customise their deployments. For instance, if you have created a new application that integrates with Open edX, you should not have to describe how to manually patch the platform settings, ``urls.py`` or ``*.env.json`` files. Instead, you can create a "tutor-myapp" plugin for Tutor. Then, users will start using your application in three simple steps:: +Tutor comes with a plugin system that allows anyone to customise the deployment of an Open edX platform very easily. The vision behind this plugin system is that users should not have to fork the Tutor repository to customise their deployments. For instance, if you have created a new application that integrates with Open edX, you should not have to describe how to manually patch the platform settings, ``urls.py`` or ``*.env.yml`` files. Instead, you can create a "tutor-myapp" plugin for Tutor. Then, users will start using your application in three simple steps:: # 1) Install the plugin pip install tutor-myapp @@ -31,6 +31,20 @@ After enabling or disabling a plugin, the environment should be re-generated wit tutor config save + +Index management:: + + # List all plugins from the stored indices + tutor plugins index list + # Show a particular plugin + tutor plugins show ecommerce + # Install a plugin + tutor plugins install ecommerce + # Add a new index stored in a remote location (or a local path) + tutor plugins index add https://... + # Remove an existing index + tutor plugins index remove https://... + The full plugins CLI is described in the :ref:`reference documentation `. .. _existing_plugins: diff --git a/docs/plugins/v0/api.rst b/docs/plugins/v0/api.rst index 380474ac21c..ea8b80089e6 100644 --- a/docs/plugins/v0/api.rst +++ b/docs/plugins/v0/api.rst @@ -189,9 +189,10 @@ Example:: import click from tutor import config as tutor_config - @click.command(help="I'm a plugin command") + @click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_obj def command(context): + """"I'm a plugin command""" config = tutor_config.load(context.root) lms_host = config["LMS_HOST"] click.echo("Hello from myplugin!") @@ -207,12 +208,14 @@ You can even define subcommands by creating `command groups =13.0.0,<14.0.0 -tutor-discovery>=13.0.0,<14.0.0 -tutor-ecommerce>=13.0.0,<14.0.0 -tutor-forum>=13.0.0,<14.0.0 -tutor-license>=13.0.0,<14.0.0 -tutor-mfe>=13.0.0,<14.0.0 -tutor-minio>=13.0.0,<14.0.0 -tutor-notes>=13.0.0,<14.0.0 -tutor-richie>=13.0.0,<14.0.0 -tutor-webui>=13.0.0,<14.0.0 -tutor-xqueue>=13.0.0,<14.0.0 +# change version ranges when upgrading from nutmeg +tutor-android>=14.0.0,<15.0.0 +tutor-discovery>=14.0.0,<15.0.0 +tutor-ecommerce>=14.0.0,<15.0.0 +tutor-forum>=14.0.0,<15.0.0 +tutor-license>=14.0.0,<15.0.0 +tutor-mfe>=14.0.0,<15.0.0 +tutor-minio>=14.0.0,<15.0.0 +tutor-notes>=14.0.0,<15.0.0 +tutor-richie>=14.0.0,<15.0.0 +tutor-webui>=14.0.0,<15.0.0 +tutor-xqueue>=14.0.0,<15.0.0 diff --git a/tests/commands/test_dev.py b/tests/commands/test_dev.py index 2f911f584f6..0b962d723a3 100644 --- a/tests/commands/test_dev.py +++ b/tests/commands/test_dev.py @@ -8,8 +8,3 @@ def test_dev_help(self) -> None: result = self.invoke(["dev", "--help"]) self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) - - def test_dev_bindmount(self) -> None: - result = self.invoke(["dev", "bindmount", "--help"]) - self.assertEqual(0, result.exit_code) - self.assertIsNone(result.exception) diff --git a/tests/test_bindmounts.py b/tests/test_bindmounts.py deleted file mode 100644 index dbc4b4b11b0..00000000000 --- a/tests/test_bindmounts.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest - -from tutor import bindmounts -from tutor.exceptions import TutorError - - -class BindMountsTests(unittest.TestCase): - def test_get_name(self) -> None: - self.assertEqual("venv", bindmounts.get_name("/openedx/venv")) - self.assertEqual("venv", bindmounts.get_name("/openedx/venv/")) - - def test_get_name_root_folder(self) -> None: - with self.assertRaises(TutorError): - bindmounts.get_name("/") - with self.assertRaises(TutorError): - bindmounts.get_name("") - - def test_parse_volumes(self) -> None: - volume_args, non_volume_args = bindmounts.parse_volumes( - [ - "run", - "--volume=/openedx/venv", - "-v", - "/tmp/openedx:/openedx", - "lms", - "echo", - "boom", - ] - ) - self.assertEqual(("/openedx/venv", "/tmp/openedx:/openedx"), volume_args) - self.assertEqual(("run", "lms", "echo", "boom"), non_volume_args) - - def test_parse_volumes_empty_list(self) -> None: - volume_args, non_volume_args = bindmounts.parse_volumes([]) - self.assertEqual((), volume_args) - self.assertEqual((), non_volume_args) diff --git a/tests/test_env.py b/tests/test_env.py index 44447be5683..f0de1747e89 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -258,6 +258,6 @@ def test_current_version_in_latest_env(self) -> None: ) as f: f.write(__version__) self.assertEqual(__version__, env.current_version(root)) - self.assertEqual("maple", env.get_env_release(root)) + self.assertEqual("nutmeg", env.get_env_release(root)) self.assertIsNone(env.should_upgrade_from_release(root)) self.assertTrue(env.is_up_to_date(root)) diff --git a/tutor/__about__.py b/tutor/__about__.py index d210a12401d..36667fd7388 100644 --- a/tutor/__about__.py +++ b/tutor/__about__.py @@ -2,7 +2,7 @@ # Increment this version number to trigger a new release. See # docs/tutor.html#versioning for information on the versioning scheme. -__version__ = "13.3.1" +__version__ = "14.0.0" # The version suffix will be appended to the actual version, separated by a # dash. Use this suffix to differentiate between the actual released version and diff --git a/tutor/bindmounts.py b/tutor/bindmounts.py deleted file mode 100644 index 874507dea75..00000000000 --- a/tutor/bindmounts.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -from typing import List, Tuple - -import click - -from .exceptions import TutorError -from .jobs import BaseComposeJobRunner -from .utils import get_user_id - - -def create( - runner: BaseComposeJobRunner, - service: str, - path: str, -) -> str: - volumes_root_path = get_root_path(runner.root) - volume_name = get_name(path) - container_volumes_root_path = "/tmp/volumes" - command = """rm -rf {volumes_path}/{volume_name} -cp -r {src_path} {volumes_path}/{volume_name} -chown -R {user_id} {volumes_path}/{volume_name}""".format( - volumes_path=container_volumes_root_path, - volume_name=volume_name, - src_path=path, - user_id=get_user_id(), - ) - - # Create volumes root dir if it does not exist. Otherwise it is created with root owner and might not be writable - # in the container, e.g: in the dev containers. - if not os.path.exists(volumes_root_path): - os.makedirs(volumes_root_path) - - runner.docker_compose( - "run", - "--rm", - "--no-deps", - "--user=0", - "--volume", - f"{volumes_root_path}:{container_volumes_root_path}", - service, - "sh", - "-e", - "-c", - command, - ) - return os.path.join(volumes_root_path, volume_name) - - -def get_path(root: str, container_bind_path: str) -> str: - bind_basename = get_name(container_bind_path) - return os.path.join(get_root_path(root), bind_basename) - - -def get_name(container_bind_path: str) -> str: - # We rstrip slashes, otherwise os.path.basename returns an empty string - # We don't use basename here as it will not work on Windows - name = container_bind_path.rstrip("/").split("/")[-1] - if not name: - raise TutorError("Mounting a container root folder is not supported") - return name - - -def get_root_path(root: str) -> str: - return os.path.join(root, "volumes") - - -def parse_volumes(docker_compose_args: List[str]) -> Tuple[List[str], List[str]]: - """ - Parse `-v/--volume` options from an arbitrary list of arguments. - """ - - @click.command(context_settings={"ignore_unknown_options": True}) - @click.option("-v", "--volume", "volumes", multiple=True) - @click.argument("args", nargs=-1) - def custom_docker_compose( - volumes: List[str], args: List[str] # pylint: disable=unused-argument - ) -> None: - pass - - if isinstance(docker_compose_args, tuple): - docker_compose_args = list(docker_compose_args) - context = custom_docker_compose.make_context("custom", docker_compose_args) - return context.params["volumes"], context.params["args"] diff --git a/tutor/commands/cli.py b/tutor/commands/cli.py index 444316f27bb..26aa4723bb2 100755 --- a/tutor/commands/cli.py +++ b/tutor/commands/cli.py @@ -89,7 +89,6 @@ def get_command( cls=TutorCli, invoke_without_command=True, add_help_option=False, # Context is incorrectly loaded when help option is automatically added - help="Tutor is the Docker-based Open edX distribution designed for peace of mind.", ) @click.version_option(version=__version__) @click.option( @@ -110,6 +109,9 @@ def get_command( ) @click.pass_context def cli(context: click.Context, root: str, show_help: bool) -> None: + """ + Tutor is the Docker-based Open edX distribution designed for peace of mind. + """ if utils.is_root(): fmt.echo_alert( "You are running Tutor as root. This is strongly not recommended. If you are doing this in order to access" @@ -121,9 +123,15 @@ def cli(context: click.Context, root: str, show_help: bool) -> None: click.echo(context.get_help()) -@click.command(help="Print this help", name="help") +@click.command( + name="help", + context_settings={"help_option_names": ["-h", "--help"]}, +) @click.pass_context def help_command(context: click.Context) -> None: + """ + Print this help + """ context.invoke(cli, show_help=True) diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index fd5d9bc5033..83cbe351901 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -4,7 +4,6 @@ import click -from tutor import bindmounts from tutor import config as tutor_config from tutor import env as tutor_env from tutor import fmt, hooks, jobs, serialize, utils @@ -156,10 +155,7 @@ def convert( ) -@click.command( - short_help="Run all or a selection of services.", - help="Run all or a selection of services. Docker images will be rebuilt where necessary.", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--skip-build", is_flag=True, help="Skip image building") @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @mount_option @@ -172,6 +168,11 @@ def start( mounts: t.Tuple[t.List[MountParam.MountType]], services: t.List[str], ) -> None: + """ + Run all or a selection of services. + + Docker images will be rebuilt where necessary. + """ command = ["up", "--remove-orphans"] if not skip_build: command.append("--build") @@ -185,36 +186,43 @@ def start( context.job_runner(config).docker_compose(*command, *services) -@click.command(help="Stop a running platform") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def stop(context: BaseComposeContext, services: t.List[str]) -> None: + """ + Stop a running platform. + """ config = tutor_config.load(context.root) context.job_runner(config).docker_compose("stop", *services) -@click.command( - short_help="Reboot an existing platform", - help="This is more than just a restart: with reboot, the platform is fully stopped before being restarted again", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode") @click.argument("services", metavar="service", nargs=-1) @click.pass_context def reboot(context: click.Context, detach: bool, services: t.List[str]) -> None: + """ + Reboot an existing platform + + This is more than just a restart: with reboot, the platform is fully stopped before being restarted again. + """ context.invoke(stop, services=services) context.invoke(start, detach=detach, services=services) -@click.command( - short_help="Restart some components from a running platform.", - help="""Specify 'openedx' to restart the lms, cms and workers, or 'all' to -restart all services. Note that this performs a 'docker-compose restart', so new images -may not be taken into account. It is useful for reloading settings, for instance. To -fully stop the platform, use the 'reboot' command.""", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("services", metavar="service", nargs=-1) @click.pass_obj def restart(context: BaseComposeContext, services: t.List[str]) -> None: + """ + Restart some components from a running platform + + Specify 'openedx' to restart the lms, cms and workers, or 'all' to restart + all services. Note that this performs a 'docker-compose restart', so new + images may not be taken into account. It is useful for reloading settings, + for instance. To fully stop the platform, use the 'reboot' command. + """ config = tutor_config.load(context.root) command = ["restart"] if "all" in services: @@ -231,7 +239,10 @@ def restart(context: BaseComposeContext, services: t.List[str]) -> None: context.job_runner(config).docker_compose(*command) -@click.command(help="Initialise all applications") +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, + help="Initialise all applications", +) @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") @mount_option @click.pass_obj @@ -246,7 +257,7 @@ def init( jobs.initialise(runner, limit_to=limit) -@click.command(help="Create an Open edX user and interactively set their password") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--superuser", is_flag=True, help="Make superuser") @click.option("--staff", is_flag=True, help="Make staff user") @click.option( @@ -265,15 +276,16 @@ def createuser( name: str, email: str, ) -> None: + """ + Create an Open edX user and interactively set their password + """ config = tutor_config.load(context.root) runner = context.job_runner(config) command = jobs.create_user_command(superuser, staff, name, email, password=password) runner.run_job("lms", command) -@click.command( - help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( "-d", "--domain", @@ -289,15 +301,23 @@ def createuser( def settheme( context: BaseComposeContext, domains: t.List[str], theme_name: str ) -> None: + """ + Assign a theme to the LMS and the CMS + + To reset to the default theme , use 'default' as the theme name. + """ config = tutor_config.load(context.root) runner = context.job_runner(config) domains = domains or jobs.get_all_openedx_domains(config) jobs.set_theme(theme_name, domains, runner) -@click.command(help="Import the demo course") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_obj def importdemocourse(context: BaseComposeContext) -> None: + """ + Import the demo course + """ config = tutor_config.load(context.root) runner = context.job_runner(config) fmt.echo_info("Importing demo course") @@ -305,13 +325,10 @@ def importdemocourse(context: BaseComposeContext) -> None: @click.command( - short_help="Run a command in a new container", - help=( - "Run a command in a new container. This is a wrapper around `docker-compose run`. Any option or argument passed" - " to this command will be forwarded to docker-compose. Thus, you may use `-v` or `-p` to mount volumes and" - " expose ports." - ), - context_settings={"ignore_unknown_options": True}, + context_settings={ + "ignore_unknown_options": True, + "help_option_names": ["-h", "--help"], + } ) @mount_option @click.argument("args", nargs=-1, required=True) @@ -321,6 +338,14 @@ def run( mounts: t.Tuple[t.List[MountParam.MountType]], args: t.List[str], ) -> None: + """ + Run a command in a new container + + Run a command in a new container. This is a wrapper around `docker-compose + run`. Any option or argument passed to this command will be forwarded to + docker-compose. Thus, you may use `-v` or `-p` to mount volumes and expose + ports. + """ process_mount_arguments(mounts) extra_args = ["--rm"] if not utils.is_a_tty(): @@ -328,31 +353,9 @@ def run( context.invoke(dc_command, command="run", args=[*extra_args, *args]) -@click.command( - name="bindmount", - help="Copy the contents of a container directory to a ready-to-bind-mount host directory", -) -@click.argument("service") -@click.argument("path") -@click.pass_obj -def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None: - """ - This command is made obsolete by the --mount arguments. - """ - fmt.echo_alert( - "The 'bindmount' command is deprecated and will be removed in a later release. Use 'copyfrom' instead." - ) - config = tutor_config.load(context.root) - host_path = bindmounts.create(context.job_runner(config), service, path) - fmt.echo_info( - f"Bind-mount volume created at {host_path}. You can now use it in all `local` and `dev` " - f"commands with the `--volume={path}` option." - ) - - @click.command( name="copyfrom", - help="Copy files/folders from a container directory to the local filesystem.", + context_settings={"help_option_names": ["-h", "--help"]}, ) @click.argument("service") @click.argument("container_path") @@ -364,6 +367,12 @@ def bindmount_command(context: BaseComposeContext, service: str, path: str) -> N def copyfrom( context: BaseComposeContext, service: str, container_path: str, host_path: str ) -> None: + """ + Copy files from a container to the local filesystem + + This command acts just like the UNIX ``cp``. If the target folder does not + exist, it will be created. + """ # Path management container_root_path = "/tmp/mount" container_dst_path = container_root_path @@ -396,30 +405,36 @@ def copyfrom( @click.command( - short_help="Run a command in a running container", - help=( - "Run a command in a running container. This is a wrapper around `docker-compose exec`. Any option or argument" - " passed to this command will be forwarded to docker-compose. Thus, you may use `-e` to manually define" - " environment variables." - ), - context_settings={"ignore_unknown_options": True}, name="exec", + context_settings={ + "ignore_unknown_options": True, + "help_option_names": ["-h", "--help"], + }, ) @click.argument("args", nargs=-1, required=True) @click.pass_context def execute(context: click.Context, args: t.List[str]) -> None: + """ + Run a command in a running container + + This is a wrapper around `docker-compose exec`. Any option or argument + passed to this command will be forwarded to docker-compose. Thus, you may + use `-e` to manually define environment variables. + """ context.invoke(dc_command, command="exec", args=args) -@click.command( - short_help="View output from containers", - help="View output from containers. This is a wrapper around `docker-compose logs`.", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-f", "--follow", is_flag=True, help="Follow log output") @click.option("--tail", type=int, help="Number of lines to show from each container") @click.argument("service", nargs=-1) @click.pass_context def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None: + """ + View output from containers + + This is a wrapper around ``docker-compose logs``. + """ args = [] if follow: args.append("--follow") @@ -429,40 +444,36 @@ def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None context.invoke(dc_command, command="logs", args=args) -@click.command(help="Print status information for containers") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def status(context: click.Context) -> None: + """ + Print status information for containers + + This is a simple wrapper around ``docker-compose ps``. + """ context.invoke(dc_command, command="ps") @click.command( - short_help="Direct interface to docker-compose.", - help=( - "Direct interface to docker-compose. This is a wrapper around `docker-compose`. Most commands, options and" - " arguments passed to this command will be forwarded as-is to docker-compose." - ), - context_settings={"ignore_unknown_options": True}, name="dc", + context_settings={ + "ignore_unknown_options": True, + "help_option_names": ["-h", "--help"], + }, ) @click.argument("command") @click.argument("args", nargs=-1) @click.pass_obj def dc_command(context: BaseComposeContext, command: str, args: t.List[str]) -> None: + """ + Direct interface to docker-compose + + This is a wrapper around ``docker-compose``. Most commands, options and + arguments passed to this command will be forwarded as-is to docker-compose. + """ config = tutor_config.load(context.root) - volumes, non_volume_args = bindmounts.parse_volumes(args) - volume_args = [] - for volume_arg in volumes: - if ":" not in volume_arg: - # This is a bind-mounted volume from the "volumes/" folder. - host_bind_path = bindmounts.get_path(context.root, volume_arg) - if not os.path.exists(host_bind_path): - raise TutorError( - f"Bind-mount volume directory {host_bind_path} does not exist. It must first be created " - f"with the '{bindmount_command.name}' command." - ) - volume_arg = f"{host_bind_path}:{volume_arg}" - volume_args += ["--volume", volume_arg] - context.job_runner(config).docker_compose(command, *volume_args, *non_volume_args) + context.job_runner(config).docker_compose(command, *args) def process_mount_arguments(mounts: t.Tuple[t.List[MountParam.MountType]]) -> None: @@ -543,7 +554,6 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(dc_command) command_group.add_command(run) command_group.add_command(copyfrom) - command_group.add_command(bindmount_command) command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index 83d347ad8f8..558546ce6ae 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -10,16 +10,16 @@ from .context import Context -@click.group( - name="config", - short_help="Configure Open edX", - help="""Configure Open edX and store configuration values in $TUTOR_ROOT/config.yml""", -) +@click.group(name="config", context_settings={"help_option_names": ["-h", "--help"]}) def config_command() -> None: - pass + """ + Configure Tutor/Open edX + + The setting values are stored in ``$(tutor config printroot)/config.yml``. + """ -@click.command(help="Create and save configuration interactively") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-i", "--interactive", is_flag=True, help="Run interactively") @click.option( "-s", @@ -48,6 +48,11 @@ def save( unset_vars: List[str], env_only: bool, ) -> None: + """ + Create and save configuration + + The configuration can be interactive, similar to the "quickstart" command. + """ config = tutor_config.load_minimal(context.root) if interactive: interactive_config.ask_questions(config) @@ -64,13 +69,19 @@ def save( env.save(context.root, config) -@click.command(help="Print the project root") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_obj def printroot(context: Context) -> None: + """ + Print the project root + """ click.echo(context.root) -@click.command(help="Print a configuration value") +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, + help="Print a configuration value", +) @click.argument("key") @click.pass_obj def printvalue(context: Context, key: str) -> None: diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 297671e3f69..6ef7c166e7f 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -1,5 +1,3 @@ -import typing as t - import click from tutor import config as tutor_config @@ -45,13 +43,19 @@ def job_runner(self, config: Config) -> DevJobRunner: return DevJobRunner(self.root, config) -@click.group(help="Run Open edX locally with development settings") +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def dev(context: click.Context) -> None: + """ + Run Open edX locally with development settings + """ context.obj = DevContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch, for development") +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, + help="Configure and run Open edX from scratch, for development", +) @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @click.pass_context @@ -103,37 +107,6 @@ def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) ) -@click.command( - help="DEPRECATED: Use 'tutor dev start ...' instead!", - context_settings={"ignore_unknown_options": True}, -) -@compose.mount_option -@click.argument("options", nargs=-1, required=False) -@click.argument("service") -@click.pass_context -def runserver( - context: click.Context, - mounts: t.Tuple[t.List[compose.MountParam.MountType]], - options: t.List[str], - service: str, -) -> None: - depr_warning = "'runserver' is deprecated and will be removed in a future release. Use 'start' instead." - for option in options: - if option.startswith("-v") or option.startswith("--volume"): - depr_warning += " Bind-mounts can be specified using '-m/--mount'." - break - fmt.echo_alert(depr_warning) - config = tutor_config.load(context.obj.root) - if service in ["lms", "cms"]: - port = 8000 if service == "lms" else 8001 - host = config["LMS_HOST"] if service == "lms" else config["CMS_HOST"] - fmt.echo_info( - f"The {service} service will be available at http://{host}:{port}" - ) - args = ["--service-ports", *options, service] - context.invoke(compose.run, mounts=mounts, args=args) - - @hooks.Actions.COMPOSE_PROJECT_STARTED.add() def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: """ @@ -146,5 +119,4 @@ def _stop_on_local_start(root: str, config: Config, project_name: str) -> None: dev.add_command(quickstart) -dev.add_command(runserver) compose.add_commands(dev) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index a069d447437..e930e423dff 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -60,15 +60,14 @@ def _add_core_images_to_push( return remote_images -@click.group(name="images", short_help="Manage docker images") +@click.group(name="images", context_settings={"help_option_names": ["-h", "--help"]}) def images_command() -> None: - pass + """ + Manage Docker images + """ -@click.command( - short_help="Build docker images", - help="Build the docker images necessary for an Open edX platform.", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("image_names", metavar="image", nargs=-1) @click.option( "--no-cache", is_flag=True, help="Do not use cache when building the image" @@ -107,6 +106,11 @@ def build( target: str, docker_args: t.List[str], ) -> None: + """ + Build Docker images + + Images both from Tutor core and plugins can be built selectively. + """ config = tutor_config.load(context.root) command_args = [] if no_cache: @@ -129,30 +133,39 @@ def build( ) -@click.command(short_help="Pull images from the Docker registry") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj def pull(context: Context, image_names: t.List[str]) -> None: + """ + Pull Docker images from their registry + """ config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PULL, image): images.pull(tag) -@click.command(short_help="Push images to the Docker registry") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj def push(context: Context, image_names: t.List[str]) -> None: + """ + Push Docker images to their registry + """ config = tutor_config.load_full(context.root) for image in image_names: for tag in find_remote_image_tags(config, hooks.Filters.IMAGES_PUSH, image): images.push(tag) -@click.command(short_help="Print tag associated to a Docker image") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("image_names", metavar="image", nargs=-1) @click.pass_obj def printtag(context: Context, image_names: t.List[str]) -> None: + """ + Print the tag associated to a Docker image + """ config = tutor_config.load_full(context.root) for image in image_names: for _name, _path, tag, _args in find_images_to_build(config, image): diff --git a/tutor/commands/k8s.py b/tutor/commands/k8s.py index 14d5e4238ca..3701f185c88 100644 --- a/tutor/commands/k8s.py +++ b/tutor/commands/k8s.py @@ -154,16 +154,22 @@ def job_runner(self, config: Config) -> K8sJobRunner: return K8sJobRunner(self.root, config) -@click.group(help="Run Open edX on Kubernetes") +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def k8s(context: click.Context) -> None: + """ + Run Open edX on Kubernetes + """ context.obj = K8sContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.pass_context def quickstart(context: click.Context, non_interactive: bool) -> None: + """ + Configure and run Open edX from scratch + """ run_upgrade_from_release = tutor_env.should_upgrade_from_release(context.obj.root) if run_upgrade_from_release is not None: click.echo(fmt.title("Upgrading from an older release")) @@ -214,16 +220,15 @@ def quickstart(context: click.Context, non_interactive: bool) -> None: ) -@click.command( - short_help="Run all configured Open edX resources", - help=( - "Run all configured Open edX resources. You may limit this command to " - "some resources by passing name arguments." - ), -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("names", metavar="name", nargs=-1) @click.pass_obj def start(context: K8sContext, names: List[str]) -> None: + """ + Run all configured Open edX resources + + You may limit this command to some resources by passing name arguments. + """ config = tutor_config.load(context.root) # Create namespace, if necessary # Note that this step should not be run for some users, in particular those @@ -264,16 +269,16 @@ def start(context: K8sContext, names: List[str]) -> None: ) -@click.command( - short_help="Stop a running platform", - help=( - "Stop a running platform by deleting all resources, except for volumes. " - "You may limit this command to some resources by passing name arguments." - ), -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("names", metavar="name", nargs=-1) @click.pass_obj def stop(context: K8sContext, names: List[str]) -> None: + """ + Stop a running platform + + All resources will be deleted, except for volumes. You may limit this + command to some resources by passing name arguments. + """ config = tutor_config.load(context.root) names = names or ["all"] for name in names: @@ -301,17 +306,23 @@ def delete_resources( ) -@click.command(help="Reboot an existing platform") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def reboot(context: click.Context) -> None: + """ + Reboot an existing platform + """ context.invoke(stop) context.invoke(start) -@click.command(help="Completely delete an existing platform") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-y", "--yes", is_flag=True, help="Do not ask for confirmation") @click.pass_obj def delete(context: K8sContext, yes: bool) -> None: + """ + Completely delete an existing platform + """ if not yes: click.confirm( "Are you sure you want to delete the platform? All data will be removed.", @@ -326,10 +337,13 @@ def delete(context: K8sContext, yes: bool) -> None: ) -@click.command(help="Initialise all applications") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-l", "--limit", help="Limit initialisation to this service or plugin") @click.pass_obj def init(context: K8sContext, limit: Optional[str]) -> None: + """ + Initialise all applications + """ config = tutor_config.load(context.root) runner = context.job_runner(config) wait_for_pod_ready(config, "caddy") @@ -339,11 +353,14 @@ def init(context: K8sContext, limit: Optional[str]) -> None: jobs.initialise(runner, limit_to=limit) -@click.command(help="Scale the number of replicas of a given deployment") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("deployment") @click.argument("replicas", type=int) @click.pass_obj def scale(context: K8sContext, deployment: str, replicas: int) -> None: + """ + Scale the number of replicas of a given deployment + """ config = tutor_config.load(context.root) utils.kubectl( "scale", @@ -357,7 +374,7 @@ def scale(context: K8sContext, deployment: str, replicas: int) -> None: ) -@click.command(help="Create an Open edX user and interactively set their password") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--superuser", is_flag=True, help="Make superuser") @click.option("--staff", is_flag=True, help="Make staff user") @click.option( @@ -378,15 +395,21 @@ def createuser( name: str, email: str, ) -> None: + """ + Create an Open edX user and interactively set their password + """ config = tutor_config.load(context.root) command = jobs.create_user_command(superuser, staff, name, email, password=password) runner = context.job_runner(config) runner.run_job("lms", command) -@click.command(help="Import the demo course") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_obj def importdemocourse(context: K8sContext) -> None: + """ + Import the demo course + """ fmt.echo_info("Importing demo course") config = tutor_config.load(context.root) runner = context.job_runner(config) @@ -394,7 +417,8 @@ def importdemocourse(context: K8sContext) -> None: @click.command( - help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name." + context_settings={"help_option_names": ["-h", "--help"]}, + help="Assign a theme to the LMS and the CMS. To reset to the default theme , use 'default' as the theme name.", ) @click.option( "-d", @@ -417,18 +441,23 @@ def settheme(context: K8sContext, domains: List[str], theme_name: str) -> None: @click.command( name="exec", - help="Execute a command in a pod of the given application", - context_settings={"ignore_unknown_options": True}, + context_settings={ + "ignore_unknown_options": True, + "help_option_names": ["-h", "--help"], + }, ) @click.argument("service") @click.argument("args", nargs=-1, required=True) @click.pass_obj def exec_command(context: K8sContext, service: str, args: List[str]) -> None: + """ + Execute a command in a pod of the given application + """ config = tutor_config.load(context.root) kubectl_exec(config, service, args) -@click.command(help="View output from containers") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("-c", "--container", help="Print the logs of this specific container") @click.option("-f", "--follow", is_flag=True, help="Follow log output") @click.option("--tail", type=int, help="Number of lines to show from each container") @@ -437,6 +466,9 @@ def exec_command(context: K8sContext, service: str, args: List[str]) -> None: def logs( context: K8sContext, container: str, follow: bool, tail: bool, service: str ) -> None: + """ + View output from containers + """ config = tutor_config.load(context.root) command = ["logs"] @@ -453,25 +485,33 @@ def logs( utils.kubectl(*command) -@click.command(help="Wait for a pod to become ready") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("name") @click.pass_obj def wait(context: K8sContext, name: str) -> None: + """ + Wait for a pod to become ready + """ config = tutor_config.load(context.root) wait_for_pod_ready(config, name) @click.command( + context_settings={"help_option_names": ["-h", "--help"]}, short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `quickstart`.", ) @click.option( "--from", "from_release", - type=click.Choice(["ironwood", "juniper", "koa", "lilac"]), + type=click.Choice(["ironwood", "juniper", "koa", "lilac", "maple"]), ) @click.pass_context def upgrade(context: click.Context, from_release: Optional[str]) -> None: + """ + Perform release-specific upgrade tasks + + To perform a full upgrade remember to run ``quickstart``. + """ if from_release is None: from_release = tutor_env.get_env_release(context.obj.root) if from_release is None: @@ -487,17 +527,21 @@ def upgrade(context: click.Context, from_release: Optional[str]) -> None: @click.command( - short_help="Direct interface to `kubectl apply`.", - help=( - "Direct interface to `kubnectl-apply`. This is a wrapper around `kubectl apply`. A;; options and" - " arguments passed to this command will be forwarded as-is to `kubectl apply`." - ), - context_settings={"ignore_unknown_options": True}, name="apply", + context_settings={ + "ignore_unknown_options": True, + "help_option_names": ["-h", "--help"], + }, ) @click.argument("args", nargs=-1) @click.pass_obj def apply_command(context: K8sContext, args: List[str]) -> None: + """ + Direct interface to ``kubectl apply`` + + This is a wrapper around `kubectl apply`. All options and arguments passed + to this command will be forwarded as-is to ``kubectl apply``. + """ kubectl_apply(context.root, *args) @@ -505,7 +549,10 @@ def kubectl_apply(root: str, *args: str) -> None: utils.kubectl("apply", "--kustomize", tutor_env.pathjoin(root), *args) -@click.command(help="Print status information for all k8s resources") +@click.command( + context_settings={"help_option_names": ["-h", "--help"]}, + help="Print status information for all k8s resources", +) @click.pass_obj def status(context: K8sContext) -> int: config = tutor_config.load(context.root) diff --git a/tutor/commands/local.py b/tutor/commands/local.py index 7e4817573d2..f5fca746f63 100644 --- a/tutor/commands/local.py +++ b/tutor/commands/local.py @@ -45,13 +45,16 @@ def job_runner(self, config: Config) -> LocalJobRunner: return LocalJobRunner(self.root, config) -@click.group(help="Run Open edX locally with docker-compose") +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def local(context: click.Context) -> None: + """ + Run Open edX locally with docker-compose + """ context.obj = LocalContext(context.obj.root) -@click.command(help="Configure and run Open edX from scratch") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @compose.mount_option @click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") @click.option("-p", "--pullimages", is_flag=True, help="Update docker images") @@ -62,6 +65,9 @@ def quickstart( non_interactive: bool, pullimages: bool, ) -> None: + """ + Configure and run Open edX from scratch + """ try: utils.check_macos_docker_memory() except exceptions.TutorError as e: @@ -147,17 +153,19 @@ def quickstart( ) -@click.command( - short_help="Perform release-specific upgrade tasks", - help="Perform release-specific upgrade tasks. To perform a full upgrade remember to run `quickstart`.", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( "--from", "from_release", - type=click.Choice(["ironwood", "juniper", "koa", "lilac"]), + type=click.Choice(["ironwood", "juniper", "koa", "lilac", "maple"]), ) @click.pass_context def upgrade(context: click.Context, from_release: t.Optional[str]) -> None: + """ + Perform release-specific upgrade tasks + + To perform a full upgrade remember to run ``quickstart``. + """ fmt.echo_alert( "This command only performs a partial upgrade of your Open edX platform. " "To perform a full upgrade, you should run `tutor local quickstart`." diff --git a/tutor/commands/plugins.py b/tutor/commands/plugins.py index 5438ec16ec5..38a6e4f7816 100644 --- a/tutor/commands/plugins.py +++ b/tutor/commands/plugins.py @@ -6,25 +6,27 @@ from tutor import config as tutor_config from tutor import exceptions, fmt, hooks, plugins -from tutor.plugins.base import PLUGINS_ROOT, PLUGINS_ROOT_ENV_VAR_NAME +from tutor.plugins.base import PLUGINS_ROOT from .context import Context -@click.group( - name="plugins", - short_help="Manage Tutor plugins", - help="Manage Tutor plugins to add new features and customise your Open edX platform", -) +@click.group(name="plugins", context_settings={"help_option_names": ["-h", "--help"]}) def plugins_command() -> None: """ - All plugin commands should work even if there is no existing config file. This is - because users might enable plugins prior to configuration or environment generation. + Manage Tutor plugins + + Plugins can be enabled to add new features and customise your Open edX platform. """ + # All plugin commands should work even if there is no existing config file. This is + # because users might enable plugins prior to configuration or environment generation. -@click.command(name="list", help="List installed plugins") +@click.command(name="list", context_settings={"help_option_names": ["-h", "--help"]}) def list_command() -> None: + """ + List installed plugins + """ lines = [] first_column_width = 1 for plugin, plugin_info in plugins.iter_info(): @@ -38,10 +40,13 @@ def list_command() -> None: print("{:{width}}\t{:10}\t{}".format(*line, width=first_column_width)) -@click.command(help="Enable a plugin") +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def enable(context: Context, plugin_names: t.List[str]) -> None: + """ + Enable a plugin + """ config = tutor_config.load_minimal(context.root) for plugin in plugin_names: plugins.load(plugin) @@ -53,13 +58,15 @@ def enable(context: Context, plugin_names: t.List[str]) -> None: ) -@click.command( - short_help="Disable a plugin", - help="Disable one or more plugins. Specify 'all' to disable all enabled plugins at once.", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("plugin_names", metavar="plugin", nargs=-1) @click.pass_obj def disable(context: Context, plugin_names: t.List[str]) -> None: + """ + Disable a plugin + + Specify 'all' to disable all enabled plugins at once. + """ config = tutor_config.load_minimal(context.root) disable_all = "all" in plugin_names disabled: t.List[str] = [] @@ -77,21 +84,27 @@ def disable(context: Context, plugin_names: t.List[str]) -> None: @click.command( - short_help="Print the location of yaml-based plugins", - help=f"""Print the location of yaml-based plugins. This location can be manually -defined by setting the {PLUGINS_ROOT_ENV_VAR_NAME} environment variable""", + context_settings={"help_option_names": ["-h", "--help"]}, ) def printroot() -> None: + """ + Print the location of yaml-based plugins + + This location can be manually overridden by setting the TUTOR_PLUGINS_ROOT environment variable + """ fmt.echo(PLUGINS_ROOT) -@click.command( - short_help="Install a plugin", - help=f"""Install a plugin, either from a local Python/YAML file or a remote, web-hosted -location. The plugin will be installed to {PLUGINS_ROOT_ENV_VAR_NAME}.""", -) +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.argument("location") def install(location: str) -> None: + """ + Install a plugin + + Plugins installed this way can come either from a local Python/YAML file or + a remote, web-hosted location. The plugin will be installed to the location + indicated by ``tutor plugins printroot``. + """ basename = os.path.basename(location) if not basename.endswith(".yml") and not basename.endswith(".py"): basename += ".py" diff --git a/tutor/commands/upgrade/k8s.py b/tutor/commands/upgrade/k8s.py index 86143b4da92..60ebe6e4845 100644 --- a/tutor/commands/upgrade/k8s.py +++ b/tutor/commands/upgrade/k8s.py @@ -1,4 +1,5 @@ from tutor import config as tutor_config +from tutor import env as tutor_env from tutor import fmt from tutor.commands import k8s from tutor.commands.context import Context @@ -27,6 +28,10 @@ def upgrade_from(context: Context, from_release: str) -> None: upgrade_from_lilac(config) running_release = "maple" + if running_release == "maple": + upgrade_from_maple(context, config) + running_release = "nutmeg" + def upgrade_from_ironwood(config: Config) -> None: if not config["RUN_MONGODB"]: @@ -102,3 +107,42 @@ def upgrade_from_lilac(config: Config) -> None: "upgrade from Lilac to Maple" ) k8s.delete_resources(config, resources=["deployments", "services"]) + + +def upgrade_from_maple(context: Context, config: Config) -> None: + fmt.echo_info("Upgrading from Maple") + # The environment needs to be updated because the backpopulate/backfill commands are from Nutmeg + tutor_env.save(context.root, config) + + # Start mysql + k8s.kubectl_apply( + context.root, + "--selector", + "app.kubernetes.io/name=mysql", + ) + k8s.wait_for_pod_ready(config, "mysql") + + # lms upgrade + k8s.kubectl_apply( + context.root, + "--selector", + "app.kubernetes.io/name=lms", + ) + k8s.wait_for_pod_ready(config, "lms") + k8s.kubectl_exec( + config, "lms", ["sh", "-e", "-c", "./manage.py lms backpopulate_user_tours"] + ) + + # cms upgrade + k8s.kubectl_apply( + context.root, + "--selector", + "app.kubernetes.io/name=cms", + ) + k8s.wait_for_pod_ready(config, "cms") + k8s.kubectl_exec( + config, "cms", ["sh", "-e", "-c", "./manage.py cms backfill_course_tabs"] + ) + k8s.kubectl_exec( + config, "cms", ["sh", "-e", "-c", "./manage.py cms simulate_publish"] + ) diff --git a/tutor/commands/upgrade/local.py b/tutor/commands/upgrade/local.py index 98fe7caf4b9..1fecbc3deb9 100644 --- a/tutor/commands/upgrade/local.py +++ b/tutor/commands/upgrade/local.py @@ -31,6 +31,10 @@ def upgrade_from(context: click.Context, from_release: str) -> None: common_upgrade.upgrade_from_lilac(config) running_release = "maple" + if running_release == "maple": + upgrade_from_maple(context, config) + running_release = "nutmeg" + def upgrade_from_ironwood(context: click.Context, config: Config) -> None: click.echo(fmt.title("Upgrading from Ironwood")) @@ -95,6 +99,24 @@ def upgrade_from_koa(context: click.Context, config: Config) -> None: upgrade_mongodb(context, config, "4.0.25", "4.0") +def upgrade_from_maple(context: click.Context, config: Config) -> None: + click.echo(fmt.title("Upgrading from Maple")) + # The environment needs to be updated because the management commands are from Nutmeg + tutor_env.save(context.obj.root, config) + context.invoke( + compose.run, + args=["lms", "sh", "-e", "-c", "./manage.py lms backpopulate_user_tours"], + ) + context.invoke( + compose.run, + args=["cms", "sh", "-e", "-c", "./manage.py cms backfill_course_tabs"], + ) + context.invoke( + compose.run, + args=["cms", "sh", "-e", "-c", "./manage.py cms simulate_publish"], + ) + + def upgrade_mongodb( context: click.Context, config: Config, diff --git a/tutor/env.py b/tutor/env.py index bd0636ce812..5d7436818d7 100644 --- a/tutor/env.py +++ b/tutor/env.py @@ -388,6 +388,7 @@ def get_release(version: str) -> str: "11": "koa", "12": "lilac", "13": "maple", + "14": "nutmeg", }[version.split(".", maxsplit=1)[0]] diff --git a/tutor/templates/apps/openedx/config/cms.env.json b/tutor/templates/apps/openedx/config/cms.env.json deleted file mode 100644 index 68aca49f139..00000000000 --- a/tutor/templates/apps/openedx/config/cms.env.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "SITE_NAME": "{{ CMS_HOST }}", - "BOOK_URL": "", - "LOG_DIR": "/openedx/data/logs", - "LOGGING_ENV": "sandbox", - "OAUTH_OIDC_ISSUER": "{{ JWT_COMMON_ISSUER }}", - "PLATFORM_NAME": "{{ PLATFORM_NAME }}", - "FEATURES": { - {{ patch("common-env-features", separator=",\n", suffix=",")|indent(4) }} - {{ patch("cms-env-features", separator=",\n", suffix=",")|indent(4) }} - "CERTIFICATES_HTML_VIEW": true, - "PREVIEW_LMS_BASE": "{{ PREVIEW_LMS_HOST }}", - "ENABLE_COURSEWARE_INDEX": true, - "ENABLE_CSMH_EXTENDED": false, - "ENABLE_LEARNER_RECORDS": false, - "ENABLE_LIBRARY_INDEX": true, - "MILESTONES_APP": true, - "ENABLE_PREREQUISITE_COURSES": true - }, - "LMS_ROOT_URL": "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}", - "CMS_ROOT_URL": "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}", - "CMS_BASE": "{{ CMS_HOST }}", - "LMS_BASE": "{{ LMS_HOST }}", - "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}", - "CELERY_BROKER_TRANSPORT": "redis", - "CELERY_BROKER_HOSTNAME": "{{ REDIS_HOST }}:{{ REDIS_PORT }}", - "CELERY_BROKER_VHOST": "{{ OPENEDX_CELERY_REDIS_DB }}", - "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", - "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", - "ALTERNATE_WORKER_QUEUES": "lms", - "ENABLE_COMPREHENSIVE_THEMING": true, - "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], - "STATIC_ROOT_BASE": "/openedx/staticfiles", - "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", - "EMAIL_HOST": "{{ SMTP_HOST }}", - "EMAIL_PORT": {{ SMTP_PORT }}, - "EMAIL_USE_TLS": {{ "true" if SMTP_USE_TLS else "false" }}, - "HTTPS": "{{ "on" if ENABLE_HTTPS else "off" }}", - "LANGUAGE_CODE": "{{ LANGUAGE_CODE }}", - "SESSION_COOKIE_DOMAIN": "{{ CMS_HOST }}", - {{ patch("cms-env", separator=",\n", suffix=",")|indent(2) }} - "CACHES": { - "default": { - "KEY_PREFIX": "default", - "VERSION": "1", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "general": { - "KEY_PREFIX": "general", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "mongo_metadata_inheritance": { - "KEY_PREFIX": "mongo_metadata_inheritance", - "TIMEOUT": 300, - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "staticfiles": { - "KEY_PREFIX": "staticfiles_cms", - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "LOCATION": "staticfiles_cms" - }, - "configuration": { - "KEY_PREFIX": "configuration", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "celery": { - "KEY_PREFIX": "celery", - "TIMEOUT": "7200", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "course_structure_cache": { - "KEY_PREFIX": "course_structure", - "TIMEOUT": "7200", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - } - }, -{% include "apps/openedx/config/partials/auth.json" %} -} diff --git a/tutor/templates/apps/openedx/config/cms.env.yml b/tutor/templates/apps/openedx/config/cms.env.yml new file mode 100644 index 00000000000..58a4ca19267 --- /dev/null +++ b/tutor/templates/apps/openedx/config/cms.env.yml @@ -0,0 +1,73 @@ +SITE_NAME: "{{ CMS_HOST }}" +BOOK_URL: "" +LOG_DIR: "/openedx/data/logs" +LOGGING_ENV: "sandbox" +OAUTH_OIDC_ISSUER: "{{ JWT_COMMON_ISSUER }}" +PLATFORM_NAME: "{{ PLATFORM_NAME }}" +FEATURES: + {{ patch("common-env-features") }} + {{ patch("cms-env-features") }} + CERTIFICATES_HTML_VIEW: true + PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" + ENABLE_COURSEWARE_INDEX: true + ENABLE_CSMH_EXTENDED: false + ENABLE_LEARNER_RECORDS: false + ENABLE_LIBRARY_INDEX: true + MILESTONES_APP: true + ENABLE_PREREQUISITE_COURSES: true +LMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}" +CMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}" +CMS_BASE: "{{ CMS_HOST }}" +LMS_BASE: "{{ LMS_HOST }}" +CONTACT_EMAIL: "{{ CONTACT_EMAIL }}" +CELERY_BROKER_TRANSPORT: "redis" +CELERY_BROKER_HOSTNAME: "{{ REDIS_HOST }}:{{ REDIS_PORT }}" +CELERY_BROKER_VHOST: "{{ OPENEDX_CELERY_REDIS_DB }}" +CELERY_BROKER_USER: "{{ REDIS_USERNAME }}" +CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}" +ALTERNATE_WORKER_QUEUES: "lms" +ENABLE_COMPREHENSIVE_THEMING: true +COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"] +STATIC_ROOT_BASE: "/openedx/staticfiles" +EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" +EMAIL_HOST: "{{ SMTP_HOST }}" +EMAIL_PORT: {{ SMTP_PORT }}, +EMAIL_USE_TLS: {{ "true" if SMTP_USE_TLS else "false" }} +HTTPS: "{{ "on" if ENABLE_HTTPS else "off" }}" +LANGUAGE_CODE: "{{ LANGUAGE_CODE }}" +SESSION_COOKIE_DOMAIN: "{{ CMS_HOST }}" +{{ patch("cms-env") }} +CACHES: + default: + KEY_PREFIX: "default" + VERSION: "1" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + general: + KEY_PREFIX: "general" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + mongo_metadata_inheritance: + KEY_PREFIX: "mongo_metadata_inheritance" + TIMEOUT: 300, + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + staticfiles: + KEY_PREFIX: "staticfiles_cms" + BACKEND: "django.core.cache.backends.locmem.LocMemCache" + LOCATION: "staticfiles_cms" + configuration: + KEY_PREFIX: "configuration" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + celery: + KEY_PREFIX: "celery" + TIMEOUT: "7200" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + course_structure_cache: + KEY_PREFIX: "course_structure" + TIMEOUT: "7200" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" +{% include "apps/openedx/config/partials/auth.yml" %} diff --git a/tutor/templates/apps/openedx/config/lms.env.json b/tutor/templates/apps/openedx/config/lms.env.json deleted file mode 100644 index b3188f94f5f..00000000000 --- a/tutor/templates/apps/openedx/config/lms.env.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "SITE_NAME": "{{ LMS_HOST }}", - "BOOK_URL": "", - "LOG_DIR": "/openedx/data/logs", - "LOGGING_ENV": "sandbox", - "OAUTH_OIDC_ISSUER": "{{ JWT_COMMON_ISSUER }}", - "PLATFORM_NAME": "{{ PLATFORM_NAME }}", - "FEATURES": { - {{ patch("common-env-features", separator=",\n", suffix=",")|indent(4) }} - {{ patch("lms-env-features", separator=",\n", suffix=",")|indent(4) }} - "CERTIFICATES_HTML_VIEW": true, - "PREVIEW_LMS_BASE": "{{ PREVIEW_LMS_HOST }}", - "ENABLE_CORS_HEADERS": true, - "ENABLE_COURSE_DISCOVERY": true, - "ENABLE_COURSEWARE_SEARCH": true, - "ENABLE_CSMH_EXTENDED": false, - "ENABLE_DASHBOARD_SEARCH": true, - "ENABLE_COMBINED_LOGIN_REGISTRATION": true, - "ENABLE_GRADE_DOWNLOADS": true, - "ENABLE_LEARNER_RECORDS": false, - "ENABLE_MOBILE_REST_API": true, - "ENABLE_OAUTH2_PROVIDER": true, - "ENABLE_THIRD_PARTY_AUTH": true, - "MILESTONES_APP": true, - "ENABLE_PREREQUISITE_COURSES": true - }, - "LMS_ROOT_URL": "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}", - "CMS_ROOT_URL": "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}", - "CMS_BASE": "{{ CMS_HOST }}", - "LMS_BASE": "{{ LMS_HOST }}", - "CONTACT_EMAIL": "{{ CONTACT_EMAIL }}", - "CELERY_BROKER_TRANSPORT": "redis", - "CELERY_BROKER_HOSTNAME": "{{ REDIS_HOST }}:{{ REDIS_PORT }}", - "CELERY_BROKER_VHOST": "{{ OPENEDX_CELERY_REDIS_DB }}", - "CELERY_BROKER_USER": "{{ REDIS_USERNAME }}", - "CELERY_BROKER_PASSWORD": "{{ REDIS_PASSWORD }}", - "ALTERNATE_WORKER_QUEUES": "cms", - "ENABLE_COMPREHENSIVE_THEMING": true, - "COMPREHENSIVE_THEME_DIRS": ["/openedx/themes"], - "STATIC_ROOT_BASE": "/openedx/staticfiles", - "EMAIL_BACKEND": "django.core.mail.backends.smtp.EmailBackend", - "EMAIL_HOST": "{{ SMTP_HOST }}", - "EMAIL_PORT": {{ SMTP_PORT }}, - "EMAIL_USE_TLS": {{ "true" if SMTP_USE_TLS else "false" }}, - "ACE_ROUTING_KEY": "edx.lms.core.default", - "HTTPS": "{{ "on" if ENABLE_HTTPS else "off" }}", - "LANGUAGE_CODE": "{{ LANGUAGE_CODE }}", - "SESSION_COOKIE_DOMAIN": "{{ LMS_HOST }}", - {{ patch("lms-env", separator=",\n", suffix=",")|indent(2) }} - "CACHES": { - "default": { - "KEY_PREFIX": "default", - "VERSION": "1", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "general": { - "KEY_PREFIX": "general", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "mongo_metadata_inheritance": { - "KEY_PREFIX": "mongo_metadata_inheritance", - "TIMEOUT": 300, - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "staticfiles": { - "KEY_PREFIX": "staticfiles_lms", - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "LOCATION": "staticfiles_lms" - }, - "configuration": { - "KEY_PREFIX": "configuration", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "celery": { - "KEY_PREFIX": "celery", - "TIMEOUT": "7200", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "course_structure_cache": { - "KEY_PREFIX": "course_structure", - "TIMEOUT": "7200", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - }, - "ora2-storage": { - "KEY_PREFIX": "ora2-storage", - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" - } - }, -{% include "apps/openedx/config/partials/auth.json" %} -} diff --git a/tutor/templates/apps/openedx/config/lms.env.yml b/tutor/templates/apps/openedx/config/lms.env.yml new file mode 100644 index 00000000000..53d320e0b3f --- /dev/null +++ b/tutor/templates/apps/openedx/config/lms.env.yml @@ -0,0 +1,86 @@ +SITE_NAME: "{{ LMS_HOST }}" +BOOK_URL: "" +LOG_DIR: "/openedx/data/logs" +LOGGING_ENV: "sandbox" +OAUTH_OIDC_ISSUER: "{{ JWT_COMMON_ISSUER }}" +PLATFORM_NAME: "{{ PLATFORM_NAME }}" +FEATURES: + {{ patch("common-env-features")|indent(2) }} + {{ patch("lms-env-features")|indent(2) }} + CERTIFICATES_HTML_VIEW: true + PREVIEW_LMS_BASE: "{{ PREVIEW_LMS_HOST }}" + ENABLE_CORS_HEADERS: true + ENABLE_COURSE_DISCOVERY: true + ENABLE_COURSEWARE_SEARCH: true + ENABLE_CSMH_EXTENDED: false + ENABLE_DASHBOARD_SEARCH: true + ENABLE_COMBINED_LOGIN_REGISTRATION: true + ENABLE_GRADE_DOWNLOADS: true + ENABLE_LEARNER_RECORDS: false + ENABLE_MOBILE_REST_API: true + ENABLE_OAUTH2_PROVIDER: true + ENABLE_PREREQUISITE_COURSES: true + ENABLE_THIRD_PARTY_AUTH: true + MILESTONES_APP: true + PERSISTENT_GRADES_ENABLED_FOR_ALL_TESTS: true +LMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ LMS_HOST }}" +CMS_ROOT_URL: "{{ "https" if ENABLE_HTTPS else "http" }}://{{ CMS_HOST }}" +CMS_BASE: "{{ CMS_HOST }}" +LMS_BASE: "{{ LMS_HOST }}" +CONTACT_EMAIL: "{{ CONTACT_EMAIL }}" +CELERY_BROKER_TRANSPORT: "redis" +CELERY_BROKER_HOSTNAME: "{{ REDIS_HOST }}:{{ REDIS_PORT }}" +CELERY_BROKER_VHOST: "{{ OPENEDX_CELERY_REDIS_DB }}" +CELERY_BROKER_USER: "{{ REDIS_USERNAME }}" +CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}" +ALTERNATE_WORKER_QUEUES: "cms" +ENABLE_COMPREHENSIVE_THEMING: true +COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"] +STATIC_ROOT_BASE: "/openedx/staticfiles" +EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" +EMAIL_HOST: "{{ SMTP_HOST }}" +EMAIL_PORT: {{ SMTP_PORT }} +EMAIL_USE_TLS: {{ "true" if SMTP_USE_TLS else "false" }} +ACE_ROUTING_KEY: "edx.lms.core.default" +HTTPS: "{{ "on" if ENABLE_HTTPS else "off" }}" +LANGUAGE_CODE: "{{ LANGUAGE_CODE }}" +SESSION_COOKIE_DOMAIN: "{{ LMS_HOST }}" +{{ patch("lms-env") }} +CACHES: + default: + KEY_PREFIX: "default" + VERSION: "1" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + general: + KEY_PREFIX: "general" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + mongo_metadata_inheritance: + KEY_PREFIX: "mongo_metadata_inheritance" + TIMEOUT: 300 + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + staticfiles: + KEY_PREFIX: "staticfiles_lms" + BACKEND: "django.core.cache.backends.locmem.LocMemCache" + LOCATION: "staticfiles_lms" + configuration: + KEY_PREFIX: "configuration" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + celery: + KEY_PREFIX: "celery" + TIMEOUT: "7200" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + course_structure_cache: + KEY_PREFIX: "course_structure" + TIMEOUT: "7200" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" + ora2-storage: + KEY_PREFIX: "ora2-storage" + BACKEND: "django_redis.cache.RedisCache" + LOCATION: "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{ REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/{{ OPENEDX_CACHE_REDIS_DB }}" +{% include "apps/openedx/config/partials/auth.yml" %} diff --git a/tutor/templates/apps/openedx/config/partials/auth.json b/tutor/templates/apps/openedx/config/partials/auth.json deleted file mode 100644 index 66e9422b605..00000000000 --- a/tutor/templates/apps/openedx/config/partials/auth.json +++ /dev/null @@ -1,26 +0,0 @@ - "SECRET_KEY": "{{ OPENEDX_SECRET_KEY }}", - "AWS_ACCESS_KEY_ID": "{{ OPENEDX_AWS_ACCESS_KEY }}", - "AWS_SECRET_ACCESS_KEY": "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}", - "CONTENTSTORE": null, - "DOC_STORE_CONFIG": null, - {{ patch("openedx-auth", separator=",\n", suffix=",")|indent(2) }} - "XQUEUE_INTERFACE": { - "django_auth": null, - "url": null - }, - "DATABASES": { - "default": { - "ENGINE": "django.db.backends.mysql", - "HOST": "{{ MYSQL_HOST }}", - "PORT": {{ MYSQL_PORT }}, - "NAME": "{{ OPENEDX_MYSQL_DATABASE }}", - "USER": "{{ OPENEDX_MYSQL_USERNAME }}", - "PASSWORD": "{{ OPENEDX_MYSQL_PASSWORD }}", - "ATOMIC_REQUESTS": true, - "OPTIONS": { - "init_command": "SET sql_mode='STRICT_TRANS_TABLES'" - } - } - }, - "EMAIL_HOST_USER": "{{ SMTP_USERNAME }}", - "EMAIL_HOST_PASSWORD": "{{ SMTP_PASSWORD }}" \ No newline at end of file diff --git a/tutor/templates/apps/openedx/config/partials/auth.yml b/tutor/templates/apps/openedx/config/partials/auth.yml new file mode 100644 index 00000000000..2ec64487076 --- /dev/null +++ b/tutor/templates/apps/openedx/config/partials/auth.yml @@ -0,0 +1,22 @@ +SECRET_KEY: "{{ OPENEDX_SECRET_KEY }}" +AWS_ACCESS_KEY_ID: "{{ OPENEDX_AWS_ACCESS_KEY }}" +AWS_SECRET_ACCESS_KEY: "{{ OPENEDX_AWS_SECRET_ACCESS_KEY }}" +CONTENTSTORE: null +DOC_STORE_CONFIG: null +{{ patch("openedx-auth") }} +XQUEUE_INTERFACE: + django_auth: null + url: null +DATABASES: + default: + ENGINE: "django.db.backends.mysql" + HOST: "{{ MYSQL_HOST }}" + PORT: {{ MYSQL_PORT }} + NAME: "{{ OPENEDX_MYSQL_DATABASE }}" + USER: "{{ OPENEDX_MYSQL_USERNAME }}" + PASSWORD: "{{ OPENEDX_MYSQL_PASSWORD }}" + ATOMIC_REQUESTS: true + OPTIONS: + init_command: "SET sql_mode='STRICT_TRANS_TABLES'" +EMAIL_HOST_USER: "{{ SMTP_USERNAME }}" +EMAIL_HOST_PASSWORD: "{{ SMTP_PASSWORD }}" diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index 2398a7e9b86..522d6f1a687 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -20,6 +20,10 @@ # Email settings DEFAULT_EMAIL_LOGO_URL = LMS_ROOT_URL + "/theming/asset/images/logo.png" +BULK_EMAIL_SEND_USING_EDX_ACE = True + +# Make it possible to hide courses by default from the studio +SEARCH_SKIP_SHOW_IN_CATALOG_FILTERING = False # Create folders if necessary for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 906ca70094b..b584ebadf00 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -47,18 +47,9 @@ RUN git config --global user.email "tutor@overhang.io" \ {{ patch("openedx-dockerfile-git-patches-default") }} {%- else %} # Patch edx-platform -# Fix forum notification for questions -# https://github.com/openedx/edx-platform/pull/29611 -RUN git fetch --depth=2 https://github.com/open-craft/edx-platform/ 03731f19459e558f188c06aac5cc9ca1bbc675c2 && git cherry-pick 03731f19459e558f188c06aac5cc9ca1bbc675c2 -# SAML security fix -# https://github.com/overhangio/edx-platform/tree/overhangio/sec-fix-saml-vulnerability -RUN git fetch --depth=2 https://github.com/overhangio/edx-platform/ 3b985f207853e88090d68a81acd52866b71f5af7 && git cherry-pick 3b985f207853e88090d68a81acd52866b71f5af7 -# Rate limiting security fix -# https://github.com/overhangio/edx-platform/tree/overhangio/sec-rate-limiting -RUN git fetch --depth=2 https://github.com/overhangio/edx-platform/ b5723e416e628cac4fa84392ca13e1b72817674f && git cherry-pick b5723e416e628cac4fa84392ca13e1b72817674f -# Logout redirect security fix -# https://github.com/overhangio/edx-platform/tree/overhangio/sec-fix-logout-redirect-vulnerability -RUN git fetch --depth=2 https://github.com/overhangio/edx-platform/ 08d8504224e3a3e728a0f264749e1b585e21b871 && git cherry-pick 08d8504224e3a3e728a0f264749e1b585e21b871 +# Fix broken "Pages" view in Studio +# https://github.com/openedx/edx-platform/pull/30550 +RUN git fetch --depth=2 https://github.com/open-craft/edx-platform/ 3d54f284f82b61e693ad652d8d6e46a226fcb36d && git cherry-pick 3d54f284f82b61e693ad652d8d6e46a226fcb36d {%- endif %} {# Example: RUN git fetch --depth=2 https://github.com/openedx/edx-platform && git cherry-pick #} @@ -88,15 +79,20 @@ COPY --from=code /openedx/edx-platform /openedx/edx-platform WORKDIR /openedx/edx-platform # Install the right version of pip/setuptools -RUN pip install setuptools==44.1.0 pip==20.0.2 wheel==0.34.2 +# https://pypi.org/project/setuptools/ +# https://pypi.org/project/pip/ +# https://pypi.org/project/wheel/ +RUN pip install setuptools==62.1.0 pip==22.0.4 wheel==0.37.1 # Install base requirements RUN pip install -r ./requirements/edx/base.txt # Install django-redis for using redis as a django cache -RUN pip install django-redis==4.12.1 +# https://pypi.org/project/django-redis/ +RUN pip install django-redis==5.2.0 # Install uwsgi +# https://pypi.org/project/uWSGI/ RUN pip install uwsgi==2.0.20 {{ patch("openedx-dockerfile-post-python-requirements") }} @@ -154,12 +150,12 @@ WORKDIR /openedx/edx-platform # Re-install local requirements, otherwise egg-info folders are missing RUN pip install -r requirements/edx/local.in -# Create folder that will store lms/cms.env.json files, as well as +# Create folder that will store lms/cms.env.yml files, as well as # the tutor-specific settings files. RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor COPY --chown=app:app revisions.yml /openedx/config/ -ENV LMS_CFG /openedx/config/lms.env.json -ENV STUDIO_CFG /openedx/config/cms.env.json +ENV LMS_CFG /openedx/config/lms.env.yml +ENV CMS_CFG /openedx/config/cms.env.yml ENV REVISION_CFG /openedx/config/revisions.yml COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ diff --git a/tutor/templates/build/openedx/revisions.yml b/tutor/templates/build/openedx/revisions.yml index c76e09e8bbc..d75f161c43d 100644 --- a/tutor/templates/build/openedx/revisions.yml +++ b/tutor/templates/build/openedx/revisions.yml @@ -1 +1 @@ -EDX_PLATFORM_REVISION: maple +EDX_PLATFORM_REVISION: nutmeg diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index 1628333a4a2..1e5d5aedfd3 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -48,7 +48,7 @@ OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_CSMH_MYSQL_DATABASE: "{{ OPENEDX_MYSQL_DATABASE }}_csmh" OPENEDX_MYSQL_USERNAME: "openedx" -OPENEDX_COMMON_VERSION: "open-release/maple.3" +OPENEDX_COMMON_VERSION: "open-release/nutmeg.master" OPENEDX_EXTRA_PIP_REQUIREMENTS: - "openedx-scorm-xblock<14.0.0,>=13.0.0" MYSQL_HOST: "mysql" diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 00b048d7aaf..12e717a08c5 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -49,13 +49,9 @@ services: lms-worker: <<: *openedx-service - # Note: we should get rid of this once celery is upgraded to 5.0.0+ after maple - # (same below in cms-worker) - tty: false cms-worker: <<: *openedx-service - tty: false # Additional service for watching theme changes watchthemes: diff --git a/tutor/templates/hooks/lms/init b/tutor/templates/hooks/lms/init index fa05a504014..0276cd83a61 100644 --- a/tutor/templates/hooks/lms/init +++ b/tutor/templates/hooks/lms/init @@ -1,4 +1,5 @@ dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s +dockerize -wait tcp://{{ MONGODB_HOST }}:{{ MONGODB_PORT }} -timeout 20s echo "Loading settings $DJANGO_SETTINGS_MODULE" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index 6f8098c238f..57e0fc8ed66 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -131,7 +131,7 @@ spec: containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "worker", "--app=cms.celery", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--maxtasksperchild", "100", "--exclude-queues=edx.lms.core.default"] + args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] env: - name: SERVICE_VARIANT value: cms @@ -231,7 +231,7 @@ spec: containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "worker", "--app=lms.celery", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--maxtasksperchild=100", "--exclude-queues=edx.cms.core.default"] + args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] env: - name: SERVICE_VARIANT value: lms diff --git a/tutor/templates/local/docker-compose.jobs.yml b/tutor/templates/local/docker-compose.jobs.yml index ed7f967f44c..8b87f9fd526 100644 --- a/tutor/templates/local/docker-compose.jobs.yml +++ b/tutor/templates/local/docker-compose.jobs.yml @@ -14,7 +14,7 @@ services: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro - ../apps/openedx/config:/openedx/config:ro - depends_on: {{ [("mysql", RUN_MYSQL)]|list_if }} + depends_on: {{ [("mysql", RUN_MYSQL), ("mongodb", RUN_MONGODB)]|list_if }} cms-job: image: {{ DOCKER_IMAGE_OPENEDX }} diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 940d41fe90c..c2834c2b06c 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -170,7 +170,7 @@ services: environment: SERVICE_VARIANT: lms DJANGO_SETTINGS_MODULE: lms.envs.tutor.production - command: celery worker --app=lms.celery --loglevel=info --hostname=edx.lms.core.default.%%h --maxtasksperchild=100 --exclude-queues=edx.cms.core.default + command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro @@ -188,7 +188,7 @@ services: environment: SERVICE_VARIANT: cms DJANGO_SETTINGS_MODULE: cms.envs.tutor.production - command: celery worker --app=cms.celery --loglevel=info --hostname=edx.cms.core.default.%%h --maxtasksperchild 100 --exclude-queues=edx.lms.core.default + command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro