From d1f266c5f771910373667dcfb8ef0ef7d5793cbe Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Wed, 6 Apr 2022 11:46:22 -0400 Subject: [PATCH] feat: introduce `tutor dev quickstart` Add `tutor dev quickstart` command, which is equivalent to `tutor local quickstart`, but uses dev containers instead of local production ones provides some setup steps that are tailored to developers. This should remove some friction from the Open edX development setup process, which previously required that users provision using local producation containers but then stop them and switch to dev containers: * tutor local quickstart * tutor local stop * tutor dev start -d Document the command and its improved workflow in ./docs/tutorials/nightly.rst Fixes https://github.com/overhangio/2u-tutor-adoption/issues/58 --- CHANGELOG.md | 2 + docs/dev.rst | 89 +++++++++++++++++++++++------- docs/install.rst | 3 +- tests/commands/test_dev.py | 8 ++- tutor/commands/config.py | 4 +- tutor/commands/context.py | 6 +++ tutor/commands/dev.py | 108 ++++++++++++++++++++++++++++++++++++- tutor/interactive.py | 94 +++++++++++++++++++------------- 8 files changed, 255 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d27f33d3bb..1e634f81bf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Note: Breaking changes between versions are indicated by "💥". ## Unreleased +- [Feature] Add `tutor dev quickstart` command, which is equivalent to `tutor local quickstart`, but uses dev containers instead of local production ones, and provides some setup steps that are tailed to developers. This should remove some friction from the Open edX development setup process, which previously required that users provision using local producation containers (`tutor local quickstart`) but then stop them and switch to dev containers (`tutor local stop && tutor dev start -d`). + ## v13.1.8 (2022-03-18) - [Bugfix] Fix "evalsymlink failure" during `k8s quickstart` (#611). diff --git a/docs/dev.rst b/docs/dev.rst index ccff6d16920..f5e81981929 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -5,37 +5,70 @@ Open edX development In addition to running Open edX in production, Tutor can be used for local development of Open edX. This means that it is possible to hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX. -The following commands assume you have previously launched a :ref:`local ` Open edX platform. If you have not done so already, you should run:: +.. warning:: - tutor local quickstart + Do not run ``tutor dev ...`` commands on a production system. They can modify your configuration and database in ways that are nonsensical or insecure for production use. -To run the platform in development mode, you **must** answer no ("n") to the question "Are you configuring a production platform". -Note that the local.overhang.io `domain `__ and its `subdomains `__ all point to 127.0.0.1. This is just a domain name that was set up to conveniently access a locally running Open edX platform. +First-time setup +---------------- -Once the local platform has been configured, you should stop it so that it does not interfere with the development environment:: +Start by launching a development platform:: - tutor local stop + tutor dev quickstart -Finally, you should build the ``openedx-dev`` docker image:: +This will perform several tasks for you. It will: - tutor dev dc build lms +* stop any existing locally-running Tutor containers, -This ``openedx-dev`` development image differs from the ``openedx`` production image: +* disable HTTPS, -- The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). -- Additional python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. -- The edx-platform `development requirements `__ are installed. +* set your ``LMS_HOST`` to `local.overhang.io`_ (a conveninece domain that simply `points at 127.0.0.1`_), + +* prompt for a platform details (with suitable defaults), + +* offer to import the demo course, + +* offer create a superuser, + +* build an ``openedx-dev`` image, which based ``openedx`` production image but is `specialized for developer usage`_, + +* start LMS, CMS, supporting services, and any plugged-in services, + +* ensure databases are created and migrated, and + +* run service initialization scripts, such as service user creation and Waffle configuration. + +Once setup is complete, the platform will be running in the background: + +* LMS will be accessible at `http://local.overhang.io:8000 `_. +* CMS will be accessible at `http://studio.local.overhang.io:8001 `_. +* Plugged-in services should be accessible at their documented URLs. + +.. _local.overhang.io: http://local.overhang.io +.. _points at 127.0.0.1: https://dnschecker.org/#A/local.overhang.io + + +Stopping the platform +--------------------- + +To bring down your platrform's containers, simply run:: + + tutor dev stop -Since the ``openedx-dev`` is based upon the ``openedx`` docker image, it should be re-built every time the ``openedx`` docker image is modified. -Run a local development webserver ---------------------------------- +Starting the platform back up +----------------------------- -:: +Once you have used ``quickstart`` once, you can start the platform in the future with the lighter-weight ``start`` command, which brings up containers but does not perform any initialization tasks:: + + tutor dev start # Run platform in the same terminal ("attached") + tutor dev start -d # Or, run platform the in the background ("detached") + +Nonetheless, ``quickstart`` is idempotent, so it is always safe to run it again in the future without risk to your data. In fact, you may find it useful to use this command as a one-stop-shop for pulling images, running migrations, initializing new plugins you have enabled, and/or executing any new initialization steps that may have been introduced since you set up Tutor:: + + tutor dev quickstart --pullimages - tutor dev runserver lms # Access the lms at http://local.overhang.io:8000 - tutor dev runserver cms # Access the cms at http://studio.local.overhang.io:8001 Running arbitrary commands -------------------------- @@ -56,6 +89,26 @@ To collect assets, you can use the ``openedx-assets`` command that ships with Tu tutor dev run lms openedx-assets build --env=dev + +.. _specialized for developer usage: + +Rebuilding the openedx-dev image +-------------------------------- + +The ``openedx-dev`` Docker image is based on the same ``openedx`` image used by ``tutor local ...`` to run LMS and CMS. However, it has a few differences to make it more convenient for developers: + +- The user that runs inside the container has the same UID as the user on the host, to avoid permission problems inside mounted volumes (and in particular in the edx-platform repository). + +- Additional python and system requirements are installed for convenient debugging: `ipython `__, `ipdb `__, vim, telnet. + +- The edx-platform `development requirements `__ are installed. + + +If you are using a custom ``openedx`` image, then you will need to rebuild ``openedx-dev`` every time you modify ``openedx``. To so, run:: + + tutor dev dc build lms + + .. _bind_mounts: Sharing directories with containers diff --git a/docs/install.rst b/docs/install.rst index a21f7db671a..9529e056318 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -91,9 +91,10 @@ To upgrade Open edX or benefit from the latest features and bug fixes, you shoul pip install --upgrade tutor[full] -Then run the ``quickstart`` command again. Depending on your deployment target, run either:: +Then run the ``quickstart`` command again. Depending on your deployment target, run one of:: tutor local quickstart # for local installations + tutor dev quickstart # for local development installations tutor k8s quickstart # for Kubernetes installation Upgrading with custom Docker images diff --git a/tests/commands/test_dev.py b/tests/commands/test_dev.py index 7a7205a1c42..1745b525c1e 100644 --- a/tests/commands/test_dev.py +++ b/tests/commands/test_dev.py @@ -3,7 +3,7 @@ from click.testing import CliRunner from tutor.commands.compose import bindmount_command -from tutor.commands.dev import dev +from tutor.commands.dev import dev, quickstart class DevTests(unittest.TestCase): @@ -13,6 +13,12 @@ def test_dev_help(self) -> None: self.assertEqual(0, result.exit_code) self.assertIsNone(result.exception) + def test_local_quickstart_help(self) -> None: + runner = CliRunner() + result = runner.invoke(quickstart, ["--help"]) + self.assertEqual(0, result.exit_code) + self.assertIsNone(result.exception) + def test_dev_bindmount(self) -> None: runner = CliRunner() result = runner.invoke(bindmount_command, ["--help"]) diff --git a/tutor/commands/config.py b/tutor/commands/config.py index a776dc3218d..2c57c6c0597 100644 --- a/tutor/commands/config.py +++ b/tutor/commands/config.py @@ -48,7 +48,9 @@ def save( unset_vars: List[str], env_only: bool, ) -> None: - config = interactive_config.load_user_config(context.root, interactive=interactive) + config = interactive_config.load_user_config( + context.root, interactive=interactive, dev_context=context.is_dev() + ) if set_vars: for key, value in dict(set_vars).items(): config[key] = env.render_unknown(config, value) diff --git a/tutor/commands/context.py b/tutor/commands/context.py index 3e859e05bec..ab02c3324b9 100644 --- a/tutor/commands/context.py +++ b/tutor/commands/context.py @@ -15,6 +15,12 @@ class Context: def __init__(self, root: str) -> None: self.root = root + def is_dev(self) -> bool: + """ + Are we running from a developer (`tutor dev ...`) context? + """ + return False + class BaseJobContext(Context): """ diff --git a/tutor/commands/dev.py b/tutor/commands/dev.py index 62fbaeaa465..11d5a235660 100644 --- a/tutor/commands/dev.py +++ b/tutor/commands/dev.py @@ -4,8 +4,9 @@ from .. import config as tutor_config from .. import env as tutor_env -from .. import fmt +from .. import exceptions, fmt, utils from ..types import Config, get_typed +from .config import save as config_save_command from . import compose @@ -34,6 +35,110 @@ class DevContext(compose.BaseComposeContext): def job_runner(self, config: Config) -> DevJobRunner: return DevJobRunner(self.root, config) + def is_dev(self) -> bool: + return True + + +@click.command(help="Configure and run Open edX from scratch, for development") +@click.option("-I", "--non-interactive", is_flag=True, help="Run non-interactively") +@click.option("-p", "--pullimages", is_flag=True, help="Update docker images") +@click.pass_context +def quickstart(context: click.Context, non_interactive: bool, pullimages: bool) -> None: + try: + utils.check_macos_docker_memory() + except exceptions.TutorError as e: + fmt.echo_alert( + f"""Could not verify sufficient RAM allocation in Docker: + + {e} + +Tutor may not work if Docker is configured with < 4 GB RAM. Please follow instructions from: + + https://docs.tutor.overhang.io/install.html""" + ) + + click.echo(fmt.title("Interactive platform configuration")) + context.invoke(config_save_command, interactive=(not non_interactive)) + config = tutor_config.load(context.obj.root) + + if non_interactive: + import_demo = False + create_superuser = False + else: + import_demo = click.confirm( + fmt.question("Import the demo course?"), + prompt_suffix=" ", + default=True, + ) + create_superuser = click.confirm( + fmt.question("Create/update a superuser?"), + prompt_suffix=" ", + default=True, + ) + if create_superuser: + superuser_username = click.prompt( + fmt.question(" Username for superuser"), + prompt_suffix=" ", + show_default=True, + default="admin", + ) + superuser_email = click.prompt( + fmt.question(" Email for superuser"), + prompt_suffix=" ", + show_default=True, + default=f"admin@{config['LMS_HOST']}", + ) + superuser_password = click.prompt( + fmt.question(" Password for superuser"), + prompt_suffix=" ", + show_default=True, + default="password", + ) + + click.echo(fmt.title("Stopping any existing platform")) + context.invoke(compose.stop) + + if pullimages: + click.echo(fmt.title("Docker image updates")) + context.invoke(compose.dc_command, command="pull") + + click.echo(fmt.title("Building Docker image for LMS and CMS development")) + context.invoke(compose.dc_command, command="build", args=["lms"]) + + click.echo(fmt.title("Starting the platform in detached mode")) + context.invoke(compose.start, detach=True) + + click.echo(fmt.title("Database creation and migrations")) + context.invoke(compose.init) + + if import_demo: + click.echo(fmt.title("Importing the demo course")) + context.invoke(compose.importdemocourse) + + if create_superuser: + click.echo(fmt.title("Creating or updating a superuser")) + context.invoke( + compose.createuser, + name=superuser_username, + password=superuser_password, + email=superuser_email, + staff=True, + superuser=True, + ) + + fmt.echo_info( + """The Open edX platform is now running in detached mode +Your Open edX platform is ready and can be accessed at the following urls: + + {http}://{lms_host} + {http}://{cms_host} + """.format( + http="https" if config["ENABLE_HTTPS"] else "http", + lms_host=config["LMS_HOST"], + cms_host=config["CMS_HOST"], + ) + ) + @click.group(help="Run Open edX locally with development settings") @click.pass_context @@ -62,5 +167,6 @@ def runserver(context: click.Context, options: List[str], service: str) -> None: context.invoke(compose.run, args=args) +dev.add_command(quickstart) dev.add_command(runserver) compose.add_commands(dev) diff --git a/tutor/interactive.py b/tutor/interactive.py index 25ba3d9f750..ffd4d2e1b49 100644 --- a/tutor/interactive.py +++ b/tutor/interactive.py @@ -7,48 +7,78 @@ from .types import Config, get_typed -def load_user_config(root: str, interactive: bool = True) -> Config: +def load_user_config( + root: str, interactive: bool = True, dev_context: bool = False +) -> Config: """ Load configuration and interactively ask questions to collect param values from the user. """ config = tutor_config.load_minimal(root) + defaults = tutor_config.get_defaults(config) + prod = should_configure_for_prod(config, interactive, dev_context=dev_context) + + if prod: + if interactive: + prompt_and_configure_prod_host(config, defaults) + else: + configure_dev_host(config) + if interactive: - ask_questions(config) + prompt_and_configure_platform_details(config, defaults) + return config -def ask_questions(config: Config) -> None: - defaults = tutor_config.get_defaults(config) - run_for_prod = config.get("LMS_HOST") != "local.overhang.io" - run_for_prod = click.confirm( +def should_configure_for_prod( + config: Config, interactive: bool, dev_context: bool +) -> bool: + if dev_context: + # If we're running from a `tutor dev ...` command, then just assume + # that we're *not* configuring a production platform. + return False + already_configured_for_dev = config.get("LMS_HOST") == "local.overhang.io" + if not interactive: + return not already_configured_for_dev + return click.confirm( fmt.question( "Are you configuring a production platform? Type 'n' if you are just testing Tutor on your local computer" ), prompt_suffix=" ", - default=run_for_prod, + default=(not already_configured_for_dev), + ) + + +def configure_dev_host(config: Config) -> None: + dev_values: Config = { + "LMS_HOST": "local.overhang.io", + "CMS_HOST": "studio.local.overhang.io", + "ENABLE_HTTPS": False, + } + fmt.echo_info( + """As you are not running this platform in production, we automatically set the following configuration values:""" ) - if not run_for_prod: - dev_values: Config = { - "LMS_HOST": "local.overhang.io", - "CMS_HOST": "studio.local.overhang.io", - "ENABLE_HTTPS": False, - } - fmt.echo_info( - """As you are not running this platform in production, we automatically set the following configuration values:""" + for k, v in dev_values.items(): + config[k] = v + fmt.echo_info(" {} = {}".format(k, v)) + + +def prompt_and_configure_prod_host(config: Config, defaults: Config) -> None: + ask("Your website domain name for students (LMS)", "LMS_HOST", config, defaults) + lms_host = get_typed(config, "LMS_HOST", str) + if "localhost" in lms_host: + raise exceptions.TutorError( + "You may not use 'localhost' as the LMS domain name. To run a local platform for testing purposes you should answer 'n' to the previous question." ) - for k, v in dev_values.items(): - config[k] = v - fmt.echo_info(" {} = {}".format(k, v)) - - if run_for_prod: - ask("Your website domain name for students (LMS)", "LMS_HOST", config, defaults) - lms_host = get_typed(config, "LMS_HOST", str) - if "localhost" in lms_host: - raise exceptions.TutorError( - "You may not use 'localhost' as the LMS domain name. To run a local platform for testing purposes you should answer 'n' to the previous question." - ) - ask("Your website domain name for teachers (CMS)", "CMS_HOST", config, defaults) + ask("Your website domain name for teachers (CMS)", "CMS_HOST", config, defaults) + ask_bool( + "Activate SSL/TLS certificates for HTTPS access?", + "ENABLE_HTTPS", + config, + defaults, + ) + +def prompt_and_configure_platform_details(config: Config, defaults: Config) -> None: ask("Your platform name/title", "PLATFORM_NAME", config, defaults) ask("Your public contact email address", "CONTACT_EMAIL", config, defaults) ask_choice( @@ -136,16 +166,6 @@ def ask_questions(config: Config) -> None: "zh-tw", ], ) - if run_for_prod: - ask_bool( - ( - "Activate SSL/TLS certificates for HTTPS access? Important note:" - " this will NOT work in a development environment." - ), - "ENABLE_HTTPS", - config, - defaults, - ) def ask(question: str, key: str, config: Config, defaults: Config) -> None: