Merge remote-tracking branch 'origin/master' into nightly

CHANGELOG.md

@@ -4,6 +4,9 @@ Note: Breaking changes between versions are indicated by "💥".
 
 ## Unreleased
 
+- [Feature] Make it possible to override job configuration in development: if they exist, `dev/docker-compose.jobs.yml` and `dev/docker-compose.jobs.override.yml` will be loaded when running jobs.
+- [Improvement] Faster `tutor local start` by building only necessary images.
+
 ## v12.1.5 (2021-10-25)
 
 - 💥[Improvement] Change the `settheme` command such that, by default, a custom theme is assigned to the LMS and the CMS, both in production and development mode.

tutor/bindmounts.py

@@ -1,22 +1,19 @@
 import os
-from typing import Callable, List, Tuple
+from typing import List, Tuple
 
 import click
-from mypy_extensions import VarArg
 
 from .exceptions import TutorError
-from .types import Config
+from .jobs import BaseComposeJobRunner
 from .utils import get_user_id
 
 
 def create(
-    root: str,
-    config: Config,
-    docker_compose_func: Callable[[str, Config, VarArg(str)], int],
+    runner: BaseComposeJobRunner,
     service: str,
     path: str,
 ) -> str:
-    volumes_root_path = get_root_path(root)
+    volumes_root_path = get_root_path(runner.root)
     volume_name = get_name(path)
     container_volumes_root_path = "/tmp/volumes"
     command = """rm -rf {volumes_path}/{volume_name}
@@ -33,9 +30,7 @@ def create(
     if not os.path.exists(volumes_root_path):
         os.makedirs(volumes_root_path)
 
-    docker_compose_func(
-        root,
-        config,
+    runner.docker_compose(
         "run",
         "--rm",
         "--no-deps",

tutor/commands/compose.py

@@ -1,8 +1,7 @@
 import os
-from typing import Callable, List
+from typing import List
 
 import click
-from mypy_extensions import VarArg
 
 from .. import bindmounts
 from .. import config as tutor_config
@@ -12,18 +11,27 @@
 from .. import jobs
 from ..types import Config
 from .. import utils
-from .context import Context
+from .context import BaseJobContext
 
 
-class ComposeJobRunner(jobs.BaseJobRunner):
-    def __init__(
-        self,
-        root: str,
-        config: Config,
-        docker_compose_func: Callable[[str, Config, VarArg(str)], int],
-    ):
+class ComposeJobRunner(jobs.BaseComposeJobRunner):
+    def __init__(self, root: str, config: Config):
         super().__init__(root, config)
-        self.docker_compose_func = docker_compose_func
+        self.project_name = ""
+        self.docker_compose_files: List[str] = []
+        self.docker_compose_job_files: List[str] = []
+
+    def docker_compose(self, *command: str) -> int:
+        """
+        Run docker-compose with the right yml files.
+        """
+        args = []
+        for docker_compose_path in self.docker_compose_files:
+            if os.path.exists(docker_compose_path):
+                args += ["-f", docker_compose_path]
+        return utils.docker_compose(
+            *args, "--project-name", self.project_name, *command
+        )
 
     def run_job(self, service: str, command: str) -> int:
         """
@@ -32,22 +40,16 @@ def run_job(self, service: str, command: str) -> int:
         service does not exist, run the service from good old regular
         docker-compose.yml.
         """
-        run_command = [
-            "-f",
-            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
-        ]
-        override_path = tutor_env.pathjoin(
-            self.root, "local", "docker-compose.jobs.override.yml"
-        )
-        if os.path.exists(override_path):
-            run_command += ["-f", override_path]
+        run_command = []
+        for docker_compose_path in self.docker_compose_job_files:
+            path = tutor_env.pathjoin(self.root, docker_compose_path)
+            if os.path.exists(path):
+                run_command += ["-f", path]
         run_command += ["run", "--rm"]
         if not utils.is_a_tty():
             run_command += ["-T"]
         job_service_name = "{}-job".format(service)
-        return self.docker_compose_func(
-            self.root,
-            self.config,
+        return self.docker_compose(
             *run_command,
             job_service_name,
             "sh",
@@ -57,31 +59,34 @@ def run_job(self, service: str, command: str) -> int:
         )
 
 
+class BaseComposeContext(BaseJobContext):
+    def job_runner(self, config: Config) -> ComposeJobRunner:
+        raise NotImplementedError
+
+
 @click.command(
     short_help="Run all or a selection of services.",
     help="Run all or a selection of services. Docker images will be rebuilt where necessary.",
 )
 @click.option("-d", "--detach", is_flag=True, help="Start in daemon mode")
 @click.argument("services", metavar="service", nargs=-1)
 @click.pass_obj
-def start(context: Context, detach: bool, services: List[str]) -> None:
-    command = ["up", "--remove-orphans"]
+def start(context: BaseComposeContext, detach: bool, services: List[str]) -> None:
+    command = ["up", "--remove-orphans", "--build"]
     if detach:
         command.append("-d")
 
-    config = tutor_config.load(context.root)
-    # Rebuild Docker images with a `build: ...` context.
-    context.docker_compose(context.root, config, "build")
     # Start services
-    context.docker_compose(context.root, config, *command, *services)
+    config = tutor_config.load(context.root)
+    context.job_runner(config).docker_compose(*command, *services)
 
 
 @click.command(help="Stop a running platform")
 @click.argument("services", metavar="service", nargs=-1)
 @click.pass_obj
-def stop(context: Context, services: List[str]) -> None:
+def stop(context: BaseComposeContext, services: List[str]) -> None:
     config = tutor_config.load(context.root)
-    context.docker_compose(context.root, config, "stop", *services)
+    context.job_runner(config).docker_compose("stop", *services)
 
 
 @click.command(
@@ -105,7 +110,7 @@ def reboot(context: click.Context, detach: bool, services: List[str]) -> None:
 )
 @click.argument("services", metavar="service", nargs=-1)
 @click.pass_obj
-def restart(context: Context, services: List[str]) -> None:
+def restart(context: BaseComposeContext, services: List[str]) -> None:
     config = tutor_config.load(context.root)
     command = ["restart"]
     if "all" in services:
@@ -119,15 +124,15 @@ def restart(context: Context, services: List[str]) -> None:
                     command += ["cms", "cms-worker"]
             else:
                 command.append(service)
-    context.docker_compose(context.root, config, *command)
+    context.job_runner(config).docker_compose(*command)
 
 
 @click.command(help="Initialise all applications")
 @click.option("-l", "--limit", help="Limit initialisation to this service or plugin")
 @click.pass_obj
-def init(context: Context, limit: str) -> None:
+def init(context: BaseComposeContext, limit: str) -> None:
     config = tutor_config.load(context.root)
-    runner = ComposeJobRunner(context.root, config, context.docker_compose)
+    runner = context.job_runner(config)
     jobs.initialise(runner, limit_to=limit)
 
 
@@ -143,10 +148,15 @@ def init(context: Context, limit: str) -> None:
 @click.argument("email")
 @click.pass_obj
 def createuser(
-    context: Context, superuser: str, staff: bool, password: str, name: str, email: str
+    context: BaseComposeContext,
+    superuser: str,
+    staff: bool,
+    password: str,
+    name: str,
+    email: str,
 ) -> None:
     config = tutor_config.load(context.root)
-    runner = ComposeJobRunner(context.root, config, context.docker_compose)
+    runner = context.job_runner(config)
     command = jobs.create_user_command(superuser, staff, name, email, password=password)
     runner.run_job("lms", command)
 
@@ -166,18 +176,18 @@ def createuser(
 )
 @click.argument("theme_name")
 @click.pass_obj
-def settheme(context: Context, domains: List[str], theme_name: str) -> None:
+def settheme(context: BaseComposeContext, domains: List[str], theme_name: str) -> None:
     config = tutor_config.load(context.root)
-    runner = ComposeJobRunner(context.root, config, context.docker_compose)
+    runner = context.job_runner(config)
     domains = domains or jobs.get_all_openedx_domains(config)
     jobs.set_theme(theme_name, domains, runner)
 
 
 @click.command(help="Import the demo course")
 @click.pass_obj
-def importdemocourse(context: Context) -> None:
+def importdemocourse(context: BaseComposeContext) -> None:
     config = tutor_config.load(context.root)
-    runner = ComposeJobRunner(context.root, config, context.docker_compose)
+    runner = context.job_runner(config)
     fmt.echo_info("Importing demo course")
     jobs.import_demo_course(runner)
 
@@ -209,11 +219,9 @@ def run(context: click.Context, args: List[str]) -> None:
 )
 @click.argument("path")
 @click.pass_obj
-def bindmount_command(context: Context, service: str, path: str) -> None:
+def bindmount_command(context: BaseComposeContext, service: str, path: str) -> None:
     config = tutor_config.load(context.root)
-    host_path = bindmounts.create(
-        context.root, config, context.docker_compose, service, path
-    )
+    host_path = bindmounts.create(context.job_runner(config), service, path)
     fmt.echo_info(
         "Bind-mount volume created at {}. You can now use it in all `local` and `dev` commands with the `--volume={}` option.".format(
             host_path, path
@@ -267,7 +275,7 @@ def logs(context: click.Context, follow: bool, tail: bool, service: str) -> None
 @click.argument("command")
 @click.argument("args", nargs=-1)
 @click.pass_obj
-def dc_command(context: Context, command: str, args: List[str]) -> None:
+def dc_command(context: BaseComposeContext, command: str, args: List[str]) -> None:
     config = tutor_config.load(context.root)
     volumes, non_volume_args = bindmounts.parse_volumes(args)
     volume_args = []
@@ -284,9 +292,7 @@ def dc_command(context: Context, command: str, args: List[str]) -> None:
                 )
             volume_arg = "{}:{}".format(host_bind_path, volume_arg)
         volume_args += ["--volume", volume_arg]
-    context.docker_compose(
-        context.root, config, command, *volume_args, *non_volume_args
-    )
+    context.job_runner(config).docker_compose(command, *volume_args, *non_volume_args)
 
 
 def add_commands(command_group: click.Group) -> None:

tutor/commands/context.py

@@ -1,15 +1,30 @@
+from ..jobs import BaseJobRunner
 from ..types import Config
 
 
-def unimplemented_docker_compose(root: str, config: Config, *command: str) -> int:
-    raise NotImplementedError
+class Context:
+    """
+    Context object that is passed to all subcommands.
 
+    The project `root` is passed to all subcommands of `tutor`; that's because
+    it is defined as an argument of the top-level command. For instance:
+
+        tutor --root=... local run ...
+    """
 
-# pylint: disable=too-few-public-methods
-class Context:
     def __init__(self, root: str) -> None:
         self.root = root
-        self.docker_compose_func = unimplemented_docker_compose
 
-    def docker_compose(self, root: str, config: Config, *command: str) -> int:
-        return self.docker_compose_func(root, config, *command)
+
+class BaseJobContext(Context):
+    """
+    Specialized context that subcommands may use.
+
+    For instance `dev`, `local` and `k8s` define custom runners to run jobs.
+    """
+
+    def job_runner(self, config: Config) -> BaseJobRunner:
+        """
+        Return a runner capable of running docker-compose/kubectl commands.
+        """
+        raise NotImplementedError

tutor/commands/dev.py

@@ -1,44 +1,44 @@
-import os
 from typing import List
 
 import click
 
 from .. import config as tutor_config
 from .. import env as tutor_env
 from .. import fmt
-from ..types import Config
-from .. import utils
+from ..types import Config, get_typed
 from . import compose
-from .context import Context
 
 
-def docker_compose(root: str, config: Config, *command: str) -> int:
-    """
-    Run docker-compose with dev arguments.
-    """
-    args = []
-    for folder in ["local", "dev"]:
-        # Add docker-compose.yml and docker-compose.override.yml (if it exists)
-        # from "local" and "dev" folders (but not docker-compose.prod.yml)
-        args += [
-            "-f",
-            tutor_env.pathjoin(root, folder, "docker-compose.yml"),
+class DevJobRunner(compose.ComposeJobRunner):
+    def __init__(self, root: str, config: Config):
+        """
+        Load docker-compose files from dev/ and local/
+        """
+        super().__init__(root, config)
+        self.project_name = get_typed(self.config, "DEV_PROJECT_NAME", str)
+        self.docker_compose_files += [
+            tutor_env.pathjoin(self.root, "local", "docker-compose.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.yml"),
+            tutor_env.pathjoin(self.root, "local", "docker-compose.override.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.override.yml"),
         ]
-        override_path = tutor_env.pathjoin(root, folder, "docker-compose.override.yml")
-        if os.path.exists(override_path):
-            args += ["-f", override_path]
-    return utils.docker_compose(
-        *args,
-        "--project-name",
-        str(config["DEV_PROJECT_NAME"]),
-        *command,
-    )
+        self.docker_compose_job_files += [
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.yml"),
+            tutor_env.pathjoin(self.root, "local", "docker-compose.jobs.override.yml"),
+            tutor_env.pathjoin(self.root, "dev", "docker-compose.jobs.override.yml"),
+        ]
+
+
+class DevContext(compose.BaseComposeContext):
+    def job_runner(self, config: Config) -> DevJobRunner:
+        return DevJobRunner(self.root, config)
 
 
 @click.group(help="Run Open edX locally with development settings")
-@click.pass_obj
-def dev(context: Context) -> None:
-    context.docker_compose_func = docker_compose
+@click.pass_context
+def dev(context: click.Context) -> None:
+    context.obj = DevContext(context.obj.root)
 
 
 @click.command(