Split act-runner into its own pod and offer as a distinct stack. (#612)

* Split act-runner into its own pod and offer as a distinct stack.

app/data/container-build/cerc-act-runner-task-executor/build.sh

@@ -2,4 +2,4 @@
 # Build a local version of the task executor for act-runner
 source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}

app/data/stacks/act-runner/README.md

@@ -0,0 +1,15 @@
+# act-runner stack
+
+## Example
+
+```
+$ laconic-so --stack act-runner deploy init --output act-runner.yml
+
+$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-1
+$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO" >> ~/opt/deployments/act-runner-1/config.env
+$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up
+
+$ laconic-so --stack act-runner deploy create --spec-file act-runner.yml --deployment-dir ~/opt/deployments/act-runner-2
+$ echo "CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR" >> ~/opt/deployments/act-runner-2/config.env
+$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up
+```

app/data/stacks/act-runner/stack.yml

@@ -0,0 +1,15 @@
+version: "1.1"
+name: act-runner
+description: "Local act-runner"
+repos:
+  - git.vdb.to/cerc-io/hosting
+  - gitea.com/gitea/act_runner
+containers:
+  - cerc/act-runner
+  - cerc/act-runner-task-executor
+pods:
+  - name: act-runner
+    repository: cerc-io/hosting
+    path: act-runner
+    pre_start_command: "pre_start.sh"
+    post_start_command: "post_start.sh"

app/data/stacks/package-registry/stack.yml

@@ -13,3 +13,8 @@ pods:
     path: gitea
     pre_start_command: "run-this-first.sh"
     post_start_command: "initialize-gitea.sh"
+  - name: act-runner
+    repository: cerc-io/hosting
+    path: act-runner
+    pre_start_command: "pre_start.sh"
+    post_start_command: "post_start.sh"

app/deploy/deploy.py

@@ -307,7 +307,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                 compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
             else:
                 if deployment:
-                    compose_file_name = os.path.join(compose_dir, "docker-compose.yml")
+                    compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod["pre_start_command"]
                     pod_post_start_command = pod["post_start_command"]
                     script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
@@ -317,7 +317,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
                         post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
                 else:
                     pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
-                    compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
+                    compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
                     pod_pre_start_command = pod["pre_start_command"]
                     pod_post_start_command = pod["post_start_command"]
                     if pod_pre_start_command is not None:

app/deploy/deployment_create.py

@@ -22,7 +22,7 @@
 from shutil import copy, copyfile, copytree
 import sys
 from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
-                      get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
+                      get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths)
 from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
 
 
@@ -106,9 +106,10 @@ def _fixup_pod_file(pod, spec, compose_dir):
                 pod["services"][container_name]["ports"] = container_ports
 
 
-def _commands_plugin_path(ctx: DeployCommandContext):
-    plugin_path = get_plugin_code_path(ctx.stack)
-    return plugin_path.joinpath("deploy", "commands.py")
+def _commands_plugin_paths(ctx: DeployCommandContext):
+    plugin_paths = get_plugin_code_paths(ctx.stack)
+    ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
+    return ret
 
 
 # See: https://stackoverflow.com/a/54625079/1701505
@@ -120,48 +121,53 @@ def call_stack_deploy_init(deploy_command_context):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "init"):
-            return imported_stack.init(deploy_command_context)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deploy_command_context)
+
+    ret = None
+    init_done = False
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "init"):
+                if not init_done:
+                    ret = imported_stack.init(deploy_command_context)
+                    init_done = True
+                else:
+                    # TODO: remove this restriction
+                    print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.")
+    return ret
 
 
 # TODO: fold this with function above
 def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetupCommand, extra_args):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deploy_command_context)
-    print(f"Path: {python_file_path}")
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "setup"):
-            return imported_stack.setup(deploy_command_context, parameters, extra_args)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deploy_command_context)
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "setup"):
+                imported_stack.setup(deploy_command_context, parameters, extra_args)
 
 
 # TODO: fold this with function above
 def call_stack_deploy_create(deployment_context, extra_args):
     # Link with the python file in the stack
     # Call a function in it
     # If no function found, return None
-    python_file_path = _commands_plugin_path(deployment_context.command_context)
-    if python_file_path.exists():
-        spec = util.spec_from_file_location("commands", python_file_path)
-        imported_stack = util.module_from_spec(spec)
-        spec.loader.exec_module(imported_stack)
-        if _has_method(imported_stack, "create"):
-            return imported_stack.create(deployment_context, extra_args)
-    else:
-        return None
+    python_file_paths = _commands_plugin_paths(deployment_context.command_context)
+    for python_file_path in python_file_paths:
+        if python_file_path.exists():
+            spec = util.spec_from_file_location("commands", python_file_path)
+            imported_stack = util.module_from_spec(spec)
+            spec.loader.exec_module(imported_stack)
+            if _has_method(imported_stack, "create"):
+                imported_stack.create(deployment_context, extra_args)
 
 
 # Inspect the pod yaml to find config files referenced in subdirectories
@@ -336,7 +342,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
         if global_options(ctx).debug:
             print(f"extra config dirs: {extra_config_dirs}")
         _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
-        with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
+        with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file:
             yaml.dump(parsed_pod_file, output_file)
         # Copy the config files for the pod, if any
         config_dirs = {pod}

app/util.py

@@ -79,16 +79,16 @@ def get_pod_list(parsed_stack):
     return result
 
 
-def get_plugin_code_path(stack):
+def get_plugin_code_paths(stack):
     parsed_stack = get_parsed_stack_config(stack)
     pods = parsed_stack["pods"]
-    # TODO: Hack
-    pod = pods[0]
-    if type(pod) is str:
-        result = get_stack_file_path(stack).parent
-    else:
-        pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
-        result = Path(os.path.join(pod_root_dir, "stack"))
+    result = []
+    for pod in pods:
+        if type(pod) is str:
+            result.append(get_stack_file_path(stack).parent)
+        else:
+            pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
+            result.append(Path(os.path.join(pod_root_dir, "stack")))
     return result