deployer: create separate new typer apps under the deployer and update the general structure of the directory

.github/workflows/deploy-grafana-dashboards.yaml

@@ -60,4 +60,4 @@ jobs:
 
       - name: Deploy grafana dashboards for ${{ matrix.cluster_name }}
         run: |
-          deployer deploy-grafana-dashboards ${{ matrix.cluster_name }}
+          deployer grafana deploy-dashboards ${{ matrix.cluster_name }}

.github/workflows/deploy-hubs.yaml

@@ -15,7 +15,7 @@ on:
       - deployer/**
       # Exclude changes to the tests directory from
       # triggering this workflow
-      - "!deployer/tests/**"
+      - "!deployer/health_check_tests/**"
       - requirements.txt
       - .github/actions/setup-deploy/**
       - helm-charts/**
@@ -27,7 +27,7 @@ on:
       - deployer/**
       # Exclude changes to the tests directory from
       # triggering this workflow
-      - "!deployer/tests/**"
+      - "!deployer/health_check_tests/**"
       - requirements.txt
       - .github/actions/setup-deploy/**
       - helm-charts/**
@@ -49,7 +49,7 @@ env:
 jobs:
   # This job runs in Pull Requests and on pushes to the default branch. It identifies
   # which files have been added or modified by recent GitHub activity and parsed a list
-  # to the generate-helm-upgrade-jobs function of the deployer. This function generates
+  # to the `deployer generate helm-upgrade-job`s command of the deployer. This command generates
   # two lists of dictionaries, which can be read by GitHub Actions as matrix jobs. The
   # first set of jobs describes which clusters need their support chart and/or staging
   # hub upgraded; and the second set of jobs describe which production hubs require
@@ -113,7 +113,7 @@ jobs:
       # by one
       - name: Generate matrix jobs
         run: |
-          deployer generate-helm-upgrade-jobs "${{ steps.changed-files.outputs.changed_files }}"
+          deployer generate helm-upgrade-jobs "${{ steps.changed-files.outputs.changed_files }}"
 
       # The comment-deployment-plan-pr.yaml workflow won't have the correct context to
       # know the PR number, so we save it to a file to pass to that workflow

.github/workflows/validate-clusters.yaml

@@ -214,4 +214,4 @@ jobs:
         env:
           TERM: xterm
         run: |
-          deployer validate ${{ matrix.jobs.cluster_name }}
+          deployer validate all ${{ matrix.jobs.cluster_name }}

deployer/__main__.py

@@ -1,17 +1,20 @@
 # Import the various subcommands here, they will be automatically
 # registered into the app
-import deployer.billing.app  # noqa: F401
-import deployer.cilogon_app  # noqa: F401
-import deployer.cloud_access  # noqa: F401
-import deployer.debug  # noqa: F401
-import deployer.deployer  # noqa: F401
-import deployer.generate.generate_aws_cluster  # noqa: F401
-import deployer.generate.generate_gcp_cluster  # noqa: F401
-import deployer.grafana.central_grafana  # noqa: F401
-import deployer.grafana.grafana_tokens  # noqa: F401
+import deployer.commands.cilogon  # noqa: F401
+import deployer.commands.debug  # noqa: F401
+import deployer.commands.deployer  # noqa: F401
+import deployer.commands.exec.cloud  # noqa: F401
+import deployer.commands.exec.infra_components  # noqa: F401
+import deployer.commands.generate.billing.cost_table  # noqa: F401
+import deployer.commands.generate.dedicated_cluster.aws  # noqa: F401
+import deployer.commands.generate.dedicated_cluster.gcp  # noqa: F401
+import deployer.commands.generate.helm_upgrade.jobs  # noqa: F401
+import deployer.commands.grafana.central_grafana  # noqa: F401
+import deployer.commands.grafana.deploy_dashboards  # noqa: F401
+import deployer.commands.grafana.tokens  # noqa: F401
+import deployer.commands.validate.config  # noqa: F401
 import deployer.keys.decrypt_age  # noqa: F401
-
-from .cli_app import app
+from deployer.cli_app import app
 
 
 def main():

deployer/cli_app.py

@@ -1,5 +1,5 @@
 """
-Export the typer app we use throughout our codebase.
+Export the typer apps we use throughout our codebase.
 
 Having this in a single file allows multiple files to provide subcommands
 for the same CLI application. So we can put deployment related stuff under
@@ -10,3 +10,36 @@
 # The typer app to which all subcommands are attached
 # Disable 'pretty' exception handling
 app = typer.Typer(pretty_exceptions_show_locals=False)
+generate_app = typer.Typer(pretty_exceptions_show_locals=False)
+cilogon_client_app = typer.Typer(pretty_exceptions_show_locals=False)
+debug_app = typer.Typer(pretty_exceptions_show_locals=False)
+exec_app = typer.Typer(pretty_exceptions_show_locals=False)
+grafana_app = typer.Typer(pretty_exceptions_show_locals=False)
+validate_app = typer.Typer(pretty_exceptions_show_locals=False)
+
+app.add_typer(
+    generate_app,
+    name="generate",
+    help="Generate various types of assets. It currently supports generating files related to billing or new, dedicated clusters.",
+)
+app.add_typer(
+    cilogon_client_app,
+    name="cilogon-client",
+    help="Manage cilogon clients for hubs' authentication.",
+)
+app.add_typer(
+    exec_app,
+    name="exec",
+    help="Execute a shell in various parts of the infra. It can be used for poking around, or debugging issues.",
+)
+app.add_typer(
+    debug_app,
+    name="debug",
+    help="Debug issues by accessing different components and their logs",
+)
+app.add_typer(grafana_app, name="grafana", help="Manages Grafana related workflows.")
+app.add_typer(
+    validate_app,
+    name="validate",
+    help="Validate configuration files such as helm chart values and cluster.yaml files.",
+)

deployer/cilogon_app.py → deployer/commands/cilogon.py

@@ -24,14 +24,14 @@
 from ruamel.yaml import YAML
 from yarl import URL
 
-from .cli_app import app
-from .file_acquisition import (
+from deployer.cli_app import cilogon_client_app
+from deployer.utils.file_acquisition import (
     build_absolute_path_to_hub_encrypted_config_file,
     get_decrypted_file,
     persist_config_in_encrypted_file,
     remove_jupyterhub_hub_config_key_from_encrypted_file,
 )
-from .utils import print_colour
+from deployer.utils.rendering import print_colour
 
 yaml = YAML(typ="safe")
 
@@ -380,8 +380,8 @@ def get_2i2c_cilogon_admin_credentials():
     )
 
 
-@app.command()
-def cilogon_client_create(
+@cilogon_client_app.command()
+def create(
     cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
     hub_name: str = typer.Argument(
         ..., help="Name of the hub for which we'll create a CILogon client"
@@ -403,8 +403,8 @@ def cilogon_client_create(
     )
 
 
-@app.command()
-def cilogon_client_update(
+@cilogon_client_app.command()
+def update(
     cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
     hub_name: str = typer.Argument(
         ..., help="Name of the hub for which we'll update a CILogon client"
@@ -420,8 +420,8 @@ def cilogon_client_update(
     update_client(admin_id, admin_secret, cluster_name, hub_name, callback_url)
 
 
-@app.command()
-def cilogon_client_get(
+@cilogon_client_app.command()
+def get(
     cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
     hub_name: str = typer.Argument(
         ..., help="Name of the hub for which we'll retrieve the CILogon client details"
@@ -432,15 +432,15 @@ def cilogon_client_get(
     get_client(admin_id, admin_secret, cluster_name, hub_name)
 
 
-@app.command()
-def cilogon_client_get_all():
+@cilogon_client_app.command()
+def get_all():
     """Retrieve details about all existing 2i2c CILogon clients."""
     admin_id, admin_secret = get_2i2c_cilogon_admin_credentials()
     get_all_clients(admin_id, admin_secret)
 
 
-@app.command()
-def cilogon_client_delete(
+@cilogon_client_app.command()
+def delete(
     cluster_name: str = typer.Argument(..., help="Name of cluster to operate"),
     hub_name: str = typer.Argument(
         ...,

deployer/commands/debug.py

@@ -0,0 +1,147 @@
+"""
+Helper commands for debugging active issues in a hub
+"""
+import string
+import subprocess
+from enum import Enum
+
+import escapism
+import typer
+from ruamel.yaml import YAML
+
+from deployer.cli_app import debug_app
+from deployer.infra_components.cluster import Cluster
+from deployer.utils.file_acquisition import find_absolute_path_to_cluster_file
+
+# Without `pure=True`, I get an exception about str / byte issues
+yaml = YAML(typ="safe", pure=True)
+
+
+class InfraComponents(Enum):
+    """
+    Enum of various components that make up a particular hub's infrastructure
+    """
+
+    hub = "hub"
+    proxy = "proxy"
+    dask_gateway_api = "dask-gateway-api"
+    dask_gateway_controller = "dask-gateway-controller"
+    dask_gateway_traefik = "dask-gateway-traefik"
+
+
+@debug_app.command()
+def component_logs(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+    component: InfraComponents = typer.Argument(
+        ..., help="Component to display logs of"
+    ),
+    follow: bool = typer.Option(True, help="Live update new logs as they show up"),
+    previous: bool = typer.Option(
+        False,
+        help="If component pod has restarted, show logs from just before the restart",
+    ),
+):
+    """
+    Display logs from a particular component on a hub on a cluster
+    """
+
+    # hub_name is also the name of the namespace the hub is in
+    cmd = ["kubectl", "-n", hub_name, "logs"]
+
+    component_map = {
+        "hub": ["-l", "component=hub", "-c", "hub"],
+        "proxy": ["-l", "component=proxy"],
+        "dask-gateway-api": ["-l", "app.kubernetes.io/component=gateway"],
+        "dask-gateway-controller": ["-l", "app.kubernetes.io/component=controller"],
+        "dask-gateway-traefik": ["-l", "app.kubernetes.io/component=traefik"],
+    }
+
+    cmd += component_map[component.value]
+
+    if follow:
+        cmd += ["-f"]
+    if previous:
+        cmd += ["--previous"]
+    config_file_path = find_absolute_path_to_cluster_file(cluster_name)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+    with cluster.auth():
+        subprocess.check_call(cmd)
+
+
+@debug_app.command()
+def user_logs(
+    cluster_name: str = typer.Argument(..., help="Name of cluster to operate on"),
+    hub_name: str = typer.Argument(..., help="Name of hub to operate on"),
+    username: str = typer.Argument(
+        ..., help="Name of the JupyterHub user to get logs for"
+    ),
+    follow: bool = typer.Option(True, help="Live update new logs as they show up"),
+    previous: bool = typer.Option(
+        False,
+        help="If user pod has restarted, show logs from just before the restart",
+    ),
+):
+    """
+    Display logs from the notebook pod of a given user
+    """
+    # This is how kubespawner determines the 'safe' username, which is
+    # used in the label. Kubernetes restricts what characters can be in
+    # the label value.
+
+    # Note: '-' is not in safe_chars, as it is being used as escape character
+    safe_chars = set(string.ascii_lowercase + string.digits)
+    escaped_username = escapism.escape(
+        username, safe=safe_chars, escape_char="-"
+    ).lower()
+
+    cmd = [
+        "kubectl",
+        "-n",
+        hub_name,
+        "logs",
+        "-l",
+        f"hub.jupyter.org/username={escaped_username}",
+        "-c",
+        "notebook",
+    ]
+
+    if follow:
+        cmd += ["-f"]
+    if previous:
+        cmd += ["--previous"]
+    config_file_path = find_absolute_path_to_cluster_file(cluster_name)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+    with cluster.auth():
+        # hub_name is also the name of the namespace the hub is in
+        subprocess.check_call(cmd)
+
+
+@debug_app.command()
+def start_docker_proxy(
+    docker_daemon_cluster: str = typer.Argument(
+        "2i2c", help="Name of cluster where the docker daemon lives"
+    )
+):
+    """
+    Proxy a docker daemon from a remote cluster to local port 23760.
+    """
+
+    print(
+        "Run `export DOCKER_HOST=tcp://localhost:23760` on another terminal to use the remote docker daemon"
+    )
+    config_file_path = find_absolute_path_to_cluster_file(docker_daemon_cluster)
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f), config_file_path.parent)
+    with cluster.auth():
+        cmd = [
+            "kubectl",
+            "--namespace=default",
+            "port-forward",
+            "deployment/dind",
+            "23760:2376",
+        ]
+
+        subprocess.check_call(cmd)