Automatically install dask-gateway CRDs for daskhubs

deployer/README.md

@@ -63,18 +63,18 @@ It takes a name of a cluster and a name of a hub (or list of names) as arguments
 **Command line usage:**
 
 ```bash
-usage: python deployer deploy [-h] [--skip-hub-health-test] [--config-path CONFIG_PATH] cluster_name [hub_name]
+usage: python deployer deploy [-h] [--config-path CONFIG_PATH] [--dask-gateway-version DASK_GATEWAY_VERSION] cluster_name [hub_name]
 
 positional arguments:
   cluster_name          The name of the cluster to perform actions on
   hub_name              The hub, or list of hubs, to install/upgrade the helm chart for
 
 optional arguments:
   -h, --help            show this help message and exit
-  --skip-hub-health-test
-                        Bypass the hub health test
   --config-path CONFIG_PATH
                         File to read secret deployment configuration from
+  --dask-gateway-version DASK_GATEWAY_VERSION
+                        For daskhubs, the version of dask-gateway to install for the CRDs. Default: 2022.6.1
 ```
 
 ### `validate`

deployer/cli.py

@@ -64,15 +64,20 @@ def main():
         nargs="?",
         help="The hub, or list of hubs, to install/upgrade the helm chart for",
     )
-    deploy_parser.add_argument(
-        "--skip-hub-health-test", action="store_true", help="Bypass the hub health test"
-    )
     deploy_parser.add_argument(
         "--config-path",
         help="File to read secret deployment configuration from",
         # This filepath is relative to the PROJECT ROOT
         default="shared/deployer/enc-auth-providers-credentials.secret.yaml",
     )
+    deploy_parser.add_argument(
+        "--dask-gateway-version",
+        type=str,
+        # This version must match what is listed in daskhub's Chart.yaml file
+        # https://github.com/2i2c-org/infrastructure/blob/HEAD/helm-charts/daskhub/Chart.yaml#L14
+        default="2022.6.1",
+        help="For daskhubs, the version of dask-gateway to install for the CRDs. Default: 2022.6.1",
+    )
 
     # Validate subcommand
     validate_parser = subparsers.add_parser(
@@ -161,6 +166,7 @@ def main():
             args.cluster_name,
             args.hub_name,
             args.config_path,
+            dask_gateway_version=args.dask_gateway_version,
         )
     elif args.action == "exec-homes-shell":
         exec_homes_shell(args.cluster_name, args.hub_name)

deployer/deploy_actions.py

@@ -180,7 +180,7 @@ def deploy_grafana_dashboards(cluster_name):
         shutil.rmtree(dashboards_dir)
 
 
-def deploy(cluster_name, hub_name, config_path):
+def deploy(cluster_name, hub_name, config_path, dask_gateway_version):
     """
     Deploy one or more hubs in a given cluster
     """
@@ -221,13 +221,13 @@ def deploy(cluster_name, hub_name, config_path):
         if hub_name:
             hub = next((hub for hub in hubs if hub.spec["name"] == hub_name), None)
             print_colour(f"Deploying hub {hub.spec['name']}...")
-            hub.deploy(k, SECRET_KEY)
+            hub.deploy(k, SECRET_KEY, dask_gateway_version)
         else:
             for i, hub in enumerate(hubs):
                 print_colour(
                     f"{i+1} / {len(hubs)}: Deploying hub {hub.spec['name']}..."
                 )
-                hub.deploy(k, SECRET_KEY)
+                hub.deploy(k, SECRET_KEY, dask_gateway_version)
 
 
 def generate_helm_upgrade_jobs(changed_filepaths):

deployer/hub.py

@@ -306,7 +306,7 @@ def exec_homes_shell(self):
 
         subprocess.check_call(cmd)
 
-    def deploy(self, auth_provider, secret_key):
+    def deploy(self, auth_provider, secret_key, dask_gateway_version):
         """
         Deploy this hub
         """
@@ -334,6 +334,16 @@ def deploy(self, auth_provider, secret_key):
 
         generated_values = self.get_generated_config(auth_provider, secret_key)
 
+        if self.spec["helm_chart"] == "daskhub":
+            # Install CRDs for daskhub before deployment
+            manifest_urls = [
+                f"https://raw.githubusercontent.com/dask/dask-gateway/{dask_gateway_version}/resources/helm/dask-gateway/crds/daskclusters.yaml",
+                f"https://raw.githubusercontent.com/dask/dask-gateway/{dask_gateway_version}/resources/helm/dask-gateway/crds/traefik.yaml",
+            ]
+
+            for manifest_url in manifest_urls:
+                subprocess.check_call(["kubectl", "apply", "-f", manifest_url])
+
         with tempfile.NamedTemporaryFile(
             mode="w"
         ) as generated_values_file, get_decrypted_files(

helm-charts/daskhub/Chart.yaml

@@ -7,6 +7,9 @@ dependencies:
   - name: basehub
     version: "0.1.0"
     repository: file://../basehub
+    # If bumping the version of dask-gateway, please also bump the default version set
+    # in the deployer's CLI
+    # https://github.com/2i2c-org/infrastructure/blob/HEAD/deployer/cli.py#L73-L80
   - name: dask-gateway
     version: "2022.6.1"
     repository: "https://helm.dask.org/"