AWS Curvenote: Install Calico for network policies

.github/workflows/cd.yml

@@ -151,8 +151,9 @@ jobs:
       - name: "Stage 1: Install and setup helm ${{ env.HELM_VERSION }}"
         run: |
           curl -sf https://raw.githubusercontent.com/helm/helm/HEAD/scripts/get-helm-3 | DESIRED_VERSION=${HELM_VERSION} bash
-          helm dependency update ./mybinder
-          helm dependency update ./mybinder-kube-system
+          for d in ./mybinder/ ./system-charts/*/; do
+            helm dependency update "$d"
+          done
 
       # Action Repo: https://github.com/sliteteam/github-action-git-crypt-unlock
       - name: "Stage 2: Unlock git-crypt secrets"
@@ -309,8 +310,9 @@ jobs:
       - name: "Stage 1: Install and setup helm ${{ env.HELM_VERSION }}"
         run: |
           curl -sf https://raw.githubusercontent.com/helm/helm/HEAD/scripts/get-helm-3 | DESIRED_VERSION=${HELM_VERSION} bash
-          helm dependency update ./mybinder
-          helm dependency update ./mybinder-kube-system
+          for d in ./mybinder/ ./system-charts/*/; do
+            helm dependency update "$d"
+          done
 
       - name: "Stage 2: Unlock git-crypt secrets"
         uses: sliteteam/github-action-git-crypt-unlock@8b1fa3ccc81e322c5c45fbab261eee46513fd3f8

.gitignore

@@ -8,9 +8,12 @@ __pycache__
 config/common/datacenter-*.yaml
 secrets/banned_hosts.txt
 secrets/config/common/bans.yaml
-mybinder*/charts
-mybinder*/requirements.lock
-mybinder*/Chart.lock
+system-charts/*/charts
+system-charts/*/requirements.lock
+system-charts/*/Chart.lock
+mybinder/charts
+mybinder/requirements.lock
+mybinder/Chart.lock
 
 .ipynb_checkpoints
 

deploy.py

@@ -329,39 +329,47 @@ def patch_coredns(dry_run=False):
     )
 
 
-def deploy_kube_system_charts(release, name=None, dry_run=False):
+def deploy_system_charts(release, name=None, dry_run=False):
     """
-    Some charts must be deployed into the kube-system namespace
+    Some charts must be deployed into other namespaces
     """
     if not name:
         name = release
-    log_name = f"mybinder-kube-system {release}"
 
-    config_files = get_config_files(release, config_dir="config-kube-system")
-    if not config_files:
-        print(BOLD + GREEN + f"No config files found for {log_name}" + NC, flush=True)
-        return
+    charts = glob.glob("system-charts/*/Chart.yaml")
+    namespaces = [c.split("/")[1] for c in charts]
 
-    print(BOLD + GREEN + f"Starting helm upgrade for {log_name}" + NC, flush=True)
-    helm = [
-        "helm",
-        "upgrade",
-        "--install",
-        "--cleanup-on-fail",
-        "--namespace=kube-system",
-        name,
-        "mybinder-kube-system",
-    ]
-    for config_file in config_files:
-        helm.extend(["-f", config_file])
+    for ns in namespaces:
+        log_name = f"mybinder-{ns} {release}"
 
-    check_call(helm, dry_run)
-    print(
-        BOLD + GREEN + f"SUCCESS: Helm upgrade for {log_name} completed" + NC,
-        flush=True,
-    )
+        config_files = get_config_files(release, config_dir=f"system-config/{ns}")
+        if not config_files:
+            print(
+                BOLD + GREEN + f"No config files found for {log_name}" + NC, flush=True
+            )
+            return
+
+        print(BOLD + GREEN + f"Starting helm upgrade for {log_name}" + NC, flush=True)
+        helm = [
+            "helm",
+            "upgrade",
+            "--install",
+            "--cleanup-on-fail",
+            f"--namespace={ns}",
+            "--create-namespace",
+            name,
+            f"system-charts/{ns}",
+        ]
+        for config_file in config_files:
+            helm.extend(["-f", config_file])
+
+        check_call(helm, dry_run)
+        print(
+            BOLD + GREEN + f"SUCCESS: Helm upgrade for {log_name} completed" + NC,
+            flush=True,
+        )
 
-    wait_for_deployments_daemonsets("kube-system", dry_run)
+        wait_for_deployments_daemonsets(ns, dry_run)
 
 
 def main():
@@ -398,7 +406,7 @@ def main():
         action="store_true",
         help="Print commands, but don't run them",
     )
-    stages = ["all", "auth", "networkban", "kubesystem", "certmanager", "mybinder"]
+    stages = ["all", "auth", "networkban", "system", "certmanager", "mybinder"]
     argparser.add_argument(
         "--stage",
         choices=stages,
@@ -454,8 +462,8 @@ def main():
 
     if args.stage in ("all", "networkban"):
         update_networkbans(cluster, args.dry_run)
-    if args.stage in ("all", "kubesystem"):
-        deploy_kube_system_charts(args.release, args.name, args.dry_run)
+    if args.stage in ("all", "system"):
+        deploy_system_charts(args.release, args.name, args.dry_run)
     if args.stage in ("all", "certmanager"):
         setup_certmanager(args.dry_run)
     if args.stage in ("all", "mybinder"):

system-charts/tigera-operator/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+description: A meta-chart for the tiger-operator charts on some mybinder.org hosts
+name: mybinder-tiger-operator
+version: "0.0.1-set.by.chartpress"
+kubeVersion: ">= 1.26.0-0"
+dependencies:
+  # https://artifacthub.io/packages/helm/aws/aws-load-balancer-controller
+  - name: tigera-operator
+    version: 3.25.2
+    repository: https://docs.tigera.io/calico/charts
+    condition: tigera-operator.enabled

system-charts/tigera-operator/values.yaml

@@ -0,0 +1,3 @@
+# Tigera operator for installing Calico
+tigera-operator:
+  enabled: false

system-config/tigera-operator/curvenote.yaml

@@ -0,0 +1,11 @@
+# Install Calico for proper NetworkPolicy support
+tigera-operator:
+  enabled: true
+  installation:
+    kubernetesProvider: EKS
+  apiServer:
+    # Doesn't seem to work on EKS
+    # https://github.com/projectcalico/calico/issues/7555
+    # It's not needed anyway since we only want standard NetworkPolicy support,
+    # not 'projectcalico.org/v3'
+    enabled: false

terraform/aws/binder-eks/README.md

@@ -1,7 +1,30 @@
 # BinderHub on AWS EKS
 
+**_Important: Deploying this EKS cluster requires some manual steps after running Terraform_**
+
 This module deploys an AWS EKS cluster with IRSA roles to support BinderHub ECR access.
 
 The module has optional support for using a limited non-administrative AWS role with a permissions boundary to deploy the cluster.
 
 For an example see [curvenote](../curvenote/README.md)
+
+## Post-deployment steps
+
+After running Terraform, you will need to perform the following steps:
+
+### Install the AWS VPC CNI add-on
+
+Using the AWS CLI:
+
+1. Get the available vpc-cni versions: `aws eks describe-addon-versions --addon-name vpc-cni `
+2. `aws eks create-addon --cluster-name binderhub --addon-name vpc-cni --addon-version v1.15.3-eksbuild.1 --resolve-conflicts OVERWRITE`
+3. Wait for the status to change to `ACTIVE`: `aws eks describe-addon --cluster-name binderhub --addon-name vpc-cni`
+
+You can also do this using the AWS EKS web console:
+
+1. Go to the AWS EKS console and open the EKS cluster
+2. Under `Add-ons` choose `Get more add-ons`
+3. Select `Amazon VPC CNI`, click `Next`
+4. Select latest version of the plugin, use the default IAM role `Inherit from node`, click `Next`
+5. Client `Create`
+6. Wait for the status to change to `Active`

terraform/aws/curvenote/README.md

@@ -60,6 +60,8 @@ and deploy again
 terraform apply
 ```
 
+[Install the AWS VPC-CNI plugin](../binder-eks/README.md#install-the-aws-vpc-cni-add-on)
+
 ## Obtaining a kubeconfig file
 
 You must have the AWS CLI (v2 is recommended) and kubectl in your PATH.