Setup shared cluster on AWS and deploy 'researchdelight' hub

eksctl/shared-hubs-cluster.jsonnet

@@ -0,0 +1,112 @@
+// Exports an eksctl config file for carbonplan cluster
+local ng = import "./libsonnet/nodegroup.jsonnet";
+
+// place all cluster nodes here
+local clusterRegion = "us-west-2";
+local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"];
+local nodeAz = "us-west-2a";
+
+// Node definitions for notebook nodes. Config here is merged
+// with our notebook node definition.
+// A `node.kubernetes.io/instance-type label is added, so pods
+// can request a particular kind of node with a nodeSelector
+local notebookNodes = [
+    { instanceType: "m5.large" },
+    { instanceType: "m5.xlarge" },
+    { instanceType: "m5.2xlarge" },
+    { instanceType: "m5.8xlarge" },
+];
+
+local daskNodes =
+    if "daskhub" == "daskhub" then [
+    // Node definitions for dask worker nodes. Config here is merged
+    // with our dask worker node definition, which uses spot instances.
+    // A `node.kubernetes.io/instance-type label is set to the name of the
+    // *first* item in instanceDistribution.instanceTypes, to match
+    // what we do with notebook nodes. Pods can request a particular
+    // kind of node with a nodeSelector
+        { instancesDistribution+: { instanceTypes: ["m5.large"] }},
+        { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }},
+        { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }},
+        { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }},
+    ];
+{
+    apiVersion: 'eksctl.io/v1alpha5',
+    kind: 'ClusterConfig',
+    metadata+: {
+        name: "shared-hubs-cluster",
+        region: clusterRegion,
+        // Warning: version 1.23 introduces some breaking changes
+        // Checkout the docs before upgrading
+        // ref: https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi-migration-faq.html
+        version: '1.22'
+    },
+    availabilityZones: masterAzs,
+    iam: {
+        withOIDC: true,
+    },
+    nodeGroups: [
+        ng {
+            name: 'core-a',
+            availabilityZones: [nodeAz],
+            ssh: {
+                publicKeyPath: 'ssh-keys/shared-hubs-cluster.key.pub'
+            },
+            instanceType: "m5.xlarge",
+            minSize: 1,
+            maxSize: 6,
+            labels+: {
+                "hub.jupyter.org/node-purpose": "core",
+                "k8s.dask.org/node-purpose": "core"
+            },
+        },
+    ] + [
+        ng {
+            // NodeGroup names can't have a '.' in them, while
+            // instanceTypes always have a .
+            name: "nb-%s" % std.strReplace(n.instanceType, ".", "-"),
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            instanceType: n.instanceType,
+            ssh: {
+                publicKeyPath: 'ssh-keys/shared-hubs-cluster.key.pub'
+            },
+            labels+: {
+                "hub.jupyter.org/node-purpose": "user",
+                "k8s.dask.org/node-purpose": "scheduler"
+            },
+            taints+: {
+                "hub.jupyter.org_dedicated": "user:NoSchedule",
+                "hub.jupyter.org/dedicated": "user:NoSchedule"
+            },
+
+        } + n for n in notebookNodes
+    ] + ( if daskNodes != null then
+        [
+        ng {
+            // NodeGroup names can't have a '.' in them, while
+            // instanceTypes always have a .
+            name: "dask-%s" % std.strReplace(n.instancesDistribution.instanceTypes[0], ".", "-"),
+            availabilityZones: [nodeAz],
+            minSize: 0,
+            maxSize: 500,
+            ssh: {
+                publicKeyPath: 'ssh-keys/shared-hubs-cluster.key.pub'
+            },
+            labels+: {
+                "k8s.dask.org/node-purpose": "worker"
+            },
+            taints+: {
+                "k8s.dask.org_dedicated" : "worker:NoSchedule",
+                "k8s.dask.org/dedicated" : "worker:NoSchedule"
+            },
+            instancesDistribution+: {
+                onDemandBaseCapacity: 0,
+                onDemandPercentageAboveBaseCapacity: 0,
+                spotAllocationStrategy: "capacity-optimized",
+            },
+        } + n for n in daskNodes
+        ] else []
+    )
+}

eksctl/ssh-keys/secret/shared-hubs-cluster.key

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:hPSfJceh/gU+/f8DCBy4L4Ywf70U2d/FARSi8dUBNwEhy+Hr0ukrZl9oJGZwYoR8FpjGFk6TBGLGh5u4jXCjBisj8Fnl4DltcsiC5niK/FnLNVu7q2o8q1wsC+apOzXHgfEqs5MTLeCSn6qSz01uIyeNkTBxikJ2CmOrykq7uZpiS1fC8gzjLz6gAJ+I3aZxh1PjsFMc2H78ne6axW9Rl2mNQAhI9PxFYa2rihUeV4kFiaDoU5jf0f6E5Z7gNY41mj+4eyG/W6fc0ox70sCoRYdgkMBHgVKuqTh0F6ApDGG8dZmm8ZAGOqCMW8Qnnb6FNooUbbmbrZwa2kBqMxERTEvQ2BGou2TG7MaWwRVO/aWOKyIj9nXnQf8L3Ewpkv+yJdh5ZWBOsqn59W15YW86+PlokAL2aXQH964fX5EBRRoeyafPmQHfK5ZMSYwfAEnfGFN3GQ/mE/e4Z0sYoch9gbVbbsnsVg06edbNpzyck3wzJTK2knmqQB+gxsrBTAvatexg5dm5MvVW6tbchnDSnTb2JZ0/YhIU3BACl1A2AONVknjoccJS2EVs3PoPt+/iXyNabzddG//zLxggpjHqnP5Rg8sfJVne23YIJsZNFiTWrXinj0ST8pAzyKciTr5MEhS2XHNJBi8/vvjwhK442qwGnWoNXTMS7LpnlBPgda+xk/YEpo7v/L2UpmFTMBrtMF/Vj4sqdGW0Qwi3/QLimws4Tev8rjefXvqhe5g3C053r9QDDaDF1YdKpeyDFtESB+sSSbMrwCBI40hC51Ul25Y3pvESqQuuw8ffnKDZgB4lWUuGyfk5MLY4lTIYld8zcWMpwo3kj3gGyeB0S1XuaeillDQa/acVjK+8KFXPeFJmhR4SpDyGc8zLeDp0WY35pO9q8aSjpbC0pN3CTwnLOmUgf1L4PMcwUkJ4dy20p3JQaZNVY5W4vFhx3evVatWjOyXZ2q9JQQctjGUcJsZZ+tx2MdZ117sfKjvfby5M+WDREUOIioKZAjaQO0hkbYxZz5eA1RGRl9Ix/tiG5GcezHat86u0PL/0q2G9Eqar9sZzSTn04GmbeyP5wboQONO84dSkxnr97w+9WT7L/bUZi8XvM49a01UQLqbCJ0IZjyUem4O+2c6ypDs6K35UZIwyTGH84EQwm+/eIuXKgaOpVyUUUDphyEpC7mvU06TPf4Oognh0dUD2Z2ATYqxbJnydK/6POdZSTjVwdNeg1KiHqRGOEohQN0KZYNif6GRoQIutxR9yoC+rWUIu7a7o/XI31cBEfN9VfXD7aXEZoXuAonFPMkYk1cz25JXYqf0vFWQFSLXOqaASCktA8bGm0i5GFTJ7AuqKSZgQZg1Z9CZO/se/eV6LNwX0N7+VqG5zllPc9AzRps60O0L85raqSn/4pnS4fZPYIbdZxBD52Nsv97r+Hysg0KGgeX1mTRryBX4YZJKcyz2tmN2SCIanWY8VwSXyLtqnDNTSd58X0TZuqoT7F/23mmWUpFWV1ZvKB69GEAo07LhfIlbR7RXO1zpkiLJXO910b3CIa/ZdlgR578E9S8dD2UqOpZN5ixcunaMhgJ1X+D3hlt2u0AbuUesims/lijvCXLzzrNQ2XLyiDxwcXrBk/W4alw/TBCOMlyrSJZh+CHIOsxmAAfmBjhZdzurVBoQglOIdPZ+h22e1LsDq8MgVOTeudZ+bmpSxsyEuQpoUJFmxlbs7bi3CDZKWWODXzzKZT8aUfEOijZ0AzsePhPPMBqICsf60LBdmXQKIrDwGs7b62D7a45y3jIKWqvaL3Le7OtqoPLfyFd06t3bEQwr0SKiIER/2K1W/sUi8qiOL7NI4hn6TDIPhSq+K4LRnR3plbysnRwOdW04h0vfpXUuNdGViu3WD3ejgR7/shgnBegg17AIYSmCIwmy0PF9w3AZcTcI0UD2gNoCdTwaaSxjwQYAYWYb3HkuZNw/lpR0MeJvfhTpkYFacXj2dB3wCJMcYKtaZzFz+l55a8wiabV0cP8D0By1bRoCKEV3roANzCoDldHm1FeMQ2JrQtfQ3OBb6MiHv0hCRprFmo/AlpQ+PSISu3SjShVtg7CjF3pu+4RvOfe4jtEimJFebaca7mRYp8ac/BcMqHKs4p+6jxk0LpSZdMtDuPOz7mrBu8i/GqSc2AzMHJvAG9n4yDZyUFMqpgmVvyBSf3F0Bv6cJnXLJw4lzfu43Hosr9GvTn9yrLOCmTF70ppIzOseXcX+jcGaYT5I/KxuwKhUzt5SVFKwUt10okpC0+6gO2gRjVBu3vi3TeIWtRNcqkydhXv6dhXXpwW4zG+lM1gJb4aZ97urqpPeZJWvendZnSAl0ArSN0Ez57D8I+ovQRqf5nwgqKOKakYXE4dAb0GvoKFUJ+CUB8Sc4BOfYbIs3JFSC4BbbudhXe930/OdS+j99vZ77PfNoRd3jL7BOidDP3zwZ3O02WPSGcz1hgmZsHwx6gl8vZjiN3cPwZqTAs+B0wdU7ght1W0J5P+kESo5TG1ihx/HuSqmqljbubTvznmZ1/ok9tYa5ucUgauVE1zz+PYus+48exr7RQDMeZitKxCkSId9eiEYqaHfrZFlZTruceOvjTMol8/KBInyglriBBvq0Q6sGWttSvEZDyAF15+H8TQZQqyHFrFTWi3KvqMYNbunG1M5Ur5actQpDmJ0FdY6JbE55cPf7qBdYyoghcvMPGlQWF8GIUy3J4/NtplkaGqlInZ14RRzF/1mmXgi2NXg71og8rQP4zdBjfUqD5MRf9j2mQgcYfXiK68coyUGAH8XQ+nK4hp3iLFJrsHdzgzmY/e4QzsXt8FUjIG90GnijvqtkNBiWl8cdFXcCMQxhvNQJkeRSXag8fCspMGSO9YbsFbJqbklEyKY9jAfCPmcYCugZti8wC6Q9xKtrfJxp18OaIy8ZbZnokLcAx4OJ820EOoM4aZZapX4ddFaQ4OrFAqIgtjUhrhQ6lNNdUd98OANxHPAGwHasq3YRxAooh08nWueFgwwVGMpMvs11WnL5fkcxqduxF8csInH/nDj4AUR/wyj1fcooKjr0SML9vxQQqI7IQvm+8CMBEBhHEcIKsOnW98MUnyT6inlKGTCjpJT99ZkFFjvre1bqJAp9n/PPerH0ru7Fa6oe2hYsSJacKTaJ53w2Jy9Msz/Cmzgl7mwR3V8GYSjqDExLQdxgvgKxEXOj+CboBqq9f4gMDcljmZJSokhNtGvysmxuyyutKWTqu5H8yDH2HJwHpsbjeo6Bs/hRHkuVyg1DJTxsbImJZipRnwSJJJg2PYhW8uETWlIEUO/TmOdw8DfkyDBd4Rx26hJsm+z8f095JC0L/dJhA3AmBRi4im6DAaq+efHQSCItkEg/pr3x4W4YXwCxHlFX2tR0DICUKFrVzc9FFiptm6JsKfFt98TGAKZsdbcgLBQg/VmAZ6w94hYIEXAq2g4UFaiMroWxjMaEcC2CIziq,iv:GpFd3mgEG/XFUIKaqrQDeGK7BINl0f8hkeYKBhZTrsI=,tag:6JiZVvnatz+E4zw99DDQmg==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": [
+			{
+				"resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs",
+				"created_at": "2022-11-30T11:19:43Z",
+				"enc": "CiUA4OM7eCzZe7vZVl3dWQ/xGiXsJanYBfOikD3K50a8O11Q/NzXEkgA+0T9hdnqI92imnVspFdEh9IBkh7LZHZjmzalHG9CU0Wh/saAr25QpzEQaAQhyaubdGENj53KuGW2w75sPU3bUhdgSS0nCZo="
+			}
+		],
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2022-11-30T11:19:43Z",
+		"mac": "ENC[AES256_GCM,data:WTiGyCMo5Jnh1jog/JVjMpUsU58KNns+JkQ7igxIQ1nk39c/PVz8cws0ROpw3QAaDvrRGuYnfhndXEIVTW1wXe4Qs4LtgwsBsD2GhmWaAyjEmnxDaiQ6bUxhU7cPz+3lPYH6Fr1Ipk9EzimmOGoKS1/2eMOkKjBlvZ/OLYpYEAg=,iv:rPZrIfkjGEV7bVl56Zb1cSic73RWC7SFMFrp2caGQd4=,tag:A+wl8o9eVia495krt1XnIg==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

eksctl/ssh-keys/shared-hubs-cluster.key.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC2GP+rT4i3Hed1cWky3F6M1Xnp+hQ3w10RDiC1qUtFu2gGXliHhoxKDpuxF0OpTgK5M3ZwS52qVKajXGwVmdRohVas1njYdJbECxdGHBj7EKYPdNmsxhdjd2IVipMRIdwOJ6ckSKUe3ypRjLhuB1IuxooI5NNAKkBI73d46yObTkcwNhxxdI+Sj+jBtaGyAzzuvg1vJiIMsegCQzJREHmOElJQLpD384O27ytXdL17ldzcQ92ydM01omWhKDP8e/j0BGxjJ+r69jxrqqYHI8P/UxIjhJqnHH6JGSJeBCQfN06dalWLaVPOGCiKfouhWAmEqFfftvkXDfMIkrc6iCOX7I7Zm070iArn+rm4CPvCFMMjXhp9fVCmScsIv6OY0WBbuJMByY0Qun0zET1/cSNq7g2vFVvQJyTJpJlPFlFA5p792etASPHH8EAU9N5VEbjm5Y6XDAfSxljP+SMNZlVRov6hpV7PFolLxeiNJROJhykcKbUnZCD0GbQadb6OKRc= [email protected]

terraform/aws/projects/shared-hubs-cluster.tfvars

@@ -0,0 +1,28 @@
+region = "us-west-2"
+
+cluster_name = "shared-hubs-cluster"
+
+cluster_nodes_location = "us-west-2a"
+
+user_buckets = {
+    "scratch-staging": {
+        "delete_after" : 7
+    },
+    "scratch": {
+        "delete_after": 7
+    },
+}
+
+
+hub_cloud_permissions = {
+  "staging" : {
+    requestor_pays: true,
+    bucket_admin_access: ["scratch-staging"],
+    extra_iam_policy: ""
+  },
+  "prod" : {
+    requestor_pays: true,
+    bucket_admin_access: ["scratch"],
+    extra_iam_policy: ""
+  },
+}