FAST: GKE multitenant infrastructure

examples/cloud-operations/binauthz/main.tf

@@ -97,8 +97,7 @@ module "cluster" {
     master_ipv4_cidr_block  = var.master_cidr_block
     master_global_access    = false
   }
-  enable_binary_authorization = true
-  workload_identity           = true
+  workload_identity = true
 }
 
 module "cluster_nodepool" {

examples/gke-serverless/README.md

@@ -0,0 +1,12 @@
+# GKE and Serverless examples
+
+The examples in this folder show implement **end-to-end scenarios** for GKE or Serveless topologies that show how to automate common configurations or leverage specific products.
+
+They are meant to be used as minimal but complete starting points to create actual infrastructure, and as playgrounds to experiment with Google Cloud features.
+
+## Examples
+
+### Multitenant GKE fleet
+
+<a href="./multitenant-fleet/" title="GKE multitenant fleet"><img src="./multitenant-fleet/diagram.png" align="left" width="280px"></a> This [example](./multitenant-fleet/) allows simple centralized management of similar sets of GKE clusters and their nodepools in a single project, and optional fleet management via GKE Hub templated configurations.
+<br clear="left">

examples/gke-serverless/multitenant-fleet/gke-clusters.tf

@@ -0,0 +1,116 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  clusters = {
+    for name, config in var.clusters :
+    name => merge(config, {
+      overrides = coalesce(config.overrides, var.cluster_defaults)
+    })
+  }
+}
+
+module "gke-cluster" {
+  source                   = "../../../modules/gke-cluster"
+  for_each                 = local.clusters
+  name                     = each.key
+  project_id               = module.gke-project-0.project_id
+  description              = each.value.description
+  location                 = each.value.location
+  network                  = var.vpc_config.vpc_self_link
+  subnetwork               = each.value.net.subnet
+  secondary_range_pods     = each.value.net.pods
+  secondary_range_services = each.value.net.services
+  labels                   = each.value.labels
+  addons = {
+    cloudrun_config                       = each.value.overrides.cloudrun_config
+    dns_cache_config                      = true
+    http_load_balancing                   = true
+    gce_persistent_disk_csi_driver_config = true
+    horizontal_pod_autoscaling            = true
+    config_connector_config               = true
+    kalm_config                           = false
+    gcp_filestore_csi_driver_config       = each.value.overrides.gcp_filestore_csi_driver_config
+    gke_backup_agent_config               = false
+    # enable only if enable_dataplane_v2 is changed to false below
+    network_policy_config = false
+    istio_config = {
+      enabled = false
+      tls     = false
+    }
+  }
+  # change these here for all clusters if absolutely needed
+  authenticator_security_group = var.authenticator_security_group
+  enable_dataplane_v2          = true
+  enable_l4_ilb_subsetting     = false
+  enable_intranode_visibility  = true
+  enable_shielded_nodes        = true
+  workload_identity            = true
+  private_cluster_config = {
+    enable_private_nodes    = true
+    enable_private_endpoint = false
+    master_ipv4_cidr_block  = each.value.net.master_range
+    master_global_access    = true
+  }
+  dns_config = each.value.dns_domain == null ? null : {
+    cluster_dns        = "CLOUD_DNS"
+    cluster_dns_scope  = "VPC_SCOPE"
+    cluster_dns_domain = "${each.key}.${var.dns_domain}"
+  }
+  logging_config    = ["SYSTEM_COMPONENTS", "WORKLOADS"]
+  monitoring_config = ["SYSTEM_COMPONENTS", "WORKLOADS"]
+
+  peering_config = var.peering_config == null ? null : {
+    export_routes = var.peering_config.export_routes
+    import_routes = var.peering_config.import_routes
+    project_id    = var.vpc_config.host_project_id
+  }
+  resource_usage_export_config = {
+    enabled = true
+    dataset = module.gke-dataset-resource-usage.dataset_id
+  }
+  # TODO: the attributes below are "primed" from project-level defaults
+  #       in locals, merge defaults with cluster-level stuff
+  # TODO(jccb): change fabric module
+  database_encryption = (
+    each.value.overrides.database_encryption_key == null
+    ? {
+      enabled  = false
+      state    = null
+      key_name = null
+    }
+    : {
+      enabled  = true
+      state    = "ENCRYPTED"
+      key_name = each.value.overrides.database_encryption_key
+    }
+  )
+  default_max_pods_per_node = each.value.overrides.max_pods_per_node
+  master_authorized_ranges  = each.value.overrides.master_authorized_ranges
+  pod_security_policy       = each.value.overrides.pod_security_policy
+  release_channel           = each.value.overrides.release_channel
+  vertical_pod_autoscaling  = each.value.overrides.vertical_pod_autoscaling
+  # dynamic "cluster_autoscaling" {
+  #   for_each = each.value.cluster_autoscaling == null ? {} : { 1 = 1 }
+  #   content {
+  #     enabled    = true
+  #     cpu_min    = each.value.cluster_autoscaling.cpu_min
+  #     cpu_max    = each.value.cluster_autoscaling.cpu_max
+  #     memory_min = each.value.cluster_autoscaling.memory_min
+  #     memory_max = each.value.cluster_autoscaling.memory_max
+  #   }
+  # }
+}

examples/gke-serverless/multitenant-fleet/gke-hub.tf

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  fleet_enabled = (
+    var.fleet_features != null || var.fleet_workload_identity
+  )
+  fleet_mcs_enabled = (
+    try(var.fleet_features.multiclusterservicediscovery, false) == true
+  )
+}
+
+module "gke-hub" {
+  source     = "../../../modules/gke-hub"
+  count      = local.fleet_enabled ? 1 : 0
+  project_id = module.gke-project-0.project_id
+  clusters = {
+    for cluster_id in keys(var.clusters) :
+    cluster_id => module.gke-cluster[cluster_id].id
+  }
+  features                   = var.fleet_features
+  configmanagement_templates = var.fleet_configmanagement_templates
+  configmanagement_clusters  = var.fleet_configmanagement_clusters
+  workload_identity_clusters = (
+    var.fleet_workload_identity ? keys(var.clusters) : []
+  )
+
+  depends_on = [
+    module.gke-nodepool
+  ]
+}

examples/gke-serverless/multitenant-fleet/gke-nodepools.tf

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  nodepools = merge([
+    for cluster, nodepools in var.nodepools : {
+      for nodepool, config in nodepools :
+      "${cluster}/${nodepool}" => merge(config, {
+        name      = nodepool
+        cluster   = cluster
+        overrides = coalesce(config.overrides, var.nodepool_defaults)
+      })
+    }
+  ]...)
+}
+
+module "gke-nodepool" {
+  source             = "../../../modules/gke-nodepool"
+  for_each           = local.nodepools
+  name               = each.value.name
+  project_id         = module.gke-project-0.project_id
+  cluster_name       = module.gke-cluster[each.value.cluster].name
+  location           = module.gke-cluster[each.value.cluster].location
+  initial_node_count = each.value.initial_node_count
+  node_machine_type  = each.value.node_type
+  node_spot          = each.value.spot
+
+  node_count = each.value.node_count
+  # node_count = (
+  #   each.value.autoscaling_config == null ? each.value.node_count : null
+  # )
+  # dynamic "autoscaling_config" {
+  #   for_each = each.value.autoscaling_config == null ? {} : { 1 = 1 }
+  #   content {
+  #     min_node_count = each.value.autoscaling_config.min_node_count
+  #     max_node_count = each.value.autoscaling_config.max_node_count
+  #   }
+  # }
+
+  # overrides
+  node_locations    = each.value.overrides.node_locations
+  max_pods_per_node = each.value.overrides.max_pods_per_node
+  node_image_type   = each.value.overrides.image_type
+  node_tags         = each.value.overrides.node_tags
+  node_taints       = each.value.overrides.node_taints
+
+  management_config = {
+    auto_repair  = true
+    auto_upgrade = true
+  }
+
+  node_service_account_create = true
+}

examples/gke-serverless/multitenant-fleet/main.tf

@@ -0,0 +1,86 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gke-project-0" {
+  source          = "../../../modules/project"
+  billing_account = var.billing_account_id
+  name            = var.project_id
+  parent          = var.folder_id
+  prefix          = var.prefix
+  group_iam       = var.group_iam
+  labels          = var.labels
+  iam = merge(var.iam, {
+    "roles/gkehub.serviceAgent" = [
+      "serviceAccount:${module.gke-project-0.service_accounts.robots.fleet}"
+    ] }
+  )
+  services = concat(
+    [
+      "anthos.googleapis.com",
+      "anthosconfigmanagement.googleapis.com",
+      "cloudresourcemanager.googleapis.com",
+      "container.googleapis.com",
+      "dns.googleapis.com",
+      "gkeconnect.googleapis.com",
+      "gkehub.googleapis.com",
+      "iam.googleapis.com",
+      "multiclusteringress.googleapis.com",
+      "multiclusterservicediscovery.googleapis.com",
+      "stackdriver.googleapis.com",
+      "trafficdirector.googleapis.com"
+    ],
+    var.project_services
+  )
+  service_config = {
+    disable_on_destroy         = false
+    disable_dependent_services = false
+  }
+  shared_vpc_service_config = {
+    attach       = true
+    host_project = var.vpc_config.host_project_id
+    service_identity_iam = merge({
+      "roles/compute.networkUser" = [
+        "cloudservices", "container-engine"
+      ]
+      "roles/container.hostServiceAgentUser" = [
+        "container-engine"
+      ]
+      },
+      !local.fleet_mcs_enabled ? {} : {
+        "roles/multiclusterservicediscovery.serviceAgent" = ["gke-mcs"]
+        "roles/compute.networkViewer"                     = ["gke-mcs-importer"]
+    })
+  }
+  # specify project-level org policies here if you need them
+  # policy_boolean = {
+  #   "constraints/compute.disableGuestAttributesAccess" = true
+  # }
+  # policy_list = {
+  #   "constraints/compute.trustedImageProjects" = {
+  #     inherit_from_parent = null
+  #     suggested_value     = null
+  #     status              = true
+  #     values              = ["projects/fl01-prod-iac-core-0"]
+  #   }
+  # }
+}
+
+module "gke-dataset-resource-usage" {
+  source        = "../../../modules/bigquery-dataset"
+  project_id    = module.gke-project-0.project_id
+  id            = "gke_resource_usage"
+  friendly_name = "GKE resource usage."
+}

examples/gke-serverless/multitenant-fleet/outputs.tf

@@ -0,0 +1,32 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tfdoc:file:description Output variables.
+
+output "clusters" {
+  description = "Cluster resources."
+  value       = module.gke-cluster
+}
+
+output "cluster_ids" {
+  description = "Cluster ids."
+  value = {
+    for k, v in module.gke-cluster : k => v.id
+  }
+}
+
+output "project_id" {
+  description = "GKE project id."
+  value       = module.gke-project-0.project_id
+}