From f3f9a4a88cedd64f6fc91b64666046aa6726a2f3 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 4 Feb 2022 17:26:43 +0100 Subject: [PATCH 01/75] GKE multitenant Co-authored-by: Daniel Marzini --- fast/stages/00-bootstrap/automation.tf | 1 + fast/stages/00-bootstrap/organization.tf | 8 + fast/stages/01-resman/README.md | 20 ++- fast/stages/01-resman/billing.tf | 1 + fast/stages/01-resman/branch-gke.tf | 108 ++++++++++++ fast/stages/01-resman/branch-networking.tf | 6 +- fast/stages/01-resman/organization.tf | 7 +- fast/stages/01-resman/outputs.tf | 45 ++++- fast/stages/02-networking-vpn/README.md | 17 +- .../data/subnets/prod/prod-gke-nodes-ew1.yaml | 8 + fast/stages/02-networking-vpn/spoke-dev.tf | 1 + fast/stages/02-networking-vpn/spoke-prod.tf | 1 + fast/stages/02-networking-vpn/variables.tf | 7 + fast/stages/03-gke-multitenant/prod/README.md | 105 ++++++++++++ .../03-gke-multitenant/prod/gke-clusters.tf | 120 +++++++++++++ .../03-gke-multitenant/prod/gke-nodepools.tf | 67 ++++++++ fast/stages/03-gke-multitenant/prod/main.tf | 122 +++++++++++++ .../stages/03-gke-multitenant/prod/outputs.tf | 15 ++ .../03-gke-multitenant/prod/variables.tf | 162 ++++++++++++++++++ 19 files changed, 794 insertions(+), 27 deletions(-) create mode 100644 fast/stages/01-resman/branch-gke.tf create mode 100644 fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml create mode 100644 fast/stages/03-gke-multitenant/prod/README.md create mode 100644 fast/stages/03-gke-multitenant/prod/gke-clusters.tf create mode 100644 fast/stages/03-gke-multitenant/prod/gke-nodepools.tf create mode 100644 fast/stages/03-gke-multitenant/prod/main.tf create mode 100644 fast/stages/03-gke-multitenant/prod/outputs.tf create mode 100644 fast/stages/03-gke-multitenant/prod/variables.tf diff --git a/fast/stages/00-bootstrap/automation.tf b/fast/stages/00-bootstrap/automation.tf index 0874fc4ff6..1cd5f15f2c 100644 --- a/fast/stages/00-bootstrap/automation.tf +++ b/fast/stages/00-bootstrap/automation.tf @@ -60,6 +60,7 @@ module "automation-project" { "cloudresourcemanager.googleapis.com", "container.googleapis.com", "compute.googleapis.com", + "container.googleapis.com", "essentialcontacts.googleapis.com", "iam.googleapis.com", "iamcredentials.googleapis.com", diff --git a/fast/stages/00-bootstrap/organization.tf b/fast/stages/00-bootstrap/organization.tf index 7fb4f07e56..51299e45d9 100644 --- a/fast/stages/00-bootstrap/organization.tf +++ b/fast/stages/00-bootstrap/organization.tf @@ -172,6 +172,14 @@ module "organization" { "compute.subnetworks.setIamPolicy", "dns.networks.bindPrivateDNSZone", "resourcemanager.projects.get", + + # if you prefer not granting permissions to create peerings to + # service accounts deploying service projects, remove these + # permissions and ask you network administrator to create any + # needed peerings (e.g. if you need to update routes for a GKE + # cluster) + "compute.networks.updatePeering", + "compute.networks.get", ] } logging_sinks = { diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index 7151f45e79..cba8060965 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -159,6 +159,7 @@ Due to its simplicity, this stage lends itself easily to customizations: adding |---|---|---|---| | [billing.tf](./billing.tf) | Billing resources for external billing use cases. | organization | google_billing_account_iam_member | | [branch-data-platform.tf](./branch-data-platform.tf) | Data Platform stages resources. | folder · gcs · iam-service-account | | +| [branch-gke.tf](./branch-gke.tf) | GKE multitenant stage resources. | folder · gcs · iam-service-account | | | [branch-networking.tf](./branch-networking.tf) | Networking stage resources. | folder · gcs · iam-service-account | | | [branch-sandbox.tf](./branch-sandbox.tf) | Sandbox stage resources. | folder · gcs · iam-service-account | | | [branch-security.tf](./branch-security.tf) | Security stage resources. | folder · gcs · iam-service-account | | @@ -194,14 +195,15 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L143) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L155) | Data for the Data Platform stage. | | | -| [networking](outputs.tf#L171) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L180) | Data for the project factories stage. | | | -| [providers](outputs.tf#L196) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L203) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L213) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L223) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L236) | Terraform variable files for the following stages. | ✓ | | +| [cicd_repositories](outputs.tf#L157) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L169) | Data for the Data Platform stage. | | | +| [gke_multitenant](outputs.tf#L237) | Data for the GKE multitenant stage. | | 03-gke-multitenant | +| [networking](outputs.tf#L185) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L194) | Data for the project factories stage. | | | +| [providers](outputs.tf#L210) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L217) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L227) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L254) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L267) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/01-resman/billing.tf b/fast/stages/01-resman/billing.tf index 3e2020e798..b3223b7540 100644 --- a/fast/stages/01-resman/billing.tf +++ b/fast/stages/01-resman/billing.tf @@ -29,6 +29,7 @@ locals { # for k, v in module.branch-teams-team-sa : v.iam_email # ], local.branch_teams_pf_sa_iam_emails, + local.branch_gke_multitenant_sa_iam_emails ) } diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf new file mode 100644 index 0000000000..0f3065f949 --- /dev/null +++ b/fast/stages/01-resman/branch-gke.tf @@ -0,0 +1,108 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description GKE multitenant stage resources. + +# top-level gke folder + +module "branch-gke-folder" { + source = "../../../modules/folder" + parent = "organizations/${var.organization.id}" + name = "GKE" + # iam = { + # "roles/logging.admin" = [module.branch-gke-sa.iam_email] + # "roles/owner" = [module.branch-gke-sa.iam_email] + # "roles/resourcemanager.folderAdmin" = [module.branch-gke-sa.iam_email] + # "roles/resourcemanager.projectCreator" = [module.branch-gke-sa.iam_email] + # } +} + +# GKE-level folders, service accounts and buckets for each individual environment + +module "branch-gke-multitenant-prod-folder" { + source = "../../../modules/folder" + parent = module.branch-gke-folder.id + name = "prod" + iam = { + "roles/owner" = [ + module.branch-gke-multitenant-prod-sa.iam_email + ] + "roles/resourcemanager.projectCreator" = [ + module.branch-gke-multitenant-prod-sa.iam_email + ] + } +} + +module "branch-gke-multitenant-prod-sa" { + source = "../../../modules/iam-service-account" + project_id = var.automation_project_id + name = "gke-prod-0" + description = "Terraform gke multitenant prod service account." + prefix = var.prefix + iam = { + # FIXME(jccb): who should we use here? + "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] + } +} + +module "branch-gke-multitenant-prod-gcs" { + source = "../../../modules/gcs" + project_id = var.automation_project_id + name = "gke-prod-0" + prefix = var.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.branch-gke-multitenant-prod-sa.iam_email] + } +} + + +module "branch-gke-multitenant-dev-folder" { + source = "../../../modules/folder" + parent = module.branch-gke-folder.id + name = "dev" + iam = { + "roles/owner" = [ + module.branch-gke-multitenant-dev-sa.iam_email + ] + "roles/resourcemanager.projectCreator" = [ + module.branch-gke-multitenant-dev-sa.iam_email + ] + } +} + +module "branch-gke-multitenant-dev-sa" { + source = "../../../modules/iam-service-account" + project_id = var.automation_project_id + name = "gke-dev-0" + description = "Terraform gke multitenant dev service account." + prefix = var.prefix + iam = { + # FIXME(jccb): who should we use here? + "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] + } +} + +module "branch-gke-multitenant-dev-gcs" { + source = "../../../modules/gcs" + project_id = var.automation_project_id + name = "gke-dev-0" + prefix = var.prefix + versioning = true + iam = { + "roles/storage.objectAdmin" = [module.branch-gke-multitenant-dev-sa.iam_email] + } +} diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf index 8757e7f894..e8e6fbc645 100644 --- a/fast/stages/01-resman/branch-networking.tf +++ b/fast/stages/01-resman/branch-networking.tf @@ -52,7 +52,8 @@ module "branch-network-prod-folder" { iam = { "roles/compute.xpnAdmin" = [ module.branch-dp-prod-sa.iam_email, - module.branch-teams-prod-pf-sa.iam_email + module.branch-teams-prod-pf-sa.iam_email, + module.branch-gke-multitenant-prod-sa.iam_email, ] } tag_bindings = { @@ -69,7 +70,8 @@ module "branch-network-dev-folder" { iam = { (local.custom_roles.service_project_network_admin) = [ module.branch-dp-dev-sa.iam_email, - module.branch-teams-dev-pf-sa.iam_email + module.branch-teams-dev-pf-sa.iam_email, + module.branch-gke-multitenant-dev-sa.iam_email, ] } tag_bindings = { diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/01-resman/organization.tf index 6ce4e9ced3..b5f636e677 100644 --- a/fast/stages/01-resman/organization.tf +++ b/fast/stages/01-resman/organization.tf @@ -28,6 +28,10 @@ locals { module.branch-teams-dev-pf-sa.iam_email, module.branch-teams-prod-pf-sa.iam_email ] + branch_gke_multitenant_sa_iam_emails = [ + module.branch-gke-multitenant-dev-sa.iam_email, + module.branch-gke-multitenant-prod-sa.iam_email + ] list_allow = { inherit_from_parent = false suggested_value = null @@ -76,7 +80,8 @@ module "organization" { # [ # for k, v in module.branch-teams-team-sa : v.iam_email # ], - local.branch_teams_pf_sa_iam_emails + local.branch_teams_pf_sa_iam_emails, + local.branch_gke_multitenant_sa_iam_emails ) } : {} ) diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index f91a843d81..65190cbace 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -60,13 +60,15 @@ locals { } folder_ids = merge( { - data-platform = module.branch-dp-dev-folder.id - networking = module.branch-network-folder.id - networking-dev = module.branch-network-dev-folder.id - networking-prod = module.branch-network-prod-folder.id - sandbox = module.branch-sandbox-folder.id - security = module.branch-security-folder.id - teams = module.branch-teams-folder.id + data-platform = module.branch-dp-dev-folder.id + gke-multitenant-dev = module.branch-gke-multitenant-dev-folder.id + gke-multitenant-prod = module.branch-gke-multitenant-prod-folder.id + networking = module.branch-network-folder.id + networking-dev = module.branch-network-dev-folder.id + networking-prod = module.branch-network-prod-folder.id + sandbox = module.branch-sandbox-folder.id + security = module.branch-security-folder.id + teams = module.branch-teams-folder.id }, { for k, v in module.branch-teams-team-folder : @@ -102,6 +104,16 @@ locals { name = "dp-prod" sa = module.branch-dp-prod-sa.email }) + "03-gke-multitenant-dev" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-multitenant-dev-gcs.name + name = "gke-multitenant-dev" + sa = module.branch-gke-multitenant-dev-sa.email + }) + "03-gke-multitenant-prod" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-multitenant-prod-gcs.name + name = "gke-multitenant-prod" + sa = module.branch-gke-multitenant-prod-sa.email + }) "03-project-factory-dev" = templatefile(local._tpl_providers, { bucket = module.branch-teams-dev-pf-gcs.name name = "team-dev" @@ -122,6 +134,8 @@ locals { { data-platform-dev = module.branch-dp-dev-sa.email data-platform-prod = module.branch-dp-prod-sa.email + gke-multitenant-dev = module.branch-gke-multitenant-dev-sa.iam_email + gke-multitenant-prod = module.branch-gke-multitenant-prod-sa.iam_email networking = module.branch-network-sa.email project-factory-dev = module.branch-teams-dev-pf-sa.email project-factory-prod = module.branch-teams-prod-pf-sa.email @@ -220,6 +234,23 @@ output "security" { } } +output "gke_multitenant" { + # tfdoc:output:consumers 03-gke-multitenant + description = "Data for the GKE multitenant stage." + value = { + "dev" = { + folder = module.branch-gke-multitenant-dev-folder.id + gcs_bucket = module.branch-gke-multitenant-dev-gcs.name + service_account = module.branch-gke-multitenant-dev-sa.email + } + "prod" = { + folder = module.branch-gke-multitenant-prod-folder.id + gcs_bucket = module.branch-gke-multitenant-prod-gcs.name + service_account = module.branch-gke-multitenant-prod-sa.email + } + } +} + output "teams" { description = "Data for the teams stage." value = { diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index 4924ca6b6f..16191edb88 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -315,19 +315,20 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | | [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [organization](variables.tf#L102) | Organization details. | object({…}) | ✓ | | 00-bootstrap | -| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | +| [organization](variables.tf#L109) | Organization details. | object({…}) | ✓ | | 00-bootstrap | +| [prefix](variables.tf#L125) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | | [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | | [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | | [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | | [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | | -| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | -| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | -| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | -| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | +| [gke_multitenant_sa](variables.tf#L84) | IAM emails for GKE multitenant service accounts. | map(string) | | {} | 01-resman | +| [l7ilb_subnets](variables.tf#L91) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | +| [outputs_location](variables.tf#L119) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [psa_ranges](variables.tf#L136) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | +| [router_onprem_configs](variables.tf#L173) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | | [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L196) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L191) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L203) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | | [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml b/fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml new file mode 100644 index 0000000000..c2b5cbe712 --- /dev/null +++ b/fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml @@ -0,0 +1,8 @@ +# skip boilerplate check + +region: europe-west1 +description: Default subnet for prod gke nodes +ip_cidr_range: 10.64.0.0/24 +secondary_ip_range: + pods: 100.64.0.0/16 + services: 192.168.1.0/24 diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index a6713eaf77..525c83c7e1 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -103,6 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, + local.service_accounts.gke_multitenant_sa, ] condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/02-networking-vpn/spoke-prod.tf index 401a08563f..084f5a2164 100644 --- a/fast/stages/02-networking-vpn/spoke-prod.tf +++ b/fast/stages/02-networking-vpn/spoke-prod.tf @@ -103,6 +103,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, + local.service_accounts.gke-multitenant-prod, ] condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/variables.tf b/fast/stages/02-networking-vpn/variables.tf index 60bd8be1d3..5557b485bb 100644 --- a/fast/stages/02-networking-vpn/variables.tf +++ b/fast/stages/02-networking-vpn/variables.tf @@ -81,6 +81,13 @@ variable "folder_ids" { }) } +variable "gke_multitenant_sa" { + # tfdoc:variable:source 01-resman + description = "IAM emails for GKE multitenant service accounts." + type = map(string) + default = {} +} + variable "l7ilb_subnets" { description = "Subnets used for L7 ILBs." type = map(list(object({ diff --git a/fast/stages/03-gke-multitenant/prod/README.md b/fast/stages/03-gke-multitenant/prod/README.md new file mode 100644 index 0000000000..be3be68628 --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/README.md @@ -0,0 +1,105 @@ +## IAM + +- project+infra cluster admin + +- cluster admins + they have IAM bindings at proj level + +- namespace admins / users + cluster user? only IAM role to fetch GKE creds +everything else is in RBAC (second part of this stage) + +- permissions to node service accounts + - logging/monitoring + - image registry + +- custom role to only allow autoprovisioning of credentials + +## Inputs from previous stages + +- vpc host and subnets + +## Network User role + +- (optional) resman stage 01 creates SA, net stage 02 sets role on net project, dev --> dev, prod --> prod +- (always possible) subnet factory assigns role + +## Resources + +### Robot service accounts + +- configured after prject activation (svpc roles, etc.) + - gke host service agent + - option for security admin + - CMEK for disks + +### Service accounts + +- node service accounts (per nodepool?) +- config sync service account (start with a single one, +wl identity) +- backup for GKE + +### Registry + +- per env? +- deep dive on partitioning per team + +### Clusters and nodepools + +- n clusters +- no default nodepool (or default nodepool has taints -- infra nodepool) +- what can change between clusters? + - stays the same + - monitoring and logging config + - autoprovisioning + - private / public + - workload identity + - cloud dns + - can change between clusters + - dataplane v2 + - nodepools + - binary auth +- nodepools per cluster + +## Addresses for ILBs + +## Filestore + +- later + +## RBAC + + +org +- GKE +- prod +- dev +- core + +branch: gke-stage3 + +resman +GKE folder + env folders + gcs + sa (extra file to drop in stage 1) + +[TF] supporting infra - stage 3 gke / infra (apply with SA from stage 1) +**single environment** (e.g. prod) +IAM for admins and users +projects for clusters +VPC wiring +logging +monitoring +container registries +source repos +workload identity config +Secrets (?) +clusters and nodepools and service accounts +managed prometheus? + +[Manifests/etc.] multitenant cluster config - stage 3 gke / cluster config (apply/runs with credentials from infra part) +config sync +gatekeeper +team namespaces +network policies +in-cluster logging and monitoring + + diff --git a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf new file mode 100644 index 0000000000..afdf08180b --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf @@ -0,0 +1,120 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + clusters = { + for name, config in var.clusters : + name => merge(config, { + overrides = coalesce(config.overrides, var.cluster_defaults) + }) + } +} + +module "gke-cluster" { + source = "../../../../modules/gke-cluster" + for_each = local.clusters + project_id = module.gke-project-0.project_id + name = each.key + description = each.value.description + location = each.value.location + network = each.value.net.vpc + subnetwork = each.value.net.subnet + secondary_range_pods = each.value.net.pods + secondary_range_services = each.value.net.services + labels = each.value.labels + addons = { + cloudrun_config = each.value.overrides.cloudrun_config + dns_cache_config = true + http_load_balancing = true + gce_persistent_disk_csi_driver_config = true + horizontal_pod_autoscaling = true + config_connector_config = true + kalm_config = false + # enable only if enable_dataplane_v2 is changed to false below + network_policy_config = false + istio_config = { + enabled = false + tls = false + } + } + # change these here for all clusters if absolutely needed + # authenticator_security_group = var.authenticator_security_group + enable_dataplane_v2 = true + enable_l4_ilb_subsetting = false + enable_intranode_visibility = true + enable_shielded_nodes = true + workload_identity = true + private_cluster_config = { + enable_private_nodes = true + enable_private_endpoint = true + master_ipv4_cidr_block = each.value.net.master_range + master_global_access = true + } + dns_config = each.value.dns_domain == null ? null : { + cluster_dns = "CLOUD_DNS" + cluster_dns_scope = "VPC_SCOPE" + cluster_dns_domain = "${each.key}.${var.dns_domain}" + } + logging_config = ["SYSTEM_COMPONENTS", "WORKLOADS"] + monitoring_config = ["SYSTEM_COMPONENTS", "WORKLOADS"] + + # if you don't have compute.networks.updatePeering in the host + # project, comment out the next line and ask your network admin to + # create the peering for you + peering_config = { + export_routes = true + import_routes = false + project_id = var.vpc_host_project + } + # resource_usage_export_config = { + # enabled = true + # dataset = module.gke-dataset-resource-usage.id + # } + # TODO: the attributes below are "primed" from project-level defaults + # in locals, merge defaults with cluster-level stuff + # TODO(jccb): change fabric module + database_encryption = ( + each.value.overrides.database_encryption_key == null ? { + enabled = false + state = null + key_name = null + } : { + enabled = true + state = "ENCRYPTED" + key_name = each.value.overrides.database_encryption_key + } + ) + default_max_pods_per_node = each.value.overrides.max_pods_per_node + enable_binary_authorization = each.value.overrides.enable_binary_authorization + master_authorized_ranges = each.value.overrides.master_authorized_ranges + pod_security_policy = each.value.overrides.pod_security_policy + release_channel = each.value.overrides.release_channel + vertical_pod_autoscaling = each.value.overrides.vertical_pod_autoscaling + # dynamic "cluster_autoscaling" { + # for_each = each.value.cluster_autoscaling == null ? {} : { 1 = 1 } + # content { + # enabled = true + # cpu_min = each.value.cluster_autoscaling.cpu_min + # cpu_max = each.value.cluster_autoscaling.cpu_max + # memory_min = each.value.cluster_autoscaling.memory_min + # memory_max = each.value.cluster_autoscaling.memory_max + # } + # } + + depends_on = [ + google_project_iam_member.host_project_bindings + ] +} diff --git a/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf b/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf new file mode 100644 index 0000000000..778a42d1d0 --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + nodepools = merge([ + for cluster, nodepools in var.nodepools : { + for nodepool, config in nodepools : + "${cluster}/${nodepool}" => merge(config, { + name = nodepool + cluster = cluster + overrides = coalesce(config.overrides, var.nodepool_defaults) + }) + } + ]...) +} + +module "gke_1_nodepool" { + source = "../../../../modules/gke-nodepool" + for_each = local.nodepools + name = each.value.name + project_id = module.gke-project-0.project_id + cluster_name = module.gke-cluster[each.value.cluster].name + location = module.gke-cluster[each.value.cluster].location + initial_node_count = each.value.node_count + node_machine_type = each.value.node_type + # TODO(jccb): can we use spot instances here? + node_preemptible = each.value.preemptible + + node_count = each.value.node_count + # node_count = ( + # each.value.autoscaling_config == null ? each.value.node_count : null + # ) + # dynamic "autoscaling_config" { + # for_each = each.value.autoscaling_config == null ? {} : { 1 = 1 } + # content { + # min_node_count = each.value.autoscaling_config.min_node_count + # max_node_count = each.value.autoscaling_config.max_node_count + # } + # } + + # overrides + node_locations = each.value.overrides.node_locations + max_pods_per_node = each.value.overrides.max_pods_per_node + node_image_type = each.value.overrides.image_type + node_tags = each.value.overrides.node_tags + node_taints = each.value.overrides.node_taints + + management_config = { + auto_repair = true + auto_upgrade = true + } + + node_service_account_create = true +} diff --git a/fast/stages/03-gke-multitenant/prod/main.tf b/fast/stages/03-gke-multitenant/prod/main.tf new file mode 100644 index 0000000000..5e709d68a7 --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/main.tf @@ -0,0 +1,122 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + labels = merge(var.labels, { environment = var.environment }) + + _gke_robot_sa = "serviceAccount:${module.gke-project-0.service_accounts.robots.container-engine}" + _cloud_services_sa = "serviceAccount:${module.gke-project-0.service_accounts.cloud_services}" + host_project_bindings = [ + { role = "roles/container.hostServiceAgentUser", member = local._gke_robot_sa }, + { role = "roles/compute.networkUser", member = local._gke_robot_sa }, + { role = "roles/compute.networkUser", member = local._cloud_services_sa } + ] +} + +module "gke-project-0" { + source = "../../../../modules/project" + billing_account = var.billing_account_id + name = "${var.environment}-gke-clusters-0" + parent = var.folder_id + prefix = var.prefix + labels = local.labels + services = [ + "container.googleapis.com", + "dns.googleapis.com", + "stackdriver.googleapis.com", + # uncomment if you need Multi-cluster Ingress / Gateway API + # "gkehub.googleapis.com", + # "multiclusterservicediscovery.googleapis.com", + # "multiclusteringress.googleapis.com", + # "trafficdirector.googleapis.com" + ] + # add here any other service ids and keys for robot accounts which are needed + # service_encryption_key_ids = { + # container = var.project_config.service_encryption_key_ids + # } + shared_vpc_service_config = { + attach = true + host_project = var.vpc_host_project + } + # specify project-level org policies here if you need them + + # policy_boolean = { + # "constraints/compute.disableGuestAttributesAccess" = true + # } + # policy_list = { + # "constraints/compute.trustedImageProjects" = { + # inherit_from_parent = null + # suggested_value = null + # status = true + # values = ["projects/fl01-prod-iac-core-0"] + # } + # } +} + +module "gke-project-1" { + source = "../../../../modules/project" + billing_account = var.billing_account_id + name = "${var.environment}-gke-clusters-2j" + parent = var.folder_id + prefix = var.prefix + labels = local.labels + services = [ + "container.googleapis.com", + "dns.googleapis.com", + "stackdriver.googleapis.com", + # uncomment if you need Multi-cluster Ingress / Gateway API + # "gkehub.googleapis.com", + # "multiclusterservicediscovery.googleapis.com", + # "multiclusteringress.googleapis.com", + # "trafficdirector.googleapis.com" + ] + # add here any other service ids and keys for robot accounts which are needed + # service_encryption_key_ids = { + # container = var.project_config.service_encryption_key_ids + # } + shared_vpc_service_config = { + attach = true + host_project = var.vpc_host_project + } + # specify project-level org policies here if you need them + + # policy_boolean = { + # "constraints/compute.disableGuestAttributesAccess" = true + # } + # policy_list = { + # "constraints/compute.trustedImageProjects" = { + # inherit_from_parent = null + # suggested_value = null + # status = true + # values = ["projects/fl01-prod-iac-core-0"] + # } + # } +} + + +module "gke-dataset-resource-usage" { + source = "../../../../modules/bigquery-dataset" + project_id = module.gke-project-0.project_id + id = "resource_usage" + friendly_name = "GKE resource usage." +} + +resource "google_project_iam_member" "host_project_bindings" { + for_each = { for i, v in local.host_project_bindings : i => v } + project = var.vpc_host_project + role = each.value.role + member = each.value.member +} diff --git a/fast/stages/03-gke-multitenant/prod/outputs.tf b/fast/stages/03-gke-multitenant/prod/outputs.tf new file mode 100644 index 0000000000..11a2ddf118 --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/fast/stages/03-gke-multitenant/prod/variables.tf b/fast/stages/03-gke-multitenant/prod/variables.tf new file mode 100644 index 0000000000..ce9203fc86 --- /dev/null +++ b/fast/stages/03-gke-multitenant/prod/variables.tf @@ -0,0 +1,162 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# we deal with one env here +# 1 project, m clusters +# cloud dns for gke? + +# variable "authenticator_security_group" { +# description = "Optional group used for Groups for GKE." +# type = string +# default = null +# } + +variable "billing_account_id" { + # tfdoc:variable:source 00-bootstrap + description = "Billing account id." + type = string +} + +variable "cluster_defaults" { + description = "Default values for optional cluster configurations." + type = object({ + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + }) + default = { + # TODO: review defaults + cloudrun_config = false + database_encryption_key = null + enable_binary_authorization = false + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + rfc1918_2 = "172.16.0.0/12" + rfc1918_3 = "192.168.0.0/16" + } + max_pods_per_node = 110 + pod_security_policy = false + release_channel = "STABLE" + vertical_pod_autoscaling = false + } +} + +variable "clusters" { + description = "" + type = map(object({ + cluster_autoscaling = object({ + cpu_min = number + cpu_max = number + memory_min = number + memory_max = number + }) + description = string + dns_domain = string + labels = map(string) + location = string + net = object({ + master_range = string + pods = string + services = string + subnet = string + vpc = string + }) + overrides = object({ + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + }) + })) +} + +variable "dns_domain" { + description = "Domain name used for clusters, prefix by each cluster name. Leave null to disable Cloud DNS for GKE." + type = string + default = null +} + +variable "environment" { + # tfdoc:variable:source 01-resman + description = "Environment abbreviation." + type = string +} + +variable "folder_id" { + # tfdoc:variable:source 01-resman + description = "Folder to be used for the networking resources in folders/nnnn format." + type = string +} + +variable "labels" { + description = "Project-level labels." + type = map(string) + default = {} +} + +variable "nodepool_defaults" { + description = "" + type = object({ + image_type = string + max_pods_per_node = number + node_locations = list(string) + node_tags = list(string) + node_taints = list(string) + }) + default = { + image_type = "COS_CONTAINERD" + max_pods_per_node = 110 + node_locations = null + node_tags = null + node_taints = [] + } +} + +variable "nodepools" { + description = "" + type = map(map(object({ + node_count = number + node_type = string + overrides = object({ + image_type = string + max_pods_per_node = number + node_locations = list(string) + node_tags = list(string) + node_taints = list(string) + }) + preemptible = bool + }))) +} + +variable "prefix" { + description = "Prefix used for resources that need unique names." + type = string +} + +variable "vpc_host_project" { + # tfdoc:variable:source 02-networking + description = "Host project for the shared VPC." + type = string +} From 801a5ed42d6e26699deaef13b194d5001636bcfe Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Sat, 5 Feb 2022 13:38:08 +0100 Subject: [PATCH 02/75] Add xpn admin to gke SAs on gke folders --- fast/stages/01-resman/branch-gke.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 0f3065f949..7adcb37376 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -43,6 +43,9 @@ module "branch-gke-multitenant-prod-folder" { "roles/resourcemanager.projectCreator" = [ module.branch-gke-multitenant-prod-sa.iam_email ] + "roles/compute.xpnAdmin" = [ + module.branch-gke-multitenant-prod-sa.iam_email + ] } } @@ -81,6 +84,9 @@ module "branch-gke-multitenant-dev-folder" { "roles/resourcemanager.projectCreator" = [ module.branch-gke-multitenant-dev-sa.iam_email ] + "roles/compute.xpnAdmin" = [ + module.branch-gke-multitenant-dev-sa.iam_email + ] } } From 740288ea9456787a038eb5f2d817228a008687e5 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Sat, 5 Feb 2022 16:59:07 +0100 Subject: [PATCH 03/75] Remove old test project --- fast/stages/03-gke-multitenant/prod/main.tf | 41 --------------------- 1 file changed, 41 deletions(-) diff --git a/fast/stages/03-gke-multitenant/prod/main.tf b/fast/stages/03-gke-multitenant/prod/main.tf index 5e709d68a7..479c4e3439 100644 --- a/fast/stages/03-gke-multitenant/prod/main.tf +++ b/fast/stages/03-gke-multitenant/prod/main.tf @@ -66,47 +66,6 @@ module "gke-project-0" { # } } -module "gke-project-1" { - source = "../../../../modules/project" - billing_account = var.billing_account_id - name = "${var.environment}-gke-clusters-2j" - parent = var.folder_id - prefix = var.prefix - labels = local.labels - services = [ - "container.googleapis.com", - "dns.googleapis.com", - "stackdriver.googleapis.com", - # uncomment if you need Multi-cluster Ingress / Gateway API - # "gkehub.googleapis.com", - # "multiclusterservicediscovery.googleapis.com", - # "multiclusteringress.googleapis.com", - # "trafficdirector.googleapis.com" - ] - # add here any other service ids and keys for robot accounts which are needed - # service_encryption_key_ids = { - # container = var.project_config.service_encryption_key_ids - # } - shared_vpc_service_config = { - attach = true - host_project = var.vpc_host_project - } - # specify project-level org policies here if you need them - - # policy_boolean = { - # "constraints/compute.disableGuestAttributesAccess" = true - # } - # policy_list = { - # "constraints/compute.trustedImageProjects" = { - # inherit_from_parent = null - # suggested_value = null - # status = true - # values = ["projects/fl01-prod-iac-core-0"] - # } - # } -} - - module "gke-dataset-resource-usage" { source = "../../../../modules/bigquery-dataset" project_id = module.gke-project-0.project_id From 5ff2286378f763d83b129e26d40a58ac564abcaa Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Sun, 6 Feb 2022 18:58:51 +0100 Subject: [PATCH 04/75] clusters creation improvement (#509) * shared_vpc_self_link variable from stage2 * removing shared_vpc_self_link * do not initial_node_count as node_count --- fast/stages/03-gke-multitenant/prod/gke-clusters.tf | 2 +- fast/stages/03-gke-multitenant/prod/gke-nodepools.tf | 2 +- fast/stages/03-gke-multitenant/prod/variables.tf | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf index afdf08180b..2adac67f81 100644 --- a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf @@ -26,8 +26,8 @@ locals { module "gke-cluster" { source = "../../../../modules/gke-cluster" for_each = local.clusters - project_id = module.gke-project-0.project_id name = each.key + project_id = each.value.project_id description = each.value.description location = each.value.location network = each.value.net.vpc diff --git a/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf b/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf index 778a42d1d0..649eeecb33 100644 --- a/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf +++ b/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf @@ -34,7 +34,7 @@ module "gke_1_nodepool" { project_id = module.gke-project-0.project_id cluster_name = module.gke-cluster[each.value.cluster].name location = module.gke-cluster[each.value.cluster].location - initial_node_count = each.value.node_count + initial_node_count = each.value.initial_node_count node_machine_type = each.value.node_type # TODO(jccb): can we use spot instances here? node_preemptible = each.value.preemptible diff --git a/fast/stages/03-gke-multitenant/prod/variables.tf b/fast/stages/03-gke-multitenant/prod/variables.tf index ce9203fc86..55e7276d9d 100644 --- a/fast/stages/03-gke-multitenant/prod/variables.tf +++ b/fast/stages/03-gke-multitenant/prod/variables.tf @@ -68,6 +68,7 @@ variable "clusters" { memory_min = number memory_max = number }) + project_id = string description = string dns_domain = string labels = map(string) @@ -139,6 +140,7 @@ variable "nodepools" { type = map(map(object({ node_count = number node_type = string + initial_node_count = number overrides = object({ image_type = string max_pods_per_node = number @@ -159,4 +161,4 @@ variable "vpc_host_project" { # tfdoc:variable:source 02-networking description = "Host project for the shared VPC." type = string -} +} \ No newline at end of file From 46af8fa72e9ee89cf5ded63690d81de2ae4d6ce2 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 9 Feb 2022 18:10:35 +0100 Subject: [PATCH 05/75] Use new project-level robot bindings --- .../03-gke-multitenant/prod/gke-clusters.tf | 5 +---- fast/stages/03-gke-multitenant/prod/main.tf | 22 ++++++++----------- .../03-gke-multitenant/prod/variables.tf | 7 +++--- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf index 2adac67f81..89ad47aca5 100644 --- a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/prod/gke-clusters.tf @@ -27,7 +27,7 @@ module "gke-cluster" { source = "../../../../modules/gke-cluster" for_each = local.clusters name = each.key - project_id = each.value.project_id + project_id = module.gke-project-0.project_id description = each.value.description location = each.value.location network = each.value.net.vpc @@ -114,7 +114,4 @@ module "gke-cluster" { # } # } - depends_on = [ - google_project_iam_member.host_project_bindings - ] } diff --git a/fast/stages/03-gke-multitenant/prod/main.tf b/fast/stages/03-gke-multitenant/prod/main.tf index 479c4e3439..1f7f3e1331 100644 --- a/fast/stages/03-gke-multitenant/prod/main.tf +++ b/fast/stages/03-gke-multitenant/prod/main.tf @@ -19,11 +19,6 @@ locals { _gke_robot_sa = "serviceAccount:${module.gke-project-0.service_accounts.robots.container-engine}" _cloud_services_sa = "serviceAccount:${module.gke-project-0.service_accounts.cloud_services}" - host_project_bindings = [ - { role = "roles/container.hostServiceAgentUser", member = local._gke_robot_sa }, - { role = "roles/compute.networkUser", member = local._gke_robot_sa }, - { role = "roles/compute.networkUser", member = local._cloud_services_sa } - ] } module "gke-project-0" { @@ -50,9 +45,17 @@ module "gke-project-0" { shared_vpc_service_config = { attach = true host_project = var.vpc_host_project + service_identity_iam = { + "roles/compute.networkUser" = [ + "cloudservices", "container-engine" + ] + "roles/container.hostServiceAgentUser" = [ + "container-engine" + ] + } } - # specify project-level org policies here if you need them + # specify project-level org policies here if you need them # policy_boolean = { # "constraints/compute.disableGuestAttributesAccess" = true # } @@ -72,10 +75,3 @@ module "gke-dataset-resource-usage" { id = "resource_usage" friendly_name = "GKE resource usage." } - -resource "google_project_iam_member" "host_project_bindings" { - for_each = { for i, v in local.host_project_bindings : i => v } - project = var.vpc_host_project - role = each.value.role - member = each.value.member -} diff --git a/fast/stages/03-gke-multitenant/prod/variables.tf b/fast/stages/03-gke-multitenant/prod/variables.tf index 55e7276d9d..425eaf8529 100644 --- a/fast/stages/03-gke-multitenant/prod/variables.tf +++ b/fast/stages/03-gke-multitenant/prod/variables.tf @@ -68,7 +68,6 @@ variable "clusters" { memory_min = number memory_max = number }) - project_id = string description = string dns_domain = string labels = map(string) @@ -138,8 +137,8 @@ variable "nodepool_defaults" { variable "nodepools" { description = "" type = map(map(object({ - node_count = number - node_type = string + node_count = number + node_type = string initial_node_count = number overrides = object({ image_type = string @@ -161,4 +160,4 @@ variable "vpc_host_project" { # tfdoc:variable:source 02-networking description = "Host project for the shared VPC." type = string -} \ No newline at end of file +} From a4c0b22f4e20c29b85456e304e0b8ce670ff4ae1 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 15 Feb 2022 00:45:43 +0100 Subject: [PATCH 06/75] Move GKE example to dev --- fast/stages/01-resman/branch-gke.tf | 12 ++++++------ .../dev-gke-nodes-ew1.yaml} | 0 .../03-gke-multitenant/{prod => dev}/README.md | 0 .../03-gke-multitenant/{prod => dev}/gke-clusters.tf | 2 +- .../{prod => dev}/gke-nodepools.tf | 4 ++-- fast/stages/03-gke-multitenant/{prod => dev}/main.tf | 0 .../03-gke-multitenant/{prod => dev}/outputs.tf | 0 .../03-gke-multitenant/{prod => dev}/variables.tf | 8 +++++++- 8 files changed, 16 insertions(+), 10 deletions(-) rename fast/stages/02-networking-vpn/data/subnets/{prod/prod-gke-nodes-ew1.yaml => dev/dev-gke-nodes-ew1.yaml} (100%) rename fast/stages/03-gke-multitenant/{prod => dev}/README.md (100%) rename fast/stages/03-gke-multitenant/{prod => dev}/gke-clusters.tf (98%) rename fast/stages/03-gke-multitenant/{prod => dev}/gke-nodepools.tf (96%) rename fast/stages/03-gke-multitenant/{prod => dev}/main.tf (100%) rename fast/stages/03-gke-multitenant/{prod => dev}/outputs.tf (100%) rename fast/stages/03-gke-multitenant/{prod => dev}/variables.tf (96%) diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 7adcb37376..5914f5f503 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -35,7 +35,7 @@ module "branch-gke-folder" { module "branch-gke-multitenant-prod-folder" { source = "../../../modules/folder" parent = module.branch-gke-folder.id - name = "prod" + name = "Production" iam = { "roles/owner" = [ module.branch-gke-multitenant-prod-sa.iam_email @@ -52,7 +52,7 @@ module "branch-gke-multitenant-prod-folder" { module "branch-gke-multitenant-prod-sa" { source = "../../../modules/iam-service-account" project_id = var.automation_project_id - name = "gke-prod-0" + name = "prod-resman-gke-0" description = "Terraform gke multitenant prod service account." prefix = var.prefix iam = { @@ -64,7 +64,7 @@ module "branch-gke-multitenant-prod-sa" { module "branch-gke-multitenant-prod-gcs" { source = "../../../modules/gcs" project_id = var.automation_project_id - name = "gke-prod-0" + name = "prod-resman-gke-0" prefix = var.prefix versioning = true iam = { @@ -76,7 +76,7 @@ module "branch-gke-multitenant-prod-gcs" { module "branch-gke-multitenant-dev-folder" { source = "../../../modules/folder" parent = module.branch-gke-folder.id - name = "dev" + name = "Development" iam = { "roles/owner" = [ module.branch-gke-multitenant-dev-sa.iam_email @@ -93,7 +93,7 @@ module "branch-gke-multitenant-dev-folder" { module "branch-gke-multitenant-dev-sa" { source = "../../../modules/iam-service-account" project_id = var.automation_project_id - name = "gke-dev-0" + name = "dev-resman-gke-0" description = "Terraform gke multitenant dev service account." prefix = var.prefix iam = { @@ -105,7 +105,7 @@ module "branch-gke-multitenant-dev-sa" { module "branch-gke-multitenant-dev-gcs" { source = "../../../modules/gcs" project_id = var.automation_project_id - name = "gke-dev-0" + name = "dev-resman-gke-0" prefix = var.prefix versioning = true iam = { diff --git a/fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml b/fast/stages/02-networking-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml similarity index 100% rename from fast/stages/02-networking-vpn/data/subnets/prod/prod-gke-nodes-ew1.yaml rename to fast/stages/02-networking-vpn/data/subnets/dev/dev-gke-nodes-ew1.yaml diff --git a/fast/stages/03-gke-multitenant/prod/README.md b/fast/stages/03-gke-multitenant/dev/README.md similarity index 100% rename from fast/stages/03-gke-multitenant/prod/README.md rename to fast/stages/03-gke-multitenant/dev/README.md diff --git a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf similarity index 98% rename from fast/stages/03-gke-multitenant/prod/gke-clusters.tf rename to fast/stages/03-gke-multitenant/dev/gke-clusters.tf index 89ad47aca5..efce81ac4e 100644 --- a/fast/stages/03-gke-multitenant/prod/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -30,7 +30,7 @@ module "gke-cluster" { project_id = module.gke-project-0.project_id description = each.value.description location = each.value.location - network = each.value.net.vpc + network = var.shared_vpc_self_link subnetwork = each.value.net.subnet secondary_range_pods = each.value.net.pods secondary_range_services = each.value.net.services diff --git a/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf b/fast/stages/03-gke-multitenant/dev/gke-nodepools.tf similarity index 96% rename from fast/stages/03-gke-multitenant/prod/gke-nodepools.tf rename to fast/stages/03-gke-multitenant/dev/gke-nodepools.tf index 649eeecb33..52febb9b20 100644 --- a/fast/stages/03-gke-multitenant/prod/gke-nodepools.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-nodepools.tf @@ -27,14 +27,14 @@ locals { ]...) } -module "gke_1_nodepool" { +module "gke-1-nodepool" { source = "../../../../modules/gke-nodepool" for_each = local.nodepools name = each.value.name project_id = module.gke-project-0.project_id cluster_name = module.gke-cluster[each.value.cluster].name location = module.gke-cluster[each.value.cluster].location - initial_node_count = each.value.initial_node_count + initial_node_count = each.value.initial_node_count node_machine_type = each.value.node_type # TODO(jccb): can we use spot instances here? node_preemptible = each.value.preemptible diff --git a/fast/stages/03-gke-multitenant/prod/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf similarity index 100% rename from fast/stages/03-gke-multitenant/prod/main.tf rename to fast/stages/03-gke-multitenant/dev/main.tf diff --git a/fast/stages/03-gke-multitenant/prod/outputs.tf b/fast/stages/03-gke-multitenant/dev/outputs.tf similarity index 100% rename from fast/stages/03-gke-multitenant/prod/outputs.tf rename to fast/stages/03-gke-multitenant/dev/outputs.tf diff --git a/fast/stages/03-gke-multitenant/prod/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf similarity index 96% rename from fast/stages/03-gke-multitenant/prod/variables.tf rename to fast/stages/03-gke-multitenant/dev/variables.tf index 425eaf8529..06769b7871 100644 --- a/fast/stages/03-gke-multitenant/prod/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -77,7 +77,6 @@ variable "clusters" { pods = string services = string subnet = string - vpc = string }) overrides = object({ cloudrun_config = bool @@ -156,6 +155,13 @@ variable "prefix" { type = string } +variable "shared_vpc_self_link" { + # tfdoc:variable:source 02-networking + description = "Self link for the shared VPC." + type = string + default = null +} + variable "vpc_host_project" { # tfdoc:variable:source 02-networking description = "Host project for the shared VPC." From 9fabfafc63232a6fbc9449f758f96fdb66d6e726 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 16 Feb 2022 11:55:49 +0100 Subject: [PATCH 07/75] Update gke stage to use contract setup --- fast/stages/01-resman/outputs.tf | 4 +- fast/stages/02-networking-vpn/README.md | 17 ++++---- fast/stages/02-networking-vpn/spoke-dev.tf | 2 +- fast/stages/02-networking-vpn/variables.tf | 9 +---- .../03-gke-multitenant/dev/gke-clusters.tf | 4 +- fast/stages/03-gke-multitenant/dev/main.tf | 10 ++--- .../03-gke-multitenant/dev/variables.tf | 40 ++++++++++--------- 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index 65190cbace..30a74a7526 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -134,8 +134,8 @@ locals { { data-platform-dev = module.branch-dp-dev-sa.email data-platform-prod = module.branch-dp-prod-sa.email - gke-multitenant-dev = module.branch-gke-multitenant-dev-sa.iam_email - gke-multitenant-prod = module.branch-gke-multitenant-prod-sa.iam_email + gke-multitenant-dev = module.branch-gke-multitenant-dev-sa.email + gke-multitenant-prod = module.branch-gke-multitenant-prod-sa.email networking = module.branch-network-sa.email project-factory-dev = module.branch-teams-dev-pf-sa.email project-factory-prod = module.branch-teams-prod-pf-sa.email diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index 16191edb88..9b8a4d1100 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -315,20 +315,19 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [automation](variables.tf#L17) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L25) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | | [folder_ids](variables.tf#L74) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [organization](variables.tf#L109) | Organization details. | object({…}) | ✓ | | 00-bootstrap | -| [prefix](variables.tf#L125) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | +| [organization](variables.tf#L102) | Organization details. | object({…}) | ✓ | | 00-bootstrap | +| [prefix](variables.tf#L118) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | | [custom_adv](variables.tf#L34) | Custom advertisement definitions in name => range format. | map(string) | | {…} | | | [custom_roles](variables.tf#L51) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | | [data_dir](variables.tf#L60) | Relative path for the folder storing configuration data for network resources. | string | | "data" | | | [dns](variables.tf#L66) | Onprem DNS resolvers. | map(list(string)) | | {…} | | -| [gke_multitenant_sa](variables.tf#L84) | IAM emails for GKE multitenant service accounts. | map(string) | | {} | 01-resman | -| [l7ilb_subnets](variables.tf#L91) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | -| [outputs_location](variables.tf#L119) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | -| [psa_ranges](variables.tf#L136) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | -| [router_onprem_configs](variables.tf#L173) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | +| [l7ilb_subnets](variables.tf#L84) | Subnets used for L7 ILBs. | map(list(object({…}))) | | {…} | | +| [outputs_location](variables.tf#L112) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | +| [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | +| [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | | [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L191) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L203) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | | [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index 525c83c7e1..169ac6ba09 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -103,7 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, - local.service_accounts.gke_multitenant_sa, + local.service_accounts.gke-multitenant-dev, ] condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/variables.tf b/fast/stages/02-networking-vpn/variables.tf index 5557b485bb..77ef3884fc 100644 --- a/fast/stages/02-networking-vpn/variables.tf +++ b/fast/stages/02-networking-vpn/variables.tf @@ -81,13 +81,6 @@ variable "folder_ids" { }) } -variable "gke_multitenant_sa" { - # tfdoc:variable:source 01-resman - description = "IAM emails for GKE multitenant service accounts." - type = map(string) - default = {} -} - variable "l7ilb_subnets" { description = "Subnets used for L7 ILBs." type = map(list(object({ @@ -194,6 +187,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string + gke-multitenant-dev = string + gke-multitenant-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index efce81ac4e..2438cc1c8e 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -30,7 +30,7 @@ module "gke-cluster" { project_id = module.gke-project-0.project_id description = each.value.description location = each.value.location - network = var.shared_vpc_self_link + network = var.vpc_self_links.dev-spoke-0 subnetwork = each.value.net.subnet secondary_range_pods = each.value.net.pods secondary_range_services = each.value.net.services @@ -77,7 +77,7 @@ module "gke-cluster" { peering_config = { export_routes = true import_routes = false - project_id = var.vpc_host_project + project_id = var.host_project_ids.dev-spoke-0 } # resource_usage_export_config = { # enabled = true diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index 1f7f3e1331..b75cac694f 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -15,7 +15,7 @@ */ locals { - labels = merge(var.labels, { environment = var.environment }) + labels = merge(var.labels, { environment = "dev" }) _gke_robot_sa = "serviceAccount:${module.gke-project-0.service_accounts.robots.container-engine}" _cloud_services_sa = "serviceAccount:${module.gke-project-0.service_accounts.cloud_services}" @@ -23,9 +23,9 @@ locals { module "gke-project-0" { source = "../../../../modules/project" - billing_account = var.billing_account_id - name = "${var.environment}-gke-clusters-0" - parent = var.folder_id + billing_account = var.billing_account.id + name = "dev-gke-clusters-0" + parent = var.folder_ids.gke-multitenant-dev prefix = var.prefix labels = local.labels services = [ @@ -44,7 +44,7 @@ module "gke-project-0" { # } shared_vpc_service_config = { attach = true - host_project = var.vpc_host_project + host_project = var.host_project_ids.dev-spoke-0 service_identity_iam = { "roles/compute.networkUser" = [ "cloudservices", "container-engine" diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 06769b7871..2a165b8eea 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -24,10 +24,13 @@ # default = null # } -variable "billing_account_id" { +variable "billing_account" { # tfdoc:variable:source 00-bootstrap - description = "Billing account id." - type = string + description = "Billing account id and organization id ('nnnnnnnn' or null)." + type = object({ + id = string + organization_id = number + }) } variable "cluster_defaults" { @@ -97,16 +100,20 @@ variable "dns_domain" { default = null } -variable "environment" { +variable "folder_ids" { # tfdoc:variable:source 01-resman - description = "Environment abbreviation." - type = string + description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created." + type = object({ + gke-multitenant-dev = string + }) } -variable "folder_id" { - # tfdoc:variable:source 01-resman - description = "Folder to be used for the networking resources in folders/nnnn format." - type = string +variable "host_project_ids" { + # tfdoc:variable:source 02-networking + description = "Host project for the shared VPC." + type = object({ + dev-spoke-0 = string + }) } variable "labels" { @@ -155,15 +162,10 @@ variable "prefix" { type = string } -variable "shared_vpc_self_link" { +variable "vpc_self_links" { # tfdoc:variable:source 02-networking description = "Self link for the shared VPC." - type = string - default = null -} - -variable "vpc_host_project" { - # tfdoc:variable:source 02-networking - description = "Host project for the shared VPC." - type = string + type = object({ + dev-spoke-0 = string + }) } From 96b01cf68c9f87314731205e5b8d58478b3e9061 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Mon, 28 Feb 2022 23:03:34 +0100 Subject: [PATCH 08/75] Fix networking-vpn tests --- tests/fast/stages/s02_networking_vpn/fixture/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/fast/stages/s02_networking_vpn/fixture/main.tf b/tests/fast/stages/s02_networking_vpn/fixture/main.tf index 6d7b8840fc..8c1da847ef 100644 --- a/tests/fast/stages/s02_networking_vpn/fixture/main.tf +++ b/tests/fast/stages/s02_networking_vpn/fixture/main.tf @@ -35,6 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" + gke-multitenant-dev = "string" + gke-multitenant-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } From 66f87903a8920cc20f0706e00a3b678605bbef8d Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 8 Mar 2022 23:08:49 +0100 Subject: [PATCH 09/75] Enable GKE resource metering --- fast/stages/03-gke-multitenant/dev/gke-clusters.tf | 8 ++++---- fast/stages/03-gke-multitenant/dev/main.tf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index 2438cc1c8e..e74750c12d 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -79,10 +79,10 @@ module "gke-cluster" { import_routes = false project_id = var.host_project_ids.dev-spoke-0 } - # resource_usage_export_config = { - # enabled = true - # dataset = module.gke-dataset-resource-usage.id - # } + resource_usage_export_config = { + enabled = true + dataset = module.gke-dataset-resource-usage.dataset_id + } # TODO: the attributes below are "primed" from project-level defaults # in locals, merge defaults with cluster-level stuff # TODO(jccb): change fabric module diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index b75cac694f..852ce0b14d 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -72,6 +72,6 @@ module "gke-project-0" { module "gke-dataset-resource-usage" { source = "../../../../modules/bigquery-dataset" project_id = module.gke-project-0.project_id - id = "resource_usage" + id = "gke_resource_usage" friendly_name = "GKE resource usage." } From 9bcae7b1809d3da9ad33a72a05328e0016ef523b Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 8 Mar 2022 23:54:35 +0100 Subject: [PATCH 10/75] Enable WIF issuer in gke-hub --- modules/gke-hub/README.md | 2 +- modules/gke-hub/main.tf | 4 ++++ modules/gke-hub/variables.tf | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/gke-hub/README.md b/modules/gke-hub/README.md index 6b2010590a..0e3acb939b 100644 --- a/modules/gke-hub/README.md +++ b/modules/gke-hub/README.md @@ -99,7 +99,7 @@ module "hub" { |---|---|:---:|:---:|:---:| | [project_id](variables.tf#L75) | GKE hub project ID. | string | ✓ | | | [features](variables.tf#L17) | GKE hub features to enable. | object({…}) | | {…} | -| [member_clusters](variables.tf#L32) | List for member cluster self links. | map(string) | | {} | +| [member_clusters](variables.tf#L32) | List for member cluster ids. | map(string) | | {} | | [member_features](variables.tf#L39) | Member features for each cluster | object({…}) | | {…} | ## Outputs diff --git a/modules/gke-hub/main.tf b/modules/gke-hub/main.tf index 4da1a0ce6e..784fe88c9b 100644 --- a/modules/gke-hub/main.tf +++ b/modules/gke-hub/main.tf @@ -24,6 +24,10 @@ resource "google_gke_hub_membership" "membership" { resource_link = each.value } } + # TODO(jccb): allow cluster member without WIF + authority { + issuer = "https://container.googleapis.com/v1/${each.value}" + } } resource "google_gke_hub_feature" "configmanagement" { diff --git a/modules/gke-hub/variables.tf b/modules/gke-hub/variables.tf index d099bd762e..3183cfff43 100644 --- a/modules/gke-hub/variables.tf +++ b/modules/gke-hub/variables.tf @@ -30,7 +30,7 @@ variable "features" { } variable "member_clusters" { - description = "List for member cluster self links." + description = "List for member cluster ids." type = map(string) default = {} nullable = false From d28dfc40b0fa16113e7ddd1c4ede37013f52bd57 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 8 Mar 2022 23:55:01 +0100 Subject: [PATCH 11/75] Enable GKE hub apis --- fast/stages/03-gke-multitenant/dev/main.tf | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index 852ce0b14d..aef450b0da 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -29,13 +29,12 @@ module "gke-project-0" { prefix = var.prefix labels = local.labels services = [ - "container.googleapis.com", - "dns.googleapis.com", + "anthosconfigmanagement.googleapis.com", + "anthos.googleapis.com", "dns.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", "stackdriver.googleapis.com", - # uncomment if you need Multi-cluster Ingress / Gateway API - # "gkehub.googleapis.com", - # "multiclusterservicediscovery.googleapis.com", - # "multiclusteringress.googleapis.com", + "container.googleapis.com", # "trafficdirector.googleapis.com" ] # add here any other service ids and keys for robot accounts which are needed From 110d94067d3a428802e87afaa215f681a56d9d70 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Sun, 3 Apr 2022 22:32:52 +0200 Subject: [PATCH 12/75] Squashing relevant changes from fast-dev-gke-marzi Co-authored-by: Daniel Marzini --- .../config/cluster/README.md | 3 + .../config/cluster/clusterrole-ns-viewer.yml | 28 ++++ ...-have-geo-constraint_v1beta_gatekeeper.yml | 43 ++++++ .../config/cluster/pod_priority_classes.yml | 47 +++++++ .../config/namespaces/README.md | 7 + .../apps_v1_configmap_cos-auditd.yml | 54 ++++++++ .../apps_v1_daemonset_cos-auditd.yml | 128 ++++++++++++++++++ .../namespaces/cos-auditd/namespace.yml | 19 +++ .../config/namespaces/teams/limit-range.yml | 34 +++++ .../network-policy-default-deny-egress.yml | 22 +++ .../namespaces/teams/team-a/namespace.yml | 18 +++ .../namespaces/teams/team-a/repo-sync.yml | 30 ++++ .../teams/team-a/resourcequotas.yml | 35 +++++ .../teams/team-a/rolebinding-ns-viewer.yml | 28 ++++ .../teams/team-a/sync-rolebinding.yml | 28 ++++ .../teams/team-b/apps_v1_deployment_v1.yml | 44 ++++++ .../namespaces/teams/team-b/namespace.yml | 20 +++ .../03-gke-multitenant/config/system/repo.yml | 23 ++++ fast/stages/03-gke-multitenant/dev/README.md | 3 + .../03-gke-multitenant/dev/gke-clusters.tf | 15 +- fast/stages/03-gke-multitenant/dev/gke-hub.tf | 59 ++++++++ fast/stages/03-gke-multitenant/dev/main.tf | 7 +- .../03-gke-multitenant/dev/variables.tf | 59 ++++---- modules/gke-hub/main.tf | 8 +- modules/gke-nodepool/main.tf | 2 +- 25 files changed, 729 insertions(+), 35 deletions(-) create mode 100644 fast/stages/03-gke-multitenant/config/cluster/README.md create mode 100644 fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml create mode 100644 fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml create mode 100644 fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/README.md create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml create mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml create mode 100644 fast/stages/03-gke-multitenant/config/system/repo.yml create mode 100644 fast/stages/03-gke-multitenant/dev/gke-hub.tf diff --git a/fast/stages/03-gke-multitenant/config/cluster/README.md b/fast/stages/03-gke-multitenant/config/cluster/README.md new file mode 100644 index 0000000000..fe3a852813 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/README.md @@ -0,0 +1,3 @@ +The cluster/ directory contains configs that apply to entire clusters, rather than to namespaces. By default, any config in the cluster/ directory applies to every cluster enrolled in Config Sync. You can limit which clusters a config can affect by using a ClusterSelector. + +Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#cluster \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml new file mode 100644 index 0000000000..3bf4cf773b --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dev-namespace-viewer +rules: +- apiGroups: ["", "metrics.k8s.io", "extensions", "apps"] + resources: + - "pods" + - "pods/log" + - "events" + - "deployments" + - "replicasets" + verbs: ["get", "watch", "list"] \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml new file mode 100644 index 0000000000..a89b549f76 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredLabels +metadata: + name: ns-must-have-geo + annotations: + configsync.gke.io/cluster-name-selector: gke-1 +spec: + enforcementAction: dryrun + match: + excludedNamespaces: + - "kube-system" + - "kube-public" + - "kube-node-lease" + - "resource-group-system" + - "config-management-monitoring" + - "config-management-system" + - "cos-auditd" + - "gatekeeper-system" + - "kube-node-lease" + - "cnrm-system" + - "gke-mcs" + - "configconnector-operator-system" + kinds: + - apiGroups: [""] + kinds: ["Namespace"] + parameters: + labels: + - key: "geo" \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml new file mode 100644 index 0000000000..070e1e9cf3 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml @@ -0,0 +1,47 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority-system +value: 1000000 +globalDefault: false +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000 +globalDefault: false +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: mid-priority +value: 100 +globalDefault: false +# preemptionPolicy: Never +description: "This priority class will cause other pods to be preempted." +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: low-priority +value: 10 +globalDefault: true +# preemptionPolicy: Never +description: "This priority class will cause other pods to be preempted." \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/README.md b/fast/stages/03-gke-multitenant/config/namespaces/README.md new file mode 100644 index 0000000000..99382474b4 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/README.md @@ -0,0 +1,7 @@ + + +The namespaces/ directory contains configs for namespaces and namespace-scoped objects. + +The structure within namespaces/ is the mechanism that drives namespace inheritance. You can limit which namespaces can inherit a config, by using a NamespaceSelector. + +Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#namespaces \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml new file mode 100644 index 0000000000..0c3a25c628 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml @@ -0,0 +1,54 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +kind: ConfigMap +apiVersion: v1 +metadata: + name: fluentd-gcp-config-cos-auditd + namespace: cos-auditd + annotations: + kubernetes.io/description: 'ConfigMap for Linux auditd logging daemonset on COS nodes.' +data: + google-fluentd.conf: |- + + @type systemd + filters [{ "SYSLOG_IDENTIFIER": "audit" }] + pos_file /var/log/gcp-journald-audit.pos + read_from_head true + tag linux-auditd + + + # Do not collect fluentd's own logs to avoid infinite loops. + + @type null + + + + @type google_cloud + + enable_monitoring false + split_logs_by_tag false + detect_subservice false + buffer_type file + buffer_path /var/log/fluentd-buffers/system.audit.buffer + buffer_queue_full_action block + buffer_chunk_limit 512k + buffer_queue_limit 2 + flush_interval 5s + max_retry_wait 30 + disable_retry_limit + num_threads 2 + use_grpc true + \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml new file mode 100644 index 0000000000..d9980318ce --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml @@ -0,0 +1,128 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cos-auditd-logging + namespace: cos-auditd + annotations: + kubernetes.io/description: 'DaemonSet that enables Linux auditd logging on COS nodes.' +spec: + selector: + matchLabels: + name: cos-auditd-logging + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: cos-auditd-logging + spec: + hostNetwork: true + hostPID: true + nodeSelector: + cloud.google.com/gke-os-distribution: cos + volumes: + - hostPath: + path: / + name: host + - hostPath: + path: /var/log + name: varlog + - hostPath: + path: /usr/lib64 + name: libsystemddir + - configMap: + defaultMode: 420 + name: fluentd-gcp-config-cos-auditd + name: config-volume + initContainers: + - name: cos-auditd-setup + image: ubuntu + command: ["chroot", "/host", "systemctl", "start", "cloud-audit-setup"] + securityContext: + privileged: true + volumeMounts: + - name: host + mountPath: /host + resources: + requests: + memory: "10Mi" + cpu: "10m" + priorityClassName: high-priority-system + containers: + - name: fluentd-gcp-cos-auditd + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/stackdriver-agents/stackdriver-logging-agent:0.6-1.6.0-1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [[ ! -e /var/log/fluentd-buffers ]]; then + exit 1; + fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then + rm -rf /var/log/fluentd-buffers; + exit 1; + fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then + exit 1; + fi; + failureThreshold: 3 + initialDelaySeconds: 600 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: varlog + - mountPath: /host/lib + name: libsystemddir + readOnly: true + - mountPath: /etc/google-fluentd/google-fluentd.conf + subPath: google-fluentd.conf + name: config-volume + dnsPolicy: Default + restartPolicy: Always + terminationGracePeriodSeconds: 30 + tolerations: + - effect: NoSchedule + key: node.alpha.kubernetes.io/ismaster + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: sandbox.gke.io/runtime + operator: Equal + value: gvisor + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml new file mode 100644 index 0000000000..fc78be56c9 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +apiVersion: v1 +kind: Namespace +metadata: + name: cos-auditd \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml new file mode 100644 index 0000000000..d50f33c3b5 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml @@ -0,0 +1,34 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# LimitRange Reference: https://kubernetes.io/docs/concepts/policy/limit-range/ +apiVersion: v1 +kind: LimitRange +metadata: + name: default-limits +spec: + limits: + - type: Container + default: + cpu: "1" + memory: 512Mi + defaultRequest: + memory: 256Mi + cpu: "0.5" + max: + cpu: "2" + memory: 1Gi + - type: PersistentVolumeClaim + max: + storage: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml new file mode 100644 index 0000000000..aabbe4e143 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml @@ -0,0 +1,22 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# apiVersion: networking.k8s.io/v1 +# kind: NetworkPolicy +# metadata: +# name: default-deny-egress +# spec: +# podSelector: {} +# policyTypes: +# - Egress diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml new file mode 100644 index 0000000000..4d8d1c48f3 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml @@ -0,0 +1,18 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: team-a diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml new file mode 100644 index 0000000000..f3615b1d57 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml @@ -0,0 +1,30 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This Namespace is configured with RepoSync +# Doc ref: https://cloud.google.com/anthos-config-management/docs/how-to/namespace-repositories#namespace-root + +apiVersion: configsync.gke.io/v1beta1 +kind: RepoSync +metadata: + name: repo-sync + namespace: team-a +spec: + sourceFormat: unstructured + git: + repo: "https://github.com/danielmarzini/configsync-application-example" + branch: main + revision: HEAD + dir: "configsync/team-a" + auth: none diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml new file mode 100644 index 0000000000..b47cdaa2d6 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml @@ -0,0 +1,35 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ResourceQuota Reference: https://kubernetes.io/docs/concepts/policy/resource-quotas/ +kind: ResourceQuota +apiVersion: v1 +metadata: + namespace: team-a + name: pvc +spec: + hard: + persistentvolumeclaims: "3" +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + namespace: team-a + name: mem-cpu-store +spec: + hard: + requests.cpu: "1" + requests.memory: 1Gi + limits.cpu: "2" + limits.memory: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml new file mode 100644 index 0000000000..72377417d6 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: namespace-viewer +roleRef: + kind: ClusterRole + name: dev-namespace-viewer + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: Group + name: team-a@marzi.gcp-pso-italy.net + apiGroup: rbac.authorization.k8s.io + namespace: team-a \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml new file mode 100644 index 0000000000..ec5de25032 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml + kind: RoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: syncs-repo + namespace: team-a + subjects: + - kind: ServiceAccount + name: ns-reconciler-team-a + namespace: config-management-system + roleRef: + kind: ClusterRole + name: edit + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml new file mode 100644 index 0000000000..3f2c61dac7 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml @@ -0,0 +1,44 @@ + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: team-b + name: whereami +spec: + replicas: 1 + selector: + matchLabels: + app: whereami + template: + metadata: + labels: + app: whereami + version: v1 + spec: + containers: + - name: whereami + image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.7 + resources: + limits: + cpu: "0.5" + memory: 128Mi + requests: + cpu: "0.5" + memory: 128Mi + ports: + - name: http + containerPort: 8080 \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml new file mode 100644 index 0000000000..075ca0859b --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: team-b + labels: + geo: "europe-west1" \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/system/repo.yml b/fast/stages/03-gke-multitenant/config/system/repo.yml new file mode 100644 index 0000000000..bfcc179a41 --- /dev/null +++ b/fast/stages/03-gke-multitenant/config/system/repo.yml @@ -0,0 +1,23 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# For the mono-repo mode, declaring a Repo resource under system/ directory is required. +# For the multi-repo mode, declaring a Repo resource under system/ directory is optional. +apiVersion: configmanagement.gke.io/v1 +kind: Repo +metadata: + name: repo +spec: + version: 1.0.0 \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index be3be68628..d1089cfa55 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -63,6 +63,9 @@ everything else is in RBAC (second part of this stage) ## Addresses for ILBs +### Private Cluster +WARNING: do not use run this stage with GKE-HUB and private clusters enabled from a machine outside the clusters network + ## Filestore - later diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index e74750c12d..8f9cb987e1 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -43,6 +43,7 @@ module "gke-cluster" { horizontal_pod_autoscaling = true config_connector_config = true kalm_config = false + gcp_filestore_csi_driver_config = false # enable only if enable_dataplane_v2 is changed to false below network_policy_config = false istio_config = { @@ -51,15 +52,15 @@ module "gke-cluster" { } } # change these here for all clusters if absolutely needed - # authenticator_security_group = var.authenticator_security_group - enable_dataplane_v2 = true - enable_l4_ilb_subsetting = false - enable_intranode_visibility = true - enable_shielded_nodes = true - workload_identity = true + authenticator_security_group = var.authenticator_security_group + enable_dataplane_v2 = true + enable_l4_ilb_subsetting = false + enable_intranode_visibility = true + enable_shielded_nodes = true + workload_identity = true private_cluster_config = { enable_private_nodes = true - enable_private_endpoint = true + enable_private_endpoint = false master_ipv4_cidr_block = each.value.net.master_range master_global_access = true } diff --git a/fast/stages/03-gke-multitenant/dev/gke-hub.tf b/fast/stages/03-gke-multitenant/dev/gke-hub.tf new file mode 100644 index 0000000000..6d21b727e0 --- /dev/null +++ b/fast/stages/03-gke-multitenant/dev/gke-hub.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gke-hub" { + source = "../../../../modules/gke-hub" + project_id = module.gke-project-0.project_id + features = { + configmanagement = true + mc_ingress = true + mc_servicediscovery = true + } + member_clusters = { + for cluster_id in keys(var.clusters) : + cluster_id => module.gke-cluster[cluster_id].id + } + member_features = { + configmanagement = { + binauthz = false + config_sync = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "fast/stages/03-gke-multitenant/config" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "fast-dev-gke-marzi" + sync_repo = "https://github.com/GoogleCloudPlatform/cloud-foundation-fabric" + sync_rev = null + } + hierarchy_controller = null + policy_controller = { + exemptable_namespaces = [ + "asm-system", + "config-management-system", + "config-management-monitoring", + "gatekeeper-system", + "kube-system", + "cos-auditd" + ] + log_denies_enabled = true + referential_rules_enabled = false + template_library_installed = true + } + version = "1.10.2" + } + } +} diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index aef450b0da..cc27ed9764 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -35,7 +35,9 @@ module "gke-project-0" { "gkehub.googleapis.com", "stackdriver.googleapis.com", "container.googleapis.com", - # "trafficdirector.googleapis.com" + "multiclusterservicediscovery.googleapis.com", + "multiclusteringress.googleapis.com", + "trafficdirector.googleapis.com" ] # add here any other service ids and keys for robot accounts which are needed # service_encryption_key_ids = { @@ -66,6 +68,9 @@ module "gke-project-0" { # values = ["projects/fl01-prod-iac-core-0"] # } # } + iam = { + "roles/container.clusterViewer" = var.cluster_viewer_users + } } module "gke-dataset-resource-usage" { diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 2a165b8eea..608ad51b41 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -18,11 +18,11 @@ # 1 project, m clusters # cloud dns for gke? -# variable "authenticator_security_group" { -# description = "Optional group used for Groups for GKE." -# type = string -# default = null -# } +variable "authenticator_security_group" { + description = "Optional group used for Groups for GKE." + type = string + default = null +} variable "billing_account" { # tfdoc:variable:source 00-bootstrap @@ -36,14 +36,15 @@ variable "billing_account" { variable "cluster_defaults" { description = "Default values for optional cluster configurations." type = object({ - cloudrun_config = bool - database_encryption_key = string - enable_binary_authorization = bool - master_authorized_ranges = map(string) - max_pods_per_node = number - pod_security_policy = bool - release_channel = string - vertical_pod_autoscaling = bool + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool }) default = { # TODO: review defaults @@ -55,10 +56,11 @@ variable "cluster_defaults" { rfc1918_2 = "172.16.0.0/12" rfc1918_3 = "192.168.0.0/16" } - max_pods_per_node = 110 - pod_security_policy = false - release_channel = "STABLE" - vertical_pod_autoscaling = false + max_pods_per_node = 110 + pod_security_policy = false + release_channel = "STABLE" + vertical_pod_autoscaling = false + gcp_filestore_csi_driver_config = false } } @@ -82,18 +84,25 @@ variable "clusters" { subnet = string }) overrides = object({ - cloudrun_config = bool - database_encryption_key = string - enable_binary_authorization = bool - master_authorized_ranges = map(string) - max_pods_per_node = number - pod_security_policy = bool - release_channel = string - vertical_pod_autoscaling = bool + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool }) })) } +variable "cluster_viewer_users" { + description = "list of users to be granted as container.clusterViewer" + type = list(any) + default = [] +} + variable "dns_domain" { description = "Domain name used for clusters, prefix by each cluster name. Leave null to disable Cloud DNS for GKE." type = string diff --git a/modules/gke-hub/main.tf b/modules/gke-hub/main.tf index 784fe88c9b..910641f4e3 100644 --- a/modules/gke-hub/main.tf +++ b/modules/gke-hub/main.tf @@ -40,7 +40,7 @@ resource "google_gke_hub_feature" "configmanagement" { resource "google_gke_hub_feature" "mci" { provider = google-beta - for_each = var.features.mc_ingress ? var.member_clusters : {} + for_each = var.features.mc_ingress ? try(var.member_clusters[0], {}) : {} project = var.project_id name = "multiclusteringress" location = "global" @@ -67,6 +67,12 @@ resource "google_gke_hub_feature_membership" "feature_member" { feature = google_gke_hub_feature.configmanagement["1"].name membership = google_gke_hub_membership.membership[each.key].membership_id + depends_on = [ + google_gke_hub_feature.configmanagement, + google_gke_hub_feature.mci, + google_gke_hub_feature.mcs, + ] + dynamic "configmanagement" { for_each = ( try(var.member_features.configmanagement, null) != null diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf index ec9bbf97c4..b1c540b5a3 100644 --- a/modules/gke-nodepool/main.tf +++ b/modules/gke-nodepool/main.tf @@ -84,7 +84,7 @@ resource "google_container_node_pool" "nodepool" { location = var.location name = var.name - initial_node_count = var.initial_node_count + initial_node_count = var.node_count == null ? var.initial_node_count : null // (dmarzi) TOFIX max_pods_per_node = var.max_pods_per_node node_count = var.autoscaling_config == null ? var.node_count : null node_locations = var.node_locations From 259afec97d3cf67d431b3cd3b948ee58377808e6 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 5 Apr 2022 12:52:04 +0200 Subject: [PATCH 13/75] Leave MCS and MCI disabled --- fast/stages/03-gke-multitenant/dev/gke-hub.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/gke-hub.tf b/fast/stages/03-gke-multitenant/dev/gke-hub.tf index 6d21b727e0..59a4ffce6d 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-hub.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-hub.tf @@ -17,11 +17,6 @@ module "gke-hub" { source = "../../../../modules/gke-hub" project_id = module.gke-project-0.project_id - features = { - configmanagement = true - mc_ingress = true - mc_servicediscovery = true - } member_clusters = { for cluster_id in keys(var.clusters) : cluster_id => module.gke-cluster[cluster_id].id From f9b808b4bc5dea9b02ba14c5633e52d558436202 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 27 May 2022 16:06:41 +0200 Subject: [PATCH 14/75] Fix permissions and update NVA and peering net stages for gke --- fast/stages/01-resman/branch-gke.tf | 8 ++++---- fast/stages/02-networking-nva/spoke-dev.tf | 1 + fast/stages/02-networking-nva/spoke-prod.tf | 1 + fast/stages/02-networking-nva/variables.tf | 2 ++ fast/stages/02-networking-peering/spoke-dev.tf | 1 + fast/stages/02-networking-peering/spoke-prod.tf | 1 + fast/stages/02-networking-peering/variables.tf | 2 ++ fast/stages/03-gke-multitenant/dev/gke-clusters.tf | 1 + tests/fast/stages/s02_networking_nva/fixture/main.tf | 2 ++ tests/fast/stages/s02_networking_peering/fixture/main.tf | 2 ++ tests/modules/gke_hub/test_plan.py | 6 +++--- 11 files changed, 20 insertions(+), 7 deletions(-) diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 5914f5f503..1cfb3d6feb 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -51,7 +51,7 @@ module "branch-gke-multitenant-prod-folder" { module "branch-gke-multitenant-prod-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-gke-0" description = "Terraform gke multitenant prod service account." prefix = var.prefix @@ -63,7 +63,7 @@ module "branch-gke-multitenant-prod-sa" { module "branch-gke-multitenant-prod-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "prod-resman-gke-0" prefix = var.prefix versioning = true @@ -92,7 +92,7 @@ module "branch-gke-multitenant-dev-folder" { module "branch-gke-multitenant-dev-sa" { source = "../../../modules/iam-service-account" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "dev-resman-gke-0" description = "Terraform gke multitenant dev service account." prefix = var.prefix @@ -104,7 +104,7 @@ module "branch-gke-multitenant-dev-sa" { module "branch-gke-multitenant-dev-gcs" { source = "../../../modules/gcs" - project_id = var.automation_project_id + project_id = var.automation.project_id name = "dev-resman-gke-0" prefix = var.prefix versioning = true diff --git a/fast/stages/02-networking-nva/spoke-dev.tf b/fast/stages/02-networking-nva/spoke-dev.tf index 843d544fdb..28ed13eccf 100644 --- a/fast/stages/02-networking-nva/spoke-dev.tf +++ b/fast/stages/02-networking-nva/spoke-dev.tf @@ -126,6 +126,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, + local.service_accounts.gke-multitenant-dev, ] condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-nva/spoke-prod.tf b/fast/stages/02-networking-nva/spoke-prod.tf index d0a22dc98a..6ba5fe30b2 100644 --- a/fast/stages/02-networking-nva/spoke-prod.tf +++ b/fast/stages/02-networking-nva/spoke-prod.tf @@ -126,6 +126,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, + local.service_accounts.gke-multitenant-prod, ] condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-nva/variables.tf b/fast/stages/02-networking-nva/variables.tf index bc06729bcb..2fe29d7853 100644 --- a/fast/stages/02-networking-nva/variables.tf +++ b/fast/stages/02-networking-nva/variables.tf @@ -209,6 +209,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string + gke-multitenant-dev = string + gke-multitenant-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/02-networking-peering/spoke-dev.tf b/fast/stages/02-networking-peering/spoke-dev.tf index a6713eaf77..169ac6ba09 100644 --- a/fast/stages/02-networking-peering/spoke-dev.tf +++ b/fast/stages/02-networking-peering/spoke-dev.tf @@ -103,6 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, + local.service_accounts.gke-multitenant-dev, ] condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-peering/spoke-prod.tf b/fast/stages/02-networking-peering/spoke-prod.tf index 401a08563f..084f5a2164 100644 --- a/fast/stages/02-networking-peering/spoke-prod.tf +++ b/fast/stages/02-networking-peering/spoke-prod.tf @@ -103,6 +103,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = [ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, + local.service_accounts.gke-multitenant-prod, ] condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-peering/variables.tf b/fast/stages/02-networking-peering/variables.tf index 60bd8be1d3..77ef3884fc 100644 --- a/fast/stages/02-networking-peering/variables.tf +++ b/fast/stages/02-networking-peering/variables.tf @@ -187,6 +187,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string + gke-multitenant-dev = string + gke-multitenant-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index 8f9cb987e1..51ba711a45 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -44,6 +44,7 @@ module "gke-cluster" { config_connector_config = true kalm_config = false gcp_filestore_csi_driver_config = false + gke_backup_agent_config = false # enable only if enable_dataplane_v2 is changed to false below network_policy_config = false istio_config = { diff --git a/tests/fast/stages/s02_networking_nva/fixture/main.tf b/tests/fast/stages/s02_networking_nva/fixture/main.tf index f0ff8ad033..b34da60a7f 100644 --- a/tests/fast/stages/s02_networking_nva/fixture/main.tf +++ b/tests/fast/stages/s02_networking_nva/fixture/main.tf @@ -35,6 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" + gke-multitenant-dev = "string" + gke-multitenant-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } diff --git a/tests/fast/stages/s02_networking_peering/fixture/main.tf b/tests/fast/stages/s02_networking_peering/fixture/main.tf index 4204095901..704c71a318 100644 --- a/tests/fast/stages/s02_networking_peering/fixture/main.tf +++ b/tests/fast/stages/s02_networking_peering/fixture/main.tf @@ -35,6 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" + gke-multitenant-dev = "string" + gke-multitenant-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } diff --git a/tests/modules/gke_hub/test_plan.py b/tests/modules/gke_hub/test_plan.py index 08e0a581c2..40d82c1f5e 100644 --- a/tests/modules/gke_hub/test_plan.py +++ b/tests/modules/gke_hub/test_plan.py @@ -23,11 +23,11 @@ def resources(plan_runner): def test_resource_count(resources): "Test number of resources created." - assert len(resources) == 8 + assert len(resources) == 6 assert sorted(r['address'] for r in resources) == [ 'module.hub.google_gke_hub_feature.configmanagement["1"]', - 'module.hub.google_gke_hub_feature.mci["mycluster1"]', - 'module.hub.google_gke_hub_feature.mci["mycluster2"]', + # 'module.hub.google_gke_hub_feature.mci["mycluster1"]', + # 'module.hub.google_gke_hub_feature.mci["mycluster2"]', 'module.hub.google_gke_hub_feature.mcs["1"]', 'module.hub.google_gke_hub_feature_membership.feature_member["mycluster1"]', 'module.hub.google_gke_hub_feature_membership.feature_member["mycluster2"]', From 744417f22f14e6a41953d0bb9ac48e8737906ebe Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 23 Jun 2022 08:52:36 +0200 Subject: [PATCH 15/75] fix boilerplate --- .../config/cluster/clusterrole-ns-viewer.yml | 23 ++- ...-have-geo-constraint_v1beta_gatekeeper.yml | 35 ++-- .../config/cluster/pod_priority_classes.yml | 8 +- .../apps_v1_configmap_cos-auditd.yml | 11 +- .../apps_v1_daemonset_cos-auditd.yml | 170 +++++++++--------- .../namespaces/cos-auditd/namespace.yml | 9 +- .../network-policy-default-deny-egress.yml | 6 +- .../teams/team-a/rolebinding-ns-viewer.yml | 15 +- .../03-gke-multitenant/config/system/repo.yml | 14 +- 9 files changed, 143 insertions(+), 148 deletions(-) diff --git a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml index 3bf4cf773b..52ab7387b1 100644 --- a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml +++ b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml @@ -1,28 +1,27 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: dev-namespace-viewer rules: -- apiGroups: ["", "metrics.k8s.io", "extensions", "apps"] - resources: - - "pods" - - "pods/log" - - "events" - - "deployments" - - "replicasets" - verbs: ["get", "watch", "list"] \ No newline at end of file + - apiGroups: ["", "metrics.k8s.io", "extensions", "apps"] + resources: + - "pods" + - "pods/log" + - "events" + - "deployments" + - "replicasets" + verbs: ["get", "watch", "list"] diff --git a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml index a89b549f76..aede039d56 100644 --- a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml +++ b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml @@ -1,18 +1,17 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - apiVersion: constraints.gatekeeper.sh/v1beta1 kind: K8sRequiredLabels metadata: @@ -22,22 +21,22 @@ metadata: spec: enforcementAction: dryrun match: - excludedNamespaces: - - "kube-system" - - "kube-public" - - "kube-node-lease" - - "resource-group-system" - - "config-management-monitoring" - - "config-management-system" - - "cos-auditd" - - "gatekeeper-system" - - "kube-node-lease" - - "cnrm-system" - - "gke-mcs" - - "configconnector-operator-system" + excludedNamespaces: + - "kube-system" + - "kube-public" + - "kube-node-lease" + - "resource-group-system" + - "config-management-monitoring" + - "config-management-system" + - "cos-auditd" + - "gatekeeper-system" + - "kube-node-lease" + - "cnrm-system" + - "gke-mcs" + - "configconnector-operator-system" kinds: - apiGroups: [""] kinds: ["Namespace"] parameters: labels: - - key: "geo" \ No newline at end of file + - key: "geo" diff --git a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml index 070e1e9cf3..216deba384 100644 --- a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml +++ b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml @@ -1,11 +1,11 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,4 +44,4 @@ metadata: value: 10 globalDefault: true # preemptionPolicy: Never -description: "This priority class will cause other pods to be preempted." \ No newline at end of file +description: "This priority class will cause other pods to be preempted." diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml index 0c3a25c628..8db587ea33 100644 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml @@ -1,25 +1,24 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - kind: ConfigMap apiVersion: v1 metadata: name: fluentd-gcp-config-cos-auditd namespace: cos-auditd annotations: - kubernetes.io/description: 'ConfigMap for Linux auditd logging daemonset on COS nodes.' + kubernetes.io/description: "ConfigMap for Linux auditd logging daemonset on COS nodes." data: google-fluentd.conf: |- @@ -51,4 +50,4 @@ data: disable_retry_limit num_threads 2 use_grpc true - \ No newline at end of file + diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml index d9980318ce..b9068cd792 100644 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml @@ -1,25 +1,24 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - apiVersion: apps/v1 kind: DaemonSet metadata: name: cos-auditd-logging namespace: cos-auditd annotations: - kubernetes.io/description: 'DaemonSet that enables Linux auditd logging on COS nodes.' + kubernetes.io/description: "DaemonSet that enables Linux auditd logging on COS nodes." spec: selector: matchLabels: @@ -36,93 +35,94 @@ spec: nodeSelector: cloud.google.com/gke-os-distribution: cos volumes: - - hostPath: - path: / - name: host - - hostPath: - path: /var/log - name: varlog - - hostPath: - path: /usr/lib64 - name: libsystemddir - - configMap: - defaultMode: 420 - name: fluentd-gcp-config-cos-auditd - name: config-volume - initContainers: - - name: cos-auditd-setup - image: ubuntu - command: ["chroot", "/host", "systemctl", "start", "cloud-audit-setup"] - securityContext: - privileged: true - volumeMounts: - - name: host - mountPath: /host - resources: - requests: - memory: "10Mi" - cpu: "10m" - priorityClassName: high-priority-system - containers: - - name: fluentd-gcp-cos-auditd - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - image: gcr.io/stackdriver-agents/stackdriver-logging-agent:0.6-1.6.0-1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /bin/sh - - -c - - | - LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [[ ! -e /var/log/fluentd-buffers ]]; then - exit 1; - fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then - rm -rf /var/log/fluentd-buffers; - exit 1; - fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then - exit 1; - fi; - failureThreshold: 3 - initialDelaySeconds: 600 - periodSeconds: 60 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: "1" - memory: 500Mi - requests: - cpu: 100m - memory: 200Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/log + - hostPath: + path: / + name: host + - hostPath: + path: /var/log name: varlog - - mountPath: /host/lib + - hostPath: + path: /usr/lib64 name: libsystemddir - readOnly: true - - mountPath: /etc/google-fluentd/google-fluentd.conf - subPath: google-fluentd.conf + - configMap: + defaultMode: 420 + name: fluentd-gcp-config-cos-auditd name: config-volume + initContainers: + - name: cos-auditd-setup + image: ubuntu + command: + ["chroot", "/host", "systemctl", "start", "cloud-audit-setup"] + securityContext: + privileged: true + volumeMounts: + - name: host + mountPath: /host + resources: + requests: + memory: "10Mi" + cpu: "10m" + priorityClassName: high-priority-system + containers: + - name: fluentd-gcp-cos-auditd + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: gcr.io/stackdriver-agents/stackdriver-logging-agent:0.6-1.6.0-1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/sh + - -c + - | + LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [[ ! -e /var/log/fluentd-buffers ]]; then + exit 1; + fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then + rm -rf /var/log/fluentd-buffers; + exit 1; + fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then + exit 1; + fi; + failureThreshold: 3 + initialDelaySeconds: 600 + periodSeconds: 60 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: "1" + memory: 500Mi + requests: + cpu: 100m + memory: 200Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/log + name: varlog + - mountPath: /host/lib + name: libsystemddir + readOnly: true + - mountPath: /etc/google-fluentd/google-fluentd.conf + subPath: google-fluentd.conf + name: config-volume dnsPolicy: Default restartPolicy: Always terminationGracePeriodSeconds: 30 tolerations: - - effect: NoSchedule - key: node.alpha.kubernetes.io/ismaster - - effect: NoExecute - operator: Exists - - effect: NoSchedule - key: sandbox.gke.io/runtime - operator: Equal - value: gvisor + - effect: NoSchedule + key: node.alpha.kubernetes.io/ismaster + - effect: NoExecute + operator: Exists + - effect: NoSchedule + key: sandbox.gke.io/runtime + operator: Equal + value: gvisor updateStrategy: rollingUpdate: maxUnavailable: 1 - type: RollingUpdate \ No newline at end of file + type: RollingUpdate diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml index fc78be56c9..d80a804eb2 100644 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml +++ b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml @@ -1,19 +1,18 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - apiVersion: v1 kind: Namespace metadata: - name: cos-auditd \ No newline at end of file + name: cos-auditd diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml index aabbe4e143..281a8f2c06 100644 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml @@ -1,11 +1,11 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml index 72377417d6..28b84ccc30 100644 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml +++ b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml @@ -1,18 +1,17 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -22,7 +21,7 @@ roleRef: name: dev-namespace-viewer apiGroup: rbac.authorization.k8s.io subjects: -- kind: Group - name: team-a@marzi.gcp-pso-italy.net - apiGroup: rbac.authorization.k8s.io - namespace: team-a \ No newline at end of file + - kind: Group + name: team-a@marzi.gcp-pso-italy.net + apiGroup: rbac.authorization.k8s.io + namespace: team-a diff --git a/fast/stages/03-gke-multitenant/config/system/repo.yml b/fast/stages/03-gke-multitenant/config/system/repo.yml index bfcc179a41..b88f7dc63e 100644 --- a/fast/stages/03-gke-multitenant/config/system/repo.yml +++ b/fast/stages/03-gke-multitenant/config/system/repo.yml @@ -1,23 +1,23 @@ # Copyright 2022 Google LLC - +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - +# # http://www.apache.org/licenses/LICENSE-2.0 - +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# mono-repo mode: a Repo resource under system/ directory is required +# multi-repo mode: a Repo resource under system/ directory is optional -# For the mono-repo mode, declaring a Repo resource under system/ directory is required. -# For the multi-repo mode, declaring a Repo resource under system/ directory is optional. apiVersion: configmanagement.gke.io/v1 kind: Repo metadata: - name: repo + name: repo spec: - version: 1.0.0 \ No newline at end of file + version: 1.0.0 From fb1d31551f3485859007c343536807fb5e92199c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 23 Jun 2022 08:55:28 +0200 Subject: [PATCH 16/75] fix tfdoc --- fast/stages/01-resman/README.md | 19 ++++++++++--------- fast/stages/02-networking-nva/README.md | 4 ++-- fast/stages/02-networking-peering/README.md | 4 ++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index 90ffa7c750..d8c5b4d9a9 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -195,14 +195,15 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L145) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L159) | Data for the Data Platform stage. | | | -| [networking](outputs.tf#L175) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L184) | Data for the project factories stage. | | | -| [providers](outputs.tf#L200) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L207) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L217) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L227) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L240) | Terraform variable files for the following stages. | ✓ | | +| [cicd_repositories](outputs.tf#L159) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L173) | Data for the Data Platform stage. | | | +| [gke_multitenant](outputs.tf#L241) | Data for the GKE multitenant stage. | | 03-gke-multitenant | +| [networking](outputs.tf#L189) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L198) | Data for the project factories stage. | | | +| [providers](outputs.tf#L214) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L221) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L231) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L258) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L271) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md index ed3c8309ac..f164c45437 100644 --- a/fast/stages/02-networking-nva/README.md +++ b/fast/stages/02-networking-nva/README.md @@ -377,8 +377,8 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [outputs_location](variables.tf#L125) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | | [psa_ranges](variables.tf#L142) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_configs](variables.tf#L183) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L206) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L218) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L206) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L220) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md index a4beab1ad5..27d0da1d71 100644 --- a/fast/stages/02-networking-peering/README.md +++ b/fast/stages/02-networking-peering/README.md @@ -301,8 +301,8 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [peering_configs](variables-peerings.tf#L19) | Peering configurations. | map(object({…})) | | {…} | | | [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | -| [vpn_onprem_configs](variables.tf#L196) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs From ed4b301a170929b558745ac68915ca0b986ff2a0 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 23 Jun 2022 09:05:20 +0200 Subject: [PATCH 17/75] align subnets between vpn and peering network stages --- .../data/subnets/dev/dev-gke-nodes-ew1.yaml | 8 ++++++++ tests/fast/stages/s02_networking_peering/test_plan.py | 1 + 2 files changed, 9 insertions(+) create mode 100644 fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml diff --git a/fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml b/fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml new file mode 100644 index 0000000000..c2b5cbe712 --- /dev/null +++ b/fast/stages/02-networking-peering/data/subnets/dev/dev-gke-nodes-ew1.yaml @@ -0,0 +1,8 @@ +# skip boilerplate check + +region: europe-west1 +description: Default subnet for prod gke nodes +ip_cidr_range: 10.64.0.0/24 +secondary_ip_range: + pods: 100.64.0.0/16 + services: 192.168.1.0/24 diff --git a/tests/fast/stages/s02_networking_peering/test_plan.py b/tests/fast/stages/s02_networking_peering/test_plan.py index 6d38e66127..b4de02f91c 100644 --- a/tests/fast/stages/s02_networking_peering/test_plan.py +++ b/tests/fast/stages/s02_networking_peering/test_plan.py @@ -39,6 +39,7 @@ def test_vpn_peering_parity(e2e_plan_runner): for VPN and VPC peering resources''' _, plan_peering = e2e_plan_runner(fixture_path=FIXTURE_PEERING) _, plan_vpn = e2e_plan_runner(fixture_path=FIXTURE_VPN) + ddiff = DeepDiff(plan_vpn, plan_peering, ignore_order=True, group_by='address', view='tree') From d6a81fb662d07006c8028fa3e49290248f35183c Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 30 Jun 2022 11:00:57 +0200 Subject: [PATCH 18/75] wip --- fast/stages/00-bootstrap/README.md | 18 +++++------ fast/stages/00-bootstrap/organization.tf | 14 ++++----- fast/stages/00-bootstrap/variables.tf | 2 ++ fast/stages/01-resman/README.md | 35 +++++++++++----------- fast/stages/01-resman/billing.tf | 4 +-- fast/stages/01-resman/branch-gke.tf | 28 ++++++++--------- fast/stages/01-resman/branch-networking.tf | 4 +-- fast/stages/01-resman/main.tf | 1 - fast/stages/01-resman/organization.tf | 21 +++++++------ fast/stages/01-resman/outputs.tf | 32 ++++++++++---------- fast/stages/01-resman/variables.tf | 2 ++ 11 files changed, 83 insertions(+), 78 deletions(-) diff --git a/fast/stages/00-bootstrap/README.md b/fast/stages/00-bootstrap/README.md index 5b7f495d79..262183663e 100644 --- a/fast/stages/00-bootstrap/README.md +++ b/fast/stages/00-bootstrap/README.md @@ -461,18 +461,18 @@ The remaining configuration is manual, as it regards the repositories themselves | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account](variables.tf#L17) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | | -| [organization](variables.tf#L179) | Organization details. | object({…}) | ✓ | | | -| [prefix](variables.tf#L194) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | +| [organization](variables.tf#L181) | Organization details. | object({…}) | ✓ | | | +| [prefix](variables.tf#L196) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | | | [bootstrap_user](variables.tf#L25) | Email of the nominal user running this stage for the first time. | string | | null | | | [cicd_repositories](variables.tf#L31) | CI/CD repository configuration. Identity providers reference keys in the `federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | | [custom_role_names](variables.tf#L83) | Names of custom roles defined at the org level. | object({…}) | | {…} | | -| [fast_features](variables.tf#L95) | Selective control for top-level FAST features. | object({…}) | | {…} | | -| [federated_identity_providers](variables.tf#L112) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | -| [groups](variables.tf#L126) | Group names to grant organization-level permissions. | map(string) | | {…} | | -| [iam](variables.tf#L140) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | -| [iam_additive](variables.tf#L146) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | -| [log_sinks](variables.tf#L154) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | -| [outputs_location](variables.tf#L188) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [fast_features](variables.tf#L95) | Selective control for top-level FAST features. | object({…}) | | {…} | | +| [federated_identity_providers](variables.tf#L114) | Workload Identity Federation pools. The `cicd_repositories` variable references keys here. | map(object({…})) | | {} | | +| [groups](variables.tf#L128) | Group names to grant organization-level permissions. | map(string) | | {…} | | +| [iam](variables.tf#L142) | Organization-level custom IAM settings in role => [principal] format. | map(list(string)) | | {} | | +| [iam_additive](variables.tf#L148) | Organization-level custom IAM settings in role => [principal] format for non-authoritative bindings. | map(list(string)) | | {} | | +| [log_sinks](variables.tf#L156) | Org-level log sinks, in name => {type, filter} format. | map(object({…})) | | {…} | | +| [outputs_location](variables.tf#L190) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | ## Outputs diff --git a/fast/stages/00-bootstrap/organization.tf b/fast/stages/00-bootstrap/organization.tf index 947269c1ff..a9fa6af0ce 100644 --- a/fast/stages/00-bootstrap/organization.tf +++ b/fast/stages/00-bootstrap/organization.tf @@ -163,6 +163,12 @@ module "organization" { ] (var.custom_role_names.service_project_network_admin) = [ "compute.globalOperations.get", + # the following two permissions are used by automation service accounts + # who manage service projects where peering creation might be needed + # (e.g. GKE), if you remove them make sure your network administrators + # should create peerings for service projects + "compute.networks.updatePeering", + "compute.networks.get", "compute.organizations.disableXpnResource", "compute.organizations.enableXpnResource", "compute.projects.get", @@ -170,14 +176,6 @@ module "organization" { "compute.subnetworks.setIamPolicy", "dns.networks.bindPrivateDNSZone", "resourcemanager.projects.get", - - # if you prefer not granting permissions to create peerings to - # service accounts deploying service projects, remove these - # permissions and ask you network administrator to create any - # needed peerings (e.g. if you need to update routes for a GKE - # cluster) - "compute.networks.updatePeering", - "compute.networks.get", ] } logging_sinks = { diff --git a/fast/stages/00-bootstrap/variables.tf b/fast/stages/00-bootstrap/variables.tf index e9cc26e352..10ffdeefaa 100644 --- a/fast/stages/00-bootstrap/variables.tf +++ b/fast/stages/00-bootstrap/variables.tf @@ -96,12 +96,14 @@ variable "fast_features" { description = "Selective control for top-level FAST features." type = object({ data_platform = bool + gke = bool project_factory = bool sandbox = bool teams = bool }) default = { data_platform = true + gke = true project_factory = true sandbox = true teams = true diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index 68a9caae16..010f5a3d9d 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -182,29 +182,30 @@ Due to its simplicity, this stage lends itself easily to customizations: adding |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L20) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L38) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | -| [organization](variables.tf#L159) | Organization details. | object({…}) | ✓ | | 00-bootstrap | -| [prefix](variables.tf#L183) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | +| [organization](variables.tf#L161) | Organization details. | object({…}) | ✓ | | 00-bootstrap | +| [prefix](variables.tf#L185) | Prefix used for resources that need unique names. Use 9 characters or less. | string | ✓ | | 00-bootstrap | | [cicd_repositories](variables.tf#L47) | CI/CD repository configuration. Identity providers reference keys in the `automation.federated_identity_providers` variable. Set to null to disable, or set individual repositories to null if not needed. | object({…}) | | null | | | [custom_roles](variables.tf#L117) | Custom roles defined at the org level, in key => id format. | object({…}) | | null | 00-bootstrap | -| [fast_features](variables.tf#L126) | Selective control for top-level FAST features. | object({…}) | | {…} | 00-bootstrap | -| [groups](variables.tf#L144) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | -| [organization_policy_configs](variables.tf#L169) | Organization policies customization. | object({…}) | | null | | -| [outputs_location](variables.tf#L177) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | -| [tag_names](variables.tf#L194) | Customized names for resource management tags. | object({…}) | | {…} | | -| [team_folders](variables.tf#L211) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | +| [fast_features](variables.tf#L126) | Selective control for top-level FAST features. | object({…}) | | {…} | 00-bootstrap | +| [groups](variables.tf#L146) | Group names to grant organization-level permissions. | map(string) | | {…} | 00-bootstrap | +| [organization_policy_configs](variables.tf#L171) | Organization policies customization. | object({…}) | | null | | +| [outputs_location](variables.tf#L179) | Enable writing provider, tfvars and CI/CD workflow files to local filesystem. Leave null to disable | string | | null | | +| [tag_names](variables.tf#L196) | Customized names for resource management tags. | object({…}) | | {…} | | +| [team_folders](variables.tf#L213) | Team folders to be created. Format is described in a code comment. | map(object({…})) | | null | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L154) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L168) | Data for the Data Platform stage. | | | -| [networking](outputs.tf#L184) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L193) | Data for the project factories stage. | | | -| [providers](outputs.tf#L209) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L216) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L230) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L240) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L253) | Terraform variable files for the following stages. | ✓ | | +| [cicd_repositories](outputs.tf#L166) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L180) | Data for the Data Platform stage. | | | +| [gke_multitenant](outputs.tf#L252) | Data for the GKE multitenant stage. | | 03-gke-multitenant | +| [networking](outputs.tf#L196) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L205) | Data for the project factories stage. | | | +| [providers](outputs.tf#L221) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L228) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L242) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L269) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L282) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/01-resman/billing.tf b/fast/stages/01-resman/billing.tf index b3223b7540..9fa57de727 100644 --- a/fast/stages/01-resman/billing.tf +++ b/fast/stages/01-resman/billing.tf @@ -24,12 +24,12 @@ locals { module.branch-security-sa.iam_email, ], local.branch_dataplatform_sa_iam_emails, + local.branch_gke_sa_iam_emails, + local.branch_pf_sa_iam_emails, # enable if individual teams can create their own projects # [ # for k, v in module.branch-teams-team-sa : v.iam_email # ], - local.branch_teams_pf_sa_iam_emails, - local.branch_gke_multitenant_sa_iam_emails ) } diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 1cfb3d6feb..72b3ffec02 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -32,24 +32,24 @@ module "branch-gke-folder" { # GKE-level folders, service accounts and buckets for each individual environment -module "branch-gke-multitenant-prod-folder" { +module "branch-gke-prod-folder" { source = "../../../modules/folder" parent = module.branch-gke-folder.id name = "Production" iam = { "roles/owner" = [ - module.branch-gke-multitenant-prod-sa.iam_email + module.branch-gke-prod-sa.iam_email ] "roles/resourcemanager.projectCreator" = [ - module.branch-gke-multitenant-prod-sa.iam_email + module.branch-gke-prod-sa.iam_email ] "roles/compute.xpnAdmin" = [ - module.branch-gke-multitenant-prod-sa.iam_email + module.branch-gke-prod-sa.iam_email ] } } -module "branch-gke-multitenant-prod-sa" { +module "branch-gke-prod-sa" { source = "../../../modules/iam-service-account" project_id = var.automation.project_id name = "prod-resman-gke-0" @@ -61,36 +61,36 @@ module "branch-gke-multitenant-prod-sa" { } } -module "branch-gke-multitenant-prod-gcs" { +module "branch-gke-prod-gcs" { source = "../../../modules/gcs" project_id = var.automation.project_id name = "prod-resman-gke-0" prefix = var.prefix versioning = true iam = { - "roles/storage.objectAdmin" = [module.branch-gke-multitenant-prod-sa.iam_email] + "roles/storage.objectAdmin" = [module.branch-gke-prod-sa.iam_email] } } -module "branch-gke-multitenant-dev-folder" { +module "branch-gke-dev-folder" { source = "../../../modules/folder" parent = module.branch-gke-folder.id name = "Development" iam = { "roles/owner" = [ - module.branch-gke-multitenant-dev-sa.iam_email + module.branch-gke-dev-sa.iam_email ] "roles/resourcemanager.projectCreator" = [ - module.branch-gke-multitenant-dev-sa.iam_email + module.branch-gke-dev-sa.iam_email ] "roles/compute.xpnAdmin" = [ - module.branch-gke-multitenant-dev-sa.iam_email + module.branch-gke-dev-sa.iam_email ] } } -module "branch-gke-multitenant-dev-sa" { +module "branch-gke-dev-sa" { source = "../../../modules/iam-service-account" project_id = var.automation.project_id name = "dev-resman-gke-0" @@ -102,13 +102,13 @@ module "branch-gke-multitenant-dev-sa" { } } -module "branch-gke-multitenant-dev-gcs" { +module "branch-gke-dev-gcs" { source = "../../../modules/gcs" project_id = var.automation.project_id name = "dev-resman-gke-0" prefix = var.prefix versioning = true iam = { - "roles/storage.objectAdmin" = [module.branch-gke-multitenant-dev-sa.iam_email] + "roles/storage.objectAdmin" = [module.branch-gke-dev-sa.iam_email] } } diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf index 79a31745cd..952c832671 100644 --- a/fast/stages/01-resman/branch-networking.tf +++ b/fast/stages/01-resman/branch-networking.tf @@ -53,7 +53,7 @@ module "branch-network-prod-folder" { "roles/compute.xpnAdmin" = compact([ try(module.branch-dp-prod-sa.0.iam_email, ""), try(module.branch-pf-prod-sa.0.iam_email, ""), - module.branch-gke-multitenant-prod-sa.iam_email, + module.branch-gke-prod-sa.iam_email, ]) } tag_bindings = { @@ -71,7 +71,7 @@ module "branch-network-dev-folder" { (local.custom_roles.service_project_network_admin) = compact([ try(module.branch-dp-dev-sa.0.iam_email, ""), try(module.branch-pf-dev-sa.0.iam_email, ""), - module.branch-gke-multitenant-dev-sa.iam_email, + module.branch-gke-dev-sa.iam_email, ]) } tag_bindings = { diff --git a/fast/stages/01-resman/main.tf b/fast/stages/01-resman/main.tf index c40957964f..590c402715 100644 --- a/fast/stages/01-resman/main.tf +++ b/fast/stages/01-resman/main.tf @@ -57,7 +57,6 @@ locals { "02-security.auto.tfvars.json" ] } - custom_roles = coalesce(var.custom_roles, {}) groups = { for k, v in var.groups : diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/01-resman/organization.tf index 24e1a5f91d..0de7193a4c 100644 --- a/fast/stages/01-resman/organization.tf +++ b/fast/stages/01-resman/organization.tf @@ -26,12 +26,15 @@ locals { ] : [] ) - # set to the empty list if you remove the teams branch - branch_gke_multitenant_sa_iam_emails = [ - module.branch-gke-multitenant-dev-sa.iam_email, - module.branch-gke-multitenant-prod-sa.iam_email - ] - branch_teams_pf_sa_iam_emails = ( + branch_gke_sa_iam_emails = ( + var.fast_features.gke + ? [ + module.branch-gke-dev-sa.iam_email, + module.branch-gke-prod-sa.iam_email + ] + : [] + ) + branch_pf_sa_iam_emails = ( var.fast_features.project_factory ? [ module.branch-pf-dev-sa.0.iam_email, @@ -76,19 +79,19 @@ module "organization" { ] }, local.billing_org ? { - "roles/billing.costsManager" = local.branch_teams_pf_sa_iam_emails + "roles/billing.costsManager" = local.branch_pf_sa_iam_emails "roles/billing.user" = concat( [ module.branch-network-sa.iam_email, module.branch-security-sa.iam_email, ], local.branch_dataplatform_sa_iam_emails, + local.branch_gke_sa_iam_emails, + local.branch_pf_sa_iam_emails, # enable if individual teams can create their own projects # [ # for k, v in module.branch-teams-team-sa : v.iam_email # ], - local.branch_teams_pf_sa_iam_emails, - local.branch_gke_multitenant_sa_iam_emails ) } : {} ) diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index 31188d7c80..e57a4ca84b 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -64,8 +64,8 @@ locals { { data-platform-dev = try(module.branch-dp-dev-folder.0.id, null) data-platform-prod = try(module.branch-dp-prod-folder.0.id, null) - gke-multitenant-dev = module.branch-gke-multitenant-dev-folder.id - gke-multitenant-prod = module.branch-gke-multitenant-prod-folder.id + gke-multitenant-dev = module.branch-gke-dev-folder.id + gke-multitenant-prod = module.branch-gke-prod-folder.id networking = module.branch-network-folder.id networking-dev = module.branch-network-dev-folder.id networking-prod = module.branch-network-prod-folder.id @@ -98,15 +98,15 @@ locals { name = "security" sa = module.branch-security-sa.email }) - "03-gke-multitenant-dev" = templatefile(local._tpl_providers, { - bucket = module.branch-gke-multitenant-dev-gcs.name - name = "gke-multitenant-dev" - sa = module.branch-gke-multitenant-dev-sa.email + "03-gke-dev" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-dev-gcs.name + name = "gke-dev" + sa = module.branch-gke-dev-sa.email }) - "03-gke-multitenant-prod" = templatefile(local._tpl_providers, { - bucket = module.branch-gke-multitenant-prod-gcs.name - name = "gke-multitenant-prod" - sa = module.branch-gke-multitenant-prod-sa.email + "03-gke-prod" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-prod-gcs.name + name = "gke-prod" + sa = module.branch-gke-prod-sa.email }) }, !var.fast_features.data_platform ? {} : { @@ -254,14 +254,14 @@ output "gke_multitenant" { description = "Data for the GKE multitenant stage." value = { "dev" = { - folder = module.branch-gke-multitenant-dev-folder.id - gcs_bucket = module.branch-gke-multitenant-dev-gcs.name - service_account = module.branch-gke-multitenant-dev-sa.email + folder = module.branch-gke-dev-folder.id + gcs_bucket = module.branch-gke-dev-gcs.name + service_account = module.branch-gke-dev-sa.email } "prod" = { - folder = module.branch-gke-multitenant-prod-folder.id - gcs_bucket = module.branch-gke-multitenant-prod-gcs.name - service_account = module.branch-gke-multitenant-prod-sa.email + folder = module.branch-gke-prod-folder.id + gcs_bucket = module.branch-gke-prod-gcs.name + service_account = module.branch-gke-prod-sa.email } } } diff --git a/fast/stages/01-resman/variables.tf b/fast/stages/01-resman/variables.tf index 35a05efedb..f52c5fc695 100644 --- a/fast/stages/01-resman/variables.tf +++ b/fast/stages/01-resman/variables.tf @@ -128,12 +128,14 @@ variable "fast_features" { description = "Selective control for top-level FAST features." type = object({ data_platform = bool + gke = bool project_factory = bool sandbox = bool teams = bool }) default = { data_platform = true + gke = true project_factory = true sandbox = true teams = true From 7b5ced7e15a537b1760bfe822780d9429e94f182 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 30 Jun 2022 18:22:57 +0200 Subject: [PATCH 19/75] stage 01 --- fast/stages/01-resman/branch-data-platform.tf | 6 +- fast/stages/01-resman/branch-gke.tf | 144 +++++++++++------- fast/stages/01-resman/branch-networking.tf | 12 +- fast/stages/01-resman/organization.tf | 4 +- fast/stages/01-resman/outputs.tf | 68 +++++---- fast/stages/02-networking-nva/spoke-dev.tf | 2 +- fast/stages/02-networking-nva/spoke-prod.tf | 2 +- fast/stages/02-networking-nva/variables.tf | 4 +- fast/stages/02-networking-peering/README.md | 6 +- .../stages/02-networking-peering/spoke-dev.tf | 2 +- .../02-networking-peering/spoke-prod.tf | 2 +- .../stages/02-networking-peering/variables.tf | 4 +- fast/stages/02-networking-vpn/README.md | 2 +- fast/stages/02-networking-vpn/spoke-dev.tf | 2 +- fast/stages/02-networking-vpn/spoke-prod.tf | 2 +- fast/stages/02-networking-vpn/variables.tf | 4 +- fast/stages/03-gke-multitenant/dev/gke-hub.tf | 2 +- fast/stages/03-gke-multitenant/dev/main.tf | 2 +- .../03-gke-multitenant/dev/variables.tf | 2 +- 19 files changed, 160 insertions(+), 112 deletions(-) diff --git a/fast/stages/01-resman/branch-data-platform.tf b/fast/stages/01-resman/branch-data-platform.tf index 237f8fb173..03a2fa9675 100644 --- a/fast/stages/01-resman/branch-data-platform.tf +++ b/fast/stages/01-resman/branch-data-platform.tf @@ -56,7 +56,8 @@ module "branch-dp-dev-folder" { } tag_bindings = { context = try( - module.organization.tag_values["${var.tag_names.environment}/development"].id, null + module.organization.tag_values["${var.tag_names.environment}/development"].id, + null ) } } @@ -82,7 +83,8 @@ module "branch-dp-prod-folder" { } tag_bindings = { context = try( - module.organization.tag_values["${var.tag_names.environment}/production"].id, null + module.organization.tag_values["${var.tag_names.environment}/production"].id, + null ) } } diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 72b3ffec02..42d640e782 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -16,99 +16,137 @@ # tfdoc:file:description GKE multitenant stage resources. -# top-level gke folder +moved { + from = module.branch-gke-folder + to = module.branch-gke-folder.0 +} module "branch-gke-folder" { source = "../../../modules/folder" + count = var.fast_features.gke ? 1 : 0 parent = "organizations/${var.organization.id}" name = "GKE" - # iam = { - # "roles/logging.admin" = [module.branch-gke-sa.iam_email] - # "roles/owner" = [module.branch-gke-sa.iam_email] - # "roles/resourcemanager.folderAdmin" = [module.branch-gke-sa.iam_email] - # "roles/resourcemanager.projectCreator" = [module.branch-gke-sa.iam_email] - # } + tag_bindings = { + context = try( + module.organization.tag_values["${var.tag_names.context}/gke"].id, null + ) + } +} + +moved { + from = module.branch-gke-dev-folder + to = module.branch-gke-dev-folder.0 +} + +module "branch-gke-dev-folder" { + source = "../../../modules/folder" + count = var.fast_features.gke ? 1 : 0 + parent = module.branch-gke-folder.0.id + name = "Development" + iam = { + "roles/owner" = [module.branch-gke-dev-sa.0.iam_email] + "roles/logging.admin" = [module.branch-gke-dev-sa.0.iam_email] + "roles/resourcemanager.folderAdmin" = [module.branch-gke-dev-sa.0.iam_email] + "roles/resourcemanager.projectCreator" = [module.branch-gke-dev-sa.0.iam_email] + "roles/compute.xpnAdmin" = [module.branch-gke-dev-sa.0.iam_email] + } + tag_bindings = { + context = try( + module.organization.tag_values["${var.tag_names.environment}/development"].id, + null + ) + } } -# GKE-level folders, service accounts and buckets for each individual environment +moved { + from = module.branch-gke-prod-folder + to = module.branch-gke-prod-folder.0 +} module "branch-gke-prod-folder" { source = "../../../modules/folder" - parent = module.branch-gke-folder.id + count = var.fast_features.gke ? 1 : 0 + parent = module.branch-gke-folder.0.id name = "Production" iam = { - "roles/owner" = [ - module.branch-gke-prod-sa.iam_email - ] - "roles/resourcemanager.projectCreator" = [ - module.branch-gke-prod-sa.iam_email - ] - "roles/compute.xpnAdmin" = [ - module.branch-gke-prod-sa.iam_email - ] + "roles/owner" = [module.branch-gke-prod-sa.0.iam_email] + "roles/logging.admin" = [module.branch-gke-prod-sa.0.iam_email] + "roles/resourcemanager.folderAdmin" = [module.branch-gke-prod-sa.0.iam_email] + "roles/resourcemanager.projectCreator" = [module.branch-gke-prod-sa.0.iam_email] + "roles/compute.xpnAdmin" = [module.branch-gke-prod-sa.0.iam_email] + } + tag_bindings = { + context = try( + module.organization.tag_values["${var.tag_names.environment}/production"].id, + null + ) + } +} + +moved { + from = module.branch-gke-dev-sa + to = module.branch-gke-dev-sa.0 +} + +module "branch-gke-dev-sa" { + source = "../../../modules/iam-service-account" + count = var.fast_features.gke ? 1 : 0 + project_id = var.automation.project_id + name = "dev-resman-gke-0" + description = "Terraform gke multitenant dev service account." + prefix = var.prefix + iam = { + "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] } } +moved { + from = module.branch-gke-prod-sa + to = module.branch-gke-prod-sa.0 +} + module "branch-gke-prod-sa" { source = "../../../modules/iam-service-account" + count = var.fast_features.gke ? 1 : 0 project_id = var.automation.project_id name = "prod-resman-gke-0" description = "Terraform gke multitenant prod service account." prefix = var.prefix iam = { - # FIXME(jccb): who should we use here? "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] } } -module "branch-gke-prod-gcs" { +moved { + from = module.branch-gke-dev-gcs + to = module.branch-gke-dev-gcs.0 +} + +module "branch-gke-dev-gcs" { source = "../../../modules/gcs" + count = var.fast_features.gke ? 1 : 0 project_id = var.automation.project_id - name = "prod-resman-gke-0" + name = "dev-resman-gke-0" prefix = var.prefix versioning = true iam = { - "roles/storage.objectAdmin" = [module.branch-gke-prod-sa.iam_email] - } -} - - -module "branch-gke-dev-folder" { - source = "../../../modules/folder" - parent = module.branch-gke-folder.id - name = "Development" - iam = { - "roles/owner" = [ - module.branch-gke-dev-sa.iam_email - ] - "roles/resourcemanager.projectCreator" = [ - module.branch-gke-dev-sa.iam_email - ] - "roles/compute.xpnAdmin" = [ - module.branch-gke-dev-sa.iam_email - ] + "roles/storage.objectAdmin" = [module.branch-gke-dev-sa.0.iam_email] } } -module "branch-gke-dev-sa" { - source = "../../../modules/iam-service-account" - project_id = var.automation.project_id - name = "dev-resman-gke-0" - description = "Terraform gke multitenant dev service account." - prefix = var.prefix - iam = { - # FIXME(jccb): who should we use here? - "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] - } +moved { + from = module.branch-gke-prod-gcs + to = module.branch-gke-prod-gcs.0 } -module "branch-gke-dev-gcs" { +module "branch-gke-prod-gcs" { source = "../../../modules/gcs" + count = var.fast_features.gke ? 1 : 0 project_id = var.automation.project_id - name = "dev-resman-gke-0" + name = "prod-resman-gke-0" prefix = var.prefix versioning = true iam = { - "roles/storage.objectAdmin" = [module.branch-gke-dev-sa.iam_email] + "roles/storage.objectAdmin" = [module.branch-gke-prod-sa.0.iam_email] } } diff --git a/fast/stages/01-resman/branch-networking.tf b/fast/stages/01-resman/branch-networking.tf index 952c832671..bf15c4681f 100644 --- a/fast/stages/01-resman/branch-networking.tf +++ b/fast/stages/01-resman/branch-networking.tf @@ -50,15 +50,16 @@ module "branch-network-prod-folder" { parent = module.branch-network-folder.id name = "Production" iam = { - "roles/compute.xpnAdmin" = compact([ + (local.custom_roles.service_project_network_admin) = compact([ try(module.branch-dp-prod-sa.0.iam_email, ""), try(module.branch-pf-prod-sa.0.iam_email, ""), - module.branch-gke-prod-sa.iam_email, + try(module.branch-gke-prod-sa.0.iam_email, ""), ]) } tag_bindings = { environment = try( - module.organization.tag_values["${var.tag_names.environment}/production"].id, null + module.organization.tag_values["${var.tag_names.environment}/production"].id, + null ) } } @@ -71,12 +72,13 @@ module "branch-network-dev-folder" { (local.custom_roles.service_project_network_admin) = compact([ try(module.branch-dp-dev-sa.0.iam_email, ""), try(module.branch-pf-dev-sa.0.iam_email, ""), - module.branch-gke-dev-sa.iam_email, + try(module.branch-gke-dev-sa.iam_email, ""), ]) } tag_bindings = { environment = try( - module.organization.tag_values["${var.tag_names.environment}/development"].id, null + module.organization.tag_values["${var.tag_names.environment}/development"].id, + null ) } } diff --git a/fast/stages/01-resman/organization.tf b/fast/stages/01-resman/organization.tf index 0de7193a4c..913a5f86e8 100644 --- a/fast/stages/01-resman/organization.tf +++ b/fast/stages/01-resman/organization.tf @@ -29,8 +29,8 @@ locals { branch_gke_sa_iam_emails = ( var.fast_features.gke ? [ - module.branch-gke-dev-sa.iam_email, - module.branch-gke-prod-sa.iam_email + module.branch-gke-dev-sa.0.iam_email, + module.branch-gke-prod-sa.0.iam_email ] : [] ) diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index e57a4ca84b..44152069e4 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -62,16 +62,16 @@ locals { } folder_ids = merge( { - data-platform-dev = try(module.branch-dp-dev-folder.0.id, null) - data-platform-prod = try(module.branch-dp-prod-folder.0.id, null) - gke-multitenant-dev = module.branch-gke-dev-folder.id - gke-multitenant-prod = module.branch-gke-prod-folder.id - networking = module.branch-network-folder.id - networking-dev = module.branch-network-dev-folder.id - networking-prod = module.branch-network-prod-folder.id - sandbox = try(module.branch-sandbox-folder.0.id, null) - security = module.branch-security-folder.id - teams = try(module.branch-teams-folder.0.id, null) + data-platform-dev = try(module.branch-dp-dev-folder.0.id, null) + data-platform-prod = try(module.branch-dp-prod-folder.0.id, null) + gke-dev = try(module.branch-gke-dev-folder.0.id, null) + gke-prod = try(module.branch-gke-prod-folder.0.id, null) + networking = module.branch-network-folder.id + networking-dev = module.branch-network-dev-folder.id + networking-prod = module.branch-network-prod-folder.id + sandbox = try(module.branch-sandbox-folder.0.id, null) + security = module.branch-security-folder.id + teams = try(module.branch-teams-folder.0.id, null) }, { for k, v in module.branch-teams-team-folder : @@ -98,16 +98,6 @@ locals { name = "security" sa = module.branch-security-sa.email }) - "03-gke-dev" = templatefile(local._tpl_providers, { - bucket = module.branch-gke-dev-gcs.name - name = "gke-dev" - sa = module.branch-gke-dev-sa.email - }) - "03-gke-prod" = templatefile(local._tpl_providers, { - bucket = module.branch-gke-prod-gcs.name - name = "gke-prod" - sa = module.branch-gke-prod-sa.email - }) }, !var.fast_features.data_platform ? {} : { "03-data-platform-dev" = templatefile(local._tpl_providers, { @@ -121,6 +111,18 @@ locals { sa = module.branch-dp-prod-sa.0.email }) }, + !var.fast_features.gke ? {} : { + "03-gke-dev" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-dev-gcs.0.name + name = "gke-dev" + sa = module.branch-gke-dev-sa.0.email + }) + "03-gke-prod" = templatefile(local._tpl_providers, { + bucket = module.branch-gke-prod-gcs.0.name + name = "gke-prod" + sa = module.branch-gke-prod-sa.0.email + }) + }, !var.fast_features.project_factory ? {} : { "03-project-factory-dev" = templatefile(local._tpl_providers, { bucket = module.branch-pf-dev-gcs.0.name @@ -252,18 +254,22 @@ output "security" { output "gke_multitenant" { # tfdoc:output:consumers 03-gke-multitenant description = "Data for the GKE multitenant stage." - value = { - "dev" = { - folder = module.branch-gke-dev-folder.id - gcs_bucket = module.branch-gke-dev-gcs.name - service_account = module.branch-gke-dev-sa.email - } - "prod" = { - folder = module.branch-gke-prod-folder.id - gcs_bucket = module.branch-gke-prod-gcs.name - service_account = module.branch-gke-prod-sa.email + value = ( + var.fast_features.gke + ? { + "dev" = { + folder = module.branch-gke-dev-folder.0.id + gcs_bucket = module.branch-gke-dev-gcs.0.name + service_account = module.branch-gke-dev-sa.0.email + } + "prod" = { + folder = module.branch-gke-prod-folder.0.id + gcs_bucket = module.branch-gke-prod-gcs.0.name + service_account = module.branch-gke-prod-sa.0.email + } } - } + : {} + ) } output "teams" { diff --git a/fast/stages/02-networking-nva/spoke-dev.tf b/fast/stages/02-networking-nva/spoke-dev.tf index f74fe8b79e..68d2645b65 100644 --- a/fast/stages/02-networking-nva/spoke-dev.tf +++ b/fast/stages/02-networking-nva/spoke-dev.tf @@ -126,7 +126,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, - local.service_accounts.gke-multitenant-dev, + local.service_accounts.gke-dev, ]) condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-nva/spoke-prod.tf b/fast/stages/02-networking-nva/spoke-prod.tf index 3e54419e5a..1edfdc754b 100644 --- a/fast/stages/02-networking-nva/spoke-prod.tf +++ b/fast/stages/02-networking-nva/spoke-prod.tf @@ -126,7 +126,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, - local.service_accounts.gke-multitenant-prod, + local.service_accounts.gke-prod, ]) condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-nva/variables.tf b/fast/stages/02-networking-nva/variables.tf index 2fe29d7853..594afd26db 100644 --- a/fast/stages/02-networking-nva/variables.tf +++ b/fast/stages/02-networking-nva/variables.tf @@ -209,8 +209,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string - gke-multitenant-dev = string - gke-multitenant-prod = string + gke-dev = string + gke-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md index 3bd20824d1..ae2ddf1792 100644 --- a/fast/stages/02-networking-peering/README.md +++ b/fast/stages/02-networking-peering/README.md @@ -151,8 +151,8 @@ There are two broad sets of variables you will need to fill in: To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files. -If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `*.auto.tfvars.json` files from this stage's folder in the path you specified. -The `*` above is set to the name of the stage that produced it, except for `globals.auto.tfvars.json` which is also generated by the bootstrap stage, containing global values compiled manually for the bootstrap stage. +If you have set a valid value for `outputs_location` in the bootstrap and in the resman stage, simply link the relevant `*.auto.tfvars.json` files from this stage's folder in the path you specified. +The `*` above is set to the name of the stage that produced it, except for `globals.auto.tfvars.json` which is also generated by the bootstrap stage, containing global values compiled manually for the bootstrap stage. For this stage, link the following files: ```bash @@ -302,7 +302,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [peering_configs](variables-peerings.tf#L19) | Peering configurations. | map(object({…})) | | {…} | | | [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | | [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-peering/spoke-dev.tf b/fast/stages/02-networking-peering/spoke-dev.tf index 1ae1be2d9a..19aaeba771 100644 --- a/fast/stages/02-networking-peering/spoke-dev.tf +++ b/fast/stages/02-networking-peering/spoke-dev.tf @@ -103,7 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, - local.service_accounts.gke-multitenant-dev, + local.service_accounts.gke-dev, ]) condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-peering/spoke-prod.tf b/fast/stages/02-networking-peering/spoke-prod.tf index 3b009daa3f..19a7903025 100644 --- a/fast/stages/02-networking-peering/spoke-prod.tf +++ b/fast/stages/02-networking-peering/spoke-prod.tf @@ -103,7 +103,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, - local.service_accounts.gke-multitenant-prod, + local.service_accounts.gke-prod, ]) condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-peering/variables.tf b/fast/stages/02-networking-peering/variables.tf index 77ef3884fc..018e398326 100644 --- a/fast/stages/02-networking-peering/variables.tf +++ b/fast/stages/02-networking-peering/variables.tf @@ -187,8 +187,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string - gke-multitenant-dev = string - gke-multitenant-prod = string + gke-dev = string + gke-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index 4f0b854095..32b144f4f5 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -327,7 +327,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | | [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | | [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | | [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | | diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index 1ae1be2d9a..19aaeba771 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -103,7 +103,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-dev, local.service_accounts.project-factory-dev, - local.service_accounts.gke-multitenant-dev, + local.service_accounts.gke-dev, ]) condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/02-networking-vpn/spoke-prod.tf index 3b009daa3f..19a7903025 100644 --- a/fast/stages/02-networking-vpn/spoke-prod.tf +++ b/fast/stages/02-networking-vpn/spoke-prod.tf @@ -103,7 +103,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = compact([ local.service_accounts.data-platform-prod, local.service_accounts.project-factory-prod, - local.service_accounts.gke-multitenant-prod, + local.service_accounts.gke-prod, ]) condition { title = "prod_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/variables.tf b/fast/stages/02-networking-vpn/variables.tf index 77ef3884fc..018e398326 100644 --- a/fast/stages/02-networking-vpn/variables.tf +++ b/fast/stages/02-networking-vpn/variables.tf @@ -187,8 +187,8 @@ variable "service_accounts" { type = object({ data-platform-dev = string data-platform-prod = string - gke-multitenant-dev = string - gke-multitenant-prod = string + gke-dev = string + gke-prod = string project-factory-dev = string project-factory-prod = string }) diff --git a/fast/stages/03-gke-multitenant/dev/gke-hub.tf b/fast/stages/03-gke-multitenant/dev/gke-hub.tf index 59a4ffce6d..b1e24a9f45 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-hub.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-hub.tf @@ -27,7 +27,7 @@ module "gke-hub" { config_sync = { gcp_service_account_email = null https_proxy = null - policy_dir = "fast/stages/03-gke-multitenant/config" + policy_dir = "fast/stages/03-gke/config" secret_type = "none" source_format = "hierarchy" sync_branch = "fast-dev-gke-marzi" diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index cc27ed9764..c4fad5deb5 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -25,7 +25,7 @@ module "gke-project-0" { source = "../../../../modules/project" billing_account = var.billing_account.id name = "dev-gke-clusters-0" - parent = var.folder_ids.gke-multitenant-dev + parent = var.folder_ids.gke-dev prefix = var.prefix labels = local.labels services = [ diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 608ad51b41..552dec5086 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -113,7 +113,7 @@ variable "folder_ids" { # tfdoc:variable:source 01-resman description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created." type = object({ - gke-multitenant-dev = string + gke-dev = string }) } From 9a3128e716132e95e87e9c3ef6be9ca821f04589 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Thu, 30 Jun 2022 20:20:14 +0200 Subject: [PATCH 20/75] wip --- .../stages/03-gke-multitenant/dev/gke-clusters.tf | 8 +++++--- fast/stages/03-gke-multitenant/dev/main.tf | 12 +----------- fast/stages/03-gke-multitenant/dev/variables.tf | 15 ++++++++------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf index 51ba711a45..bf3ee05957 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/dev/gke-clusters.tf @@ -74,7 +74,7 @@ module "gke-cluster" { monitoring_config = ["SYSTEM_COMPONENTS", "WORKLOADS"] # if you don't have compute.networks.updatePeering in the host - # project, comment out the next line and ask your network admin to + # project, comment the next lines and ask your network admin to # create the peering for you peering_config = { export_routes = true @@ -89,11 +89,13 @@ module "gke-cluster" { # in locals, merge defaults with cluster-level stuff # TODO(jccb): change fabric module database_encryption = ( - each.value.overrides.database_encryption_key == null ? { + each.value.overrides.database_encryption_key == null + ? { enabled = false state = null key_name = null - } : { + } + : { enabled = true state = "ENCRYPTED" key_name = each.value.overrides.database_encryption_key diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index c4fad5deb5..bbcf1ddb27 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -16,9 +16,6 @@ locals { labels = merge(var.labels, { environment = "dev" }) - - _gke_robot_sa = "serviceAccount:${module.gke-project-0.service_accounts.robots.container-engine}" - _cloud_services_sa = "serviceAccount:${module.gke-project-0.service_accounts.cloud_services}" } module "gke-project-0" { @@ -27,6 +24,7 @@ module "gke-project-0" { name = "dev-gke-clusters-0" parent = var.folder_ids.gke-dev prefix = var.prefix + group_iam = var.group_iam labels = local.labels services = [ "anthosconfigmanagement.googleapis.com", @@ -39,10 +37,6 @@ module "gke-project-0" { "multiclusteringress.googleapis.com", "trafficdirector.googleapis.com" ] - # add here any other service ids and keys for robot accounts which are needed - # service_encryption_key_ids = { - # container = var.project_config.service_encryption_key_ids - # } shared_vpc_service_config = { attach = true host_project = var.host_project_ids.dev-spoke-0 @@ -55,7 +49,6 @@ module "gke-project-0" { ] } } - # specify project-level org policies here if you need them # policy_boolean = { # "constraints/compute.disableGuestAttributesAccess" = true @@ -68,9 +61,6 @@ module "gke-project-0" { # values = ["projects/fl01-prod-iac-core-0"] # } # } - iam = { - "roles/container.clusterViewer" = var.cluster_viewer_users - } } module "gke-dataset-resource-usage" { diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 552dec5086..083cbb74ca 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -97,14 +97,8 @@ variable "clusters" { })) } -variable "cluster_viewer_users" { - description = "list of users to be granted as container.clusterViewer" - type = list(any) - default = [] -} - variable "dns_domain" { - description = "Domain name used for clusters, prefix by each cluster name. Leave null to disable Cloud DNS for GKE." + description = "Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE." type = string default = null } @@ -125,6 +119,13 @@ variable "host_project_ids" { }) } +variable "group_iam" { + description = "Project-level IAM bindings for groups. Use group emails as keys, list of roles as values." + type = map(list(string)) + default = {} + nullable = false +} + variable "labels" { description = "Project-level labels." type = map(string) From a3ea8bed8b540e56b23009415299c0bc90c846f3 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 5 Jul 2022 10:07:50 +0300 Subject: [PATCH 21/75] wip --- fast/stages/03-gke-multitenant/main.tf | 23 +++++++++++++++++++ .../{dev => module}/README.md | 0 .../{dev => module}/gke-clusters.tf | 0 .../{dev => module}/gke-hub.tf | 0 .../{dev => module}/gke-nodepools.tf | 2 +- .../{dev => module}/main.tf | 13 +++++++---- .../{dev => module}/outputs.tf | 0 .../{dev => module}/variables.tf | 0 modules/gke-hub/README.md | 17 +++++++------- 9 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 fast/stages/03-gke-multitenant/main.tf rename fast/stages/03-gke-multitenant/{dev => module}/README.md (100%) rename fast/stages/03-gke-multitenant/{dev => module}/gke-clusters.tf (100%) rename fast/stages/03-gke-multitenant/{dev => module}/gke-hub.tf (100%) rename fast/stages/03-gke-multitenant/{dev => module}/gke-nodepools.tf (98%) rename fast/stages/03-gke-multitenant/{dev => module}/main.tf (88%) rename fast/stages/03-gke-multitenant/{dev => module}/outputs.tf (100%) rename fast/stages/03-gke-multitenant/{dev => module}/variables.tf (100%) diff --git a/fast/stages/03-gke-multitenant/main.tf b/fast/stages/03-gke-multitenant/main.tf new file mode 100644 index 0000000000..86d2e9ea7f --- /dev/null +++ b/fast/stages/03-gke-multitenant/main.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_container_cluster" "cluster" { + provider = google-beta + project = "tf-playground-svpc-gke" + name = "cluster1" + location = "europe-west1-b" + initial_node_count = 1 +} diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/module/README.md similarity index 100% rename from fast/stages/03-gke-multitenant/dev/README.md rename to fast/stages/03-gke-multitenant/module/README.md diff --git a/fast/stages/03-gke-multitenant/dev/gke-clusters.tf b/fast/stages/03-gke-multitenant/module/gke-clusters.tf similarity index 100% rename from fast/stages/03-gke-multitenant/dev/gke-clusters.tf rename to fast/stages/03-gke-multitenant/module/gke-clusters.tf diff --git a/fast/stages/03-gke-multitenant/dev/gke-hub.tf b/fast/stages/03-gke-multitenant/module/gke-hub.tf similarity index 100% rename from fast/stages/03-gke-multitenant/dev/gke-hub.tf rename to fast/stages/03-gke-multitenant/module/gke-hub.tf diff --git a/fast/stages/03-gke-multitenant/dev/gke-nodepools.tf b/fast/stages/03-gke-multitenant/module/gke-nodepools.tf similarity index 98% rename from fast/stages/03-gke-multitenant/dev/gke-nodepools.tf rename to fast/stages/03-gke-multitenant/module/gke-nodepools.tf index 52febb9b20..703c9d71e7 100644 --- a/fast/stages/03-gke-multitenant/dev/gke-nodepools.tf +++ b/fast/stages/03-gke-multitenant/module/gke-nodepools.tf @@ -27,7 +27,7 @@ locals { ]...) } -module "gke-1-nodepool" { +module "gke-nodepool" { source = "../../../../modules/gke-nodepool" for_each = local.nodepools name = each.value.name diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/module/main.tf similarity index 88% rename from fast/stages/03-gke-multitenant/dev/main.tf rename to fast/stages/03-gke-multitenant/module/main.tf index bbcf1ddb27..9b8eecf418 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/module/main.tf @@ -28,14 +28,17 @@ module "gke-project-0" { labels = local.labels services = [ "anthosconfigmanagement.googleapis.com", - "anthos.googleapis.com", "dns.googleapis.com", + # "anthos.googleapis.com", + "cloudresourcemanager.googleapis.com", + "container.googleapis.com", + "dns.googleapis.com", "gkeconnect.googleapis.com", "gkehub.googleapis.com", + "iam.googleapis.com", + # "multiclusteringress.googleapis.com", + # "multiclusterservicediscovery.googleapis.com", "stackdriver.googleapis.com", - "container.googleapis.com", - "multiclusterservicediscovery.googleapis.com", - "multiclusteringress.googleapis.com", - "trafficdirector.googleapis.com" + # "trafficdirector.googleapis.com" ] shared_vpc_service_config = { attach = true diff --git a/fast/stages/03-gke-multitenant/dev/outputs.tf b/fast/stages/03-gke-multitenant/module/outputs.tf similarity index 100% rename from fast/stages/03-gke-multitenant/dev/outputs.tf rename to fast/stages/03-gke-multitenant/module/outputs.tf diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/module/variables.tf similarity index 100% rename from fast/stages/03-gke-multitenant/dev/variables.tf rename to fast/stages/03-gke-multitenant/module/variables.tf diff --git a/modules/gke-hub/README.md b/modules/gke-hub/README.md index 0e3acb939b..992ac39a52 100644 --- a/modules/gke-hub/README.md +++ b/modules/gke-hub/README.md @@ -3,13 +3,12 @@ This module allows simplified creation and management of a GKE Hub object and its features for a given set of clusters. The given list of clusters will be registered inside the Hub and all the configured features will be activated. To use this module you must ensure the following APIs are enabled in the target project: -``` -"gkehub.googleapis.com" -"gkeconnect.googleapis.com" -"anthosconfigmanagement.googleapis.com" -"multiclusteringress.googleapis.com" -"multiclusterservicediscovery.googleapis.com" -``` + +- `anthosconfigmanagement.googleapis.com` +- `gkeconnect.googleapis.com` +- `gkehub.googleapis.com` +- `multiclusteringress.googleapis.com` +- `multiclusterservicediscovery.googleapis.com` ## Full GKE Hub example @@ -20,10 +19,10 @@ module "project" { name = "gkehub-test" parent = "folders/12345" services = [ + "anthosconfigmanagement.googleapis.com", "container.googleapis.com", - "gkehub.googleapis.com", "gkeconnect.googleapis.com", - "anthosconfigmanagement.googleapis.com", + "gkehub.googleapis.com", "multiclusteringress.googleapis.com", "multiclusterservicediscovery.googleapis.com", ] From e7bc11e6b9160e51358d897bbce171ef4e98f253 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 12 Jul 2022 12:10:39 +0200 Subject: [PATCH 22/75] Add gke SAs to outputs --- fast/stages/01-resman/outputs.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fast/stages/01-resman/outputs.tf b/fast/stages/01-resman/outputs.tf index 44152069e4..f5ccc6bab5 100644 --- a/fast/stages/01-resman/outputs.tf +++ b/fast/stages/01-resman/outputs.tf @@ -27,6 +27,7 @@ locals { tf_providers_file = "03-data-platform-prod-providers.tf" tf_var_files = local.cicd_workflow_var_files.stage_3 } + # TODO(jccb): add gke here networking = { service_account = try(module.branch-network-sa-cicd.0.email, null) tf_providers_file = "02-networking-providers.tf" @@ -147,6 +148,8 @@ locals { { data-platform-dev = try(module.branch-dp-dev-sa.0.email, null) data-platform-prod = try(module.branch-dp-prod-sa.0.email, null) + gke-dev = try(module.branch-gke-dev-sa.0.email, null) + gke-prod = try(module.branch-gke-prod-sa.0.email, null) networking = module.branch-network-sa.email project-factory-dev = try(module.branch-pf-dev-sa.0.email, null) project-factory-prod = try(module.branch-pf-prod-sa.0.email, null) From 1bc352bb7b0b129238d871ccc05412f073728550 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 12 Jul 2022 12:19:05 +0200 Subject: [PATCH 23/75] Fix docs --- fast/stages/01-resman/README.md | 20 ++++++++++---------- fast/stages/02-networking-nva/README.md | 2 +- fast/stages/02-networking-peering/README.md | 2 +- fast/stages/02-networking-vpn/README.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fast/stages/01-resman/README.md b/fast/stages/01-resman/README.md index 8f8bee196a..ee763d28c6 100644 --- a/fast/stages/01-resman/README.md +++ b/fast/stages/01-resman/README.md @@ -196,15 +196,15 @@ Due to its simplicity, this stage lends itself easily to customizations: adding | name | description | sensitive | consumers | |---|---|:---:|---| -| [cicd_repositories](outputs.tf#L166) | WIF configuration for CI/CD repositories. | | | -| [dataplatform](outputs.tf#L180) | Data for the Data Platform stage. | | | -| [gke_multitenant](outputs.tf#L252) | Data for the GKE multitenant stage. | | 03-gke-multitenant | -| [networking](outputs.tf#L196) | Data for the networking stage. | | | -| [project_factories](outputs.tf#L205) | Data for the project factories stage. | | | -| [providers](outputs.tf#L221) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | -| [sandbox](outputs.tf#L228) | Data for the sandbox stage. | | xx-sandbox | -| [security](outputs.tf#L242) | Data for the networking stage. | | 02-security | -| [teams](outputs.tf#L269) | Data for the teams stage. | | | -| [tfvars](outputs.tf#L282) | Terraform variable files for the following stages. | ✓ | | +| [cicd_repositories](outputs.tf#L171) | WIF configuration for CI/CD repositories. | | | +| [dataplatform](outputs.tf#L185) | Data for the Data Platform stage. | | | +| [gke_multitenant](outputs.tf#L257) | Data for the GKE multitenant stage. | | 03-gke-multitenant | +| [networking](outputs.tf#L201) | Data for the networking stage. | | | +| [project_factories](outputs.tf#L210) | Data for the project factories stage. | | | +| [providers](outputs.tf#L226) | Terraform provider files for this stage and dependent stages. | ✓ | 02-networking · 02-security · 03-dataplatform · xx-sandbox · xx-teams | +| [sandbox](outputs.tf#L233) | Data for the sandbox stage. | | xx-sandbox | +| [security](outputs.tf#L247) | Data for the networking stage. | | 02-security | +| [teams](outputs.tf#L278) | Data for the teams stage. | | | +| [tfvars](outputs.tf#L291) | Terraform variable files for the following stages. | ✓ | | diff --git a/fast/stages/02-networking-nva/README.md b/fast/stages/02-networking-nva/README.md index b290880da0..dbf04a766f 100644 --- a/fast/stages/02-networking-nva/README.md +++ b/fast/stages/02-networking-nva/README.md @@ -378,7 +378,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [outputs_location](variables.tf#L125) | Path where providers and tfvars files for the following stages are written. Leave empty to disable. | string | | null | | | [psa_ranges](variables.tf#L142) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_configs](variables.tf#L183) | Configurations for CRs and onprem routers. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L206) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [service_accounts](variables.tf#L206) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | | [vpn_onprem_configs](variables.tf#L220) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-peering/README.md b/fast/stages/02-networking-peering/README.md index ae2ddf1792..351af478a3 100644 --- a/fast/stages/02-networking-peering/README.md +++ b/fast/stages/02-networking-peering/README.md @@ -302,7 +302,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [peering_configs](variables-peerings.tf#L19) | Peering configurations. | map(object({…})) | | {…} | | | [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | | [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | ## Outputs diff --git a/fast/stages/02-networking-vpn/README.md b/fast/stages/02-networking-vpn/README.md index 32b144f4f5..36cfc7e9d9 100644 --- a/fast/stages/02-networking-vpn/README.md +++ b/fast/stages/02-networking-vpn/README.md @@ -327,7 +327,7 @@ DNS configurations are centralised in the `dns-*.tf` files. Spokes delegate DNS | [psa_ranges](variables.tf#L129) | IP ranges used for Private Service Access (e.g. CloudSQL). | object({…}) | | null | | | [router_onprem_configs](variables.tf#L166) | Configurations for routers used for onprem connectivity. | map(object({…})) | | {…} | | | [router_spoke_configs](variables-vpn.tf#L18) | Configurations for routers used for internal connectivity. | map(object({…})) | | {…} | | -| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | +| [service_accounts](variables.tf#L184) | Automation service accounts in name => email format. | object({…}) | | null | 01-resman | | [vpn_onprem_configs](variables.tf#L198) | VPN gateway configuration for onprem interconnection. | map(object({…})) | | {…} | | | [vpn_spoke_configs](variables-vpn.tf#L37) | VPN gateway configuration for spokes. | map(object({…})) | | {…} | | From 07c5da4283db6a86f3997ac69cf4523193448d10 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 12 Jul 2022 12:39:00 +0200 Subject: [PATCH 24/75] Fix tests --- fast/stages/02-networking-vpn/spoke-dev.tf | 2 +- fast/stages/02-networking-vpn/spoke-prod.tf | 2 +- tests/fast/stages/s02_networking_nva/fixture/main.tf | 4 ++-- tests/fast/stages/s02_networking_peering/fixture/main.tf | 4 ++-- tests/fast/stages/s02_networking_vpn/fixture/main.tf | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index 5dea40f4a3..c0749bf9d9 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -105,7 +105,7 @@ resource "google_project_iam_binding" "dev_spoke_project_iam_delegated" { members = compact([ try(local.service_accounts.data-platform-dev, null), try(local.service_accounts.project-factory-dev, null), - try(local.service_accounts.gke-dev, null) + try(local.service_accounts.gke-dev, null), ]) condition { title = "dev_stage3_sa_delegated_grants" diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/02-networking-vpn/spoke-prod.tf index c40600c2c7..768a2012f0 100644 --- a/fast/stages/02-networking-vpn/spoke-prod.tf +++ b/fast/stages/02-networking-vpn/spoke-prod.tf @@ -105,7 +105,7 @@ resource "google_project_iam_binding" "prod_spoke_project_iam_delegated" { members = compact([ try(local.service_accounts.data-platform-prod, null), try(local.service_accounts.project-factory-prod, null), - tru(local.service_accounts.gke-prod, null), + try(local.service_accounts.gke-prod, null), ]) condition { title = "prod_stage3_sa_delegated_grants" diff --git a/tests/fast/stages/s02_networking_nva/fixture/main.tf b/tests/fast/stages/s02_networking_nva/fixture/main.tf index b34da60a7f..2fc7480144 100644 --- a/tests/fast/stages/s02_networking_nva/fixture/main.tf +++ b/tests/fast/stages/s02_networking_nva/fixture/main.tf @@ -35,8 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" - gke-multitenant-dev = "string" - gke-multitenant-prod = "string" + gke-dev = "string" + gke-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } diff --git a/tests/fast/stages/s02_networking_peering/fixture/main.tf b/tests/fast/stages/s02_networking_peering/fixture/main.tf index 704c71a318..b7a0577f63 100644 --- a/tests/fast/stages/s02_networking_peering/fixture/main.tf +++ b/tests/fast/stages/s02_networking_peering/fixture/main.tf @@ -35,8 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" - gke-multitenant-dev = "string" - gke-multitenant-prod = "string" + gke-dev = "string" + gke-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } diff --git a/tests/fast/stages/s02_networking_vpn/fixture/main.tf b/tests/fast/stages/s02_networking_vpn/fixture/main.tf index 8c1da847ef..a3c1885ea1 100644 --- a/tests/fast/stages/s02_networking_vpn/fixture/main.tf +++ b/tests/fast/stages/s02_networking_vpn/fixture/main.tf @@ -35,8 +35,8 @@ module "stage" { service_accounts = { data-platform-dev = "string" data-platform-prod = "string" - gke-multitenant-dev = "string" - gke-multitenant-prod = "string" + gke-dev = "string" + gke-prod = "string" project-factory-dev = "string" project-factory-prod = "string" } From 1260db923e8179de72cf63b79f1381791b30885f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 29 Jul 2022 10:49:50 +0200 Subject: [PATCH 25/75] Update main.tf --- fast/stages/03-gke-multitenant/module/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fast/stages/03-gke-multitenant/module/main.tf b/fast/stages/03-gke-multitenant/module/main.tf index 9b8eecf418..227d1972ec 100644 --- a/fast/stages/03-gke-multitenant/module/main.tf +++ b/fast/stages/03-gke-multitenant/module/main.tf @@ -26,14 +26,15 @@ module "gke-project-0" { prefix = var.prefix group_iam = var.group_iam labels = local.labels + # TODO: depend GKE hub services on GKE hub variable/activation services = [ - "anthosconfigmanagement.googleapis.com", + # "anthosconfigmanagement.googleapis.com", # "anthos.googleapis.com", "cloudresourcemanager.googleapis.com", "container.googleapis.com", "dns.googleapis.com", - "gkeconnect.googleapis.com", - "gkehub.googleapis.com", + # "gkeconnect.googleapis.com", + # "gkehub.googleapis.com", "iam.googleapis.com", # "multiclusteringress.googleapis.com", # "multiclusterservicediscovery.googleapis.com", From 133fd078232ef202140450d921bb8018b60e700f Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 29 Jul 2022 11:31:34 +0200 Subject: [PATCH 26/75] unfinished gke hub IAM --- .../03-gke-multitenant/module/gke-hub.tf | 52 +++++-------- fast/stages/03-gke-multitenant/module/main.tf | 8 +- .../03-gke-multitenant/module/variables.tf | 77 +++++++++++++++++-- modules/project/service-accounts.tf | 17 ++-- 4 files changed, 109 insertions(+), 45 deletions(-) diff --git a/fast/stages/03-gke-multitenant/module/gke-hub.tf b/fast/stages/03-gke-multitenant/module/gke-hub.tf index b1e24a9f45..a037eba59a 100644 --- a/fast/stages/03-gke-multitenant/module/gke-hub.tf +++ b/fast/stages/03-gke-multitenant/module/gke-hub.tf @@ -14,41 +14,31 @@ * limitations under the License. */ +# TODO: service account +# https://cloud.google.com/kubernetes-engine/docs/how-to/msc-setup-with-shared-vpc-networks#shared-service-project-iam +# TODO: add roles/multiclusterservicediscovery.serviceAgent and +# roles/compute.networkViewer to IAM condition for GKE stage SA + +locals { + fleet_enabled = ( + var.fleet_features != null || var.fleet_workload_identity + ) + # TODO: add condition + fleet_mcs_enabled = false +} + module "gke-hub" { source = "../../../../modules/gke-hub" + count = local.fleet_enabled ? 1 : 0 project_id = module.gke-project-0.project_id - member_clusters = { + clusters = { for cluster_id in keys(var.clusters) : cluster_id => module.gke-cluster[cluster_id].id } - member_features = { - configmanagement = { - binauthz = false - config_sync = { - gcp_service_account_email = null - https_proxy = null - policy_dir = "fast/stages/03-gke/config" - secret_type = "none" - source_format = "hierarchy" - sync_branch = "fast-dev-gke-marzi" - sync_repo = "https://github.com/GoogleCloudPlatform/cloud-foundation-fabric" - sync_rev = null - } - hierarchy_controller = null - policy_controller = { - exemptable_namespaces = [ - "asm-system", - "config-management-system", - "config-management-monitoring", - "gatekeeper-system", - "kube-system", - "cos-auditd" - ] - log_denies_enabled = true - referential_rules_enabled = false - template_library_installed = true - } - version = "1.10.2" - } - } + features = var.fleet_features + configmanagement_templates = var.fleet_configmanagement_templates + configmanagement_clusters = var.fleet_configmanagement_clusters + workload_identity_clusters = ( + var.fleet_workload_identity ? keys(var.clusters) : [] + ) } diff --git a/fast/stages/03-gke-multitenant/module/main.tf b/fast/stages/03-gke-multitenant/module/main.tf index 227d1972ec..626c36b482 100644 --- a/fast/stages/03-gke-multitenant/module/main.tf +++ b/fast/stages/03-gke-multitenant/module/main.tf @@ -44,14 +44,18 @@ module "gke-project-0" { shared_vpc_service_config = { attach = true host_project = var.host_project_ids.dev-spoke-0 - service_identity_iam = { + service_identity_iam = merge({ "roles/compute.networkUser" = [ "cloudservices", "container-engine" ] "roles/container.hostServiceAgentUser" = [ "container-engine" ] - } + }, + !local.fleet_mcs_enabled ? {} : { + "roles/multiclusterservicediscovery.serviceAgent" = ["gke-mcs"] + "roles/compute.networkViewer" = ["gke-mcs-importer"] + }) } # specify project-level org policies here if you need them # policy_boolean = { diff --git a/fast/stages/03-gke-multitenant/module/variables.tf b/fast/stages/03-gke-multitenant/module/variables.tf index 083cbb74ca..1df64622a1 100644 --- a/fast/stages/03-gke-multitenant/module/variables.tf +++ b/fast/stages/03-gke-multitenant/module/variables.tf @@ -103,6 +103,69 @@ variable "dns_domain" { default = null } +variable "fleet_configmanagement_clusters" { + description = "Config management features enabled on specific sets of member clusters, in config name => [cluster name] format." + type = map(list(string)) + default = {} + nullable = false +} + + +variable "fleet_configmanagement_templates" { + description = "Sets of config management configurations that can be applied to member clusters, in config name => {options} format." + type = map(object({ + binauthz = bool + config_sync = object({ + git = object({ + gcp_service_account_email = string + https_proxy = string + policy_dir = string + secret_type = string + sync_branch = string + sync_repo = string + sync_rev = string + sync_wait_secs = number + }) + prevent_drift = string + source_format = string + }) + hierarchy_controller = object({ + enable_hierarchical_resource_quota = bool + enable_pod_tree_labels = bool + }) + policy_controller = object({ + audit_interval_seconds = number + exemptable_namespaces = list(string) + log_denies_enabled = bool + referential_rules_enabled = bool + template_library_installed = bool + }) + version = string + })) + default = {} + nullable = false +} + +variable "fleet_features" { + description = "Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used." + type = object({ + appdevexperience = bool + configmanagement = bool + identityservice = bool + multiclusteringress = string + multiclusterservicediscovery = bool + servicemesh = bool + }) + default = null +} + +variable "fleet_workload_identity" { + description = "Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true." + type = bool + default = true + nullable = false +} + variable "folder_ids" { # tfdoc:variable:source 01-resman description = "Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created." @@ -111,6 +174,13 @@ variable "folder_ids" { }) } +variable "group_iam" { + description = "Project-level IAM bindings for groups. Use group emails as keys, list of roles as values." + type = map(list(string)) + default = {} + nullable = false +} + variable "host_project_ids" { # tfdoc:variable:source 02-networking description = "Host project for the shared VPC." @@ -119,13 +189,6 @@ variable "host_project_ids" { }) } -variable "group_iam" { - description = "Project-level IAM bindings for groups. Use group emails as keys, list of roles as values." - type = map(list(string)) - default = {} - nullable = false -} - variable "labels" { description = "Project-level labels." type = map(string) diff --git a/modules/project/service-accounts.tf b/modules/project/service-accounts.tf index af3873644e..7d584fa715 100644 --- a/modules/project/service-accounts.tf +++ b/modules/project/service-accounts.tf @@ -40,7 +40,9 @@ locals { fleet = "service-%s@gcp-sa-gkehub" gae-flex = "service-%s@gae-api-prod" # TODO: deprecate gcf - gcf = "service-%s@gcf-admin-robot" + gcf = "service-%s@gcf-admin-robot" + # TODO: jit? + gke-mcs = "service-%s@gcp-sa-mcsd" monitoring-notifications = "service-%s@gcp-sa-monitoring-notification" pubsub = "service-%s@gcp-sa-pubsub" secretmanager = "service-%s@gcp-sa-secretmanager" @@ -55,10 +57,15 @@ locals { service_account_cloud_services = ( "${local.project.number}@cloudservices.gserviceaccount.com" ) - service_accounts_robots = { - for k, v in local._service_accounts_robot_services : - k => "${format(v, local.project.number)}.iam.gserviceaccount.com" - } + service_accounts_robots = merge( + { + for k, v in local._service_accounts_robot_services : + k => "${format(v, local.project.number)}.iam.gserviceaccount.com" + }, + { + gke-mcs-importer = "${local.project.project_id}.svc.id.goog[gke-mcs/gke-mcs-importer]" + } + ) service_accounts_jit_services = [ "cloudasset.googleapis.com", "gkehub.googleapis.com", From c24e66138339bb5c599e52cd88236267acac4611 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 29 Jul 2022 14:01:35 +0200 Subject: [PATCH 27/75] fleet mcs service accounts --- .../03-gke-multitenant/module/gke-hub.tf | 10 ++---- fast/stages/03-gke-multitenant/module/main.tf | 33 ++++++++++--------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/fast/stages/03-gke-multitenant/module/gke-hub.tf b/fast/stages/03-gke-multitenant/module/gke-hub.tf index a037eba59a..8979605cba 100644 --- a/fast/stages/03-gke-multitenant/module/gke-hub.tf +++ b/fast/stages/03-gke-multitenant/module/gke-hub.tf @@ -14,17 +14,13 @@ * limitations under the License. */ -# TODO: service account -# https://cloud.google.com/kubernetes-engine/docs/how-to/msc-setup-with-shared-vpc-networks#shared-service-project-iam -# TODO: add roles/multiclusterservicediscovery.serviceAgent and -# roles/compute.networkViewer to IAM condition for GKE stage SA - locals { fleet_enabled = ( var.fleet_features != null || var.fleet_workload_identity ) - # TODO: add condition - fleet_mcs_enabled = false + fleet_mcs_enabled = local.fleet_enabled && lookup( + coalesce(var.fleet_features, {}), "multiclusterservicediscovery", false + ) == true } module "gke-hub" { diff --git a/fast/stages/03-gke-multitenant/module/main.tf b/fast/stages/03-gke-multitenant/module/main.tf index 626c36b482..5f112dcdfc 100644 --- a/fast/stages/03-gke-multitenant/module/main.tf +++ b/fast/stages/03-gke-multitenant/module/main.tf @@ -26,21 +26,24 @@ module "gke-project-0" { prefix = var.prefix group_iam = var.group_iam labels = local.labels - # TODO: depend GKE hub services on GKE hub variable/activation - services = [ - # "anthosconfigmanagement.googleapis.com", - # "anthos.googleapis.com", - "cloudresourcemanager.googleapis.com", - "container.googleapis.com", - "dns.googleapis.com", - # "gkeconnect.googleapis.com", - # "gkehub.googleapis.com", - "iam.googleapis.com", - # "multiclusteringress.googleapis.com", - # "multiclusterservicediscovery.googleapis.com", - "stackdriver.googleapis.com", - # "trafficdirector.googleapis.com" - ] + services = concat( + [ + "cloudresourcemanager.googleapis.com", + "container.googleapis.com", + "dns.googleapis.com", + "iam.googleapis.com", + "stackdriver.googleapis.com", + ], + !local.fleet_enabled ? [] : [ + "anthosconfigmanagement.googleapis.com", + "anthos.googleapis.com", + "gkeconnect.googleapis.com", + "gkehub.googleapis.com", + "multiclusteringress.googleapis.com", + "multiclusterservicediscovery.googleapis.com", + "trafficdirector.googleapis.com" + ] + ) shared_vpc_service_config = { attach = true host_project = var.host_project_ids.dev-spoke-0 From 7c23aeae58a32b0ac6aa056d7e566c00e9e65cdf Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 29 Jul 2022 15:09:57 +0200 Subject: [PATCH 28/75] implement dev, fix variables, add stub READMEs --- fast/stages/03-data-platform/dev/main.tf | 2 +- fast/stages/03-gke-multitenant/README.md | 10 + .../03-gke-multitenant/_module/README.md | 68 ++++++ .../{module => _module}/gke-clusters.tf | 4 +- .../{module => _module}/gke-hub.tf | 0 .../{module => _module}/gke-nodepools.tf | 0 .../{module => _module}/main.tf | 14 +- .../system/repo.yml => _module/outputs.tf} | 27 ++- .../03-gke-multitenant/_module/variables.tf | 229 ++++++++++++++++++ .../config/cluster/README.md | 3 - .../config/cluster/clusterrole-ns-viewer.yml | 27 --- ...-have-geo-constraint_v1beta_gatekeeper.yml | 42 ---- .../config/cluster/pod_priority_classes.yml | 47 ---- .../config/namespaces/README.md | 7 - .../apps_v1_configmap_cos-auditd.yml | 53 ---- .../apps_v1_daemonset_cos-auditd.yml | 128 ---------- .../namespaces/cos-auditd/namespace.yml | 18 -- .../config/namespaces/teams/limit-range.yml | 34 --- .../network-policy-default-deny-egress.yml | 22 -- .../namespaces/teams/team-a/namespace.yml | 18 -- .../namespaces/teams/team-a/repo-sync.yml | 30 --- .../teams/team-a/resourcequotas.yml | 35 --- .../teams/team-a/rolebinding-ns-viewer.yml | 27 --- .../teams/team-a/sync-rolebinding.yml | 28 --- .../teams/team-b/apps_v1_deployment_v1.yml | 44 ---- .../namespaces/teams/team-b/namespace.yml | 20 -- fast/stages/03-gke-multitenant/dev/README.md | 74 ++++++ fast/stages/03-gke-multitenant/dev/main.tf | 40 +++ fast/stages/03-gke-multitenant/dev/outputs.tf | 70 ++++++ .../{module => dev}/variables.tf | 0 fast/stages/03-gke-multitenant/main.tf | 23 -- .../03-gke-multitenant/module/README.md | 108 --------- .../03-gke-multitenant/module/outputs.tf | 15 -- 33 files changed, 517 insertions(+), 750 deletions(-) create mode 100644 fast/stages/03-gke-multitenant/README.md create mode 100644 fast/stages/03-gke-multitenant/_module/README.md rename fast/stages/03-gke-multitenant/{module => _module}/gke-clusters.tf (97%) rename fast/stages/03-gke-multitenant/{module => _module}/gke-hub.tf (100%) rename fast/stages/03-gke-multitenant/{module => _module}/gke-nodepools.tf (100%) rename fast/stages/03-gke-multitenant/{module => _module}/main.tf (89%) rename fast/stages/03-gke-multitenant/{config/system/repo.yml => _module/outputs.tf} (56%) create mode 100644 fast/stages/03-gke-multitenant/_module/variables.tf delete mode 100644 fast/stages/03-gke-multitenant/config/cluster/README.md delete mode 100644 fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml delete mode 100644 fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml delete mode 100644 fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/README.md delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml delete mode 100644 fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml create mode 100644 fast/stages/03-gke-multitenant/dev/README.md create mode 100644 fast/stages/03-gke-multitenant/dev/main.tf create mode 100644 fast/stages/03-gke-multitenant/dev/outputs.tf rename fast/stages/03-gke-multitenant/{module => dev}/variables.tf (100%) delete mode 100644 fast/stages/03-gke-multitenant/main.tf delete mode 100644 fast/stages/03-gke-multitenant/module/README.md delete mode 100644 fast/stages/03-gke-multitenant/module/outputs.tf diff --git a/fast/stages/03-data-platform/dev/main.tf b/fast/stages/03-data-platform/dev/main.tf index f26a528856..6f33e7d5f7 100644 --- a/fast/stages/03-data-platform/dev/main.tf +++ b/fast/stages/03-data-platform/dev/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -# tfdoc:file:description Data Platformy. +# tfdoc:file:description Data Platform. module "data-platform" { source = "../../../../examples/data-solutions/data-platform-foundations" diff --git a/fast/stages/03-gke-multitenant/README.md b/fast/stages/03-gke-multitenant/README.md new file mode 100644 index 0000000000..93473f7d69 --- /dev/null +++ b/fast/stages/03-gke-multitenant/README.md @@ -0,0 +1,10 @@ +# GKE Multitenant stage + +TODO: one paragraph description + +This directory contains + +- a reference [`dev` environment](./dev/) that can be used as-is, and cloned with few changes to implement further environments +- the [GKE stage "module"](./_module) that implements the underlying stage and is then wrapped for specific environments or configurations like `dev` above + +Refer to the example [`dev/README.md`](./dev/README.md) for configuration details. diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md new file mode 100644 index 0000000000..e18762582b --- /dev/null +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -0,0 +1,68 @@ +# GKE Multitenant Module + +TODO: add description and diagram + +

+ GKE multitenant +

+ +## Design overview and choices + +TODO + +### Cluster and nodepool configuration + +TODO + +### Fleet management + +TODO + +### Variable configuration + +TODO + + + + +## Files + +| name | description | modules | +|---|---|---| +| [gke-clusters.tf](./gke-clusters.tf) | None | gke-cluster | +| [gke-hub.tf](./gke-hub.tf) | None | gke-hub | +| [gke-nodepools.tf](./gke-nodepools.tf) | None | gke-nodepool | +| [main.tf](./main.tf) | Module-level locals and resources. | bigquery-dataset · project | +| [outputs.tf](./outputs.tf) | Output variables. | | +| [variables.tf](./variables.tf) | Module variables. | | + +## Variables + +| name | description | type | required | default | producer | +|---|---|:---:|:---:|:---:|:---:| +| [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | +| [clusters](variables.tf#L63) | | map(object({…})) | ✓ | | | +| [folder_id](variables.tf#L165) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | +| [nodepools](variables.tf#L201) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L218) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_config](variables.tf#L223) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | +| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L96) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L102) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L110) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L145) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L158) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L170) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [labels](variables.tf#L177) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L183) | | object({…}) | | {…} | | + +## Outputs + +| name | description | sensitive | consumers | +|---|---|:---:|---| +| [cluster_ids](outputs.tf#L22) | Cluster ids. | | | +| [clusters](outputs.tf#L17) | Cluster resources. | | | +| [project_id](outputs.tf#L29) | GKE project id. | | | + + diff --git a/fast/stages/03-gke-multitenant/module/gke-clusters.tf b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf similarity index 97% rename from fast/stages/03-gke-multitenant/module/gke-clusters.tf rename to fast/stages/03-gke-multitenant/_module/gke-clusters.tf index bf3ee05957..4a968aee39 100644 --- a/fast/stages/03-gke-multitenant/module/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf @@ -30,7 +30,7 @@ module "gke-cluster" { project_id = module.gke-project-0.project_id description = each.value.description location = each.value.location - network = var.vpc_self_links.dev-spoke-0 + network = var.vpc_config.vpc_self_link subnetwork = each.value.net.subnet secondary_range_pods = each.value.net.pods secondary_range_services = each.value.net.services @@ -79,7 +79,7 @@ module "gke-cluster" { peering_config = { export_routes = true import_routes = false - project_id = var.host_project_ids.dev-spoke-0 + project_id = var.vpc_config.host_project_id } resource_usage_export_config = { enabled = true diff --git a/fast/stages/03-gke-multitenant/module/gke-hub.tf b/fast/stages/03-gke-multitenant/_module/gke-hub.tf similarity index 100% rename from fast/stages/03-gke-multitenant/module/gke-hub.tf rename to fast/stages/03-gke-multitenant/_module/gke-hub.tf diff --git a/fast/stages/03-gke-multitenant/module/gke-nodepools.tf b/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf similarity index 100% rename from fast/stages/03-gke-multitenant/module/gke-nodepools.tf rename to fast/stages/03-gke-multitenant/_module/gke-nodepools.tf diff --git a/fast/stages/03-gke-multitenant/module/main.tf b/fast/stages/03-gke-multitenant/_module/main.tf similarity index 89% rename from fast/stages/03-gke-multitenant/module/main.tf rename to fast/stages/03-gke-multitenant/_module/main.tf index 5f112dcdfc..1ea59ed897 100644 --- a/fast/stages/03-gke-multitenant/module/main.tf +++ b/fast/stages/03-gke-multitenant/_module/main.tf @@ -14,18 +14,14 @@ * limitations under the License. */ -locals { - labels = merge(var.labels, { environment = "dev" }) -} - module "gke-project-0" { source = "../../../../modules/project" - billing_account = var.billing_account.id - name = "dev-gke-clusters-0" - parent = var.folder_ids.gke-dev + billing_account = var.billing_account_id + name = "gke-clusters-0" + parent = var.folder_id prefix = var.prefix group_iam = var.group_iam - labels = local.labels + labels = var.labels services = concat( [ "cloudresourcemanager.googleapis.com", @@ -46,7 +42,7 @@ module "gke-project-0" { ) shared_vpc_service_config = { attach = true - host_project = var.host_project_ids.dev-spoke-0 + host_project = var.vpc_config.host_project_id service_identity_iam = merge({ "roles/compute.networkUser" = [ "cloudservices", "container-engine" diff --git a/fast/stages/03-gke-multitenant/config/system/repo.yml b/fast/stages/03-gke-multitenant/_module/outputs.tf similarity index 56% rename from fast/stages/03-gke-multitenant/config/system/repo.yml rename to fast/stages/03-gke-multitenant/_module/outputs.tf index b88f7dc63e..e2676246bc 100644 --- a/fast/stages/03-gke-multitenant/config/system/repo.yml +++ b/fast/stages/03-gke-multitenant/_module/outputs.tf @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -12,12 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -# mono-repo mode: a Repo resource under system/ directory is required -# multi-repo mode: a Repo resource under system/ directory is optional +# tfdoc:file:description Output variables. -apiVersion: configmanagement.gke.io/v1 -kind: Repo -metadata: - name: repo -spec: - version: 1.0.0 +output "clusters" { + description = "Cluster resources." + value = module.gke-cluster +} + +output "cluster_ids" { + description = "Cluster ids." + value = { + for k, v in module.gke-cluster : k => v.id + } +} + +output "project_id" { + description = "GKE project id." + value = module.gke-project-0.project_id +} diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf new file mode 100644 index 0000000000..abf237420c --- /dev/null +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -0,0 +1,229 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# we deal with one env here +# 1 project, m clusters +# cloud dns for gke? + +variable "authenticator_security_group" { + description = "Optional group used for Groups for GKE." + type = string + default = null +} + +variable "billing_account_id" { + description = "Billing account id." + type = string +} + +variable "cluster_defaults" { + description = "Default values for optional cluster configurations." + type = object({ + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool + }) + default = { + # TODO: review defaults + cloudrun_config = false + database_encryption_key = null + enable_binary_authorization = false + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + rfc1918_2 = "172.16.0.0/12" + rfc1918_3 = "192.168.0.0/16" + } + max_pods_per_node = 110 + pod_security_policy = false + release_channel = "STABLE" + vertical_pod_autoscaling = false + gcp_filestore_csi_driver_config = false + } +} + +variable "clusters" { + description = "" + type = map(object({ + cluster_autoscaling = object({ + cpu_min = number + cpu_max = number + memory_min = number + memory_max = number + }) + description = string + dns_domain = string + labels = map(string) + location = string + net = object({ + master_range = string + pods = string + services = string + subnet = string + }) + overrides = object({ + cloudrun_config = bool + database_encryption_key = string + enable_binary_authorization = bool + master_authorized_ranges = map(string) + max_pods_per_node = number + pod_security_policy = bool + release_channel = string + vertical_pod_autoscaling = bool + gcp_filestore_csi_driver_config = bool + }) + })) +} + +variable "dns_domain" { + description = "Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE." + type = string + default = null +} + +variable "fleet_configmanagement_clusters" { + description = "Config management features enabled on specific sets of member clusters, in config name => [cluster name] format." + type = map(list(string)) + default = {} + nullable = false +} + + +variable "fleet_configmanagement_templates" { + description = "Sets of config management configurations that can be applied to member clusters, in config name => {options} format." + type = map(object({ + binauthz = bool + config_sync = object({ + git = object({ + gcp_service_account_email = string + https_proxy = string + policy_dir = string + secret_type = string + sync_branch = string + sync_repo = string + sync_rev = string + sync_wait_secs = number + }) + prevent_drift = string + source_format = string + }) + hierarchy_controller = object({ + enable_hierarchical_resource_quota = bool + enable_pod_tree_labels = bool + }) + policy_controller = object({ + audit_interval_seconds = number + exemptable_namespaces = list(string) + log_denies_enabled = bool + referential_rules_enabled = bool + template_library_installed = bool + }) + version = string + })) + default = {} + nullable = false +} + +variable "fleet_features" { + description = "Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used." + type = object({ + appdevexperience = bool + configmanagement = bool + identityservice = bool + multiclusteringress = string + multiclusterservicediscovery = bool + servicemesh = bool + }) + default = null +} + +variable "fleet_workload_identity" { + description = "Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true." + type = bool + default = true + nullable = false +} + +variable "folder_id" { + description = "Folder used for the GKE project in folders/nnnnnnnnnnn format." + type = string +} + +variable "group_iam" { + description = "Project-level IAM bindings for groups. Use group emails as keys, list of roles as values." + type = map(list(string)) + default = {} + nullable = false +} + +variable "labels" { + description = "Project-level labels." + type = map(string) + default = {} +} + +variable "nodepool_defaults" { + description = "" + type = object({ + image_type = string + max_pods_per_node = number + node_locations = list(string) + node_tags = list(string) + node_taints = list(string) + }) + default = { + image_type = "COS_CONTAINERD" + max_pods_per_node = 110 + node_locations = null + node_tags = null + node_taints = [] + } +} + +variable "nodepools" { + description = "" + type = map(map(object({ + node_count = number + node_type = string + initial_node_count = number + overrides = object({ + image_type = string + max_pods_per_node = number + node_locations = list(string) + node_tags = list(string) + node_taints = list(string) + }) + preemptible = bool + }))) +} + +variable "prefix" { + description = "Prefix used for resources that need unique names." + type = string +} + +variable "vpc_config" { + description = "Shared VPC project and VPC details." + type = object({ + host_project_id = string + vpc_self_link = string + }) +} diff --git a/fast/stages/03-gke-multitenant/config/cluster/README.md b/fast/stages/03-gke-multitenant/config/cluster/README.md deleted file mode 100644 index fe3a852813..0000000000 --- a/fast/stages/03-gke-multitenant/config/cluster/README.md +++ /dev/null @@ -1,3 +0,0 @@ -The cluster/ directory contains configs that apply to entire clusters, rather than to namespaces. By default, any config in the cluster/ directory applies to every cluster enrolled in Config Sync. You can limit which clusters a config can affect by using a ClusterSelector. - -Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#cluster \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml deleted file mode 100644 index 52ab7387b1..0000000000 --- a/fast/stages/03-gke-multitenant/config/cluster/clusterrole-ns-viewer.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dev-namespace-viewer -rules: - - apiGroups: ["", "metrics.k8s.io", "extensions", "apps"] - resources: - - "pods" - - "pods/log" - - "events" - - "deployments" - - "replicasets" - verbs: ["get", "watch", "list"] diff --git a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml b/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml deleted file mode 100644 index aede039d56..0000000000 --- a/fast/stages/03-gke-multitenant/config/cluster/ns-must-have-geo-constraint_v1beta_gatekeeper.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: constraints.gatekeeper.sh/v1beta1 -kind: K8sRequiredLabels -metadata: - name: ns-must-have-geo - annotations: - configsync.gke.io/cluster-name-selector: gke-1 -spec: - enforcementAction: dryrun - match: - excludedNamespaces: - - "kube-system" - - "kube-public" - - "kube-node-lease" - - "resource-group-system" - - "config-management-monitoring" - - "config-management-system" - - "cos-auditd" - - "gatekeeper-system" - - "kube-node-lease" - - "cnrm-system" - - "gke-mcs" - - "configconnector-operator-system" - kinds: - - apiGroups: [""] - kinds: ["Namespace"] - parameters: - labels: - - key: "geo" diff --git a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml b/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml deleted file mode 100644 index 216deba384..0000000000 --- a/fast/stages/03-gke-multitenant/config/cluster/pod_priority_classes.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: high-priority-system -value: 1000000 -globalDefault: false -description: "This priority class will cause other pods to be preempted." ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: high-priority -value: 1000 -globalDefault: false -description: "This priority class will cause other pods to be preempted." ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: mid-priority -value: 100 -globalDefault: false -# preemptionPolicy: Never -description: "This priority class will cause other pods to be preempted." ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: low-priority -value: 10 -globalDefault: true -# preemptionPolicy: Never -description: "This priority class will cause other pods to be preempted." diff --git a/fast/stages/03-gke-multitenant/config/namespaces/README.md b/fast/stages/03-gke-multitenant/config/namespaces/README.md deleted file mode 100644 index 99382474b4..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/README.md +++ /dev/null @@ -1,7 +0,0 @@ - - -The namespaces/ directory contains configs for namespaces and namespace-scoped objects. - -The structure within namespaces/ is the mechanism that drives namespace inheritance. You can limit which namespaces can inherit a config, by using a NamespaceSelector. - -Doc ref: https://cloud.google.com/anthos-config-management/docs/concepts/hierarchical-repo#namespaces \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml deleted file mode 100644 index 8db587ea33..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_configmap_cos-auditd.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: ConfigMap -apiVersion: v1 -metadata: - name: fluentd-gcp-config-cos-auditd - namespace: cos-auditd - annotations: - kubernetes.io/description: "ConfigMap for Linux auditd logging daemonset on COS nodes." -data: - google-fluentd.conf: |- - - @type systemd - filters [{ "SYSLOG_IDENTIFIER": "audit" }] - pos_file /var/log/gcp-journald-audit.pos - read_from_head true - tag linux-auditd - - - # Do not collect fluentd's own logs to avoid infinite loops. - - @type null - - - - @type google_cloud - - enable_monitoring false - split_logs_by_tag false - detect_subservice false - buffer_type file - buffer_path /var/log/fluentd-buffers/system.audit.buffer - buffer_queue_full_action block - buffer_chunk_limit 512k - buffer_queue_limit 2 - flush_interval 5s - max_retry_wait 30 - disable_retry_limit - num_threads 2 - use_grpc true - diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml deleted file mode 100644 index b9068cd792..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/apps_v1_daemonset_cos-auditd.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cos-auditd-logging - namespace: cos-auditd - annotations: - kubernetes.io/description: "DaemonSet that enables Linux auditd logging on COS nodes." -spec: - selector: - matchLabels: - name: cos-auditd-logging - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: cos-auditd-logging - spec: - hostNetwork: true - hostPID: true - nodeSelector: - cloud.google.com/gke-os-distribution: cos - volumes: - - hostPath: - path: / - name: host - - hostPath: - path: /var/log - name: varlog - - hostPath: - path: /usr/lib64 - name: libsystemddir - - configMap: - defaultMode: 420 - name: fluentd-gcp-config-cos-auditd - name: config-volume - initContainers: - - name: cos-auditd-setup - image: ubuntu - command: - ["chroot", "/host", "systemctl", "start", "cloud-audit-setup"] - securityContext: - privileged: true - volumeMounts: - - name: host - mountPath: /host - resources: - requests: - memory: "10Mi" - cpu: "10m" - priorityClassName: high-priority-system - containers: - - name: fluentd-gcp-cos-auditd - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - image: gcr.io/stackdriver-agents/stackdriver-logging-agent:0.6-1.6.0-1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /bin/sh - - -c - - | - LIVENESS_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-300}; STUCK_THRESHOLD_SECONDS=${LIVENESS_THRESHOLD_SECONDS:-900}; if [[ ! -e /var/log/fluentd-buffers ]]; then - exit 1; - fi; touch -d "${STUCK_THRESHOLD_SECONDS} seconds ago" /tmp/marker-stuck; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-stuck -print -quit)" ]]; then - rm -rf /var/log/fluentd-buffers; - exit 1; - fi; touch -d "${LIVENESS_THRESHOLD_SECONDS} seconds ago" /tmp/marker-liveness; if [[ -z "$(find /var/log/fluentd-buffers -type f -newer /tmp/marker-liveness -print -quit)" ]]; then - exit 1; - fi; - failureThreshold: 3 - initialDelaySeconds: 600 - periodSeconds: 60 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: "1" - memory: 500Mi - requests: - cpu: 100m - memory: 200Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/log - name: varlog - - mountPath: /host/lib - name: libsystemddir - readOnly: true - - mountPath: /etc/google-fluentd/google-fluentd.conf - subPath: google-fluentd.conf - name: config-volume - dnsPolicy: Default - restartPolicy: Always - terminationGracePeriodSeconds: 30 - tolerations: - - effect: NoSchedule - key: node.alpha.kubernetes.io/ismaster - - effect: NoExecute - operator: Exists - - effect: NoSchedule - key: sandbox.gke.io/runtime - operator: Equal - value: gvisor - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate diff --git a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml deleted file mode 100644 index d80a804eb2..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/cos-auditd/namespace.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - name: cos-auditd diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml deleted file mode 100644 index d50f33c3b5..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/limit-range.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# LimitRange Reference: https://kubernetes.io/docs/concepts/policy/limit-range/ -apiVersion: v1 -kind: LimitRange -metadata: - name: default-limits -spec: - limits: - - type: Container - default: - cpu: "1" - memory: 512Mi - defaultRequest: - memory: 256Mi - cpu: "0.5" - max: - cpu: "2" - memory: 1Gi - - type: PersistentVolumeClaim - max: - storage: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml deleted file mode 100644 index 281a8f2c06..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/network-policy-default-deny-egress.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# apiVersion: networking.k8s.io/v1 -# kind: NetworkPolicy -# metadata: -# name: default-deny-egress -# spec: -# podSelector: {} -# policyTypes: -# - Egress diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml deleted file mode 100644 index 4d8d1c48f3..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/namespace.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - name: team-a diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml deleted file mode 100644 index f3615b1d57..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/repo-sync.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This Namespace is configured with RepoSync -# Doc ref: https://cloud.google.com/anthos-config-management/docs/how-to/namespace-repositories#namespace-root - -apiVersion: configsync.gke.io/v1beta1 -kind: RepoSync -metadata: - name: repo-sync - namespace: team-a -spec: - sourceFormat: unstructured - git: - repo: "https://github.com/danielmarzini/configsync-application-example" - branch: main - revision: HEAD - dir: "configsync/team-a" - auth: none diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml deleted file mode 100644 index b47cdaa2d6..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/resourcequotas.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ResourceQuota Reference: https://kubernetes.io/docs/concepts/policy/resource-quotas/ -kind: ResourceQuota -apiVersion: v1 -metadata: - namespace: team-a - name: pvc -spec: - hard: - persistentvolumeclaims: "3" ---- -apiVersion: v1 -kind: ResourceQuota -metadata: - namespace: team-a - name: mem-cpu-store -spec: - hard: - requests.cpu: "1" - requests.memory: 1Gi - limits.cpu: "2" - limits.memory: 2Gi \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml deleted file mode 100644 index 28b84ccc30..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/rolebinding-ns-viewer.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: namespace-viewer -roleRef: - kind: ClusterRole - name: dev-namespace-viewer - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: Group - name: team-a@marzi.gcp-pso-italy.net - apiGroup: rbac.authorization.k8s.io - namespace: team-a diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml deleted file mode 100644 index ec5de25032..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-a/sync-rolebinding.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ROOT_REPO/namespaces/NAMESPACE/sync-rolebinding.yaml - kind: RoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: syncs-repo - namespace: team-a - subjects: - - kind: ServiceAccount - name: ns-reconciler-team-a - namespace: config-management-system - roleRef: - kind: ClusterRole - name: edit - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml deleted file mode 100644 index 3f2c61dac7..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/apps_v1_deployment_v1.yml +++ /dev/null @@ -1,44 +0,0 @@ - -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: apps/v1 -kind: Deployment -metadata: - namespace: team-b - name: whereami -spec: - replicas: 1 - selector: - matchLabels: - app: whereami - template: - metadata: - labels: - app: whereami - version: v1 - spec: - containers: - - name: whereami - image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.7 - resources: - limits: - cpu: "0.5" - memory: 128Mi - requests: - cpu: "0.5" - memory: 128Mi - ports: - - name: http - containerPort: 8080 \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml b/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml deleted file mode 100644 index 075ca0859b..0000000000 --- a/fast/stages/03-gke-multitenant/config/namespaces/teams/team-b/namespace.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Namespace -metadata: - name: team-b - labels: - geo: "europe-west1" \ No newline at end of file diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md new file mode 100644 index 0000000000..894c8698c7 --- /dev/null +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -0,0 +1,74 @@ +# GKE Multitenant + +TODO: add description and diagram + +

+ GKE multitenant +

+ +## Design overview and choices + +TODO + +### Cluster and nodepool configuration + +TODO + +### Fleet management + +TODO + +## How to run this stage + +TODO + +### Providers configuration + +TODO + +### Variable configuration + +TODO + + + + +## Files + +| name | description | modules | resources | +|---|---|---|---| +| [main.tf](./main.tf) | GKE multitenant for development environment. | _module | | +| [outputs.tf](./outputs.tf) | Output variables. | | google_storage_bucket_object · local_file | +| [variables.tf](./variables.tf) | Module variables. | | | + +## Variables + +| name | description | type | required | default | producer | +|---|---|:---:|:---:|:---:|:---:| +| [billing_account](variables.tf#L27) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | +| [clusters](variables.tf#L67) | | map(object({…})) | ✓ | | | +| [folder_ids](variables.tf#L169) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | +| [host_project_ids](variables.tf#L184) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L216) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L233) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L238) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | +| [cluster_defaults](variables.tf#L36) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L100) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L106) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L114) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L149) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L162) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L177) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [labels](variables.tf#L192) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L198) | | object({…}) | | {…} | | + +## Outputs + +| name | description | sensitive | consumers | +|---|---|:---:|---| +| [cluster_ids](outputs.tf#L62) | Cluster ids. | | | +| [clusters](outputs.tf#L57) | Cluster resources. | | | +| [project_id](outputs.tf#L67) | GKE project id. | | | + + diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf new file mode 100644 index 0000000000..bb999ef70a --- /dev/null +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description GKE multitenant for development environment. + +module "gke-multitenant" { + source = "../_module" + billing_account_id = var.billing_account.id + folder_id = var.folder_ids.gke-dev + group_iam = var.group_iam + labels = merge(var.labels, { environment = "dev" }) + prefix = "${var.prefix}-dev" + vpc_config = { + host_project_id = var.host_project_ids.dev-spoke-0 + vpc_self_link = var.vpc_self_links.dev-spoke-0 + } + cluster_defaults = var.cluster_defaults + nodepool_defaults = var.nodepool_defaults + clusters = var.clusters + nodepools = var.nodepools + authenticator_security_group = var.authenticator_security_group + dns_domain = var.dns_domain + fleet_configmanagement_clusters = var.fleet_configmanagement_clusters + fleet_configmanagement_templates = var.fleet_configmanagement_templates + fleet_features = var.fleet_features + fleet_workload_identity = var.fleet_workload_identity +} diff --git a/fast/stages/03-gke-multitenant/dev/outputs.tf b/fast/stages/03-gke-multitenant/dev/outputs.tf new file mode 100644 index 0000000000..afa203fe4a --- /dev/null +++ b/fast/stages/03-gke-multitenant/dev/outputs.tf @@ -0,0 +1,70 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tfdoc:file:description Output variables. + +locals { + tfvars = { + clusters = module.gke-multitenant.cluster_ids + project_ids = { + gke-dev = module.gke-multitenant.project_id + } + } +} + +# generate tfvars file for subsequent stages + +resource "local_file" "tfvars" { + for_each = var.outputs_location == null ? {} : { 1 = 1 } + file_permission = "0644" + filename = "${pathexpand(var.outputs_location)}/tfvars/03-gke-dev.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +resource "google_storage_bucket_object" "tfvars" { + bucket = var.automation.outputs_bucket + name = "tfvars/03-gke-dev.auto.tfvars.json" + content = jsonencode(local.tfvars) +} + +# outputs + +output "clusters" { + description = "Cluster resources." + value = module.gke-multitenant.clusters +} + +output "cluster_ids" { + description = "Cluster ids." + value = module.gke-multitenant.cluster_ids +} + +output "project_id" { + description = "GKE project id." + value = module.gke-multitenant.project_id +} diff --git a/fast/stages/03-gke-multitenant/module/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf similarity index 100% rename from fast/stages/03-gke-multitenant/module/variables.tf rename to fast/stages/03-gke-multitenant/dev/variables.tf diff --git a/fast/stages/03-gke-multitenant/main.tf b/fast/stages/03-gke-multitenant/main.tf deleted file mode 100644 index 86d2e9ea7f..0000000000 --- a/fast/stages/03-gke-multitenant/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_container_cluster" "cluster" { - provider = google-beta - project = "tf-playground-svpc-gke" - name = "cluster1" - location = "europe-west1-b" - initial_node_count = 1 -} diff --git a/fast/stages/03-gke-multitenant/module/README.md b/fast/stages/03-gke-multitenant/module/README.md deleted file mode 100644 index d1089cfa55..0000000000 --- a/fast/stages/03-gke-multitenant/module/README.md +++ /dev/null @@ -1,108 +0,0 @@ -## IAM - -- project+infra cluster admin - -- cluster admins - they have IAM bindings at proj level - -- namespace admins / users - cluster user? only IAM role to fetch GKE creds -everything else is in RBAC (second part of this stage) - -- permissions to node service accounts - - logging/monitoring - - image registry - -- custom role to only allow autoprovisioning of credentials - -## Inputs from previous stages - -- vpc host and subnets - -## Network User role - -- (optional) resman stage 01 creates SA, net stage 02 sets role on net project, dev --> dev, prod --> prod -- (always possible) subnet factory assigns role - -## Resources - -### Robot service accounts - -- configured after prject activation (svpc roles, etc.) - - gke host service agent - - option for security admin - - CMEK for disks - -### Service accounts - -- node service accounts (per nodepool?) -- config sync service account (start with a single one, +wl identity) -- backup for GKE - -### Registry - -- per env? -- deep dive on partitioning per team - -### Clusters and nodepools - -- n clusters -- no default nodepool (or default nodepool has taints -- infra nodepool) -- what can change between clusters? - - stays the same - - monitoring and logging config - - autoprovisioning - - private / public - - workload identity - - cloud dns - - can change between clusters - - dataplane v2 - - nodepools - - binary auth -- nodepools per cluster - -## Addresses for ILBs - -### Private Cluster -WARNING: do not use run this stage with GKE-HUB and private clusters enabled from a machine outside the clusters network - -## Filestore - -- later - -## RBAC - - -org -- GKE -- prod -- dev -- core - -branch: gke-stage3 - -resman -GKE folder + env folders + gcs + sa (extra file to drop in stage 1) - -[TF] supporting infra - stage 3 gke / infra (apply with SA from stage 1) -**single environment** (e.g. prod) -IAM for admins and users -projects for clusters -VPC wiring -logging -monitoring -container registries -source repos -workload identity config -Secrets (?) -clusters and nodepools and service accounts -managed prometheus? - -[Manifests/etc.] multitenant cluster config - stage 3 gke / cluster config (apply/runs with credentials from infra part) -config sync -gatekeeper -team namespaces -network policies -in-cluster logging and monitoring - - diff --git a/fast/stages/03-gke-multitenant/module/outputs.tf b/fast/stages/03-gke-multitenant/module/outputs.tf deleted file mode 100644 index 11a2ddf118..0000000000 --- a/fast/stages/03-gke-multitenant/module/outputs.tf +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ From f00b67ae88ad16a0a3b6f0a95dc6714172e00d58 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 29 Jul 2022 15:11:36 +0200 Subject: [PATCH 29/75] linting --- fast/stages/03-data-platform/dev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/03-data-platform/dev/README.md b/fast/stages/03-data-platform/dev/README.md index d62a1c0063..6b885c9709 100644 --- a/fast/stages/03-data-platform/dev/README.md +++ b/fast/stages/03-data-platform/dev/README.md @@ -158,7 +158,7 @@ You can find examples in the `[demo](../../../../examples/data-solutions/data-pl | name | description | modules | resources | |---|---|---|---| -| [main.tf](./main.tf) | Data Platformy. | data-platform-foundations | | +| [main.tf](./main.tf) | Data Platform. | data-platform-foundations | | | [outputs.tf](./outputs.tf) | Output variables. | | google_storage_bucket_object · local_file | | [variables.tf](./variables.tf) | Terraform Variables. | | | From c9fa0fbd55af56334b117ae35a9b85af2a8c95d9 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Fri, 29 Jul 2022 23:45:15 +0200 Subject: [PATCH 30/75] fixes --- fast/stages/03-gke-multitenant/dev/outputs.tf | 1 + fast/stages/03-gke-multitenant/dev/variables.tf | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/fast/stages/03-gke-multitenant/dev/outputs.tf b/fast/stages/03-gke-multitenant/dev/outputs.tf index afa203fe4a..216db95dd1 100644 --- a/fast/stages/03-gke-multitenant/dev/outputs.tf +++ b/fast/stages/03-gke-multitenant/dev/outputs.tf @@ -57,6 +57,7 @@ resource "google_storage_bucket_object" "tfvars" { output "clusters" { description = "Cluster resources." value = module.gke-multitenant.clusters + sensitive = true } output "cluster_ids" { diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 1df64622a1..974b62dff4 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -18,6 +18,14 @@ # 1 project, m clusters # cloud dns for gke? +variable "automation" { + # tfdoc:variable:source 00-bootstrap + description = "Automation resources created by the bootstrap stage." + type = object({ + outputs_bucket = string + }) +} + variable "authenticator_security_group" { description = "Optional group used for Groups for GKE." type = string @@ -230,6 +238,12 @@ variable "nodepools" { }))) } +variable "outputs_location" { + description = "Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable." + type = string + default = null +} + variable "prefix" { description = "Prefix used for resources that need unique names." type = string From 171d0d10519cfe05567cfa2111c198cde94baaee Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Sat, 30 Jul 2022 00:54:05 +0200 Subject: [PATCH 31/75] fix mci integration in the gke-hub --- modules/gke-hub/main.tf | 2 +- modules/gke-hub/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gke-hub/main.tf b/modules/gke-hub/main.tf index 66688613c0..13526f2e0b 100644 --- a/modules/gke-hub/main.tf +++ b/modules/gke-hub/main.tf @@ -53,7 +53,7 @@ resource "google_gke_hub_membership" "default" { resource "google_gke_hub_feature" "default" { provider = google-beta - for_each = { for k, v in var.features : k => v if coalesce(v, false) != false } + for_each = { for k, v in var.features : k => v if v != false && v != "" } project = var.project_id name = each.key location = "global" diff --git a/modules/gke-hub/variables.tf b/modules/gke-hub/variables.tf index c7133c07fb..f3a481ba62 100644 --- a/modules/gke-hub/variables.tf +++ b/modules/gke-hub/variables.tf @@ -77,7 +77,7 @@ variable "features" { appdevexperience = false configmanagement = false identityservice = false - multiclusteringress = null + multiclusteringress = "" servicemesh = false multiclusterservicediscovery = false } From e4de73febb7c280e26e69d13777ad1a7205cfa03 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Sat, 30 Jul 2022 09:37:34 +0200 Subject: [PATCH 32/75] Revert "fix mci integration in the gke-hub" This reverts commit 171d0d10519cfe05567cfa2111c198cde94baaee. --- modules/gke-hub/main.tf | 2 +- modules/gke-hub/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gke-hub/main.tf b/modules/gke-hub/main.tf index 13526f2e0b..66688613c0 100644 --- a/modules/gke-hub/main.tf +++ b/modules/gke-hub/main.tf @@ -53,7 +53,7 @@ resource "google_gke_hub_membership" "default" { resource "google_gke_hub_feature" "default" { provider = google-beta - for_each = { for k, v in var.features : k => v if v != false && v != "" } + for_each = { for k, v in var.features : k => v if coalesce(v, false) != false } project = var.project_id name = each.key location = "global" diff --git a/modules/gke-hub/variables.tf b/modules/gke-hub/variables.tf index f3a481ba62..c7133c07fb 100644 --- a/modules/gke-hub/variables.tf +++ b/modules/gke-hub/variables.tf @@ -77,7 +77,7 @@ variable "features" { appdevexperience = false configmanagement = false identityservice = false - multiclusteringress = "" + multiclusteringress = null servicemesh = false multiclusterservicediscovery = false } From c51ba73e3840b3ae2d23b9b16af853bb29c55558 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 30 Jul 2022 15:59:45 +0200 Subject: [PATCH 33/75] change to binary_authorization, add support for additional services --- .../03-gke-multitenant/_module/gke-clusters.tf | 12 ++++++------ fast/stages/03-gke-multitenant/_module/main.tf | 1 + .../03-gke-multitenant/_module/variables.tf | 17 ++++++++++++----- fast/stages/03-gke-multitenant/dev/main.tf | 1 + fast/stages/03-gke-multitenant/dev/variables.tf | 17 ++++++++++++----- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf index 4a968aee39..7eaeddf2f9 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf @@ -101,12 +101,12 @@ module "gke-cluster" { key_name = each.value.overrides.database_encryption_key } ) - default_max_pods_per_node = each.value.overrides.max_pods_per_node - enable_binary_authorization = each.value.overrides.enable_binary_authorization - master_authorized_ranges = each.value.overrides.master_authorized_ranges - pod_security_policy = each.value.overrides.pod_security_policy - release_channel = each.value.overrides.release_channel - vertical_pod_autoscaling = each.value.overrides.vertical_pod_autoscaling + default_max_pods_per_node = each.value.overrides.max_pods_per_node + binary_authorization = each.value.overrides.binary_authorization + master_authorized_ranges = each.value.overrides.master_authorized_ranges + pod_security_policy = each.value.overrides.pod_security_policy + release_channel = each.value.overrides.release_channel + vertical_pod_autoscaling = each.value.overrides.vertical_pod_autoscaling # dynamic "cluster_autoscaling" { # for_each = each.value.cluster_autoscaling == null ? {} : { 1 = 1 } # content { diff --git a/fast/stages/03-gke-multitenant/_module/main.tf b/fast/stages/03-gke-multitenant/_module/main.tf index 1ea59ed897..3bc2a9cc55 100644 --- a/fast/stages/03-gke-multitenant/_module/main.tf +++ b/fast/stages/03-gke-multitenant/_module/main.tf @@ -30,6 +30,7 @@ module "gke-project-0" { "iam.googleapis.com", "stackdriver.googleapis.com", ], + var.project_services, !local.fleet_enabled ? [] : [ "anthosconfigmanagement.googleapis.com", "anthos.googleapis.com", diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf index abf237420c..9e2822cccb 100644 --- a/fast/stages/03-gke-multitenant/_module/variables.tf +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -34,7 +34,7 @@ variable "cluster_defaults" { type = object({ cloudrun_config = bool database_encryption_key = string - enable_binary_authorization = bool + binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -44,9 +44,9 @@ variable "cluster_defaults" { }) default = { # TODO: review defaults - cloudrun_config = false - database_encryption_key = null - enable_binary_authorization = false + cloudrun_config = false + database_encryption_key = null + binary_authorization = false master_authorized_ranges = { rfc1918_1 = "10.0.0.0/8" rfc1918_2 = "172.16.0.0/12" @@ -82,7 +82,7 @@ variable "clusters" { overrides = object({ cloudrun_config = bool database_encryption_key = string - enable_binary_authorization = bool + binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -220,6 +220,13 @@ variable "prefix" { type = string } +variable "project_services" { + description = "Additional project services to enable." + type = list(string) + default = [] + nullable = false +} + variable "vpc_config" { description = "Shared VPC project and VPC details." type = object({ diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index bb999ef70a..09d898d18e 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -23,6 +23,7 @@ module "gke-multitenant" { group_iam = var.group_iam labels = merge(var.labels, { environment = "dev" }) prefix = "${var.prefix}-dev" + project_services = var.project_services vpc_config = { host_project_id = var.host_project_ids.dev-spoke-0 vpc_self_link = var.vpc_self_links.dev-spoke-0 diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 974b62dff4..12f2342763 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -46,7 +46,7 @@ variable "cluster_defaults" { type = object({ cloudrun_config = bool database_encryption_key = string - enable_binary_authorization = bool + binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -56,9 +56,9 @@ variable "cluster_defaults" { }) default = { # TODO: review defaults - cloudrun_config = false - database_encryption_key = null - enable_binary_authorization = false + cloudrun_config = false + database_encryption_key = null + binary_authorization = false master_authorized_ranges = { rfc1918_1 = "10.0.0.0/8" rfc1918_2 = "172.16.0.0/12" @@ -94,7 +94,7 @@ variable "clusters" { overrides = object({ cloudrun_config = bool database_encryption_key = string - enable_binary_authorization = bool + binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -249,6 +249,13 @@ variable "prefix" { type = string } +variable "project_services" { + description = "Additional project services to enable." + type = list(string) + default = [] + nullable = false +} + variable "vpc_self_links" { # tfdoc:variable:source 02-networking description = "Self link for the shared VPC." From f51e40b60afd176a93a0abce3fd50edd8e7a473e Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 30 Jul 2022 16:00:10 +0200 Subject: [PATCH 34/75] tfdoc --- .../03-gke-multitenant/_module/README.md | 7 +-- fast/stages/03-gke-multitenant/dev/README.md | 43 ++++++++++--------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index e18762582b..7afc74d5a4 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -41,13 +41,13 @@ TODO | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | -| [clusters](variables.tf#L63) | | map(object({…})) | ✓ | | | +| [clusters](variables.tf#L63) | | map(object({…})) | ✓ | | | | [folder_id](variables.tf#L165) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | | [nodepools](variables.tf#L201) | | map(map(object({…}))) | ✓ | | | | [prefix](variables.tf#L218) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L223) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [vpc_config](variables.tf#L230) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | -| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L96) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | | [fleet_configmanagement_clusters](variables.tf#L102) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | | [fleet_configmanagement_templates](variables.tf#L110) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | @@ -56,6 +56,7 @@ TODO | [group_iam](variables.tf#L170) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | | [labels](variables.tf#L177) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L183) | | object({…}) | | {…} | | +| [project_services](variables.tf#L223) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 894c8698c7..6ed71da3a1 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -45,30 +45,33 @@ TODO | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account](variables.tf#L27) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | -| [clusters](variables.tf#L67) | | map(object({…})) | ✓ | | | -| [folder_ids](variables.tf#L169) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [host_project_ids](variables.tf#L184) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L216) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L233) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L238) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | -| [cluster_defaults](variables.tf#L36) | Default values for optional cluster configurations. | object({…}) | | {…} | | -| [dns_domain](variables.tf#L100) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | -| [fleet_configmanagement_clusters](variables.tf#L106) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L114) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L149) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L162) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | -| [group_iam](variables.tf#L177) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [labels](variables.tf#L192) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L198) | | object({…}) | | {…} | | +| [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | +| [billing_account](variables.tf#L35) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | +| [clusters](variables.tf#L75) | | map(object({…})) | ✓ | | | +| [folder_ids](variables.tf#L177) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | +| [host_project_ids](variables.tf#L192) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L224) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L247) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L259) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | +| [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L108) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L114) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L122) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L157) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L170) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L185) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [labels](variables.tf#L200) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L206) | | object({…}) | | {…} | | +| [outputs_location](variables.tf#L241) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L252) | Additional project services to enable. | list(string) | | [] | | ## Outputs | name | description | sensitive | consumers | |---|---|:---:|---| -| [cluster_ids](outputs.tf#L62) | Cluster ids. | | | -| [clusters](outputs.tf#L57) | Cluster resources. | | | -| [project_id](outputs.tf#L67) | GKE project id. | | | +| [cluster_ids](outputs.tf#L63) | Cluster ids. | | | +| [clusters](outputs.tf#L57) | Cluster resources. | ✓ | | +| [project_id](outputs.tf#L68) | GKE project id. | | | From 24f3545de799a80f6e698fedbc04a154395410b4 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 30 Jul 2022 16:05:39 +0200 Subject: [PATCH 35/75] add project-level iam variable --- .../03-gke-multitenant/_module/README.md | 13 +++++++------ .../stages/03-gke-multitenant/_module/main.tf | 1 + .../03-gke-multitenant/_module/variables.tf | 7 +++++++ fast/stages/03-gke-multitenant/dev/README.md | 19 ++++++++++--------- fast/stages/03-gke-multitenant/dev/main.tf | 1 + .../03-gke-multitenant/dev/variables.tf | 9 ++++++++- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index 7afc74d5a4..fe14b4634d 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -43,9 +43,9 @@ TODO | [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | | [clusters](variables.tf#L63) | | map(object({…})) | ✓ | | | | [folder_id](variables.tf#L165) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | -| [nodepools](variables.tf#L201) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L218) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L230) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [nodepools](variables.tf#L208) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L225) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_config](variables.tf#L237) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L96) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -54,9 +54,10 @@ TODO | [fleet_features](variables.tf#L145) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | | [fleet_workload_identity](variables.tf#L158) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | | [group_iam](variables.tf#L170) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [labels](variables.tf#L177) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L183) | | object({…}) | | {…} | | -| [project_services](variables.tf#L223) | Additional project services to enable. | list(string) | | [] | | +| [iam](variables.tf#L177) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L184) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L190) | | object({…}) | | {…} | | +| [project_services](variables.tf#L230) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/_module/main.tf b/fast/stages/03-gke-multitenant/_module/main.tf index 3bc2a9cc55..470d403497 100644 --- a/fast/stages/03-gke-multitenant/_module/main.tf +++ b/fast/stages/03-gke-multitenant/_module/main.tf @@ -21,6 +21,7 @@ module "gke-project-0" { parent = var.folder_id prefix = var.prefix group_iam = var.group_iam + iam = var.iam labels = var.labels services = concat( [ diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf index 9e2822cccb..15f1807859 100644 --- a/fast/stages/03-gke-multitenant/_module/variables.tf +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -174,6 +174,13 @@ variable "group_iam" { nullable = false } +variable "iam" { + description = "Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + variable "labels" { description = "Project-level labels." type = map(string) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 6ed71da3a1..fb237343a2 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -49,10 +49,10 @@ TODO | [billing_account](variables.tf#L35) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | | [clusters](variables.tf#L75) | | map(object({…})) | ✓ | | | | [folder_ids](variables.tf#L177) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [host_project_ids](variables.tf#L192) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L224) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L247) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L259) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [host_project_ids](variables.tf#L199) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L231) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L254) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L266) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | | [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L108) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -60,11 +60,12 @@ TODO | [fleet_configmanagement_templates](variables.tf#L122) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | | [fleet_features](variables.tf#L157) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | | [fleet_workload_identity](variables.tf#L170) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | -| [group_iam](variables.tf#L185) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [labels](variables.tf#L200) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L206) | | object({…}) | | {…} | | -| [outputs_location](variables.tf#L241) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L252) | Additional project services to enable. | list(string) | | [] | | +| [group_iam](variables.tf#L185) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L192) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L207) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L213) | | object({…}) | | {…} | | +| [outputs_location](variables.tf#L248) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L259) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index 09d898d18e..5ac7e332f9 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -21,6 +21,7 @@ module "gke-multitenant" { billing_account_id = var.billing_account.id folder_id = var.folder_ids.gke-dev group_iam = var.group_iam + iam = var.iam labels = merge(var.labels, { environment = "dev" }) prefix = "${var.prefix}-dev" project_services = var.project_services diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 12f2342763..997ba13f2d 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -183,7 +183,14 @@ variable "folder_ids" { } variable "group_iam" { - description = "Project-level IAM bindings for groups. Use group emails as keys, list of roles as values." + description = "Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam" { + description = "Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format." type = map(list(string)) default = {} nullable = false From 9b371a3d2c2c6486b5d97a5df4999e444e0e4793 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 31 Jul 2022 14:54:14 +0200 Subject: [PATCH 36/75] support GKE specific network roles in stages 01 and 02 --- fast/stages/02-networking-nva/main.tf | 2 ++ fast/stages/02-networking-nva/spoke-dev.tf | 3 ++- fast/stages/02-networking-nva/spoke-prod.tf | 3 ++- fast/stages/02-networking-peering/main.tf | 2 ++ fast/stages/02-networking-peering/spoke-dev.tf | 3 ++- fast/stages/02-networking-peering/spoke-prod.tf | 3 ++- fast/stages/02-networking-vpn/main.tf | 2 ++ fast/stages/02-networking-vpn/spoke-dev.tf | 1 + fast/stages/02-networking-vpn/spoke-prod.tf | 1 + 9 files changed, 16 insertions(+), 4 deletions(-) diff --git a/fast/stages/02-networking-nva/main.tf b/fast/stages/02-networking-nva/main.tf index c680f4442e..4db5061ba0 100644 --- a/fast/stages/02-networking-nva/main.tf +++ b/fast/stages/02-networking-nva/main.tf @@ -31,7 +31,9 @@ locals { stage3_sas_delegated_grants = [ "roles/composer.sharedVpcAgent", "roles/compute.networkUser", + "roles/compute.networkViewer", "roles/container.hostServiceAgentUser", + "roles/multiclusterservicediscovery.serviceAgent", "roles/vpcaccess.user", ] } diff --git a/fast/stages/02-networking-nva/spoke-dev.tf b/fast/stages/02-networking-nva/spoke-dev.tf index 75e17d610f..225c282962 100644 --- a/fast/stages/02-networking-nva/spoke-dev.tf +++ b/fast/stages/02-networking-nva/spoke-dev.tf @@ -41,7 +41,8 @@ module "dev-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ - try(local.service_accounts.project-factory-dev, null) + try(local.service_accounts.gke-dev, null), + try(local.service_accounts.project-factory-dev, null), ]) } } diff --git a/fast/stages/02-networking-nva/spoke-prod.tf b/fast/stages/02-networking-nva/spoke-prod.tf index b3d4647d67..e3fa7c8cac 100644 --- a/fast/stages/02-networking-nva/spoke-prod.tf +++ b/fast/stages/02-networking-nva/spoke-prod.tf @@ -41,7 +41,8 @@ module "prod-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ - try(local.service_accounts.project-factory-prod, null) + try(local.service_accounts.gke-prod, null), + try(local.service_accounts.project-factory-prod, null), ]) } } diff --git a/fast/stages/02-networking-peering/main.tf b/fast/stages/02-networking-peering/main.tf index 9e013fd178..de62646499 100644 --- a/fast/stages/02-networking-peering/main.tf +++ b/fast/stages/02-networking-peering/main.tf @@ -32,7 +32,9 @@ locals { stage3_sas_delegated_grants = [ "roles/composer.sharedVpcAgent", "roles/compute.networkUser", + "roles/compute.networkViewer", "roles/container.hostServiceAgentUser", + "roles/multiclusterservicediscovery.serviceAgent", "roles/vpcaccess.user", ] service_accounts = { diff --git a/fast/stages/02-networking-peering/spoke-dev.tf b/fast/stages/02-networking-peering/spoke-dev.tf index c0749bf9d9..f2c6572801 100644 --- a/fast/stages/02-networking-peering/spoke-dev.tf +++ b/fast/stages/02-networking-peering/spoke-dev.tf @@ -42,7 +42,8 @@ module "dev-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ - try(local.service_accounts.project-factory-dev, null) + try(local.service_accounts.gke-dev, null), + try(local.service_accounts.project-factory-dev, null), ]) } } diff --git a/fast/stages/02-networking-peering/spoke-prod.tf b/fast/stages/02-networking-peering/spoke-prod.tf index 768a2012f0..30608d3a9d 100644 --- a/fast/stages/02-networking-peering/spoke-prod.tf +++ b/fast/stages/02-networking-peering/spoke-prod.tf @@ -42,7 +42,8 @@ module "prod-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ - try(local.service_accounts.project-factory-prod, null) + try(local.service_accounts.gke-prod, null), + try(local.service_accounts.project-factory-prod, null), ]) } } diff --git a/fast/stages/02-networking-vpn/main.tf b/fast/stages/02-networking-vpn/main.tf index 9e013fd178..de62646499 100644 --- a/fast/stages/02-networking-vpn/main.tf +++ b/fast/stages/02-networking-vpn/main.tf @@ -32,7 +32,9 @@ locals { stage3_sas_delegated_grants = [ "roles/composer.sharedVpcAgent", "roles/compute.networkUser", + "roles/compute.networkViewer", "roles/container.hostServiceAgentUser", + "roles/multiclusterservicediscovery.serviceAgent", "roles/vpcaccess.user", ] service_accounts = { diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index c0749bf9d9..ccd75da322 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -42,6 +42,7 @@ module "dev-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ + try(local.service_accounts.gke-dev, null), try(local.service_accounts.project-factory-dev, null) ]) } diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/02-networking-vpn/spoke-prod.tf index 768a2012f0..70b81a15e3 100644 --- a/fast/stages/02-networking-vpn/spoke-prod.tf +++ b/fast/stages/02-networking-vpn/spoke-prod.tf @@ -42,6 +42,7 @@ module "prod-spoke-project" { metric_scopes = [module.landing-project.project_id] iam = { "roles/dns.admin" = compact([ + try(local.service_accounts.gke-prod, null), try(local.service_accounts.project-factory-prod, null) ]) } From cd3b4463def40146f839a65107fd7d8a7b75e550 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 31 Jul 2022 15:00:09 +0200 Subject: [PATCH 37/75] align net stages --- fast/stages/02-networking-vpn/spoke-dev.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/02-networking-vpn/spoke-dev.tf b/fast/stages/02-networking-vpn/spoke-dev.tf index ccd75da322..f2c6572801 100644 --- a/fast/stages/02-networking-vpn/spoke-dev.tf +++ b/fast/stages/02-networking-vpn/spoke-dev.tf @@ -43,7 +43,7 @@ module "dev-spoke-project" { iam = { "roles/dns.admin" = compact([ try(local.service_accounts.gke-dev, null), - try(local.service_accounts.project-factory-dev, null) + try(local.service_accounts.project-factory-dev, null), ]) } } From dedddbed20d517bfeaa3275ca71cb3df265debea Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sun, 31 Jul 2022 15:13:39 +0200 Subject: [PATCH 38/75] align net stages --- fast/stages/02-networking-vpn/spoke-prod.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/02-networking-vpn/spoke-prod.tf b/fast/stages/02-networking-vpn/spoke-prod.tf index 70b81a15e3..30608d3a9d 100644 --- a/fast/stages/02-networking-vpn/spoke-prod.tf +++ b/fast/stages/02-networking-vpn/spoke-prod.tf @@ -43,7 +43,7 @@ module "prod-spoke-project" { iam = { "roles/dns.admin" = compact([ try(local.service_accounts.gke-prod, null), - try(local.service_accounts.project-factory-prod, null) + try(local.service_accounts.project-factory-prod, null), ]) } } From 6314afb21ea5cec46787dd87a0e31f3873f46575 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 2 Aug 2022 18:26:53 +0200 Subject: [PATCH 39/75] Fix gke-cluster enable_binary_authorization being deprecated --- modules/gke-cluster/README.md | 59 ++++++++++++++++---------------- modules/gke-cluster/main.tf | 1 - modules/gke-cluster/variables.tf | 6 ---- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md index ba88eb2579..efb40de486 100644 --- a/modules/gke-cluster/README.md +++ b/modules/gke-cluster/README.md @@ -68,13 +68,13 @@ module "cluster-1" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [location](variables.tf#L161) | Cluster zone or region. | string | ✓ | | -| [name](variables.tf#L228) | Cluster name. | string | ✓ | | -| [network](variables.tf#L233) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | -| [project_id](variables.tf#L277) | Cluster project id. | string | ✓ | | -| [secondary_range_pods](variables.tf#L300) | Subnet secondary range name used for pods. | string | ✓ | | -| [secondary_range_services](variables.tf#L305) | Subnet secondary range name used for services. | string | ✓ | | -| [subnetwork](variables.tf#L310) | VPC subnetwork name or self link. | string | ✓ | | +| [location](variables.tf#L155) | Cluster zone or region. | string | ✓ | | +| [name](variables.tf#L222) | Cluster name. | string | ✓ | | +| [network](variables.tf#L227) | Name or self link of the VPC used for the cluster. Use the self link for Shared VPC. | string | ✓ | | +| [project_id](variables.tf#L271) | Cluster project id. | string | ✓ | | +| [secondary_range_pods](variables.tf#L294) | Subnet secondary range name used for pods. | string | ✓ | | +| [secondary_range_services](variables.tf#L299) | Subnet secondary range name used for services. | string | ✓ | | +| [subnetwork](variables.tf#L304) | VPC subnetwork name or self link. | string | ✓ | | | [addons](variables.tf#L17) | Addons enabled in the cluster (true means enabled). | object({…}) | | {…} | | [authenticator_security_group](variables.tf#L53) | RBAC security group for Google Groups for GKE, format is gke-security-groups@yourdomain.com. | string | | null | | [cluster_autoscaling](variables.tf#L59) | Enable and configure limits for Node Auto-Provisioning with Cluster Autoscaler. | object({…}) | | {…} | @@ -83,29 +83,28 @@ module "cluster-1" { | [description](variables.tf#L97) | Cluster description. | string | | null | | [dns_config](variables.tf#L103) | Configuration for Using Cloud DNS for GKE. | object({…}) | | null | | [enable_autopilot](variables.tf#L113) | Create cluster in autopilot mode. With autopilot there's no need to create node-pools and some features are not supported (e.g. setting default_max_pods_per_node). | bool | | false | -| [enable_binary_authorization](variables.tf#L119) | Enable Google Binary Authorization. | bool | | null | -| [enable_dataplane_v2](variables.tf#L125) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false | -| [enable_intranode_visibility](variables.tf#L131) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null | -| [enable_l4_ilb_subsetting](variables.tf#L137) | Enable L4ILB Subsetting. | bool | | null | -| [enable_shielded_nodes](variables.tf#L143) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null | -| [enable_tpu](variables.tf#L149) | Enable Cloud TPU resources in this cluster. | bool | | null | -| [labels](variables.tf#L155) | Cluster resource labels. | map(string) | | null | -| [logging_config](variables.tf#L166) | Logging configuration (enabled components). | list(string) | | null | -| [logging_service](variables.tf#L172) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" | -| [maintenance_config](variables.tf#L178) | Maintenance window configuration. | object({…}) | | {…} | -| [master_authorized_ranges](variables.tf#L204) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} | -| [min_master_version](variables.tf#L210) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | -| [monitoring_config](variables.tf#L216) | Monitoring configuration (enabled components). | list(string) | | null | -| [monitoring_service](variables.tf#L222) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" | -| [node_locations](variables.tf#L238) | Zones in which the cluster's nodes are located. | list(string) | | [] | -| [notification_config](variables.tf#L244) | GKE Cluster upgrade notifications via PubSub. | bool | | false | -| [peering_config](variables.tf#L250) | Configure peering with the master VPC for private clusters. | object({…}) | | null | -| [pod_security_policy](variables.tf#L260) | Enable the PodSecurityPolicy feature. | bool | | null | -| [private_cluster_config](variables.tf#L266) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null | -| [release_channel](variables.tf#L282) | Release channel for GKE upgrades. | string | | null | -| [resource_usage_export_config](variables.tf#L288) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} | -| [vertical_pod_autoscaling](variables.tf#L315) | Enable the Vertical Pod Autoscaling feature. | bool | | null | -| [workload_identity](variables.tf#L321) | Enable the Workload Identity feature. | bool | | true | +| [enable_dataplane_v2](variables.tf#L119) | Enable Dataplane V2 on the cluster, will disable network_policy addons config. | bool | | false | +| [enable_intranode_visibility](variables.tf#L125) | Enable intra-node visibility to make same node pod to pod traffic visible. | bool | | null | +| [enable_l4_ilb_subsetting](variables.tf#L131) | Enable L4ILB Subsetting. | bool | | null | +| [enable_shielded_nodes](variables.tf#L137) | Enable Shielded Nodes features on all nodes in this cluster. | bool | | null | +| [enable_tpu](variables.tf#L143) | Enable Cloud TPU resources in this cluster. | bool | | null | +| [labels](variables.tf#L149) | Cluster resource labels. | map(string) | | null | +| [logging_config](variables.tf#L160) | Logging configuration (enabled components). | list(string) | | null | +| [logging_service](variables.tf#L166) | Logging service (disable with an empty string). | string | | "logging.googleapis.com/kubernetes" | +| [maintenance_config](variables.tf#L172) | Maintenance window configuration. | object({…}) | | {…} | +| [master_authorized_ranges](variables.tf#L198) | External Ip address ranges that can access the Kubernetes cluster master through HTTPS. | map(string) | | {} | +| [min_master_version](variables.tf#L204) | Minimum version of the master, defaults to the version of the most recent official release. | string | | null | +| [monitoring_config](variables.tf#L210) | Monitoring configuration (enabled components). | list(string) | | null | +| [monitoring_service](variables.tf#L216) | Monitoring service (disable with an empty string). | string | | "monitoring.googleapis.com/kubernetes" | +| [node_locations](variables.tf#L232) | Zones in which the cluster's nodes are located. | list(string) | | [] | +| [notification_config](variables.tf#L238) | GKE Cluster upgrade notifications via PubSub. | bool | | false | +| [peering_config](variables.tf#L244) | Configure peering with the master VPC for private clusters. | object({…}) | | null | +| [pod_security_policy](variables.tf#L254) | Enable the PodSecurityPolicy feature. | bool | | null | +| [private_cluster_config](variables.tf#L260) | Enable and configure private cluster, private nodes must be true if used. | object({…}) | | null | +| [release_channel](variables.tf#L276) | Release channel for GKE upgrades. | string | | null | +| [resource_usage_export_config](variables.tf#L282) | Configure the ResourceUsageExportConfig feature. | object({…}) | | {…} | +| [vertical_pod_autoscaling](variables.tf#L309) | Enable the Vertical Pod Autoscaling feature. | bool | | null | +| [workload_identity](variables.tf#L315) | Enable the Workload Identity feature. | bool | | true | ## Outputs diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf index 095ee5a97f..56f7ea75a1 100644 --- a/modules/gke-cluster/main.tf +++ b/modules/gke-cluster/main.tf @@ -43,7 +43,6 @@ resource "google_container_cluster" "cluster" { monitoring_service = var.monitoring_config == null ? var.monitoring_service : null resource_labels = var.labels default_max_pods_per_node = var.enable_autopilot ? null : var.default_max_pods_per_node - enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting enable_shielded_nodes = var.enable_shielded_nodes diff --git a/modules/gke-cluster/variables.tf b/modules/gke-cluster/variables.tf index 6793902090..58be03c9d6 100644 --- a/modules/gke-cluster/variables.tf +++ b/modules/gke-cluster/variables.tf @@ -116,12 +116,6 @@ variable "enable_autopilot" { default = false } -variable "enable_binary_authorization" { - description = "Enable Google Binary Authorization." - type = bool - default = null -} - variable "enable_dataplane_v2" { description = "Enable Dataplane V2 on the cluster, will disable network_policy addons config." type = bool From 45723233c8274e3ae23e92792b29c1a4fc14ce55 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 2 Aug 2022 18:27:44 +0200 Subject: [PATCH 40/75] fix permissions and binauthz --- fast/stages/03-gke-multitenant/dev/variables.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 997ba13f2d..32d9525b79 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -46,7 +46,6 @@ variable "cluster_defaults" { type = object({ cloudrun_config = bool database_encryption_key = string - binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -58,7 +57,7 @@ variable "cluster_defaults" { # TODO: review defaults cloudrun_config = false database_encryption_key = null - binary_authorization = false + # binary_authorization = false master_authorized_ranges = { rfc1918_1 = "10.0.0.0/8" rfc1918_2 = "172.16.0.0/12" @@ -92,9 +91,9 @@ variable "clusters" { subnet = string }) overrides = object({ - cloudrun_config = bool - database_encryption_key = string - binary_authorization = bool + cloudrun_config = bool + database_encryption_key = string + # binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool From 170d585473f846dcb09b164e06a6c31160d8a116 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 2 Aug 2022 18:28:06 +0200 Subject: [PATCH 41/75] fix permissions and binauthz --- fast/stages/03-gke-multitenant/_module/gke-clusters.tf | 1 - fast/stages/03-gke-multitenant/_module/gke-hub.tf | 6 +++--- fast/stages/03-gke-multitenant/_module/variables.tf | 8 +++----- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf index 7eaeddf2f9..79794e5660 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf @@ -102,7 +102,6 @@ module "gke-cluster" { } ) default_max_pods_per_node = each.value.overrides.max_pods_per_node - binary_authorization = each.value.overrides.binary_authorization master_authorized_ranges = each.value.overrides.master_authorized_ranges pod_security_policy = each.value.overrides.pod_security_policy release_channel = each.value.overrides.release_channel diff --git a/fast/stages/03-gke-multitenant/_module/gke-hub.tf b/fast/stages/03-gke-multitenant/_module/gke-hub.tf index 8979605cba..9830c7fc26 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-hub.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-hub.tf @@ -18,9 +18,9 @@ locals { fleet_enabled = ( var.fleet_features != null || var.fleet_workload_identity ) - fleet_mcs_enabled = local.fleet_enabled && lookup( - coalesce(var.fleet_features, {}), "multiclusterservicediscovery", false - ) == true + fleet_mcs_enabled = ( + try(var.fleet_features.multiclusterservicediscovery, false) == true + ) } module "gke-hub" { diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf index 15f1807859..35df5615ba 100644 --- a/fast/stages/03-gke-multitenant/_module/variables.tf +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -34,7 +34,6 @@ variable "cluster_defaults" { type = object({ cloudrun_config = bool database_encryption_key = string - binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool @@ -46,7 +45,6 @@ variable "cluster_defaults" { # TODO: review defaults cloudrun_config = false database_encryption_key = null - binary_authorization = false master_authorized_ranges = { rfc1918_1 = "10.0.0.0/8" rfc1918_2 = "172.16.0.0/12" @@ -80,9 +78,9 @@ variable "clusters" { subnet = string }) overrides = object({ - cloudrun_config = bool - database_encryption_key = string - binary_authorization = bool + cloudrun_config = bool + database_encryption_key = string + # binary_authorization = bool master_authorized_ranges = map(string) max_pods_per_node = number pod_security_policy = bool From ce7e4bc59cb4f0dbfb849a56b57ee6ef8dd75bcd Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 2 Aug 2022 19:12:52 +0200 Subject: [PATCH 42/75] linting --- .../03-gke-multitenant/_module/README.md | 32 ++++++++--------- fast/stages/03-gke-multitenant/dev/README.md | 36 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index fe14b4634d..4528d5b073 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -41,23 +41,23 @@ TODO | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| | [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | -| [clusters](variables.tf#L63) | | map(object({…})) | ✓ | | | -| [folder_id](variables.tf#L165) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | -| [nodepools](variables.tf#L208) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L225) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L237) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [clusters](variables.tf#L61) | | map(object({…})) | ✓ | | | +| [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | +| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L223) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_config](variables.tf#L235) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | -| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | -| [dns_domain](variables.tf#L96) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | -| [fleet_configmanagement_clusters](variables.tf#L102) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L110) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L145) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L158) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | -| [group_iam](variables.tf#L170) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L177) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L184) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L190) | | object({…}) | | {…} | | -| [project_services](variables.tf#L230) | Additional project services to enable. | list(string) | | [] | | +| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L100) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L108) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L143) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L156) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L168) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | +| [project_services](variables.tf#L228) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index fb237343a2..12c5cd6f28 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -47,25 +47,25 @@ TODO |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L35) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | -| [clusters](variables.tf#L75) | | map(object({…})) | ✓ | | | -| [folder_ids](variables.tf#L177) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [host_project_ids](variables.tf#L199) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L231) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L254) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L266) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [clusters](variables.tf#L74) | | map(object({…})) | ✓ | | | +| [folder_ids](variables.tf#L176) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | +| [host_project_ids](variables.tf#L198) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L253) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L265) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | | [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | -| [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | -| [dns_domain](variables.tf#L108) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | -| [fleet_configmanagement_clusters](variables.tf#L114) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L122) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L157) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L170) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | -| [group_iam](variables.tf#L185) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L192) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L207) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L213) | | object({…}) | | {…} | | -| [outputs_location](variables.tf#L248) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L259) | Additional project services to enable. | list(string) | | [] | | +| [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L107) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L113) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L121) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L156) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L169) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L184) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L191) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L206) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L212) | | object({…}) | | {…} | | +| [outputs_location](variables.tf#L247) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L258) | Additional project services to enable. | list(string) | | [] | | ## Outputs From 0516c30011f7a09911b4c4e464feee82a1651c32 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 2 Aug 2022 19:15:03 +0200 Subject: [PATCH 43/75] fix binauthz example --- examples/cloud-operations/binauthz/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/cloud-operations/binauthz/main.tf b/examples/cloud-operations/binauthz/main.tf index af34cb9262..b9af6c41c3 100644 --- a/examples/cloud-operations/binauthz/main.tf +++ b/examples/cloud-operations/binauthz/main.tf @@ -97,8 +97,7 @@ module "cluster" { master_ipv4_cidr_block = var.master_cidr_block master_global_access = false } - enable_binary_authorization = true - workload_identity = true + workload_identity = true } module "cluster_nodepool" { From 1cf963b443eb1dcbed166dff9f964a0600355a2e Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 2 Aug 2022 21:41:36 +0200 Subject: [PATCH 44/75] added spot vm --- fast/stages/03-gke-multitenant/_module/README.md | 8 ++++---- fast/stages/03-gke-multitenant/_module/gke-nodepools.tf | 1 + fast/stages/03-gke-multitenant/_module/variables.tf | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index 4528d5b073..8c2d5cd345 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -43,9 +43,9 @@ TODO | [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | | [clusters](variables.tf#L61) | | map(object({…})) | ✓ | | | | [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | -| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L223) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L235) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L224) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_config](variables.tf#L236) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -57,7 +57,7 @@ TODO | [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | -| [project_services](variables.tf#L228) | Additional project services to enable. | list(string) | | [] | | +| [project_services](variables.tf#L229) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf b/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf index 703c9d71e7..ed95a02b0a 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf @@ -38,6 +38,7 @@ module "gke-nodepool" { node_machine_type = each.value.node_type # TODO(jccb): can we use spot instances here? node_preemptible = each.value.preemptible + node_spot = each.value.spot node_count = each.value.node_count # node_count = ( diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf index 35df5615ba..0f3fe77c4c 100644 --- a/fast/stages/03-gke-multitenant/_module/variables.tf +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -217,6 +217,7 @@ variable "nodepools" { node_taints = list(string) }) preemptible = bool + spot = bool }))) } From b85b5e265ab769d61478cc4620f86b853bbf51ba Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Tue, 2 Aug 2022 21:41:47 +0200 Subject: [PATCH 45/75] added spot vm --- fast/stages/03-gke-multitenant/dev/README.md | 10 +++++----- fast/stages/03-gke-multitenant/dev/variables.tf | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 12c5cd6f28..99451ff9f0 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -50,9 +50,9 @@ TODO | [clusters](variables.tf#L74) | | map(object({…})) | ✓ | | | | [folder_ids](variables.tf#L176) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | | [host_project_ids](variables.tf#L198) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L253) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L265) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L254) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L266) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | | [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L107) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -64,8 +64,8 @@ TODO | [iam](variables.tf#L191) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L206) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L212) | | object({…}) | | {…} | | -| [outputs_location](variables.tf#L247) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L258) | Additional project services to enable. | list(string) | | [] | | +| [outputs_location](variables.tf#L248) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L259) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 32d9525b79..0b57037e1d 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -241,6 +241,7 @@ variable "nodepools" { node_taints = list(string) }) preemptible = bool + spot = bool }))) } From 3745b2885ef1a43b46c663221e77aba0f2eee817 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 6 Aug 2022 11:00:46 +0200 Subject: [PATCH 46/75] remove support for preemptible nodes --- fast/stages/03-gke-multitenant/_module/README.md | 8 ++++---- .../stages/03-gke-multitenant/_module/gke-nodepools.tf | 4 +--- fast/stages/03-gke-multitenant/_module/variables.tf | 3 +-- fast/stages/03-gke-multitenant/dev/README.md | 10 +++++----- fast/stages/03-gke-multitenant/dev/variables.tf | 3 +-- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index 8c2d5cd345..6189adaa30 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -43,9 +43,9 @@ TODO | [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | | [clusters](variables.tf#L61) | | map(object({…})) | ✓ | | | | [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | -| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L224) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L236) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L223) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_config](variables.tf#L235) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -57,7 +57,7 @@ TODO | [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | -| [project_services](variables.tf#L229) | Additional project services to enable. | list(string) | | [] | | +| [project_services](variables.tf#L228) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf b/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf index ed95a02b0a..deafb2e08b 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf @@ -36,9 +36,7 @@ module "gke-nodepool" { location = module.gke-cluster[each.value.cluster].location initial_node_count = each.value.initial_node_count node_machine_type = each.value.node_type - # TODO(jccb): can we use spot instances here? - node_preemptible = each.value.preemptible - node_spot = each.value.spot + node_spot = each.value.spot node_count = each.value.node_count # node_count = ( diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/fast/stages/03-gke-multitenant/_module/variables.tf index 0f3fe77c4c..5c60991881 100644 --- a/fast/stages/03-gke-multitenant/_module/variables.tf +++ b/fast/stages/03-gke-multitenant/_module/variables.tf @@ -216,8 +216,7 @@ variable "nodepools" { node_tags = list(string) node_taints = list(string) }) - preemptible = bool - spot = bool + spot = bool }))) } diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 99451ff9f0..ae06167485 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -50,9 +50,9 @@ TODO | [clusters](variables.tf#L74) | | map(object({…})) | ✓ | | | | [folder_ids](variables.tf#L176) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | | [host_project_ids](variables.tf#L198) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L254) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L266) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L253) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L265) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | | [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L107) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -64,8 +64,8 @@ TODO | [iam](variables.tf#L191) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L206) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L212) | | object({…}) | | {…} | | -| [outputs_location](variables.tf#L248) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L259) | Additional project services to enable. | list(string) | | [] | | +| [outputs_location](variables.tf#L247) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L258) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 0b57037e1d..6b9437709e 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -240,8 +240,7 @@ variable "nodepools" { node_tags = list(string) node_taints = list(string) }) - preemptible = bool - spot = bool + spot = bool }))) } From 34d5156eeb3555190fd152930cfe7dbb23722adf Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Sat, 6 Aug 2022 11:07:22 +0200 Subject: [PATCH 47/75] top-level README --- fast/stages/03-gke-multitenant/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/README.md b/fast/stages/03-gke-multitenant/README.md index 93473f7d69..1974f9a723 100644 --- a/fast/stages/03-gke-multitenant/README.md +++ b/fast/stages/03-gke-multitenant/README.md @@ -1,10 +1,10 @@ # GKE Multitenant stage -TODO: one paragraph description +This directory contains a stage that can be used to centralize management of GKE multinenant clusters. -This directory contains +The Terraform code follows the same general approach used for the [project factory](../03-project-factory/) and [data platform](../03-data-platform/) stages, where a "fat module" contains the stage code and is used by thin code wrappers that localize it for each environment or specialized configuration: -- a reference [`dev` environment](./dev/) that can be used as-is, and cloned with few changes to implement further environments -- the [GKE stage "module"](./_module) that implements the underlying stage and is then wrapped for specific environments or configurations like `dev` above +- the [`dev` folder](./dev/) contains an example setup for a generic development environment, and can be used as-is or cloned to implement other environments, or more specialized setups +- the [`_module` folder](./_module) implements the actual stage code -Refer to the example [`dev/README.md`](./dev/README.md) for configuration details. +Refer to [the `dev` documentation](./dev/README.md) configuration details, and to [the `_module` documentation](./_module/README.md) for the architectural design and decisions taken. From 6af9953a4951f4cf06c655f282476057e8cfd092 Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Mon, 8 Aug 2022 11:04:46 +0200 Subject: [PATCH 48/75] Update README.md README WIP --- fast/stages/03-gke-multitenant/dev/README.md | 135 ++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index ae06167485..2c6eb8bae0 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -12,15 +12,144 @@ TODO ### Cluster and nodepool configuration -TODO +Clusters and nodepools configuration comes with variables that let you define common configuration (cluster_default) and define the single cluster details when needed. This stage is designed with multi-tenancy in mind, this means that the GKE clusters within this stage will be similar more often than not. + +``` +cluster_defaults = { + cloudrun_config = false + database_encryption_key = null + gcp_filestore_csi_driver_config = false + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + rfc1918_2 = "172.16.0.0/12" + rfc1918_3 = "192.168.0.0/16" + on_prem = "100.xxx.xxx.xxx/xx" + } + max_pods_per_node = 110 + pod_security_policy = false + release_channel = "STABLE" + vertical_pod_autoscaling = false +} +``` +The variable cluster_defaults is used as a shared configuration between all the clusters within the projects, clusters variable instead can be used to define the required specific details: + +``` +clusters = { + "gke-1" = { + cluster_autoscaling = null + description = "gke-1" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = "https://www.googleapis.com/compute/v1/projects/XXXX-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1" + } + overrides = null + } + "gke-2" = { + cluster_autoscaling = null + description = "gke-2" + dns_domain = null + location = "europe-west3" + labels = {} + net = { + master_range = "172.17.17.0/28" + pods = "pods" + services = "services" + subnet = "https://www.googleapis.com/compute/v1/projects/XXXX-dev-net-spoke-0/regions/europe-west3/subnetworks/dev-gke-nodes-ew3" + } + overrides = null + } +} + +``` +The same design principle used for the clusters variable is used to define the nodepool and attach them to the right GKE cluster (previously defined) + +``` +nodepools = { + "gke-1" = { + "np-001" = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } + } + "gke-2" = { + "np-002" = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = true + } + } +} +``` + +On the top of all the clusters configuration, the variable authenticator_security_group can be used to define the google group that should be used within Google Groups for RBAC feature as authenticator security group. ### Fleet management -TODO +Fleet management is achieved by the configuration of the fleet_configmanagement_templates, fleet_configmanagement_clusters and fleet_features variables exposed by the module in _module. In details fleet_features lets you activate the fleet features, fleet_configmanagement_templates lets you define one o more fleet configmanagement configuration template to be activated onto one or more GKE clusters. Configured features and settings can be applied to clusters by leveraging fleet_configmanagement_clusters where a single template can be applied to one or more clusters. + +In the example below, we're defining a configuring the configmanagement feature with the name default, only config_sync is configured, other features have been left inactive. + +The entire fleet (fleet_features) has been configured to have multiclusterservicediscovery and multiclusteringress active; pay attention that multiclusteringress is not a bool, it's a string since MCI requires a configuration cluster. + +The variable fleet_configmanagement_clusters is used to activate the configmanagement feature on the given set of clusters. + +``` +fleet_configmanagement_templates = { + default = { + binauthz = false + config_sync = { + git = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "configsync" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "main" + sync_repo = "https://github.com/.../..." + sync_rev = null + sync_wait_secs = null + } + prevent_drift = true + source_format = "hierarchy" + } + hierarchy_controller = null + policy_controller = null + version = "1.10.2" + } +} + +fleet_configmanagement_clusters = { + default = ["gke-1", "gke-2"] +} + +fleet_features = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = "gke-1" + multiclusterservicediscovery = true + servicemesh = false +} +``` + ## How to run this stage -TODO +This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run. + +It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below: + +... ### Providers configuration From a16cf9e2a8ec54f2ca83184acd851cfd60eb81ad Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Mon, 8 Aug 2022 13:54:06 +0200 Subject: [PATCH 49/75] documentation WIP --- .../03-gke-multitenant/_module/README.md | 22 +-- .../_module/gke-clusters.tf | 2 +- fast/stages/03-gke-multitenant/dev/README.md | 126 ++++++++++-------- .../03-gke-multitenant/dev/variables.tf | 1 - 4 files changed, 70 insertions(+), 81 deletions(-) diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/fast/stages/03-gke-multitenant/_module/README.md index 6189adaa30..d4f956ce85 100644 --- a/fast/stages/03-gke-multitenant/_module/README.md +++ b/fast/stages/03-gke-multitenant/_module/README.md @@ -1,26 +1,6 @@ # GKE Multitenant Module -TODO: add description and diagram - -

- GKE multitenant -

- -## Design overview and choices - -TODO - -### Cluster and nodepool configuration - -TODO - -### Fleet management - -TODO - -### Variable configuration - -TODO +TODO: add brief explanation and refer back to dev folder? diff --git a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf index 79794e5660..4b102bd311 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf +++ b/fast/stages/03-gke-multitenant/_module/gke-clusters.tf @@ -43,7 +43,7 @@ module "gke-cluster" { horizontal_pod_autoscaling = true config_connector_config = true kalm_config = false - gcp_filestore_csi_driver_config = false + gcp_filestore_csi_driver_config = each.value.overrides.gcp_filestore_csi_driver_config gke_backup_agent_config = false # enable only if enable_dataplane_v2 is changed to false below network_policy_config = false diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 2c6eb8bae0..db65df14ef 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -10,34 +10,33 @@ TODO: add description and diagram TODO -### Cluster and nodepool configuration +## How to run this stage -Clusters and nodepools configuration comes with variables that let you define common configuration (cluster_default) and define the single cluster details when needed. This stage is designed with multi-tenancy in mind, this means that the GKE clusters within this stage will be similar more often than not. +TODO -``` -cluster_defaults = { - cloudrun_config = false - database_encryption_key = null - gcp_filestore_csi_driver_config = false - master_authorized_ranges = { - rfc1918_1 = "10.0.0.0/8" - rfc1918_2 = "172.16.0.0/12" - rfc1918_3 = "192.168.0.0/16" - on_prem = "100.xxx.xxx.xxx/xx" - } - max_pods_per_node = 110 - pod_security_policy = false - release_channel = "STABLE" - vertical_pod_autoscaling = false -} -``` -The variable cluster_defaults is used as a shared configuration between all the clusters within the projects, clusters variable instead can be used to define the required specific details: +### Providers configuration -``` +TODO + +### Variable configuration + +#### Cluster and nodepools + +This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Stage variables are designed to support this approach for both clusters and nodepools: + +- the `cluster_default` variable allows defining common defaults for cluster +- the `clusters` variable is used to declare the actual GKE clusters and allows overriding defaults on a per-cluster basis +- the `nodepool_defaults` variable allows definining common defaults for nodepools +- the `nodepools` variable is used to declare cluster nodepools and allows overriding defaults on a per-cluster basis + +This is an example of that shows the use of the above variables: + +```hcl +# the `cluster_defaults` variable defaults are used and not shown here clusters = { - "gke-1" = { + "gke-00" = { cluster_autoscaling = null - description = "gke-1" + description = "gke-00" dns_domain = null location = "europe-west1" labels = {} @@ -45,13 +44,13 @@ clusters = { master_range = "172.17.16.0/28" pods = "pods" services = "services" - subnet = "https://www.googleapis.com/compute/v1/projects/XXXX-dev-net-spoke-0/regions/europe-west1/subnetworks/dev-gke-nodes-ew1" + subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] } overrides = null } - "gke-2" = { + "gke-01" = { cluster_autoscaling = null - description = "gke-2" + description = "gke-01" dns_domain = null location = "europe-west3" labels = {} @@ -59,19 +58,25 @@ clusters = { master_range = "172.17.17.0/28" pods = "pods" services = "services" - subnet = "https://www.googleapis.com/compute/v1/projects/XXXX-dev-net-spoke-0/regions/europe-west3/subnetworks/dev-gke-nodes-ew3" + subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] + } + overrides = { + cloudrun_config = false + database_encryption_key = null + gcp_filestore_csi_driver_config = true + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + } + max_pods_per_node = 64 + pod_security_policy = true + release_channel = "STABLE" + vertical_pod_autoscaling = false } - overrides = null } } - -``` -The same design principle used for the clusters variable is used to define the nodepool and attach them to the right GKE cluster (previously defined) - -``` nodepools = { - "gke-1" = { - "np-001" = { + "gke-0" = { + "gke-00-000" = { initial_node_count = 1 node_count = 1 node_type = "n2-standard-4" @@ -79,21 +84,27 @@ nodepools = { spot = false } } - "gke-2" = { - "np-002" = { + "gke-1" = { + "gke-01-000" = { initial_node_count = 1 node_count = 1 node_type = "n2-standard-4" - overrides = null + overrides = { + image_type = "UBUNTU_CONTAINERD" + max_pods_per_node = 64 + node_locations = [] + node_tags = [] + node_taints = [] + } spot = true } } } ``` -On the top of all the clusters configuration, the variable authenticator_security_group can be used to define the google group that should be used within Google Groups for RBAC feature as authenticator security group. +There are two additional variables that influence cluster configuration: `authenticator_security_group` to configure Google Groups for RBAC, `dns_domain` to configure Cloud DNS for GKE. -### Fleet management +#### Fleet management Fleet management is achieved by the configuration of the fleet_configmanagement_templates, fleet_configmanagement_clusters and fleet_features variables exposed by the module in _module. In details fleet_features lets you activate the fleet features, fleet_configmanagement_templates lets you define one o more fleet configmanagement configuration template to be activated onto one or more GKE clusters. Configured features and settings can be applied to clusters by leveraging fleet_configmanagement_clusters where a single template can be applied to one or more clusters. @@ -142,7 +153,6 @@ fleet_features = { } ``` - ## How to run this stage This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run. @@ -176,25 +186,25 @@ TODO |---|---|:---:|:---:|:---:|:---:| | [automation](variables.tf#L21) | Automation resources created by the bootstrap stage. | object({…}) | ✓ | | 00-bootstrap | | [billing_account](variables.tf#L35) | Billing account id and organization id ('nnnnnnnn' or null). | object({…}) | ✓ | | 00-bootstrap | -| [clusters](variables.tf#L74) | | map(object({…})) | ✓ | | | -| [folder_ids](variables.tf#L176) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | -| [host_project_ids](variables.tf#L198) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | -| [nodepools](variables.tf#L230) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L253) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_self_links](variables.tf#L265) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [clusters](variables.tf#L73) | | map(object({…})) | ✓ | | | +| [folder_ids](variables.tf#L175) | Folders to be used for the networking resources in folders/nnnnnnnnnnn format. If null, folder will be created. | object({…}) | ✓ | | 01-resman | +| [host_project_ids](variables.tf#L197) | Host project for the shared VPC. | object({…}) | ✓ | | 02-networking | +| [nodepools](variables.tf#L229) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L252) | Prefix used for resources that need unique names. | string | ✓ | | | +| [vpc_self_links](variables.tf#L264) | Self link for the shared VPC. | object({…}) | ✓ | | 02-networking | | [authenticator_security_group](variables.tf#L29) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L44) | Default values for optional cluster configurations. | object({…}) | | {…} | | -| [dns_domain](variables.tf#L107) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | -| [fleet_configmanagement_clusters](variables.tf#L113) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L121) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L156) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L169) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | -| [group_iam](variables.tf#L184) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L191) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L206) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L212) | | object({…}) | | {…} | | -| [outputs_location](variables.tf#L247) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | -| [project_services](variables.tf#L258) | Additional project services to enable. | list(string) | | [] | | +| [dns_domain](variables.tf#L106) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L112) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L120) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L155) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L168) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [group_iam](variables.tf#L183) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L190) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L205) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L211) | | object({…}) | | {…} | | +| [outputs_location](variables.tf#L246) | Path where providers, tfvars files, and lists for the following stages are written. Leave empty to disable. | string | | null | | +| [project_services](variables.tf#L257) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 6b9437709e..7547f168ec 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -54,7 +54,6 @@ variable "cluster_defaults" { gcp_filestore_csi_driver_config = bool }) default = { - # TODO: review defaults cloudrun_config = false database_encryption_key = null # binary_authorization = false From af336f82fe26b015aad1c54855823e188143a3af Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 10 Aug 2022 15:37:20 +0200 Subject: [PATCH 50/75] move gke fat module to examples --- examples/gke/README.md | 0 .../_module => examples/gke/multitenant-fleet}/README.md | 0 .../gke/multitenant-fleet}/gke-clusters.tf | 2 +- .../_module => examples/gke/multitenant-fleet}/gke-hub.tf | 2 +- .../gke/multitenant-fleet}/gke-nodepools.tf | 2 +- .../_module => examples/gke/multitenant-fleet}/main.tf | 4 ++-- .../_module => examples/gke/multitenant-fleet}/outputs.tf | 0 .../_module => examples/gke/multitenant-fleet}/variables.tf | 0 fast/stages/03-gke-multitenant/dev/main.tf | 2 +- 9 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 examples/gke/README.md rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/README.md (100%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/gke-clusters.tf (98%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/gke-hub.tf (96%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/gke-nodepools.tf (97%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/main.tf (95%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/outputs.tf (100%) rename {fast/stages/03-gke-multitenant/_module => examples/gke/multitenant-fleet}/variables.tf (100%) diff --git a/examples/gke/README.md b/examples/gke/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fast/stages/03-gke-multitenant/_module/README.md b/examples/gke/multitenant-fleet/README.md similarity index 100% rename from fast/stages/03-gke-multitenant/_module/README.md rename to examples/gke/multitenant-fleet/README.md diff --git a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf b/examples/gke/multitenant-fleet/gke-clusters.tf similarity index 98% rename from fast/stages/03-gke-multitenant/_module/gke-clusters.tf rename to examples/gke/multitenant-fleet/gke-clusters.tf index 4b102bd311..f94039dc5d 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-clusters.tf +++ b/examples/gke/multitenant-fleet/gke-clusters.tf @@ -24,7 +24,7 @@ locals { } module "gke-cluster" { - source = "../../../../modules/gke-cluster" + source = "../../../modules/gke-cluster" for_each = local.clusters name = each.key project_id = module.gke-project-0.project_id diff --git a/fast/stages/03-gke-multitenant/_module/gke-hub.tf b/examples/gke/multitenant-fleet/gke-hub.tf similarity index 96% rename from fast/stages/03-gke-multitenant/_module/gke-hub.tf rename to examples/gke/multitenant-fleet/gke-hub.tf index 9830c7fc26..b8679e4cb0 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-hub.tf +++ b/examples/gke/multitenant-fleet/gke-hub.tf @@ -24,7 +24,7 @@ locals { } module "gke-hub" { - source = "../../../../modules/gke-hub" + source = "../../../modules/gke-hub" count = local.fleet_enabled ? 1 : 0 project_id = module.gke-project-0.project_id clusters = { diff --git a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf b/examples/gke/multitenant-fleet/gke-nodepools.tf similarity index 97% rename from fast/stages/03-gke-multitenant/_module/gke-nodepools.tf rename to examples/gke/multitenant-fleet/gke-nodepools.tf index deafb2e08b..6f72484dac 100644 --- a/fast/stages/03-gke-multitenant/_module/gke-nodepools.tf +++ b/examples/gke/multitenant-fleet/gke-nodepools.tf @@ -28,7 +28,7 @@ locals { } module "gke-nodepool" { - source = "../../../../modules/gke-nodepool" + source = "../../../modules/gke-nodepool" for_each = local.nodepools name = each.value.name project_id = module.gke-project-0.project_id diff --git a/fast/stages/03-gke-multitenant/_module/main.tf b/examples/gke/multitenant-fleet/main.tf similarity index 95% rename from fast/stages/03-gke-multitenant/_module/main.tf rename to examples/gke/multitenant-fleet/main.tf index 470d403497..411d51c51b 100644 --- a/fast/stages/03-gke-multitenant/_module/main.tf +++ b/examples/gke/multitenant-fleet/main.tf @@ -15,7 +15,7 @@ */ module "gke-project-0" { - source = "../../../../modules/project" + source = "../../../modules/project" billing_account = var.billing_account_id name = "gke-clusters-0" parent = var.folder_id @@ -73,7 +73,7 @@ module "gke-project-0" { } module "gke-dataset-resource-usage" { - source = "../../../../modules/bigquery-dataset" + source = "../../../modules/bigquery-dataset" project_id = module.gke-project-0.project_id id = "gke_resource_usage" friendly_name = "GKE resource usage." diff --git a/fast/stages/03-gke-multitenant/_module/outputs.tf b/examples/gke/multitenant-fleet/outputs.tf similarity index 100% rename from fast/stages/03-gke-multitenant/_module/outputs.tf rename to examples/gke/multitenant-fleet/outputs.tf diff --git a/fast/stages/03-gke-multitenant/_module/variables.tf b/examples/gke/multitenant-fleet/variables.tf similarity index 100% rename from fast/stages/03-gke-multitenant/_module/variables.tf rename to examples/gke/multitenant-fleet/variables.tf diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index 5ac7e332f9..f1d29ad428 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -17,7 +17,7 @@ # tfdoc:file:description GKE multitenant for development environment. module "gke-multitenant" { - source = "../_module" + source = "../../../../examples/gke/multitenant-fleet" billing_account_id = var.billing_account.id folder_id = var.folder_ids.gke-dev group_iam = var.group_iam From c2b0782c1861b12fb0e381d58938e8f55e5363c2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 10 Aug 2022 15:59:51 +0200 Subject: [PATCH 51/75] wip Co-authored-by: Julio Castillo --- fast/stages/03-project-factory/dev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/03-project-factory/dev/README.md b/fast/stages/03-project-factory/dev/README.md index 4722b359eb..3c29c5dce7 100644 --- a/fast/stages/03-project-factory/dev/README.md +++ b/fast/stages/03-project-factory/dev/README.md @@ -53,7 +53,7 @@ It's of course possible to run this stage in isolation, by making sure the archi ### Providers configuration -If you're running this on top of Fast, you should run the following commands to create the providers file, and populate the required variables from the previous stage. +If you're running this on top of FAST, you should run the following commands to create the providers file, and populate the required variables from the previous stage. ```bash # Variable `outputs_location` is set to `~/fast-config` in stage 01-resman From ac575119c6fb0305d6063a9b829c96998437b656 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Wed, 10 Aug 2022 15:59:56 +0200 Subject: [PATCH 52/75] wip --- examples/gke/multitenant-fleet/README.md | 112 +++++++++++ fast/stages/03-gke-multitenant/dev/README.md | 188 ++++++------------- 2 files changed, 170 insertions(+), 130 deletions(-) diff --git a/examples/gke/multitenant-fleet/README.md b/examples/gke/multitenant-fleet/README.md index d4f956ce85..6acfd3b260 100644 --- a/examples/gke/multitenant-fleet/README.md +++ b/examples/gke/multitenant-fleet/README.md @@ -2,6 +2,118 @@ TODO: add brief explanation and refer back to dev folder? +This is an example of that shows the use of the above variables: + +```hcl +# the `cluster_defaults` variable defaults are used and not shown here +clusters = { + "gke-00" = { + cluster_autoscaling = null + description = "gke-00" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] + } + overrides = null + } + "gke-01" = { + cluster_autoscaling = null + description = "gke-01" + dns_domain = null + location = "europe-west3" + labels = {} + net = { + master_range = "172.17.17.0/28" + pods = "pods" + services = "services" + subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] + } + overrides = { + cloudrun_config = false + database_encryption_key = null + gcp_filestore_csi_driver_config = true + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + } + max_pods_per_node = 64 + pod_security_policy = true + release_channel = "STABLE" + vertical_pod_autoscaling = false + } + } +} +nodepools = { + "gke-0" = { + "gke-00-000" = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } + } + "gke-1" = { + "gke-01-000" = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = { + image_type = "UBUNTU_CONTAINERD" + max_pods_per_node = 64 + node_locations = [] + node_tags = [] + node_taints = [] + } + spot = true + } + } +} +``` + +```hcl +fleet_configmanagement_templates = { + default = { + binauthz = false + config_sync = { + git = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "configsync" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "main" + sync_repo = "https://github.com/.../..." + sync_rev = null + sync_wait_secs = null + } + prevent_drift = true + source_format = "hierarchy" + } + hierarchy_controller = null + policy_controller = null + version = "1.10.2" + } +} + +fleet_configmanagement_clusters = { + default = ["gke-1", "gke-2"] +} + +fleet_features = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = "gke-1" + multiclusterservicediscovery = true + servicemesh = false +} +``` + diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index db65df14ef..8689405add 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -1,6 +1,8 @@ # GKE Multitenant -TODO: add description and diagram +This stage allows creation and management of a fleet of GKE multitenant clusters, optionally leveraging GKE Hub to configure additional features. It's designed to be replicated once for every homogeneous set of clusters, either per environment or with more granularity as needed (e.g. teams or sets of teams sharing similar requirements). + +The following diagram illustrates the high-level design of created resources and a schema of the VPC SC design, which can be adapted to specific requirements via variables:

GKE multitenant @@ -12,146 +14,80 @@ TODO ## How to run this stage -TODO +This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run. + +It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below: +- on the organization or network folder level + - `roles/xpnAdmin` or a custom role which includes the following permissions + - `compute.organizations.enableXpnResource`, + - `compute.organizations.disableXpnResource`, + - `compute.subnetworks.setIamPolicy`, +- on each folder where projects are created + - `roles/logging.admin` + - `roles/owner` + - `roles/resourcemanager.folderAdmin` + - `roles/resourcemanager.projectCreator` + - `roles/xpnAdmin` +- on the host project for the Shared VPC + - `roles/browser` + - `roles/compute.viewer` +- on the organization or billing account + - `roles/billing.admin` + +The VPC host project, VPC and subnets should already exist. + ### Providers configuration -TODO +If you're running this on top of FAST, you should run the following commands to create the providers file, and populate the required variables from the previous stage. + +```bash +# Variable `outputs_location` is set to `~/fast-config` in stage 01-resman +$ cd fabric-fast/stages/03-gke-multitenant/dev +ln -s ~/fast-config/providers/03-gke-dev-providers.tf . +``` ### Variable configuration +There are two broad sets of variables you will need to fill in: + +- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.) +- variables specific to resources managed by this stage + +#### Variables passed in from other stages + +To avoid the tedious job of filling in the first group of variables with values derived from other stages' outputs, the same mechanism used above for the provider configuration can be used to leverage pre-configured `.tfvars` files. + +If you configured a valid path for `outputs_location` in the bootstrap and networking stage, simply link the relevant `terraform-*.auto.tfvars.json` files from this stage's outputs folder (under the path you specified), where the `*` above is set to the name of the stage that produced it. For this stage, a single `.tfvars` file is available: + +```bash +# Variable `outputs_location` is set to `~/fast-config` +ln -s ~/fast-config/tfvars/00-bootstrap.auto.tfvars.json . +ln -s ~/fast-config/tfvars/01-resman.auto.tfvars.json . +ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json . +``` + +If you're not using FAST, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. + #### Cluster and nodepools -This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Stage variables are designed to support this approach for both clusters and nodepools: +This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Variables are designed to support this approach for both clusters and nodepools: - the `cluster_default` variable allows defining common defaults for cluster - the `clusters` variable is used to declare the actual GKE clusters and allows overriding defaults on a per-cluster basis - the `nodepool_defaults` variable allows definining common defaults for nodepools - the `nodepools` variable is used to declare cluster nodepools and allows overriding defaults on a per-cluster basis -This is an example of that shows the use of the above variables: - -```hcl -# the `cluster_defaults` variable defaults are used and not shown here -clusters = { - "gke-00" = { - cluster_autoscaling = null - description = "gke-00" - dns_domain = null - location = "europe-west1" - labels = {} - net = { - master_range = "172.17.16.0/28" - pods = "pods" - services = "services" - subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] - } - overrides = null - } - "gke-01" = { - cluster_autoscaling = null - description = "gke-01" - dns_domain = null - location = "europe-west3" - labels = {} - net = { - master_range = "172.17.17.0/28" - pods = "pods" - services = "services" - subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] - } - overrides = { - cloudrun_config = false - database_encryption_key = null - gcp_filestore_csi_driver_config = true - master_authorized_ranges = { - rfc1918_1 = "10.0.0.0/8" - } - max_pods_per_node = 64 - pod_security_policy = true - release_channel = "STABLE" - vertical_pod_autoscaling = false - } - } -} -nodepools = { - "gke-0" = { - "gke-00-000" = { - initial_node_count = 1 - node_count = 1 - node_type = "n2-standard-4" - overrides = null - spot = false - } - } - "gke-1" = { - "gke-01-000" = { - initial_node_count = 1 - node_count = 1 - node_type = "n2-standard-4" - overrides = { - image_type = "UBUNTU_CONTAINERD" - max_pods_per_node = 64 - node_locations = [] - node_tags = [] - node_taints = [] - } - spot = true - } - } -} -``` - There are two additional variables that influence cluster configuration: `authenticator_security_group` to configure Google Groups for RBAC, `dns_domain` to configure Cloud DNS for GKE. #### Fleet management -Fleet management is achieved by the configuration of the fleet_configmanagement_templates, fleet_configmanagement_clusters and fleet_features variables exposed by the module in _module. In details fleet_features lets you activate the fleet features, fleet_configmanagement_templates lets you define one o more fleet configmanagement configuration template to be activated onto one or more GKE clusters. Configured features and settings can be applied to clusters by leveraging fleet_configmanagement_clusters where a single template can be applied to one or more clusters. - -In the example below, we're defining a configuring the configmanagement feature with the name default, only config_sync is configured, other features have been left inactive. - -The entire fleet (fleet_features) has been configured to have multiclusterservicediscovery and multiclusteringress active; pay attention that multiclusteringress is not a bool, it's a string since MCI requires a configuration cluster. +Fleet management is entirely optional, and uses three separate variables: -The variable fleet_configmanagement_clusters is used to activate the configmanagement feature on the given set of clusters. - -``` -fleet_configmanagement_templates = { - default = { - binauthz = false - config_sync = { - git = { - gcp_service_account_email = null - https_proxy = null - policy_dir = "configsync" - secret_type = "none" - source_format = "hierarchy" - sync_branch = "main" - sync_repo = "https://github.com/.../..." - sync_rev = null - sync_wait_secs = null - } - prevent_drift = true - source_format = "hierarchy" - } - hierarchy_controller = null - policy_controller = null - version = "1.10.2" - } -} - -fleet_configmanagement_clusters = { - default = ["gke-1", "gke-2"] -} - -fleet_features = { - appdevexperience = false - configmanagement = false - identityservice = false - multiclusteringress = "gke-1" - multiclusterservicediscovery = true - servicemesh = false -} -``` +- `fleet_features`, that specifies the [GKE fleet](https://cloud.google.com/anthos/fleet-management/docs/fleet-concepts#fleet-enabled-components) features you want activate +- `fleet_configmanagement_templates`, that allows defing configuration templates for specific sets of features ([Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/install-anthos-config-management) currently) +- `fleet_configmanagement_clusters`, that specifies which clusters are managed by fleet features, and the optional Config Management template for each cluster +- `fleet_workload_identity` that enables optional centralized [Workload Identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) ## How to run this stage @@ -161,14 +97,6 @@ It's of course possible to run this stage in isolation, by making sure the archi ... -### Providers configuration - -TODO - -### Variable configuration - -TODO - From 00e8666063deb8dda94da057dcfe61bc44e71e4b Mon Sep 17 00:00:00 2001 From: Daniel Marzini <44803752+danielmarzini@users.noreply.github.com> Date: Thu, 11 Aug 2022 17:04:45 +0200 Subject: [PATCH 53/75] TODO: adjust external load balancer policy --- fast/stages/03-gke-multitenant/dev/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 8689405add..9b0771d904 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -89,6 +89,10 @@ Fleet management is entirely optional, and uses three separate variables: - `fleet_configmanagement_clusters`, that specifies which clusters are managed by fleet features, and the optional Config Management template for each cluster - `fleet_workload_identity` that enables optional centralized [Workload Identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) +## TODO +Adjusting External Load balancer Policy: +Error ensuring load balancer: Insert: Constraint constraints/compute.restrictLoadBalancerCreationForTypes violated for projects/0000-dev-gke-clusters-0. Forwarding Rule projects/000-dev-gke-clusters-0/global/forwardingRules/mci-jz0ri8-fw-apps-whereami-ingress of type EXTERNAL_HTTP_HTTPS + ## How to run this stage This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run. From 5762d5925dba5ceedb99c7439fb5f21019044351 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 12 Aug 2022 11:24:21 +0200 Subject: [PATCH 54/75] rename example folder, add diagram --- examples/gke-serverless/README.md | 12 ++++++++++++ .../multitenant-fleet/README.md | 4 ++++ .../multitenant-fleet/diagram.png | Bin 0 -> 44405 bytes .../multitenant-fleet/gke-clusters.tf | 0 .../multitenant-fleet/gke-hub.tf | 0 .../multitenant-fleet/gke-nodepools.tf | 0 .../multitenant-fleet/main.tf | 0 .../multitenant-fleet/outputs.tf | 0 .../multitenant-fleet/variables.tf | 0 examples/gke/README.md | 0 fast/stages/03-gke-multitenant/dev/diagram.png | Bin 0 -> 44405 bytes 11 files changed, 16 insertions(+) create mode 100644 examples/gke-serverless/README.md rename examples/{gke => gke-serverless}/multitenant-fleet/README.md (99%) create mode 100644 examples/gke-serverless/multitenant-fleet/diagram.png rename examples/{gke => gke-serverless}/multitenant-fleet/gke-clusters.tf (100%) rename examples/{gke => gke-serverless}/multitenant-fleet/gke-hub.tf (100%) rename examples/{gke => gke-serverless}/multitenant-fleet/gke-nodepools.tf (100%) rename examples/{gke => gke-serverless}/multitenant-fleet/main.tf (100%) rename examples/{gke => gke-serverless}/multitenant-fleet/outputs.tf (100%) rename examples/{gke => gke-serverless}/multitenant-fleet/variables.tf (100%) delete mode 100644 examples/gke/README.md create mode 100644 fast/stages/03-gke-multitenant/dev/diagram.png diff --git a/examples/gke-serverless/README.md b/examples/gke-serverless/README.md new file mode 100644 index 0000000000..b9e71e42a1 --- /dev/null +++ b/examples/gke-serverless/README.md @@ -0,0 +1,12 @@ +# GKE and Serverless examples + +The examples in this folder show implement **end-to-end scenarios** for GKE or Serveless topologies that show how to automate common configurations or leverage specific products. + +They are meant to be used as minimal but complete starting points to create actual infrastructure, and as playgrounds to experiment with Google Cloud features. + +## Examples + +### Multitenant GKE fleet + + This [example](./multitenant-fleet/) allows simple centralized management of similar sets of GKE clusters and their nodepools in a single project, and optional fleet management via GKE Hub templated configurations. +
diff --git a/examples/gke/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md similarity index 99% rename from examples/gke/multitenant-fleet/README.md rename to examples/gke-serverless/multitenant-fleet/README.md index 6acfd3b260..f0e0cd2426 100644 --- a/examples/gke/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -2,6 +2,10 @@ TODO: add brief explanation and refer back to dev folder? +

+ GKE multitenant +

+ This is an example of that shows the use of the above variables: ```hcl diff --git a/examples/gke-serverless/multitenant-fleet/diagram.png b/examples/gke-serverless/multitenant-fleet/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..a282e7d5e6cc21b6febeadaedfb5af9d90387968 GIT binary patch literal 44405 zcmeFZWl&t(x-Luzu7L;+!Gk-&-911`rdEYTZl@uh=QSebfq4xhEhen$rn}pO98IC_UV5iyvL}16BR6!q+aS>p6ayC< zvua5JwmPJeyhXUW=j3e0U4DD)92A2-X;8}YHr%@cUAMdzBQPSuy8LIgu-{KuO$m{{ zezWtG%#_T9n(djSV!>vWxRTv%=t31;(e}Vc2|3sG9>+|khMUv4#jD67r8h5H1L36n z{(M2jTE&Gyl*f$>$k8x=y8Ot5ggX%Z|NOL6HWcnnA@oz$Tk0p+um5yqXD_n}!BYJ3 zQUbxgPz7}irN#a6mtl~Vl#cNJ_^rS{3u&LAK%vVU1JeItXKxBG(f+(87-ZE17qjxbzs&lR z93REt55>;ThSK^hr2hwP_yv3#`VXRv76Jj{1icirRrn9u0Mu;yA4K_oubPF%;l_VC z;rE2U8~^^FwS9tYWphUQYnbzeEO39yFoogP?FWp+x8(mp6YtYoOG{ze1iVFA-YHqX zu~`WJvqIA4{Q`QYj|>j}Y)aYtfJ(-_KXMEG*^=xy0aG&V3TgWDI{fRY0b9BtU5ELz z3$#uHzL`!$avu5T)oo`2PB0&m{FLI)+cbTio$cV_HO2eq)fIXIUgAt*h!Oqgum7LT z|BvSX7*GB`nSI*I=xQi=A*FRQYTBQ3i~kuZbE2mY8E$M}H2kR~c6Y|N(5sraXOu5<$Uz7+=Pf8L$Y0N~!0+(G}dkl(ue|FuLS zDV{0j9V+U1BH%DaEKh}oL8A~atUCrK558yA5Q)6m%$htn5IEs!zPOtlUYrcQ5Ohqi%k!8AvbN2hKouTE{1%}H@+qjpighga@rM-TU?jt)CtNUpQOBJVDIc7zG%8mf1>*Q*8ZTCFcrTo)^TrRO&{ zPenv!G=5yXY5&ZpRg4){T`^ZtlAlHAZAT#>9^-_hLo^VkYqQt#v1ifq@;m%S*blG8 zGGCMZlLW(x5ufAR1T0*ltnU}izE0B>IvP8CY5yd!M8NSZu8P&1so=+<~EZO$kT%8-&85yZNoy& zB5a9L`q<-|fhfXGw&Qs3!B@63`uv;7Uk?6r^UB&^Xai3(Gne7p=;x7 zsin~^)x^4#c<(Kaa4R-4US#(1EAAVgmz?SzbD+?6-69!FK1SZ`B_J41-#*&KzoHn| z2fwlB9=D-Ms=oJ|E0_#go!wXY0oy7MH$`qoAzwgR2|?Y;4kj$jvN~GVXC3z+ALa#b zJwMK|P==pr_q@32e5?0k>PPr?WH;xPc3r*t#69D<;K`iKMvp^d*ZoOwopGYW;*Ia_ zt8vL4(&go0CSL+Yft%~OFTAs9N27xFH`*ivM+wK;^eYhdK$)3%G_I3!#`*YW>BRJE z{;J3%jh22ebstMfimS=ug8M}TZraB|el~~ZbW0-Je9k`m7&#$Gj!CEd^}}Jv0$a`8 z*tMLR;bn7{Z&#ynF;!gD{k7uK5p`t$xCvWU=-%D@hvkimMo0D z#e(LRe$h(&d>zQsn$#(tbwmu_zZI+~u!YD`#n#jBhe6Bg--;ms+uPU_rk{J1*{ ziVxFF5Ow2;x0MbzyLy5J0!(+P!M7)HSx8xWH=BBI%#9iCzn;s&MQJ$A?`lyFuHzP4 z_twpRa~U8g;vk+dh9(djI?_+b33AuO=Tw(b#mRUxTwIOhm-I!HT-~Wc9@ZV3Qm0Bc z3DT(l$#JbHfVGssOjs!7XKSK!=!*Vj_ex=KullYbU+FTy+ft;CCz#tKf+Om&`9{WS znpRD*Fq)rEJdkL68e>>w5Qa=2wigrVY1LKwW|BHSD~S!p?gAUxY=W>t727L zk!Y#wwzbNaVq!9`(PjIRz5pEi&F8}RzR6io4kexjfU?;vqlm`##i7* zJ!+tbS4AmhsN^HN#s`a6QE%bXIUM`(RZgyz@l1RcG_(3{we#r(ED21kN&;kRTOPhQ zj86rH@%c{p*oQ7E3&LMI{~dlJ0Q_ytX8vD7>oewjQErH;bZzoRqKVggaZvtRKwkpa z8VmzSUnl4jq2)`}d^~++@~uG3hhU4@RGlhp|8L3Jm7ii7_LH+f=ks6mRQkI5S)>zV zZyU5Ce#4!7YmjCoN7U@onwL9lIxk;`@eA#c7VbA( ztk+R`lI`l=F42JKdix7CEm92Y?h{a;%5@+cb*Y`>S{Z9i%6NCo9X2b0II5%s=HeF@ zYd$`d6%22E@IpWm@07ULC8_^$XFGsmNEnEM{jcu)zKZQt5s~pgVk{ay7UM*|a@PG1 z(Gvmcr}QNm-~_HPhNs1FrZm!Dv4yK+9ZT+Xi!sKP@7TIP(n`|n(NcvDA>d!#%Iqha z2A+Kpo_(2gWc@D!iw3e|eUkCyM>6Yf(o=-hco24CISeSyKCecb{^+;Tf@DL-3cv{| z9F;O`g)(b~b!fDfirVH*J zw_XNBdYwVz1jn#Fr&`qqT4^-PyEo>G#i0}%kEMp)3A(2$KRtIprBObfh4Q7AOS$I; z|J$L4h22rZ;|k@L;)e4cZsn>iCV@N4?bDG8HIG+vJ(QEhs!A!=iThM$^X|$s@f9rx zlY+U|#`AlnUvcYGgwN8VZX=*u6tcHI2TO<>#2k2Ky9;kPCi-!UK}sOaZ>IZXZX0`( z{T(ip0loNX9f=te+d~xRf7bxnE`Jp=bpP&cl33-h3KJux=W~FN4vpGClu%u=P*F-w zQ9eG9@l}!?pqe&UD5YbdZyq~_j?AAIXD4C{tu@;u*j6@V5b2%|YV-n?-oRAd$Is37qg8%Fu8LCTm9C8FCpwi2rI<`P zOYBcx@VX|dSeoK(VE$d=N5VG-#++{YF|FW zigxLLcRDUuvIHPo5T|KHf@Xwc*hp^kZ#IlKRIdXyORIa~Eem=jP-%<8nY$=pMKF3k z(@+;u$o6ZCm^4Yac+r(WjMu)#3$YC`xavYE*oR(T9e&rV$E=wvH{DljQ)4ZXvp4e* zW=;3mdAq=?z5`{)kg?D%F$V2F(Wag4ulFiV2eY}RRefV94z3~uiWf51-f9{RWo691 znSxtiOOu3Kt*IVmaRbT_kWb?N57UT~b`p<+dSfN&hQ5eB8TbSc3gb(yuDe!)kiXI3 z_J>EO%y|TU%>TejJNhSAM%Tef3Q%JehmX-Qnvy2|ww7Ju7Cn&;6R^J~%Cy3Y(>T0~ zv*-}NYYIY6ch;*-_zTv2=6MRNwLIL7du|P)bv#}4h6B@B=P7X{Z~REjdzE9 zJ?|NH)td9QUl9bR->g1mmc`>MVZj!c^No|>ylWn`N@wF2= z#|Bw;sdh=!plJH)SKrgITgFTu;h6uvpsr4T&I*YjjBFP!M-Q+#yI}~>TH|8d_?BoedIA! zy_bZYXuGY>n>7w}=yYL)rFi6*Pp`XxOS167Q1=+V;@|KO1Kj|;oy`uCFa z{~&<>2RQ$RY`U&2}PRu*wvq;hIRitO_VP2 zXl)<=ivjrs`@ORxH8o3?Mr2MwGvwPBVUOwBkL2PMA31bK`B@E{@{fK0B~h;M#{lyr z=idU1nH%iqv}1Q1^GH%^{93% zY0#yttPU9P38P%+{C51h5C*Xh9CggUeq~A+SR@8Ck(1`%kRX1hAN`K(27j{D2j)~H zBX(j5o`Vlyl?);lJC`v(Ekuz8LJXI#iWa&LIz-4~UfgE!=;#l(^a!t0$LtI#vJ->% zH|0oI2inz7L45y`$V3VZ{5a232J6Vb+;71ij;g>AmoQXh(z)v_f{4uDgF5?rYbKEn z`pF$NM}aaoQ^Baf57rJ$ED%{itohTPV*iv{|1Gg6DUFeQ+K06$NH>+6q2O`|ckn_1 zn&wjt_Nx=;3pcQX5Kqz44>K5Q$vf!1n(XT+?|B+48-@0jPHp~D?S)_~Vz)b)XB$+v z-2_T<80S8~*47$Uwu&@MJ?#Rpb=nxzLlI9H z@xeE5s*JJUD85eWGqSXOuBp`~CI(BO@EB57|BgkGU8gs68C?OtkW~l2MgC(;e zeXPuQ52|euqeecrCv?`^4`JE$rA(2tHA8I&9NG5Xh1OARKUUcXG2F{N4>-Qf2ciQJ zKc1SIn>+N!k9XQtEm0q(^xGc+dU*P8)kbF41~kvRX6NU3*n1m=NM$b|6MCPCpH_G5 zs40cAcfXuVmcGIoxJ=0YWa8rn4Gs%#M^Z56UxO5W=?zl#J*v{&T7 z=#rrPN~yw3}HX*F5@Z3RNYYqinkqUd7Ty6_2LiO?XKH9o0zzGSIj)h3A5LWI}% z;B4No=(iDrt@NpPG;s5H>3Ap0Q+7(mBGLsvoA-lgqXl9Pe~rYGP6mg&KI4P3NTEP2ls5RKv!#H}prQ!JqsHa3erxP{ua1q>YHAO>lr{fYz|QB{j69SX zrVpNoP*oGvdJ}awk6b-g#-9$7CD+46S_zGlm-zuc#-9n`-nCL(8sx8{T9qoSUr$YXu!`RjpyXLejesjeD6DV6fYb!z6?=gU$@S4&x zw+j|fYJ}~MyZjv^i6j9%+mjlE;C~cqLZ+D5}{pbtB856TlYdWD{lU#bVpX-(^He-Vw6M3LPMUC4>#1^&Ys85 zAL4iI%9jf%{+ua=kLbC)X{qqJC(HRXR*0Uaict+o#R>5X;ujqfwOj1FGljCp&iwAl!1{-3ZeC)X|bDV#yOUZ zd5%J4MeRj6s+T2XXHnQOBd19 zf#>>!49#%2nS_muG1Q&uZ$iItgj3I1KxV?88c>>vP0W2G{}9OE5&ouXKXz9$smJ`I z={&9D=BXRf)3_($9ZlQ1i1kKOqj>5mss>q(u2;`vlDapOW$Ib5MwjdfHZ2FqHeWIZ z*THU*voHbMI-Cn;MlfwOLtoEnK#|#&9-xeIbluDRuJx_!CH3@?Vkd+3>M=eQHV=ZS zm)ZHoPrO0$P4_rse(yBz8}M@<=4YMn$4U~#nr{x{XNC{svDJkenV+@kG~`|$1&(ea ztzkjhiCz~GDFPCa^t-SH16VhL@NPRQ1E>IS|E>UdBY|YTBGQJz@yy@jxoRu}2exgS z8(Gj17?)HLKaG6U>_~b-Rda8Ux?@L!MW z8mp+K!}TD|{mnT1lV}e)9!7<$my=b`bt$bq$TfDiWDF!FXCUv@=9prfVp-0nhSm54 zpS7*8)4Mq5*>#l`_Cg85+z)SqpAic#1Y{7=RR2(XZ*AFEe5u>0Zx0B_OGGN$17Ink zU74;3k8Cu`-)!{1`xie7H0M1VRjjaFzQ`upprP*#5}$`|D2+xl^|Wa06}jvrh0*q= z9gJz0*GFTgGmevJyFR;uRJmJ@>k&Lpn$XoF$~oOmF-hZX$7QX5VRlpbC+KrsNUWqTwW+@H;XF zOviul`{Fn2(!YE!8jBS#Ec;VQu3wZjgraD3?7fgQbIYudD@7n8hWyID_q#8K8ul#` zpNT0Cf&GU+Cl`K9yKk-Kv&*<6$`_T$POki!@As-XL3RS0154Cp8J!GwM=uw{aQ>cC z`2w^H9W223xq3iC=}P(UVKyuK_*bvhbE;8NIV#vqj4r!!ZzP*0v&;f)o=y{fE`eQj z-jW_t=VWY+B-&F3_+fFrI@MrDNRp~j-;7wflv)fJXK!EbGk>zEO7@fqYthnTq|3^b z64LCf%GBb^cK^)e75MQa-R&P2Lm^|z;t=%lmxD`#^Y7-G-a@Y#t82t0bed_8bsMmM z7nJkJTH`3ckV4s?D{5>q0^MG z@m<%?^by9d?Z9*t<@kUB73FkVPJ{tfrAJKhPmI5)vBnhuwQ)02>?%FQGGiBZwwWti z9eYll#zH)Klz5Np;!=CE0OAZG+uuOC3rJZG+EPuw*gF|>A)a-F11_S0XpFq>avM=A^2v>=2qwXbl_-E!}YdMS=A|z-8 zZui%cw7YTeuPdx)g9;bV ziys$*V$aUkhlzWQTuo(+Wu&c(RB{E2faKI99=`&igpHso^l_sB zil9KZeG$0l$ZyY`D95omG%^!=w?7u>Fr%cRY$PL^)~y&4~kWM8N^dKL1P z8ss8Z0`$QwWK22asOE4#dDuI3N8uol$&_|01S5LHO^v4fhzY~0rkAe_ICr5u)Whi4 zm$gLDt?@s|u<2}Rp7{hGgfBxNoQYRTF_UBr2Jw>JojVoSjZR#Gq(O~?8>RUN(ZSv? zk)2J(LXsGRk2=dqfYiHuZ&_eBzYeF{h^y9dF8SF86r|Tb`aLs9lShM|kZF1F%cs+a ze&^>%_#1`m{>7eJ|YhcTD?JcpVX1%QDO2r4wvGcSvEI2|gk2^Y>H2&=8? zlod91g1h9m6N>_XU)NllHe1cF68aZTug7>Cp-;$gKGg22>-_o*xk+o2UiJxiv=Kz8b|kd{th;5lk}6#7bF!(&;%b9*QF`jgW|_m=XCaILsh?upU&9 zGGzU0W&rv2M~<|WxqL3aHdqr=l8TyFXxnYw3isjIqG7qBgg?^z^jTm9`)p)2=(2{m@#EI~f@AD&p#|r~07+z`41l9>!ii>rd2Syyv!uXUNQK>z zF?FrALmk+AogC&Cn$LkYcnOF*Y=cS6ZdDmccc69Ro0EocLVs;^_qGLcHoc{%{p;{J zBcECw_-eRHV)%q`4Y38;BoVh`CpJEtrwsml9}4)TOq>b#W%U)mjBUw~{-P;FW5-HU z%!nxbyhXO{r2g(uTG$6NbQU+MVCboZ@1vMwBc{vY6QryJ3gkJEh82G-3b3Klk1*=` zH;nqlE=(!Qwc4(=;;c-FW9HHw1Bb7n)Kke1p&2LcZOeVBnn6jUU&2}#9-w)h2n_0& zev-Xah`1V8us%}?lp&Vb!ES*BMGk93X(q0Zh6NAm0U67!KR0ntBk2;J-@SYPLoQ2H ztj3clf9>!2V*Q~pe%D@-Z`4m@Zynrm9N^!Ma1*68V&wi3;VFakSLEOkJzXLG!Kx&C zEwW=nQ z25=+mEt{o0_*TAKO+|a8}D0W*M)Uy5SSfuN5O3>@$c6}fqiIAj!N#b=OcoNFi?yjBw=5-1~p&YVR*RteP?2#@E+E^k%JS=!BnO_l~dJ6{{)AbW{%Ml}{IRD1n^~9dJ$cbF7U>xKgmRjXNgD|wM+=e8-+|X|pvn1^XNrnH z80Bg2^~SH^LSdzZcNuyf`$EUp;58bS?gsTOExCMR#TlpR@MLV~LMgk8Q(nd=@iV%# z$(a5+mN*Q3T0AlK9Z9ks5br?`Ia$YPU?zK8CcAP9p&>F&B@^!vo{UNF2^mKL#Z$V~ zz9LFLq9sDA6&vkPk1>btzER@-g1MhxVe8v8ZoAL9e4H-OJjn3iyln+SA`#Zg2f4+S zA_iXBEz1zA*ki?sZxJ2Vmp7@XkQ|=`AR>1V|Zf@BW&#;EfdGfCml2 zP#M7!Hl>Gv6%%;D4s}hk?WH#8Poj1#T!L4aD@7^FX<^Yx3uivq`&7O`c+A#ukl2lN~f901{SixQ6F*C)|`?x@NH7-Bb9?mfkfP z=`tRl9V_?6*R767u=d$^ClQrnag>gS&HXn%UJoxkQXOO%(TCg)JI#cQWtvzBmHO_) zer+r%H?D51C~C6>H=OrpbLbbQ%2obQ{4p`9YpHUUI>LVKjuAii#*ic zZhtnL34y2rY{9e6Sc6q`jRookOnew^T2Hye8}m|F&S2B>CS%y=EIrA?M`P1e>4axS zR1nDgJ(vclVqZ1zjB-t-DODHv*a^ewK2ta0OP1c=cI_>hHLN}O6;Jz_afm!B;dx;} zcI-i@Sb^VYI|5@k!M11KY4oJ_apg`b15auoi4gB!G?IqwLce24ICS8{kOToYrU>m5fBkUU1Tvx z7=&Eq*>Qpwj)GqVE-)VsF$W1+@jYHdNRX0vdwW}&<+CuAs0(w%pcvA=6r+m!K!+nv z_LQQE5riobB*cjbMn{Jfk716mr2GVRqEbjp^dPk!aQ?c$H#^?IvLm1kl-fpL zQ;&-!>A42m&Wc4V?3*PwxTMb|QXBK#>+^@3%51cVH7140vYZKC@&cM((oP5~-Ntdr z^6Q`@Cyj+SL*aa)WFp3ONn@qh2nJ<2iI?)!^SGHb`yOu?MO`i%Y#d?VT8FL_+}9di zan%`(W2+ult&QaQ`&aNS_eo*I*Mq&S@vnK=DT~u<+t*Hp1O;)nwQQT5g{K5hQigsh zKAb8Z4_@OxXj%EGE5MBBhh z$XXG8sWUa5Z{V1={_$-#naTaC;ys-2O^q+FEi)_RcB)H&Q&DxAp-@g*%|@hwb)=8? zrfws%UA^TReokVvbn9nj&=nk3o?FYjFTtx(y6f^_smfs!m;Kkg>p%;(7X`BRrJ(4M>C8U!Qt-Rks*r$(?KdmPtwy4 zX{INMdVcwfj@Ei+v9SS7-rhH6walZR-cos}R=BlfidP8+NAo_NH!hT`nl8qy+JgD4 zk%jH7%O6GMjh%|6uHiwJyNdVxgECmRlWAzmO*`?d?k92YfVX_*iI_*|&)9%1*Mp4j zio}`9vJdHR7zwYrifpXEX7{f7hdu8IhfJ$JIbOD`3z2eYL7W=3Jx(D+RQ zu+s{NT0AOARA`MpG-ZjF@x<#|16HMebffTdm7-mcB|ekbj&PJ6(zw2TD)h;-9gA2E zdZ>2B%7*X#2I=#3o_w*$|`A}U8R8-eTLC@2R1-RY>Ag@y9NqK&Yzg62>LuPGfG zmh!`j?=2=rXkyRF?k>H+X%6N)rIG*-fGCM^8=X5Yo(4bZQN^Cmf%8aP2HGRT*w5JZ zM5xr~GHkiFJJ!sZ1|zTRY(~mhA$eTanS*(>_zRPZu8*@yL2@XLnSKqnyK*!*v6ikZ$LsAKySlU$!Kv9&gz@XpbWLQ( z)|#S4VXJvHOvDw>gHAZr*kHLM5qGdA6Z5jD?@Zc7M*khCQTK3@_JUciIU;fpXDWB@Fp^qUlIwGpQ5qw6$5Pbc7D#-f{0tRQCP-Zh_4&48@? zp_b|N7rp%~=u;y3QvMah*&Ui~p_=fCAqvClU~G7DU73TD^E$dF15@zUZdy8B3W`YL zr4&k+Vl@$$ecguB?6=YtMMp{<5*`;|;jF0Fg!~Wf8VY`vdxRt==Xpk7FUsN+jD6_o za9Cd{Bk)luT9l$quD1lmlY{_;%(8Y&SwN$nsyZAZ#c-0%|f zpaL8yF$Sz1nm#j%*LD6aPjh5#Of->$nzg={EbZeK)J0X#@s)6y9nI^Q;dnm0P|u-x_{C~tPM$Q$9P<p@4W9NnOTGn9`fDzR&Dg5@iMGOpr>UoLAE`Q|8B0lM+oq0AqQq36TJ zARv3z1*pR@WGl#D_BlM79Bwok?G9;0ETyN`EXQee{a7cOVslJ;jsQ`ECwsU!x;8Ew zGd4qE7Wsr|o|<~x2kQ(*Co%0qRbf>f&%NqxV7jsSD55S&(}d)pb_Jt4{C-?r50v#e zTXWmosZ zED>eMSORW3tDOr)i@rv-NACX9>lx?WO{ecW3taEc`dV)Lwg=Yf${H*6MQH94rSrL$ z;(e1$ltO>SQab3}A17__<78d_pg!y*u7Yjv+uTw*&ajw$AzCCwIDvNC|>;bOrW}@`1Fy&%@ry#aoXq2X=*CATxcIGL!*loxBu9ykL z*dG~F!jM=y9YbHA)(=_ZTf)TftL~f{?%=?A`%{iDKUarDR%=u+@xtg$<3$#fRb)_n zLK=_NtHvf^6(SMrytuMm(g;$Ipljv^S^io#Mo?xiUrQN=tDE)ESJgY(Kax~Qd&f5Rv-YwuAhicDLR*|nc&{}llnH#7uf zqT>q|G5#+1B@kC$jG1dTJcEJU%t7)fLq(p8f1Qqv9io#@L-VFVx|2=$p&wN2X~LbC z?cr;Qf568TSYs7t4_QPZ#^!|w@t~&T^%U(|^kmWw1fX1LPYvfAs1$w3u9@0`xEW+- z#}4(NRzbvs_weZ(?h=}rG?V6@wcJ~!5UPde`wR3EcOi=nvIKZVb(# zFCeP0t?91aXC`$pTXFritC_`?b+^>fk2C3R>(>Cz()2Aj(foUO$(8U4FI>XycXuo1 zvdV(qy#n&(twS|w$mDucPANs9oLOwaL$6RpxknxOuItjZH4jf40b51<3peR5-`k_E zq?eP*!XR_$%=C*kRHjyKD0Q4%%3+V#h}=;U@@F}Aagk8CxX{dQvN{E-I)`MA9Ez2# z3*Uze`-^$kqR_(hi_P395erZXcHeFo{q$^$Zwt2GdTd7rf~0VD@lSO=$O2pfnySTh zo&&L5J}gwJ{#N~yj_kWrDq9r!W_HaxX*hlwkf~H9m?9Ud{q0jXevCS#xKwORX)Nk{ zA!d^4EpF-=`L98m3Ub4O77hsO7YZ#Y){hn}H@h3eAX4Ti+g#NIGBcl|a3!KMhHiI0Uj5Lh3sXcY&@Tt)SKu&CDDVFJ>YLqKn% zu1~4d9W*!T7_}0zN_aI5WUclEU2oCm5Vce+kcUtHV*8o^lOuA9%13r=2!+G0-u;^1 zdS#O_BPq7br}9OIK8BNZ*yF34{RYJW?p2SgvVJCTT?^PJMMQdCf;B0+tdCE09eeFl zoz?~sS$@B9ME*$}!ZoMn8IZ)(A|i20p8_z^*{}Bf^WPLrd1inN+(-hAJFofc#LS0C z6)Lae#HyO<;EZ{E*o9zha~fgFLov4qF(B))FcZ_!Pd)irY+MZh&%XOzdMCO#=9E?j zSM}y`6k12@i!l|&0eKkX&zIk7nyeQ_>K)qp_!L*i9=9>b@4H?X#ebfh`Xco~*PQAF z{P;|B{?0ok(Mrz^VbTcU%9Uvz&zVBFtU4|9;IMOW}-VP5M{ zF(teF9*i`ikpv}iZcl`>nZs^hk_SQudV;;P;g6^!rr@LHj_lrAAPw}f%IE(YdKi9GW;s=NvMpFfrK+7K4JuIco3hr z>?oYbb+INHZFPZ~XZHc7K|>jD#J!6<<6`kn^$aJ$TK(Az)nj_O4iqllMD+mZ+_8_w zr0xTyT^Svx)cN5r# zY+7L*TLd`yfUg?M!I%P4!=q&CrO?ub8~q}9XFl$HvX;>w(T|_L)~RZ~aFo_^l_rxq zt)u8jGgy7Swu=%OIF5Zi+hrqJp&w@nNsr2DyB4xT)2dkCzR%g zCiwS;ar`@ZgN@&u>M|RJn;!0OHsB^@7JsE)e96|Ft#N8Xp^f{54f+siHWQT+PEv&@ zxU4|}=1u~)i+RxmQ5N;4iTsQc78?5grrX9EpDNBqj*=hu(={{U8E{+xRjAddu}9OX8J{~o#9u^K;EPAIXl!YN$7GY)l(FHDf>i_D^8Ribo?1rNkm3h z$e9zQJHljmv#O-qL#=V{I`B&4THS~7^|2)!idX9ISpc4QtsVBFUQQR%qOJJZTupJYAUewN+rMW(&d-s)pivFjP(!%|OJoiKDv9J%U| z37&Vz9P$th34kt;@9rJ}G!P-lZ+`i`qOh~ek5Vpn4``V?SMqS(G&TIMQDsrrKm2mC#8sm*F4B*h7)ax{y6I2K=y zm_O@B@7Uk&DoK35k-oU#6>gB?-O!8 zOI(kG?Dit)Tu{5dQ2!cZ!|}0)$>LZxkR8JJu3E8B~)0~EhV)M$5N;g494c; zI~p+Lm-H2}OVnK}wR9Art{}dZ4mm2nP_bumJ<+|jamKloXsY*V{qCP0T=1q|gc^FD zz`P$EELVM-5aryA-ksGP_@T9PsqT0OxQDyOSwCdA(b!vAW&iNi^b2DjbH9;h zg&%%NOkmR3K0b;_GRQY)`ev}=whl3ORy)|L8MnbbM9aFWHsK8#M== z_?wizD`Q535>m>I1Qc4f8w(bM^}dR=eVF&=b98G4>e?0%oj-Z0pAMjbAl?Qi!l`0V z@ly2QtFcHPvTxv|W``@xW5m?mL+WXDvJBe0@6r7*idp@nKF;P#5=gYtVM#Nnih_4e z)3q{*v2w8a2!Vt}I?JY+&@Pnt2g_26M<}ju$~+b zmKhxjx!J8JG3h$qZ$qi&|r3%?f9lLxvQx$9~ydke+cV zN?c%*boH2`i@;aM&=ZTU+A(KfW-xrIQ*I<`%Lyhl9kZvHAXvmcGz&>FD^a{VNjOc= zfAAOmoMr_KU(_~Mxr&lFEaTiaEha5|Aqo1Sd%4PNm#sLt{Siukhe)qkC-%DXBH2Lj zdrw-cLb64bGK^Jw^W%ZF3KvwUOv>QI;vMVcvmA<+u1T`OJbCvm$`hokI|kYFyuFdN zy9BrdAdSq$nM^e0WzSq5h@wCvOb2ezQZM;24@&ediD$vydood)R(Cw;}MwWxvP?c#p$|R@E?Eg0u!jAXGj9 z&43TDQ;P4iSOU27W4QpYg-{|wNFz7lZm5BtDPaQ-wnYuk^iq`M-SaoSa&J3T4Epn6 zx7}m_ zAB_{!K8);K>gyUir-jT1^xk^ks|cQ{I69{nle28#n-KLnG+b&V?WRp8HVd)3UUP^i zxo6gqS5))0$|PPUCsS)5C_~i#Na7) zDQik#t3H8GPy>jkJXf%)wC7ZhWF9K(fI_oStWZE6>M|U7} z&ox(gH2_->3o%}3f1y8tY@KhoJ(AhY;MgVbykZy5Z2yO-Fet-z<-<$W%Gs8=g(aYGt&CHIo^~^YsQil zcJ+&KqC!KuItxAhG`uV2bKi&>CSX?J)~W}?Or$3IWK7&=E5hO zj>e1SGd1>SoPcrk9J_uww3PKF5MFLd3Vtf{IV*CUez%TFx05cqe;2CgC9=0`>8+ur zzzXu9q2B8yZ1)P45U!p(NIm0?3b43)Uzwlqhy}Q-Ug`)7SCeV;)nOo6ea(*bP>JOy zg(kF=JzG^_<9sp=6?f5gCtANARhZ{2IG;oLTwJNjAvEY^4N5K{ zE!8m_T>jkTPbK``MmBTkyrqyCk3rVNGh+gwK zcIH;6iexqaO6-7{V%5t%G4p=in}SXDnmSGof@5F;?zvkPy;jqtcdiIg>!iPkuSkrX zw8w0C$}1*(W`cF*q#((&A3zz6+{v3|NuQy< z14~B*rRxUGLAr|@&C-O$4ijm==4OF2UX7WmdmqjNQl@6H4=Eh^U;%>x*rc+)Ik9y# zlAEcnb)S~BiwZ`Z`12AjqOt8HJ_|lcekuW51iYuGTGOG? zhX1j3kJuqT15A_xhlX4U{VkB7UldkfgcSIp3cAN>SrZYRH>k#_Kok>tUzFF$cV+6z z?-^2Zm^Yu^p=Aohs5hwfxC}+z<^hAZ_KMGWzbXSwb2`e-6W|PYx)dPwZB#{LkXrR5 zdZY1|NKZ`9pZKuKc)zBMpT^mX8e5TUjt=8H8SuRG>lbXkbUNbU_F3Y@gim-*^5)!J z92mw^3kstyV|zq+2%0m5zsYS?6(wXe_nSK3s>oT_9j>q9lGoeE#1Je}TalJC4+U?j z>t=VLg|*LO7@`IwGK+*?Zoo?|{Bn_i2UA}2kORrID?-b4+YJ3>u6Z1wY)iU);Ol3!xCpB8AIlwcDwY2g67+6ky|Apeaqm8^x0VUcgNX~iF?ZJct+ zKQA?OIbPnFR@Ji81S@VZ-Q{LG%o+W_D2WB;!Dzo9u>yyPB(dxQvaw&=5KWDdQNqnl z%@(aJ*ZL3nQU~THoqWuHMFu=GX7ppQ_@8Ggok;;TG2#N&)Y-jm`O!}WF5&fwryt_t zHpo^`@YFY!R=j0SSQsRViQ7>B2AC5Tcn!p#GoeUDXH7yJXCVQ?FOPXY>7y z#WZxbKtt8-nhW9a9Zo=~0^)+V$vSS+A7YmkhFRp%0Q#2spq;U%*clG4F zl|`FtY%g)aTo;B_04#oOteah;1sKDLWc$6MDTYE9C-YjacdaL;C)?z4aXQYQ8g}A| z`><45{jJa&#$F9i+Ke9!TF%#+e<3g+QDP8wKHt?y=XJ$6UKD!mm%;103~-sM4~rg; zy)ae1ILsaUwceM6ye?>nn8ZH{&xGAUim)*kJ8FkBrtzQq;Q z3KX+fO%KWVG$xW4)~9MG%fRdz{OT$_`c)ceV`c6Ll2@XP zljb|=XN%_czx4$OxW;U!LA%v%EIE1krA*KLl#AENlQ(xhbM{?C+W={8SJ&HDO~mU$ zZ#L_$_{#GuAz;dVIX@hCACF$+Oc(9_2F!jL0T!NLKAadKM5gx{k*T!~+Mr+PZ1_te zHaqf#Br&aS$kQhQ2Q~&6Zv|c5Dyl`>YogW;tI0LsyjzzTczlkmb+t45ZQT$OZ?ITRq-@^f-XyWx(=Nv^v2Vond+F<15*4mtr|m-m9OTkO;^>n5E?aqum5yh&Pq ztZegr>r>3aP`ap@$JamIu&g2h2Xg`=9q8@HE zF-DKB5-nJS*PI&;BLjIBoC6Wn3(#)wE&yLN$SxZXc3oWkzRrpMbcG-dpx6DGe2&6o zm^t5__r|&=;$*N!eydNONgIK66&{Glhr7xvDUptkkKcNfxm&Cx`F>CmNXdBEEAB9? zm%Q>XGyxmTOk1SV^1b!SMLI8^?yOIKcR*ZEph@%x<;q70?J@Y1ue&^6_U99r%AF3W zHMut{UF$tzOFQ+9J-_$A;51%gtmz7heOYu6@@x*uzXoFCn)cX}F~s>;<-4||E;X;+ zD67c(lyZCbsI+uM*_IRd+oKKZcRA}DDw0A^L-l5^10R4_tsFEv!WnR#Nze%!;&`I3kaTl`nu9zCpmK79eCh^tc%?w~{xaBNDjDVxe3NtixfNW4|;m5ck zy70&*e>W`+@7`b1+WRl&J*zUyb~=lY^T+rLM%rU!`9p3S1&CB}~TS+}|Bt;IJ>_K(&dt4~J;PDT=2 zu7YHh(4kG~+I3BM9vuTOm<;xo97h~KMj`#@Q#=t{m-%uD+n7%VZVS6VI;id6FKIC@ zKe@KC+PD3cX>S&h+A~*Bmn3Z$^x1{p9+*_OUP14J^B0;027=o|)B>Y}Ycje6%^>Mz zs+;{HGBkm=##l%?Bpa87rzJu{%Fdfsa5 zX*e7mQ)ptD!XWhaUBeG$8N;Nyv5O}}XsIJVDY)g{Fjsa{>J^8ydu1%jqNXZMA_W;p zbnRC95GLN&s%+ss12BqFM5FgnCdnF4o?59MR8;rQWcN@B@|r%N645&T;PdYH*9!OT zQ8EXs<7o;1$(1d~pRDV6O4s%1AM)wL$42MX@y0w}8BVcR3!pk@j+rixTBvI_IH-+t zfXl9Bblk4{lIq#;8!)PE-{7^8(LL0LaYt+%JFshl=|U;M!)N!X`A$Hbn}v)NJf-eg zjA-Y9u#BmRR}0dw3~GsSjlM~%34hug-I|R@cpa7D zvv@thB|1q&!AxXbEw86!>!S|X7x|~=O*~*vin9K7p#YolKh(>Ft{%t^;NR`XkG9RH z#8eh$>H2M?Qf}v4Pb}4_!w=8|>?GeG>A`rU>YXYGVlv7~^!;yYRh$CJjtu0r6Ug(| z=5IUJcw`Wm+RkDWMQpbEiUe(c&dXhFdq^`@R||~D#$v}X@Uw(JV{@G)EpgBwCoZV* zXSAlJ#9c~sy=-HwF!DZ1yAy>LJb6q20>HO!_sEVfGyF|nDHS=6ZOE;?9CwY*xo`Ss z(0qD?Bf4{M@eO}gc>0TMoa35K@pgs`A^uHzripR{Z~)GOUH#j^ORc@c{zbCBti_MH z3SWxQ&|jN-&qxg=X^pFiiMHmTcnS)Z3Yk&mSK|beI8Y2YDm&UF#iA<>`G3-mc8&66 zO+~B|j`RlBGFB>Uh+|}}l-q2*hPmzoM%7U4&eIa`RXW=UO?DX1LfWfxjZ%V<8AFtX zvhb&NHxhhLYI&;01UCET$Ng;E(1z=r^=)avC$9Ri`Lj#bimV(KI-qK zf1920$(d=9AGP4XuNE%LpPwhwc2ld^z0kV%&Sk7Uo`yp0{xi$WiYa5$Qw+9++n-b4 zfDvf-#dpsS^g!hPq=ocN^alpt_Qx!g^#Sody1aDpDzQr%IK}CqU(k*Ef`a(%J-8X& z(O??> zm6J9#A50z+#*ZnP`#BBVf!t6a|Lq$p9Qr@9S@5WvL-2WPAKz&sxKAi-$&rKZXOU4P z`KkuJ1tKURjrp|`o0HE0_e&5(cif2 z{8b3Du|CxI{OeKGRAI*^;FXd{WHC=)6ju(`Agy&}eaVF(zn78Y$4o^ysGi=`>JGqn z2XlW4-Ef^rIaU(}V-mmMK>c^)Rj(vWHdIQR|N1l~AtrW*r);mF=3Z1kUL>E>D`gU7 z4^w(8cU$!M@Yh%RS>-IWni(27E}PbgiHOyTj;zcPK0ar-H_Q;2Ma^OEVp*wHIPCSzqS15xav!&7-owN5{>T ztc<&28#_)Z=Jq_0Kj*o>iPY0uTG=nrcN5xGZmLAEQ7R@p16W|r#Gy)Fpi{o@W1^?oc8;}z{U?6mK~+P;rZqFFWy=1g#Z zq=!kOV|OLCf8gx}#wG2dCvVjEr>&XkVB{rtDHOUmL_x~vP}Ifh%}p)U5M|^2AmCLz z)cnUY|M7&F!{K)(DpCaL7eyL`>L{*%O)%8()cgk?{dn5fYCV7bmZ}Yc@53LyygZqs zu1Ap|Iv^U?DRg($t2P}n0@4!r4)R@UNnLq2*~-}{(nrf|Hd%U!xLCl*GUa|-?s$LxE0>DTNU~nxwBErRWC12x9WP|=#|t5SYdNg%=H*+hx6?0D2Md7SLL~l8-j7;H3qjFW zuBzNdXMW^5<>dZmi@$hSRhZrMskXK_N!4k_K9&@z%#}g#@oePLbq29g*qwkc$y-qBh&Vd(68ezDrzWV)mDQTI#Ak-Z>`jraXdO^XHs*HR0HyjIEWG88B^_o|_ z&ME*CtA=YOEC&=uy2YJ}tcGWNxW27Hnr3-%5f^8;>ua^Q)U_A8(T2}I6Q7){wfCvN zv1U5iueBzm&yvrTm7rpW(2^8vu!UVg+g>$Y-}b4?Mr@LfK0&C-KL9ceCjMIGL0r%O z#_><{=Yt7V1b0(4bRo&is9czN6#ipPeC^U`K$JL^sIUhu^0*<=*eLTG_wi`s0DkKd zCkvC2y->5}==bz*NUO?U;t`4BQ~+t#G&otr=xeth z+C9d-&J5zS56psZsey61F8ObK_Mhe{B+?4`(gAfB-U3X}+QaYkiSyW~Q1Y?Ulr&(d zM$lpLcaH0(Rwya4_#g~CMt5(No`ZOh56~=joyULaJ#e-{$5NM45a!S5^$}l40Z1E0PKtMfN!Ubm*PVSD`xRXzJ_kp-jXcC#r9h!wh zqidhT4A^mQd{<}G@P};%iCB_wsOpcXq38_pUDv)5VEE{hwgfB5GhE|NbQ;_q*Qf zs6^rJz)}D5WqsnBh`166N6NVv}#Q zvvb$}#x4Vek&EpJaFT8N{Lc^%;EPRLsJNsyq8NBE=B(VZJ za1A?iO$@{w+ATj9dli0|VQIiKBlu@F?vE<0!4QeI{_9-8(sZ#P;oA5)6%Xnogsnvg zG&J4*J3k5*ekKCGz5Y4!+J6TBfW}y>(O!q&cMGu#zj=ZkTr2ax7+kxIb)c_55cqGy zP5#--zr%#Bu*tSAmcMaO|Cd8{QLb3}{ecqu&(xsbmv&pqKQ`~R*IZ6ZoYsY3lkp{w zC1?I9N7PF6oh3BzNxXUlZBySn`~+J0?c-i)Imch>QahSHeK<`{qmOS<-MAXr8W|pb z$2d+82^a(2Os?Nf9x6a-~Pn{V4uT&j9emShKuIQKsZ`H@R8vV{d)3p zk@NN<^`Yeb`R3s8FpuAJqZ_Y@{$BWs&k4)6-=){Ha@!ehqoWP?^{n3Lo4Dj(K3&1p z=A!Wn0ZS1CAGbovH%B?^tG0TIZCX<4^+${Zc$e+ME?pBI6>a0sfGKINxgIa4XBN&d zGg6m+xl@H*hpBtrl25z6&jUp3Jy&l_k#QwS$|6jStc(zpj_R=u2gME}USq{J6@tRT zt?yct+hvINPS!~M2(N`7)*s^1in~2CVe-T7N0JB@8&4 zBRLD-ud~G{K9!aA<8_H2F)|`rOQL$G^#~jj$_-a~%Vv7-K*WrgPVIqeZatqJJVm!w zmer4n^#g2?M&hp<3se3E4`vXz5YpK4e@E` zd)X~V=xdC4|8Obn8f9-GvvEuCc8HLEW7D1WP%Ik~bb^_#f3e}eto$!m zgK5by(6IaoERSRS_wxU8^7N#81a!Pm%j)(299pLyv{D4cNi)>JtCMq#xQWOg_?lfTa=rro5q@a+3hp6fm*2 z@ZcjNTp?+&eRg#A8h-$|6~#3%Q7AAT(xiSG=A|2 zHz}03v`nzQPxFbmm*Tsn5xeR8gx3#lx=?24_+F9x%7JFlU?(USYEzGTB3G;)kW^h& zW!ki6)8F6!BA{ij^9$rp&HAAwDO1gy3I+E&cQzu6%OpMn7r7h2vI4xijn~@;c!I5Up;`aoHj_D5! zx=c$fa7sw*qrH#GBIX)kNvTJbimGCp*ZN0CpHW_Iy9W5)G?UOE;ed#-&3^K5lZw2#NizCL~M7%%`f>wktCy=9f0XH6K z&qUwC{R9|+>dvn%@(&;4uHaJH;Np@O0=ccUAtTbVJtpD&TljX#VaPOBAi=^o5RP#? zV2`)}%e@YWV8m@>2U1$@MS+^!l|$NhQty$s%3tvb#Ngn7mHP;oulU>pT$k26Z%A}E z>E6TyD`wX6i7WHAV~7|*-J z@&P7y8_0CS4H%L{DbNv^ML;f4S=(4WTW;{b@gxjMcTaXav3;Bz*wT4+g(|Eqg)!p( z`k(ezenh1dyicwmjH3gu%rk;;RCwq~|3yVO*aOuj9xnDku7Lwl2fLK}+EPodzzB|} z!U?F8SY9Abb-+shO&^ic2P&;sPb045l0N_^VL=LxS_Nn>8+V}o{-=ZZ0Hcb%xe62^ zIA@@DeuJ}Yx2gW$coG~>xMNoB%iSN`{Fyct1Lp5TDDT$hes`sT6cdsA1rmS@8O6Sz z3@H5~$mpCd7hk_D^BClrG`K#nyI5Xu*?UcoAiF>nrFnwAftvujotXq&^}YS?)VOVb zowO?Us=1S3Lj#=lD9yX3G3G0vvb?`VD6Y|;WTkLfR3vTk^3BYp$VeT=n#S+BTp}F< zUc(~@{SB144?0VPjZi%AI?o+FNftKy!T}8NecHis=4-B=GKz`TS+j+I9$dqr21IGc zTgk^Mn5(f*eTa=N6*Q%9{MsGucxZMp)gK*+!=2#m;vD4YX#U5}M)7IVf%`0dI442`BeH}|=dhh95(KAnp&XI^pTEX(6K&Ls5PdoQaF|k&XbH+k9eoFsP zN(9RGz4?=`V!aWO;!7^=p`M=gVf2&Cd)-zM?;82-;%|FL<-t8)+9oM8L`nQ~3MSnn z{}Ka}Ak|uT-fw46b2Qs2^=!z;yI=X{RWmf0ahzE7*_f8=Plz3 zd^$SdNRQvGx8%?8ZpJp{2#^ejn^m;uvI?AEhy+!ete$jQA9NZgE~*G5onE^%)ebxJ zda8R_-;>nGW%p2W&s~o**Rwp=D0f&v>(NaQnX4f;D5=pYfGtMlZj*LElxuG@ft8gPet5Npvb zy4YpAZDLA{Mh3m-!x*=svvf_pyypYDcBcjOPrSXY^(rR!;(}oZ<(i3${R8~ln?@s~BQCi448zR|m(Zh|Sp3)F$Y!L3%z-d~ns1 zI=I*!1N(Ulyq9Ajr!4S3S>~6(>ySaF$-|J(z-u$LzA$I>Ev(4#DP1r*b&!J z=Xtu&DbTZDR{W}wVC?(5bb8SCz;8WTNTk+2dD3BYp#gALMr~6rGYmns_B8i}dfM%F z7uwiN`PysQBnp2ek)tBn;Mxt)$@lXiyK_OjbCil3enZZ;akbBhwcHQ?s2t9E7=Nli zTjb@p?Ee-y6JUs5%cFzpYT>g%#hC*d9^1BmM>P4^Bzdnt4hQ~X1FygBbF-NdEaHX2kC_-tUF9m9MBDz=K?!yMB$z zBkh&wfP5N3&_o_xzuvC&e&zSRI7`;nAky-D+eoB!64_T!c|tYz;uT@H`kxT^dC_Db zEJoP}oh-?cjT?hm@A3TOu&Rw}I{Ela%=}X@aB~V^8EluJSB_cNz)AZ$1j0cv5~WU+ z=}4b87hT!?aQ-R;eZzA$QCCE19i=&N2hqV9z=!JmF4_wHzCy}lU6t!nW`Ly*;)8zl z5x7eZhDV)l2@c(C{gd=t#KP5N-%O4(8CM^9SQ*K?_|=qG>$Hr=xyAX;(jW4)ex*gc5PLY|eox!ERcWM18a(9n7$@%%ROwgB2$}0D}49ru> zXW5d{co`z*bE$zNP-Xg|yEa|4h_`d!*lAy-oah$H++i>DY>3e{ye~5O*3SVEuH~b6 zTBb_lqkIr;3Ja|fiP_x*|S6*_{zjP2?XB*f>|YNpG%tCJwu= z3OFgH-Vd@ht+_Z^FKNGFh96@i(oPC(uay(7d=@zp1%AF7;%)Q((tZGPu`2kP3Bm!x4G$cZUD11BHu&-k*g?M@RYO@}rV=_-!+GFL-I}C+1<@Xhj zVoJFdF~pzFQchrvPtK953T2Om-m<_KhZ}t3s!S%PcZa$^Qe3ki=3Asj?WYIV&77AX z91&}s;(sEX#(|xPEy$BMLB`qu=7c$F|e^p!dC&HIRw*#Y*K{oDz0T z)1t0gew_(Al8BehB+bzGQoe)-`1M}VJro98Ug>c&{&tF!k2^ zCx}>xY2H$?^H9Do`SaJ$A@`sI;w3|d#`xp-Sbf@+89}=fmlXF^57lMOlVq#!K=Tj( zrTM$dkF;O&Kfmv@272kEDjk+H-?UG!EsM}-!`{FS% zF&$f;O$l}SB1SW~PZvLx8oVY|WFYJ!rsY)}9t6CQvkp^M>5N@LfrRvhrrENqc6`kaUHi*aIQo z>S96P==%=IMY!puRuJ?F6{_^wn)`96_@CCXwPh-lBiGyG#keeds7(i^dZt%Sl!uT(9YtBIRz^1JwF8m-G4l0j6YAgaaatpAQNGNy5l_uDKyQ{ zl~4kK?w)1J!Tb@!aJ61O-@r#^*B2OAS7y3u6g_<}^6B@DC%Z&uCNEz}1gkyZa$-vi zB^*T1BD7nf@$T9l{iSXN3F52uLuv`4F-vlL)Xs@)oE)d}kjOOrKnfE|%8dFBEb=efz@o z-Z1p$nmk5VbDr|-q>zNS8A@Zckn+uHS}+B3AR{U5EwbwC;hV5e$E1zo@~Z1AgP8_h zG>ce{0r*$I1-I6kcaQj1O)2x%R3xo#^}S|Q;FS`JYxWN$RnP5==~fTuzwaEHyw^`N z|9Ummkm9`Oh4E+MR%HaYS5@u%FJ(5Uc(<;&;hL5*-qq8wFG3hX-U1j^Dcv5xk|{hN z<+GVAwM%(;@taU;rRVeN2*#vcnaDI?Pikg~Dxm9O^ZxvU zs6ONw?PWF-Swkv17)F)%sf7j3u+jM^iCx~?AZiSg<2_I}2B@(tP@|Emkyee5uJ%P& z^0CPh{5tyGh+leIfl%9CC-vGR5XzF2)>UvnRey#+Hgx-uNsi9Ybka1gQ`T(3H9r?h z{>0q(nr9HO?IT-h7WuqGjIk0t{mr%)>@iR7P;i)@(!JGx6bVJ(k-L_U)xof#tA%9+{YLApRD$v^D~#KjjeZJFzoBZDfllyB6N8Eki9tfx z{9w)gbB;IUIo>1MH@6AqW&Fcn{=n#o)V4KfRE`)~ruy0GdWi!x36wnJ>@`)1?x7-E!g;me*pBuJ5G?z5I!@7PwwFjmA~#^}^Bg&eMX`cl@fwa@sr=^Th5(7F`| zt)Ek9y~{j}3b{zujKo<}3De>0?edb_=27Pck7q*;z;D8`(T#g#sf`i^~zd zPlg;iYD^>XsSdxKTs<3xYwzn7-4|hkDqBU98IjkkZI|m7V7Wuu7!oz?QMy|D9lY2z5=j;2r(L=G9)arF}(G9zB_)b>sS%PHNc_Ep4bZo!T7>`?Y0=%5$gg>4Mvd z)9YND$WAXg&b#mg>YxOzHPY^Cj`qpzs1b8fQ+t~jk-U&3e znpmgoTx-22`IATvbGOg-{j_=WBBiy_&71yBjE|RYZzRlguHlbJ;`neBzu_4FG47O< z@otnWf!|cI?=bOp-EsAGb)O0MuGH~+bVLTcR46l5+vwH#dCK@WyNx!te6RQJ+m($S z^?9A0l2z)qbsjI<7D=o6?0n57pC`DK=px&psiSobr#l@BKTq=oDeIAs=$Oa_X@^cX zhVWiB6Rk}+b_AdAba>V2Ldf>K}b&xbm2&vT#auBJ-)YVi(^>A{j7_aeQB zMvj{Y`F4o43=xI)4q=aq1yAABi}p+VPbw`9&m?~qT+&d;d-~4&WC488m33kEF%20b zJAGmLcg!U)1Ie%9uoydcDO~NH+RIN80~z8}mq%hos-d(Z z@Xw`9BJ4z(z@9#czVj`h%9q54L;PV;?U0%g^WqKEpx&tg-LfL#2WfO2*ki?`p)EOM zI}!e#!@8?4)ShhG`@8DNt?(4hfDpvmNAY%WZOoHPh9>BdjqDyJ%8}~Z%vXYwGa1RB zM-K!nI-t)_iM22o(E_!5aoCi5bqIx??cn9t5*7)06@(I_WUe7SsOsg>*f`>y*Cb zXnA0&k3PO$z9yE{TN!0rEsqQwj?|N;3NiXu_le_buU;YL-WlW#tvYB{-tYWzf5w?n zz(@0IYi?Dc!wQ4-=a3xoky}zF9y7|X$L4h1PW!|`M92%blpsv`qpK`MawE{UoQ^th zOUoh%QG3!KYGC@j<~VY7s(*f&vZlwL`#CFIZciy=G6begQdZ~g`D(siFczVMP|aaMjvM{grYN0@xaEJ*;zJ{+iP}0OQYhb$agJ2H;QZ1FSf>?9{(U6 z$?Y{A_O!`*r$pb31@ZE6iN}1(>x!d<&A?Y@w>VMFpIq+peKWz=zDmBA@ZNMC07)XM zlLr=!p=6VHjK`g;JsyW{qHtpfq-SO#*2r{h84FKgc5d~~%db9q>7RMk|0*H8r`G1w zb}nTQ&yX@WJ(q=ZBaC+DZaTp-0VlmM9517V>DL1bU-R+o@tA1{n<)X0LtOIPXVVI>FwkqB(Bfo%)r`ujE#X6I*+IdO^=mWZLs^>Ria2pap|@ z48qTPYSXghGyxkaL%q^({;uglfd#nVm zapzekN606%;zJYzOPSlH}`_qpg+)7`WS3$&U&zKC-^{soqF5T&Iho)uk z_xW9~iDUd;ybhbec-C`!@mjA$gh;-#I#bNPcsj3FyYsE{7G%qc_&}vat#c}^x1X!d zI2a|km#4^2DdPEgD$9@EV9AlEq-u<)ZQ6LPk>Kgb^4fQp+N6eb=fY66M&sxn!Q<=m z6E%c9G^uWiUkO}zhiwE^wIGsdGdTgSc zmn}%gMXBZ0wRSb+N!MHht~N(ZWZ8uK;tsX@LIr{$`QV{GvOn(EMz)ImuP;X!B()5J zmnM}qH~X5qcge9Y@?EN9Wlh}`!8}8b{ZZ&f0;Q8+N*%;89JQ)_RIDpcBcG24XL!8! zL2JM&-09kl+{470wVC{?HMEyIhqJ2@haE`83_$VMI^^rE);eHeiI~)J<6m6$#iLo~ zP50r8r?}}?fe9cTdL!zsmIk95hJtYHm4^)ImVQ%?IsSN>--RJAhh<342CsJUJ6oL1 zIl^h@F~rVN{0$Y7`lp2n4^NIE0w1&yNa?r`1YcIi&0XSYZ8Ady@?07JZGqYYFSRfu zVR?nymu3isTUlz`q}`z~ZB^dk!`z1uH1Y-S`)sB1&CSyxa6s#cj5EL24BhE+c6X?& zO@|}8c8h}eeL`x1{Fq9u$L}`vwnE_vVbUkl75E$u=z@G(!3k%T`hC?$UB9%?eM0&6 zQ}Qiqg;gX6)#BgzU#MJGV*}puqx<^jS28h!B2yrpw3@zhu6oKL_@uBxfNmz!Rg1|u zOs%12rjbOe8DT~1tGabp!A(%=hWu4DlD^YujBVny4kyca!p#^iaiCaXJ#BC__*UZg zs#-_2>7U?R$3G@JbvX;v;vb=n%eCGZx!3PG-9d}hCRAd#=f7Pc-;fF3eQ2&|mc3_ulJ?GMuC72f1%uN}71-voth; zwpHX=@EqcSuRf(C)xmsIn{(UP8)AcJ`|0{GN!)Ko>dq<IS1Ee-5lua&W_`c8NLYSATS@Q#S0g&Wo3x z`|l7tGwC3Zcu!Bin9fsb1#4Ws^6)pRA25G2CWzoL%-Oov47G6GOrJ71g>>Z@P6uSg z>UkQE*2fLe(tACVK+IA962IUl?0C@@m~q8viL4AXgkx`d_XNLDcE){NzSdLI zGhlT&(})114=%N(mGDhGV?)(zgoSu9;s#A@2e-{UUbBB(6A?683)=5@teCV{(vcde z_dl?CersN;jBBl=lndi7*?B7QC1XeQb5Ur)AXAR#1n;=bgH|K02Hu-)X253P#{fzc zb)DAF=7u)!AOD^lM&CfyYz}p>wU&% zk*jNQ^%;t($Q~1G@!I`~XwltQIGt^Z8)64AS%+UxEpxEO;;@pIPkV0i^RhBtdTH?Gp|Kn1 z4l#CRy4*3WmacNhcFCrKUJ~^ArYC3Cu&r7^ehFIG2OT>OmHaC9h7{C&YOzJF+j_|7 zC3(7xu3z;XuUHt_ba}d0CDEOhkK8P2>akuwo!?~??Tsi(R73WEr{b_kcu2LBKfFXx zxnFuq)Y#264B_~~MlQ*m|G8js@nra8pS28NUW!bo`o~gcpntESiFo^`@ZG*h;Y%;|1{zIfL}r8lem`pqZ} zM*^sMx2?!beSuMtXs!mN+3x5U*UA6!NYljV%S0}j4gRi^m^Fjz1;b43)=QLP)sPxK z%Fw2Q=e+c_x}S>GYO=Mb`8%W^FZ371ovT97#jIopKkkxf1&!>ts!=;)3OodvYb3wY z__R+C$AP4j{@Eb(QRz2*;Xj?XQ2*s}F9ty=(0tLN6p~*%f5=Mar{bfrB^4dp?i5Hj zaH5U<51Qh*o*)(N*4?x94aX8U@kofZ-0pSw^*AjLj&7viUR>%R{G8in!Y+$Mce|;e zVcKi=y`(u&8#bQKDz|Q+bZq|sy!cY!u(_&Tbkfo+B>)!m@B)6wqr_GtSqqSX`bak$ zwSrEgK309q0)fb5Bu6dZilkLYCHc)aUmZIpX|WKVS#Ga2N<|@XBwTzM$A?XX2w6=6%uT%hPv0i4n3_@as(Ozh~1mTYj`! zE*YYq!F!$Ine9|m`}lNL`5v)W)uw#hPJI=(`(3nroA*YVuO*4(Dr+kBA%;7-{t1a~nl%V5J*`Cupq*Wwn z7XMv{;k)M3PjpusdD861P;3zuH8dkN;B;q$?nn>g^wwO~B$Kt#x}V(C5Oe_S20l8^U31k& z%s^rc4+f%zyKtMS2~JL7QxD(f((yVG{cr71_`yht`K-2*gI}_}R=P6dCpA=N9nA2A zXs`r!urWUQFtXjmyE@;OgxXKBX@e^1^w)idY=trhZ?rd?Hj9sJ6#2Aoqt2LtR_$dzso5KkDVfUZZdCXNjgByE|y%127-o|FQGN zdQvzL3HD%%@RCsJFZPpzd`zr9-+*`bWvO5za`W`WtLra=?@kuDBJX_u-9Q1E_?Iit z2>mr`!nYU?%6s}huz&^~E#%2TTK9#Gr`ufYOa`9P)Obm~#O1B}VY7%a&7_4d6uE(K zCrutOv4WcA294GlW%3DEJR=k6k}uU zpP(pIRit9Q$HkU8p*fqYhg{so+$B?bVJXupNJ*fRFadhe>M>6Y`u z8mI0s(HilR@To4g%wR?0L0DokNe7>#WA~u~|o5u_PCKeqj zLO;W$d#SQyakO~by;zkjcaVgGfr;x`A=x8e_m3|IG^{MdS&y|rc*9vbd5ZnhLs*0(ac<49G)8S zi#;6N-`w2fx0|lH@X5H?ET6Ddn%H+IHgH?busnTHrUp2Ch*N)mu-0~U{}<)K2?T^( zoWh3;k>pGLN0eFa9=_c0Y_(olN{p+AjDApSMpjvC=Y3nxhEmIcewpG;%9<>TIm(7X zN6!qbot-wXGMxu@wsnnj{JF0&d^Nk?ZQT4`Ps9ycp>OJ;^eGhiwNdI4fnHtZFWwBa zh-6maR(^MUojo6bgGaDmKNH7mlzMSUD0TF+^vJ{rGb-z?X;kcXu(tnDW*|wah2E@q zt*auG)$!3!?Ndl32mIXZU0hAMcm;1LT^duSpWipT>6A!rcL&b^BbD`A^*f*pIM@0T zQMd==kZ9R9Y*Dp{tgrZ(_fS7_1iJD<`tp>zgxaq{?a)`k>4WwVLP;LC`pFzZMhf5I3q5X#=_x`lE?92l!D z*3&O(g-#E0*d@f&YOt=A9C)4cyMif}t?o0Y)42uVd&@m>*qL_wocTY%;&#Mef!sZ>uy8DZzxwY^#S%lY7bCXS0oc2g2rn{oVXki zpDgarT!0^Df2WKhZ--c2U*TZJs0VjQ#CxQ|xBst{nz27V@RZARYH^3w@@jW~J9uRs z3AJ~{off?K5=pqZhi&2PmxMu6ml~rl3|g*<@zw4$-#%rcSTOI5|Ai4)N{=63ROG>G zJR@(U?0$OOVPZPe{rct}u@@gIZMbv^zf%47_V<`HS_khH|yLfL+w{akvqAwsL=-PhY2ek*04X!@2{LJj&cymugzl4 zA2;s6hY&DR2viDZ{D^p-wA3=2msZFsx&Wart#lr&mj@g{8S~t)`V^?*iX*waDn(K) zP(gbC^LBq@5A-IT`{)HOg1pRWu=yPD%sTPEaDam9KVn7oT1x_^b+(fpdtG&Hj<{%E z@)EW$R9X}qnV)Utkcy$O49?X34zp*7m$4vyX@zQxWyt|pHz<4x^y;tXiDLr4ATu*3 zoE;ZU!x4QmPKz;mo?RxTS5wMK7=2(dT%abt*_|T32-?Cueh!;TzMQ2bNkivA03|uM zvB05L+?YQdhGiMLHCfwEm@gZmy-AY&dIhoU5j%}eqPD1e6kiqdvUcJhic~q&WK>Gi z-7dzvv)(^L$u88opuUs(XW8ID>t5qYaUYIFugPHD%qWKRbN)MB@wwaDrkg<^n3jXz{P|hY zdLTuF;*J1E>M6c#{YqMpqD|(ct|?X(a8HHbme=T;kd!vsVI>H-YU{8}>B!TUcG=<( z!jKp_kk!P*g9OPJVaSKESsyi2u`0cTGl2ULtO0n)Urr&%J@*MrhDqO=?dtZgukW;z z?Gba!6^qSV`w{|Ixa5$###GKupEm;-jXX9_Y%9R(v_{1o*S@*Jr-+ z9pW8Ky;Z4B5AZHLx&F#ia2wggoZghGZ$sr*l{Y!%DyN)*KKA@&)>HU}3VhyHL#%dIOCX*qvz z459jZc?@bR$yFYAw~7CZKP}UHf^5%=aJz{Lw#X3FS`cQ+Qq+^q5N;rv9U7)?ReKM< zKdToLSv-jwgRro<$kcy-;3x?^Ju`eNkigb~>mOpN

PV0?Fmre z)_z-wkCrKV1>UwM_B0ltyVZ~!`#2q;F4<87*x?JVSZm>3bs|&nx}1b9n`uZNRHi3~ z-?KqD7&MQXGWMF2IRu&1#g8Bro9T3Utl(`_*tew+Yn@JL7JKsCi($lsv@n^#_T#9=u)^pQy)rt3bJpU3Zn@UOQjpT+Bze_J?qN?@H2LyX>o zDz~3Wdcf1^h1jRR#~eR*kTgObx{{fwO9@!?A}^v&tS7{H`UXCA1>Ogh_HEV!c6i>{ zIOpm3>B)OFv~;S$0P7nUlasTtf;W)?tiGNN!2u!mX+z6KlhHRMDtlfSA|BYkO;@Hh zkVJ%UnW)txSB_ON`xl?pP-F-!STz+3p>$%0yOx~KWg1puyvH+YIVo56i+afZsFk0- zRFpiAk}dXF3kj%Tnc@A#K!WXBD$VN>2?N8+jQ$ zKFB~C@jL!T$@i^i7|Ct{9lk%Kz0&W)HsMMya>BuHq=p+qeM-9SzNjfB8hv0s0j(*A zvR$5(1gpjKS4r2M)cM%|+U`VxRFSRcB}~*(aq|emYX9b?6OhIGsZ}a82)JBInX;ZV zvYo7|T7E%1%(30G#{~C!t%&Xm z@A}J0oyy&s_i8t}Gb>jf{y4&Q_dp7t>6p<;ulFnQ#yIpyR9vci$xui2y|zko9vDu7 znT}|=$F=5>Ikh(`2}sU1cM!CSwwPTP+&=kOFP7H zV~2;D>sdJFwwUSzOY@%86h^i94iB0V==UX&^Q6ERz>U1(!=Z;8vqL=Yy}9X{iO31- z<-?WyiaE{CxdGI|oe6V*p2i~NXpYtv?EBK>R(-zJaXI$7^go3jfZQbOzEXC6O@HZ0 zy-;4D3P!1Ul(aoj6%3^m`n+MB<_kiTNGROWfnx{?(v;S-lpLe?InOKr)i+V4gP0>& z3y~OcilC*&WeX=St;!x4@BHmNf}N_Bk&+OYAvTp zb6n=Hzmk+uc=`VYVaKJa#r+GXUa|Q%(l-y}Nn8}Yb{yjzoy>i|d1LYtkQo52yt|iG zWisMZH{(xc74Fh$8^I?lfD=)F{8Ii_NJqDI1Z&58^yhi>4-W<`a@lIWd)g6E0W8yo zXmq5JRLTx9E1^hhb+<*t?C{eRAo5$ew-N-=_R{_9g~XZ*@gOO%NQj2wT)uOSqc0Q#a1){UTLVdOvqWsBPV78N+JO*#&VR zZ^fcswROWgT~ zXmN5>Alnl+qPzbcC;LAF`2T}j$71@+0j3}O?a6Y&zfIU*xN?9!#{<}NEbR7gIQ%~l zdau8pB>mrbl1K~BR6&0U8Es!m1iIUzsU=Gcmy#U<1VtQtLyQLjnEyJ?Yk1#10G|hx zQf=bklHZHHqJQ=;@tnW}0&u>5UGD$uI~DFIR1FwDaQ%IYcA$J>RiLJsIMPsO^3hdX zJ;)bnIX}<{VGYi)ln}JXJ;Sx$;f$qOWeK;&1sl_^2fua1QT-S0xli&x{RHILoZwBj zNm8E$fbzH3&%&|`=%88xohbjbYZLCh6^xD3 z2Z~?O4 zHdgu2djfonpnGIvyiCbVm5JH-iLTn^#M$_Dy~;ZIC&4tT_@KGXl)doHva0B@6Q1cl z3b-KD80EM>S`KR4iPD=9TSp)W1Dz>4${gYfaHhJ2g2oZ4YA!)O< zVAva_pwja4L6hu5riag>j@bvu6>xo6{*U&~JsRqDjpGtRZAIu%HnxNj;!qK$^{%ilmF=GMS;>&HXw__Fxl+Vv?w2OsAb>kc>+Pk#cE7HpaO7op+d@bas2Kv-Udw z9skYx{btSk{GR9gJn!?q>;3%Zmvv|-^*;pAcJdy?BmAFC`v3I{-%avEaP~Zb@?au< zXyntmxn$pQJixCU_W#JoA?B92G5}hDtjGfH*Dg8&Gn&AMRksv3P`k(QJF_8!|(+>f-~sxW;xcf<6BNW0L!{!aW41oq``ifa4=hm9LpCK;PBJmmsj2|*QfLbaa zf09=+{@G27&%FuVT?A~u5f1YtIwQ!#KEq}3_jFuH_)e#}JJD&o>{YP?-FTlJw9wDC zg@?Plcb|2wKO2Q|^}EKAt_)l8rc;k4IO!0+eegMTQbsIJujQl8Lennx7U*r(){9I#7ti1|(B`~DY4HL` zb}`vxHiWYduJddt8upfz2aJ3N*Y&tQ=r0=_Y}mOnKb5>`C+m1opnt#sB*FhQosa` zh&pLA)9>o1;)TWHTMNHLHC~`!*B5r^44H{A>19;ox+$Swi0)pm>Y5r3Ri~K^jJiXK z3lNqYFJ)O9YmwvGs1j^0K0$wDJ}kb~-`u_vDv{0^fu^=prpu?(4Z4xev$$hMVwLuZ zB1^utVl(iAxdlp!u+ay2E|=xzKlb2J1Uur5X1vnm3ssF@q&jGB$Zo@G<{0$Ql=XhLFm5g4(gB0pN;#qtm4Rrw>A80`xhKK5n4 z$5R#piyg;M@s8zF4U85W>4nZ1ZMvpsI%afVZ^a(yG*KgoACCD9pncKE0E8iL&f}E8^PmS_>-MKUIAe_u88 zi=&X^&(Z!(kRvZqK1#e-^y1tLB}R*JS(qn}TvD5O z8!+u{Cb{KJrJA&-f1Gf4g?GT!n|#SoRUC!j9N=lFbw)>SsJN;jG_7BNlfDkyK`5h2 zjnm;!-iQe!n+@KErusrVclywB zR*tc=k`k*V3ky7$zwJYWUk7Hk4{FPBa92-l8U{y2Yz66Vab2!QSZV%URrU&PbnZlf za_1pLR;KP?H*V$~iIu@dZ0gGN-&s%9c0kHbspz4|VGvGdnE9hp6qZCK&qM zRX_GxD!C=WSYWvUfY)hDC4DeGOxQ~FO3?)xglcYt>O6T6m{Y&L47wfQ<+80{0MbCJ zh^ERgcBABYQYoQx>s=Z|il3vxB2MIZPdt$`j;p6;Ea-g_Mb6i&X4A_~kU}*2{E$I1 zu7#Q4-~DSYT&O-h@9m8vZl&up}&sLBOP==bN?Um&;;USP^!nCa@CW zxQEnW_S+gtit2*@%o%WUCF-c9TKp`ImA&m$SJM1p?tRx#R+~l*@Y%trHLrS2nm&)C z3^j1I3RVokJn63U2OWYQ*x~3|iG+0{5xA>kC$NvMYR;xx*GjRT+<`TiSq9?Ka&bNZ z16$;_J@a`}yPdOcMaZXz7dH8er7(4-qy0;4_441B^fGv}I(krwI&Nbq@R#*FSjhwx zv6OV|Tn{tj0y;s^Fvof3xD!36L0y?#$I_FtVjxJRYNYs{?3c`SZf^&?D+*P%zU@mk zsl9)`=AT)U-ma|Cq*!V>sjZ@bfHz-2IPNaA%n%A{{_N#J5IJ^FkH(okWa2M@)l3b^ z-Hacu@7(++cka|lU)sn+d>8AkiF*6<|LsDhXFV2f56SKfX-&6f$fkE9{E% zxLY0QHEOVgYrtz!@N+u3SKw;1{YKc!n)VWZ3@)OgxL5>*isXi^mSXpm+0GB!~vuy)a$@HWxOD{XkYG?8$-nKnVd4yz)1Q#=z*7W6HhXZ-a$FQ=n7e8~NQf zxF7+jP^(oi_%f$7eQkDtc>#vt;883hEz97W%baTB!>%SVr3UFMOxr4-zzlao;d#z*Io&4V_rhQFwtciwwp0As}=RGc-ogk)~U`OeDj*n!|af;X< zg_i5u%HQC;n`7lAPEd%7Knjr`26|Si2TmCH6%(ri{@DYoF8y1gnSwAHUWQMLlfY}6 PxVR1(9Wg92aJ~F*aq-O- literal 0 HcmV?d00001 diff --git a/examples/gke/multitenant-fleet/gke-clusters.tf b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf similarity index 100% rename from examples/gke/multitenant-fleet/gke-clusters.tf rename to examples/gke-serverless/multitenant-fleet/gke-clusters.tf diff --git a/examples/gke/multitenant-fleet/gke-hub.tf b/examples/gke-serverless/multitenant-fleet/gke-hub.tf similarity index 100% rename from examples/gke/multitenant-fleet/gke-hub.tf rename to examples/gke-serverless/multitenant-fleet/gke-hub.tf diff --git a/examples/gke/multitenant-fleet/gke-nodepools.tf b/examples/gke-serverless/multitenant-fleet/gke-nodepools.tf similarity index 100% rename from examples/gke/multitenant-fleet/gke-nodepools.tf rename to examples/gke-serverless/multitenant-fleet/gke-nodepools.tf diff --git a/examples/gke/multitenant-fleet/main.tf b/examples/gke-serverless/multitenant-fleet/main.tf similarity index 100% rename from examples/gke/multitenant-fleet/main.tf rename to examples/gke-serverless/multitenant-fleet/main.tf diff --git a/examples/gke/multitenant-fleet/outputs.tf b/examples/gke-serverless/multitenant-fleet/outputs.tf similarity index 100% rename from examples/gke/multitenant-fleet/outputs.tf rename to examples/gke-serverless/multitenant-fleet/outputs.tf diff --git a/examples/gke/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf similarity index 100% rename from examples/gke/multitenant-fleet/variables.tf rename to examples/gke-serverless/multitenant-fleet/variables.tf diff --git a/examples/gke/README.md b/examples/gke/README.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/fast/stages/03-gke-multitenant/dev/diagram.png b/fast/stages/03-gke-multitenant/dev/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..a282e7d5e6cc21b6febeadaedfb5af9d90387968 GIT binary patch literal 44405 zcmeFZWl&t(x-Luzu7L;+!Gk-&-911`rdEYTZl@uh=QSebfq4xhEhen$rn}pO98IC_UV5iyvL}16BR6!q+aS>p6ayC< zvua5JwmPJeyhXUW=j3e0U4DD)92A2-X;8}YHr%@cUAMdzBQPSuy8LIgu-{KuO$m{{ zezWtG%#_T9n(djSV!>vWxRTv%=t31;(e}Vc2|3sG9>+|khMUv4#jD67r8h5H1L36n z{(M2jTE&Gyl*f$>$k8x=y8Ot5ggX%Z|NOL6HWcnnA@oz$Tk0p+um5yqXD_n}!BYJ3 zQUbxgPz7}irN#a6mtl~Vl#cNJ_^rS{3u&LAK%vVU1JeItXKxBG(f+(87-ZE17qjxbzs&lR z93REt55>;ThSK^hr2hwP_yv3#`VXRv76Jj{1icirRrn9u0Mu;yA4K_oubPF%;l_VC z;rE2U8~^^FwS9tYWphUQYnbzeEO39yFoogP?FWp+x8(mp6YtYoOG{ze1iVFA-YHqX zu~`WJvqIA4{Q`QYj|>j}Y)aYtfJ(-_KXMEG*^=xy0aG&V3TgWDI{fRY0b9BtU5ELz z3$#uHzL`!$avu5T)oo`2PB0&m{FLI)+cbTio$cV_HO2eq)fIXIUgAt*h!Oqgum7LT z|BvSX7*GB`nSI*I=xQi=A*FRQYTBQ3i~kuZbE2mY8E$M}H2kR~c6Y|N(5sraXOu5<$Uz7+=Pf8L$Y0N~!0+(G}dkl(ue|FuLS zDV{0j9V+U1BH%DaEKh}oL8A~atUCrK558yA5Q)6m%$htn5IEs!zPOtlUYrcQ5Ohqi%k!8AvbN2hKouTE{1%}H@+qjpighga@rM-TU?jt)CtNUpQOBJVDIc7zG%8mf1>*Q*8ZTCFcrTo)^TrRO&{ zPenv!G=5yXY5&ZpRg4){T`^ZtlAlHAZAT#>9^-_hLo^VkYqQt#v1ifq@;m%S*blG8 zGGCMZlLW(x5ufAR1T0*ltnU}izE0B>IvP8CY5yd!M8NSZu8P&1so=+<~EZO$kT%8-&85yZNoy& zB5a9L`q<-|fhfXGw&Qs3!B@63`uv;7Uk?6r^UB&^Xai3(Gne7p=;x7 zsin~^)x^4#c<(Kaa4R-4US#(1EAAVgmz?SzbD+?6-69!FK1SZ`B_J41-#*&KzoHn| z2fwlB9=D-Ms=oJ|E0_#go!wXY0oy7MH$`qoAzwgR2|?Y;4kj$jvN~GVXC3z+ALa#b zJwMK|P==pr_q@32e5?0k>PPr?WH;xPc3r*t#69D<;K`iKMvp^d*ZoOwopGYW;*Ia_ zt8vL4(&go0CSL+Yft%~OFTAs9N27xFH`*ivM+wK;^eYhdK$)3%G_I3!#`*YW>BRJE z{;J3%jh22ebstMfimS=ug8M}TZraB|el~~ZbW0-Je9k`m7&#$Gj!CEd^}}Jv0$a`8 z*tMLR;bn7{Z&#ynF;!gD{k7uK5p`t$xCvWU=-%D@hvkimMo0D z#e(LRe$h(&d>zQsn$#(tbwmu_zZI+~u!YD`#n#jBhe6Bg--;ms+uPU_rk{J1*{ ziVxFF5Ow2;x0MbzyLy5J0!(+P!M7)HSx8xWH=BBI%#9iCzn;s&MQJ$A?`lyFuHzP4 z_twpRa~U8g;vk+dh9(djI?_+b33AuO=Tw(b#mRUxTwIOhm-I!HT-~Wc9@ZV3Qm0Bc z3DT(l$#JbHfVGssOjs!7XKSK!=!*Vj_ex=KullYbU+FTy+ft;CCz#tKf+Om&`9{WS znpRD*Fq)rEJdkL68e>>w5Qa=2wigrVY1LKwW|BHSD~S!p?gAUxY=W>t727L zk!Y#wwzbNaVq!9`(PjIRz5pEi&F8}RzR6io4kexjfU?;vqlm`##i7* zJ!+tbS4AmhsN^HN#s`a6QE%bXIUM`(RZgyz@l1RcG_(3{we#r(ED21kN&;kRTOPhQ zj86rH@%c{p*oQ7E3&LMI{~dlJ0Q_ytX8vD7>oewjQErH;bZzoRqKVggaZvtRKwkpa z8VmzSUnl4jq2)`}d^~++@~uG3hhU4@RGlhp|8L3Jm7ii7_LH+f=ks6mRQkI5S)>zV zZyU5Ce#4!7YmjCoN7U@onwL9lIxk;`@eA#c7VbA( ztk+R`lI`l=F42JKdix7CEm92Y?h{a;%5@+cb*Y`>S{Z9i%6NCo9X2b0II5%s=HeF@ zYd$`d6%22E@IpWm@07ULC8_^$XFGsmNEnEM{jcu)zKZQt5s~pgVk{ay7UM*|a@PG1 z(Gvmcr}QNm-~_HPhNs1FrZm!Dv4yK+9ZT+Xi!sKP@7TIP(n`|n(NcvDA>d!#%Iqha z2A+Kpo_(2gWc@D!iw3e|eUkCyM>6Yf(o=-hco24CISeSyKCecb{^+;Tf@DL-3cv{| z9F;O`g)(b~b!fDfirVH*J zw_XNBdYwVz1jn#Fr&`qqT4^-PyEo>G#i0}%kEMp)3A(2$KRtIprBObfh4Q7AOS$I; z|J$L4h22rZ;|k@L;)e4cZsn>iCV@N4?bDG8HIG+vJ(QEhs!A!=iThM$^X|$s@f9rx zlY+U|#`AlnUvcYGgwN8VZX=*u6tcHI2TO<>#2k2Ky9;kPCi-!UK}sOaZ>IZXZX0`( z{T(ip0loNX9f=te+d~xRf7bxnE`Jp=bpP&cl33-h3KJux=W~FN4vpGClu%u=P*F-w zQ9eG9@l}!?pqe&UD5YbdZyq~_j?AAIXD4C{tu@;u*j6@V5b2%|YV-n?-oRAd$Is37qg8%Fu8LCTm9C8FCpwi2rI<`P zOYBcx@VX|dSeoK(VE$d=N5VG-#++{YF|FW zigxLLcRDUuvIHPo5T|KHf@Xwc*hp^kZ#IlKRIdXyORIa~Eem=jP-%<8nY$=pMKF3k z(@+;u$o6ZCm^4Yac+r(WjMu)#3$YC`xavYE*oR(T9e&rV$E=wvH{DljQ)4ZXvp4e* zW=;3mdAq=?z5`{)kg?D%F$V2F(Wag4ulFiV2eY}RRefV94z3~uiWf51-f9{RWo691 znSxtiOOu3Kt*IVmaRbT_kWb?N57UT~b`p<+dSfN&hQ5eB8TbSc3gb(yuDe!)kiXI3 z_J>EO%y|TU%>TejJNhSAM%Tef3Q%JehmX-Qnvy2|ww7Ju7Cn&;6R^J~%Cy3Y(>T0~ zv*-}NYYIY6ch;*-_zTv2=6MRNwLIL7du|P)bv#}4h6B@B=P7X{Z~REjdzE9 zJ?|NH)td9QUl9bR->g1mmc`>MVZj!c^No|>ylWn`N@wF2= z#|Bw;sdh=!plJH)SKrgITgFTu;h6uvpsr4T&I*YjjBFP!M-Q+#yI}~>TH|8d_?BoedIA! zy_bZYXuGY>n>7w}=yYL)rFi6*Pp`XxOS167Q1=+V;@|KO1Kj|;oy`uCFa z{~&<>2RQ$RY`U&2}PRu*wvq;hIRitO_VP2 zXl)<=ivjrs`@ORxH8o3?Mr2MwGvwPBVUOwBkL2PMA31bK`B@E{@{fK0B~h;M#{lyr z=idU1nH%iqv}1Q1^GH%^{93% zY0#yttPU9P38P%+{C51h5C*Xh9CggUeq~A+SR@8Ck(1`%kRX1hAN`K(27j{D2j)~H zBX(j5o`Vlyl?);lJC`v(Ekuz8LJXI#iWa&LIz-4~UfgE!=;#l(^a!t0$LtI#vJ->% zH|0oI2inz7L45y`$V3VZ{5a232J6Vb+;71ij;g>AmoQXh(z)v_f{4uDgF5?rYbKEn z`pF$NM}aaoQ^Baf57rJ$ED%{itohTPV*iv{|1Gg6DUFeQ+K06$NH>+6q2O`|ckn_1 zn&wjt_Nx=;3pcQX5Kqz44>K5Q$vf!1n(XT+?|B+48-@0jPHp~D?S)_~Vz)b)XB$+v z-2_T<80S8~*47$Uwu&@MJ?#Rpb=nxzLlI9H z@xeE5s*JJUD85eWGqSXOuBp`~CI(BO@EB57|BgkGU8gs68C?OtkW~l2MgC(;e zeXPuQ52|euqeecrCv?`^4`JE$rA(2tHA8I&9NG5Xh1OARKUUcXG2F{N4>-Qf2ciQJ zKc1SIn>+N!k9XQtEm0q(^xGc+dU*P8)kbF41~kvRX6NU3*n1m=NM$b|6MCPCpH_G5 zs40cAcfXuVmcGIoxJ=0YWa8rn4Gs%#M^Z56UxO5W=?zl#J*v{&T7 z=#rrPN~yw3}HX*F5@Z3RNYYqinkqUd7Ty6_2LiO?XKH9o0zzGSIj)h3A5LWI}% z;B4No=(iDrt@NpPG;s5H>3Ap0Q+7(mBGLsvoA-lgqXl9Pe~rYGP6mg&KI4P3NTEP2ls5RKv!#H}prQ!JqsHa3erxP{ua1q>YHAO>lr{fYz|QB{j69SX zrVpNoP*oGvdJ}awk6b-g#-9$7CD+46S_zGlm-zuc#-9n`-nCL(8sx8{T9qoSUr$YXu!`RjpyXLejesjeD6DV6fYb!z6?=gU$@S4&x zw+j|fYJ}~MyZjv^i6j9%+mjlE;C~cqLZ+D5}{pbtB856TlYdWD{lU#bVpX-(^He-Vw6M3LPMUC4>#1^&Ys85 zAL4iI%9jf%{+ua=kLbC)X{qqJC(HRXR*0Uaict+o#R>5X;ujqfwOj1FGljCp&iwAl!1{-3ZeC)X|bDV#yOUZ zd5%J4MeRj6s+T2XXHnQOBd19 zf#>>!49#%2nS_muG1Q&uZ$iItgj3I1KxV?88c>>vP0W2G{}9OE5&ouXKXz9$smJ`I z={&9D=BXRf)3_($9ZlQ1i1kKOqj>5mss>q(u2;`vlDapOW$Ib5MwjdfHZ2FqHeWIZ z*THU*voHbMI-Cn;MlfwOLtoEnK#|#&9-xeIbluDRuJx_!CH3@?Vkd+3>M=eQHV=ZS zm)ZHoPrO0$P4_rse(yBz8}M@<=4YMn$4U~#nr{x{XNC{svDJkenV+@kG~`|$1&(ea ztzkjhiCz~GDFPCa^t-SH16VhL@NPRQ1E>IS|E>UdBY|YTBGQJz@yy@jxoRu}2exgS z8(Gj17?)HLKaG6U>_~b-Rda8Ux?@L!MW z8mp+K!}TD|{mnT1lV}e)9!7<$my=b`bt$bq$TfDiWDF!FXCUv@=9prfVp-0nhSm54 zpS7*8)4Mq5*>#l`_Cg85+z)SqpAic#1Y{7=RR2(XZ*AFEe5u>0Zx0B_OGGN$17Ink zU74;3k8Cu`-)!{1`xie7H0M1VRjjaFzQ`upprP*#5}$`|D2+xl^|Wa06}jvrh0*q= z9gJz0*GFTgGmevJyFR;uRJmJ@>k&Lpn$XoF$~oOmF-hZX$7QX5VRlpbC+KrsNUWqTwW+@H;XF zOviul`{Fn2(!YE!8jBS#Ec;VQu3wZjgraD3?7fgQbIYudD@7n8hWyID_q#8K8ul#` zpNT0Cf&GU+Cl`K9yKk-Kv&*<6$`_T$POki!@As-XL3RS0154Cp8J!GwM=uw{aQ>cC z`2w^H9W223xq3iC=}P(UVKyuK_*bvhbE;8NIV#vqj4r!!ZzP*0v&;f)o=y{fE`eQj z-jW_t=VWY+B-&F3_+fFrI@MrDNRp~j-;7wflv)fJXK!EbGk>zEO7@fqYthnTq|3^b z64LCf%GBb^cK^)e75MQa-R&P2Lm^|z;t=%lmxD`#^Y7-G-a@Y#t82t0bed_8bsMmM z7nJkJTH`3ckV4s?D{5>q0^MG z@m<%?^by9d?Z9*t<@kUB73FkVPJ{tfrAJKhPmI5)vBnhuwQ)02>?%FQGGiBZwwWti z9eYll#zH)Klz5Np;!=CE0OAZG+uuOC3rJZG+EPuw*gF|>A)a-F11_S0XpFq>avM=A^2v>=2qwXbl_-E!}YdMS=A|z-8 zZui%cw7YTeuPdx)g9;bV ziys$*V$aUkhlzWQTuo(+Wu&c(RB{E2faKI99=`&igpHso^l_sB zil9KZeG$0l$ZyY`D95omG%^!=w?7u>Fr%cRY$PL^)~y&4~kWM8N^dKL1P z8ss8Z0`$QwWK22asOE4#dDuI3N8uol$&_|01S5LHO^v4fhzY~0rkAe_ICr5u)Whi4 zm$gLDt?@s|u<2}Rp7{hGgfBxNoQYRTF_UBr2Jw>JojVoSjZR#Gq(O~?8>RUN(ZSv? zk)2J(LXsGRk2=dqfYiHuZ&_eBzYeF{h^y9dF8SF86r|Tb`aLs9lShM|kZF1F%cs+a ze&^>%_#1`m{>7eJ|YhcTD?JcpVX1%QDO2r4wvGcSvEI2|gk2^Y>H2&=8? zlod91g1h9m6N>_XU)NllHe1cF68aZTug7>Cp-;$gKGg22>-_o*xk+o2UiJxiv=Kz8b|kd{th;5lk}6#7bF!(&;%b9*QF`jgW|_m=XCaILsh?upU&9 zGGzU0W&rv2M~<|WxqL3aHdqr=l8TyFXxnYw3isjIqG7qBgg?^z^jTm9`)p)2=(2{m@#EI~f@AD&p#|r~07+z`41l9>!ii>rd2Syyv!uXUNQK>z zF?FrALmk+AogC&Cn$LkYcnOF*Y=cS6ZdDmccc69Ro0EocLVs;^_qGLcHoc{%{p;{J zBcECw_-eRHV)%q`4Y38;BoVh`CpJEtrwsml9}4)TOq>b#W%U)mjBUw~{-P;FW5-HU z%!nxbyhXO{r2g(uTG$6NbQU+MVCboZ@1vMwBc{vY6QryJ3gkJEh82G-3b3Klk1*=` zH;nqlE=(!Qwc4(=;;c-FW9HHw1Bb7n)Kke1p&2LcZOeVBnn6jUU&2}#9-w)h2n_0& zev-Xah`1V8us%}?lp&Vb!ES*BMGk93X(q0Zh6NAm0U67!KR0ntBk2;J-@SYPLoQ2H ztj3clf9>!2V*Q~pe%D@-Z`4m@Zynrm9N^!Ma1*68V&wi3;VFakSLEOkJzXLG!Kx&C zEwW=nQ z25=+mEt{o0_*TAKO+|a8}D0W*M)Uy5SSfuN5O3>@$c6}fqiIAj!N#b=OcoNFi?yjBw=5-1~p&YVR*RteP?2#@E+E^k%JS=!BnO_l~dJ6{{)AbW{%Ml}{IRD1n^~9dJ$cbF7U>xKgmRjXNgD|wM+=e8-+|X|pvn1^XNrnH z80Bg2^~SH^LSdzZcNuyf`$EUp;58bS?gsTOExCMR#TlpR@MLV~LMgk8Q(nd=@iV%# z$(a5+mN*Q3T0AlK9Z9ks5br?`Ia$YPU?zK8CcAP9p&>F&B@^!vo{UNF2^mKL#Z$V~ zz9LFLq9sDA6&vkPk1>btzER@-g1MhxVe8v8ZoAL9e4H-OJjn3iyln+SA`#Zg2f4+S zA_iXBEz1zA*ki?sZxJ2Vmp7@XkQ|=`AR>1V|Zf@BW&#;EfdGfCml2 zP#M7!Hl>Gv6%%;D4s}hk?WH#8Poj1#T!L4aD@7^FX<^Yx3uivq`&7O`c+A#ukl2lN~f901{SixQ6F*C)|`?x@NH7-Bb9?mfkfP z=`tRl9V_?6*R767u=d$^ClQrnag>gS&HXn%UJoxkQXOO%(TCg)JI#cQWtvzBmHO_) zer+r%H?D51C~C6>H=OrpbLbbQ%2obQ{4p`9YpHUUI>LVKjuAii#*ic zZhtnL34y2rY{9e6Sc6q`jRookOnew^T2Hye8}m|F&S2B>CS%y=EIrA?M`P1e>4axS zR1nDgJ(vclVqZ1zjB-t-DODHv*a^ewK2ta0OP1c=cI_>hHLN}O6;Jz_afm!B;dx;} zcI-i@Sb^VYI|5@k!M11KY4oJ_apg`b15auoi4gB!G?IqwLce24ICS8{kOToYrU>m5fBkUU1Tvx z7=&Eq*>Qpwj)GqVE-)VsF$W1+@jYHdNRX0vdwW}&<+CuAs0(w%pcvA=6r+m!K!+nv z_LQQE5riobB*cjbMn{Jfk716mr2GVRqEbjp^dPk!aQ?c$H#^?IvLm1kl-fpL zQ;&-!>A42m&Wc4V?3*PwxTMb|QXBK#>+^@3%51cVH7140vYZKC@&cM((oP5~-Ntdr z^6Q`@Cyj+SL*aa)WFp3ONn@qh2nJ<2iI?)!^SGHb`yOu?MO`i%Y#d?VT8FL_+}9di zan%`(W2+ult&QaQ`&aNS_eo*I*Mq&S@vnK=DT~u<+t*Hp1O;)nwQQT5g{K5hQigsh zKAb8Z4_@OxXj%EGE5MBBhh z$XXG8sWUa5Z{V1={_$-#naTaC;ys-2O^q+FEi)_RcB)H&Q&DxAp-@g*%|@hwb)=8? zrfws%UA^TReokVvbn9nj&=nk3o?FYjFTtx(y6f^_smfs!m;Kkg>p%;(7X`BRrJ(4M>C8U!Qt-Rks*r$(?KdmPtwy4 zX{INMdVcwfj@Ei+v9SS7-rhH6walZR-cos}R=BlfidP8+NAo_NH!hT`nl8qy+JgD4 zk%jH7%O6GMjh%|6uHiwJyNdVxgECmRlWAzmO*`?d?k92YfVX_*iI_*|&)9%1*Mp4j zio}`9vJdHR7zwYrifpXEX7{f7hdu8IhfJ$JIbOD`3z2eYL7W=3Jx(D+RQ zu+s{NT0AOARA`MpG-ZjF@x<#|16HMebffTdm7-mcB|ekbj&PJ6(zw2TD)h;-9gA2E zdZ>2B%7*X#2I=#3o_w*$|`A}U8R8-eTLC@2R1-RY>Ag@y9NqK&Yzg62>LuPGfG zmh!`j?=2=rXkyRF?k>H+X%6N)rIG*-fGCM^8=X5Yo(4bZQN^Cmf%8aP2HGRT*w5JZ zM5xr~GHkiFJJ!sZ1|zTRY(~mhA$eTanS*(>_zRPZu8*@yL2@XLnSKqnyK*!*v6ikZ$LsAKySlU$!Kv9&gz@XpbWLQ( z)|#S4VXJvHOvDw>gHAZr*kHLM5qGdA6Z5jD?@Zc7M*khCQTK3@_JUciIU;fpXDWB@Fp^qUlIwGpQ5qw6$5Pbc7D#-f{0tRQCP-Zh_4&48@? zp_b|N7rp%~=u;y3QvMah*&Ui~p_=fCAqvClU~G7DU73TD^E$dF15@zUZdy8B3W`YL zr4&k+Vl@$$ecguB?6=YtMMp{<5*`;|;jF0Fg!~Wf8VY`vdxRt==Xpk7FUsN+jD6_o za9Cd{Bk)luT9l$quD1lmlY{_;%(8Y&SwN$nsyZAZ#c-0%|f zpaL8yF$Sz1nm#j%*LD6aPjh5#Of->$nzg={EbZeK)J0X#@s)6y9nI^Q;dnm0P|u-x_{C~tPM$Q$9P<p@4W9NnOTGn9`fDzR&Dg5@iMGOpr>UoLAE`Q|8B0lM+oq0AqQq36TJ zARv3z1*pR@WGl#D_BlM79Bwok?G9;0ETyN`EXQee{a7cOVslJ;jsQ`ECwsU!x;8Ew zGd4qE7Wsr|o|<~x2kQ(*Co%0qRbf>f&%NqxV7jsSD55S&(}d)pb_Jt4{C-?r50v#e zTXWmosZ zED>eMSORW3tDOr)i@rv-NACX9>lx?WO{ecW3taEc`dV)Lwg=Yf${H*6MQH94rSrL$ z;(e1$ltO>SQab3}A17__<78d_pg!y*u7Yjv+uTw*&ajw$AzCCwIDvNC|>;bOrW}@`1Fy&%@ry#aoXq2X=*CATxcIGL!*loxBu9ykL z*dG~F!jM=y9YbHA)(=_ZTf)TftL~f{?%=?A`%{iDKUarDR%=u+@xtg$<3$#fRb)_n zLK=_NtHvf^6(SMrytuMm(g;$Ipljv^S^io#Mo?xiUrQN=tDE)ESJgY(Kax~Qd&f5Rv-YwuAhicDLR*|nc&{}llnH#7uf zqT>q|G5#+1B@kC$jG1dTJcEJU%t7)fLq(p8f1Qqv9io#@L-VFVx|2=$p&wN2X~LbC z?cr;Qf568TSYs7t4_QPZ#^!|w@t~&T^%U(|^kmWw1fX1LPYvfAs1$w3u9@0`xEW+- z#}4(NRzbvs_weZ(?h=}rG?V6@wcJ~!5UPde`wR3EcOi=nvIKZVb(# zFCeP0t?91aXC`$pTXFritC_`?b+^>fk2C3R>(>Cz()2Aj(foUO$(8U4FI>XycXuo1 zvdV(qy#n&(twS|w$mDucPANs9oLOwaL$6RpxknxOuItjZH4jf40b51<3peR5-`k_E zq?eP*!XR_$%=C*kRHjyKD0Q4%%3+V#h}=;U@@F}Aagk8CxX{dQvN{E-I)`MA9Ez2# z3*Uze`-^$kqR_(hi_P395erZXcHeFo{q$^$Zwt2GdTd7rf~0VD@lSO=$O2pfnySTh zo&&L5J}gwJ{#N~yj_kWrDq9r!W_HaxX*hlwkf~H9m?9Ud{q0jXevCS#xKwORX)Nk{ zA!d^4EpF-=`L98m3Ub4O77hsO7YZ#Y){hn}H@h3eAX4Ti+g#NIGBcl|a3!KMhHiI0Uj5Lh3sXcY&@Tt)SKu&CDDVFJ>YLqKn% zu1~4d9W*!T7_}0zN_aI5WUclEU2oCm5Vce+kcUtHV*8o^lOuA9%13r=2!+G0-u;^1 zdS#O_BPq7br}9OIK8BNZ*yF34{RYJW?p2SgvVJCTT?^PJMMQdCf;B0+tdCE09eeFl zoz?~sS$@B9ME*$}!ZoMn8IZ)(A|i20p8_z^*{}Bf^WPLrd1inN+(-hAJFofc#LS0C z6)Lae#HyO<;EZ{E*o9zha~fgFLov4qF(B))FcZ_!Pd)irY+MZh&%XOzdMCO#=9E?j zSM}y`6k12@i!l|&0eKkX&zIk7nyeQ_>K)qp_!L*i9=9>b@4H?X#ebfh`Xco~*PQAF z{P;|B{?0ok(Mrz^VbTcU%9Uvz&zVBFtU4|9;IMOW}-VP5M{ zF(teF9*i`ikpv}iZcl`>nZs^hk_SQudV;;P;g6^!rr@LHj_lrAAPw}f%IE(YdKi9GW;s=NvMpFfrK+7K4JuIco3hr z>?oYbb+INHZFPZ~XZHc7K|>jD#J!6<<6`kn^$aJ$TK(Az)nj_O4iqllMD+mZ+_8_w zr0xTyT^Svx)cN5r# zY+7L*TLd`yfUg?M!I%P4!=q&CrO?ub8~q}9XFl$HvX;>w(T|_L)~RZ~aFo_^l_rxq zt)u8jGgy7Swu=%OIF5Zi+hrqJp&w@nNsr2DyB4xT)2dkCzR%g zCiwS;ar`@ZgN@&u>M|RJn;!0OHsB^@7JsE)e96|Ft#N8Xp^f{54f+siHWQT+PEv&@ zxU4|}=1u~)i+RxmQ5N;4iTsQc78?5grrX9EpDNBqj*=hu(={{U8E{+xRjAddu}9OX8J{~o#9u^K;EPAIXl!YN$7GY)l(FHDf>i_D^8Ribo?1rNkm3h z$e9zQJHljmv#O-qL#=V{I`B&4THS~7^|2)!idX9ISpc4QtsVBFUQQR%qOJJZTupJYAUewN+rMW(&d-s)pivFjP(!%|OJoiKDv9J%U| z37&Vz9P$th34kt;@9rJ}G!P-lZ+`i`qOh~ek5Vpn4``V?SMqS(G&TIMQDsrrKm2mC#8sm*F4B*h7)ax{y6I2K=y zm_O@B@7Uk&DoK35k-oU#6>gB?-O!8 zOI(kG?Dit)Tu{5dQ2!cZ!|}0)$>LZxkR8JJu3E8B~)0~EhV)M$5N;g494c; zI~p+Lm-H2}OVnK}wR9Art{}dZ4mm2nP_bumJ<+|jamKloXsY*V{qCP0T=1q|gc^FD zz`P$EELVM-5aryA-ksGP_@T9PsqT0OxQDyOSwCdA(b!vAW&iNi^b2DjbH9;h zg&%%NOkmR3K0b;_GRQY)`ev}=whl3ORy)|L8MnbbM9aFWHsK8#M== z_?wizD`Q535>m>I1Qc4f8w(bM^}dR=eVF&=b98G4>e?0%oj-Z0pAMjbAl?Qi!l`0V z@ly2QtFcHPvTxv|W``@xW5m?mL+WXDvJBe0@6r7*idp@nKF;P#5=gYtVM#Nnih_4e z)3q{*v2w8a2!Vt}I?JY+&@Pnt2g_26M<}ju$~+b zmKhxjx!J8JG3h$qZ$qi&|r3%?f9lLxvQx$9~ydke+cV zN?c%*boH2`i@;aM&=ZTU+A(KfW-xrIQ*I<`%Lyhl9kZvHAXvmcGz&>FD^a{VNjOc= zfAAOmoMr_KU(_~Mxr&lFEaTiaEha5|Aqo1Sd%4PNm#sLt{Siukhe)qkC-%DXBH2Lj zdrw-cLb64bGK^Jw^W%ZF3KvwUOv>QI;vMVcvmA<+u1T`OJbCvm$`hokI|kYFyuFdN zy9BrdAdSq$nM^e0WzSq5h@wCvOb2ezQZM;24@&ediD$vydood)R(Cw;}MwWxvP?c#p$|R@E?Eg0u!jAXGj9 z&43TDQ;P4iSOU27W4QpYg-{|wNFz7lZm5BtDPaQ-wnYuk^iq`M-SaoSa&J3T4Epn6 zx7}m_ zAB_{!K8);K>gyUir-jT1^xk^ks|cQ{I69{nle28#n-KLnG+b&V?WRp8HVd)3UUP^i zxo6gqS5))0$|PPUCsS)5C_~i#Na7) zDQik#t3H8GPy>jkJXf%)wC7ZhWF9K(fI_oStWZE6>M|U7} z&ox(gH2_->3o%}3f1y8tY@KhoJ(AhY;MgVbykZy5Z2yO-Fet-z<-<$W%Gs8=g(aYGt&CHIo^~^YsQil zcJ+&KqC!KuItxAhG`uV2bKi&>CSX?J)~W}?Or$3IWK7&=E5hO zj>e1SGd1>SoPcrk9J_uww3PKF5MFLd3Vtf{IV*CUez%TFx05cqe;2CgC9=0`>8+ur zzzXu9q2B8yZ1)P45U!p(NIm0?3b43)Uzwlqhy}Q-Ug`)7SCeV;)nOo6ea(*bP>JOy zg(kF=JzG^_<9sp=6?f5gCtANARhZ{2IG;oLTwJNjAvEY^4N5K{ zE!8m_T>jkTPbK``MmBTkyrqyCk3rVNGh+gwK zcIH;6iexqaO6-7{V%5t%G4p=in}SXDnmSGof@5F;?zvkPy;jqtcdiIg>!iPkuSkrX zw8w0C$}1*(W`cF*q#((&A3zz6+{v3|NuQy< z14~B*rRxUGLAr|@&C-O$4ijm==4OF2UX7WmdmqjNQl@6H4=Eh^U;%>x*rc+)Ik9y# zlAEcnb)S~BiwZ`Z`12AjqOt8HJ_|lcekuW51iYuGTGOG? zhX1j3kJuqT15A_xhlX4U{VkB7UldkfgcSIp3cAN>SrZYRH>k#_Kok>tUzFF$cV+6z z?-^2Zm^Yu^p=Aohs5hwfxC}+z<^hAZ_KMGWzbXSwb2`e-6W|PYx)dPwZB#{LkXrR5 zdZY1|NKZ`9pZKuKc)zBMpT^mX8e5TUjt=8H8SuRG>lbXkbUNbU_F3Y@gim-*^5)!J z92mw^3kstyV|zq+2%0m5zsYS?6(wXe_nSK3s>oT_9j>q9lGoeE#1Je}TalJC4+U?j z>t=VLg|*LO7@`IwGK+*?Zoo?|{Bn_i2UA}2kORrID?-b4+YJ3>u6Z1wY)iU);Ol3!xCpB8AIlwcDwY2g67+6ky|Apeaqm8^x0VUcgNX~iF?ZJct+ zKQA?OIbPnFR@Ji81S@VZ-Q{LG%o+W_D2WB;!Dzo9u>yyPB(dxQvaw&=5KWDdQNqnl z%@(aJ*ZL3nQU~THoqWuHMFu=GX7ppQ_@8Ggok;;TG2#N&)Y-jm`O!}WF5&fwryt_t zHpo^`@YFY!R=j0SSQsRViQ7>B2AC5Tcn!p#GoeUDXH7yJXCVQ?FOPXY>7y z#WZxbKtt8-nhW9a9Zo=~0^)+V$vSS+A7YmkhFRp%0Q#2spq;U%*clG4F zl|`FtY%g)aTo;B_04#oOteah;1sKDLWc$6MDTYE9C-YjacdaL;C)?z4aXQYQ8g}A| z`><45{jJa&#$F9i+Ke9!TF%#+e<3g+QDP8wKHt?y=XJ$6UKD!mm%;103~-sM4~rg; zy)ae1ILsaUwceM6ye?>nn8ZH{&xGAUim)*kJ8FkBrtzQq;Q z3KX+fO%KWVG$xW4)~9MG%fRdz{OT$_`c)ceV`c6Ll2@XP zljb|=XN%_czx4$OxW;U!LA%v%EIE1krA*KLl#AENlQ(xhbM{?C+W={8SJ&HDO~mU$ zZ#L_$_{#GuAz;dVIX@hCACF$+Oc(9_2F!jL0T!NLKAadKM5gx{k*T!~+Mr+PZ1_te zHaqf#Br&aS$kQhQ2Q~&6Zv|c5Dyl`>YogW;tI0LsyjzzTczlkmb+t45ZQT$OZ?ITRq-@^f-XyWx(=Nv^v2Vond+F<15*4mtr|m-m9OTkO;^>n5E?aqum5yh&Pq ztZegr>r>3aP`ap@$JamIu&g2h2Xg`=9q8@HE zF-DKB5-nJS*PI&;BLjIBoC6Wn3(#)wE&yLN$SxZXc3oWkzRrpMbcG-dpx6DGe2&6o zm^t5__r|&=;$*N!eydNONgIK66&{Glhr7xvDUptkkKcNfxm&Cx`F>CmNXdBEEAB9? zm%Q>XGyxmTOk1SV^1b!SMLI8^?yOIKcR*ZEph@%x<;q70?J@Y1ue&^6_U99r%AF3W zHMut{UF$tzOFQ+9J-_$A;51%gtmz7heOYu6@@x*uzXoFCn)cX}F~s>;<-4||E;X;+ zD67c(lyZCbsI+uM*_IRd+oKKZcRA}DDw0A^L-l5^10R4_tsFEv!WnR#Nze%!;&`I3kaTl`nu9zCpmK79eCh^tc%?w~{xaBNDjDVxe3NtixfNW4|;m5ck zy70&*e>W`+@7`b1+WRl&J*zUyb~=lY^T+rLM%rU!`9p3S1&CB}~TS+}|Bt;IJ>_K(&dt4~J;PDT=2 zu7YHh(4kG~+I3BM9vuTOm<;xo97h~KMj`#@Q#=t{m-%uD+n7%VZVS6VI;id6FKIC@ zKe@KC+PD3cX>S&h+A~*Bmn3Z$^x1{p9+*_OUP14J^B0;027=o|)B>Y}Ycje6%^>Mz zs+;{HGBkm=##l%?Bpa87rzJu{%Fdfsa5 zX*e7mQ)ptD!XWhaUBeG$8N;Nyv5O}}XsIJVDY)g{Fjsa{>J^8ydu1%jqNXZMA_W;p zbnRC95GLN&s%+ss12BqFM5FgnCdnF4o?59MR8;rQWcN@B@|r%N645&T;PdYH*9!OT zQ8EXs<7o;1$(1d~pRDV6O4s%1AM)wL$42MX@y0w}8BVcR3!pk@j+rixTBvI_IH-+t zfXl9Bblk4{lIq#;8!)PE-{7^8(LL0LaYt+%JFshl=|U;M!)N!X`A$Hbn}v)NJf-eg zjA-Y9u#BmRR}0dw3~GsSjlM~%34hug-I|R@cpa7D zvv@thB|1q&!AxXbEw86!>!S|X7x|~=O*~*vin9K7p#YolKh(>Ft{%t^;NR`XkG9RH z#8eh$>H2M?Qf}v4Pb}4_!w=8|>?GeG>A`rU>YXYGVlv7~^!;yYRh$CJjtu0r6Ug(| z=5IUJcw`Wm+RkDWMQpbEiUe(c&dXhFdq^`@R||~D#$v}X@Uw(JV{@G)EpgBwCoZV* zXSAlJ#9c~sy=-HwF!DZ1yAy>LJb6q20>HO!_sEVfGyF|nDHS=6ZOE;?9CwY*xo`Ss z(0qD?Bf4{M@eO}gc>0TMoa35K@pgs`A^uHzripR{Z~)GOUH#j^ORc@c{zbCBti_MH z3SWxQ&|jN-&qxg=X^pFiiMHmTcnS)Z3Yk&mSK|beI8Y2YDm&UF#iA<>`G3-mc8&66 zO+~B|j`RlBGFB>Uh+|}}l-q2*hPmzoM%7U4&eIa`RXW=UO?DX1LfWfxjZ%V<8AFtX zvhb&NHxhhLYI&;01UCET$Ng;E(1z=r^=)avC$9Ri`Lj#bimV(KI-qK zf1920$(d=9AGP4XuNE%LpPwhwc2ld^z0kV%&Sk7Uo`yp0{xi$WiYa5$Qw+9++n-b4 zfDvf-#dpsS^g!hPq=ocN^alpt_Qx!g^#Sody1aDpDzQr%IK}CqU(k*Ef`a(%J-8X& z(O??> zm6J9#A50z+#*ZnP`#BBVf!t6a|Lq$p9Qr@9S@5WvL-2WPAKz&sxKAi-$&rKZXOU4P z`KkuJ1tKURjrp|`o0HE0_e&5(cif2 z{8b3Du|CxI{OeKGRAI*^;FXd{WHC=)6ju(`Agy&}eaVF(zn78Y$4o^ysGi=`>JGqn z2XlW4-Ef^rIaU(}V-mmMK>c^)Rj(vWHdIQR|N1l~AtrW*r);mF=3Z1kUL>E>D`gU7 z4^w(8cU$!M@Yh%RS>-IWni(27E}PbgiHOyTj;zcPK0ar-H_Q;2Ma^OEVp*wHIPCSzqS15xav!&7-owN5{>T ztc<&28#_)Z=Jq_0Kj*o>iPY0uTG=nrcN5xGZmLAEQ7R@p16W|r#Gy)Fpi{o@W1^?oc8;}z{U?6mK~+P;rZqFFWy=1g#Z zq=!kOV|OLCf8gx}#wG2dCvVjEr>&XkVB{rtDHOUmL_x~vP}Ifh%}p)U5M|^2AmCLz z)cnUY|M7&F!{K)(DpCaL7eyL`>L{*%O)%8()cgk?{dn5fYCV7bmZ}Yc@53LyygZqs zu1Ap|Iv^U?DRg($t2P}n0@4!r4)R@UNnLq2*~-}{(nrf|Hd%U!xLCl*GUa|-?s$LxE0>DTNU~nxwBErRWC12x9WP|=#|t5SYdNg%=H*+hx6?0D2Md7SLL~l8-j7;H3qjFW zuBzNdXMW^5<>dZmi@$hSRhZrMskXK_N!4k_K9&@z%#}g#@oePLbq29g*qwkc$y-qBh&Vd(68ezDrzWV)mDQTI#Ak-Z>`jraXdO^XHs*HR0HyjIEWG88B^_o|_ z&ME*CtA=YOEC&=uy2YJ}tcGWNxW27Hnr3-%5f^8;>ua^Q)U_A8(T2}I6Q7){wfCvN zv1U5iueBzm&yvrTm7rpW(2^8vu!UVg+g>$Y-}b4?Mr@LfK0&C-KL9ceCjMIGL0r%O z#_><{=Yt7V1b0(4bRo&is9czN6#ipPeC^U`K$JL^sIUhu^0*<=*eLTG_wi`s0DkKd zCkvC2y->5}==bz*NUO?U;t`4BQ~+t#G&otr=xeth z+C9d-&J5zS56psZsey61F8ObK_Mhe{B+?4`(gAfB-U3X}+QaYkiSyW~Q1Y?Ulr&(d zM$lpLcaH0(Rwya4_#g~CMt5(No`ZOh56~=joyULaJ#e-{$5NM45a!S5^$}l40Z1E0PKtMfN!Ubm*PVSD`xRXzJ_kp-jXcC#r9h!wh zqidhT4A^mQd{<}G@P};%iCB_wsOpcXq38_pUDv)5VEE{hwgfB5GhE|NbQ;_q*Qf zs6^rJz)}D5WqsnBh`166N6NVv}#Q zvvb$}#x4Vek&EpJaFT8N{Lc^%;EPRLsJNsyq8NBE=B(VZJ za1A?iO$@{w+ATj9dli0|VQIiKBlu@F?vE<0!4QeI{_9-8(sZ#P;oA5)6%Xnogsnvg zG&J4*J3k5*ekKCGz5Y4!+J6TBfW}y>(O!q&cMGu#zj=ZkTr2ax7+kxIb)c_55cqGy zP5#--zr%#Bu*tSAmcMaO|Cd8{QLb3}{ecqu&(xsbmv&pqKQ`~R*IZ6ZoYsY3lkp{w zC1?I9N7PF6oh3BzNxXUlZBySn`~+J0?c-i)Imch>QahSHeK<`{qmOS<-MAXr8W|pb z$2d+82^a(2Os?Nf9x6a-~Pn{V4uT&j9emShKuIQKsZ`H@R8vV{d)3p zk@NN<^`Yeb`R3s8FpuAJqZ_Y@{$BWs&k4)6-=){Ha@!ehqoWP?^{n3Lo4Dj(K3&1p z=A!Wn0ZS1CAGbovH%B?^tG0TIZCX<4^+${Zc$e+ME?pBI6>a0sfGKINxgIa4XBN&d zGg6m+xl@H*hpBtrl25z6&jUp3Jy&l_k#QwS$|6jStc(zpj_R=u2gME}USq{J6@tRT zt?yct+hvINPS!~M2(N`7)*s^1in~2CVe-T7N0JB@8&4 zBRLD-ud~G{K9!aA<8_H2F)|`rOQL$G^#~jj$_-a~%Vv7-K*WrgPVIqeZatqJJVm!w zmer4n^#g2?M&hp<3se3E4`vXz5YpK4e@E` zd)X~V=xdC4|8Obn8f9-GvvEuCc8HLEW7D1WP%Ik~bb^_#f3e}eto$!m zgK5by(6IaoERSRS_wxU8^7N#81a!Pm%j)(299pLyv{D4cNi)>JtCMq#xQWOg_?lfTa=rro5q@a+3hp6fm*2 z@ZcjNTp?+&eRg#A8h-$|6~#3%Q7AAT(xiSG=A|2 zHz}03v`nzQPxFbmm*Tsn5xeR8gx3#lx=?24_+F9x%7JFlU?(USYEzGTB3G;)kW^h& zW!ki6)8F6!BA{ij^9$rp&HAAwDO1gy3I+E&cQzu6%OpMn7r7h2vI4xijn~@;c!I5Up;`aoHj_D5! zx=c$fa7sw*qrH#GBIX)kNvTJbimGCp*ZN0CpHW_Iy9W5)G?UOE;ed#-&3^K5lZw2#NizCL~M7%%`f>wktCy=9f0XH6K z&qUwC{R9|+>dvn%@(&;4uHaJH;Np@O0=ccUAtTbVJtpD&TljX#VaPOBAi=^o5RP#? zV2`)}%e@YWV8m@>2U1$@MS+^!l|$NhQty$s%3tvb#Ngn7mHP;oulU>pT$k26Z%A}E z>E6TyD`wX6i7WHAV~7|*-J z@&P7y8_0CS4H%L{DbNv^ML;f4S=(4WTW;{b@gxjMcTaXav3;Bz*wT4+g(|Eqg)!p( z`k(ezenh1dyicwmjH3gu%rk;;RCwq~|3yVO*aOuj9xnDku7Lwl2fLK}+EPodzzB|} z!U?F8SY9Abb-+shO&^ic2P&;sPb045l0N_^VL=LxS_Nn>8+V}o{-=ZZ0Hcb%xe62^ zIA@@DeuJ}Yx2gW$coG~>xMNoB%iSN`{Fyct1Lp5TDDT$hes`sT6cdsA1rmS@8O6Sz z3@H5~$mpCd7hk_D^BClrG`K#nyI5Xu*?UcoAiF>nrFnwAftvujotXq&^}YS?)VOVb zowO?Us=1S3Lj#=lD9yX3G3G0vvb?`VD6Y|;WTkLfR3vTk^3BYp$VeT=n#S+BTp}F< zUc(~@{SB144?0VPjZi%AI?o+FNftKy!T}8NecHis=4-B=GKz`TS+j+I9$dqr21IGc zTgk^Mn5(f*eTa=N6*Q%9{MsGucxZMp)gK*+!=2#m;vD4YX#U5}M)7IVf%`0dI442`BeH}|=dhh95(KAnp&XI^pTEX(6K&Ls5PdoQaF|k&XbH+k9eoFsP zN(9RGz4?=`V!aWO;!7^=p`M=gVf2&Cd)-zM?;82-;%|FL<-t8)+9oM8L`nQ~3MSnn z{}Ka}Ak|uT-fw46b2Qs2^=!z;yI=X{RWmf0ahzE7*_f8=Plz3 zd^$SdNRQvGx8%?8ZpJp{2#^ejn^m;uvI?AEhy+!ete$jQA9NZgE~*G5onE^%)ebxJ zda8R_-;>nGW%p2W&s~o**Rwp=D0f&v>(NaQnX4f;D5=pYfGtMlZj*LElxuG@ft8gPet5Npvb zy4YpAZDLA{Mh3m-!x*=svvf_pyypYDcBcjOPrSXY^(rR!;(}oZ<(i3${R8~ln?@s~BQCi448zR|m(Zh|Sp3)F$Y!L3%z-d~ns1 zI=I*!1N(Ulyq9Ajr!4S3S>~6(>ySaF$-|J(z-u$LzA$I>Ev(4#DP1r*b&!J z=Xtu&DbTZDR{W}wVC?(5bb8SCz;8WTNTk+2dD3BYp#gALMr~6rGYmns_B8i}dfM%F z7uwiN`PysQBnp2ek)tBn;Mxt)$@lXiyK_OjbCil3enZZ;akbBhwcHQ?s2t9E7=Nli zTjb@p?Ee-y6JUs5%cFzpYT>g%#hC*d9^1BmM>P4^Bzdnt4hQ~X1FygBbF-NdEaHX2kC_-tUF9m9MBDz=K?!yMB$z zBkh&wfP5N3&_o_xzuvC&e&zSRI7`;nAky-D+eoB!64_T!c|tYz;uT@H`kxT^dC_Db zEJoP}oh-?cjT?hm@A3TOu&Rw}I{Ela%=}X@aB~V^8EluJSB_cNz)AZ$1j0cv5~WU+ z=}4b87hT!?aQ-R;eZzA$QCCE19i=&N2hqV9z=!JmF4_wHzCy}lU6t!nW`Ly*;)8zl z5x7eZhDV)l2@c(C{gd=t#KP5N-%O4(8CM^9SQ*K?_|=qG>$Hr=xyAX;(jW4)ex*gc5PLY|eox!ERcWM18a(9n7$@%%ROwgB2$}0D}49ru> zXW5d{co`z*bE$zNP-Xg|yEa|4h_`d!*lAy-oah$H++i>DY>3e{ye~5O*3SVEuH~b6 zTBb_lqkIr;3Ja|fiP_x*|S6*_{zjP2?XB*f>|YNpG%tCJwu= z3OFgH-Vd@ht+_Z^FKNGFh96@i(oPC(uay(7d=@zp1%AF7;%)Q((tZGPu`2kP3Bm!x4G$cZUD11BHu&-k*g?M@RYO@}rV=_-!+GFL-I}C+1<@Xhj zVoJFdF~pzFQchrvPtK953T2Om-m<_KhZ}t3s!S%PcZa$^Qe3ki=3Asj?WYIV&77AX z91&}s;(sEX#(|xPEy$BMLB`qu=7c$F|e^p!dC&HIRw*#Y*K{oDz0T z)1t0gew_(Al8BehB+bzGQoe)-`1M}VJro98Ug>c&{&tF!k2^ zCx}>xY2H$?^H9Do`SaJ$A@`sI;w3|d#`xp-Sbf@+89}=fmlXF^57lMOlVq#!K=Tj( zrTM$dkF;O&Kfmv@272kEDjk+H-?UG!EsM}-!`{FS% zF&$f;O$l}SB1SW~PZvLx8oVY|WFYJ!rsY)}9t6CQvkp^M>5N@LfrRvhrrENqc6`kaUHi*aIQo z>S96P==%=IMY!puRuJ?F6{_^wn)`96_@CCXwPh-lBiGyG#keeds7(i^dZt%Sl!uT(9YtBIRz^1JwF8m-G4l0j6YAgaaatpAQNGNy5l_uDKyQ{ zl~4kK?w)1J!Tb@!aJ61O-@r#^*B2OAS7y3u6g_<}^6B@DC%Z&uCNEz}1gkyZa$-vi zB^*T1BD7nf@$T9l{iSXN3F52uLuv`4F-vlL)Xs@)oE)d}kjOOrKnfE|%8dFBEb=efz@o z-Z1p$nmk5VbDr|-q>zNS8A@Zckn+uHS}+B3AR{U5EwbwC;hV5e$E1zo@~Z1AgP8_h zG>ce{0r*$I1-I6kcaQj1O)2x%R3xo#^}S|Q;FS`JYxWN$RnP5==~fTuzwaEHyw^`N z|9Ummkm9`Oh4E+MR%HaYS5@u%FJ(5Uc(<;&;hL5*-qq8wFG3hX-U1j^Dcv5xk|{hN z<+GVAwM%(;@taU;rRVeN2*#vcnaDI?Pikg~Dxm9O^ZxvU zs6ONw?PWF-Swkv17)F)%sf7j3u+jM^iCx~?AZiSg<2_I}2B@(tP@|Emkyee5uJ%P& z^0CPh{5tyGh+leIfl%9CC-vGR5XzF2)>UvnRey#+Hgx-uNsi9Ybka1gQ`T(3H9r?h z{>0q(nr9HO?IT-h7WuqGjIk0t{mr%)>@iR7P;i)@(!JGx6bVJ(k-L_U)xof#tA%9+{YLApRD$v^D~#KjjeZJFzoBZDfllyB6N8Eki9tfx z{9w)gbB;IUIo>1MH@6AqW&Fcn{=n#o)V4KfRE`)~ruy0GdWi!x36wnJ>@`)1?x7-E!g;me*pBuJ5G?z5I!@7PwwFjmA~#^}^Bg&eMX`cl@fwa@sr=^Th5(7F`| zt)Ek9y~{j}3b{zujKo<}3De>0?edb_=27Pck7q*;z;D8`(T#g#sf`i^~zd zPlg;iYD^>XsSdxKTs<3xYwzn7-4|hkDqBU98IjkkZI|m7V7Wuu7!oz?QMy|D9lY2z5=j;2r(L=G9)arF}(G9zB_)b>sS%PHNc_Ep4bZo!T7>`?Y0=%5$gg>4Mvd z)9YND$WAXg&b#mg>YxOzHPY^Cj`qpzs1b8fQ+t~jk-U&3e znpmgoTx-22`IATvbGOg-{j_=WBBiy_&71yBjE|RYZzRlguHlbJ;`neBzu_4FG47O< z@otnWf!|cI?=bOp-EsAGb)O0MuGH~+bVLTcR46l5+vwH#dCK@WyNx!te6RQJ+m($S z^?9A0l2z)qbsjI<7D=o6?0n57pC`DK=px&psiSobr#l@BKTq=oDeIAs=$Oa_X@^cX zhVWiB6Rk}+b_AdAba>V2Ldf>K}b&xbm2&vT#auBJ-)YVi(^>A{j7_aeQB zMvj{Y`F4o43=xI)4q=aq1yAABi}p+VPbw`9&m?~qT+&d;d-~4&WC488m33kEF%20b zJAGmLcg!U)1Ie%9uoydcDO~NH+RIN80~z8}mq%hos-d(Z z@Xw`9BJ4z(z@9#czVj`h%9q54L;PV;?U0%g^WqKEpx&tg-LfL#2WfO2*ki?`p)EOM zI}!e#!@8?4)ShhG`@8DNt?(4hfDpvmNAY%WZOoHPh9>BdjqDyJ%8}~Z%vXYwGa1RB zM-K!nI-t)_iM22o(E_!5aoCi5bqIx??cn9t5*7)06@(I_WUe7SsOsg>*f`>y*Cb zXnA0&k3PO$z9yE{TN!0rEsqQwj?|N;3NiXu_le_buU;YL-WlW#tvYB{-tYWzf5w?n zz(@0IYi?Dc!wQ4-=a3xoky}zF9y7|X$L4h1PW!|`M92%blpsv`qpK`MawE{UoQ^th zOUoh%QG3!KYGC@j<~VY7s(*f&vZlwL`#CFIZciy=G6begQdZ~g`D(siFczVMP|aaMjvM{grYN0@xaEJ*;zJ{+iP}0OQYhb$agJ2H;QZ1FSf>?9{(U6 z$?Y{A_O!`*r$pb31@ZE6iN}1(>x!d<&A?Y@w>VMFpIq+peKWz=zDmBA@ZNMC07)XM zlLr=!p=6VHjK`g;JsyW{qHtpfq-SO#*2r{h84FKgc5d~~%db9q>7RMk|0*H8r`G1w zb}nTQ&yX@WJ(q=ZBaC+DZaTp-0VlmM9517V>DL1bU-R+o@tA1{n<)X0LtOIPXVVI>FwkqB(Bfo%)r`ujE#X6I*+IdO^=mWZLs^>Ria2pap|@ z48qTPYSXghGyxkaL%q^({;uglfd#nVm zapzekN606%;zJYzOPSlH}`_qpg+)7`WS3$&U&zKC-^{soqF5T&Iho)uk z_xW9~iDUd;ybhbec-C`!@mjA$gh;-#I#bNPcsj3FyYsE{7G%qc_&}vat#c}^x1X!d zI2a|km#4^2DdPEgD$9@EV9AlEq-u<)ZQ6LPk>Kgb^4fQp+N6eb=fY66M&sxn!Q<=m z6E%c9G^uWiUkO}zhiwE^wIGsdGdTgSc zmn}%gMXBZ0wRSb+N!MHht~N(ZWZ8uK;tsX@LIr{$`QV{GvOn(EMz)ImuP;X!B()5J zmnM}qH~X5qcge9Y@?EN9Wlh}`!8}8b{ZZ&f0;Q8+N*%;89JQ)_RIDpcBcG24XL!8! zL2JM&-09kl+{470wVC{?HMEyIhqJ2@haE`83_$VMI^^rE);eHeiI~)J<6m6$#iLo~ zP50r8r?}}?fe9cTdL!zsmIk95hJtYHm4^)ImVQ%?IsSN>--RJAhh<342CsJUJ6oL1 zIl^h@F~rVN{0$Y7`lp2n4^NIE0w1&yNa?r`1YcIi&0XSYZ8Ady@?07JZGqYYFSRfu zVR?nymu3isTUlz`q}`z~ZB^dk!`z1uH1Y-S`)sB1&CSyxa6s#cj5EL24BhE+c6X?& zO@|}8c8h}eeL`x1{Fq9u$L}`vwnE_vVbUkl75E$u=z@G(!3k%T`hC?$UB9%?eM0&6 zQ}Qiqg;gX6)#BgzU#MJGV*}puqx<^jS28h!B2yrpw3@zhu6oKL_@uBxfNmz!Rg1|u zOs%12rjbOe8DT~1tGabp!A(%=hWu4DlD^YujBVny4kyca!p#^iaiCaXJ#BC__*UZg zs#-_2>7U?R$3G@JbvX;v;vb=n%eCGZx!3PG-9d}hCRAd#=f7Pc-;fF3eQ2&|mc3_ulJ?GMuC72f1%uN}71-voth; zwpHX=@EqcSuRf(C)xmsIn{(UP8)AcJ`|0{GN!)Ko>dq<IS1Ee-5lua&W_`c8NLYSATS@Q#S0g&Wo3x z`|l7tGwC3Zcu!Bin9fsb1#4Ws^6)pRA25G2CWzoL%-Oov47G6GOrJ71g>>Z@P6uSg z>UkQE*2fLe(tACVK+IA962IUl?0C@@m~q8viL4AXgkx`d_XNLDcE){NzSdLI zGhlT&(})114=%N(mGDhGV?)(zgoSu9;s#A@2e-{UUbBB(6A?683)=5@teCV{(vcde z_dl?CersN;jBBl=lndi7*?B7QC1XeQb5Ur)AXAR#1n;=bgH|K02Hu-)X253P#{fzc zb)DAF=7u)!AOD^lM&CfyYz}p>wU&% zk*jNQ^%;t($Q~1G@!I`~XwltQIGt^Z8)64AS%+UxEpxEO;;@pIPkV0i^RhBtdTH?Gp|Kn1 z4l#CRy4*3WmacNhcFCrKUJ~^ArYC3Cu&r7^ehFIG2OT>OmHaC9h7{C&YOzJF+j_|7 zC3(7xu3z;XuUHt_ba}d0CDEOhkK8P2>akuwo!?~??Tsi(R73WEr{b_kcu2LBKfFXx zxnFuq)Y#264B_~~MlQ*m|G8js@nra8pS28NUW!bo`o~gcpntESiFo^`@ZG*h;Y%;|1{zIfL}r8lem`pqZ} zM*^sMx2?!beSuMtXs!mN+3x5U*UA6!NYljV%S0}j4gRi^m^Fjz1;b43)=QLP)sPxK z%Fw2Q=e+c_x}S>GYO=Mb`8%W^FZ371ovT97#jIopKkkxf1&!>ts!=;)3OodvYb3wY z__R+C$AP4j{@Eb(QRz2*;Xj?XQ2*s}F9ty=(0tLN6p~*%f5=Mar{bfrB^4dp?i5Hj zaH5U<51Qh*o*)(N*4?x94aX8U@kofZ-0pSw^*AjLj&7viUR>%R{G8in!Y+$Mce|;e zVcKi=y`(u&8#bQKDz|Q+bZq|sy!cY!u(_&Tbkfo+B>)!m@B)6wqr_GtSqqSX`bak$ zwSrEgK309q0)fb5Bu6dZilkLYCHc)aUmZIpX|WKVS#Ga2N<|@XBwTzM$A?XX2w6=6%uT%hPv0i4n3_@as(Ozh~1mTYj`! zE*YYq!F!$Ine9|m`}lNL`5v)W)uw#hPJI=(`(3nroA*YVuO*4(Dr+kBA%;7-{t1a~nl%V5J*`Cupq*Wwn z7XMv{;k)M3PjpusdD861P;3zuH8dkN;B;q$?nn>g^wwO~B$Kt#x}V(C5Oe_S20l8^U31k& z%s^rc4+f%zyKtMS2~JL7QxD(f((yVG{cr71_`yht`K-2*gI}_}R=P6dCpA=N9nA2A zXs`r!urWUQFtXjmyE@;OgxXKBX@e^1^w)idY=trhZ?rd?Hj9sJ6#2Aoqt2LtR_$dzso5KkDVfUZZdCXNjgByE|y%127-o|FQGN zdQvzL3HD%%@RCsJFZPpzd`zr9-+*`bWvO5za`W`WtLra=?@kuDBJX_u-9Q1E_?Iit z2>mr`!nYU?%6s}huz&^~E#%2TTK9#Gr`ufYOa`9P)Obm~#O1B}VY7%a&7_4d6uE(K zCrutOv4WcA294GlW%3DEJR=k6k}uU zpP(pIRit9Q$HkU8p*fqYhg{so+$B?bVJXupNJ*fRFadhe>M>6Y`u z8mI0s(HilR@To4g%wR?0L0DokNe7>#WA~u~|o5u_PCKeqj zLO;W$d#SQyakO~by;zkjcaVgGfr;x`A=x8e_m3|IG^{MdS&y|rc*9vbd5ZnhLs*0(ac<49G)8S zi#;6N-`w2fx0|lH@X5H?ET6Ddn%H+IHgH?busnTHrUp2Ch*N)mu-0~U{}<)K2?T^( zoWh3;k>pGLN0eFa9=_c0Y_(olN{p+AjDApSMpjvC=Y3nxhEmIcewpG;%9<>TIm(7X zN6!qbot-wXGMxu@wsnnj{JF0&d^Nk?ZQT4`Ps9ycp>OJ;^eGhiwNdI4fnHtZFWwBa zh-6maR(^MUojo6bgGaDmKNH7mlzMSUD0TF+^vJ{rGb-z?X;kcXu(tnDW*|wah2E@q zt*auG)$!3!?Ndl32mIXZU0hAMcm;1LT^duSpWipT>6A!rcL&b^BbD`A^*f*pIM@0T zQMd==kZ9R9Y*Dp{tgrZ(_fS7_1iJD<`tp>zgxaq{?a)`k>4WwVLP;LC`pFzZMhf5I3q5X#=_x`lE?92l!D z*3&O(g-#E0*d@f&YOt=A9C)4cyMif}t?o0Y)42uVd&@m>*qL_wocTY%;&#Mef!sZ>uy8DZzxwY^#S%lY7bCXS0oc2g2rn{oVXki zpDgarT!0^Df2WKhZ--c2U*TZJs0VjQ#CxQ|xBst{nz27V@RZARYH^3w@@jW~J9uRs z3AJ~{off?K5=pqZhi&2PmxMu6ml~rl3|g*<@zw4$-#%rcSTOI5|Ai4)N{=63ROG>G zJR@(U?0$OOVPZPe{rct}u@@gIZMbv^zf%47_V<`HS_khH|yLfL+w{akvqAwsL=-PhY2ek*04X!@2{LJj&cymugzl4 zA2;s6hY&DR2viDZ{D^p-wA3=2msZFsx&Wart#lr&mj@g{8S~t)`V^?*iX*waDn(K) zP(gbC^LBq@5A-IT`{)HOg1pRWu=yPD%sTPEaDam9KVn7oT1x_^b+(fpdtG&Hj<{%E z@)EW$R9X}qnV)Utkcy$O49?X34zp*7m$4vyX@zQxWyt|pHz<4x^y;tXiDLr4ATu*3 zoE;ZU!x4QmPKz;mo?RxTS5wMK7=2(dT%abt*_|T32-?Cueh!;TzMQ2bNkivA03|uM zvB05L+?YQdhGiMLHCfwEm@gZmy-AY&dIhoU5j%}eqPD1e6kiqdvUcJhic~q&WK>Gi z-7dzvv)(^L$u88opuUs(XW8ID>t5qYaUYIFugPHD%qWKRbN)MB@wwaDrkg<^n3jXz{P|hY zdLTuF;*J1E>M6c#{YqMpqD|(ct|?X(a8HHbme=T;kd!vsVI>H-YU{8}>B!TUcG=<( z!jKp_kk!P*g9OPJVaSKESsyi2u`0cTGl2ULtO0n)Urr&%J@*MrhDqO=?dtZgukW;z z?Gba!6^qSV`w{|Ixa5$###GKupEm;-jXX9_Y%9R(v_{1o*S@*Jr-+ z9pW8Ky;Z4B5AZHLx&F#ia2wggoZghGZ$sr*l{Y!%DyN)*KKA@&)>HU}3VhyHL#%dIOCX*qvz z459jZc?@bR$yFYAw~7CZKP}UHf^5%=aJz{Lw#X3FS`cQ+Qq+^q5N;rv9U7)?ReKM< zKdToLSv-jwgRro<$kcy-;3x?^Ju`eNkigb~>mOpN

PV0?Fmre z)_z-wkCrKV1>UwM_B0ltyVZ~!`#2q;F4<87*x?JVSZm>3bs|&nx}1b9n`uZNRHi3~ z-?KqD7&MQXGWMF2IRu&1#g8Bro9T3Utl(`_*tew+Yn@JL7JKsCi($lsv@n^#_T#9=u)^pQy)rt3bJpU3Zn@UOQjpT+Bze_J?qN?@H2LyX>o zDz~3Wdcf1^h1jRR#~eR*kTgObx{{fwO9@!?A}^v&tS7{H`UXCA1>Ogh_HEV!c6i>{ zIOpm3>B)OFv~;S$0P7nUlasTtf;W)?tiGNN!2u!mX+z6KlhHRMDtlfSA|BYkO;@Hh zkVJ%UnW)txSB_ON`xl?pP-F-!STz+3p>$%0yOx~KWg1puyvH+YIVo56i+afZsFk0- zRFpiAk}dXF3kj%Tnc@A#K!WXBD$VN>2?N8+jQ$ zKFB~C@jL!T$@i^i7|Ct{9lk%Kz0&W)HsMMya>BuHq=p+qeM-9SzNjfB8hv0s0j(*A zvR$5(1gpjKS4r2M)cM%|+U`VxRFSRcB}~*(aq|emYX9b?6OhIGsZ}a82)JBInX;ZV zvYo7|T7E%1%(30G#{~C!t%&Xm z@A}J0oyy&s_i8t}Gb>jf{y4&Q_dp7t>6p<;ulFnQ#yIpyR9vci$xui2y|zko9vDu7 znT}|=$F=5>Ikh(`2}sU1cM!CSwwPTP+&=kOFP7H zV~2;D>sdJFwwUSzOY@%86h^i94iB0V==UX&^Q6ERz>U1(!=Z;8vqL=Yy}9X{iO31- z<-?WyiaE{CxdGI|oe6V*p2i~NXpYtv?EBK>R(-zJaXI$7^go3jfZQbOzEXC6O@HZ0 zy-;4D3P!1Ul(aoj6%3^m`n+MB<_kiTNGROWfnx{?(v;S-lpLe?InOKr)i+V4gP0>& z3y~OcilC*&WeX=St;!x4@BHmNf}N_Bk&+OYAvTp zb6n=Hzmk+uc=`VYVaKJa#r+GXUa|Q%(l-y}Nn8}Yb{yjzoy>i|d1LYtkQo52yt|iG zWisMZH{(xc74Fh$8^I?lfD=)F{8Ii_NJqDI1Z&58^yhi>4-W<`a@lIWd)g6E0W8yo zXmq5JRLTx9E1^hhb+<*t?C{eRAo5$ew-N-=_R{_9g~XZ*@gOO%NQj2wT)uOSqc0Q#a1){UTLVdOvqWsBPV78N+JO*#&VR zZ^fcswROWgT~ zXmN5>Alnl+qPzbcC;LAF`2T}j$71@+0j3}O?a6Y&zfIU*xN?9!#{<}NEbR7gIQ%~l zdau8pB>mrbl1K~BR6&0U8Es!m1iIUzsU=Gcmy#U<1VtQtLyQLjnEyJ?Yk1#10G|hx zQf=bklHZHHqJQ=;@tnW}0&u>5UGD$uI~DFIR1FwDaQ%IYcA$J>RiLJsIMPsO^3hdX zJ;)bnIX}<{VGYi)ln}JXJ;Sx$;f$qOWeK;&1sl_^2fua1QT-S0xli&x{RHILoZwBj zNm8E$fbzH3&%&|`=%88xohbjbYZLCh6^xD3 z2Z~?O4 zHdgu2djfonpnGIvyiCbVm5JH-iLTn^#M$_Dy~;ZIC&4tT_@KGXl)doHva0B@6Q1cl z3b-KD80EM>S`KR4iPD=9TSp)W1Dz>4${gYfaHhJ2g2oZ4YA!)O< zVAva_pwja4L6hu5riag>j@bvu6>xo6{*U&~JsRqDjpGtRZAIu%HnxNj;!qK$^{%ilmF=GMS;>&HXw__Fxl+Vv?w2OsAb>kc>+Pk#cE7HpaO7op+d@bas2Kv-Udw z9skYx{btSk{GR9gJn!?q>;3%Zmvv|-^*;pAcJdy?BmAFC`v3I{-%avEaP~Zb@?au< zXyntmxn$pQJixCU_W#JoA?B92G5}hDtjGfH*Dg8&Gn&AMRksv3P`k(QJF_8!|(+>f-~sxW;xcf<6BNW0L!{!aW41oq``ifa4=hm9LpCK;PBJmmsj2|*QfLbaa zf09=+{@G27&%FuVT?A~u5f1YtIwQ!#KEq}3_jFuH_)e#}JJD&o>{YP?-FTlJw9wDC zg@?Plcb|2wKO2Q|^}EKAt_)l8rc;k4IO!0+eegMTQbsIJujQl8Lennx7U*r(){9I#7ti1|(B`~DY4HL` zb}`vxHiWYduJddt8upfz2aJ3N*Y&tQ=r0=_Y}mOnKb5>`C+m1opnt#sB*FhQosa` zh&pLA)9>o1;)TWHTMNHLHC~`!*B5r^44H{A>19;ox+$Swi0)pm>Y5r3Ri~K^jJiXK z3lNqYFJ)O9YmwvGs1j^0K0$wDJ}kb~-`u_vDv{0^fu^=prpu?(4Z4xev$$hMVwLuZ zB1^utVl(iAxdlp!u+ay2E|=xzKlb2J1Uur5X1vnm3ssF@q&jGB$Zo@G<{0$Ql=XhLFm5g4(gB0pN;#qtm4Rrw>A80`xhKK5n4 z$5R#piyg;M@s8zF4U85W>4nZ1ZMvpsI%afVZ^a(yG*KgoACCD9pncKE0E8iL&f}E8^PmS_>-MKUIAe_u88 zi=&X^&(Z!(kRvZqK1#e-^y1tLB}R*JS(qn}TvD5O z8!+u{Cb{KJrJA&-f1Gf4g?GT!n|#SoRUC!j9N=lFbw)>SsJN;jG_7BNlfDkyK`5h2 zjnm;!-iQe!n+@KErusrVclywB zR*tc=k`k*V3ky7$zwJYWUk7Hk4{FPBa92-l8U{y2Yz66Vab2!QSZV%URrU&PbnZlf za_1pLR;KP?H*V$~iIu@dZ0gGN-&s%9c0kHbspz4|VGvGdnE9hp6qZCK&qM zRX_GxD!C=WSYWvUfY)hDC4DeGOxQ~FO3?)xglcYt>O6T6m{Y&L47wfQ<+80{0MbCJ zh^ERgcBABYQYoQx>s=Z|il3vxB2MIZPdt$`j;p6;Ea-g_Mb6i&X4A_~kU}*2{E$I1 zu7#Q4-~DSYT&O-h@9m8vZl&up}&sLBOP==bN?Um&;;USP^!nCa@CW zxQEnW_S+gtit2*@%o%WUCF-c9TKp`ImA&m$SJM1p?tRx#R+~l*@Y%trHLrS2nm&)C z3^j1I3RVokJn63U2OWYQ*x~3|iG+0{5xA>kC$NvMYR;xx*GjRT+<`TiSq9?Ka&bNZ z16$;_J@a`}yPdOcMaZXz7dH8er7(4-qy0;4_441B^fGv}I(krwI&Nbq@R#*FSjhwx zv6OV|Tn{tj0y;s^Fvof3xD!36L0y?#$I_FtVjxJRYNYs{?3c`SZf^&?D+*P%zU@mk zsl9)`=AT)U-ma|Cq*!V>sjZ@bfHz-2IPNaA%n%A{{_N#J5IJ^FkH(okWa2M@)l3b^ z-Hacu@7(++cka|lU)sn+d>8AkiF*6<|LsDhXFV2f56SKfX-&6f$fkE9{E% zxLY0QHEOVgYrtz!@N+u3SKw;1{YKc!n)VWZ3@)OgxL5>*isXi^mSXpm+0GB!~vuy)a$@HWxOD{XkYG?8$-nKnVd4yz)1Q#=z*7W6HhXZ-a$FQ=n7e8~NQf zxF7+jP^(oi_%f$7eQkDtc>#vt;883hEz97W%baTB!>%SVr3UFMOxr4-zzlao;d#z*Io&4V_rhQFwtciwwp0As}=RGc-ogk)~U`OeDj*n!|af;X< zg_i5u%HQC;n`7lAPEd%7Knjr`26|Si2TmCH6%(ri{@DYoF8y1gnSwAHUWQMLlfY}6 PxVR1(9Wg92aJ~F*aq-O- literal 0 HcmV?d00001 From 95b12ac2f7015548e0c7e1b53ab0ba5748acdc26 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Fri, 12 Aug 2022 11:24:46 +0200 Subject: [PATCH 55/75] rename example folder --- fast/stages/03-gke-multitenant/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index f1d29ad428..779f13d692 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -17,7 +17,7 @@ # tfdoc:file:description GKE multitenant for development environment. module "gke-multitenant" { - source = "../../../../examples/gke/multitenant-fleet" + source = "../../../../examples/gke-serverless/multitenant-fleet" billing_account_id = var.billing_account.id folder_id = var.folder_ids.gke-dev group_iam = var.group_iam From d345cf8223ce64e1f456feab73a195afb3843882 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 24 Aug 2022 15:40:42 +0200 Subject: [PATCH 56/75] Update gke multitenant README --- fast/stages/03-gke-multitenant/dev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 9b0771d904..819f889da0 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -108,7 +108,7 @@ It's of course possible to run this stage in isolation, by making sure the archi | name | description | modules | resources | |---|---|---|---| -| [main.tf](./main.tf) | GKE multitenant for development environment. | _module | | +| [main.tf](./main.tf) | GKE multitenant for development environment. | multitenant-fleet | | | [outputs.tf](./outputs.tf) | Output variables. | | google_storage_bucket_object · local_file | | [variables.tf](./variables.tf) | Module variables. | | | From 909739039b535535d35390a0a3d20f0825709c10 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 24 Aug 2022 15:53:55 +0200 Subject: [PATCH 57/75] Fix internal links --- fast/stages/03-gke-multitenant/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fast/stages/03-gke-multitenant/README.md b/fast/stages/03-gke-multitenant/README.md index 1974f9a723..25c88ac91d 100644 --- a/fast/stages/03-gke-multitenant/README.md +++ b/fast/stages/03-gke-multitenant/README.md @@ -4,7 +4,6 @@ This directory contains a stage that can be used to centralize management of GKE The Terraform code follows the same general approach used for the [project factory](../03-project-factory/) and [data platform](../03-data-platform/) stages, where a "fat module" contains the stage code and is used by thin code wrappers that localize it for each environment or specialized configuration: -- the [`dev` folder](./dev/) contains an example setup for a generic development environment, and can be used as-is or cloned to implement other environments, or more specialized setups -- the [`_module` folder](./_module) implements the actual stage code +The [`dev` folder](./dev/) contains an example setup for a generic development environment, and can be used as-is or cloned to implement other environments, or more specialized setups -Refer to [the `dev` documentation](./dev/README.md) configuration details, and to [the `_module` documentation](./_module/README.md) for the architectural design and decisions taken. +Refer to [the `dev` documentation](./dev/README.md) configuration details, and to [the `gke-serverless` documentation](../../../examples/gke-serverless/multitenant-fleet) for the architectural design and decisions taken. From fa8990dc76b0528e905b962a3c700c7003b742dc Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 24 Aug 2022 16:43:48 +0200 Subject: [PATCH 58/75] gke stage docs --- fast/stages/03-gke-multitenant/dev/README.md | 35 ++++++++++---------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 819f889da0..f005cbc8c9 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -51,7 +51,7 @@ ln -s ~/fast-config/providers/03-gke-dev-providers.tf . There are two broad sets of variables you will need to fill in: -- variables shared by other stages (org id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.) +- variables shared by other stages (organization id, billing account id, etc.), or derived from a resource managed by a different stage (folder id, automation project id, etc.) - variables specific to resources managed by this stage #### Variables passed in from other stages @@ -69,35 +69,36 @@ ln -s ~/fast-config/tfvars/02-networking.auto.tfvars.json . If you're not using FAST, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. -#### Cluster and nodepools +#### Cluster and node pools -This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Variables are designed to support this approach for both clusters and nodepools: +This stage is designed with multi-tenancy in mind, and the expectation is that GKE clusters will mostly share a common set of defaults. Variables are designed to support this approach for both clusters and node pools: -- the `cluster_default` variable allows defining common defaults for cluster +- the `cluster_default` variable allows defining common defaults for all clusters - the `clusters` variable is used to declare the actual GKE clusters and allows overriding defaults on a per-cluster basis -- the `nodepool_defaults` variable allows definining common defaults for nodepools -- the `nodepools` variable is used to declare cluster nodepools and allows overriding defaults on a per-cluster basis +- the `nodepool_defaults` variable allows definining common defaults for all node pools +- the `nodepools` variable is used to declare cluster node pools and allows overriding defaults on a per-cluster basis -There are two additional variables that influence cluster configuration: `authenticator_security_group` to configure Google Groups for RBAC, `dns_domain` to configure Cloud DNS for GKE. +There are two additional variables that influence cluster configuration: `authenticator_security_group` to configure [Google Groups for RBAC](https://cloud.google.com/kubernetes-engine/docs/how-to/google-groups-rbac), `dns_domain` to configure [Cloud DNS for GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/cloud-dns). #### Fleet management Fleet management is entirely optional, and uses three separate variables: -- `fleet_features`, that specifies the [GKE fleet](https://cloud.google.com/anthos/fleet-management/docs/fleet-concepts#fleet-enabled-components) features you want activate -- `fleet_configmanagement_templates`, that allows defing configuration templates for specific sets of features ([Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/install-anthos-config-management) currently) -- `fleet_configmanagement_clusters`, that specifies which clusters are managed by fleet features, and the optional Config Management template for each cluster -- `fleet_workload_identity` that enables optional centralized [Workload Identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) +- `fleet_features`: specifies the [GKE fleet](https://cloud.google.com/anthos/fleet-management/docs/fleet-concepts#fleet-enabled-components) features you want activate +- `fleet_configmanagement_templates`: defines configuration templates for specific sets of features ([Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/install-anthos-config-management) currently) +- `fleet_configmanagement_clusters`: specifies which clusters are managed by fleet features, and the optional Config Management template for each cluster +- `fleet_workload_identity`: to enables optional centralized [Workload Identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) -## TODO -Adjusting External Load balancer Policy: -Error ensuring load balancer: Insert: Constraint constraints/compute.restrictLoadBalancerCreationForTypes violated for projects/0000-dev-gke-clusters-0. Forwarding Rule projects/000-dev-gke-clusters-0/global/forwardingRules/mci-jz0ri8-fw-apps-whereami-ingress of type EXTERNAL_HTTP_HTTPS +Leave all these variables unset (or set to `null`) to disable fleet management. -## How to run this stage +## Running Terraform -This stage is meant to be executed after "foundational stages" (i.e., stages [`00-bootstrap`](../../00-bootstrap), [`01-resman`](../../01-resman), 02-networking (either [VPN](../../02-networking-vpn) or [NVA](../../02-networking-nva)) and [`02-security`](../../02-security)) have been run. +Once the [providers](#providers-configuration) and [variable](#variable-configuration) configuration is complete, you can apply this stage: -It's of course possible to run this stage in isolation, by making sure the architectural prerequisites are satisfied (e.g., networking), and that the Service Account running the stage is granted the roles/permissions below: +```bash +terraform init +terraform apply +``` ... From c3e6a03eaae743a49da36cefa82f3469ae26a6a3 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 25 Aug 2022 14:24:39 +0200 Subject: [PATCH 59/75] Allow fleet project to be specified by the user --- examples/gke-serverless/multitenant-fleet/README.md | 5 +++-- examples/gke-serverless/multitenant-fleet/main.tf | 2 +- examples/gke-serverless/multitenant-fleet/variables.tf | 5 +++++ fast/stages/03-gke-multitenant/dev/main.tf | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index f0e0cd2426..c1ae7476f5 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -141,7 +141,8 @@ fleet_features = { | [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | | [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | | [prefix](variables.tf#L223) | Prefix used for resources that need unique names. | string | ✓ | | | -| [vpc_config](variables.tf#L235) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [project_id](variables.tf#L228) | ID of the project that will contain all the clusters. | string | ✓ | | | +| [vpc_config](variables.tf#L240) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -153,7 +154,7 @@ fleet_features = { | [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | -| [project_services](variables.tf#L228) | Additional project services to enable. | list(string) | | [] | | +| [project_services](variables.tf#L233) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/examples/gke-serverless/multitenant-fleet/main.tf b/examples/gke-serverless/multitenant-fleet/main.tf index 411d51c51b..64cc864099 100644 --- a/examples/gke-serverless/multitenant-fleet/main.tf +++ b/examples/gke-serverless/multitenant-fleet/main.tf @@ -17,7 +17,7 @@ module "gke-project-0" { source = "../../../modules/project" billing_account = var.billing_account_id - name = "gke-clusters-0" + name = var.project_id parent = var.folder_id prefix = var.prefix group_iam = var.group_iam diff --git a/examples/gke-serverless/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf index 5c60991881..2354b136d7 100644 --- a/examples/gke-serverless/multitenant-fleet/variables.tf +++ b/examples/gke-serverless/multitenant-fleet/variables.tf @@ -225,6 +225,11 @@ variable "prefix" { type = string } +variable "project_id" { + description = "ID of the project that will contain all the clusters." + type = string +} + variable "project_services" { description = "Additional project services to enable." type = list(string) diff --git a/fast/stages/03-gke-multitenant/dev/main.tf b/fast/stages/03-gke-multitenant/dev/main.tf index 779f13d692..708461e21b 100644 --- a/fast/stages/03-gke-multitenant/dev/main.tf +++ b/fast/stages/03-gke-multitenant/dev/main.tf @@ -20,6 +20,7 @@ module "gke-multitenant" { source = "../../../../examples/gke-serverless/multitenant-fleet" billing_account_id = var.billing_account.id folder_id = var.folder_ids.gke-dev + project_id = "gke-clusters-0" group_iam = var.group_iam iam = var.iam labels = merge(var.labels, { environment = "dev" }) From a82ef7550e811eaebadcd31c6c2fa5dfe42bb09f Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 25 Aug 2022 15:11:44 +0200 Subject: [PATCH 60/75] Allow gke stage to write to automation bucket --- fast/stages/01-resman/branch-gke.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fast/stages/01-resman/branch-gke.tf b/fast/stages/01-resman/branch-gke.tf index 42d640e782..cbf05dbae8 100644 --- a/fast/stages/01-resman/branch-gke.tf +++ b/fast/stages/01-resman/branch-gke.tf @@ -98,6 +98,9 @@ module "branch-gke-dev-sa" { iam = { "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } } moved { @@ -115,6 +118,9 @@ module "branch-gke-prod-sa" { iam = { "roles/iam.serviceAccountTokenCreator" = ["group:${local.groups.gcp-devops}"] } + iam_storage_roles = { + (var.automation.outputs_bucket) = ["roles/storage.admin"] + } } moved { From cad37158b7af54dee4f35425403a87efa451c09b Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 25 Aug 2022 17:35:51 +0200 Subject: [PATCH 61/75] Fix dependencies in gke multitenant stage --- .../multitenant-fleet/gke-hub.tf | 4 ++++ .../gke-serverless/multitenant-fleet/main.tf | 24 ++++++++++++------- .../03-gke-multitenant/dev/variables.tf | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/gke-hub.tf b/examples/gke-serverless/multitenant-fleet/gke-hub.tf index b8679e4cb0..a66f1bd26a 100644 --- a/examples/gke-serverless/multitenant-fleet/gke-hub.tf +++ b/examples/gke-serverless/multitenant-fleet/gke-hub.tf @@ -37,4 +37,8 @@ module "gke-hub" { workload_identity_clusters = ( var.fleet_workload_identity ? keys(var.clusters) : [] ) + + depends_on = [ + module.gke-nodepool + ] } diff --git a/examples/gke-serverless/multitenant-fleet/main.tf b/examples/gke-serverless/multitenant-fleet/main.tf index 64cc864099..912ae4a7e6 100644 --- a/examples/gke-serverless/multitenant-fleet/main.tf +++ b/examples/gke-serverless/multitenant-fleet/main.tf @@ -21,27 +21,33 @@ module "gke-project-0" { parent = var.folder_id prefix = var.prefix group_iam = var.group_iam - iam = var.iam labels = var.labels + iam = merge(var.iam, { + "roles/gkehub.serviceAgent" = [ + "serviceAccount:${module.gke-project-0.service_accounts.robots.fleet}" + ] } + ) services = concat( [ + "anthos.googleapis.com", + "anthosconfigmanagement.googleapis.com", "cloudresourcemanager.googleapis.com", "container.googleapis.com", "dns.googleapis.com", - "iam.googleapis.com", - "stackdriver.googleapis.com", - ], - var.project_services, - !local.fleet_enabled ? [] : [ - "anthosconfigmanagement.googleapis.com", - "anthos.googleapis.com", "gkeconnect.googleapis.com", "gkehub.googleapis.com", + "iam.googleapis.com", "multiclusteringress.googleapis.com", "multiclusterservicediscovery.googleapis.com", + "stackdriver.googleapis.com", "trafficdirector.googleapis.com" - ] + ], + var.project_services ) + service_config = { + disable_on_destroy = false + disable_dependent_services = false + } shared_vpc_service_config = { attach = true host_project = var.vpc_config.host_project_id diff --git a/fast/stages/03-gke-multitenant/dev/variables.tf b/fast/stages/03-gke-multitenant/dev/variables.tf index 7547f168ec..14a9d5e812 100644 --- a/fast/stages/03-gke-multitenant/dev/variables.tf +++ b/fast/stages/03-gke-multitenant/dev/variables.tf @@ -168,7 +168,7 @@ variable "fleet_features" { variable "fleet_workload_identity" { description = "Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true." type = bool - default = true + default = false nullable = false } From fd06a841b7cc0ed4756d565f4efbbaff123dbe9c Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 25 Aug 2022 18:37:55 +0200 Subject: [PATCH 62/75] Update gke multitenant README --- fast/stages/03-gke-multitenant/dev/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index f005cbc8c9..f7eea314de 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -10,7 +10,6 @@ The following diagram illustrates the high-level design of created resources and ## Design overview and choices -TODO ## How to run this stage @@ -131,7 +130,7 @@ terraform apply | [fleet_configmanagement_clusters](variables.tf#L112) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | | [fleet_configmanagement_templates](variables.tf#L120) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | | [fleet_features](variables.tf#L155) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L168) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [fleet_workload_identity](variables.tf#L168) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | | [group_iam](variables.tf#L183) | Project-level authoritative IAM bindings for groups in {GROUP_EMAIL => [ROLES]} format. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | | [iam](variables.tf#L190) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L205) | Project-level labels. | map(string) | | {} | | From b1d9b27ac3087d9f2bcc592b2818da84602088f5 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 30 Aug 2022 20:39:47 +0200 Subject: [PATCH 63/75] Allow peering configuration to be passed as variable to fleet example. --- examples/gke-serverless/multitenant-fleet/README.md | 9 +++++---- .../multitenant-fleet/gke-clusters.tf | 10 +++------- .../gke-serverless/multitenant-fleet/variables.tf | 13 +++++++++++++ fast/stages/03-gke-multitenant/dev/README.md | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index c1ae7476f5..a4d6382c42 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -140,9 +140,9 @@ fleet_features = { | [clusters](variables.tf#L61) | | map(object({…})) | ✓ | | | | [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | | [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L223) | Prefix used for resources that need unique names. | string | ✓ | | | -| [project_id](variables.tf#L228) | ID of the project that will contain all the clusters. | string | ✓ | | | -| [vpc_config](variables.tf#L240) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [prefix](variables.tf#L236) | Prefix used for resources that need unique names. | string | ✓ | | | +| [project_id](variables.tf#L241) | ID of the project that will contain all the clusters. | string | ✓ | | | +| [vpc_config](variables.tf#L253) | Shared VPC project and VPC details. | object({…}) | ✓ | | | | [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | | [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | | [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | @@ -154,7 +154,8 @@ fleet_features = { | [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | | [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | -| [project_services](variables.tf#L233) | Additional project services to enable. | list(string) | | [] | | +| [peering_config](variables.tf#L223) | Configure peering with the control plane VPC. Requires compute.networks.updatePeering. Set to null if you don't want to update the default peering configuration. | object({…}) | | {…} | | +| [project_services](variables.tf#L246) | Additional project services to enable. | list(string) | | [] | | ## Outputs diff --git a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf index f94039dc5d..8e04d780bf 100644 --- a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf +++ b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf @@ -73,12 +73,9 @@ module "gke-cluster" { logging_config = ["SYSTEM_COMPONENTS", "WORKLOADS"] monitoring_config = ["SYSTEM_COMPONENTS", "WORKLOADS"] - # if you don't have compute.networks.updatePeering in the host - # project, comment the next lines and ask your network admin to - # create the peering for you - peering_config = { - export_routes = true - import_routes = false + peering_config = var.peering_config == null ? null : { + export_routes = var.peering_config.export_routes + import_routes = var.peering_config.import_routes project_id = var.vpc_config.host_project_id } resource_usage_export_config = { @@ -116,5 +113,4 @@ module "gke-cluster" { # memory_max = each.value.cluster_autoscaling.memory_max # } # } - } diff --git a/examples/gke-serverless/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf index 2354b136d7..37138c3798 100644 --- a/examples/gke-serverless/multitenant-fleet/variables.tf +++ b/examples/gke-serverless/multitenant-fleet/variables.tf @@ -220,6 +220,19 @@ variable "nodepools" { }))) } +variable "peering_config" { + description = "Configure peering with the control plane VPC. Requires compute.networks.updatePeering. Set to null if you don't want to update the default peering configuration." + type = object({ + export_routes = bool + import_routes = bool + }) + default = { + export_routes = true + // TODO(jccb) is there any situation where the control plane VPC would export any routes? + import_routes = false + } +} + variable "prefix" { description = "Prefix used for resources that need unique names." type = string diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index f7eea314de..969828616a 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -92,7 +92,7 @@ Leave all these variables unset (or set to `null`) to disable fleet management. ## Running Terraform -Once the [providers](#providers-configuration) and [variable](#variable-configuration) configuration is complete, you can apply this stage: +Once the [provider](#providers-configuration) and [variable](#variable-configuration) configuration is complete, you can apply this stage: ```bash terraform init From 2ddd68ee2a123dc7ff650a03a79a29e9670755bc Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 30 Aug 2022 20:41:34 +0200 Subject: [PATCH 64/75] Fix comment exaplaining serviceProjectAdmin permissions --- fast/stages/00-bootstrap/organization.tf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fast/stages/00-bootstrap/organization.tf b/fast/stages/00-bootstrap/organization.tf index 1129698eff..c0f5e798b5 100644 --- a/fast/stages/00-bootstrap/organization.tf +++ b/fast/stages/00-bootstrap/organization.tf @@ -170,10 +170,11 @@ module "organization" { ] (var.custom_role_names.service_project_network_admin) = [ "compute.globalOperations.get", - # the following two permissions are used by automation service accounts - # who manage service projects where peering creation might be needed - # (e.g. GKE), if you remove them make sure your network administrators - # should create peerings for service projects + # compute.networks.updatePeering and compute.networks.get are + # used by automation service accounts who manage service + # projects where peering creation might be needed (e.g. GKE). If + # you remove them your network administrators should create + # peerings for service projects "compute.networks.updatePeering", "compute.networks.get", "compute.organizations.disableXpnResource", From 1e8ec71d54a33998ce7853c24a0473adf159c60c Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 30 Aug 2022 21:33:09 +0200 Subject: [PATCH 65/75] multitenat fleet readme, first pass --- .../multitenant-fleet/README.md | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index a4d6382c42..54dc594633 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -1,6 +1,30 @@ -# GKE Multitenant Module +# GKE Multitenant Example -TODO: add brief explanation and refer back to dev folder? +This example presents an opinionated architecture to handle multiple homogeneous GKE clusters. The general idea behind this example is to deploy a single project hosting multiple clusters leveraging several useful GKE features. This pattern is useful, for example, in cases where multiple clusters host/support the same workloads, such as in the case of a multi-regional deployment. + +In addition to supporting multiple clusters, the architecture assumes that multiple tenants (e.g. teams, applications) will share the cluster. As such, several options are provided to isolate tenants from each other. + +- Private clusters +- VPC-native only. Route-based clusters are not (and will not be) supported +- Metering enabled, and data is stored in a BQ dataset +- DB encryption +- Optional gke fleet support with support for workload identity, config sync, hierarchy controller and policy controller +- logging monitoring to cloud operations by default +- support for groups for gke to allow flexible RBAC policies +- optional etcd database encryption with KMS +- support to customize peering configuration of the control plane vpc +- features enabled by default + - workload identity + - shielded nodes + - dataplane v2 + - intranode visibility + - dns cache + - http load balancing + - gce persistent disk csi driver + - node auto upgrade and auto repair for all nodepools + + +This example is used as part of the [FAST GKE stage](../../../fast/stages/03-gke-multitenant/) but it can also be used independently if desired.

GKE multitenant From d83e3ad83bb5198854fb861c470e9c02d966465f Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Fri, 2 Sep 2022 11:53:35 +0200 Subject: [PATCH 66/75] fleet example readme --- .../multitenant-fleet/README.md | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 54dc594633..895ce3c259 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -2,27 +2,7 @@ This example presents an opinionated architecture to handle multiple homogeneous GKE clusters. The general idea behind this example is to deploy a single project hosting multiple clusters leveraging several useful GKE features. This pattern is useful, for example, in cases where multiple clusters host/support the same workloads, such as in the case of a multi-regional deployment. -In addition to supporting multiple clusters, the architecture assumes that multiple tenants (e.g. teams, applications) will share the cluster. As such, several options are provided to isolate tenants from each other. - -- Private clusters -- VPC-native only. Route-based clusters are not (and will not be) supported -- Metering enabled, and data is stored in a BQ dataset -- DB encryption -- Optional gke fleet support with support for workload identity, config sync, hierarchy controller and policy controller -- logging monitoring to cloud operations by default -- support for groups for gke to allow flexible RBAC policies -- optional etcd database encryption with KMS -- support to customize peering configuration of the control plane vpc -- features enabled by default - - workload identity - - shielded nodes - - dataplane v2 - - intranode visibility - - dns cache - - http load balancing - - gce persistent disk csi driver - - node auto upgrade and auto repair for all nodepools - +In addition to supporting multiple clusters, the architecture presented here assumes that multiple tenants (e.g. teams, applications) will share the cluster. As such, several options are provided to isolate tenants from each other. This example is used as part of the [FAST GKE stage](../../../fast/stages/03-gke-multitenant/) but it can also be used independently if desired. @@ -30,6 +10,42 @@ This example is used as part of the [FAST GKE stage](../../../fast/stages/03-gke GKE multitenant

+The overall architecture is based on the following design decisions: + +- All clusters are assumed to be [private](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), therefore only [VPC-native clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips) are supported. +- Logging and monitoring configured to use Cloud Operations for system components and user workloads. +- [GKE metering](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-usage-metering) enabled by default and stored in a bigquery dataset created withing the project. +- Optional [GKE Fleet](https://cloud.google.com/kubernetes-engine/docs/fleets-overview) support with the possibility to enable any of the following features: + - [Fleet workload identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) + - [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/overview) + - [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/overview) + - [Anthos Identity Service](https://cloud.google.com/anthos/identity/setup/fleet) + - [Multi-cluster services](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-services) + - [Multi-cluster ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-ingress). +- Support for [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview), [Hierarchy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/hierarchy-controller), and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) when using Anthos Config Management. +- [Groups for GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/google-groups-rbac) can be enabled to facilitate the creation of flexible RBAC policies referencing group principals. +- Support for [application layer secret encryption](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets). +- Support to customize peering configuration of the control plane VPC (e.g. to import/export routes to the peered network) +- Some features are enabled by default in all clusters: + - [Intranode visibility](https://cloud.google.com/kubernetes-engine/docs/how-to/intranode-visibility) + - [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2) + - [Shielded GKE nodes](https://cloud.google.com/kubernetes-engine/docs/how-to/shielded-gke-nodes) + - [Workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) + - [Node local DNS cache](https://cloud.google.com/kubernetes-engine/docs/how-to/nodelocal-dns-cache) + - [Use of the GCE persistent disk CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) + - Node [auto-upgrade](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) and [auto-repair](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair) for all node pools + + + +## Basic usage + +## Fleet configuration + +## Multi-tenant usage + + This is an example of that shows the use of the above variables: ```hcl From a62fda5b66d4f5068df3636432e3503de71eb40c Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Tue, 6 Sep 2022 15:24:25 +0200 Subject: [PATCH 67/75] Update gke docs --- .../multitenant-fleet/README.md | 37 ++++++++++++++++++- .../multitenant-fleet/gke-clusters.tf | 2 +- .../multitenant-fleet/variables.tf | 2 +- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 895ce3c259..e13b724bfd 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -1,8 +1,8 @@ # GKE Multitenant Example -This example presents an opinionated architecture to handle multiple homogeneous GKE clusters. The general idea behind this example is to deploy a single project hosting multiple clusters leveraging several useful GKE features. This pattern is useful, for example, in cases where multiple clusters host/support the same workloads, such as in the case of a multi-regional deployment. +This example presents an opinionated architecture to handle multiple homogeneous GKE clusters. The general idea behind this example is to deploy a single project hosting multiple clusters leveraging several useful GKE features. -In addition to supporting multiple clusters, the architecture presented here assumes that multiple tenants (e.g. teams, applications) will share the cluster. As such, several options are provided to isolate tenants from each other. +The pattern used in this design is useful, for example, in cases where multiple clusters host/support the same workloads, such as in the case of a multi-regional deployment. Furthermore, combined with Anthos Config Sync and proper RBAC, this architecture can be used to host multiple tenants (e.g. teams, applications) sharing the clusters. This example is used as part of the [FAST GKE stage](../../../fast/stages/03-gke-multitenant/) but it can also be used independently if desired. @@ -41,6 +41,39 @@ The overall architecture is based on the following design decisions: ## Basic usage +The following example shows how to deploy a single cluster and a single node pool + +```hcl +clusters = { + "mycluster" = { + cluster_autoscaling = null + description = "mycluster" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = "//www.googleapis.com/compute/v1/projects//regions/europe-west1/subnetworks/" + } + overrides = null + } +} +nodepools = { + "mycluster" = { + "mynodepool" = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } + } +} + +``` + ## Fleet configuration ## Multi-tenant usage diff --git a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf index 8e04d780bf..18188b2e6e 100644 --- a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf +++ b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf @@ -97,7 +97,7 @@ module "gke-cluster" { state = "ENCRYPTED" key_name = each.value.overrides.database_encryption_key } - ) + default_max_pods_per_node = each.value.overrides.max_pods_per_node master_authorized_ranges = each.value.overrides.master_authorized_ranges pod_security_policy = each.value.overrides.pod_security_policy diff --git a/examples/gke-serverless/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf index 37138c3798..48602af21a 100644 --- a/examples/gke-serverless/multitenant-fleet/variables.tf +++ b/examples/gke-serverless/multitenant-fleet/variables.tf @@ -156,7 +156,7 @@ variable "fleet_features" { variable "fleet_workload_identity" { description = "Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true." type = bool - default = true + default = false nullable = false } From fce4170fe8dea2edd0bac5acc35171794c985119 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 7 Sep 2022 13:45:06 +0200 Subject: [PATCH 68/75] Add missing paren. --- examples/gke-serverless/multitenant-fleet/gke-clusters.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf index 18188b2e6e..8e04d780bf 100644 --- a/examples/gke-serverless/multitenant-fleet/gke-clusters.tf +++ b/examples/gke-serverless/multitenant-fleet/gke-clusters.tf @@ -97,7 +97,7 @@ module "gke-cluster" { state = "ENCRYPTED" key_name = each.value.overrides.database_encryption_key } - + ) default_max_pods_per_node = each.value.overrides.max_pods_per_node master_authorized_ranges = each.value.overrides.master_authorized_ranges pod_security_policy = each.value.overrides.pod_security_policy From 218b374fc5675944e26ada55d60a99e060ba933e Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 7 Sep 2022 13:48:32 +0200 Subject: [PATCH 69/75] Update multitenant-fleet README --- examples/gke-serverless/multitenant-fleet/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index e13b724bfd..29f53deeee 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -222,7 +222,7 @@ fleet_features = { | [fleet_configmanagement_clusters](variables.tf#L100) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | | [fleet_configmanagement_templates](variables.tf#L108) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | | [fleet_features](variables.tf#L143) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L156) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | true | | +| [fleet_workload_identity](variables.tf#L156) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | | [group_iam](variables.tf#L168) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | | [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | | [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | From 3ffdd0dd4301f2f8e93af9abc64828cf1ad351bf Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Wed, 7 Sep 2022 16:31:42 +0200 Subject: [PATCH 70/75] More examples for multitenant-fleet example --- .../multitenant-fleet/README.md | 335 ++++++++++++------ .../multitenant-fleet/variables.tf | 1 - tests/doc_examples/variables.tf | 2 +- 3 files changed, 222 insertions(+), 116 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 29f53deeee..471a0c44df 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -44,151 +44,258 @@ The overall architecture is based on the following design decisions: The following example shows how to deploy a single cluster and a single node pool ```hcl -clusters = { - "mycluster" = { - cluster_autoscaling = null - description = "mycluster" - dns_domain = null - location = "europe-west1" - labels = {} - net = { - master_range = "172.17.16.0/28" - pods = "pods" - services = "services" - subnet = "//www.googleapis.com/compute/v1/projects//regions/europe-west1/subnetworks/" +module "gke" { + source = "./fabric/examples/gke-serverless/multitenant-fleet/" + project_id = var.project_id + billing_account_id = var.billing_account_id + folder_id = var.folder_id + prefix = "myprefix" + vpc_config = { + host_project_id = "my-host-project-id" + vpc_self_link = "projects/my-host-project-id/global/networks/my-network" + } + +authenticator_security_group = "gke-rbac-base@example.com" + group_iam = { + "gke-admin@example.com" = [ + "roles/container.admin" + ] + } + iam = { + "roles/container.clusterAdmin" = [ + "cicd@my-cicd-project.iam.gserviceaccount.com" + ] + } + + clusters = { + mycluster = { + cluster_autoscaling = null + description = "My cluster" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = "projects/my-host-project-id/regions/europe-west1/subnetworks/mycluster-subnet" + } + overrides = null } - overrides = null } -} -nodepools = { - "mycluster" = { - "mynodepool" = { - initial_node_count = 1 - node_count = 1 - node_type = "n2-standard-4" - overrides = null - spot = false + nodepools = { + mycluster = { + mynodepool = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } } } } - +# tftest modules=1 resources=0 ``` -## Fleet configuration +## Creating Multiple Clusters -## Multi-tenant usage +The following example shows how to deploy two clusters with different configurations. +The first cluster `cluster-euw1` defines the mandatory configuration parameters (description, location, network setup) and inherits the some defaults from the `cluster_defaults` and `nodepool_detaults` variables. These two variables are used whenever the `override` key of the `clusters` and `nodepools` variables are set to `null`. + +On the other hand, the second cluster (`cluster-euw3`) defines its own configuration by providing a value to the `overrides` key. -This is an example of that shows the use of the above variables: ```hcl -# the `cluster_defaults` variable defaults are used and not shown here -clusters = { - "gke-00" = { - cluster_autoscaling = null - description = "gke-00" - dns_domain = null - location = "europe-west1" - labels = {} - net = { - master_range = "172.17.16.0/28" - pods = "pods" - services = "services" - subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] - } - overrides = null +module "gke" { + source = "./fabric/examples/gke-serverless/multitenant-fleet/" + project_id = var.project_id + billing_account_id = var.billing_account_id + folder_id = var.folder_id + prefix = "myprefix" + vpc_config = { + host_project_id = "my-host-project-id" + vpc_self_link = "projects/my-host-project-id/global/networks/my-network" } - "gke-01" = { - cluster_autoscaling = null - description = "gke-01" - dns_domain = null - location = "europe-west3" - labels = {} - net = { - master_range = "172.17.17.0/28" - pods = "pods" - services = "services" - subnet = local.vpc.subnet_self_links["europe-west3/gke-dev-0"] + clusters = { + cluster-euw1 = { + cluster_autoscaling = null + description = "Cluster for europ-west1" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = "projects/my-host-project-id/regions/europe-west1/subnetworks/euw1-subnet" + } + overrides = null } - overrides = { - cloudrun_config = false - database_encryption_key = null - gcp_filestore_csi_driver_config = true - master_authorized_ranges = { - rfc1918_1 = "10.0.0.0/8" + cluster-euw3 = { + cluster_autoscaling = null + description = "Cluster for europe-west3" + dns_domain = null + location = "europe-west3" + labels = {} + net = { + master_range = "172.17.17.0/28" + pods = "pods" + services = "services" + subnet = "projects/my-host-project-id/regions/europe-west3/subnetworks/euw3-subnet" + } + overrides = { + cloudrun_config = false + database_encryption_key = null + gcp_filestore_csi_driver_config = true + master_authorized_ranges = { + rfc1918_1 = "10.0.0.0/8" + } + max_pods_per_node = 64 + pod_security_policy = true + release_channel = "STABLE" + vertical_pod_autoscaling = false } - max_pods_per_node = 64 - pod_security_policy = true - release_channel = "STABLE" - vertical_pod_autoscaling = false } } -} -nodepools = { - "gke-0" = { - "gke-00-000" = { - initial_node_count = 1 - node_count = 1 - node_type = "n2-standard-4" - overrides = null - spot = false + nodepools = { + cluster-euw1 = { + pool-euw1 = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } } - } - "gke-1" = { - "gke-01-000" = { - initial_node_count = 1 - node_count = 1 - node_type = "n2-standard-4" - overrides = { - image_type = "UBUNTU_CONTAINERD" - max_pods_per_node = 64 - node_locations = [] - node_tags = [] - node_taints = [] + cluster-euw3 = { + pool-euw3 = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = { + image_type = "UBUNTU_CONTAINERD" + max_pods_per_node = 64 + node_locations = [] + node_tags = [] + node_taints = [] + } + spot = true } - spot = true } } } +# tftest modules=1 resources=0 ``` +## Multitenant configuration + + +## Fleet configuration + + ```hcl -fleet_configmanagement_templates = { - default = { - binauthz = false - config_sync = { - git = { - gcp_service_account_email = null - https_proxy = null - policy_dir = "configsync" - secret_type = "none" - source_format = "hierarchy" - sync_branch = "main" - sync_repo = "https://github.com/.../..." - sync_rev = null - sync_wait_secs = null +module "gke" { + source = "./fabric/examples/gke-serverless/multitenant-fleet/" + project_id = var.project_id + billing_account_id = var.billing_account_id + folder_id = var.folder_id + prefix = "myprefix" + vpc_config = { + host_project_id = "my-host-project-id" + vpc_self_link = "projects/my-host-project-id/global/networks/my-network" + } + clusters = { + cluster-euw1 = { + cluster_autoscaling = null + description = "Cluster for europe-west1" + dns_domain = null + location = "europe-west1" + labels = {} + net = { + master_range = "172.17.16.0/28" + pods = "pods" + services = "services" + subnet = "projects/my-host-project-id/regions/europe-west1/subnetworks/euw1-subnet" + } + overrides = null + } + cluster-euw3 = { + cluster_autoscaling = null + description = "Cluster for europe-west3" + dns_domain = null + location = "europe-west3" + labels = {} + net = { + master_range = "172.17.17.0/28" + pods = "pods" + services = "services" + subnet = "projects/my-host-project-id/regions/europe-west3/subnetworks/euw3-subnet" + } + overrides = null + } + } + nodepools = { + cluster-euw1 = { + pool-euw1 = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = false + } + } + cluster-euw3 = { + pool-euw3 = { + initial_node_count = 1 + node_count = 1 + node_type = "n2-standard-4" + overrides = null + spot = true } - prevent_drift = true - source_format = "hierarchy" } - hierarchy_controller = null - policy_controller = null - version = "1.10.2" } -} -fleet_configmanagement_clusters = { - default = ["gke-1", "gke-2"] -} + fleet_configmanagement_templates = { + default = { + binauthz = false + config_sync = { + git = { + gcp_service_account_email = null + https_proxy = null + policy_dir = "configsync" + secret_type = "none" + source_format = "hierarchy" + sync_branch = "main" + sync_repo = "https://github.com/.../..." + sync_rev = null + sync_wait_secs = null + } + prevent_drift = true + source_format = "hierarchy" + } + hierarchy_controller = null + policy_controller = null + version = "1.10.2" + } + } -fleet_features = { - appdevexperience = false - configmanagement = false - identityservice = false - multiclusteringress = "gke-1" - multiclusterservicediscovery = true - servicemesh = false + fleet_configmanagement_clusters = { + default = ["cluster-euw1", "cluster-euw3"] + } + + fleet_features = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = "cluster-euw1" + multiclusterservicediscovery = true + servicemesh = false + } } + +# tftest modules=1 resources=0 ``` diff --git a/examples/gke-serverless/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf index 48602af21a..dd7e4ababa 100644 --- a/examples/gke-serverless/multitenant-fleet/variables.tf +++ b/examples/gke-serverless/multitenant-fleet/variables.tf @@ -104,7 +104,6 @@ variable "fleet_configmanagement_clusters" { nullable = false } - variable "fleet_configmanagement_templates" { description = "Sets of config management configurations that can be applied to member clusters, in config name => {options} format." type = map(object({ diff --git a/tests/doc_examples/variables.tf b/tests/doc_examples/variables.tf index 7e148dc1dc..35f7b06c11 100644 --- a/tests/doc_examples/variables.tf +++ b/tests/doc_examples/variables.tf @@ -37,7 +37,7 @@ variable "folder_id" { } variable "project_id" { - default = "projects/project-id" + default = "project-id" } variable "region" { From 15e45cb3b95b1dfc2ae18ba6338d5437539dccc4 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 8 Sep 2022 09:26:04 +0200 Subject: [PATCH 71/75] Multipe-cluster example for multitenant-fleet example --- .../multitenant-fleet/README.md | 47 ++++++++++++------- .../multitenant-fleet/variables.tf | 4 -- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 471a0c44df..99bbdbb09a 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -55,7 +55,7 @@ module "gke" { vpc_self_link = "projects/my-host-project-id/global/networks/my-network" } -authenticator_security_group = "gke-rbac-base@example.com" + authenticator_security_group = "gke-rbac-base@example.com" group_iam = { "gke-admin@example.com" = [ "roles/container.admin" @@ -189,11 +189,14 @@ module "gke" { # tftest modules=1 resources=0 ``` -## Multitenant configuration +## Multiple clusters with GKE Fleet +This example deploys two clusters and configures the several GKE Fleet features: -## Fleet configuration - +- Enables [multi-cluster ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-ingress) and sets the configuration cluster to be `cluster-eu1`. +- Enables [Multi-cluster services](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-services) and gives assigns the [required roles](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-cluster-services#authenticating) to its service accounts. +- A `default` Config Management template is created with binary authorization, config sync enabled with a git repository, hierarchy controller, and policy controller. +- The two clusters are configured to use the `default` Config Management template. ```hcl module "gke" { @@ -257,9 +260,18 @@ module "gke" { } } + fleet_features = { + appdevexperience = false + configmanagement = true + identityservice = true + multiclusteringress = "cluster-euw1" + multiclusterservicediscovery = true + servicemesh = true + } + fleet_workload_identity = true fleet_configmanagement_templates = { default = { - binauthz = false + binauthz = true config_sync = { git = { gcp_service_account_email = null @@ -268,31 +280,30 @@ module "gke" { secret_type = "none" source_format = "hierarchy" sync_branch = "main" - sync_repo = "https://github.com/.../..." + sync_repo = "https://github.com/myorg/myrepo" sync_rev = null sync_wait_secs = null } prevent_drift = true source_format = "hierarchy" } - hierarchy_controller = null - policy_controller = null + hierarchy_controller = { + enable_hierarchical_resource_quota = true + enable_pod_tree_labels = true + } + policy_controller = { + audit_interval_seconds = 30 + exemptable_namespaces = ["kube-system"] + log_denies_enabled = true + referential_rules_enabled = true + template_library_installed = true + } version = "1.10.2" } } - fleet_configmanagement_clusters = { default = ["cluster-euw1", "cluster-euw3"] } - - fleet_features = { - appdevexperience = false - configmanagement = false - identityservice = false - multiclusteringress = "cluster-euw1" - multiclusterservicediscovery = true - servicemesh = false - } } # tftest modules=1 resources=0 diff --git a/examples/gke-serverless/multitenant-fleet/variables.tf b/examples/gke-serverless/multitenant-fleet/variables.tf index dd7e4ababa..bb61bff261 100644 --- a/examples/gke-serverless/multitenant-fleet/variables.tf +++ b/examples/gke-serverless/multitenant-fleet/variables.tf @@ -14,10 +14,6 @@ * limitations under the License. */ -# we deal with one env here -# 1 project, m clusters -# cloud dns for gke? - variable "authenticator_security_group" { description = "Optional group used for Groups for GKE." type = string From 7d82143e9399dcc3c18d2ce74ecbc193e850b7b1 Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 8 Sep 2022 09:31:42 +0200 Subject: [PATCH 72/75] Update README --- .../multitenant-fleet/README.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 99bbdbb09a..209d2ab9b7 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -327,26 +327,26 @@ module "gke" { | name | description | type | required | default | producer | |---|---|:---:|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L27) | Billing account id. | string | ✓ | | | -| [clusters](variables.tf#L61) | | map(object({…})) | ✓ | | | -| [folder_id](variables.tf#L163) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | -| [nodepools](variables.tf#L206) | | map(map(object({…}))) | ✓ | | | -| [prefix](variables.tf#L236) | Prefix used for resources that need unique names. | string | ✓ | | | -| [project_id](variables.tf#L241) | ID of the project that will contain all the clusters. | string | ✓ | | | -| [vpc_config](variables.tf#L253) | Shared VPC project and VPC details. | object({…}) | ✓ | | | -| [authenticator_security_group](variables.tf#L21) | Optional group used for Groups for GKE. | string | | null | | -| [cluster_defaults](variables.tf#L32) | Default values for optional cluster configurations. | object({…}) | | {…} | | -| [dns_domain](variables.tf#L94) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | -| [fleet_configmanagement_clusters](variables.tf#L100) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | -| [fleet_configmanagement_templates](variables.tf#L108) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | -| [fleet_features](variables.tf#L143) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | -| [fleet_workload_identity](variables.tf#L156) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | -| [group_iam](variables.tf#L168) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | -| [iam](variables.tf#L175) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | -| [labels](variables.tf#L182) | Project-level labels. | map(string) | | {} | | -| [nodepool_defaults](variables.tf#L188) | | object({…}) | | {…} | | -| [peering_config](variables.tf#L223) | Configure peering with the control plane VPC. Requires compute.networks.updatePeering. Set to null if you don't want to update the default peering configuration. | object({…}) | | {…} | | -| [project_services](variables.tf#L246) | Additional project services to enable. | list(string) | | [] | | +| [billing_account_id](variables.tf#L23) | Billing account id. | string | ✓ | | | +| [clusters](variables.tf#L57) | | map(object({…})) | ✓ | | | +| [folder_id](variables.tf#L158) | Folder used for the GKE project in folders/nnnnnnnnnnn format. | string | ✓ | | | +| [nodepools](variables.tf#L201) | | map(map(object({…}))) | ✓ | | | +| [prefix](variables.tf#L231) | Prefix used for resources that need unique names. | string | ✓ | | | +| [project_id](variables.tf#L236) | ID of the project that will contain all the clusters. | string | ✓ | | | +| [vpc_config](variables.tf#L248) | Shared VPC project and VPC details. | object({…}) | ✓ | | | +| [authenticator_security_group](variables.tf#L17) | Optional group used for Groups for GKE. | string | | null | | +| [cluster_defaults](variables.tf#L28) | Default values for optional cluster configurations. | object({…}) | | {…} | | +| [dns_domain](variables.tf#L90) | Domain name used for clusters, prefixed by each cluster name. Leave null to disable Cloud DNS for GKE. | string | | null | | +| [fleet_configmanagement_clusters](variables.tf#L96) | Config management features enabled on specific sets of member clusters, in config name => [cluster name] format. | map(list(string)) | | {} | | +| [fleet_configmanagement_templates](variables.tf#L103) | Sets of config management configurations that can be applied to member clusters, in config name => {options} format. | map(object({…})) | | {} | | +| [fleet_features](variables.tf#L138) | Enable and configue fleet features. Set to null to disable GKE Hub if fleet workload identity is not used. | object({…}) | | null | | +| [fleet_workload_identity](variables.tf#L151) | Use Fleet Workload Identity for clusters. Enables GKE Hub if set to true. | bool | | false | | +| [group_iam](variables.tf#L163) | Project-level IAM bindings for groups. Use group emails as keys, list of roles as values. | map(list(string)) | | {} | | +| [iam](variables.tf#L170) | Project-level authoritative IAM bindings for users and service accounts in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | +| [labels](variables.tf#L177) | Project-level labels. | map(string) | | {} | | +| [nodepool_defaults](variables.tf#L183) | | object({…}) | | {…} | | +| [peering_config](variables.tf#L218) | Configure peering with the control plane VPC. Requires compute.networks.updatePeering. Set to null if you don't want to update the default peering configuration. | object({…}) | | {…} | | +| [project_services](variables.tf#L241) | Additional project services to enable. | list(string) | | [] | | ## Outputs From 444b370e06a5e2aa0d055ccf5f850bb3a57f578c Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 8 Sep 2022 22:23:00 +0200 Subject: [PATCH 73/75] README fixes --- examples/gke-serverless/multitenant-fleet/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/gke-serverless/multitenant-fleet/README.md b/examples/gke-serverless/multitenant-fleet/README.md index 209d2ab9b7..7106fca3f6 100644 --- a/examples/gke-serverless/multitenant-fleet/README.md +++ b/examples/gke-serverless/multitenant-fleet/README.md @@ -14,7 +14,7 @@ The overall architecture is based on the following design decisions: - All clusters are assumed to be [private](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), therefore only [VPC-native clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips) are supported. - Logging and monitoring configured to use Cloud Operations for system components and user workloads. -- [GKE metering](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-usage-metering) enabled by default and stored in a bigquery dataset created withing the project. +- [GKE metering](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-usage-metering) enabled by default and stored in a bigquery dataset created within the project. - Optional [GKE Fleet](https://cloud.google.com/kubernetes-engine/docs/fleets-overview) support with the possibility to enable any of the following features: - [Fleet workload identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) - [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/overview) @@ -102,7 +102,7 @@ module "gke" { The following example shows how to deploy two clusters with different configurations. -The first cluster `cluster-euw1` defines the mandatory configuration parameters (description, location, network setup) and inherits the some defaults from the `cluster_defaults` and `nodepool_detaults` variables. These two variables are used whenever the `override` key of the `clusters` and `nodepools` variables are set to `null`. +The first cluster `cluster-euw1` defines the mandatory configuration parameters (description, location, network setup) and inherits the some defaults from the `cluster_defaults` and `nodepool_deaults` variables. These two variables are used whenever the `override` key of the `clusters` and `nodepools` variables are set to `null`. On the other hand, the second cluster (`cluster-euw3`) defines its own configuration by providing a value to the `overrides` key. @@ -191,10 +191,10 @@ module "gke" { ## Multiple clusters with GKE Fleet -This example deploys two clusters and configures the several GKE Fleet features: +This example deploys two clusters and configures several GKE Fleet features: - Enables [multi-cluster ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-ingress) and sets the configuration cluster to be `cluster-eu1`. -- Enables [Multi-cluster services](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-services) and gives assigns the [required roles](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-cluster-services#authenticating) to its service accounts. +- Enables [Multi-cluster services](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-services) and assigns the [required roles](https://cloud.google.com/kubernetes-engine/docs/how-to/multi-cluster-services#authenticating) to its service accounts. - A `default` Config Management template is created with binary authorization, config sync enabled with a git repository, hierarchy controller, and policy controller. - The two clusters are configured to use the `default` Config Management template. From ff7e39828d61095d9e89c8f77637c879541e312f Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 8 Sep 2022 22:33:14 +0200 Subject: [PATCH 74/75] Finished readme of fast gke stage --- fast/stages/03-gke-multitenant/dev/README.md | 35 +++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 969828616a..56ff60d89d 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -2,7 +2,7 @@ This stage allows creation and management of a fleet of GKE multitenant clusters, optionally leveraging GKE Hub to configure additional features. It's designed to be replicated once for every homogeneous set of clusters, either per environment or with more granularity as needed (e.g. teams or sets of teams sharing similar requirements). -The following diagram illustrates the high-level design of created resources and a schema of the VPC SC design, which can be adapted to specific requirements via variables: +The following diagram illustrates the high-level design of created resources, which can be adapted to specific requirements via variables:

GKE multitenant @@ -10,6 +10,33 @@ The following diagram illustrates the high-level design of created resources and ## Design overview and choices +> The detailed architecture of the underlying resources is explained in the documentation of [GKE multitenant module](../../../../examples/gke-serverless/multitenant-fleet/README.md). + +This stage creates containing a cluster and as many clusters and node pools as requested by the user through the [variables](#variables) explained below. The GKE clusters are created with the with the following setup + +- All clusters are assumed to be [private](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), therefore only [VPC-native clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips) are supported. +- Logging and monitoring configured to use Cloud Operations for system components and user workloads. +- [GKE metering](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-usage-metering) enabled by default and stored in a bigquery dataset created within the project. +- Optional [GKE Fleet](https://cloud.google.com/kubernetes-engine/docs/fleets-overview) support with the possibility to enable any of the following features: + - [Fleet workload identity](https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity) + - [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/overview) + - [Anthos Service Mesh](https://cloud.google.com/service-mesh/docs/overview) + - [Anthos Identity Service](https://cloud.google.com/anthos/identity/setup/fleet) + - [Multi-cluster services](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-services) + - [Multi-cluster ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/multi-cluster-ingress). +- Support for [Config Sync](https://cloud.google.com/anthos-config-management/docs/config-sync-overview), [Hierarchy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/hierarchy-controller), and [Policy Controller](https://cloud.google.com/anthos-config-management/docs/concepts/policy-controller) when using Anthos Config Management. +- [Groups for GKE](https://cloud.google.com/kubernetes-engine/docs/how-to/google-groups-rbac) can be enabled to facilitate the creation of flexible RBAC policies referencing group principals. +- Support for [application layer secret encryption](https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets). +- Support to customize peering configuration of the control plane VPC (e.g. to import/export routes to the peered network) +- Some features are enabled by default in all clusters: + - [Intranode visibility](https://cloud.google.com/kubernetes-engine/docs/how-to/intranode-visibility) + - [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2) + - [Shielded GKE nodes](https://cloud.google.com/kubernetes-engine/docs/how-to/shielded-gke-nodes) + - [Workload identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) + - [Node local DNS cache](https://cloud.google.com/kubernetes-engine/docs/how-to/nodelocal-dns-cache) + - [Use of the GCE persistent disk CSI driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver) + - Node [auto-upgrade](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) and [auto-repair](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair) for all node pools + ## How to run this stage @@ -33,9 +60,9 @@ It's of course possible to run this stage in isolation, by making sure the archi - `roles/compute.viewer` - on the organization or billing account - `roles/billing.admin` - + The VPC host project, VPC and subnets should already exist. - + ### Providers configuration If you're running this on top of FAST, you should run the following commands to create the providers file, and populate the required variables from the previous stage. @@ -99,8 +126,6 @@ terraform init terraform apply ``` -... - From a530620f4d79d8b83970f5413c280f4dcb3b642a Mon Sep 17 00:00:00 2001 From: Julio Castillo Date: Thu, 8 Sep 2022 22:35:01 +0200 Subject: [PATCH 75/75] Fixing typos --- fast/stages/03-gke-multitenant/dev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fast/stages/03-gke-multitenant/dev/README.md b/fast/stages/03-gke-multitenant/dev/README.md index 56ff60d89d..4a18a42ef7 100644 --- a/fast/stages/03-gke-multitenant/dev/README.md +++ b/fast/stages/03-gke-multitenant/dev/README.md @@ -12,7 +12,7 @@ The following diagram illustrates the high-level design of created resources, wh > The detailed architecture of the underlying resources is explained in the documentation of [GKE multitenant module](../../../../examples/gke-serverless/multitenant-fleet/README.md). -This stage creates containing a cluster and as many clusters and node pools as requested by the user through the [variables](#variables) explained below. The GKE clusters are created with the with the following setup +This stage creates a project containing and as many clusters and node pools as requested by the user through the [variables](#variables) explained below. The GKE clusters are created with the with the following setup: - All clusters are assumed to be [private](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), therefore only [VPC-native clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips) are supported. - Logging and monitoring configured to use Cloud Operations for system components and user workloads.