From a9430683005fe7908171c8e537eb929985964f48 Mon Sep 17 00:00:00 2001 From: Max Portocarrero CI&T <105444618+maxi-cit@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:39:12 -0500 Subject: [PATCH 01/33] add serverTLSpolicy to google_compute_region_target_https_proxy (#9105) --- .../compute/RegionTargetHttpsProxy.yaml | 26 +++++ .../region_target_https_proxy_mtls.tf.erb | 101 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 mmv1/templates/terraform/examples/region_target_https_proxy_mtls.tf.erb diff --git a/mmv1/products/compute/RegionTargetHttpsProxy.yaml b/mmv1/products/compute/RegionTargetHttpsProxy.yaml index 38042bfb1c9c..94cc7c0aa635 100644 --- a/mmv1/products/compute/RegionTargetHttpsProxy.yaml +++ b/mmv1/products/compute/RegionTargetHttpsProxy.yaml @@ -58,6 +58,18 @@ examples: region_url_map_name: 'url-map' region_backend_service_name: 'backend-service' region_health_check_name: 'http-health-check' + - !ruby/object:Provider::Terraform::Examples + name: 'region_target_https_proxy_mtls' + primary_resource_id: 'default' + min_version: 'beta' + vars: + target_https_proxy_name: 'test-mtls-proxy' + ssl_certificate_name: 'my-certificate' + url_map_name: 'url-map' + backend_service_name: 'backend-service' + http_health_check_name: 'http-health-check' + server_tls_policy_name: 'my-tls-policy' + trust_config_name: 'my-trust-config' - !ruby/object:Provider::Terraform::Examples name: 'region_target_https_proxy_certificate_manager_certificate' primary_resource_id: 'default' @@ -179,3 +191,17 @@ properties: update_verb: :POST custom_expand: 'templates/terraform/custom_expand/resourceref_with_validation.go.erb' update_url: 'projects/{{project}}/regions/{{region}}/targetHttpsProxies/{{name}}/setUrlMap' + - !ruby/object:Api::Type::ResourceRef + name: 'serverTlsPolicy' + resource: 'SslPolicy' + imports: 'selfLink' + description: | + A URL referring to a networksecurity.ServerTlsPolicy + resource that describes how the proxy should authenticate inbound + traffic. serverTlsPolicy only applies to a global TargetHttpsProxy + attached to globalForwardingRules with the loadBalancingScheme + set to INTERNAL_SELF_MANAGED or EXTERNAL or EXTERNAL_MANAGED. + For details which ServerTlsPolicy resources are accepted with + INTERNAL_SELF_MANAGED and which with EXTERNAL, EXTERNAL_MANAGED + loadBalancingScheme consult ServerTlsPolicy documentation. + If left blank, communications are not encrypted. diff --git a/mmv1/templates/terraform/examples/region_target_https_proxy_mtls.tf.erb b/mmv1/templates/terraform/examples/region_target_https_proxy_mtls.tf.erb new file mode 100644 index 000000000000..7ad309adeb10 --- /dev/null +++ b/mmv1/templates/terraform/examples/region_target_https_proxy_mtls.tf.erb @@ -0,0 +1,101 @@ +data "google_project" "project" { + provider = google-beta +} + +resource "google_compute_region_target_https_proxy" "<%= ctx[:primary_resource_id] %>" { + provider = google-beta + region = "us-central1" + name = "<%= ctx[:vars]['target_https_proxy_name'] %>" + url_map = google_compute_region_url_map.default.id + ssl_certificates = [google_compute_region_ssl_certificate.default.id] + server_tls_policy = google_network_security_server_tls_policy.default.id +} + +resource "google_certificate_manager_trust_config" "default" { + provider = google-beta + location = "us-central1" + name = "<%= ctx[:vars]["trust_config_name"] %>" + description = "sample description for trust config" + + trust_stores { + trust_anchors { + pem_certificate = file("test-fixtures/ca_cert.pem") + } + intermediate_cas { + pem_certificate = file("test-fixtures/ca_cert.pem") + } + } + + labels = { + foo = "bar" + } +} + +resource "google_network_security_server_tls_policy" "default" { + provider = google-beta + location = "us-central1" + name = "<%= ctx[:vars]['server_tls_policy_name'] %>" + description = "my description" + allow_open = "false" + mtls_policy { + client_validation_mode = "REJECT_INVALID" + client_validation_trust_config = "projects/${data.google_project.project.number}/locations/us-central1/trustConfigs/${google_certificate_manager_trust_config.default.name}" + } +} + +resource "google_compute_region_ssl_certificate" "default" { + provider = google-beta + region = "us-central1" + name = "<%= ctx[:vars]['ssl_certificate_name'] %>" + private_key = file("path/to/private.key") + certificate = file("path/to/certificate.crt") +} + +resource "google_compute_region_url_map" "default" { + provider = google-beta + region = "us-central1" + name = "<%= ctx[:vars]['url_map_name'] %>" + description = "a description" + + default_service = google_compute_region_backend_service.default.id + + host_rule { + hosts = ["mysite.com"] + path_matcher = "allpaths" + } + + path_matcher { + name = "allpaths" + default_service = google_compute_region_backend_service.default.id + + path_rule { + paths = ["/*"] + service = google_compute_region_backend_service.default.id + } + } +} + +resource "google_compute_region_backend_service" "default" { + provider = google-beta + region = "us-central1" + name = "<%= ctx[:vars]['backend_service_name'] %>" + port_name = "http" + protocol = "HTTP" + timeout_sec = 10 + + load_balancing_scheme = "INTERNAL_MANAGED" + + health_checks = [google_compute_region_health_check.default.id] +} + +resource "google_compute_region_health_check" "default" { + provider = google-beta + region = "us-central1" + name = "<%= ctx[:vars]['http_health_check_name'] %>" + check_interval_sec = 1 + timeout_sec = 1 + + http_health_check { + port = 80 + } +} From d21a270947923ed7a62637007aa8e0a616203b78 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Tue, 23 Apr 2024 12:01:55 -0700 Subject: [PATCH 02/33] Lower timeout for dangling cluster creation to 10s (#10502) --- .../services/container/resource_container_cluster_test.go.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb index fb7d81942690..f69cf2e5b507 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.erb @@ -3789,7 +3789,7 @@ func TestAccContainerCluster_errorCleanDanglingCluster(t *testing.T) { initConfig := testAccContainerCluster_withInitialCIDR(containerNetName, clusterName) overlapConfig := testAccContainerCluster_withCIDROverlap(initConfig, clusterNameError) - overlapConfigWithTimeout := testAccContainerCluster_withCIDROverlapWithTimeout(initConfig, clusterNameErrorWithTimeout, "40s") + overlapConfigWithTimeout := testAccContainerCluster_withCIDROverlapWithTimeout(initConfig, clusterNameErrorWithTimeout, "1s") checkTaintApplied := func(st *terraform.State) error { // Return an error if there is no tainted (i.e. marked for deletion) cluster. @@ -3835,7 +3835,7 @@ func TestAccContainerCluster_errorCleanDanglingCluster(t *testing.T) { Check: checkTaintApplied, }, { - // Next attempt to create the overlapping cluster with a 40s timeout. This will fail with a different error. + // Next attempt to create the overlapping cluster with a 1s timeout. This will fail with a different error. Config: overlapConfigWithTimeout, ExpectError: regexp.MustCompile("timeout while waiting for state to become 'DONE'"), }, From 9059b44c37c1a6a92ed7a2c171337d559a8a2662 Mon Sep 17 00:00:00 2001 From: Matheus Guilherme Souza Aleixo <82680416+matheusaleixo-cit@users.noreply.github.com> Date: Tue, 23 Apr 2024 16:28:57 -0300 Subject: [PATCH 03/33] Added new resource "Security Policy Rule" (#10434) --- mmv1/products/compute/SecurityPolicyRule.yaml | 289 +++++++++++ .../security_policy_rule_basic.tf.erb | 19 + .../security_policy_rule_default_rule.tf.erb | 37 ++ ...security_policy_rule_multiple_rules.tf.erb | 33 ++ ...e_compute_security_policy_rule_test.go.erb | 454 ++++++++++++++++++ 5 files changed, 832 insertions(+) create mode 100644 mmv1/products/compute/SecurityPolicyRule.yaml create mode 100644 mmv1/templates/terraform/examples/security_policy_rule_basic.tf.erb create mode 100644 mmv1/templates/terraform/examples/security_policy_rule_default_rule.tf.erb create mode 100644 mmv1/templates/terraform/examples/security_policy_rule_multiple_rules.tf.erb create mode 100644 mmv1/third_party/terraform/services/compute/resource_compute_security_policy_rule_test.go.erb diff --git a/mmv1/products/compute/SecurityPolicyRule.yaml b/mmv1/products/compute/SecurityPolicyRule.yaml new file mode 100644 index 000000000000..b5d59cafa0f5 --- /dev/null +++ b/mmv1/products/compute/SecurityPolicyRule.yaml @@ -0,0 +1,289 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +name: 'SecurityPolicyRule' +base_url: 'projects/{{project}}/global/securityPolicies/{{security_policy}}' +self_link: 'projects/{{project}}/global/securityPolicies/{{security_policy}}/getRule?priority={{priority}}' +create_url: 'projects/{{project}}/global/securityPolicies/{{security_policy}}/addRule?priority={{priority}}' +update_verb: :POST +update_mask: true +update_url: 'projects/{{project}}/global/securityPolicies/{{security_policy}}/patchRule?priority={{priority}}' +delete_verb: :POST +delete_url: 'projects/{{project}}/global/securityPolicies/{{security_policy}}/removeRule?priority={{priority}}' +description: | + A rule for the SecurityPolicy. +references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Creating global security policy rules': 'https://cloud.google.com/armor/docs/configure-security-policies' + api: + 'https://cloud.google.com/compute/docs/reference/rest/v1/securityPolicies/addRule' +id_format: 'projects/{{project}}/global/securityPolicies/{{security_policy}}/priority/{{priority}}' +import_format: ['projects/{{project}}/global/securityPolicies/{{security_policy}}/priority/{{priority}}'] +async: !ruby/object:Api::OpAsync + operation: !ruby/object:Api::OpAsync::Operation + kind: 'compute#operation' + path: 'name' + base_url: 'projects/{{project}}/global/operations/{{op_id}}' + wait_ms: 1000 + result: !ruby/object:Api::OpAsync::Result + path: 'targetLink' + status: !ruby/object:Api::OpAsync::Status + path: 'status' + complete: 'DONE' + allowed: + - 'PENDING' + - 'RUNNING' + - 'DONE' + error: !ruby/object:Api::OpAsync::Error + path: 'error/errors' + message: 'message' +examples: + - !ruby/object:Provider::Terraform::Examples + name: 'security_policy_rule_basic' + primary_resource_id: 'policy_rule' + vars: + sec_policy_name: 'policyruletest' + - !ruby/object:Provider::Terraform::Examples + name: 'security_policy_rule_default_rule' + primary_resource_id: 'policy_rule' + skip_test: true + vars: + sec_policy_name: 'policyruletest' + project_id: :PROJECT_NAME + - !ruby/object:Provider::Terraform::Examples + name: 'security_policy_rule_multiple_rules' + primary_resource_id: 'policy_rule_one' + vars: + sec_policy_name: 'policywithmultiplerules' +parameters: + - !ruby/object:Api::Type::String + name: 'security_policy' + required: true + immutable: true + description: | + The name of the security policy this rule belongs to. + url_param_only: true +properties: + - !ruby/object:Api::Type::String + name: 'description' + description: | + An optional description of this resource. Provide this property when you create the resource. + - !ruby/object:Api::Type::Integer + name: 'priority' + description: | + An integer indicating the priority of a rule in the list. + The priority must be a positive value between 0 and 2147483647. + Rules are evaluated from highest to lowest priority where 0 is the highest priority and 2147483647 is the lowest priority. + required: true + immutable: true + - !ruby/object:Api::Type::NestedObject + name: 'match' + description: | + A match condition that incoming traffic is evaluated against. + If it evaluates to true, the corresponding 'action' is enforced. + properties: + - !ruby/object:Api::Type::Enum + name: 'versionedExpr' + description: | + Preconfigured versioned expression. If this field is specified, config must also be specified. + Available preconfigured expressions along with their requirements are: SRC_IPS_V1 - must specify the corresponding srcIpRange field in config. + values: + - :SRC_IPS_V1 + - !ruby/object:Api::Type::NestedObject + name: 'expr' + description: | + User defined CEVAL expression. A CEVAL expression is used to specify match criteria such as origin.ip, source.region_code and contents in the request header. + properties: + - !ruby/object:Api::Type::String + name: 'expression' + required: true + description: | + Textual representation of an expression in Common Expression Language syntax. The application context of the containing message determines which well-known feature set of CEL is supported. + # >> These fields are not yet supported, following the global security policy resource. + # - !ruby/object:Api::Type::String + # name: 'title' + # description: | + # Optional. Title for the expression, i.e. a short string describing its purpose. This can be used e.g. in UIs which allow to enter the expression. + # - !ruby/object:Api::Type::String + # name: 'description' + # description: | + # Optional. Description of the expression. This is a longer text which describes the expression, e.g. when hovered over it in a UI. + # - !ruby/object:Api::Type::String + # name: 'location' + # description: | + # Optional. String indicating the location of the expression for error reporting, e.g. a file name and a position in the file. + - !ruby/object:Api::Type::NestedObject + name: 'config' + description: | + The configuration options available when specifying versionedExpr. + This field must be specified if versionedExpr is specified and cannot be specified if versionedExpr is not specified. + properties: + - !ruby/object:Api::Type::Array + name: 'srcIpRanges' + description: | + CIDR IP address range. Maximum number of srcIpRanges allowed is 10. + item_type: Api::Type::String + - !ruby/object:Api::Type::NestedObject + name: 'preconfiguredWafConfig' + description: | + Preconfigured WAF configuration to be applied for the rule. + If the rule does not evaluate preconfigured WAF rules, i.e., if evaluatePreconfiguredWaf() is not used, this field will have no effect. + properties: + - !ruby/object:Api::Type::Array + name: 'exclusion' + api_name: 'exclusions' + description: | + An exclusion to apply during preconfigured WAF evaluation. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::Array + name: 'requestHeader' + api_name: 'requestHeadersToExclude' + description: | + Request header whose value will be excluded from inspection during preconfigured WAF evaluation. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'operator' + api_name: 'op' + required: true + description: | + You can specify an exact match or a partial match by using a field operator and a field value. + Available options: + EQUALS: The operator matches if the field value equals the specified value. + STARTS_WITH: The operator matches if the field value starts with the specified value. + ENDS_WITH: The operator matches if the field value ends with the specified value. + CONTAINS: The operator matches if the field value contains the specified value. + EQUALS_ANY: The operator matches if the field value is any value. + validation: !ruby/object:Provider::Terraform::Validation + function: validation.StringInSlice([]string{"EQUALS", "STARTS_WITH", "ENDS_WITH", "CONTAINS", "EQUALS_ANY"}, false) + - !ruby/object:Api::Type::String + name: 'value' + api_name: 'val' + description: | + A request field matching the specified value will be excluded from inspection during preconfigured WAF evaluation. + The field value must be given if the field operator is not EQUALS_ANY, and cannot be given if the field operator is EQUALS_ANY. + - !ruby/object:Api::Type::Array + name: 'requestCookie' + api_name: 'requestCookiesToExclude' + description: | + Request cookie whose value will be excluded from inspection during preconfigured WAF evaluation. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'operator' + api_name: 'op' + required: true + description: | + You can specify an exact match or a partial match by using a field operator and a field value. + Available options: + EQUALS: The operator matches if the field value equals the specified value. + STARTS_WITH: The operator matches if the field value starts with the specified value. + ENDS_WITH: The operator matches if the field value ends with the specified value. + CONTAINS: The operator matches if the field value contains the specified value. + EQUALS_ANY: The operator matches if the field value is any value. + validation: !ruby/object:Provider::Terraform::Validation + function: validation.StringInSlice([]string{"EQUALS", "STARTS_WITH", "ENDS_WITH", "CONTAINS", "EQUALS_ANY"}, false) + - !ruby/object:Api::Type::String + name: 'value' + api_name: 'val' + description: | + A request field matching the specified value will be excluded from inspection during preconfigured WAF evaluation. + The field value must be given if the field operator is not EQUALS_ANY, and cannot be given if the field operator is EQUALS_ANY. + - !ruby/object:Api::Type::Array + name: 'requestUri' + api_name: 'requestUrisToExclude' + description: | + Request URI from the request line to be excluded from inspection during preconfigured WAF evaluation. + When specifying this field, the query or fragment part should be excluded. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'operator' + api_name: 'op' + required: true + description: | + You can specify an exact match or a partial match by using a field operator and a field value. + Available options: + EQUALS: The operator matches if the field value equals the specified value. + STARTS_WITH: The operator matches if the field value starts with the specified value. + ENDS_WITH: The operator matches if the field value ends with the specified value. + CONTAINS: The operator matches if the field value contains the specified value. + EQUALS_ANY: The operator matches if the field value is any value. + validation: !ruby/object:Provider::Terraform::Validation + function: validation.StringInSlice([]string{"EQUALS", "STARTS_WITH", "ENDS_WITH", "CONTAINS", "EQUALS_ANY"}, false) + - !ruby/object:Api::Type::String + name: 'value' + api_name: 'val' + description: | + A request field matching the specified value will be excluded from inspection during preconfigured WAF evaluation. + The field value must be given if the field operator is not EQUALS_ANY, and cannot be given if the field operator is EQUALS_ANY. + - !ruby/object:Api::Type::Array + name: 'requestQueryParam' + api_name: 'requestQueryParamsToExclude' + description: | + Request query parameter whose value will be excluded from inspection during preconfigured WAF evaluation. + Note that the parameter can be in the query string or in the POST body. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'operator' + api_name: 'op' + required: true + description: | + You can specify an exact match or a partial match by using a field operator and a field value. + Available options: + EQUALS: The operator matches if the field value equals the specified value. + STARTS_WITH: The operator matches if the field value starts with the specified value. + ENDS_WITH: The operator matches if the field value ends with the specified value. + CONTAINS: The operator matches if the field value contains the specified value. + EQUALS_ANY: The operator matches if the field value is any value. + validation: !ruby/object:Provider::Terraform::Validation + function: validation.StringInSlice([]string{"EQUALS", "STARTS_WITH", "ENDS_WITH", "CONTAINS", "EQUALS_ANY"}, false) + - !ruby/object:Api::Type::String + name: 'value' + api_name: 'val' + description: | + A request field matching the specified value will be excluded from inspection during preconfigured WAF evaluation. + The field value must be given if the field operator is not EQUALS_ANY, and cannot be given if the field operator is EQUALS_ANY. + - !ruby/object:Api::Type::String + name: 'targetRuleSet' + required: true + description: | + Target WAF rule set to apply the preconfigured WAF exclusion. + - !ruby/object:Api::Type::Array + name: 'targetRuleIds' + description: | + A list of target rule IDs under the WAF rule set to apply the preconfigured WAF exclusion. + If omitted, it refers to all the rule IDs under the WAF rule set. + item_type: Api::Type::String + - !ruby/object:Api::Type::String + name: 'action' + description: | + The Action to perform when the rule is matched. The following are the valid actions: + + * allow: allow access to target. + + * deny(STATUS): deny access to target, returns the HTTP response code specified. Valid values for STATUS are 403, 404, and 502. + + * rate_based_ban: limit client traffic to the configured threshold and ban the client if the traffic exceeds the threshold. Configure parameters for this action in RateLimitOptions. Requires rateLimitOptions to be set. + + * redirect: redirect to a different target. This can either be an internal reCAPTCHA redirect, or an external URL-based redirect via a 302 response. Parameters for this action can be configured via redirectOptions. This action is only supported in Global Security Policies of type CLOUD_ARMOR. + + * throttle: limit client traffic to the configured threshold. Configure parameters for this action in rateLimitOptions. Requires rateLimitOptions to be set for this. + required: true + - !ruby/object:Api::Type::Boolean + name: 'preview' + description: | + If set to true, the specified action is not enforced. diff --git a/mmv1/templates/terraform/examples/security_policy_rule_basic.tf.erb b/mmv1/templates/terraform/examples/security_policy_rule_basic.tf.erb new file mode 100644 index 000000000000..4206bd4ec389 --- /dev/null +++ b/mmv1/templates/terraform/examples/security_policy_rule_basic.tf.erb @@ -0,0 +1,19 @@ +resource "google_compute_security_policy" "default" { + name = "<%= ctx[:vars]['sec_policy_name'] %>" + description = "basic global security policy" + type = "CLOUD_ARMOR" +} + +resource "google_compute_security_policy_rule" "<%= ctx[:primary_resource_id] %>" { + security_policy = google_compute_security_policy.default.name + description = "new rule" + priority = 100 + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.10.0.0/16"] + } + } + action = "allow" + preview = true +} diff --git a/mmv1/templates/terraform/examples/security_policy_rule_default_rule.tf.erb b/mmv1/templates/terraform/examples/security_policy_rule_default_rule.tf.erb new file mode 100644 index 000000000000..b7b269030bf7 --- /dev/null +++ b/mmv1/templates/terraform/examples/security_policy_rule_default_rule.tf.erb @@ -0,0 +1,37 @@ +resource "google_compute_security_policy" "default" { + name = "<%= ctx[:vars]['sec_policy_name'] %>" + description = "basic global security policy" + type = "CLOUD_ARMOR" +} + +# A default rule is generated when creating the security_policy resource, import is needed to patch it +# import { +# id = "projects/<%= ctx[:test_env_vars]['project_id'] %>/global/securityPolicies/<%= ctx[:vars]['sec_policy_name'] %>/priority/2147483647" +# to = google_compute_security_policy_rule.default_rule +# } +resource "google_compute_security_policy_rule" "default_rule" { + security_policy = google_compute_security_policy.default.name + description = "default rule" + action = "allow" + priority = "2147483647" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["*"] + } + } +} + +resource "google_compute_security_policy_rule" "<%= ctx[:primary_resource_id] %>" { + security_policy = google_compute_security_policy.default.name + description = "new rule" + priority = 100 + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.10.0.0/16"] + } + } + action = "allow" + preview = true +} diff --git a/mmv1/templates/terraform/examples/security_policy_rule_multiple_rules.tf.erb b/mmv1/templates/terraform/examples/security_policy_rule_multiple_rules.tf.erb new file mode 100644 index 000000000000..c2f42cd2f162 --- /dev/null +++ b/mmv1/templates/terraform/examples/security_policy_rule_multiple_rules.tf.erb @@ -0,0 +1,33 @@ +resource "google_compute_security_policy" "default" { + name = "<%= ctx[:vars]['sec_policy_name'] %>" + description = "basic global security policy" + type = "CLOUD_ARMOR" +} + +resource "google_compute_security_policy_rule" "<%= ctx[:primary_resource_id] %>" { + security_policy = google_compute_security_policy.default.name + description = "new rule one" + priority = 100 + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.10.0.0/16"] + } + } + action = "allow" + preview = true +} + +resource "google_compute_security_policy_rule" "policy_rule_two" { + security_policy = google_compute_security_policy.default.name + description = "new rule two" + priority = 101 + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["192.168.0.0/16", "10.0.0.0/8"] + } + } + action = "allow" + preview = true +} diff --git a/mmv1/third_party/terraform/services/compute/resource_compute_security_policy_rule_test.go.erb b/mmv1/third_party/terraform/services/compute/resource_compute_security_policy_rule_test.go.erb new file mode 100644 index 000000000000..893aa09cd498 --- /dev/null +++ b/mmv1/third_party/terraform/services/compute/resource_compute_security_policy_rule_test.go.erb @@ -0,0 +1,454 @@ +<% autogen_exception -%> +package compute_test + +import ( + "regexp" + "testing" + "github.com/hashicorp/terraform-provider-google/google/acctest" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccComputeSecurityPolicyRule_basicUpdate(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckComputeSecurityPolicyRuleDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeSecurityPolicyRule_preBasicUpdate(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSecurityPolicyRule_postBasicUpdate(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccComputeSecurityPolicyRule_withRuleExpr(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckComputeSecurityPolicyRuleDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeSecurityPolicyRule_withRuleExpr(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccComputeSecurityPolicyRule_extendedUpdate(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckComputeSecurityPolicyRuleDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeSecurityPolicyRule_extPreUpdate(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSecurityPolicyRule_extPosUpdateSamePriority(context), + ExpectError: regexp.MustCompile("Cannot have rules with the same priorities."), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSecurityPolicyRule_extPosUpdate(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func TestAccComputeSecurityPolicyRule_withPreconfiguredWafConfig(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckComputeSecurityPolicyRuleDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_create(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_update(context), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_clear(context), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckNoResourceAttr("google_compute_security_policy_rule.policy_rule", "preconfigured_waf_config.0"), + ), + }, + { + ResourceName: "google_compute_security_policy_rule.policy_rule", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccComputeSecurityPolicyRule_preBasicUpdate(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" + type = "CLOUD_ARMOR" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "basic rule pre update" + action = "allow" + priority = 100 + preview = false + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["192.168.0.0/16", "10.0.0.0/8"] + } + } +} +`, context) +} + +func testAccComputeSecurityPolicyRule_postBasicUpdate(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" + type = "CLOUD_ARMOR" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "basic rule post update" + action = "deny(403)" + priority = 100 + preview = true + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["172.16.0.0/12"] + } + } +} +`, context) +} + +func testAccComputeSecurityPolicyRule_withRuleExpr(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "basic description" + action = "allow" + priority = "2000" + match { + expr { + expression = "evaluatePreconfiguredExpr('xss-canary')" + } + } + preview = true +} +`, context) +} + +func testAccComputeSecurityPolicyRule_extPreUpdate(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "basic description" + action = "allow" + priority = "2000" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.0.0.0/24"] + } + } + preview = true +} +`, context) +} + +func testAccComputeSecurityPolicyRule_extPosUpdateSamePriority(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" +} + +//add this +resource "google_compute_security_policy_rule" "policy_rule2" { + security_policy = google_compute_security_policy.default.name + description = "basic description" + action = "deny(403)" + priority = "2000" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.0.0.0/24"] + } + } + preview = true +} + +//keep this +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "basic description" + action = "allow" + priority = "2000" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.0.0.0/24"] + } + } + preview = true +} +`, context) +} + +func testAccComputeSecurityPolicyRule_extPosUpdate(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "default" { + name = "tf-test%{random_suffix}" + description = "basic global security policy" +} + +//add this +resource "google_compute_security_policy_rule" "policy_rule2" { + security_policy = google_compute_security_policy.default.name + description = "basic description" + action = "deny(403)" + priority = "1000" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.0.0.0/24"] + } + } + preview = true +} + +//update this +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.default.name + description = "updated description" + action = "allow" + priority = "2000" + match { + versioned_expr = "SRC_IPS_V1" + config { + src_ip_ranges = ["10.0.0.0/24"] + } + } + preview = true +} +`, context) +} + +func testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_create(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "policy" { + name = "tf-test%{random_suffix}" + description = "Global security policy - create" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.policy.name + description = "Rule with preconfiguredWafConfig - create" + action = "deny" + priority = "1000" + match { + expr { + expression = "evaluatePreconfiguredWaf('sqli-stable')" + } + } + preconfigured_waf_config { + exclusion { + request_cookie { + operator = "EQUALS_ANY" + } + request_header { + operator = "EQUALS" + value = "Referer" + } + request_uri { + operator = "STARTS_WITH" + value = "/admin" + } + request_query_param { + operator = "EQUALS" + value = "password" + } + request_query_param { + operator = "STARTS_WITH" + value = "freeform" + } + target_rule_set = "sqli-stable" + } + exclusion { + request_query_param { + operator = "CONTAINS" + value = "password" + } + request_query_param { + operator = "STARTS_WITH" + value = "freeform" + } + target_rule_set = "xss-stable" + } + } + preview = false +} +`, context) +} + +func testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "policy" { + name = "tf-test%{random_suffix}" + description = "Global security policy - update" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.policy.name + description = "Rule with preconfiguredWafConfig - update" + action = "deny" + priority = "1000" + match { + expr { + expression = "evaluatePreconfiguredWaf('rce-stable') || evaluatePreconfiguredWaf('xss-stable')" + } + } + preconfigured_waf_config { + exclusion { + request_uri { + operator = "STARTS_WITH" + value = "/admin" + } + target_rule_set = "rce-stable" + } + exclusion { + request_query_param { + operator = "CONTAINS" + value = "password" + } + request_query_param { + operator = "STARTS_WITH" + value = "freeform" + } + request_query_param { + operator = "EQUALS" + value = "description" + } + request_cookie { + operator = "CONTAINS" + value = "TokenExpired" + } + target_rule_set = "xss-stable" + target_rule_ids = [ + "owasp-crs-v030001-id941330-xss", + "owasp-crs-v030001-id941340-xss", + ] + } + } + preview = false +} +`, context) +} + +func testAccComputeSecurityPolicyRule_withPreconfiguredWafConfig_clear(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_compute_security_policy" "policy" { + name = "tf-test%{random_suffix}" + description = "Global security policy - clear" +} + +resource "google_compute_security_policy_rule" "policy_rule" { + security_policy = google_compute_security_policy.policy.name + description = "Rule with preconfiguredWafConfig - clear" + action = "deny" + priority = "1000" + match { + expr { + expression = "evaluatePreconfiguredWaf('rce-stable') || evaluatePreconfiguredWaf('xss-stable')" + } + } + preview = false +} +`, context) +} From 22e1f44fcfaa7605b0bb5321ad6907ddbf3e2d7b Mon Sep 17 00:00:00 2001 From: varsharmavs Date: Wed, 24 Apr 2024 06:21:02 +0000 Subject: [PATCH 04/33] terraform Support for Managing Entitlements. (#10334) Co-authored-by: Nick Elliot --- .../privilegedaccessmanager/Entitlement.yaml | 264 ++++++++++++++++++ .../privilegedaccessmanager/product.yaml | 40 +++ ...ivileged_access_manager_entitlement.go.erb | 23 ++ ...ed_access_manager_entitlement_basic.tf.erb | 39 +++ ...ivileged_access_manager_entitlement.go.erb | 13 + .../components/inputs/services_beta.kt | 5 + .../components/inputs/services_ga.kt | 5 + ...ged_access_manager_entitlement_test.go.erb | 137 +++++++++ 8 files changed, 526 insertions(+) create mode 100644 mmv1/products/privilegedaccessmanager/Entitlement.yaml create mode 100644 mmv1/products/privilegedaccessmanager/product.yaml create mode 100644 mmv1/templates/terraform/constants/privileged_access_manager_entitlement.go.erb create mode 100644 mmv1/templates/terraform/examples/privileged_access_manager_entitlement_basic.tf.erb create mode 100644 mmv1/templates/terraform/pre_update/privileged_access_manager_entitlement.go.erb create mode 100644 mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb diff --git a/mmv1/products/privilegedaccessmanager/Entitlement.yaml b/mmv1/products/privilegedaccessmanager/Entitlement.yaml new file mode 100644 index 000000000000..06991b2c2426 --- /dev/null +++ b/mmv1/products/privilegedaccessmanager/Entitlement.yaml @@ -0,0 +1,264 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +base_url: '{{parent}}/locations/{{location}}/entitlements' +create_url: '{{parent}}/locations/{{location}}/entitlements?entitlementId={{entitlement_id}}' +delete_url: '{{parent}}/locations/{{location}}/entitlements/{{entitlement_id}}?force=true' +self_link: '{{parent}}/locations/{{location}}/entitlements/{{entitlement_id}}' +id_format: '{{parent}}/locations/{{location}}/entitlements/{{entitlement_id}}' +import_format: ['{{%parent}}/locations/{{location}}/entitlements/{{entitlement_id}}'] +name: Entitlement +description: | + An Entitlement defines the eligibility of a set of users to obtain a predefined access for some time possibly after going through an approval workflow. +update_verb: :PATCH +update_mask: true +min_version: beta +autogen_async: true +examples: + - !ruby/object:Provider::Terraform::Examples + name: "privileged_access_manager_entitlement_basic" + min_version: beta + primary_resource_id: "tfentitlement" + vars: + entitlement_id: "example-entitlement" + test_env_vars: + project: :PROJECT_NAME +properties: + - !ruby/object:Api::Type::String + name: name + description: | + Output Only. The entitlement's name follows a hierarchical structure, comprising the organization, folder, or project, alongside the region and a unique entitlement ID. + Formats: organizations/{organization-number}/locations/{region}/entitlements/{entitlement-id}, folders/{folder-number}/locations/{region}/entitlements/{entitlement-id}, and projects/{project-id|project-number}/locations/{region}/entitlements/{entitlement-id}. + output: true + - !ruby/object:Api::Type::String + name: createTime + description: | + Output only. Create time stamp. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z" + output: true + - !ruby/object:Api::Type::String + name: updateTime + description: | + Output only. Update time stamp. A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z". + output: true + - !ruby/object:Api::Type::Array + name: eligibleUsers + description: | + Who can create Grants using Entitlement. This list should contain at most one entry + required: true + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::Array + name: principals + is_set: true + required: true + description: | + Users who are being allowed for the operation. Each entry should be a valid v1 IAM Principal Identifier. Format for these is documented at "https://cloud.google.com/iam/docs/principal-identifiers#v1" + item_type: Api::Type::String + item_validation: !ruby/object:Provider::Terraform::Validation + function: 'validateDeletedPrincipals' + - !ruby/object:Api::Type::NestedObject + name: approvalWorkflow + immutable: true + description: | + The approvals needed before access will be granted to a requester. + No approvals will be needed if this field is null. Different types of approval workflows that can be used to gate privileged access granting. + properties: + - !ruby/object:Api::Type::NestedObject + name: manualApprovals + required: true + description: | + A manual approval workflow where users who are designated as approvers need to call the ApproveGrant/DenyGrant APIs for an Grant. + The workflow can consist of multiple serial steps where each step defines who can act as Approver in that step and how many of those users should approve before the workflow moves to the next step. + This can be used to create approval workflows such as + * Require an approval from any user in a group G. + * Require an approval from any k number of users from a Group G. + * Require an approval from any user in a group G and then from a user U. etc. + A single user might be part of `approvers` ACL for multiple steps in this workflow but they can only approve once and that approval will only be considered to satisfy the approval step at which it was granted. + properties: + - !ruby/object:Api::Type::Boolean + name: requireApproverJustification + description: | + Optional. Do the approvers need to provide a justification for their actions? + - !ruby/object:Api::Type::Array + name: steps + required: true + description: | + List of approval steps in this workflow. These steps would be followed in the specified order sequentially. 1 step is supported for now. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::Array + name: approvers + required: true + min_size: 1 + max_size: 1 + description: | + The potential set of approvers in this step. This list should contain at only one entry. + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::Array + name: principals + required: true + min_size: 1 + is_set: true + item_type: Api::Type::String + item_validation: !ruby/object:Provider::Terraform::Validation + function: 'validateDeletedPrincipals' + description: | + Users who are being allowed for the operation. Each entry should be a valid v1 IAM Principal Identifier. Format for these is documented at: https://cloud.google.com/iam/docs/principal-identifiers#v1 + - !ruby/object:Api::Type::Integer + name: approvalsNeeded + description: | + How many users from the above list need to approve. + If there are not enough distinct users in the list above then the workflow + will indefinitely block. Should always be greater than 0. Currently 1 is the only + supported value. + - !ruby/object:Api::Type::Array + name: approverEmailRecipients + item_type: Api::Type::String + is_set: true + description: | + Optional. Additional email addresses to be notified when a grant is pending approval. + - !ruby/object:Api::Type::NestedObject + name: privilegedAccess + description: | + Privileged access that this service can be used to gate. + required: true + properties: + - !ruby/object:Api::Type::NestedObject + name: gcpIamAccess + description: | + GcpIamAccess represents IAM based access control on a GCP resource. Refer to https://cloud.google.com/iam/docs to understand more about IAM. + required: true + properties: + - !ruby/object:Api::Type::String + name: resourceType + description: | + The type of this resource. + required: true + - !ruby/object:Api::Type::String + name: resource + description: | + Name of the resource. + required: true + - !ruby/object:Api::Type::Array + name: roleBindings + description: | + Role bindings to be created on successful grant. + required: true + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: role + description: | + IAM role to be granted. https://cloud.google.com/iam/docs/roles-overview. + required: true + - !ruby/object:Api::Type::String + name: conditionExpression + description: | + The expression field of the IAM condition to be associated with the role. If specified, a user with an active grant for this entitlement would be able to access the resource only if this condition evaluates to true for their request. + https://cloud.google.com/iam/docs/conditions-overview#attributes. + - !ruby/object:Api::Type::String + name: maxRequestDuration + description: | + The maximum amount of time for which access would be granted for a request. + A requester can choose to ask for access for less than this duration but never more. + Format: calculate the time in seconds and concatenate it with 's' i.e. 2 hours = "7200s", 45 minutes = "2700s" + required: true + - !ruby/object:Api::Type::String + name: state + description: Output only. The current state of the Entitlement. + output: true + - !ruby/object:Api::Type::Fingerprint + name: etag + description: | + For Resource freshness validation (https://google.aip.dev/154) + output: true + - !ruby/object:Api::Type::NestedObject + name: requesterJustificationConfig + description: | + Defines the ways in which a requester should provide the justification while requesting for access. + required: true + allow_empty_object: true + send_empty_value: true + properties: + - !ruby/object:Api::Type::NestedObject + name: notMandatory + description: | + The justification is not mandatory but can be provided in any of the supported formats. + properties: [] + allow_empty_object: true + send_empty_value: true + conflicts: + - unstructured + - !ruby/object:Api::Type::NestedObject + name: unstructured + description: | + The requester has to provide a justification in the form of free flowing text. + properties: [] + allow_empty_object: true + send_empty_value: true + conflicts: + - not_mandatory + - !ruby/object:Api::Type::NestedObject + name: additionalNotificationTargets + allow_empty_object: true + send_empty_value: true + description: | + AdditionalNotificationTargets includes email addresses to be notified. + properties: + - !ruby/object:Api::Type::Array + name: adminEmailRecipients + is_set: true + description: | + Optional. Additional email addresses to be notified when a principal(requester) is granted access. + item_type: Api::Type::String + allow_empty_object: true + - !ruby/object:Api::Type::Array + name: requesterEmailRecipients + item_type: Api::Type::String + is_set: true + description: | + Optional. Additional email address to be notified about an eligible entitlement. + allow_empty_object: true +parameters: + - !ruby/object:Api::Type::String + name: location + description: | + The region of the Entitlement resource. + url_param_only: true + required: true + immutable: true + - !ruby/object:Api::Type::String + name: entitlementId + description: | + The ID to use for this Entitlement. This will become the last part of the resource name. + This value should be 4-63 characters, and valid characters are "[a-z]", "[0-9]", and "-". The first character should be from [a-z]. + This value should be unique among all other Entitlements under the specified `parent`. + url_param_only: true + required: true + immutable: true + validation: !ruby/object:Provider::Terraform::Validation + function: validateEntitlementId + - !ruby/object:Api::Type::String + name: parent + immutable: true + required: true + url_param_only: true + description: | + Format: project/{project_id} or organization/{organization_number} or folder/{folder_number} +custom_code: !ruby/object:Provider::Terraform::CustomCode + pre_update: templates/terraform/pre_update/privileged_access_manager_entitlement.go.erb + constants: templates/terraform/constants/privileged_access_manager_entitlement.go.erb diff --git a/mmv1/products/privilegedaccessmanager/product.yaml b/mmv1/products/privilegedaccessmanager/product.yaml new file mode 100644 index 000000000000..557e754ca62a --- /dev/null +++ b/mmv1/products/privilegedaccessmanager/product.yaml @@ -0,0 +1,40 @@ +# Copyright 2020 google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Product +versions: + - !ruby/object:Api::Product::Version + base_url: https://privilegedaccessmanager.googleapis.com/v1beta/ + name: beta +name: PrivilegedAccessManager +display_name: Privileged Access Manager +scopes: + - https://www.googleapis.com/auth/cloud-platform +async: !ruby/object:Api::OpAsync + operation: !ruby/object:Api::OpAsync::Operation + path: name + base_url: "{{op_id}}" + wait_ms: 1000 + timeouts: + result: !ruby/object:Api::OpAsync::Result + path: response + resource_inside_response: true + status: !ruby/object:Api::OpAsync::Status + path: done + complete: true + allowed: + - true + - false + error: !ruby/object:Api::OpAsync::Error + path: error + message: message diff --git a/mmv1/templates/terraform/constants/privileged_access_manager_entitlement.go.erb b/mmv1/templates/terraform/constants/privileged_access_manager_entitlement.go.erb new file mode 100644 index 000000000000..43044924a06e --- /dev/null +++ b/mmv1/templates/terraform/constants/privileged_access_manager_entitlement.go.erb @@ -0,0 +1,23 @@ +const deletedRegexp = `^deleted:` + +func validateDeletedPrincipals(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + if regexp.MustCompile(deletedRegexp).MatchString(value) { + errors = append(errors, fmt.Errorf( + "Terraform does not support IAM policies for deleted principals: %s", k)) + } + + return +} + +const entitlementIdRegexp = `^[a-z][a-z0-9-]{3,62}$` + +func validateEntitlementId(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + if !regexp.MustCompile(entitlementIdRegexp).MatchString(value) { + errors = append(errors, fmt.Errorf( + "Entitlement Id should be 4-63 characters, and valid characters are '[a-z]', '[0-9]', and '-'. The first character should be from [a-z]. : %s", k)) + } + + return +} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/privileged_access_manager_entitlement_basic.tf.erb b/mmv1/templates/terraform/examples/privileged_access_manager_entitlement_basic.tf.erb new file mode 100644 index 000000000000..abed6ce9d28b --- /dev/null +++ b/mmv1/templates/terraform/examples/privileged_access_manager_entitlement_basic.tf.erb @@ -0,0 +1,39 @@ +resource "google_privileged_access_manager_entitlement" "<%= ctx[:primary_resource_id] %>" { + provider = google-beta + entitlement_id = "<%= ctx[:vars]['entitlement_id'] %>" + location = "global" + max_request_duration = "43200s" + parent = "projects/<%= ctx[:test_env_vars]['project'] %>" + requester_justification_config { + unstructured{} + } + eligible_users { + principals = ["group:test@google.com"] + } + privileged_access{ + gcp_iam_access{ + role_bindings{ + role = "roles/storage.admin" + condition_expression = "request.time < timestamp(\"2024-04-23T18:30:00.000Z\")" + } + resource = "//cloudresourcemanager.googleapis.com/projects/<%= ctx[:test_env_vars]['project'] %>" + resource_type = "cloudresourcemanager.googleapis.com/Project" + } + } + additional_notification_targets { + admin_email_recipients = ["user@example.com"] + requester_email_recipients = ["user@example.com"] + } + approval_workflow { + manual_approvals { + require_approver_justification = true + steps { + approvals_needed = 1 + approver_email_recipients = ["user@example.com"] + approvers { + principals = ["group:test@google.com"] + } + } + } + } +} diff --git a/mmv1/templates/terraform/pre_update/privileged_access_manager_entitlement.go.erb b/mmv1/templates/terraform/pre_update/privileged_access_manager_entitlement.go.erb new file mode 100644 index 000000000000..3855742cd66c --- /dev/null +++ b/mmv1/templates/terraform/pre_update/privileged_access_manager_entitlement.go.erb @@ -0,0 +1,13 @@ + approvalWorkflowProp, err := expandPrivilegedAccessManagerEntitlementApprovalWorkflow(d.Get("approval_workflow"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("approval_workflow"); !tpgresource.IsEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, approvalWorkflowProp)) { + obj["approvalWorkflow"] = approvalWorkflowProp + } + if d.HasChange("approval_workflow") { + updateMask = append(updateMask, "approvalWorkflow") + } + url, err = transport_tpg.AddQueryParams(url, map[string]string{"updateMask": strings.Join(updateMask, ",")}) + if err != nil { + return err + } diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt index e75b67fab05b..1ca3c967249c 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt @@ -561,6 +561,11 @@ var ServicesListBeta = mapOf( "displayName" to "Privateca", "path" to "./google-beta/services/privateca" ), + "privilegedaccessmanager" to mapOf( + "name" to "privilegedaccessmanager", + "displayName" to "Privilegedaccessmanager", + "path" to "./google-beta/services/privilegedaccessmanager" + ), "publicca" to mapOf( "name" to "publicca", "displayName" to "Publicca", diff --git a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt index f7cc140075d5..64b7c62fe060 100644 --- a/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt +++ b/mmv1/third_party/terraform/.teamcity/components/inputs/services_ga.kt @@ -556,6 +556,11 @@ var ServicesListGa = mapOf( "displayName" to "Privateca", "path" to "./google/services/privateca" ), + "privilegedaccessmanager" to mapOf( + "name" to "privilegedaccessmanager", + "displayName" to "Privilegedaccessmanager", + "path" to "./google/services/privilegedaccessmanager" + ), "publicca" to mapOf( "name" to "publicca", "displayName" to "Publicca", diff --git a/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb b/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb new file mode 100644 index 000000000000..466924a6445f --- /dev/null +++ b/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb @@ -0,0 +1,137 @@ +<% autogen_exception -%> +package privilegedaccessmanager_test +<% unless version == 'ga' -%> + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + + "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/envvar" +) + +func TestAccPrivilegedAccessManagerEntitlement_privilegedAccessManagerEntitlementProjectExample_update(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + "project_name": envvar.GetTestProjectFromEnv(), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckPrivilegedAccessManagerEntitlementDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccPrivilegedAccessManagerEntitlement_privilegedAccessManagerEntitlementBasicExample_basic(context), + }, + { + ResourceName: "google_privileged_access_manager_entitlement.tfentitlement", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "entitlement_id", "parent"}, + }, + { + Config: testAccPrivilegedAccessManagerEntitlement_privilegedAccessManagerEntitlementBasicExample_update(context), + }, + { + ResourceName: "google_privileged_access_manager_entitlement.tfentitlement", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "entitlement_id", "parent"}, + }, + }, + }) +} + +func testAccPrivilegedAccessManagerEntitlement_privilegedAccessManagerEntitlementBasicExample_basic(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_privileged_access_manager_entitlement" "tfentitlement" { + provider = google-beta + entitlement_id = "tf-test-example-entitlement%{random_suffix}" + location = "global" + max_request_duration = "43200s" + parent = "projects/%{project_name}" + requester_justification_config { + unstructured{} + } + eligible_users { + principals = ["group:test@google.com"] + } + privileged_access{ + gcp_iam_access{ + role_bindings{ + role = "roles/storage.admin" + condition_expression = "request.time < timestamp("2024-04-23T18:30:00.000Z")" + } + resource = "//cloudresourcemanager.googleapis.com/projects/%{project_name}" + resource_type = "cloudresourcemanager.googleapis.com/Project" + } + } + additional_notification_targets { + admin_email_recipients = ["user@example.com"] + requester_email_recipients = ["user@example.com"] + } + approval_workflow { + manual_approvals { + require_approver_justification = true + steps { + approvals_needed = 1 + approver_email_recipients = ["user@example.com"] + approvers { + principals = ["group:test@google.com"] + } + } + } + } +} +`, context) +} + +func testAccPrivilegedAccessManagerEntitlement_privilegedAccessManagerEntitlementBasicExample_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_privileged_access_manager_entitlement" "tfentitlement" { + provider = google-beta + entitlement_id = "tf-test-example-entitlement%{random_suffix}" + location = "global" + max_request_duration = "4300s" + parent = "projects/%{project_name}" + requester_justification_config { + not_mandatory{} + } + eligible_users { + principals = ["group:test@google.com"] + } + privileged_access{ + gcp_iam_access{ + role_bindings{ + role = "roles/storage.admin" + condition_expression = "request.time < timestamp("2024-04-23T18:30:00.000Z")" + } + resource = "//cloudresourcemanager.googleapis.com/projects/%{project_name}" + resource_type = "cloudresourcemanager.googleapis.com/Project" + } + } + additional_notification_targets { + admin_email_recipients = ["user1@example.com"] + requester_email_recipients = ["user2@example.com"] + } + approval_workflow { + manual_approvals { + require_approver_justification = false + steps { + approvals_needed = 1 + approver_email_recipients = ["user3@example.com"] + approvers { + principals = ["group:test@google.com"] + } + } + } + } +} +`, context) +} + +<% end -%> From 6ea6f333801d112cb53ba29b8341aff403b25c2c Mon Sep 17 00:00:00 2001 From: Obada Alabbadi <76101898+obada-ab@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:20:45 +0200 Subject: [PATCH 05/33] Promote `external_dataset_reference` in `bigquery_dataset` to GA (#10516) --- mmv1/products/bigquery/Dataset.yaml | 7 ++----- .../bigquery_dataset_external_reference_aws.tf.erb | 5 ++--- ...=> bigquery_dataset_external_reference_aws_test.tf.erb} | 5 ++--- 3 files changed, 6 insertions(+), 11 deletions(-) rename mmv1/templates/terraform/examples/{bigquery_dataset_external_reference_aws_docs.tf.erb => bigquery_dataset_external_reference_aws_test.tf.erb} (59%) diff --git a/mmv1/products/bigquery/Dataset.yaml b/mmv1/products/bigquery/Dataset.yaml index c865bc242b68..25e64c26e4ec 100644 --- a/mmv1/products/bigquery/Dataset.yaml +++ b/mmv1/products/bigquery/Dataset.yaml @@ -79,16 +79,14 @@ examples: dataset_id: 'example_dataset' account_name: 'bqowner' - !ruby/object:Provider::Terraform::Examples - name: 'bigquery_dataset_external_reference_aws' + name: 'bigquery_dataset_external_reference_aws_test' primary_resource_id: 'dataset' - min_version: beta skip_docs: true vars: dataset_id: 'example_dataset' - !ruby/object:Provider::Terraform::Examples - name: 'bigquery_dataset_external_reference_aws_docs' + name: 'bigquery_dataset_external_reference_aws' primary_resource_id: 'dataset' - min_version: beta skip_test: true vars: dataset_id: 'example_dataset' @@ -309,7 +307,6 @@ properties: name: 'externalDatasetReference' description: | Information about the external metadata storage where the dataset is defined. - min_version: beta immutable: true properties: - !ruby/object:Api::Type::String diff --git a/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws.tf.erb b/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws.tf.erb index 9018a26c6a78..b407150989e5 100644 --- a/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws.tf.erb +++ b/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws.tf.erb @@ -1,12 +1,11 @@ resource "google_bigquery_dataset" "<%= ctx[:primary_resource_id] %>" { - provider = google-beta dataset_id = "<%= ctx[:vars]['dataset_id'] %>" friendly_name = "test" description = "This is a test description" location = "aws-us-east-1" external_dataset_reference { - external_source = "aws-glue://arn:aws:glue:us-east-1:772042918353:database/db_other_formats_external" - connection = "projects/bigquerytestdefault/locations/aws-us-east-1/connections/external_test-connection" + external_source = "aws-glue://arn:aws:glue:us-east-1:999999999999:database/database" + connection = "projects/project/locations/aws-us-east-1/connections/connection" } } diff --git a/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_docs.tf.erb b/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_test.tf.erb similarity index 59% rename from mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_docs.tf.erb rename to mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_test.tf.erb index a509fb47c6d1..76af83b522f0 100644 --- a/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_docs.tf.erb +++ b/mmv1/templates/terraform/examples/bigquery_dataset_external_reference_aws_test.tf.erb @@ -1,12 +1,11 @@ resource "google_bigquery_dataset" "<%= ctx[:primary_resource_id] %>" { - provider = google-beta dataset_id = "<%= ctx[:vars]['dataset_id'] %>" friendly_name = "test" description = "This is a test description" location = "aws-us-east-1" external_dataset_reference { - external_source = "aws-glue://arn:aws:glue:us-east-1:999999999999:database/database" - connection = "projects/project/locations/aws-us-east-1/connections/connection" + external_source = "aws-glue://arn:aws:glue:us-east-1:772042918353:database/db_other_formats_external" + connection = "projects/bigquerytestdefault/locations/aws-us-east-1/connections/external_test-connection" } } From 41c94af5b71deda5f8778d29095ab5b13a7a874c Mon Sep 17 00:00:00 2001 From: Rohit Jangid Date: Wed, 24 Apr 2024 19:42:26 +0530 Subject: [PATCH 06/33] Deprecate createSampleWorkflows and provisionGmek fields. Add createSampleIntegrations (#10478) --- mmv1/products/integrations/Client.yaml | 23 ++++++++++++++++++- .../integrations_auth_config_advance.tf.erb | 1 - ...integrations_auth_config_auth_token.tf.erb | 1 - .../integrations_auth_config_basic.tf.erb | 1 - ...auth_config_client_certificate_only.tf.erb | 1 - .../integrations_auth_config_jwt.tf.erb | 1 - ...th_config_oauth2_authorization_code.tf.erb | 1 - ...th_config_oauth2_client_credentials.tf.erb | 1 - ...integrations_auth_config_oidc_token.tf.erb | 1 - ...rations_auth_config_service_account.tf.erb | 1 - ...s_auth_config_username_and_password.tf.erb | 1 - .../examples/integrations_client_basic.tf.erb | 1 - ...tegrations_client_deprecated_fields.tf.erb | 5 ++++ ...tf.erb => integrations_client_full.tf.erb} | 5 +--- .../pre_create/integrations_client.go.erb | 5 ++++ 15 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 mmv1/templates/terraform/examples/integrations_client_deprecated_fields.tf.erb rename mmv1/templates/terraform/examples/{integrations_client_advance.tf.erb => integrations_client_full.tf.erb} (81%) create mode 100644 mmv1/templates/terraform/pre_create/integrations_client.go.erb diff --git a/mmv1/products/integrations/Client.yaml b/mmv1/products/integrations/Client.yaml index af65456395bc..9cc18cf1794e 100644 --- a/mmv1/products/integrations/Client.yaml +++ b/mmv1/products/integrations/Client.yaml @@ -34,6 +34,7 @@ import_format: mutex: Client/{{location}} custom_code: !ruby/object:Provider::Terraform::CustomCode decoder: templates/terraform/decoders/integrations_client.go.erb + pre_create: templates/terraform/pre_create/integrations_client.go.erb parameters: - !ruby/object:Api::Type::String name: 'location' @@ -83,18 +84,34 @@ properties: the kms key is stored at the same project as customer's project and ecrypted with CMEK, otherwise, the kms key is stored in the tenant project and encrypted with GMEK. + conflicts: + - provision_gmek - !ruby/object:Api::Type::Boolean name: 'createSampleWorkflows' description: | Indicates if sample workflow should be created along with provisioning. immutable: true ignore_read: true + deprecation_message: "`create_sample_workflows` is deprecated and will be removed in a future major release. Use `create_sample_integrations` instead." + conflicts: + - create_sample_integrations + - !ruby/object:Api::Type::Boolean + name: 'createSampleIntegrations' + description: | + Indicates if sample integrations should be created along with provisioning. + immutable: true + ignore_read: true + conflicts: + - create_sample_workflows - !ruby/object:Api::Type::Boolean name: 'provisionGmek' description: | Indicates provision with GMEK or CMEK. + deprecation_message: "`provision_gmek` is deprecated and will be removed in a future major release. Client would be provisioned as gmek if `cloud_kms_config` is not given." immutable: true ignore_read: true + conflicts: + - cloud_kms_config - !ruby/object:Api::Type::String name: 'runAsServiceAccount' description: | @@ -106,8 +123,12 @@ examples: name: "integrations_client_basic" primary_resource_id: "example" - !ruby/object:Provider::Terraform::Examples - name: "integrations_client_advance" + name: "integrations_client_full" primary_resource_id: "example" skip_vcr: true vars: key_ring_name: my-keyring + - !ruby/object:Provider::Terraform::Examples + name: "integrations_client_deprecated_fields" + primary_resource_id: "example" + skip_docs: true diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_advance.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_advance.tf.erb index d0aeb4c13158..d0f9ccb50233 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_advance.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_advance.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "asia-east2" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_auth_token.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_auth_token.tf.erb index b9f2e7892548..6444f0b21d56 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_auth_token.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_auth_token.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "us-west2" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_basic.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_basic.tf.erb index 13b1a5be514a..54393f47ef6a 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_basic.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_basic.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "us-west1" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_client_certificate_only.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_client_certificate_only.tf.erb index 95707a3a4a0f..9fc84c2f466b 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_client_certificate_only.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_client_certificate_only.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "us-west3" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_jwt.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_jwt.tf.erb index 255158776eef..b1dbc3c7165e 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_jwt.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_jwt.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "us-west4" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_authorization_code.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_authorization_code.tf.erb index d51cf5563ac9..6cc6c256044d 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_authorization_code.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_authorization_code.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "asia-east1" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_client_credentials.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_client_credentials.tf.erb index 8683a5d94413..6d40bf1f5b68 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_client_credentials.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_oauth2_client_credentials.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "southamerica-east1" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_oidc_token.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_oidc_token.tf.erb index c4714366144d..9292e02ca31d 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_oidc_token.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_oidc_token.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "us-south1" - provision_gmek = true } resource "google_service_account" "service_account" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_service_account.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_service_account.tf.erb index 2408065c04a2..676014ca7cca 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_service_account.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_service_account.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "northamerica-northeast1" - provision_gmek = true } resource "google_service_account" "service_account" { diff --git a/mmv1/templates/terraform/examples/integrations_auth_config_username_and_password.tf.erb b/mmv1/templates/terraform/examples/integrations_auth_config_username_and_password.tf.erb index d7283675acc4..5168aef8cc41 100644 --- a/mmv1/templates/terraform/examples/integrations_auth_config_username_and_password.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_auth_config_username_and_password.tf.erb @@ -1,6 +1,5 @@ resource "google_integrations_client" "client" { location = "northamerica-northeast2" - provision_gmek = true } resource "google_integrations_auth_config" "<%= ctx[:primary_resource_id] %>" { diff --git a/mmv1/templates/terraform/examples/integrations_client_basic.tf.erb b/mmv1/templates/terraform/examples/integrations_client_basic.tf.erb index 77f4bf717813..1fc724bcf539 100644 --- a/mmv1/templates/terraform/examples/integrations_client_basic.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_client_basic.tf.erb @@ -1,4 +1,3 @@ resource "google_integrations_client" "<%= ctx[:primary_resource_id] %>" { location = "us-central1" - provision_gmek = true } \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/integrations_client_deprecated_fields.tf.erb b/mmv1/templates/terraform/examples/integrations_client_deprecated_fields.tf.erb new file mode 100644 index 000000000000..ccac00b1f5f5 --- /dev/null +++ b/mmv1/templates/terraform/examples/integrations_client_deprecated_fields.tf.erb @@ -0,0 +1,5 @@ +resource "google_integrations_client" "<%= ctx[:primary_resource_id] %>" { + location = "asia-south1" + provision_gmek = true + create_sample_workflows = true +} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/integrations_client_advance.tf.erb b/mmv1/templates/terraform/examples/integrations_client_full.tf.erb similarity index 81% rename from mmv1/templates/terraform/examples/integrations_client_advance.tf.erb rename to mmv1/templates/terraform/examples/integrations_client_full.tf.erb index f874971cd2fa..cf125c39d5ce 100644 --- a/mmv1/templates/terraform/examples/integrations_client_advance.tf.erb +++ b/mmv1/templates/terraform/examples/integrations_client_full.tf.erb @@ -10,12 +10,10 @@ resource "google_kms_crypto_key" "cryptokey" { name = "crypto-key-example" key_ring = google_kms_key_ring.keyring.id rotation_period = "7776000s" - depends_on = [google_kms_key_ring.keyring] } resource "google_kms_crypto_key_version" "test_key" { crypto_key = google_kms_crypto_key.cryptokey.id - depends_on = [google_kms_crypto_key.cryptokey] } resource "google_service_account" "service_account" { @@ -25,7 +23,7 @@ resource "google_service_account" "service_account" { resource "google_integrations_client" "<%= ctx[:primary_resource_id] %>" { location = "us-east1" - create_sample_workflows = true + create_sample_integrations = true run_as_service_account = google_service_account.service_account.email cloud_kms_config { kms_location = "us-east1" @@ -34,5 +32,4 @@ resource "google_integrations_client" "<%= ctx[:primary_resource_id] %>" { key_version = google_kms_crypto_key_version.test_key.id kms_project_id = data.google_project.test_project.project_id } - depends_on = [google_kms_crypto_key_version.test_key, google_service_account.service_account] } \ No newline at end of file diff --git a/mmv1/templates/terraform/pre_create/integrations_client.go.erb b/mmv1/templates/terraform/pre_create/integrations_client.go.erb new file mode 100644 index 000000000000..8e40faf8ba58 --- /dev/null +++ b/mmv1/templates/terraform/pre_create/integrations_client.go.erb @@ -0,0 +1,5 @@ +// Translate `createSampleIntegrations` to `createSampleWorkflows` +if val, ok := obj["createSampleIntegrations"]; ok { + delete(obj, "createSampleIntegrations") + obj["createSampleWorkflows"] = val +} \ No newline at end of file From ce618c8079cb4624dfd32d00cab28e5528ba552a Mon Sep 17 00:00:00 2001 From: "Stephen Lewis (Burrows)" Date: Wed, 24 Apr 2024 08:44:48 -0700 Subject: [PATCH 07/33] Update enrolled_teams.yml (#10514) --- tools/issue-labeler/labeler/enrolled_teams.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/issue-labeler/labeler/enrolled_teams.yml b/tools/issue-labeler/labeler/enrolled_teams.yml index 9d6439ea25cb..6aa16a51a36d 100755 --- a/tools/issue-labeler/labeler/enrolled_teams.yml +++ b/tools/issue-labeler/labeler/enrolled_teams.yml @@ -43,6 +43,9 @@ service/apikeys: service/appengine: resources: - google_app_engine_.* +service/apphub: + resources: + - google_apphub_.* service/artifactregistry: team: artifact-registry resources: @@ -106,6 +109,7 @@ service/cloudkms: - google_kms_.* service/cloudresourcemanager-crm: resources: + - google_active_folder - google_folder - google_folder_iam_.* - google_folders @@ -571,12 +575,14 @@ service/spanner: service/sqladmin: resources: - google_sql_database + - google_sql_databases + - google_sql_user +service/sqladmin-cp: + resources: - google_sql_database_instance - google_sql_database_instances - - google_sql_databases - google_sql_source_representation_instance - google_sql_tiers - - google_sql_user service/sqladmin-dp: resources: - google_sql_backup_run From 731a277bc275cc8dfaeec8b2f4252a6fb284d4ac Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Wed, 24 Apr 2024 13:56:15 -0700 Subject: [PATCH 08/33] fix: update resource template to remove beta and remove egress (#10524) --- .../terraform/examples/cloudrunv2_service_directvpc.tf.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmv1/templates/terraform/examples/cloudrunv2_service_directvpc.tf.erb b/mmv1/templates/terraform/examples/cloudrunv2_service_directvpc.tf.erb index 626c979b98d7..530931f74504 100644 --- a/mmv1/templates/terraform/examples/cloudrunv2_service_directvpc.tf.erb +++ b/mmv1/templates/terraform/examples/cloudrunv2_service_directvpc.tf.erb @@ -1,7 +1,7 @@ resource "google_cloud_run_v2_service" "<%= ctx[:primary_resource_id] %>" { name = "<%= ctx[:vars]['cloud_run_service_name'] %>" location = "us-central1" - launch_stage = "BETA" + launch_stage = "GA" template { containers { image = "us-docker.pkg.dev/cloudrun/container/hello" @@ -12,7 +12,6 @@ resource "google_cloud_run_v2_service" "<%= ctx[:primary_resource_id] %>" { subnetwork = "default" tags = ["tag1", "tag2", "tag3"] } - egress = "ALL_TRAFFIC" } } } From 1a466f71281b277ad1ce3bd3d57ee61298653980 Mon Sep 17 00:00:00 2001 From: Thomas Rodgers Date: Wed, 24 Apr 2024 14:55:55 -0700 Subject: [PATCH 09/33] Make storage bucket object respect custom endpoints (#10525) --- .../storage/data_source_google_storage_bucket_object.go | 5 ++++- .../storage/data_source_google_storage_bucket_objects.go | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_object.go b/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_object.go index 83b5c093b1a4..a3c3e62c7a83 100644 --- a/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_object.go +++ b/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_object.go @@ -41,7 +41,10 @@ func dataSourceGoogleStorageBucketObjectRead(d *schema.ResourceData, meta interf name = url.QueryEscape(name) } // Using REST apis because the storage go client doesn't support folders - url := fmt.Sprintf("https://www.googleapis.com/storage/v1/b/%s/o/%s", bucket, name) + url, err := tpgresource.ReplaceVars(d, config, fmt.Sprintf("{{StorageBasePath}}b/%s/o/%s", bucket, name)) + if err != nil { + return fmt.Errorf("Error formatting url for storage bucket object: %s", err) + } res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ Config: config, diff --git a/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_objects.go b/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_objects.go index dbbb76ca994a..f40f6d3ca0a4 100644 --- a/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_objects.go +++ b/mmv1/third_party/terraform/services/storage/data_source_google_storage_bucket_objects.go @@ -70,7 +70,10 @@ func datasourceGoogleStorageBucketObjectsRead(d *schema.ResourceData, meta inter for { bucket := d.Get("bucket").(string) - url := fmt.Sprintf("https://storage.googleapis.com/storage/v1/b/%s/o", bucket) + url, err := tpgresource.ReplaceVars(d, config, fmt.Sprintf("{{StorageBasePath}}b/%s/o", bucket)) + if err != nil { + return err + } if v, ok := d.GetOk("match_glob"); ok { params["matchGlob"] = v.(string) @@ -80,7 +83,7 @@ func datasourceGoogleStorageBucketObjectsRead(d *schema.ResourceData, meta inter params["prefix"] = v.(string) } - url, err := transport_tpg.AddQueryParams(url, params) + url, err = transport_tpg.AddQueryParams(url, params) if err != nil { return err } From 3b74011837ccfc52c35add146c46540527d396e4 Mon Sep 17 00:00:00 2001 From: vmiglani <142545940+vmiglani@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:08:34 -0700 Subject: [PATCH 10/33] [AlloyDB] Maintenance Windows Support (#10499) --- mmv1/products/alloydb/Cluster.yaml | 50 +++++++ .../alloydb/resource_alloydb_cluster_test.go | 128 ++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/mmv1/products/alloydb/Cluster.yaml b/mmv1/products/alloydb/Cluster.yaml index 8d8445405e28..67b4d85f02fd 100644 --- a/mmv1/products/alloydb/Cluster.yaml +++ b/mmv1/products/alloydb/Cluster.yaml @@ -497,6 +497,56 @@ properties: description: | Name of the primary cluster must be in the format 'projects/{project}/locations/{location}/clusters/{cluster_id}' + - !ruby/object:Api::Type::NestedObject + name: 'maintenanceUpdatePolicy' + description: | + MaintenanceUpdatePolicy defines the policy for system updates. + properties: + - !ruby/object:Api::Type::Array + name: 'maintenanceWindows' + description: | + Preferred windows to perform maintenance. Currently limited to 1. + item_type: !ruby/object:Api::Type::NestedObject + name: 'maintenanceWindow' + description: | + specifies a preferred day and time for maintenance. + properties: + - !ruby/object:Api::Type::Enum + name: 'day' + required: true + description: | + Preferred day of the week for maintenance, e.g. MONDAY, TUESDAY, etc. + values: + - :MONDAY + - :TUESDAY + - :WEDNESDAY + - :THURSDAY + - :FRIDAY + - :SATURDAY + - :SUNDAY + - !ruby/object:Api::Type::NestedObject + name: 'startTime' + required: true + description: | + Preferred time to start the maintenance operation on the specified day. Maintenance will start within 1 hour of this time. + properties: + - !ruby/object:Api::Type::Integer + name: hours + required: true + description: | + Hours of day in 24 hour format. Should be from 0 to 23. + - !ruby/object:Api::Type::Integer + name: minutes + description: | + Minutes of hour of day. Currently, only the value 0 is supported. + - !ruby/object:Api::Type::Integer + name: seconds + description: | + Seconds of minutes of the time. Currently, only the value 0 is supported. + - !ruby/object:Api::Type::Integer + name: nanos + description: | + Fractions of seconds in nanoseconds. Currently, only the value 0 is supported. virtual_fields: - !ruby/object:Api::Type::Enum name: 'deletion_policy' diff --git a/mmv1/third_party/terraform/services/alloydb/resource_alloydb_cluster_test.go b/mmv1/third_party/terraform/services/alloydb/resource_alloydb_cluster_test.go index bf1255bc788e..c620a8c9bcbc 100644 --- a/mmv1/third_party/terraform/services/alloydb/resource_alloydb_cluster_test.go +++ b/mmv1/third_party/terraform/services/alloydb/resource_alloydb_cluster_test.go @@ -1,6 +1,7 @@ package alloydb_test import ( + "regexp" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" @@ -1167,3 +1168,130 @@ resource "google_compute_global_address" "private_ip_alloc" { `, context) } + +// Ensures cluster creation works with correctly specified maintenance update policy. +func TestAccAlloydbCluster_withMaintenanceWindows(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckAlloydbClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccAlloydbCluster_withMaintenanceWindows(context), + }, + { + ResourceName: "google_alloydb_cluster.default", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccAlloydbCluster_withMaintenanceWindows(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_alloydb_cluster" "default" { + cluster_id = "tf-test-alloydb-cluster%{random_suffix}" + location = "us-central1" + network_config { + network = "projects/${data.google_project.project.number}/global/networks/${google_compute_network.default.name}" + } + maintenance_update_policy { + maintenance_windows { + day = "WEDNESDAY" + start_time { + hours = 12 + minutes = 0 + seconds = 0 + nanos = 0 + } + } + } +} +data "google_project" "project" {} +resource "google_compute_network" "default" { + name = "tf-test-alloydb-cluster%{random_suffix}" +} +`, context) +} + +// Ensures cluster creation throws expected errors for incorrect configurations of maintenance update policy. +func TestAccAlloydbCluster_withMaintenanceWindowsMissingFields(t *testing.T) { + t.Parallel() + acctest.SkipIfVcr(t) + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckAlloydbClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccAlloydbCluster_withMaintenanceWindowMissingStartTime(context), + ExpectError: regexp.MustCompile("Error: Insufficient start_time blocks"), + }, + { + Config: testAccAlloydbCluster_withMaintenanceWindowMissingDay(context), + ExpectError: regexp.MustCompile("Error: Missing required argument"), + }, + }, + }) +} + +func testAccAlloydbCluster_withMaintenanceWindowMissingStartTime(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_alloydb_cluster" "default" { + cluster_id = "tf-test-alloydb-cluster%{random_suffix}" + location = "us-central1" + network = "projects/${data.google_project.project.number}/global/networks/${google_compute_network.default.name}" + + maintenance_update_policy { + maintenance_windows { + day = "WEDNESDAY" + } + } +} + +resource "google_compute_network" "default" { + name = "tf-test-alloydb-cluster%{random_suffix}" +} + +data "google_project" "project" {} +`, context) +} + +func testAccAlloydbCluster_withMaintenanceWindowMissingDay(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_alloydb_cluster" "default" { + cluster_id = "tf-test-alloydb-cluster%{random_suffix}" + location = "us-central1" + network = "projects/${data.google_project.project.number}/global/networks/${google_compute_network.default.name}" + + maintenance_update_policy { + maintenance_windows { + start_time { + hours = 12 + minutes = 0 + seconds = 0 + nanos = 0 + } + } + } +} + +resource "google_compute_network" "default" { + name = "tf-test-alloydb-cluster%{random_suffix}" +} + +data "google_project" "project" {} +`, context) +} From 210578d95111726e35616730a7d6e004bb536a1b Mon Sep 17 00:00:00 2001 From: Calvin Liu Date: Thu, 25 Apr 2024 05:50:36 -0700 Subject: [PATCH 11/33] Add redis_configs parameter to Redis Cluster (#10515) --- mmv1/products/redis/Cluster.yaml | 6 + .../examples/redis_cluster_ha.tf.erb | 3 + .../redis/resource_redis_cluster_test.go.erb | 217 ++++++++++++------ 3 files changed, 152 insertions(+), 74 deletions(-) diff --git a/mmv1/products/redis/Cluster.yaml b/mmv1/products/redis/Cluster.yaml index d815eb7b9a0a..e4fdc0649c05 100644 --- a/mmv1/products/redis/Cluster.yaml +++ b/mmv1/products/redis/Cluster.yaml @@ -231,3 +231,9 @@ properties: description: | Required. Number of shards for the Redis cluster. required: true + - !ruby/object:Api::Type::KeyValuePairs + name: 'redisConfigs' + description: | + Configure Redis Cluster behavior using a subset of native Redis configuration parameters. + Please check Memorystore documentation for the list of supported parameters: + https://cloud.google.com/memorystore/docs/cluster/supported-instance-configurations diff --git a/mmv1/templates/terraform/examples/redis_cluster_ha.tf.erb b/mmv1/templates/terraform/examples/redis_cluster_ha.tf.erb index 24273a469e55..6f97550fe142 100644 --- a/mmv1/templates/terraform/examples/redis_cluster_ha.tf.erb +++ b/mmv1/templates/terraform/examples/redis_cluster_ha.tf.erb @@ -9,6 +9,9 @@ resource "google_redis_cluster" "<%= ctx[:primary_resource_id] %>" { node_type = "REDIS_SHARED_CORE_NANO" transit_encryption_mode = "TRANSIT_ENCRYPTION_MODE_DISABLED" authorization_mode = "AUTH_MODE_DISABLED" + redis_configs = { + maxmemory-policy = "volatile-ttl" + } depends_on = [ google_network_connectivity_service_connection_policy.default ] diff --git a/mmv1/third_party/terraform/services/redis/resource_redis_cluster_test.go.erb b/mmv1/third_party/terraform/services/redis/resource_redis_cluster_test.go.erb index 928eb49c857c..4397f6e5a5a7 100644 --- a/mmv1/third_party/terraform/services/redis/resource_redis_cluster_test.go.erb +++ b/mmv1/third_party/terraform/services/redis/resource_redis_cluster_test.go.erb @@ -4,6 +4,7 @@ package redis_test import ( "fmt" + "strings" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" @@ -11,31 +12,31 @@ import ( ) func TestAccRedisCluster_createClusterWithNodeType(t *testing.T) { - t.Parallel() - - name := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), - CheckDestroy: testAccCheckRedisClusterDestroyProducer(t), - Steps: []resource.TestStep{ - { - // create cluster with replica count 1 - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 1, /* shardCount = */ 3, true, /*nodeType = */ "REDIS_STANDARD_SMALL"), - }, - { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"psc_configs"}, - }, - { - // clean up the resource - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 0, /* shardCount = */ 3, false, /*nodeType = */ "REDIS_STANDARD_SMALL"), - }, - }, - }) + t.Parallel() + + name := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckRedisClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + // create cluster with replica count 1 + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 3, preventDestroy: true, nodeType: "REDIS_STANDARD_SMALL"}), + }, + { + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"psc_configs"}, + }, + { + // clean up the resource + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 0, shardCount: 3, preventDestroy: false, nodeType: "REDIS_STANDARD_SMALL"}), + }, + }, + }) } // Validate that replica count is updated for the cluster @@ -51,41 +52,41 @@ func TestAccRedisCluster_updateReplicaCount(t *testing.T) { Steps: []resource.TestStep{ { // create cluster with replica count 1 - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 1, /* shardCount = */ 3, true, /* nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 3, preventDestroy: true}), }, { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, ImportStateVerifyIgnore: []string{"psc_configs"}, }, { // update replica count to 2 - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 2, /* shardCount = */ 3, true, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 2, shardCount: 3, preventDestroy: true}), }, { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, ImportStateVerifyIgnore: []string{"psc_configs"}, }, { // clean up the resource - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 2, /* shardCount = */ 3, false, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 3, preventDestroy: false}), }, { // update replica count to 0 - Config: createOrUpdateRedisCluster(name, /* replicaCount = */ 0, /* shardCount = */ 3, true, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 0, shardCount: 3, preventDestroy: true}), }, { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, ImportStateVerifyIgnore: []string{"psc_configs"}, }, { // clean up the resource - Config: createOrUpdateRedisCluster(name /* replicaCount = */, 0 /* shardCount = */, 3, false, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 0, shardCount: 3, preventDestroy: false}), }, }, }) @@ -104,83 +105,151 @@ func TestAccRedisCluster_updateShardCount(t *testing.T) { Steps: []resource.TestStep{ { // create cluster with shard count 3 - Config: createOrUpdateRedisCluster(name /* replicaCount = */, 1 /* shardCount = */, 3, true, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 3, preventDestroy: true}), }, { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, ImportStateVerifyIgnore: []string{"psc_configs"}, - }, { // update shard count to 5 - Config: createOrUpdateRedisCluster(name /* replicaCount = */, 1 /* shardCount = */, 5, true, /*nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 5, preventDestroy: true}), }, { - ResourceName: "google_redis_cluster.test", - ImportState: true, - ImportStateVerify: true, + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, ImportStateVerifyIgnore: []string{"psc_configs"}, }, { // clean up the resource - Config: createOrUpdateRedisCluster(name /* replicaCount = */, 1 /* shardCount = */, 5, false, /* nodeType = */ ""), + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, replicaCount: 1, shardCount: 5, preventDestroy: false}), }, }, }) } -func createOrUpdateRedisCluster(name string, replicaCount int, shardCount int, preventDestroy bool, nodeType string) string { +// Validate that redisConfigs is updated for the cluster +func TestAccRedisCluster_updateRedisConfigs(t *testing.T) { + t.Parallel() + + name := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderBetaFactories(t), + CheckDestroy: testAccCheckRedisClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + // create cluster + Config: createOrUpdateRedisCluster(&ClusterParams{ + name: name, + shardCount: 3, + redisConfigs: map[string]string{ + "maxmemory-policy": "volatile-ttl", + }}), + }, + { + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"psc_configs"}, + }, + { + // add a new redis config key-value pair and update existing redis config + Config: createOrUpdateRedisCluster(&ClusterParams{ + name: name, + shardCount: 3, + redisConfigs: map[string]string{ + "maxmemory-policy": "allkeys-lru", + "maxmemory-clients": "90%", + }}), + }, + { + ResourceName: "google_redis_cluster.test", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"psc_configs"}, + }, + { + // remove all redis configs + Config: createOrUpdateRedisCluster(&ClusterParams{name: name, shardCount: 3}), + }, + + }, + }) +} + +type ClusterParams struct { + name string + replicaCount int + shardCount int + preventDestroy bool + nodeType string + redisConfigs map[string]string +} + +func createOrUpdateRedisCluster(params *ClusterParams) string { lifecycleBlock := "" - if preventDestroy { + if params.preventDestroy { lifecycleBlock = ` lifecycle { prevent_destroy = true }` } + var strBuilder strings.Builder + for key, value := range params.redisConfigs { + strBuilder.WriteString(fmt.Sprintf("%s = \"%s\"\n", key, value)) + } + return fmt.Sprintf(` resource "google_redis_cluster" "test" { - provider = google-beta - name = "%s" + provider = google-beta + name = "%s" replica_count = %d shard_count = %d node_type = "%s" - region = "us-central1" + region = "us-central1" psc_configs { network = google_compute_network.producer_net.id } + redis_configs = { + %s + } depends_on = [ - google_network_connectivity_service_connection_policy.default - ] + google_network_connectivity_service_connection_policy.default + ] %s } resource "google_network_connectivity_service_connection_policy" "default" { - provider = google-beta - name = "%s" - location = "us-central1" - service_class = "gcp-memorystore-redis" - description = "my basic service connection policy" - network = google_compute_network.producer_net.id - psc_config { - subnetworks = [google_compute_subnetwork.producer_subnet.id] - } + provider = google-beta + name = "%s" + location = "us-central1" + service_class = "gcp-memorystore-redis" + description = "my basic service connection policy" + network = google_compute_network.producer_net.id + psc_config { + subnetworks = [google_compute_subnetwork.producer_subnet.id] + } } resource "google_compute_subnetwork" "producer_subnet" { - provider = google-beta - name = "%s" - ip_cidr_range = "10.0.0.248/29" - region = "us-central1" - network = google_compute_network.producer_net.id + provider = google-beta + name = "%s" + ip_cidr_range = "10.0.0.248/29" + region = "us-central1" + network = google_compute_network.producer_net.id } resource "google_compute_network" "producer_net" { - provider = google-beta - name = "%s" - auto_create_subnetworks = false + provider = google-beta + name = "%s" + auto_create_subnetworks = false } -`, name, replicaCount, shardCount, nodeType, lifecycleBlock, name, name, name) +`, params.name, params.replicaCount, params.shardCount, params.nodeType, strBuilder.String(), lifecycleBlock, params.name, params.name, params.name) } + <% end -%> From cc409a3b301d099d601ea9d817f5a240bd3638c2 Mon Sep 17 00:00:00 2001 From: Ryan Oaks Date: Thu, 25 Apr 2024 12:19:27 -0400 Subject: [PATCH 12/33] Add DocAI warehouse permissions to test runner (#10528) --- .ci/infra/terraform/main.tf | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.ci/infra/terraform/main.tf b/.ci/infra/terraform/main.tf index 0567107882fd..877918c0bb89 100644 --- a/.ci/infra/terraform/main.tf +++ b/.ci/infra/terraform/main.tf @@ -33,6 +33,12 @@ resource "google_organization_iam_member" "sa_access_boundary_admin" { member = google_service_account.sa.member } +resource "google_organization_iam_member" "sa_apphub_admin" { + org_id = data.google_organization.org.org_id + role = "roles/apphub.admin" + member = google_service_account.sa.member +} + resource "google_organization_iam_member" "sa_assuredworkloads_admin" { org_id = data.google_organization.org.org_id role = "roles/assuredworkloads.admin" @@ -63,6 +69,18 @@ resource "google_organization_iam_member" "sa_compute_xpn_admin" { member = google_service_account.sa.member } +resource "google_organization_iam_member" "sa_contentwarehouse_admin" { + org_id = data.google_organization.org.org_id + role = "roles/contentwarehouse.admin" + member = google_service_account.sa.member +} + +resource "google_organization_iam_member" "sa_contentwarehouse_document_admin" { + org_id = data.google_organization.org.org_id + role = "roles/contentwarehouse.documentAdmin" + member = google_service_account.sa.member +} + resource "google_organization_iam_member" "sa_deny_admin" { org_id = data.google_organization.org.org_id role = "roles/iam.denyAdmin" @@ -135,12 +153,6 @@ resource "google_organization_iam_member" "sa_storage_admin" { member = google_service_account.sa.member } -resource "google_organization_iam_member" "apphub_admin" { - org_id = data.google_organization.org.org_id - role = "roles/apphub.admin" - member = google_service_account.sa.member -} - resource "google_billing_account_iam_member" "sa_master_billing_admin" { billing_account_id = data.google_billing_account.master_acct.id role = "roles/billing.admin" From 2b3d1569bfcb8b34144a94509cf77caa8f83d640 Mon Sep 17 00:00:00 2001 From: Ryan Oaks Date: Thu, 25 Apr 2024 12:32:51 -0400 Subject: [PATCH 13/33] Fix DocAI Warehouse tests to work properly with a test project (#10526) --- .../documentaiwarehouse/DocumentSchema.yaml | 44 +- .../documentaiwarehouse/Location.yaml | 3 +- ..._warehouse_document_schema_datetime.tf.erb | 24 - ...t_ai_warehouse_document_schema_enum.tf.erb | 31 - ..._ai_warehouse_document_schema_float.tf.erb | 23 - ...i_warehouse_document_schema_integer.tf.erb | 24 - ...nt_ai_warehouse_document_schema_map.tf.erb | 24 - ..._warehouse_document_schema_property.tf.erb | 41 -- ...house_document_schema_property_enum.tf.erb | 48 -- ...t_ai_warehouse_document_schema_text.tf.erb | 2 +- ...warehouse_document_schema_timestamp.tf.erb | 24 - ...ument_ai_warehouse_document_schema_test.go | 610 ++++++++++++++++++ 12 files changed, 615 insertions(+), 283 deletions(-) delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_datetime.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_enum.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_float.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_integer.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_map.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property_enum.tf.erb delete mode 100644 mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_timestamp.tf.erb create mode 100644 mmv1/third_party/terraform/services/documentaiwarehouse/resource_document_ai_warehouse_document_schema_test.go diff --git a/mmv1/products/documentaiwarehouse/DocumentSchema.yaml b/mmv1/products/documentaiwarehouse/DocumentSchema.yaml index 9c8746f6b72e..b7af4882817a 100644 --- a/mmv1/products/documentaiwarehouse/DocumentSchema.yaml +++ b/mmv1/products/documentaiwarehouse/DocumentSchema.yaml @@ -27,50 +27,10 @@ custom_code: !ruby/object:Provider::Terraform::CustomCode custom_import: templates/terraform/custom_import/document_ai_warehouse_document_schema.go.erb examples: - !ruby/object:Provider::Terraform::Examples + # docs only, testing is done in an update test + skip_test: true name: "document_ai_warehouse_document_schema_text" primary_resource_id: "example_text" - vars: - document_schema_name: "schema-name-text" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_integer" - primary_resource_id: "example_integer" - vars: - document_schema_name: "schema-name-integer" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_float" - primary_resource_id: "example_float" - vars: - document_schema_name: "schema-name-float" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_property" - primary_resource_id: "example_property" - vars: - document_schema_name: "schema-name-property" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_property_enum" - primary_resource_id: "example_property_enum" - vars: - document_schema_name: "schema-name-property-enum" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_enum" - primary_resource_id: "example_enum" - vars: - document_schema_name: "schema-name-enum" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_map" - primary_resource_id: "example_map" - vars: - document_schema_name: "schema-name-map" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_datetime" - primary_resource_id: "example_datetime" - vars: - document_schema_name: "schema-name-datetime" - - !ruby/object:Provider::Terraform::Examples - name: "document_ai_warehouse_document_schema_timestamp" - primary_resource_id: "example_timestamp" - vars: - document_schema_name: "schema-name-timestamp" parameters: - !ruby/object:Api::Type::String name: 'project_number' diff --git a/mmv1/products/documentaiwarehouse/Location.yaml b/mmv1/products/documentaiwarehouse/Location.yaml index cbf1a999e150..36d6b5ab72b6 100644 --- a/mmv1/products/documentaiwarehouse/Location.yaml +++ b/mmv1/products/documentaiwarehouse/Location.yaml @@ -48,9 +48,10 @@ skip_sweeper: true exclude_import: true examples: - !ruby/object:Provider::Terraform::Examples + # docs only, testing is done in a DocumentSchema update test + skip_test: true name: "document_ai_warehouse_location" primary_resource_id: "example" - skip_import_test: true timeouts: !ruby/object:Api::Timeouts insert_minutes: 30 update_minutes: 30 diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_datetime.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_datetime.tf.erb deleted file mode 100644 index 1e3e4f35b060..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_datetime.tf.erb +++ /dev/null @@ -1,24 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_datetime" { - project_number = data.google_project.project.number - display_name = "test-property-date_time" - location = "us" - - property_definitions { - name = "prop7" - display_name = "propdisp7" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - date_time_type_options {} - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_enum.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_enum.tf.erb deleted file mode 100644 index b763f2fad385..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_enum.tf.erb +++ /dev/null @@ -1,31 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_enum" { - project_number = data.google_project.project.number - display_name = "test-property-enum" - location = "us" - - property_definitions { - name = "prop6" - display_name = "propdisp6" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - enum_type_options { - possible_values = [ - "M", - "F", - "X" - ] - validation_check_disabled = false - } - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_float.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_float.tf.erb deleted file mode 100644 index 96d30784152b..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_float.tf.erb +++ /dev/null @@ -1,23 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_float" { - project_number = data.google_project.project.number - display_name = "test-property-float" - location = "us" - - property_definitions { - name = "prop2" - display_name = "propdisp2" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - float_type_options {} - } -} -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_integer.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_integer.tf.erb deleted file mode 100644 index 9c0ef30e6b4f..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_integer.tf.erb +++ /dev/null @@ -1,24 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_integer" { - project_number = data.google_project.project.number - display_name = "test-property-integer" - location = "us" - - property_definitions { - name = "prop1" - display_name = "propdisp1" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - integer_type_options {} - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_map.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_map.tf.erb deleted file mode 100644 index 11796b01b0e8..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_map.tf.erb +++ /dev/null @@ -1,24 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_map" { - project_number = data.google_project.project.number - display_name = "test-property-map" - location = "us" - - property_definitions { - name = "prop4" - display_name = "propdisp4" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - map_type_options {} - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property.tf.erb deleted file mode 100644 index bd0ba6ee0976..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property.tf.erb +++ /dev/null @@ -1,41 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_property" { - project_number = data.google_project.project.number - display_name = "test-property-property" - location = "us" - document_is_folder = false - - property_definitions { - name = "prop8" - display_name = "propdisp8" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - property_type_options { - property_definitions { - name = "prop8_nested" - display_name = "propdisp8_nested" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source_nested" - processor_type = "dummy_processor_nested" - } - text_type_options {} - } - } - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property_enum.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property_enum.tf.erb deleted file mode 100644 index 19eb0553b6c7..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_property_enum.tf.erb +++ /dev/null @@ -1,48 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_property_enum" { - project_number = data.google_project.project.number - display_name = "test-property-property" - location = "us" - document_is_folder = false - - property_definitions { - name = "prop8" - display_name = "propdisp8" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - property_type_options { - property_definitions { - name = "prop8_nested" - display_name = "propdisp8_nested" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source_nested" - processor_type = "dummy_processor_nested" - } - enum_type_options { - possible_values = [ - "M", - "F", - "X" - ] - validation_check_disabled = false - } - } - } - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_text.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_text.tf.erb index d7f797fbe114..c6bb89e99c5f 100644 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_text.tf.erb +++ b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_text.tf.erb @@ -1,4 +1,4 @@ -resource "google_document_ai_warehouse_document_schema" "example_text" { +resource "google_document_ai_warehouse_document_schema" "<%= ctx[:primary_resource_id] %>" { project_number = data.google_project.project.number display_name = "test-property-text" location = "us" diff --git a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_timestamp.tf.erb b/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_timestamp.tf.erb deleted file mode 100644 index 417d81fe70fa..000000000000 --- a/mmv1/templates/terraform/examples/document_ai_warehouse_document_schema_timestamp.tf.erb +++ /dev/null @@ -1,24 +0,0 @@ -resource "google_document_ai_warehouse_document_schema" "example_timestamp" { - project_number = data.google_project.project.number - display_name = "test-property-timestamp" - location = "us" - - property_definitions { - name = "prop5" - display_name = "propdisp5" - is_repeatable = false - is_filterable = true - is_searchable = true - is_metadata = false - is_required = false - retrieval_importance = "HIGHEST" - schema_sources { - name = "dummy_source" - processor_type = "dummy_processor" - } - timestamp_type_options {} - } -} - -data "google_project" "project" { -} \ No newline at end of file diff --git a/mmv1/third_party/terraform/services/documentaiwarehouse/resource_document_ai_warehouse_document_schema_test.go b/mmv1/third_party/terraform/services/documentaiwarehouse/resource_document_ai_warehouse_document_schema_test.go new file mode 100644 index 000000000000..2087d6cbdfd3 --- /dev/null +++ b/mmv1/third_party/terraform/services/documentaiwarehouse/resource_document_ai_warehouse_document_schema_test.go @@ -0,0 +1,610 @@ +package documentaiwarehouse_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" + + "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/envvar" + "github.com/hashicorp/terraform-provider-google/google/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" +) + +func TestAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseFull(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "org_id": envvar.GetTestOrgFromEnv(t), + "billing_account": envvar.GetTestBillingAccountFromEnv(t), + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + ExternalProviders: map[string]resource.ExternalProvider{ + "time": {}, + }, + CheckDestroy: testAccCheckDocumentAIWarehouseDocumentSchemaDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseInit(context), + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaTextExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_text", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaIntegerExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_integer", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaFloatExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_float", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaPropertyExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_property", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaPropertyEnumExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_property_enum", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaEnumExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_enum", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaMapExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_map", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaDatetimeExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_datetime", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + { + Config: testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaTimestampExample(context), + }, + { + ResourceName: "google_document_ai_warehouse_document_schema.example_timestamp", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"project_number", "location"}, + }, + }, + }) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseInit(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_project_service" "contentwarehouse" { + project = google_project.project.project_id + service = "contentwarehouse.googleapis.com" + disable_on_destroy = false +} + +resource "time_sleep" "wait_120s" { + create_duration = "120s" + + depends_on = [google_project_service.contentwarehouse] +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" + + depends_on = [time_sleep.wait_120s] +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaTextExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_text" { + project_number = google_project.project.number + display_name = "test-property-text" + location = "us" + document_is_folder = false + + property_definitions { + name = "prop3" + display_name = "propdisp3" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + text_type_options {} + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaIntegerExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_integer" { + project_number = google_project.project.number + display_name = "test-property-integer" + location = "us" + + property_definitions { + name = "prop1" + display_name = "propdisp1" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + integer_type_options {} + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaFloatExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_float" { + project_number = google_project.project.number + display_name = "test-property-float" + location = "us" + + property_definitions { + name = "prop2" + display_name = "propdisp2" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + float_type_options {} + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaPropertyExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_property" { + project_number = google_project.project.number + display_name = "test-property-property" + location = "us" + document_is_folder = false + + property_definitions { + name = "prop8" + display_name = "propdisp8" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + property_type_options { + property_definitions { + name = "prop8_nested" + display_name = "propdisp8_nested" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source_nested" + processor_type = "dummy_processor_nested" + } + text_type_options {} + } + } + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaPropertyEnumExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_property_enum" { + project_number = google_project.project.number + display_name = "test-property-property" + location = "us" + document_is_folder = false + + property_definitions { + name = "prop8" + display_name = "propdisp8" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + property_type_options { + property_definitions { + name = "prop8_nested" + display_name = "propdisp8_nested" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source_nested" + processor_type = "dummy_processor_nested" + } + enum_type_options { + possible_values = [ + "M", + "F", + "X" + ] + validation_check_disabled = false + } + } + } + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaEnumExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_enum" { + project_number = google_project.project.number + display_name = "test-property-enum" + location = "us" + + property_definitions { + name = "prop6" + display_name = "propdisp6" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + enum_type_options { + possible_values = [ + "M", + "F", + "X" + ] + validation_check_disabled = false + } + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaMapExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_map" { + project_number = google_project.project.number + display_name = "test-property-map" + location = "us" + + property_definitions { + name = "prop4" + display_name = "propdisp4" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + map_type_options {} + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaDatetimeExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_datetime" { + project_number = google_project.project.number + display_name = "test-property-date_time" + location = "us" + + property_definitions { + name = "prop7" + display_name = "propdisp7" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + date_time_type_options {} + } +} +`, context) +} + +func testAccDocumentAIWarehouseDocumentSchema_documentAiWarehouseDocumentSchemaTimestampExample(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_project" "project" { + project_id = "tf-test-%{random_suffix}" + name = "tf-test-%{random_suffix}" + org_id = "%{org_id}" + billing_account = "%{billing_account}" +} + +resource "google_document_ai_warehouse_location" "loc" { + location = "us" + project_number = google_project.project.number + access_control_mode = "ACL_MODE_DOCUMENT_LEVEL_ACCESS_CONTROL_GCI" + database_type = "DB_INFRA_SPANNER" + document_creator_default_role = "DOCUMENT_ADMIN" +} + +resource "google_document_ai_warehouse_document_schema" "example_timestamp" { + project_number = google_project.project.number + display_name = "test-property-timestamp" + location = "us" + + property_definitions { + name = "prop5" + display_name = "propdisp5" + is_repeatable = false + is_filterable = true + is_searchable = true + is_metadata = false + is_required = false + retrieval_importance = "HIGHEST" + schema_sources { + name = "dummy_source" + processor_type = "dummy_processor" + } + timestamp_type_options {} + } +} +`, context) +} + +func testAccCheckDocumentAIWarehouseDocumentSchemaDestroyProducer(t *testing.T) func(s *terraform.State) error { + return func(s *terraform.State) error { + for name, rs := range s.RootModule().Resources { + if rs.Type != "google_document_ai_warehouse_document_schema" { + continue + } + if strings.HasPrefix(name, "data.") { + continue + } + + config := acctest.GoogleProviderConfig(t) + + url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{DocumentAIWarehouseBasePath}}{{name}}") + if err != nil { + return err + } + + billingProject := "" + + if config.BillingProject != "" { + billingProject = config.BillingProject + } + + _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + Project: billingProject, + RawURL: url, + UserAgent: config.UserAgent, + }) + if err == nil { + return fmt.Errorf("DocumentAIWarehouseDocumentSchema still exists at %s", url) + } + } + + return nil + } +} From 5bb072189ea291c2b7485a31902a48989afabc6e Mon Sep 17 00:00:00 2001 From: Riley Karson Date: Thu, 25 Apr 2024 10:57:08 -0700 Subject: [PATCH 14/33] Address lint warnings in products/documentaiwarehouse/Location.yaml (#10530) --- mmv1/products/documentaiwarehouse/DocumentSchema.yaml | 1 - mmv1/products/documentaiwarehouse/Location.yaml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mmv1/products/documentaiwarehouse/DocumentSchema.yaml b/mmv1/products/documentaiwarehouse/DocumentSchema.yaml index b7af4882817a..fd4ab554d83d 100644 --- a/mmv1/products/documentaiwarehouse/DocumentSchema.yaml +++ b/mmv1/products/documentaiwarehouse/DocumentSchema.yaml @@ -303,4 +303,3 @@ properties: send_empty_value: true allow_empty_object: true properties: [] - diff --git a/mmv1/products/documentaiwarehouse/Location.yaml b/mmv1/products/documentaiwarehouse/Location.yaml index 36d6b5ab72b6..dccdbac1b858 100644 --- a/mmv1/products/documentaiwarehouse/Location.yaml +++ b/mmv1/products/documentaiwarehouse/Location.yaml @@ -23,7 +23,7 @@ description: | id_format: 'projects/{{project_number}}/locations/{{location}}' autogen_async: true async: !ruby/object:Api::OpAsync - actions: ['create'] + actions: ['create'] operation: !ruby/object:Api::OpAsync::Operation path: 'name' base_url: '{{op_id}}' @@ -102,4 +102,4 @@ properties: values: - :DOCUMENT_ADMIN - :DOCUMENT_EDITOR - - :DOCUMENT_VIEWER \ No newline at end of file + - :DOCUMENT_VIEWER From 221bb3867b604d96dc95389d1ed9faf1e2baa6a3 Mon Sep 17 00:00:00 2001 From: Shuya Ma <87669292+shuyama1@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:36:03 -0700 Subject: [PATCH 15/33] add privilegedaccessmanager (#10534) --- .ci/infra/terraform/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/infra/terraform/main.tf b/.ci/infra/terraform/main.tf index 877918c0bb89..358cde716f82 100644 --- a/.ci/infra/terraform/main.tf +++ b/.ci/infra/terraform/main.tf @@ -298,6 +298,7 @@ module "project-services" { "oslogin.googleapis.com", "parallelstore.googleapis.com", "privateca.googleapis.com", + "privilegedaccessmanager.googleapis.com", "pubsub.googleapis.com", "pubsublite.googleapis.com", "publicca.googleapis.com", From 0afb6b573d7554b6b232f7738de08945d5fc1f9b Mon Sep 17 00:00:00 2001 From: bestefreund <81525627+bestefreund@users.noreply.github.com> Date: Thu, 25 Apr 2024 21:41:58 +0200 Subject: [PATCH 16/33] Add data source for retrieving multiple GCS buckets from a project (#10444) --- .../provider/provider_mmv1_resources.go.erb | 1 + .../data_source_google_storage_buckets.go | 151 ++++++++++++++++++ ...data_source_google_storage_buckets_test.go | 123 ++++++++++++++ .../docs/d/storage_buckets.html.markdown | 44 +++++ 4 files changed, 319 insertions(+) create mode 100644 mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets.go create mode 100644 mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets_test.go create mode 100644 mmv1/third_party/terraform/website/docs/d/storage_buckets.html.markdown diff --git a/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb b/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb index abe868b51ab3..fae027d235ad 100644 --- a/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb +++ b/mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb @@ -183,6 +183,7 @@ var handwrittenDatasources = map[string]*schema.Resource{ "google_sql_database_instances": sql.DataSourceSqlDatabaseInstances(), "google_service_networking_peered_dns_domain": servicenetworking.DataSourceGoogleServiceNetworkingPeeredDNSDomain(), "google_storage_bucket": storage.DataSourceGoogleStorageBucket(), + "google_storage_buckets": storage.DataSourceGoogleStorageBuckets(), "google_storage_bucket_object": storage.DataSourceGoogleStorageBucketObject(), "google_storage_bucket_objects": storage.DataSourceGoogleStorageBucketObjects(), "google_storage_bucket_object_content": storage.DataSourceGoogleStorageBucketObjectContent(), diff --git a/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets.go b/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets.go new file mode 100644 index 000000000000..e8178ec7c961 --- /dev/null +++ b/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets.go @@ -0,0 +1,151 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 +package storage + +import ( + "fmt" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-provider-google/google/tpgresource" + transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" +) + +func DataSourceGoogleStorageBuckets() *schema.Resource { + return &schema.Resource{ + Read: datasourceGoogleStorageBucketsRead, + Schema: map[string]*schema.Schema{ + "prefix": { + Type: schema.TypeString, + Optional: true, + }, + "project": { + Type: schema.TypeString, + Optional: true, + }, + "buckets": { + Type: schema.TypeList, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "labels": { + Type: schema.TypeMap, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "location": { + Type: schema.TypeString, + Computed: true, + }, + "name": { + Type: schema.TypeString, + Computed: true, + }, + "self_link": { + Type: schema.TypeString, + Computed: true, + }, + "storage_class": { + Type: schema.TypeString, + Computed: true, + }, + }, + }, + }, + }, + } +} + +func datasourceGoogleStorageBucketsRead(d *schema.ResourceData, meta interface{}) error { + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + params := make(map[string]string) + buckets := make([]map[string]interface{}, 0) + + for { + url := "https://storage.googleapis.com/storage/v1/b" + + params["project"], err = tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for bucket: %s", err) + } + + if v, ok := d.GetOk("prefix"); ok { + params["prefix"] = v.(string) + } + + url, err = transport_tpg.AddQueryParams(url, params) + if err != nil { + return err + } + + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "GET", + RawURL: url, + UserAgent: userAgent, + }) + if err != nil { + return fmt.Errorf("Error retrieving buckets: %s", err) + } + + pageBuckets := flattenDatasourceGoogleBucketsList(res["items"]) + buckets = append(buckets, pageBuckets...) + + pToken, ok := res["nextPageToken"] + if ok && pToken != nil && pToken.(string) != "" { + params["pageToken"] = pToken.(string) + } else { + break + } + } + + if err := d.Set("buckets", buckets); err != nil { + return fmt.Errorf("Error retrieving buckets: %s", err) + } + + d.SetId(params["project"]) + + return nil +} + +func flattenDatasourceGoogleBucketsList(v interface{}) []map[string]interface{} { + if v == nil { + return make([]map[string]interface{}, 0) + } + + ls := v.([]interface{}) + buckets := make([]map[string]interface{}, 0, len(ls)) + for _, raw := range ls { + o := raw.(map[string]interface{}) + + var mLabels, mLocation, mName, mSelfLink, mStorageClass interface{} + if oLabels, ok := o["labels"]; ok { + mLabels = oLabels + } + if oLocation, ok := o["location"]; ok { + mLocation = oLocation + } + if oName, ok := o["name"]; ok { + mName = oName + } + if oSelfLink, ok := o["selfLink"]; ok { + mSelfLink = oSelfLink + } + if oStorageClass, ok := o["storageClass"]; ok { + mStorageClass = oStorageClass + } + buckets = append(buckets, map[string]interface{}{ + "labels": mLabels, + "location": mLocation, + "name": mName, + "self_link": mSelfLink, + "storage_class": mStorageClass, + }) + } + + return buckets +} diff --git a/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets_test.go b/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets_test.go new file mode 100644 index 000000000000..63cbf32b2d9f --- /dev/null +++ b/mmv1/third_party/terraform/services/storage/data_source_google_storage_buckets_test.go @@ -0,0 +1,123 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 +package storage_test + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/envvar" +) + +func TestAccDataSourceGoogleStorageBuckets_basic(t *testing.T) { + t.Parallel() + + static_prefix := "tf-bucket-test" + random_suffix := acctest.RandString(t, 10) + + context := map[string]interface{}{ + "billing_account": envvar.GetTestBillingAccountFromEnv(t), + "bucket1": static_prefix + "-1-" + random_suffix, + "bucket2": static_prefix + "-2-" + random_suffix, + "project_id": static_prefix + "-" + random_suffix, + "organization": envvar.GetTestOrgFromEnv(t), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + Steps: []resource.TestStep{ + { + Config: testAccCheckGoogleStorageBucketsConfig(context), + Check: resource.ComposeTestCheckFunc( + // Test schema + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.location"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.name"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.self_link"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.0.storage_class"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.location"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.name"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.self_link"), + resource.TestCheckResourceAttrSet("data.google_storage_buckets.all", "buckets.1.storage_class"), + // Test content + resource.TestCheckResourceAttr("data.google_storage_buckets.all", "project", context["project_id"].(string)), + resource.TestCheckResourceAttr("data.google_storage_buckets.all", "buckets.0.name", context["bucket1"].(string)), + resource.TestCheckResourceAttr("data.google_storage_buckets.all", "buckets.1.name", context["bucket2"].(string)), + // Test with project + resource.TestCheckResourceAttr("data.google_storage_buckets.one", "buckets.0.name", context["bucket1"].(string)), + // Test prefix + resource.TestCheckResourceAttr("data.google_storage_buckets.two", "buckets.0.name", context["bucket2"].(string)), + ), + }, + }, + }) +} + +func testAccCheckGoogleStorageBucketsConfig(context map[string]interface{}) string { + return fmt.Sprintf(` +locals { + billing_account = "%s" + bucket_one = "%s" + bucket_two = "%s" + organization = "%s" + project_id = "%s" +} + +resource "google_project" "acceptance" { + name = local.project_id + project_id = local.project_id + org_id = local.organization + billing_account = local.billing_account +} + +resource "google_storage_bucket" "one" { + force_destroy = true + location = "EU" + name = local.bucket_one + project = google_project.acceptance.project_id + uniform_bucket_level_access = true +} + +resource "google_storage_bucket" "two" { + force_destroy = true + location = "EU" + name = local.bucket_two + project = google_project.acceptance.project_id + uniform_bucket_level_access = true +} + +data "google_storage_buckets" "all" { + project = google_project.acceptance.project_id + + depends_on = [ + google_storage_bucket.one, + google_storage_bucket.two, + ] +} + +data "google_storage_buckets" "one" { + prefix = "tf-bucket-test-1" + project = google_project.acceptance.project_id + + depends_on = [ + google_storage_bucket.one, + ] +} + +data "google_storage_buckets" "two" { + prefix = "tf-bucket-test-2" + project = google_project.acceptance.project_id + + depends_on = [ + google_storage_bucket.two, + ] +}`, + context["billing_account"].(string), + context["bucket1"].(string), + context["bucket2"].(string), + context["organization"].(string), + context["project_id"].(string), + ) +} diff --git a/mmv1/third_party/terraform/website/docs/d/storage_buckets.html.markdown b/mmv1/third_party/terraform/website/docs/d/storage_buckets.html.markdown new file mode 100644 index 000000000000..047002d792f1 --- /dev/null +++ b/mmv1/third_party/terraform/website/docs/d/storage_buckets.html.markdown @@ -0,0 +1,44 @@ +--- +subcategory: "Cloud Storage" +description: |- + Retrieve information about a set of GCS buckets in a project. +--- + + +# google\_storage\_buckets + +Gets a list of existing GCS buckets. +See [the official documentation](https://cloud.google.com/storage/docs/introduction) +and [API](https://cloud.google.com/storage/docs/json_api/v1/buckets/list). + +## Example Usage + +Example GCS buckets. + +```hcl +data "google_storage_buckets" "example" { + project = "example-project" +} +``` + +## Argument Reference + +The following arguments are supported: + +* `prefix` - (Optional) Filter results to buckets whose names begin with this prefix. +* `project` - (Optional) The ID of the project. If it is not provided, the provider project is used. + + +## Attributes Reference + +The following attributes are exported: + +* `buckets` - A list of all retrieved GCS buckets. Structure is [defined below](#nested_buckets). + +The `buckets` block supports: + +* `labels` - User-provided bucket labels, in key/value pairs. +* `location` - The location of the bucket. +* `name` - The name of the bucket. +* `self_link` - A url reference to the bucket. +* `storage_class` - The [StorageClass](https://cloud.google.com/storage/docs/storage-classes) of the bucket. From 5a6eec220a101a4a2f6929d975060d4d801ae2d7 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Thu, 25 Apr 2024 12:46:13 -0700 Subject: [PATCH 17/33] include PAM setup in ci readme (#10535) --- .ci/infra/terraform/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/infra/terraform/README.md b/.ci/infra/terraform/README.md index 76c872f13f0c..fb1898e47119 100644 --- a/.ci/infra/terraform/README.md +++ b/.ci/infra/terraform/README.md @@ -48,6 +48,7 @@ After applying this configuration: - Create a `support@` group in the Google Workspace Admin Console, add new service account as a member, and make it an owner - Enroll in Cloud Armor Managed Protection Plus tier - Add Cloud Identity Premium Plan to the Google Workspace domain +- Perform the Privileged Access Manager set-up https://pantheon.corp.google.com/iam-admin/pam/setup Quotas that will need to be adjusted to support all tests: - Project quota for the new service account From 37446c423a1491876fecac3a0168e1646d9d41ad Mon Sep 17 00:00:00 2001 From: aston-github <39973638+aston-github@users.noreply.github.com> Date: Thu, 25 Apr 2024 20:35:16 +0000 Subject: [PATCH 18/33] feat(google_container_node_pool): support secondary boot disks (#10511) --- .../services/container/node_config.go.erb | 47 +++++++++++++++ .../resource_container_node_pool_test.go.erb | 58 +++++++++++++++++++ .../docs/r/container_cluster.html.markdown | 8 +++ 3 files changed, 113 insertions(+) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.erb b/mmv1/third_party/terraform/services/container/node_config.go.erb index 3c60164f42c4..2d40aa68f51c 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.erb +++ b/mmv1/third_party/terraform/services/container/node_config.go.erb @@ -257,6 +257,30 @@ func schemaNodeConfig() *schema.Schema { }, }, + "secondary_boot_disks": { + Type: schema.TypeList, + Optional: true, + MaxItems: 127, + Description: `Secondary boot disks for preloading data or container images.`, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "disk_image": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: `Disk image to create the secondary boot disk from`, + }, + "mode": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `Mode for how the secondary boot disk is used.`, + }, + }, + }, + }, + "gcfs_config": schemaGcfsConfig(true), "gvnic": { @@ -809,6 +833,14 @@ func expandNodeConfig(v interface{}) *container.NodeConfig { } } + if v, ok := nodeConfig["secondary_boot_disks"]; ok && len(v.([]interface{})) > 0 { + conf := v.([]interface{})[0].(map[string]interface{}) + nc.SecondaryBootDisks = append(nc.SecondaryBootDisks, &container.SecondaryBootDisk{ + DiskImage: conf["disk_image"].(string), + Mode: conf["mode"].(string), + }) + } + if v, ok := nodeConfig["gcfs_config"]; ok && len(v.([]interface{})) > 0 { conf := v.([]interface{})[0].(map[string]interface{}) nc.GcfsConfig = &container.GcfsConfig{ @@ -1213,6 +1245,7 @@ func flattenNodeConfig(c *container.NodeConfig, v interface{}) []map[string]inte "resource_labels": c.ResourceLabels, "tags": c.Tags, "preemptible": c.Preemptible, + "secondary_boot_disks": flattenSecondaryBootDisks(c.SecondaryBootDisks), "spot": c.Spot, "min_cpu_platform": c.MinCpuPlatform, "shielded_instance_config": flattenShieldedInstanceConfig(c.ShieldedInstanceConfig), @@ -1335,6 +1368,20 @@ func flattenEphemeralStorageLocalSsdConfig(c *container.EphemeralStorageLocalSsd return result } +func flattenSecondaryBootDisks(c []*container.SecondaryBootDisk) []map[string]interface{} { + result := []map[string]interface{}{} + if c != nil { + for _, disk := range c { + secondaryBootDisk := map[string]interface{}{ + "disk_image": disk.DiskImage, + "mode": disk.Mode, + } + result = append(result, secondaryBootDisk) + } + } + return result +} + func flattenLoggingVariant(c *container.NodePoolLoggingConfig) string { variant := "DEFAULT" if c != nil && c.VariantConfig != nil && c.VariantConfig.Variant != "" { diff --git a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.erb b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.erb index 9c5b2af61963..8df42c8a458a 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.erb +++ b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.erb @@ -1644,6 +1644,64 @@ resource "google_container_node_pool" "np" { `, cluster, networkName, subnetworkName, np) } +func TestAccContainerNodePool_secondaryBootDisks(t *testing.T) { + t.Parallel() + + cluster := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + np := fmt.Sprintf("tf-test-nodepool-%s", acctest.RandString(t, 10)) + networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster") + subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerNodePoolDestroyProducer(t), + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccContainerNodePool_secondaryBootDisks(cluster, np, networkName, subnetworkName), + }, + resource.TestStep{ + ResourceName: "google_container_node_pool.np", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccContainerNodePool_secondaryBootDisks(cluster, np, networkName, subnetworkName string) string { + return fmt.Sprintf(` +resource "google_container_cluster" "cluster" { + name = "%s" + location = "us-central1-a" + initial_node_count = 1 + deletion_protection = false + network = "%s" + subnetwork = "%s" + min_master_version = "1.28" +} + +resource "google_container_node_pool" "np" { + name = "%s" + location = "us-central1-a" + cluster = google_container_cluster.cluster.name + initial_node_count = 1 + + node_config { + machine_type = "n1-standard-8" + image_type = "COS_CONTAINERD" + gcfs_config { + enabled = true + } + secondary_boot_disks { + disk_image = "" + mode = "CONTAINER_IMAGE_CACHE" + } + } +} +`, cluster, networkName, subnetworkName, np) +} + func TestAccContainerNodePool_gcfsConfig(t *testing.T) { t.Parallel() diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 9502e01f4b8a..00a90406f53a 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -821,6 +821,8 @@ ephemeral_storage_local_ssd_config { * `logging_variant` (Optional) Parameter for specifying the type of logging agent used in a node pool. This will override any [cluster-wide default value](#nested_node_pool_defaults). Valid values include DEFAULT and MAX_THROUGHPUT. See [Increasing logging agent throughput](https://cloud.google.com/stackdriver/docs/solutions/gke/managing-logs#throughput) for more information. +* `secondary_boot_disks` - (Optional) Parameters for secondary boot disks to preload container images and data on new nodes. Structure is [documented below](#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. `min_master_version` must also be set to use GKE 1.28.3-gke.106700 or later versions. + * `gcfs_config` - (Optional) Parameters for the Google Container Filesystem (GCFS). If unspecified, GCFS will not be enabled on the node pool. When enabling this feature you must specify `image_type = "COS_CONTAINERD"` and `node_version` from GKE versions 1.19 or later to use it. For GKE versions 1.19, 1.20, and 1.21, the recommended minimum `node_version` would be 1.19.15-gke.1300, 1.20.11-gke.1300, and 1.21.5-gke.1300 respectively. @@ -990,6 +992,12 @@ sole_tenant_config { * `local_ssd_count` (Required) - Number of raw-block local NVMe SSD disks to be attached to the node. Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. -> Note: Local NVMe SSD storage available in GKE versions v1.25.3-gke.1800 and later. +The `secondary_boot_disks` block supports: + +* `disk_image` (Required) - Path to disk image to create the secondary boot disk from. After using the [gke-disk-image-builder](https://github.com/GoogleCloudPlatform/ai-on-gke/tree/main/tools/gke-disk-image-builder), this argument should be `global/images/DISK_IMAGE_NAME`. +* `mode` (Optional) - Mode for how the secondary boot disk is used. An example mode is `CONTAINER_IMAGE_CACHE`. + + The `gcfs_config` block supports: * `enabled` (Required) - Whether or not the Google Container Filesystem (GCFS) is enabled From d2ad872dc282389788159febc564a575407c65e7 Mon Sep 17 00:00:00 2001 From: Jing Date: Thu, 25 Apr 2024 14:23:01 -0700 Subject: [PATCH 19/33] Add database_id section in firebase rule release sample (#10529) Co-authored-by: Sichen Liu --- .../firebaserules/samples/release/firestore_release.tf.tmpl | 2 +- .../firebaserules/samples/release/firestore_release.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tpgtools/overrides/firebaserules/samples/release/firestore_release.tf.tmpl b/tpgtools/overrides/firebaserules/samples/release/firestore_release.tf.tmpl index 3d41ea93c6cf..f88267b16a2e 100644 --- a/tpgtools/overrides/firebaserules/samples/release/firestore_release.tf.tmpl +++ b/tpgtools/overrides/firebaserules/samples/release/firestore_release.tf.tmpl @@ -1,5 +1,5 @@ resource "google_firebaserules_release" "primary" { - name = "cloud.firestore" + name = "cloud.firestore/{{database}}" ruleset_name = "projects/{{project}}/rulesets/${google_firebaserules_ruleset.firestore.name}" project = "{{project}}" diff --git a/tpgtools/overrides/firebaserules/samples/release/firestore_release.yaml b/tpgtools/overrides/firebaserules/samples/release/firestore_release.yaml index 814b4c37d22a..4a8a4112e96a 100644 --- a/tpgtools/overrides/firebaserules/samples/release/firestore_release.yaml +++ b/tpgtools/overrides/firebaserules/samples/release/firestore_release.yaml @@ -20,4 +20,6 @@ versions: resource: ./firestore_release.tf.tmpl variables: - name: project - type: project \ No newline at end of file + type: project +- name: database + type: resource_name \ No newline at end of file From ed92bd333e11a03000fd1fc1a23ea07a972e1863 Mon Sep 17 00:00:00 2001 From: Gustavo Kotarsky <167570715+kotarsky@users.noreply.github.com> Date: Thu, 25 Apr 2024 21:40:59 +0000 Subject: [PATCH 20/33] add compute route tests (#10521) --- mmv1/templates/tgc/resource_converters.go.erb | 1 + .../tgc/tests/data/example_compute_route.json | 23 +++++++++++++++++++ .../tgc/tests/data/example_compute_route.tf | 19 +++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 mmv1/third_party/tgc/tests/data/example_compute_route.json create mode 100644 mmv1/third_party/tgc/tests/data/example_compute_route.tf diff --git a/mmv1/templates/tgc/resource_converters.go.erb b/mmv1/templates/tgc/resource_converters.go.erb index 6f2e4f37e154..aad7ac4d6c6f 100644 --- a/mmv1/templates/tgc/resource_converters.go.erb +++ b/mmv1/templates/tgc/resource_converters.go.erb @@ -43,6 +43,7 @@ func ResourceConverters() map[string][]cai.ResourceConverter { "google_compute_global_forwarding_rule": {compute.ResourceConverterComputeGlobalForwardingRule()}, "google_compute_instance": {compute.ResourceConverterComputeInstance()}, "google_compute_network": {compute.ResourceConverterComputeNetwork()}, + "google_compute_route": {compute.ResourceConverterComputeRoute()}, "google_compute_security_policy": {resourceConverterComputeSecurityPolicy()}, "google_compute_snapshot": {compute.ResourceConverterComputeSnapshot()}, "google_compute_subnetwork": {compute.ResourceConverterComputeSubnetwork()}, diff --git a/mmv1/third_party/tgc/tests/data/example_compute_route.json b/mmv1/third_party/tgc/tests/data/example_compute_route.json new file mode 100644 index 000000000000..27d4c23684ce --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_compute_route.json @@ -0,0 +1,23 @@ +[ + { + "name": "//compute.googleapis.com/projects/{{.Provider.project}}/global/routes/my-route", + "ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}", + "asset_type": "compute.googleapis.com/Route", + "resource": { + "version": "beta", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/beta/rest", + "discovery_name": "Route", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "destRange": "10.1.0.0/16", + "name": "my-route", + "network": "projects/{{.Provider.project}}/global/networks/my-network", + "nextHopIp": "10.0.0.1", + "priority": 1000 + } + }, + "ancestors": [ + "organizations/{{.OrgID}}" + ] + } +] \ No newline at end of file diff --git a/mmv1/third_party/tgc/tests/data/example_compute_route.tf b/mmv1/third_party/tgc/tests/data/example_compute_route.tf new file mode 100644 index 000000000000..7ff8314dfc4a --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_compute_route.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google-beta" + version = "~> {{.Provider.version}}" + } + } +} + +provider "google" { + {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}} +} + +resource "google_compute_route" "my_route" { + name = "my-route" + dest_range = "10.1.0.0/16" + next_hop_ip = "10.0.0.1" + network = "my-network" +} \ No newline at end of file From 21c467b1edcbb1e3128fd209fbb5053eb3ea5795 Mon Sep 17 00:00:00 2001 From: Shuya Ma <87669292+shuyama1@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:57:05 -0700 Subject: [PATCH 21/33] =?UTF-8?q?fix=20TestAccPrivilegedAccessManagerEntit?= =?UTF-8?q?lement=5FprivilegedAccessManager=E2=80=A6=20(#10533)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...resource_privileged_access_manager_entitlement_test.go.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb b/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb index 466924a6445f..ea752221157c 100644 --- a/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb +++ b/mmv1/third_party/terraform/services/privilegedaccessmanager/resource_privileged_access_manager_entitlement_test.go.erb @@ -64,7 +64,7 @@ resource "google_privileged_access_manager_entitlement" "tfentitlement" { gcp_iam_access{ role_bindings{ role = "roles/storage.admin" - condition_expression = "request.time < timestamp("2024-04-23T18:30:00.000Z")" + condition_expression = "request.time < timestamp(\"2024-04-23T18:30:00.000Z\")" } resource = "//cloudresourcemanager.googleapis.com/projects/%{project_name}" resource_type = "cloudresourcemanager.googleapis.com/Project" @@ -108,7 +108,7 @@ resource "google_privileged_access_manager_entitlement" "tfentitlement" { gcp_iam_access{ role_bindings{ role = "roles/storage.admin" - condition_expression = "request.time < timestamp("2024-04-23T18:30:00.000Z")" + condition_expression = "request.time < timestamp(\"2024-04-23T18:30:00.000Z\")" } resource = "//cloudresourcemanager.googleapis.com/projects/%{project_name}" resource_type = "cloudresourcemanager.googleapis.com/Project" From 5d20188acdce85707b1b5cd0551c4b6f7d60dc9e Mon Sep 17 00:00:00 2001 From: delimaneto <167232526+delimaneto@users.noreply.github.com> Date: Thu, 25 Apr 2024 22:27:52 +0000 Subject: [PATCH 22/33] =?UTF-8?q?test:=20adding=20test=20for=20example=5Fg?= =?UTF-8?q?oogle=5Fcompute=5Fnetwork=5Fendpoint=5Fgroup=20r=E2=80=A6=20(#1?= =?UTF-8?q?0523)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...google_compute_network_endpoint_group.json | 36 ++++++++++++++ ...e_google_compute_network_endpoint_group.tf | 48 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.json create mode 100644 mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.tf diff --git a/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.json b/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.json new file mode 100644 index 000000000000..07edc9ebf61c --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.json @@ -0,0 +1,36 @@ +[ + { + "name": "//compute.googleapis.com/projects/{{.Provider.project}}/global/networks/neg-network", + "asset_type": "compute.googleapis.com/Network", + "resource": { + "version": "beta", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/beta/rest", + "discovery_name": "Network", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "autoCreateSubnetworks": false, + "name": "neg-network" + } + }, + "ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}" + }, + { + "name": "//compute.googleapis.com/projects/{{.Provider.project}}/regions/us-central1/subnetworks/neg-subnetwork", + "asset_type": "compute.googleapis.com/Subnetwork", + "resource": { + "version": "beta", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/compute/beta/rest", + "discovery_name": "Subnetwork", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "ipCidrRange": "10.0.0.0/16", + "logConfig": { + "enable": false + }, + "name": "neg-subnetwork", + "region": "projects/{{.Provider.project}}/global/regions/us-central1" + } + }, + "ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}" + } +] \ No newline at end of file diff --git a/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.tf b/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.tf new file mode 100644 index 000000000000..cb4e71ec24fe --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_google_compute_network_endpoint_group.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 3.84" + } + } +} + +provider "google" { + {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}} +} + +resource "google_compute_network_endpoint_group" "neg" { + name = "my-lb-neg" + network = google_compute_network.default.id + subnetwork = google_compute_subnetwork.default.id + default_port = "90" + zone = "us-central1-a" +} + +resource "google_compute_network" "default" { + name = "neg-network" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "default" { + name = "neg-subnetwork" + ip_cidr_range = "10.0.0.0/16" + region = "us-central1" + network = google_compute_network.default.id +} \ No newline at end of file From 75d1c6408307487c88296260f4a41c3b50a2da10 Mon Sep 17 00:00:00 2001 From: Jennifer Davis Date: Fri, 26 Apr 2024 05:00:33 -0700 Subject: [PATCH 23/33] fix:update resource template to remove beta and egree (#10536) --- .../terraform/examples/cloudrunv2_job_directvpc.tf.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmv1/templates/terraform/examples/cloudrunv2_job_directvpc.tf.erb b/mmv1/templates/terraform/examples/cloudrunv2_job_directvpc.tf.erb index 9b38d9722a59..d8117c68cd1a 100644 --- a/mmv1/templates/terraform/examples/cloudrunv2_job_directvpc.tf.erb +++ b/mmv1/templates/terraform/examples/cloudrunv2_job_directvpc.tf.erb @@ -1,7 +1,7 @@ resource "google_cloud_run_v2_job" "<%= ctx[:primary_resource_id] %>" { name = "<%= ctx[:vars]['cloud_run_job_name'] %>" location = "us-central1" - launch_stage = "BETA" + launch_stage = "GA" template { template{ containers { @@ -13,7 +13,6 @@ resource "google_cloud_run_v2_job" "<%= ctx[:primary_resource_id] %>" { subnetwork = "default" tags = ["tag1", "tag2", "tag3"] } - egress = "ALL_TRAFFIC" } } } From a63881aee1ba6d956d3d79d2b7e70bf0fda64f02 Mon Sep 17 00:00:00 2001 From: wj-chen Date: Fri, 26 Apr 2024 08:43:24 -0700 Subject: [PATCH 24/33] Fix the failing tests TestAccBigQueryJob_bigqueryJobCopy* (#10520) --- .../terraform/examples/bigquery_job_copy.tf.erb | 8 ++++++-- .../examples/bigquery_job_copy_table_reference.tf.erb | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mmv1/templates/terraform/examples/bigquery_job_copy.tf.erb b/mmv1/templates/terraform/examples/bigquery_job_copy.tf.erb index 1790003b1166..8025df9b886b 100644 --- a/mmv1/templates/terraform/examples/bigquery_job_copy.tf.erb +++ b/mmv1/templates/terraform/examples/bigquery_job_copy.tf.erb @@ -1,6 +1,10 @@ +locals { + count = 2 +} + resource "google_bigquery_table" "source" { deletion_protection = false - count = length(google_bigquery_dataset.source) + count = local.count dataset_id = google_bigquery_dataset.source[count.index].dataset_id table_id = "<%= ctx[:vars]['job_id'] %>_${count.index}_table" @@ -27,7 +31,7 @@ EOF } resource "google_bigquery_dataset" "source" { - count = 2 + count = local.count dataset_id = "<%= ctx[:vars]['job_id'] %>_${count.index}_dataset" friendly_name = "test" diff --git a/mmv1/templates/terraform/examples/bigquery_job_copy_table_reference.tf.erb b/mmv1/templates/terraform/examples/bigquery_job_copy_table_reference.tf.erb index 903e5b398d27..6112d5dee125 100644 --- a/mmv1/templates/terraform/examples/bigquery_job_copy_table_reference.tf.erb +++ b/mmv1/templates/terraform/examples/bigquery_job_copy_table_reference.tf.erb @@ -1,6 +1,10 @@ +locals { + count = 2 +} + resource "google_bigquery_table" "source" { deletion_protection = false - count = length(google_bigquery_dataset.source) + count = local.count dataset_id = google_bigquery_dataset.source[count.index].dataset_id table_id = "<%= ctx[:vars]['job_id'] %>_${count.index}_table" @@ -24,10 +28,12 @@ resource "google_bigquery_table" "source" { } ] EOF + + depends_on = [google_bigquery_dataset.source] } resource "google_bigquery_dataset" "source" { - count = 2 + count = local.count dataset_id = "<%= ctx[:vars]['job_id'] %>_${count.index}_dataset" friendly_name = "test" From bc5f7cc71b071a42736dc3f38cb7d6652789d36c Mon Sep 17 00:00:00 2001 From: Gustavo Kotarsky <167570715+kotarsky@users.noreply.github.com> Date: Fri, 26 Apr 2024 17:03:55 +0000 Subject: [PATCH 25/33] adding new tests for artifactregistry.googleapis.com/Repository (#10539) --- mmv1/templates/tgc/resource_converters.go.erb | 1 + .../example_artifact_registry_repository.json | 19 +++++++++++++++++++ .../example_artifact_registry_repository.tf | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.json create mode 100644 mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.tf diff --git a/mmv1/templates/tgc/resource_converters.go.erb b/mmv1/templates/tgc/resource_converters.go.erb index aad7ac4d6c6f..3fb28d691912 100644 --- a/mmv1/templates/tgc/resource_converters.go.erb +++ b/mmv1/templates/tgc/resource_converters.go.erb @@ -35,6 +35,7 @@ import ( // N:1 = [ResourceConverter{Convert: convertAbc, merge: mergeAbc}] (len=1) func ResourceConverters() map[string][]cai.ResourceConverter { return map[string][]cai.ResourceConverter{ + "google_artifact_registry_repository": {artifactregistry.ResourceConverterArtifactRegistryRepository()}, "google_compute_address": {compute.ResourceConverterComputeAddress()}, "google_compute_firewall": {compute.ResourceConverterComputeFirewall()}, "google_compute_disk": {compute.ResourceConverterComputeDisk()}, diff --git a/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.json b/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.json new file mode 100644 index 000000000000..8c2596d216a3 --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.json @@ -0,0 +1,19 @@ +[ + { + "name": "//artifactregistry.googleapis.com/projects/{{.Provider.project}}/locations/us-central1/repositories/my-repository", + "ancestry_path": "{{.Ancestry}}/project/{{.Provider.project}}", + "asset_type": "artifactregistry.googleapis.com/Repository", + "resource": { + "version": "v1", + "discovery_document_uri": "https://www.googleapis.com/discovery/v1/apis/artifactregistry/v1/rest", + "discovery_name": "Repository", + "parent": "//cloudresourcemanager.googleapis.com/projects/{{.Provider.project}}", + "data": { + "description": "example docker repository", + "format": "DOCKER", + "mode": "STANDARD_REPOSITORY" + } + }, + "ancestors": ["organizations/{{.OrgID}}"] + } +] \ No newline at end of file diff --git a/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.tf b/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.tf new file mode 100644 index 000000000000..6078dfb7d24a --- /dev/null +++ b/mmv1/third_party/tgc/tests/data/example_artifact_registry_repository.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google-beta" + version = "~> {{.Provider.version}}" + } + } +} + +provider "google" { + {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}} +} + +resource "google_artifact_registry_repository" "my-repo" { + location = "us-central1" + repository_id = "my-repository" + description = "example docker repository" + format = "DOCKER" +} \ No newline at end of file From 705f3f2a38cf4c367ff063863f466f20822ae81c Mon Sep 17 00:00:00 2001 From: Sarah French <15078782+SarahFrench@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:09:28 +0100 Subject: [PATCH 26/33] Add comment to `GoogleProviderConfig` test util function (#10541) --- mmv1/third_party/terraform/acctest/provider_test_utils.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mmv1/third_party/terraform/acctest/provider_test_utils.go b/mmv1/third_party/terraform/acctest/provider_test_utils.go index 14fc80072a09..651b6371f682 100644 --- a/mmv1/third_party/terraform/acctest/provider_test_utils.go +++ b/mmv1/third_party/terraform/acctest/provider_test_utils.go @@ -31,6 +31,9 @@ func init() { } } +// GoogleProviderConfig returns a configured SDKv2 provider. +// This function is typically used in CheckDestroy functions in acceptance tests. The provider client is used to make GET requests to check a resource is destroyed. +// Either a preexisting configured SDKv2 provider for the given test name is returned, or a new one is configured with empty (but non-nil) terraform.ResourceConfig func GoogleProviderConfig(t *testing.T) *transport_tpg.Config { configsLock.RLock() config, ok := configs[t.Name()] From 6004821d666f3267ef921326d4165cf3dcc39d8e Mon Sep 17 00:00:00 2001 From: "Stephen Lewis (Burrows)" Date: Mon, 29 Apr 2024 09:22:16 -0700 Subject: [PATCH 27/33] Request review after PR author comments (#10544) --- .github/workflows/request-reviewer.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/request-reviewer.yml b/.github/workflows/request-reviewer.yml index 870abb2b57d6..79bec4dd6ece 100644 --- a/.github/workflows/request-reviewer.yml +++ b/.github/workflows/request-reviewer.yml @@ -9,13 +9,13 @@ on: - ready_for_review - reopened - synchronize - branches: - - 'main' - - 'FEATURE-BRANCH-*' + issue_comment: + types: + - created jobs: request-review: - if: github.event.pull_request.draft == false + if: github.event.pull_request && github.event.pull_request.draft == false && (github.event.sender.login == github.event.pull_request.user.login || github.event.action != 'created') runs-on: ubuntu-latest permissions: pull-requests: write From 2abb74f38b67c029192384957da4a3fe9c9ce093 Mon Sep 17 00:00:00 2001 From: Damon Date: Mon, 29 Apr 2024 10:36:18 -0700 Subject: [PATCH 28/33] Update template parameters documentation (#10545) --- .../website/docs/r/dataflow_flex_template_job.html.markdown | 6 +++--- .../terraform/website/docs/r/dataflow_job.html.markdown | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mmv1/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown index eb068d14b591..f8b9c019bd3e 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown @@ -95,9 +95,9 @@ Template. * `autoscaling_algorithm` - (Optional) The algorithm to use for autoscaling. -* `parameters` - (Optional) Key/Value pairs to be passed to the Dataflow job (as -used in the template). Additional [pipeline options](https://cloud.google.com/dataflow/docs/guides/specifying-exec-params#setting-other-cloud-dataflow-pipeline-options) -such as `serviceAccount`, `workerMachineType`, etc can be specified here. +* `parameters` - **Template specific** Key/Value pairs to be forwarded to the pipeline's options; keys are + case-sensitive based on the language on which the pipeline is coded, mostly Java. + **Note**: do not configure Dataflow options here in parameters. * `enable_streaming_engine` - (Optional) Immutable. Indicates if the job should use the streaming engine feature. diff --git a/mmv1/third_party/terraform/website/docs/r/dataflow_job.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataflow_job.html.markdown index b33badf77885..be60472125bc 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataflow_job.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataflow_job.html.markdown @@ -99,7 +99,9 @@ The following arguments are supported: - - - -* `parameters` - (Optional) Key/Value pairs to be passed to the Dataflow job (as used in the template). +* `parameters` - **Template specific** Key/Value pairs to be forwarded to the pipeline's options; keys are + case-sensitive based on the language on which the pipeline is coded, mostly Java. + **Note**: do not configure Dataflow options here in parameters. * `labels` - (Optional) User labels to be specified for the job. Keys and values should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions) page. **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. Please refer to the field `effective_labels` for all of the labels present on the resource. From 552d835ed5ba412770659dc926e07c5625ca1506 Mon Sep 17 00:00:00 2001 From: Jean Bruno <57093332+jeanbrunogabriel@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:53:32 -0300 Subject: [PATCH 29/33] Update container_cluster: add MPS strategy on gpu_sharing block (#10471) --- .../terraform/website/docs/r/container_cluster.html.markdown | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index 00a90406f53a..be7a13e951f8 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -1032,6 +1032,7 @@ sole_tenant_config { * `gpu_sharing_strategy` (Required) - The type of GPU sharing strategy to enable on the GPU node. Accepted values are: * `"TIME_SHARING"`: Allow multiple containers to have [time-shared](https://cloud.google.com/kubernetes-engine/docs/concepts/timesharing-gpus) access to a single GPU device. + * `"MPS"`: Enable co-operative multi-process CUDA workloads to run concurrently on a single GPU device with [MPS](https://cloud.google.com/kubernetes-engine/docs/how-to/nvidia-mps-gpus) * `max_shared_clients_per_gpu` (Required) - The maximum number of containers that can share a GPU. From ce1b3f87897739b3339b3f20706fa539097316de Mon Sep 17 00:00:00 2001 From: Khaled Mohamed Refaat Date: Mon, 29 Apr 2024 22:21:32 +0200 Subject: [PATCH 30/33] [Composer] Reveal Image Version Updates to GA. (#10454) Co-authored-by: Khaled Hassan --- .../resource_composer_environment.go.erb | 11 +- .../resource_composer_environment_test.go.erb | 104 ++++++++++++++++++ 2 files changed, 107 insertions(+), 8 deletions(-) diff --git a/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb b/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb index 1164d8d7904d..e374bbf4724c 100644 --- a/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb +++ b/mmv1/third_party/terraform/services/composer/resource_composer_environment.go.erb @@ -170,13 +170,13 @@ func ResourceComposerEnvironment() *schema.Resource { tpgresource.DefaultProviderProject, tpgresource.DefaultProviderRegion, tpgresource.SetLabelsDiff, -<% unless version == "ga" -%> - customdiff.ForceNewIf("config.0.node_config.0.network", forceNewCustomDiff("config.0.node_config.0.network")), - customdiff.ForceNewIf("config.0.node_config.0.subnetwork", forceNewCustomDiff("config.0.node_config.0.subnetwork")), customdiff.Sequence( customdiff.ValidateChange("config.0.software_config.0.image_version", imageVersionChangeValidationFunc), versionValidationCustomizeDiffFunc, ), +<% unless version == "ga" -%> + customdiff.ForceNewIf("config.0.node_config.0.network", forceNewCustomDiff("config.0.node_config.0.network")), + customdiff.ForceNewIf("config.0.node_config.0.subnetwork", forceNewCustomDiff("config.0.node_config.0.subnetwork")), <% end -%> ), @@ -478,9 +478,6 @@ func ResourceComposerEnvironment() *schema.Resource { Type: schema.TypeString, Computed: true, Optional: true, -<% if version == "ga" -%> - ForceNew: true, -<% end -%> AtLeastOneOf: composerSoftwareConfigKeys, ValidateFunc: verify.ValidateRegexp(composerEnvironmentVersionRegexp), DiffSuppressFunc: composerImageVersionDiffSuppress, @@ -1291,7 +1288,6 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{}) } <% end -%> -<% unless version == "ga" -%> if d.HasChange("config.0.software_config.0.image_version") { patchObj := &composer.Environment{ Config: &composer.EnvironmentConfig{ @@ -1306,7 +1302,6 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{}) return err } } -<% end -%> if d.HasChange("config.0.software_config.0.scheduler_count") { patchObj := &composer.Environment{ diff --git a/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb b/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb index e2ee2668fb80..1f351dd85350 100644 --- a/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb +++ b/mmv1/third_party/terraform/services/composer/resource_composer_environment_test.go.erb @@ -491,6 +491,42 @@ func TestAccComposerEnvironment_ComposerV2(t *testing.T) { }) } +func TestAccComposerEnvironment_UpdateComposerV2ImageVersion(t *testing.T) { + t.Parallel() + + envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, acctest.RandInt(t)) + network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, acctest.RandInt(t)) + subnetwork := network + "-1" + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccComposerEnvironmentDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComposerEnvironment_composerV250(envName, network, subnetwork), + }, + { + Config: testAccComposerEnvironment_composerV260(envName, network, subnetwork), + }, + { + ResourceName: "google_composer_environment.test", + ImportState: true, + ImportStateVerify: true, + }, + // This is a terrible clean-up step in order to get destroy to succeed, + // due to dangling firewall rules left by the Composer Environment blocking network deletion. + // TODO(dzarmola): Remove this check if firewall rules bug gets fixed by Composer. + { + PlanOnly: true, + ExpectNonEmptyPlan: false, + Config: testAccComposerEnvironment_composerV260(envName, network, subnetwork), + Check: testAccCheckClearComposerEnvironmentFirewalls(t, network), + }, + }, + }) +} + func TestAccComposerEnvironment_UpdateComposerV2ResilienceMode(t *testing.T) { t.Parallel() @@ -2152,6 +2188,74 @@ resource "google_compute_subnetwork" "test" { `, envName, network, subnetwork) } +func testAccComposerEnvironment_composerV250(envName, network, subnetwork string) string { + return fmt.Sprintf(` +resource "google_composer_environment" "test" { + name = "%s" + region = "us-east1" + + config { + node_config { + network = google_compute_network.test.self_link + subnetwork = google_compute_subnetwork.test.self_link + } + + software_config { + image_version = "composer-2.5.0-airflow-2.6.3" + } + } +} + +resource "google_compute_network" "test" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "test" { + name = "%s" + ip_cidr_range = "10.2.0.0/16" + region = "us-east1" + network = google_compute_network.test.self_link + private_ip_google_access = true +} + +`, envName, network, subnetwork) +} + +func testAccComposerEnvironment_composerV260(envName, network, subnetwork string) string { + return fmt.Sprintf(` +resource "google_composer_environment" "test" { + name = "%s" + region = "us-east1" + + config { + node_config { + network = google_compute_network.test.self_link + subnetwork = google_compute_subnetwork.test.self_link + } + + software_config { + image_version = "composer-2.6.0-airflow-2.6.3" + } + } +} + +resource "google_compute_network" "test" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "test" { + name = "%s" + ip_cidr_range = "10.2.0.0/16" + region = "us-east1" + network = google_compute_network.test.self_link + private_ip_google_access = true +} + +`, envName, network, subnetwork) +} + func testAccComposerEnvironment_composerV2HighResilience(envName, network, subnetwork string) string { return fmt.Sprintf(` resource "google_composer_environment" "test" { From ce9e306a030d2a22e86dd4f108c98dbd62c47661 Mon Sep 17 00:00:00 2001 From: Nick Elliot Date: Mon, 29 Apr 2024 13:27:31 -0700 Subject: [PATCH 31/33] add Read command to go template (#10550) --- mmv1/api/resource.go | 17 +- mmv1/templates/terraform/resource.go.tmpl | 260 +++++++++++++++++++++- 2 files changed, 270 insertions(+), 7 deletions(-) diff --git a/mmv1/api/resource.go b/mmv1/api/resource.go index 54fb2336796a..64de022edd3d 100644 --- a/mmv1/api/resource.go +++ b/mmv1/api/resource.go @@ -964,13 +964,18 @@ func (r Resource) GetIdFormat() string { // ==================== // Template Methods // ==================== +// Functions used to create slices of resource properties that could not otherwise be called from within generating templates. -// Prints a dot notation path to where the field is nested within the parent -// object when called on a property. eg: parent.meta.label.foo -// Redefined on Resource to terminate the calls up the parent chain. +func (r Resource) ReadProperties() []*Type { + return google.Reject(r.GettableProperties(), func(p *Type) bool { + return p.IgnoreRead + }) +} -// checks a resource for if it has properties that have FlattenObject=true on fields where IgnoreRead=false -// used to decide whether or not to import "google.golang.org/api/googleapi" func (r Resource) FlattenedProperties() []*Type { - return google.Select(google.Reject(r.GettableProperties(), func(p *Type) bool { return p.IgnoreRead }), func(p *Type) bool { return p.FlattenObject }) + return google.Select(r.ReadProperties(), func(p *Type) bool { + return p.FlattenObject + }) } + + diff --git a/mmv1/templates/terraform/resource.go.tmpl b/mmv1/templates/terraform/resource.go.tmpl index ac5cc5a2d16d..c02558a7367f 100644 --- a/mmv1/templates/terraform/resource.go.tmpl +++ b/mmv1/templates/terraform/resource.go.tmpl @@ -227,7 +227,8 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ } log.Printf("[DEBUG] Creating new {{ $.Name -}}: %#v", obj) -{{if $.NestedQuery.ModifyByPatch -}} +{{if $.NestedQuery -}} +{{if $.NestedQuery.ModifyByPatch }} {{/*# Keep this after mutex - patch request data relies on current resource state */}} obj, err = resource{{ $.ResourceName -}}PatchCreateEncoder(d, meta, obj) if err != nil { @@ -239,6 +240,7 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ return err } {{- end}} +{{- end}} {{- end}} billingProject := "" @@ -337,6 +339,7 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ } {{- end}} +{{if $.NestedQuery -}} {{if $.NestedQuery.Keys -}} if _, ok := opRes["{{ index $.NestedQuery.Keys 0 -}}"]; ok { opRes, err = flattenNested{{ $.ResourceName -}}(d, meta, opRes) @@ -356,6 +359,7 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ return err } {{- end}} +{{- end}} {{- end}} // This may have caused the ID to update - update it if so. @@ -409,3 +413,257 @@ func resource{{ $.ResourceName -}}Create(d *schema.ResourceData, meta interface{ return resource{{ $.ResourceName -}}Read(d, meta) {{ end -}} } + +{{if ($.GetAsync.IsA "OpAsync")}} +func resource{{ $.ResourceName -}}PollRead(d *schema.ResourceData, meta interface{}) transport_tpg.PollReadFunc { + return func() (map[string]interface{}, error) { +{{if $.GetAsync.CustomPollRead -}} +//TODO custom poll read +{{ else -}} + config := meta.(*transport_tpg.Config) + + + url, err := tpgresource.ReplaceVars{{if $.LegacyLongFormProject -}}ForId{{ end -}}(d, config, "{{"{{"}}{{$.ProductMetadata.Name}}BasePath{{"}}"}}/{{$.SelfLinkUri}}") + + if err != nil { + return nil, err + } + + billingProject := "" + +{{if $.HasProject -}} + project, err := tpgresource.GetProject(d, config) + if err != nil { + return nil, fmt.Errorf("Error fetching project for {{ $.Name -}}: %s", err) + } +{{if $.LegacyLongFormProject -}} + billingProject = strings.TrimPrefix(project, "projects/") +{{ else -}} + billingProject = project +{{- end}} +{{- end}} + +{{if $.SupportsIndirectUserProjectOverride -}} + if parts := regexp.MustCompile(`projects\/([^\/]+)\/`).FindStringSubmatch(url); parts != nil { + billingProject = parts[1] + } +{{- end}} + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return nil, err + } + + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "{{ upper $.ReadVerb -}}", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, +{{if $.ErrorRetryPredicates -}} + ErrorRetryPredicates: []transport_tpg.RetryErrorPredicateFunc{{"{"}}{{ join $.ErrorRetryPredicates "," -}}{{"}"}}, +{{- end}} +{{if $.ErrorAbortPredicates -}} + ErrorAbortPredicates: []transport_tpg.RetryErrorPredicateFunc{{"{"}}{{ join $.ErrorRetryPredicates "," -}}{{"}"}}, +{{- end}} + }) + if err != nil { + return res, err + } +{{if $.NestedQuery -}} + res, err = flattenNested{{ $.ResourceName -}}(d, meta, res) + if err != nil { + return nil, err + } + + if res == nil { + return nil, tpgresource.Fake404("nested", "{{ $.ResourceName }}") + } + +{{- end}} +{{if $.CustomCode.Decoder -}} + res, err = resource{{ $.ResourceName -}}Decoder(d, meta, res) + if err != nil { + return nil, err + } + if res == nil { + return nil, tpgresource.Fake404("decoded", "{{ $.ResourceName }}") + } + +{{- end}} + return res, nil +{{ end -}} + } +} +{{ end -}} + +func resource{{ $.ResourceName -}}Read(d *schema.ResourceData, meta interface{}) error { +{{if $.SkipRead -}} + // This resource could not be read from the API. + return nil +{{ else -}} + config := meta.(*transport_tpg.Config) + userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent) + if err != nil { + return err + } + + url, err := tpgresource.ReplaceVars{{if $.LegacyLongFormProject -}}ForId{{ end -}}(d, config, "{{"{{"}}{{$.ProductMetadata.Name}}BasePath{{"}}"}}/{{$.SelfLinkUri}}{{$.ReadQueryParams}}") + if err != nil { + return err + } + + billingProject := "" + +{{if $.HasProject -}} + project, err := tpgresource.GetProject(d, config) + if err != nil { + return fmt.Errorf("Error fetching project for {{ $.Name -}}: %s", err) + } +{{if $.LegacyLongFormProject -}} + billingProject = strings.TrimPrefix(project, "projects/") +{{ else -}} + billingProject = project +{{- end}} +{{- end}} + +{{if $.SupportsIndirectUserProjectOverride -}} + if parts := regexp.MustCompile(`projects\/([^\/]+)\/`).FindStringSubmatch(url); parts != nil { + billingProject = parts[1] + } +{{- end}} + + // err == nil indicates that the billing_project value was found + if bp, err := tpgresource.GetBillingProject(d, config); err == nil { + billingProject = bp + } + + headers := make(http.Header) + {{ if $.CustomCode.PreRead -}} + //todo preread + {{ end }} + res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ + Config: config, + Method: "{{ upper $.ReadVerb -}}", + Project: billingProject, + RawURL: url, + UserAgent: userAgent, +{{if $.ErrorRetryPredicates -}} + ErrorRetryPredicates: []transport_tpg.RetryErrorPredicateFunc{{"{"}}{{ join $.ErrorRetryPredicates "," -}}{{"}"}}, +{{- end}} +{{if $.ErrorAbortPredicates -}} + ErrorAbortPredicates: []transport_tpg.RetryErrorPredicateFunc{{"{"}}{{ join $.ErrorRetryPredicates "," -}}{{"}"}}, +{{- end}} + }) + if err != nil { +{{if $.ReadErrorTransform -}} + return transport_tpg.HandleNotFoundError({{ $.ReadErrorTransform }}(err), d, fmt.Sprintf("{{ $.ResourceName -}} %q", d.Id())) +{{ else -}} + return transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("{{ $.ResourceName -}} %q", d.Id())) +{{- end}} + } + +{{if $.NestedQuery -}} + res, err = flattenNested{{ $.ResourceName -}}(d, meta, res) + if err != nil { + return err + } + + if res == nil { + // Object isn't there any more - remove it from the state. + log.Printf("[DEBUG] Removing {{ $.ResourceName -}} because it couldn't be matched.") + d.SetId("") + return nil + } + +{{- end}} + +{{if $.CustomCode.Decoder -}} + res, err = resource{{ $.ResourceName -}}Decoder(d, meta, res) + if err != nil { + return err + } + + if res == nil { + // Decoding the $ has resulted in it being gone. It may be marked deleted + log.Printf("[DEBUG] Removing {{ $.ResourceName -}} because it no longer exists.") + d.SetId("") + return nil + } + +{{- end}} +{{- if $.VirtualFields -}} + // Explicitly set virtual fields to default values if unset +{{- range $prop := $.VirtualFields }} +{{ if $prop.DefaultValue -}} + if _, ok := d.GetOkExists("{{ $prop.Name -}}"); !ok { + if err := d.Set("{{ $prop.Name -}}", {{ $prop.DefaultValue -}}); err != nil { + return fmt.Errorf("Error setting {{ $prop.Name -}}: %s", err) + } + } +{{- end}} +{{- end}} +{{- end}} +{{if $.HasProject -}} + if err := d.Set("project", project); err != nil { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", err) + } +{{- end}} + +{{if $.HasRegion -}} + region, err := tpgresource.GetRegion(d, config) + if err != nil { + return err + } + if err := d.Set("region", region); err != nil { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", err) + } +{{- end}} + +{{if $.HasZone -}} + zone, err := tpgresource.GetZone(d, config) + if err != nil { + return err + } + if err := d.Set("zone", zone); err != nil { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", err) + } +{{- end}} + +{{- range $prop := $.ReadProperties }} +{{if $prop.FlattenObject -}} +// Terraform must set the top level schema field, but since this $ contains collapsed properties +// it's difficult to know what the top level should be. Instead we just loop over the map returned from flatten. + if flattenedProp := flatten{{ if $.NestedQuery -}}Nested{{end}}{{ $.ResourceName -}}{{ camelize $prop.Name "upper" -}}(res["{{ $prop.ApiName -}}"], d, config); flattenedProp != nil { + if gerr, ok := flattenedProp.(*googleapi.Error); ok { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", gerr) + } + casted := flattenedProp.([]interface{})[0] + if casted != nil { + for k, v := range casted.(map[string]interface{}) { + if err := d.Set(k, v); err != nil { + return fmt.Errorf("Error setting %s: %s", k, err) + } + } + } + } +{{ else -}} + if err := d.Set("{{ underscore $prop.Name -}}", flatten{{ if $.NestedQuery -}}Nested{{end}}{{ $.ResourceName -}}{{ camelize $prop.Name "upper" -}}(res["{{ $prop.ApiName -}}"], d, config)); err != nil { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", err) + } +{{- end}} +{{- end}} +{{if $.HasSelfLink -}} + if err := d.Set("self_link", tpgresource.ConvertSelfLinkToV1(res["selfLink"].(string))); err != nil { + return fmt.Errorf("Error reading {{ $.Name -}}: %s", err) + } +{{- end}} + + return nil +{{ end -}} +} From 02fc8673d093b1b76f33cbd7334d248bf3a05773 Mon Sep 17 00:00:00 2001 From: Zhenhua Li Date: Mon, 29 Apr 2024 14:36:07 -0700 Subject: [PATCH 32/33] Revert "Set environment variable for tpu v2 vm (#9788)" (#10555) --- .ci/gcb-generate-diffs-new.yml | 4 +--- .ci/gcb-push-downstream.yml | 3 --- .ci/gcb-vcr-nightly.yml | 4 +--- .ci/magician/cmd/check_cassettes.go | 1 - .ci/magician/cmd/test_terraform_vcr.go | 1 - 5 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.ci/gcb-generate-diffs-new.yml b/.ci/gcb-generate-diffs-new.yml index 620645e69411..34d8c4ecd970 100644 --- a/.ci/gcb-generate-diffs-new.yml +++ b/.ci/gcb-generate-diffs-new.yml @@ -255,7 +255,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: gcb-tpg-vcr-test entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"] + secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"] waitFor: ["diff"] env: - BASE_BRANCH=$_BASE_BRANCH @@ -318,5 +318,3 @@ availableSecrets: env: SA_KEY - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION - - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest - env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml index 9188e749a00d..e351011f9f2b 100644 --- a/.ci/gcb-push-downstream.yml +++ b/.ci/gcb-push-downstream.yml @@ -206,7 +206,6 @@ steps: - "GOOGLE_SERVICE_ACCOUNT" - "SA_KEY" - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION" - - "GOOGLE_TPU_V2_VM_RUNTIME_VERSION" env: - "COMMIT_SHA=$COMMIT_SHA" - "GOOGLE_REGION=us-central1" @@ -253,5 +252,3 @@ availableSecrets: env: SA_KEY - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION - - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest - env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION diff --git a/.ci/gcb-vcr-nightly.yml b/.ci/gcb-vcr-nightly.yml index 3664d6312d34..ee48902cb8c5 100644 --- a/.ci/gcb-vcr-nightly.yml +++ b/.ci/gcb-vcr-nightly.yml @@ -3,7 +3,7 @@ steps: - name: 'gcr.io/graphite-docker-images/go-plus' id: gcb-vcr-nightly entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh' - secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"] + secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"] args: - $BUILD_ID @@ -41,5 +41,3 @@ availableSecrets: env: SA_KEY - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION - - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest - env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION diff --git a/.ci/magician/cmd/check_cassettes.go b/.ci/magician/cmd/check_cassettes.go index d31cebe44ddd..e19f3f7c9ef3 100644 --- a/.ci/magician/cmd/check_cassettes.go +++ b/.ci/magician/cmd/check_cassettes.go @@ -28,7 +28,6 @@ var ccEnvironmentVariables = [...]string{ "GOOGLE_REGION", "GOOGLE_SERVICE_ACCOUNT", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", - "GOOGLE_TPU_V2_VM_RUNTIME_VERSION", "GOOGLE_ZONE", "PATH", "SA_KEY", diff --git a/.ci/magician/cmd/test_terraform_vcr.go b/.ci/magician/cmd/test_terraform_vcr.go index c3e0b54037be..74a37c5bbe12 100644 --- a/.ci/magician/cmd/test_terraform_vcr.go +++ b/.ci/magician/cmd/test_terraform_vcr.go @@ -30,7 +30,6 @@ var ttvEnvironmentVariables = [...]string{ "GOOGLE_REGION", "GOOGLE_SERVICE_ACCOUNT", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", - "GOOGLE_TPU_V2_VM_RUNTIME_VERSION", "GOOGLE_ZONE", "HOME", "PATH", From 2832e16ced51f640c45808a98a773b829d4f0542 Mon Sep 17 00:00:00 2001 From: Zhenhua Li Date: Mon, 29 Apr 2024 15:47:46 -0700 Subject: [PATCH 33/33] Rewrite the code to handle handwritten resources (#10542) --- mmv1/api/resource.go | 2 +- mmv1/main.go | 30 +- mmv1/provider/template_data.go | 4 - mmv1/provider/terraform.go | 1265 ++++++++++------- .../third_party/terraform/.copywrite.hcl.tmpl | 33 + .../terraform/.goreleaser.yml.tmpl | 83 ++ mmv1/third_party/terraform/go.mod | 110 ++ mmv1/third_party/terraform/main.go.tmpl | 60 + .../terraform/release-metadata.hcl.tmpl | 2 + ...urce_container_analysis_occurrence_test.go | 2 +- ...yload.json.tmpl => generated_payload.json} | 0 ...urce_deployment_manager_deployment_test.go | 4 +- ...e_account.yml.tmpl => service_account.yml} | 0 .../terraform-registry-manifest.json | 6 + .../provider_versionfive_upgrade_test.go.erb | 646 --------- ...compute_firewall_policy_rule.html.markdown | 207 --- 16 files changed, 1063 insertions(+), 1391 deletions(-) create mode 100644 mmv1/third_party/terraform/.copywrite.hcl.tmpl create mode 100644 mmv1/third_party/terraform/.goreleaser.yml.tmpl create mode 100644 mmv1/third_party/terraform/go.mod create mode 100644 mmv1/third_party/terraform/main.go.tmpl create mode 100644 mmv1/third_party/terraform/release-metadata.hcl.tmpl rename mmv1/third_party/terraform/services/containeranalysis/test-fixtures/{generated_payload.json.tmpl => generated_payload.json} (100%) rename mmv1/third_party/terraform/services/deploymentmanager/test-fixtures/{service_account.yml.tmpl => service_account.yml} (100%) create mode 100644 mmv1/third_party/terraform/terraform-registry-manifest.json delete mode 100644 mmv1/third_party/terraform/utils/provider_versionfive_upgrade_test.go.erb delete mode 100644 mmv1/third_party/terraform/website/docs/r/compute_firewall_policy_rule.html.markdown diff --git a/mmv1/api/resource.go b/mmv1/api/resource.go index 64de022edd3d..fd65b71b6b9b 100644 --- a/mmv1/api/resource.go +++ b/mmv1/api/resource.go @@ -132,7 +132,7 @@ type Resource struct { // // [Optional] (Api::Resource::IamPolicy) Configuration of a resource's // resource-specific IAM Policy. - IamPolicy resource.IamPolicy `yaml:"iam_policy"` + IamPolicy *resource.IamPolicy `yaml:"iam_policy"` // [Optional] If set to true, don't generate the resource itself; only // generate the IAM policy. diff --git a/mmv1/main.go b/mmv1/main.go index 2167e309793c..59088cf3bb2d 100644 --- a/mmv1/main.go +++ b/mmv1/main.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sort" "strings" + "time" "golang.org/x/exp/slices" @@ -68,6 +69,7 @@ func main() { log.Fatalf("No product.yaml file found.") } + startTime := time.Now() log.Printf("Generating MM output to '%s'", *outputPath) log.Printf("Using %s version", *version) @@ -80,6 +82,7 @@ func main() { return false }) + var productsForVersion []map[string]interface{} for _, productName := range allProductFiles { productYamlPath := path.Join(productName, "go_product.yaml") @@ -137,7 +140,7 @@ func main() { productApi.Validate() // TODO Q2: set other providers via flag - providerToGenerate := provider.NewTerraform(productApi, *version) + providerToGenerate := provider.NewTerraform(productApi, *version, startTime) if !slices.Contains(productsToGenerate, productName) { log.Printf("%s not specified, skipping generation", productName) @@ -146,8 +149,33 @@ func main() { log.Printf("%s: Generating files", productName) providerToGenerate.Generate(*outputPath, productName, generateCode, generateDocs) + + // we need to preserve a single provider instance to use outside of this loop. + productsForVersion = append(productsForVersion, map[string]interface{}{ + "Definitions": productApi, + "Provider": providerToGenerate, + }) } // TODO Q2: copy common files } + + slices.SortFunc(productsForVersion, func(p1, p2 map[string]interface{}) int { + return strings.Compare(strings.ToLower(p1["Definitions"].(*api.Product).Name), strings.ToLower(p2["Definitions"].(*api.Product).Name)) + }) + + // In order to only copy/compile files once per provider this must be called outside + // of the products loop. This will get called with the provider from the final iteration + // of the loop + finalProduct := productsForVersion[len(productsForVersion)-1] + provider := finalProduct["Provider"].(*provider.Terraform) + + provider.CopyCommonFiles(*outputPath, generateCode, generateDocs) + + log.Printf("Compiling common files for terraform") + if generateCode { + provider.CompileCommonFiles(*outputPath, productsForVersion, "") + + // TODO Q2: product overrides + } } diff --git a/mmv1/provider/template_data.go b/mmv1/provider/template_data.go index 15f9e0ba84d9..4dd8a2126c4d 100644 --- a/mmv1/provider/template_data.go +++ b/mmv1/provider/template_data.go @@ -161,10 +161,6 @@ func (td *TemplateData) GenerateFile(filePath, templatePath string, input any, g } sourceByte := contents.Bytes() - // Replace import path based on version (beta/alpha) - if td.TerraformResourceDirectory != "google" { - sourceByte = bytes.Replace(sourceByte, []byte("github.com/hashicorp/terraform-provider-google/google"), []byte(td.TerraformProviderModule+"/"+td.TerraformResourceDirectory), -1) - } // if goFormat { // sourceByte, err = format.Source(sourceByte) diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go index 0e3b926f5f97..7a08685141c7 100644 --- a/mmv1/provider/terraform.go +++ b/mmv1/provider/terraform.go @@ -14,12 +14,19 @@ package provider import ( + "bytes" + "errors" "fmt" + "go/format" + "io/fs" "log" + "maps" "os" "path" + "path/filepath" "reflect" "strings" + "time" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api" "github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product" @@ -38,26 +45,30 @@ type Terraform struct { IAMResourceCount int - ResourcesForVersion []api.Resource + ResourcesForVersion []map[string]string TargetVersionName string Version product.Version Product api.Product + + StartTime time.Time } -func NewTerraform(product *api.Product, versionName string) *Terraform { +func NewTerraform(product *api.Product, versionName string, startTime time.Time) *Terraform { t := Terraform{ ResourceCount: 0, IAMResourceCount: 0, Product: *product, TargetVersionName: versionName, - Version: *product.VersionObjOrClosest(versionName)} + Version: *product.VersionObjOrClosest(versionName), + StartTime: startTime, + } t.Product.SetPropertiesBasedOnVersion(&t.Version) for _, r := range t.Product.Objects { - r.SetCompiler(reflect.TypeOf(t).Name()) + r.SetCompiler(t.providerName()) } return &t @@ -222,571 +233,767 @@ func (t *Terraform) FullResourceName(object api.Resource) string { return fmt.Sprintf("%s_%s", productName, name) } +// def copy_common_files(output_folder, generate_code, generate_docs, provider_name = nil) +func (t Terraform) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) { + log.Printf("Copying common files for %s", t.providerName()) + + files := t.getCommonCopyFiles(t.TargetVersionName, generateCode, generateDocs) + t.CopyFileList(outputFolder, files) +} + +// To compile a new folder, add the folder to foldersCopiedToRootDir or foldersCopiedToGoogleDir. +// To compile a file, add the file to singleFiles +func (t Terraform) getCommonCopyFiles(versionName string, generateCode, generateDocs bool) map[string]string { + // key is the target file and value is the source file + commonCopyFiles := make(map[string]string, 0) + + // Case 1: When copy all of files except .tmpl in a folder to the root directory of downstream repository, + // save the folder name to foldersCopiedToRootDir + foldersCopiedToRootDir := []string{"third_party/terraform/META.d", "third_party/terraform/version"} + // Copy TeamCity-related Kotlin & Markdown files to TPG only, not TPGB + if versionName == "ga" { + foldersCopiedToRootDir = append(foldersCopiedToRootDir, "third_party/terraform/.teamcity") + } + if generateCode { + foldersCopiedToRootDir = append(foldersCopiedToRootDir, "third_party/terraform/scripts") + } + if generateDocs { + foldersCopiedToRootDir = append(foldersCopiedToRootDir, "third_party/terraform/website") + } + for _, folder := range foldersCopiedToRootDir { + files := t.getCopyFilesInFolder(folder, ".") + maps.Copy(commonCopyFiles, files) + } + + // Case 2: When copy all of files except .tmpl in a folder to the google directory of downstream repository, + // save the folder name to foldersCopiedToGoogleDir + var foldersCopiedToGoogleDir []string + if generateCode { + foldersCopiedToGoogleDir = []string{"third_party/terraform/services", "third_party/terraform/acctest", "third_party/terraform/sweeper", "third_party/terraform/provider", "third_party/terraform/tpgdclresource", "third_party/terraform/tpgiamresource", "third_party/terraform/tpgresource", "third_party/terraform/transport", "third_party/terraform/fwmodels", "third_party/terraform/fwprovider", "third_party/terraform/fwtransport", "third_party/terraform/fwresource", "third_party/terraform/verify", "third_party/terraform/envvar", "third_party/terraform/functions", "third_party/terraform/test-fixtures"} + } + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + // Copy files to google(or google-beta or google-private) folder in downstream + for _, folder := range foldersCopiedToGoogleDir { + files := t.getCopyFilesInFolder(folder, googleDir) + maps.Copy(commonCopyFiles, files) + } + + // Case 3: When copy a single file, save the target as key and source as value to the map singleFiles + singleFiles := map[string]string{ + "go.sum": "third_party/terraform/go.sum", + "go.mod": "third_party/terraform/go.mod", + ".go-version": "third_party/terraform/.go-version", + "terraform-registry-manifest.json": "third_party/terraform/terraform-registry-manifest.json", + } + maps.Copy(commonCopyFiles, singleFiles) + + return commonCopyFiles +} + +func (t Terraform) getCopyFilesInFolder(folderPath, targetDir string) map[string]string { + m := make(map[string]string, 0) + filepath.WalkDir(folderPath, func(path string, di fs.DirEntry, err error) error { + if !di.IsDir() && !strings.HasSuffix(di.Name(), ".tmpl") && !strings.HasSuffix(di.Name(), ".erb") { + fname := strings.TrimPrefix(path, "third_party/terraform/") + target := fname + if targetDir != "." { + target = fmt.Sprintf("%s/%s", targetDir, fname) + } + m[target] = path + } + return nil + }) + + return m +} + +// def copy_file_list(output_folder, files) +func (t Terraform) CopyFileList(outputFolder string, files map[string]string) { + for target, source := range files { + targetFile := filepath.Join(outputFolder, target) + targetDir := filepath.Dir(targetFile) + + if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { + log.Println(fmt.Errorf("error creating output directory %v: %v", targetDir, err)) + } + // If we've modified a file since starting an MM run, it's a reasonable + // assumption that it was this run that modified it. + if info, err := os.Stat(targetFile); !errors.Is(err, os.ErrNotExist) && t.StartTime.Before(info.ModTime()) { + log.Fatalf("%s was already modified during this run at %s", targetFile, info.ModTime().String()) + } + + sourceByte, err := os.ReadFile(source) + if err != nil { + log.Fatalf("Cannot read source file %s while copying: %s", source, err) + } + + err = os.WriteFile(targetFile, sourceByte, 0644) + if err != nil { + log.Fatalf("Cannot write target file %s while copying: %s", target, err) + } + + // Replace import path based on version (beta/alpha) + if filepath.Ext(target) == ".go" || filepath.Ext(target) == ".mod" { + t.replaceImportPath(outputFolder, target) + } + + if filepath.Ext(target) == ".go" { + t.addHashicorpCopyRightHeader(outputFolder, target) + } + } +} + +// Compiles files that are shared at the provider level +// +// def compile_common_files( +// output_folder, +// products, +// common_compile_file, +// override_path = nil +// ) +func (t Terraform) CompileCommonFiles(outputFolder string, products []map[string]interface{}, overridePath string) { + t.generateResourcesForVersion(products) + files := t.getCommonCompileFiles(t.TargetVersionName) + templateData := NewTemplateData(outputFolder, t.Version) + t.CompileFileList(outputFolder, files, *templateData) +} + +// To compile a new folder, add the folder to foldersCompiledToRootDir or foldersCompiledToGoogleDir. +// To compile a file, add the file to singleFiles +func (t Terraform) getCommonCompileFiles(versionName string) map[string]string { + // key is the target file and the value is the source file + commonCompileFiles := make(map[string]string, 0) + + // Case 1: When compile all of files except .tmpl in a folder to the root directory of downstream repository, + // save the folder name to foldersCopiedToRootDir + foldersCompiledToRootDir := []string{"third_party/terraform/scripts"} + for _, folder := range foldersCompiledToRootDir { + files := t.getCompileFilesInFolder(folder, ".") + maps.Copy(commonCompileFiles, files) + } + + // Case 2: When compile all of files except .tmpl in a folder to the google directory of downstream repository, + // save the folder name to foldersCopiedToGoogleDir + foldersCompiledToGoogleDir := []string{"third_party/terraform/services", "third_party/terraform/acctest", "third_party/terraform/sweeper", "third_party/terraform/provider", "third_party/terraform/tpgdclresource", "third_party/terraform/tpgiamresource", "third_party/terraform/tpgresource", "third_party/terraform/transport", "third_party/terraform/fwmodels", "third_party/terraform/fwprovider", "third_party/terraform/fwtransport", "third_party/terraform/fwresource", "third_party/terraform/verify", "third_party/terraform/envvar", "third_party/terraform/functions", "third_party/terraform/test-fixtures"} + googleDir := "google" + if versionName != "ga" { + googleDir = fmt.Sprintf("google-%s", versionName) + } + for _, folder := range foldersCompiledToGoogleDir { + files := t.getCompileFilesInFolder(folder, googleDir) + maps.Copy(commonCompileFiles, files) + } + + // Case 3: When compile a single file, save the target as key and source as value to the map singleFiles + singleFiles := map[string]string{ + "main.go": "third_party/terraform/main.go.tmpl", + ".goreleaser.yml": "third_party/terraform/.goreleaser.yml.tmpl", + ".release/release-metadata.hcl": "third_party/terraform/release-metadata.hcl.tmpl", + ".copywrite.hcl": "third_party/terraform/.copywrite.hcl.tmpl", + } + maps.Copy(commonCompileFiles, singleFiles) + + return commonCompileFiles +} + +func (t Terraform) getCompileFilesInFolder(folderPath, targetDir string) map[string]string { + m := make(map[string]string, 0) + filepath.WalkDir(folderPath, func(path string, di fs.DirEntry, err error) error { + if !di.IsDir() && strings.HasSuffix(di.Name(), ".tmpl") { + fname := strings.TrimPrefix(path, "third_party/terraform/") + fname = strings.TrimSuffix(fname, ".tmpl") + target := fname + if targetDir != "" { + target = fmt.Sprintf("%s/%s", targetDir, fname) + } + m[target] = path + } + return nil + }) + + return m +} + +// def compile_file_list(output_folder, files, file_template, pwd = Dir.pwd) +func (t Terraform) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData) { + if err := os.MkdirAll(outputFolder, os.ModePerm); err != nil { + log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err)) + } + + // TODO: is this needed? + // err := os.Chdir(outputFolder) + // if err != nil { + // log.Fatalf("Could not move into the directory %s", outputFolder) + // } + + for target, source := range files { + targetFile := filepath.Join(outputFolder, target) + targetDir := filepath.Dir(targetFile) + if err := os.MkdirAll(targetDir, os.ModePerm); err != nil { + log.Println(fmt.Errorf("error creating output directory %v: %v", targetDir, err)) + } + + templates := []string{ + source, + } + + fileTemplate.GenerateFile(targetFile, source, t, true, templates...) + t.replaceImportPath(outputFolder, target) + t.addHashicorpCopyRightHeader(outputFolder, target) + } + // TODO: is this needed? + // Dir.chdir pwd +} + +// def add_hashicorp_copyright_header(output_folder, target) +func (t Terraform) addHashicorpCopyRightHeader(outputFolder, target string) { + if !expectedOutputFolder(outputFolder) { + log.Printf("Unexpected output folder (%s) detected"+ + "when deciding to add HashiCorp copyright headers.\n"+ + "Watch out for unexpected changes to copied files", outputFolder) + } + // only add copyright headers when generating TPG and TPGB + if !(strings.HasSuffix(outputFolder, "terraform-provider-google") || strings.HasSuffix(outputFolder, "terraform-provider-google-beta")) { + return + } + + // Prevent adding copyright header to files with paths or names matching the strings below + // NOTE: these entries need to match the content of the .copywrite.hcl file originally + // created in https://github.com/GoogleCloudPlatform/magic-modules/pull/7336 + // The test-fixtures folder is not included here as it's copied as a whole, + // not file by file + ignoredFolders := []string{".release/", ".changelog/", "examples/", "scripts/", "META.d/"} + ignoredFiles := []string{"go.mod", ".goreleaser.yml", ".golangci.yml", "terraform-registry-manifest.json"} + shouldAddHeader := true + for _, folder := range ignoredFolders { + // folder will be path leading to file + if strings.HasPrefix(target, folder) { + shouldAddHeader = false + break + } + } + if !shouldAddHeader { + return + } + + for _, file := range ignoredFiles { + // file will be the filename and extension, with no preceding path + if strings.HasSuffix(target, file) { + shouldAddHeader = false + break + } + } + if !shouldAddHeader { + return + } + + lang := languageFromFilename(target) + // Some file types we don't want to add headers to + // e.g. .sh where headers are functional + // Also, this guards against new filetypes being added and triggering build errors + if lang == "unsupported" { + return + } + + // File is not ignored and is appropriate file type to add header to + copyrightHeader := []string{"Copyright (c) HashiCorp, Inc.", "SPDX-License-Identifier: MPL-2.0"} + header := commentBlock(copyrightHeader, lang) + + targetFile := filepath.Join(outputFolder, target) + sourceByte, err := os.ReadFile(targetFile) + if err != nil { + log.Fatalf("Cannot read file %s to add Hashicorp copy right: %s", targetFile, err) + } + + sourceByte = google.Concat([]byte(header), sourceByte) + err = os.WriteFile(targetFile, sourceByte, 0644) + if err != nil { + log.Fatalf("Cannot write file %s to add Hashicorp copy right: %s", target, err) + } +} + +// def expected_output_folder?(output_folder) +func expectedOutputFolder(outputFolder string) bool { + expectedFolders := []string{"terraform-provider-google", "terraform-provider-google-beta", "terraform-next", "terraform-google-conversion", "tfplan2cai"} + folderName := filepath.Base(outputFolder) // Possible issue with Windows OS + isExpected := false + for _, folder := range expectedFolders { + if folderName == folder { + isExpected = true + break + } + } + + return isExpected +} + +// def replace_import_path(output_folder, target) +func (t Terraform) replaceImportPath(outputFolder, target string) { + targetFile := filepath.Join(outputFolder, target) + sourceByte, err := os.ReadFile(targetFile) + if err != nil { + log.Fatalf("Cannot read file %s to replace import path: %s", targetFile, err) + } + + data := string(sourceByte) + + betaImportPath := fmt.Sprintf("%s/%s", TERRAFORM_PROVIDER_BETA, RESOURCE_DIRECTORY_BETA) + gaImportPath := fmt.Sprintf("%s/%s", TERRAFORM_PROVIDER_GA, RESOURCE_DIRECTORY_GA) + if strings.Contains(data, betaImportPath) { + log.Fatalf("Importing a package from module %s is not allowed in file %s. Please import a package from module %s.", betaImportPath, filepath.Base(target), gaImportPath) + } + + if t.TargetVersionName == "ga" { + return + } + + // Replace the import pathes in utility files + var tpg, dir string + switch t.TargetVersionName { + case "beta": + tpg = TERRAFORM_PROVIDER_BETA + dir = RESOURCE_DIRECTORY_BETA + default: + tpg = TERRAFORM_PROVIDER_PRIVATE + dir = RESOURCE_DIRECTORY_PRIVATE + + } + + sourceByte = bytes.Replace(sourceByte, []byte(gaImportPath), []byte(tpg+"/"+dir), -1) + sourceByte = bytes.Replace(sourceByte, []byte(TERRAFORM_PROVIDER_GA+"/version"), []byte(tpg+"/version"), -1) + sourceByte = bytes.Replace(sourceByte, []byte("module "+TERRAFORM_PROVIDER_GA), []byte("module "+tpg), -1) + + sourceByte, err = format.Source(sourceByte) + if err != nil { + log.Fatalf("error formatting %s", targetFile) + } + + err = os.WriteFile(targetFile, sourceByte, 0644) + if err != nil { + log.Fatalf("Cannot write file %s to replace import path: %s", target, err) + } +} + +// # Gets the list of services dependent on the version ga, beta, and private +// # If there are some resources of a servcie is in GA, +// # then this service is in GA. Otherwise, the service is in BETA +// def get_mmv1_services_in_version(products, version) +// +// services = [] +// products.map do |product| +// product_definition = product[:definitions] +// if version == 'ga' +// some_resource_in_ga = false +// product_definition.objects.each do |object| +// break if some_resource_in_ga +// +// if !object.exclude && +// !object.not_in_version?(product_definition.version_obj_or_closest(version)) +// some_resource_in_ga = true +// end +// end +// +// services << product[:definitions].name.downcase if some_resource_in_ga +// else +// services << product[:definitions].name.downcase +// end +// end +// services // -// # generate_code and generate_docs are actually used because all of the variables -// # in scope in this method are made available within the templates by the compile call. -// # rubocop:disable Lint/UnusedMethodArgument -// def copy_common_files(output_folder, generate_code, generate_docs, provider_name = nil) -// # version_name is actually used because all of the variables in scope in this method -// # are made available within the templates by the compile call. -// # TODO: remove version_name, use @target_version_name or pass it in expicitly -// # rubocop:disable Lint/UselessAssignment -// version_name = @target_version_name -// # rubocop:enable Lint/UselessAssignment -// provider_name ||= self.class.name.split('::').last.downcase -// return unless File.exist?("provider/#{provider_name}/common~copy.yaml") -// -// Google::LOGGER.info "Copying common files for #{provider_name}" -// files = YAML.safe_load(compile("provider/#{provider_name}/common~copy.yaml")) -// copy_file_list(output_folder, files) -// end -// # rubocop:enable Lint/UnusedMethodArgument -// -// def copy_file_list(output_folder, files) -// files.map do |target, source| -// Thread.new do -// target_file = File.join(output_folder, target) -// target_dir = File.dirname(target_file) -// Google::LOGGER.debug "Copying #{source} => #{target}" -// FileUtils.mkpath target_dir -// -// # If we've modified a file since starting an MM run, it's a reasonable -// # assumption that it was this run that modified it. -// if File.exist?(target_file) && File.mtime(target_file) > @start_time -// raise "#{target_file} was already modified during this run. #{File.mtime(target_file)}" -// end -// -// FileUtils.copy_entry source, target_file -// -// add_hashicorp_copyright_header(output_folder, target) if File.extname(target) == '.go' -// if File.extname(target) == '.go' || File.extname(target) == '.mod' -// replace_import_path(output_folder, target) -// end -// end -// end.map(&:join) -// end -// -// # Compiles files that are shared at the provider level -// def compile_common_files( -// output_folder, -// products, -// common_compile_file, -// override_path = nil -// ) -// return unless File.exist?(common_compile_file) -// -// generate_resources_for_version(products, @target_version_name) -// -// files = YAML.safe_load(compile(common_compile_file)) -// return unless files -// -// file_template = ProviderFileTemplate.new( -// output_folder, -// @target_version_name, -// build_env, -// products, -// override_path -// ) -// compile_file_list(output_folder, files, file_template) -// end -// -// def compile_file_list(output_folder, files, file_template, pwd = Dir.pwd) -// FileUtils.mkpath output_folder -// Dir.chdir output_folder -// files.map do |target, source| -// Thread.new do -// Google::LOGGER.debug "Compiling #{source} => #{target}" -// file_template.generate(pwd, source, target, self) -// -// add_hashicorp_copyright_header(output_folder, target) -// replace_import_path(output_folder, target) -// end -// end.map(&:join) -// Dir.chdir pwd -// end -// -// def add_hashicorp_copyright_header(output_folder, target) -// unless expected_output_folder?(output_folder) -// Google::LOGGER.info "Unexpected output folder (#{output_folder}) detected " \ -// 'when deciding to add HashiCorp copyright headers. ' \ -// 'Watch out for unexpected changes to copied files' -// end -// # only add copyright headers when generating TPG and TPGB -// return unless output_folder.end_with?('terraform-provider-google') || -// output_folder.end_with?('terraform-provider-google-beta') -// -// # Prevent adding copyright header to files with paths or names matching the strings below -// # NOTE: these entries need to match the content of the .copywrite.hcl file originally -// # created in https://github.com/GoogleCloudPlatform/magic-modules/pull/7336 -// # The test-fixtures folder is not included here as it's copied as a whole, -// # not file by file (see common~copy.yaml) -// ignored_folders = [ -// '.release/', -// '.changelog/', -// 'examples/', -// 'scripts/', -// 'META.d/' -// ] -// ignored_files = [ -// 'go.mod', -// '.goreleaser.yml', -// '.golangci.yml', -// 'terraform-registry-manifest.json' -// ] -// should_add_header = true -// ignored_folders.each do |folder| -// # folder will be path leading to file -// next unless target.start_with? folder -// -// Google::LOGGER.debug 'Not adding HashiCorp copyright headers in ' \ -// "ignored folder #{folder} : #{target}" -// should_add_header = false -// end -// return unless should_add_header -// -// ignored_files.each do |file| -// # file will be the filename and extension, with no preceding path -// next unless target.end_with? file -// -// Google::LOGGER.debug 'Not adding HashiCorp copyright headers to ' \ -// "ignored file #{file} : #{target}" -// should_add_header = false -// end -// return unless should_add_header -// -// Google::LOGGER.debug "Adding HashiCorp copyright header to : #{target}" -// data = File.read("#{output_folder}/#{target}") -// -// copyright_header = ['Copyright (c) HashiCorp, Inc.', 'SPDX-License-Identifier: MPL-2.0'] -// lang = language_from_filename(target) -// -// # Some file types we don't want to add headers to -// # e.g. .sh where headers are functional -// # Also, this guards against new filetypes being added and triggering build errors -// return unless lang != :unsupported -// -// # File is not ignored and is appropriate file type to add header to -// header = comment_block(copyright_header, lang) -// File.write("#{output_folder}/#{target}", header) -// -// File.write("#{output_folder}/#{target}", data, mode: 'a') # append mode -// end -// -// def expected_output_folder?(output_folder) -// expected_folders = %w[ -// terraform-provider-google -// terraform-provider-google-beta -// terraform-next -// terraform-google-conversion -// tfplan2cai -// ] -// folder_name = output_folder.split('/')[-1] # Possible issue with Windows OS -// is_expected = false -// expected_folders.each do |folder| -// next unless folder_name == folder -// -// is_expected = true -// break -// end -// is_expected -// end -// -// def replace_import_path(output_folder, target) -// data = File.read("#{output_folder}/#{target}") -// -// if data.include? "#{TERRAFORM_PROVIDER_BETA}/#{RESOURCE_DIRECTORY_BETA}" -// raise 'Importing a package from module ' \ -// "#{TERRAFORM_PROVIDER_BETA}/#{RESOURCE_DIRECTORY_BETA} " \ -// "is not allowed in file #{target.split('/').last}. " \ -// 'Please import a package from module ' \ -// "#{TERRAFORM_PROVIDER_GA}/#{RESOURCE_DIRECTORY_GA}." -// end -// -// return if @target_version_name == 'ga' +// end // -// # Replace the import pathes in utility files -// case @target_version_name -// when 'beta' -// tpg = TERRAFORM_PROVIDER_BETA -// dir = RESOURCE_DIRECTORY_BETA -// else -// tpg = TERRAFORM_PROVIDER_PRIVATE -// dir = RESOURCE_DIRECTORY_PRIVATE -// end +// def generate_newyaml(pwd, data) // -// data = data.gsub( -// "#{TERRAFORM_PROVIDER_GA}/#{RESOURCE_DIRECTORY_GA}", -// "#{tpg}/#{dir}" -// ) -// data = data.gsub( -// "#{TERRAFORM_PROVIDER_GA}/version", -// "#{tpg}/version" -// ) +// # @api.api_name is the service folder name +// product_name = @api.api_name +// target_folder = File.join(folder_name(data.version), 'services', product_name) +// FileUtils.mkpath target_folder +// data.generate(pwd, +// '/templates/terraform/yaml_conversion.erb', +// "#{target_folder}/go_#{data.object.name}.yaml", +// self) +// return if File.exist?("#{target_folder}/go_product.yaml") // -// data = data.gsub( -// "module #{TERRAFORM_PROVIDER_GA}", -// "module #{tpg}" -// ) -// File.write("#{output_folder}/#{target}", data) -// end +// data.generate(pwd, +// '/templates/terraform/product_yaml_conversion.erb', +// "#{target_folder}/go_product.yaml", +// self) // +// end // -// # Gets the list of services dependent on the version ga, beta, and private -// # If there are some resources of a servcie is in GA, -// # then this service is in GA. Otherwise, the service is in BETA -// def get_mmv1_services_in_version(products, version) -// services = [] -// products.map do |product| -// product_definition = product[:definitions] -// if version == 'ga' -// some_resource_in_ga = false -// product_definition.objects.each do |object| -// break if some_resource_in_ga -// -// if !object.exclude && -// !object.not_in_version?(product_definition.version_obj_or_closest(version)) -// some_resource_in_ga = true -// end -// end -// -// services << product[:definitions].name.downcase if some_resource_in_ga -// else -// services << product[:definitions].name.downcase -// end -// end -// services -// end +// def build_env // -// def generate_newyaml(pwd, data) -// # @api.api_name is the service folder name -// product_name = @api.api_name -// target_folder = File.join(folder_name(data.version), 'services', product_name) -// FileUtils.mkpath target_folder -// data.generate(pwd, -// '/templates/terraform/yaml_conversion.erb', -// "#{target_folder}/go_#{data.object.name}.yaml", -// self) -// return if File.exist?("#{target_folder}/go_product.yaml") +// { +// goformat_enabled: @go_format_enabled, +// start_time: @start_time +// } // -// data.generate(pwd, -// '/templates/terraform/product_yaml_conversion.erb', -// "#{target_folder}/go_product.yaml", -// self) -// end +// end // -// def build_env -// { -// goformat_enabled: @go_format_enabled, -// start_time: @start_time -// } -// end +// # used to determine and separate objects that have update methods +// # that target individual fields +// def field_specific_update_methods(properties) // -// # used to determine and separate objects that have update methods -// # that target individual fields -// def field_specific_update_methods(properties) -// properties_by_custom_update(properties).length.positive? -// end +// properties_by_custom_update(properties).length.positive? // -// # Filter the properties to keep only the ones requiring custom update -// # method and group them by update url & verb. -// def properties_by_custom_update(properties) -// update_props = properties.reject do |p| -// p.update_url.nil? || p.update_verb.nil? || p.update_verb == :NOOP || -// p.is_a?(Api::Type::KeyValueTerraformLabels) || -// p.is_a?(Api::Type::KeyValueLabels) # effective_labels is used for update -// end +// end // -// update_props.group_by do |p| -// { -// update_url: p.update_url, -// update_verb: p.update_verb, -// update_id: p.update_id, -// fingerprint_name: p.fingerprint_name -// } -// end -// end +// # Filter the properties to keep only the ones requiring custom update +// # method and group them by update url & verb. +// def properties_by_custom_update(properties) // -// # Filter the properties to keep only the ones don't have custom update -// # method and group them by update url & verb. -// def properties_without_custom_update(properties) -// properties.select do |p| -// p.update_url.nil? || p.update_verb.nil? || p.update_verb == :NOOP -// end -// end +// update_props = properties.reject do |p| +// p.update_url.nil? || p.update_verb.nil? || p.update_verb == :NOOP || +// p.is_a?(Api::Type::KeyValueTerraformLabels) || +// p.is_a?(Api::Type::KeyValueLabels) # effective_labels is used for update +// end // -// # Takes a update_url and returns the list of custom updatable properties -// # that can be updated at that URL. This allows flattened objects -// # to determine which parent property in the API should be updated with -// # the contents of the flattened object -// def custom_update_properties_by_key(properties, key) -// properties_by_custom_update(properties).select do |k, _| -// k[:update_url] == key[:update_url] && -// k[:update_id] == key[:update_id] && -// k[:fingerprint_name] == key[:fingerprint_name] -// end.first.last -// # .first is to grab the element from the select which returns a list -// # .last is because properties_by_custom_update returns a list of -// # [{update_url}, [properties,...]] and we only need the 2nd part -// end +// update_props.group_by do |p| +// { +// update_url: p.update_url, +// update_verb: p.update_verb, +// update_id: p.update_id, +// fingerprint_name: p.fingerprint_name +// } +// end // -// def update_url(resource, url_part) -// [resource.__product.base_url, update_uri(resource, url_part)].flatten.join -// end +// end // -// def update_uri(resource, url_part) -// return resource.self_link_uri if url_part.nil? +// # Filter the properties to keep only the ones don't have custom update +// # method and group them by update url & verb. +// def properties_without_custom_update(properties) // -// url_part -// end +// properties.select do |p| +// p.update_url.nil? || p.update_verb.nil? || p.update_verb == :NOOP +// end // -// def generating_hashicorp_repo? -// # The default Provider is used to generate TPG and TPGB in HashiCorp-owned repos. -// # The compiler deviates from the default behaviour with a -f flag to produce -// # non-HashiCorp downstreams. -// true -// end +// end // -// # ProductFileTemplate with Terraform specific fields -// class TerraformProductFileTemplate < Provider::ProductFileTemplate -// # The async object used for making operations. -// # We assume that all resources share the same async properties. -// attr_accessor :async +// # Takes a update_url and returns the list of custom updatable properties +// # that can be updated at that URL. This allows flattened objects +// # to determine which parent property in the API should be updated with +// # the contents of the flattened object +// def custom_update_properties_by_key(properties, key) // -// # When generating OiCS examples, we attach the example we're -// # generating to the data object. -// attr_accessor :example +// properties_by_custom_update(properties).select do |k, _| +// k[:update_url] == key[:update_url] && +// k[:update_id] == key[:update_id] && +// k[:fingerprint_name] == key[:fingerprint_name] +// end.first.last +// # .first is to grab the element from the select which returns a list +// # .last is because properties_by_custom_update returns a list of +// # [{update_url}, [properties,...]] and we only need the 2nd part // -// attr_accessor :resource_name -// end +// end // -// # Sorts properties in the order they should appear in the TF schema: -// # Required, Optional, Computed -// def order_properties(properties) -// properties.select(&:required).sort_by(&:name) + -// properties.reject(&:required).reject(&:output).sort_by(&:name) + -// properties.select(&:output).sort_by(&:name) -// end +// def update_url(resource, url_part) // -// def tf_type(property) -// tf_types[property.class] -// end +// [resource.__product.base_url, update_uri(resource, url_part)].flatten.join // -// # "Namespace" - prefix with product and resource - a property with -// # information from the "object" variable -// def namespace_property_from_object(property, object) -// name = property.name.camelize -// until property.parent.nil? -// property = property.parent -// name = property.name.camelize + name -// end +// end // -// "#{property.__resource.__product.api_name.camelize(:lower)}#{object.name}#{name}" -// end +// def update_uri(resource, url_part) // -// # Converts between the Magic Modules type of an object and its type in the -// # TF schema -// def tf_types -// { -// Api::Type::Boolean => 'schema.TypeBool', -// Api::Type::Double => 'schema.TypeFloat', -// Api::Type::Integer => 'schema.TypeInt', -// Api::Type::String => 'schema.TypeString', -// # Anonymous string property used in array of strings. -// 'Api::Type::String' => 'schema.TypeString', -// Api::Type::Time => 'schema.TypeString', -// Api::Type::Enum => 'schema.TypeString', -// Api::Type::ResourceRef => 'schema.TypeString', -// Api::Type::NestedObject => 'schema.TypeList', -// Api::Type::Array => 'schema.TypeList', -// Api::Type::KeyValuePairs => 'schema.TypeMap', -// Api::Type::KeyValueLabels => 'schema.TypeMap', -// Api::Type::KeyValueTerraformLabels => 'schema.TypeMap', -// Api::Type::KeyValueEffectiveLabels => 'schema.TypeMap', -// Api::Type::KeyValueAnnotations => 'schema.TypeMap', -// Api::Type::Map => 'schema.TypeSet', -// Api::Type::Fingerprint => 'schema.TypeString' -// } -// end +// return resource.self_link_uri if url_part.nil? // -// def updatable?(resource, properties) -// !resource.immutable || !properties.reject { |p| p.update_url.nil? }.empty? -// end +// url_part // -// def force_new?(property, resource) -// ( -// (!property.output || property.is_a?(Api::Type::KeyValueEffectiveLabels)) && -// (property.immutable || -// (resource.immutable && property.update_url.nil? && property.immutable.nil? && -// (property.parent.nil? || -// (force_new?(property.parent, resource) && -// !(property.parent.flatten_object && property.is_a?(Api::Type::KeyValueLabels)) -// ) -// ) -// ) -// ) -// ) || -// (property.is_a?(Api::Type::KeyValueTerraformLabels) && -// !updatable?(resource, resource.all_user_properties) && !resource.root_labels? -// ) -// end +// end // -// # Returns tuples of (fieldName, list of update masks) for -// # top-level updatable fields. Schema path refers to a given Terraform -// # field name (e.g. d.GetChange('fieldName)') -// def get_property_update_masks_groups(properties, mask_prefix: '') -// mask_groups = [] -// properties.each do |prop| -// if prop.flatten_object -// mask_groups += get_property_update_masks_groups( -// prop.properties, mask_prefix: "#{prop.api_name}." -// ) -// elsif prop.update_mask_fields -// mask_groups << [prop.name.underscore, prop.update_mask_fields] -// else -// mask_groups << [prop.name.underscore, [mask_prefix + prop.api_name]] -// end -// end -// mask_groups -// end +// def generating_hashicorp_repo? // -// # Returns an updated path for a given Terraform field path (e.g. -// # 'a_field', 'parent_field.0.child_name'). Returns nil if the property -// # is not included in the resource's properties and removes keys that have -// # been flattened -// # FYI: Fields that have been renamed should use the new name, however, flattened -// # fields still need to be included, ie: -// # flattenedField > newParent > renameMe should be passed to this function as -// # flattened_field.0.new_parent.0.im_renamed -// # TODO(emilymye): Change format of input for -// # exactly_one_of/at_least_one_of/etc to use camelcase, MM properities and -// # convert to snake in this method -// def get_property_schema_path(schema_path, resource) -// nested_props = resource.properties -// prop = nil -// path_tkns = schema_path.split('.0.').map do |pname| -// camel_pname = pname.camelize(:lower) -// prop = nested_props.find { |p| p.name == camel_pname } -// # if we couldn't find it, see if it was renamed at the top level -// prop = nested_props.find { |p| p.name == schema_path } if prop.nil? -// return nil if prop.nil? -// -// nested_props = prop.nested_properties || [] -// prop.flatten_object ? nil : pname.underscore -// end -// if path_tkns.empty? || path_tkns[-1].nil? -// nil -// else -// path_tkns.compact.join('.0.') -// end -// end +// # The default Provider is used to generate TPG and TPGB in HashiCorp-owned repos. +// # The compiler deviates from the default behaviour with a -f flag to produce +// # non-HashiCorp downstreams. +// true // -// # Transforms a format string with field markers to a regex string with -// # capture groups. -// # -// # For instance, -// # projects/{{project}}/global/networks/{{name}} -// # is transformed to -// # projects/(?P[^/]+)/global/networks/(?P[^/]+) -// # -// # Values marked with % are URL-encoded, and will match any number of /'s. -// # -// # Note: ?P indicates a Python-compatible named capture group. Named groups -// # aren't common in JS-based regex flavours, but are in Perl-based ones -// def format2regex(format) -// format -// .gsub(/\{\{%([[:word:]]+)\}\}/, '(?P<\1>.+)') -// .gsub(/\{\{([[:word:]]+)\}\}/, '(?P<\1>[^/]+)') -// end +// end // -// # Capitalize the first letter of a property name. -// # E.g. "creationTimestamp" becomes "CreationTimestamp". -// def titlelize_property(property) -// property.name.camelize(:upper) -// end +// # ProductFileTemplate with Terraform specific fields +// class TerraformProductFileTemplate < Provider::ProductFileTemplate // -// # Generates the list of resources, and gets the count of resources and iam resources -// # dependent on the version ga, beta or private. -// # The resource object has the format -// # { -// # terraform_name: -// # resource_name: -// # iam_class_name: -// # } -// # The variable resources_for_version is used to generate resources in file -// # mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb -// def generate_resources_for_version(products, version) -// products.each do |product| -// product_definition = product[:definitions] -// service = product_definition.name.downcase -// product_definition.objects.each do |object| -// if object.exclude || -// object.not_in_version?(product_definition.version_obj_or_closest(version)) -// next -// end -// -// @resource_count += 1 unless object&.exclude_resource -// -// tf_product = (object.__product.legacy_name || product_definition.name).underscore -// terraform_name = object.legacy_name || "google_#{tf_product}_#{object.name.underscore}" -// -// unless object&.exclude_resource -// resource_name = "#{service}.Resource#{product_definition.name}#{object.name}" -// end -// -// iam_policy = object&.iam_policy -// -// @iam_resource_count += 3 unless iam_policy.nil? || iam_policy.exclude -// -// unless iam_policy.nil? || iam_policy.exclude || -// (iam_policy.min_version && iam_policy.min_version < version) -// iam_class_name = "#{service}.#{product_definition.name}#{object.name}" -// end -// -// @resources_for_version << { terraform_name:, resource_name:, iam_class_name: } -// end -// end +// # The async object used for making operations. +// # We assume that all resources share the same async properties. +// attr_accessor :async // -// @resources_for_version = @resources_for_version.compact -// end +// # When generating OiCS examples, we attach the example we're +// # generating to the data object. +// attr_accessor :example // -// # TODO(nelsonjr): Review all object interfaces and move to private methods -// # that should not be exposed outside the object hierarchy. -// private +// attr_accessor :resource_name // -// def provider_name -// self.class.name.split('::').last.downcase -// end +// end // -// # Adapted from the method used in templating -// # See: mmv1/compile/core.rb -// def comment_block(text, lang) -// case lang -// when :ruby, :python, :yaml, :git, :gemfile -// header = text.map { |t| t&.empty? ? '#' : "# #{t}" } -// when :go -// header = text.map { |t| t&.empty? ? '//' : "// #{t}" } -// else -// raise "Unknown language for comment: #{lang}" -// end +// # Sorts properties in the order they should appear in the TF schema: +// # Required, Optional, Computed +// def order_properties(properties) // -// header_string = header.join("\n") -// "#{header_string}\n" # add trailing newline to returned value -// end +// properties.select(&:required).sort_by(&:name) + +// properties.reject(&:required).reject(&:output).sort_by(&:name) + +// properties.select(&:output).sort_by(&:name) // -// def language_from_filename(filename) -// extension = filename.split('.')[-1] -// case extension -// when 'go' -// :go -// when 'rb' -// :ruby -// when 'yaml', 'yml' -// :yaml -// else -// :unsupported -// end -// end +// end // +// def tf_type(property) +// +// tf_types[property.class] +// +// end +// +// # "Namespace" - prefix with product and resource - a property with +// # information from the "object" variable +// def namespace_property_from_object(property, object) +// +// name = property.name.camelize +// until property.parent.nil? +// property = property.parent +// name = property.name.camelize + name +// end +// +// "#{property.__resource.__product.api_name.camelize(:lower)}#{object.name}#{name}" +// +// end +// +// # Converts between the Magic Modules type of an object and its type in the +// # TF schema +// def tf_types +// +// { +// Api::Type::Boolean => 'schema.TypeBool', +// Api::Type::Double => 'schema.TypeFloat', +// Api::Type::Integer => 'schema.TypeInt', +// Api::Type::String => 'schema.TypeString', +// # Anonymous string property used in array of strings. +// 'Api::Type::String' => 'schema.TypeString', +// Api::Type::Time => 'schema.TypeString', +// Api::Type::Enum => 'schema.TypeString', +// Api::Type::ResourceRef => 'schema.TypeString', +// Api::Type::NestedObject => 'schema.TypeList', +// Api::Type::Array => 'schema.TypeList', +// Api::Type::KeyValuePairs => 'schema.TypeMap', +// Api::Type::KeyValueLabels => 'schema.TypeMap', +// Api::Type::KeyValueTerraformLabels => 'schema.TypeMap', +// Api::Type::KeyValueEffectiveLabels => 'schema.TypeMap', +// Api::Type::KeyValueAnnotations => 'schema.TypeMap', +// Api::Type::Map => 'schema.TypeSet', +// Api::Type::Fingerprint => 'schema.TypeString' +// } +// +// end +// +// def updatable?(resource, properties) +// +// !resource.immutable || !properties.reject { |p| p.update_url.nil? }.empty? +// +// end +// +// def force_new?(property, resource) +// +// ( +// (!property.output || property.is_a?(Api::Type::KeyValueEffectiveLabels)) && +// (property.immutable || +// (resource.immutable && property.update_url.nil? && property.immutable.nil? && +// (property.parent.nil? || +// (force_new?(property.parent, resource) && +// !(property.parent.flatten_object && property.is_a?(Api::Type::KeyValueLabels)) +// ) +// ) +// ) +// ) +// ) || +// (property.is_a?(Api::Type::KeyValueTerraformLabels) && +// !updatable?(resource, resource.all_user_properties) && !resource.root_labels? +// ) +// +// end +// +// # Returns tuples of (fieldName, list of update masks) for +// # top-level updatable fields. Schema path refers to a given Terraform +// # field name (e.g. d.GetChange('fieldName)') +// def get_property_update_masks_groups(properties, mask_prefix: ”) +// +// mask_groups = [] +// properties.each do |prop| +// if prop.flatten_object +// mask_groups += get_property_update_masks_groups( +// prop.properties, mask_prefix: "#{prop.api_name}." +// ) +// elsif prop.update_mask_fields +// mask_groups << [prop.name.underscore, prop.update_mask_fields] +// else +// mask_groups << [prop.name.underscore, [mask_prefix + prop.api_name]] +// end +// end +// mask_groups +// +// end +// +// # Returns an updated path for a given Terraform field path (e.g. +// # 'a_field', 'parent_field.0.child_name'). Returns nil if the property +// # is not included in the resource's properties and removes keys that have +// # been flattened +// # FYI: Fields that have been renamed should use the new name, however, flattened +// # fields still need to be included, ie: +// # flattenedField > newParent > renameMe should be passed to this function as +// # flattened_field.0.new_parent.0.im_renamed +// # TODO(emilymye): Change format of input for +// # exactly_one_of/at_least_one_of/etc to use camelcase, MM properities and +// # convert to snake in this method +// def get_property_schema_path(schema_path, resource) +// +// nested_props = resource.properties +// prop = nil +// path_tkns = schema_path.split('.0.').map do |pname| +// camel_pname = pname.camelize(:lower) +// prop = nested_props.find { |p| p.name == camel_pname } +// # if we couldn't find it, see if it was renamed at the top level +// prop = nested_props.find { |p| p.name == schema_path } if prop.nil? +// return nil if prop.nil? +// +// nested_props = prop.nested_properties || [] +// prop.flatten_object ? nil : pname.underscore +// end +// if path_tkns.empty? || path_tkns[-1].nil? +// nil +// else +// path_tkns.compact.join('.0.') +// end +// +// end +// +// # Transforms a format string with field markers to a regex string with +// # capture groups. +// # +// # For instance, +// # projects/{{project}}/global/networks/{{name}} +// # is transformed to +// # projects/(?P[^/]+)/global/networks/(?P[^/]+) +// # +// # Values marked with % are URL-encoded, and will match any number of /'s. +// # +// # Note: ?P indicates a Python-compatible named capture group. Named groups +// # aren't common in JS-based regex flavours, but are in Perl-based ones +// def format2regex(format) +// +// format +// .gsub(/\{\{%([[:word:]]+)\}\}/, '(?P<\1>.+)') +// .gsub(/\{\{([[:word:]]+)\}\}/, '(?P<\1>[^/]+)') +// +// end +// +// # Capitalize the first letter of a property name. +// # E.g. "creationTimestamp" becomes "CreationTimestamp". +// def titlelize_property(property) +// +// property.name.camelize(:upper) +// +// end +// +// # Generates the list of resources, and gets the count of resources and iam resources +// # dependent on the version ga, beta or private. +// # The resource object has the format +// # { +// # terraform_name: +// # resource_name: +// # iam_class_name: +// # } +// # The variable resources_for_version is used to generate resources in file +// # mmv1/third_party/terraform/provider/provider_mmv1_resources.go.erb +// def generate_resources_for_version(products, version) +func (t *Terraform) generateResourcesForVersion(products []map[string]interface{}) { + // products.each do |product| + for _, product := range products { + productDefinition := product["Definitions"].(*api.Product) + service := strings.ToLower(productDefinition.Name) + for _, object := range productDefinition.Objects { + if object.Exclude || object.NotInVersion(productDefinition.VersionObjOrClosest(t.TargetVersionName)) { + continue + } + + var resourceName string + + if !object.ExcludeResource { + t.ResourceCount++ + resourceName = fmt.Sprintf("%s.Resource%s", service, object.ResourceName()) + } + + var iamClassName string + iamPolicy := object.IamPolicy + if iamPolicy != nil && !iamPolicy.Exclude { + t.IAMResourceCount += 3 + + if !(iamPolicy.MinVersion != "" && iamPolicy.MinVersion < t.TargetVersionName) { + iamClassName = fmt.Sprintf("%s.Resource%s", service, object.ResourceName()) + } + } + + t.ResourcesForVersion = append(t.ResourcesForVersion, map[string]string{ + "TerraformName": object.TerraformName(), + "ResourceName": resourceName, + "IamClassName": iamClassName, + }) + } + } + + // @resources_for_version = @resources_for_version.compact +} + +// # TODO(nelsonjr): Review all object interfaces and move to private methods +// # that should not be exposed outside the object hierarchy. +// def provider_name +func (t Terraform) providerName() string { + return reflect.TypeOf(t).Name() +} + +// # Adapted from the method used in templating +// # See: mmv1/compile/core.rb +// def comment_block(text, lang) +func commentBlock(text []string, lang string) string { + var headers []string + switch lang { + case "ruby", "python", "yaml", "gemfile": + headers = commentText(text, "#") + case "go": + headers = commentText(text, "//") + default: + log.Fatalf("Unknown language for comment: %s", lang) + } + + headerString := strings.Join(headers, "\n") + return fmt.Sprintf("%s\n", headerString) // add trailing newline to returned value +} + +func commentText(text []string, symbols string) []string { + var header []string + for _, t := range text { + var comment string + if t == "" { + comment = symbols + } else { + comment = fmt.Sprintf("%s %s", symbols, t) + } + header = append(header, comment) + } + return header +} + +// def language_from_filename(filename) +func languageFromFilename(filename string) string { + switch extension := filepath.Ext(filename); extension { + case ".go": + return "go" + case ".rb": + return "rb" + case ".yaml", ".yml": + return "yaml" + default: + return "unsupported" + } +} + // # Finds the folder name for a given version of the terraform provider // def folder_name(version) // version == 'ga' ? 'google' : "google-#{version}" diff --git a/mmv1/third_party/terraform/.copywrite.hcl.tmpl b/mmv1/third_party/terraform/.copywrite.hcl.tmpl new file mode 100644 index 000000000000..693bde6062ee --- /dev/null +++ b/mmv1/third_party/terraform/.copywrite.hcl.tmpl @@ -0,0 +1,33 @@ +schema_version = 1 + +project { + license = "MPL-2.0" + copyright_year = 2017 + + # (OPTIONAL) A list of globs that should not have copyright/license headers. + # Supports doublestar glob patterns for more flexibility in defining which + # files or folders should be ignored + header_ignore = [ + # Some ignores here are not strictly needed, but protects us if we change the types of files we put in those folders + # See here for file extensions altered by copywrite CLI (all other extensions are ignored) + # https://github.com/hashicorp/copywrite/blob/4af928579f5aa8f1dece9de1bb3098218903053d/addlicense/main.go#L357-L394 + ".github/**", + ".release/**", + ".changelog/**", + "examples/**", + "scripts/**", +{{- if or (eq $.TargetVersionName "") (eq $.TargetVersionName "ga")}} + "google/**/test-fixtures/**", +{{- else}} + "google-{{$.TargetVersionName}}/**/test-fixtures/**", +{{- end}} + "META.d/*.yml", + "META.d/*.yaml", + ".golangci.yml", + ".goreleaser.yml", + ] + + # (OPTIONAL) Links to an upstream repo for determining repo relationships + # This is for special cases and should not normally be set. + upstream = "GoogleCloudPlatform/magic-modules" +} diff --git a/mmv1/third_party/terraform/.goreleaser.yml.tmpl b/mmv1/third_party/terraform/.goreleaser.yml.tmpl new file mode 100644 index 000000000000..33eb62131c19 --- /dev/null +++ b/mmv1/third_party/terraform/.goreleaser.yml.tmpl @@ -0,0 +1,83 @@ +archives: + - files: + # Include built binary and license files in archive + - 'LICENSE' + format: zip + name_template: '{{"{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"}}' +builds: + - # Special binary naming is only necessary for Terraform CLI 0.12 + binary: '{{"{{ .ProjectName }}_v{{ .Version }}"}}_x5' + env: + - CGO_ENABLED=0 + flags: + - -trimpath + goos: + - darwin + - freebsd + - linux + - windows + goarch: + - '386' + - amd64 + - arm + - arm64 + ignore: + - goarch: arm + goos: windows + - goarch: arm64 + goos: freebsd + - goarch: arm64 + goos: windows + ldflags: + - -s -w -X github.com/hashicorp/terraform-provider-google{{- if ne $.TargetVersionName "ga" -}}-{{$.TargetVersionName}}{{- end }}/version.ProviderVersion={{"{{.Version}}"}} + mod_timestamp: '{{"{{ .CommitTimestamp }}"}}' +checksum: + extra_files: + - glob: 'terraform-registry-manifest.json' + name_template: '{{"{{ .ProjectName }}_{{ .Version }}"}}_manifest.json' + name_template: '{{"{{ .ProjectName }}_{{ .Version }}"}}_SHA256SUMS' + algorithm: sha256 +publishers: + - name: upload + checksum: true + extra_files: + - glob: 'terraform-registry-manifest.json' + name_template: '{{"{{ .ProjectName }}_{{ .Version }}"}}_manifest.json' + signature: true + cmd: hc-releases upload -product {{"{{ .ProjectName }} -version {{ .Version }} -file={{ .ArtifactPath }}={{ .ArtifactName }}"}} -header="x-terraform-protocol-version=5.0" -header="x-terraform-protocol-versions=5.0" + env: + - HC_RELEASES_HOST={{"{{ .Env.HC_RELEASES_HOST }}"}} + - HC_RELEASES_KEY={{"{{ .Env.HC_RELEASES_KEY }}"}} +release: + extra_files: + - glob: 'terraform-registry-manifest.json' + name_template: '{{"{{ .ProjectName }}_{{ .Version }}"}}_manifest.json' + ids: + - none +signs: + # Default Signature file (i.e. terraform-provider-awscc_VERSION_SHA256SUMS.sig) + - cmd: sh + args: + - -c + - >- + signore + sign + --dearmor + --file ${artifact} + --out ${signature} + artifacts: checksum + # Signature file with GPG Public Key ID in filename (i.e. terraform-provider-awscc_VERSION_SHA256SUMS.7685B676.sig) + - id: sig-with-gpg-public-key-id + signature: ${artifact}.72D7468F.sig + cmd: sh + args: + - -c + - >- + signore + sign + --dearmor + --file ${artifact} + --out ${signature} + artifacts: checksum +snapshot: + name_template: "{{"{{ .Tag }}"}}-next" diff --git a/mmv1/third_party/terraform/go.mod b/mmv1/third_party/terraform/go.mod new file mode 100644 index 000000000000..89f79e3d7263 --- /dev/null +++ b/mmv1/third_party/terraform/go.mod @@ -0,0 +1,110 @@ +module github.com/hashicorp/terraform-provider-google + +go 1.21 + +require ( + cloud.google.com/go/bigtable v1.19.0 + github.com/GoogleCloudPlatform/declarative-resource-client-library v1.64.0 + github.com/apparentlymart/go-cidr v1.1.0 + github.com/davecgh/go-spew v1.1.1 + github.com/dnaeon/go-vcr v1.0.1 + github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92 + github.com/google/go-cmp v0.6.0 + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 + github.com/hashicorp/errwrap v1.0.0 + github.com/hashicorp/go-cleanhttp v0.5.2 + github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-version v1.6.0 + github.com/hashicorp/terraform-plugin-framework v1.7.0 + github.com/hashicorp/terraform-plugin-framework-validators v0.9.0 + github.com/hashicorp/terraform-plugin-go v0.22.1 + github.com/hashicorp/terraform-plugin-log v0.9.0 + github.com/hashicorp/terraform-plugin-mux v0.15.0 + github.com/hashicorp/terraform-plugin-sdk/v2 v2.33.0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/hashstructure v1.1.0 + github.com/sirupsen/logrus v1.8.1 + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 + golang.org/x/net v0.22.0 + golang.org/x/oauth2 v0.18.0 + google.golang.org/api v0.171.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c + google.golang.org/grpc v1.62.1 + google.golang.org/protobuf v1.33.0 +) + +require ( + bitbucket.org/creachadair/stringset v0.0.8 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.4 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.6 // indirect + cloud.google.com/go/longrunning v0.5.5 // indirect + github.com/ProtonMail/go-crypto v1.1.0-alpha.0 // indirect + github.com/agext/levenshtein v1.2.2 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect + github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect + github.com/envoyproxy/go-control-plane v0.12.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/glog v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cpy v0.0.0-20211218193943-a9c933c06932 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/hashicorp/go-checkpoint v0.5.0 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-plugin v1.6.0 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/hc-install v0.6.3 // indirect + github.com/hashicorp/hcl/v2 v2.19.1 // indirect + github.com/hashicorp/logutils v1.0.0 // indirect + github.com/hashicorp/terraform-exec v0.20.0 // indirect + github.com/hashicorp/terraform-json v0.21.0 // indirect + github.com/hashicorp/terraform-registry-address v0.2.3 // indirect + github.com/hashicorp/terraform-svchost v0.1.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/oklog/run v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/zclconf/go-cty v1.14.2 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/mmv1/third_party/terraform/main.go.tmpl b/mmv1/third_party/terraform/main.go.tmpl new file mode 100644 index 000000000000..70e1fa734bc3 --- /dev/null +++ b/mmv1/third_party/terraform/main.go.tmpl @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "flag" + "log" + + "github.com/hashicorp/terraform-plugin-framework/providerserver" + "github.com/hashicorp/terraform-plugin-go/tfprotov5" + "github.com/hashicorp/terraform-plugin-go/tfprotov5/tf5server" + "github.com/hashicorp/terraform-plugin-mux/tf5muxserver" + + "github.com/hashicorp/terraform-provider-google/google/fwprovider" + "github.com/hashicorp/terraform-provider-google/google/provider" + ver "github.com/hashicorp/terraform-provider-google/version" +) + +var ( + // these will be set by the goreleaser configuration + // to appropriate values for the compiled binary + version string = ver.ProviderVersion + + // goreleaser can also pass the specific commit if you want + // commit string = "" +) + +func main() { + var debug bool + + flag.BoolVar(&debug, "debug", false, "set to true to run the provider with support for debuggers like delve") + flag.Parse() + + // concat with sdkv2 provider + providers := []func() tfprotov5.ProviderServer{ + providerserver.NewProtocol5(fwprovider.New(version)), // framework provider + provider.Provider().GRPCProvider, // sdk provider + } + + // use the muxer + muxServer, err := tf5muxserver.NewMuxServer(context.Background(), providers...) + if err != nil { + log.Fatalf(err.Error()) + } + + var serveOpts []tf5server.ServeOpt + + if debug { + serveOpts = append(serveOpts, tf5server.WithManagedDebug()) + } + + err = tf5server.Serve( + "registry.terraform.io/hashicorp/google{{- if ne $.TargetVersionName "ga" -}}-{{$.TargetVersionName}}{{- end }}", + muxServer.ProviderServer, + serveOpts..., + ) + + if err != nil { + log.Fatal(err) + } +} diff --git a/mmv1/third_party/terraform/release-metadata.hcl.tmpl b/mmv1/third_party/terraform/release-metadata.hcl.tmpl new file mode 100644 index 000000000000..ca8618b3be33 --- /dev/null +++ b/mmv1/third_party/terraform/release-metadata.hcl.tmpl @@ -0,0 +1,2 @@ +url_source_repository = "https://github.com/hashicorp/terraform-provider-google{{- if ne $.TargetVersionName "ga" -}}-{{$.TargetVersionName}}{{- end }}" +url_license = "https://github.com/hashicorp/terraform-provider-google{{- if ne $.TargetVersionName "ga" -}}-{{$.TargetVersionName}}{{- end }}/blob/main/LICENSE" diff --git a/mmv1/third_party/terraform/services/containeranalysis/resource_container_analysis_occurrence_test.go b/mmv1/third_party/terraform/services/containeranalysis/resource_container_analysis_occurrence_test.go index 1e80394cc56d..d0f73e97df6e 100644 --- a/mmv1/third_party/terraform/services/containeranalysis/resource_container_analysis_occurrence_test.go +++ b/mmv1/third_party/terraform/services/containeranalysis/resource_container_analysis_occurrence_test.go @@ -18,7 +18,7 @@ import ( const testAttestationOccurrenceImageUrl = "gcr.io/cloud-marketplace/google/ubuntu1804" const testAttestationOccurrenceImageDigest = "sha256:3593cd4ac7d782d460dc86ba9870a3beaf81c8f5cdbcc8880bf9a5ef6af10c5a" -const testAttestationOccurrencePayloadTemplate = "test-fixtures/generated_payload.json.tmpl" +const testAttestationOccurrencePayloadTemplate = "test-fixtures/generated_payload.json" var testAttestationOccurrenceFullImagePath = fmt.Sprintf("%s@%s", testAttestationOccurrenceImageUrl, testAttestationOccurrenceImageDigest) diff --git a/mmv1/third_party/terraform/services/containeranalysis/test-fixtures/generated_payload.json.tmpl b/mmv1/third_party/terraform/services/containeranalysis/test-fixtures/generated_payload.json similarity index 100% rename from mmv1/third_party/terraform/services/containeranalysis/test-fixtures/generated_payload.json.tmpl rename to mmv1/third_party/terraform/services/containeranalysis/test-fixtures/generated_payload.json diff --git a/mmv1/third_party/terraform/services/deploymentmanager/resource_deployment_manager_deployment_test.go b/mmv1/third_party/terraform/services/deploymentmanager/resource_deployment_manager_deployment_test.go index e7092236e149..46186eb77978 100644 --- a/mmv1/third_party/terraform/services/deploymentmanager/resource_deployment_manager_deployment_test.go +++ b/mmv1/third_party/terraform/services/deploymentmanager/resource_deployment_manager_deployment_test.go @@ -22,7 +22,7 @@ func TestAccDeploymentManagerDeployment_basicFile(t *testing.T) { randSuffix := acctest.RandString(t, 10) deploymentId := "tf-dm-" + randSuffix accountId := "tf-dm-account-" + randSuffix - yamlPath := createYamlConfigFileForTest(t, "test-fixtures/service_account.yml.tmpl", map[string]interface{}{ + yamlPath := createYamlConfigFileForTest(t, "test-fixtures/service_account.yml", map[string]interface{}{ "account_id": accountId, }) @@ -97,7 +97,7 @@ func TestAccDeploymentManagerDeployment_imports(t *testing.T) { randStr := acctest.RandString(t, 10) deploymentName := "tf-dm-" + randStr accountId := "tf-dm-" + randStr - importFilepath := createYamlConfigFileForTest(t, "test-fixtures/service_account.yml.tmpl", map[string]interface{}{ + importFilepath := createYamlConfigFileForTest(t, "test-fixtures/service_account.yml", map[string]interface{}{ "account_id": "{{ env['name'] }}", }) diff --git a/mmv1/third_party/terraform/services/deploymentmanager/test-fixtures/service_account.yml.tmpl b/mmv1/third_party/terraform/services/deploymentmanager/test-fixtures/service_account.yml similarity index 100% rename from mmv1/third_party/terraform/services/deploymentmanager/test-fixtures/service_account.yml.tmpl rename to mmv1/third_party/terraform/services/deploymentmanager/test-fixtures/service_account.yml diff --git a/mmv1/third_party/terraform/terraform-registry-manifest.json b/mmv1/third_party/terraform/terraform-registry-manifest.json new file mode 100644 index 000000000000..1931b0e00217 --- /dev/null +++ b/mmv1/third_party/terraform/terraform-registry-manifest.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "metadata": { + "protocol_versions": ["5.0"] + } +} diff --git a/mmv1/third_party/terraform/utils/provider_versionfive_upgrade_test.go.erb b/mmv1/third_party/terraform/utils/provider_versionfive_upgrade_test.go.erb deleted file mode 100644 index 161ec6ccf34d..000000000000 --- a/mmv1/third_party/terraform/utils/provider_versionfive_upgrade_test.go.erb +++ /dev/null @@ -1,646 +0,0 @@ -<% autogen_exception -%> -package google - -import ( - "fmt" - "strings" - "testing" - - "github.com/hashicorp/terraform-provider-google/google/acctest" - "github.com/hashicorp/terraform-provider-google/google/envvar" - "github.com/hashicorp/terraform-provider-google/google/tpgresource" - transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport" - - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" - "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" -) - -func TestProvider_versionfive_upgrade(t *testing.T) { - acctest.SkipIfVcr(t) - t.Parallel() - - org := envvar.GetTestOrgFromEnv(t) - billingId := envvar.GetTestBillingAccountFromEnv(t) - project := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - - name1 := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - name2 := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - name3 := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - name4 := fmt.Sprintf("tf-test-%d", acctest.RandInt(t)) - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - Steps: []resource.TestStep{ - { - ExternalProviders: map[string]resource.ExternalProvider{ - "google": { - VersionConstraint: "4.80.0", - Source: "hashicorp/google-beta", - }, - }, - Config: testProvider_versionfive_upgrades(project, org, billingId, name1, name2, name3, name4), - }, - { - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - Config: testProvider_versionfive_upgrades(project, org, billingId, name1, name2, name3, name4), - PlanOnly: true, - }, - { - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - ResourceName: "google_data_fusion_instance.unset", - ImportState: true, - ImportStateVerify: true, - }, - { - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - ResourceName: "google_data_fusion_instance.set", - ImportState: true, - ImportStateVerify: true, - }, - { - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - ResourceName: "google_data_fusion_instance.reference", - ImportState: true, - ImportStateVerify: true, - }, - }, - }) -} - -func TestProvider_versionfive_ignorereads_upgrade(t *testing.T) { - acctest.SkipIfVcr(t) - t.Parallel() - - var itName = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var tpName = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var igmName = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var autoscalerName = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - - diskName := fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - policyName := fmt.Sprintf("tf-test-policy-%s", acctest.RandString(t, 10)) - - endpointContext := map[string]interface{}{ - "random_suffix": acctest.RandString(t, 10), - } - - var itNameRegion = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var tpNameRegion = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var igmNameRegion = fmt.Sprintf("tf-test-%s", acctest.RandString(t, 10)) - var autoscalerNameRegion = fmt.Sprintf("tf-test-region-autoscaler-%s", acctest.RandString(t, 10)) - - policyContext := map[string]interface{}{ - "random_suffix": acctest.RandString(t, 10), - } - - attachmentContext := map[string]interface{}{ - "random_suffix": acctest.RandString(t, 10), - } - - acctest.VcrTest(t, resource.TestCase{ - PreCheck: func() { acctest.AccTestPreCheck(t) }, - CheckDestroy: resource.ComposeTestCheckFunc(testAccCheckComputeResourcePolicyDestroyProducer(t), - testAccCheckComputeRegionAutoscalerDestroyProducer(t), - testAccCheckComputeNetworkEndpointGroupDestroyProducer(t), - testAccCheckComputeAutoscalerDestroyProducer(t), - ), - Steps: []resource.TestStep{ - { - ExternalProviders: map[string]resource.ExternalProvider{ - "google": { - VersionConstraint: "4.80.0", - Source: "hashicorp/google-beta", - }, - }, - Config: testProvider_versionfive_upgrades_ignorereads(itName, tpName, igmName, autoscalerName, diskName, policyName, - itNameRegion, tpNameRegion, igmNameRegion, autoscalerNameRegion, endpointContext, - policyContext, attachmentContext), - }, - { - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), - Config: testProvider_versionfive_upgrades_ignorereads(itName, tpName, igmName, autoscalerName, diskName, policyName, - itNameRegion, tpNameRegion, igmNameRegion, autoscalerNameRegion, endpointContext, - policyContext, attachmentContext), - PlanOnly: true, - }, - }, - }) -} - -func testProvider_versionfive_upgrades(project, org, billing, name1, name2, name3, name4 string) string { - return fmt.Sprintf(` -resource "google_project" "host" { - project_id = "%s" - name = "%s" - org_id = "%s" - billing_account = "%s" -} - -resource "google_project_service" "dfapi" { - project = google_project.host.project_id - service = "datafusion.googleapis.com" - - disable_dependent_services = false -} - -resource "google_data_fusion_instance" "unset" { - name = "%s" - type = "BASIC" - options = { - prober_test_run = "true" - } -} - -resource "google_data_fusion_instance" "set" { - name = "%s" - region = "us-west1" - type = "BASIC" - options = { - prober_test_run = "true" - } -} - -resource "google_data_fusion_instance" "reference" { - project = google_project.host.project_id - name = "%s" - type = "DEVELOPER" - options = { - prober_test_run = "true" - } - zone = "us-west1-a" - depends_on = [ - google_project_service.dfapi - ] -} - -resource "google_redis_instance" "overridewithnonstandardlogic" { - name = "%s" - memory_size_gb = 1 - location_id = "us-south1-a" -} - - -`, project, project, org, billing, name1, name2, name3, name4) -} - -func testProvider_versionfive_upgrades_ignorereads(itName, tpName, igmName, autoscalerName, diskName, policyName, itNameRegion, tpNameRegion, igmNameRegion, autoscalerNameRegion string, endpointContext, policyContext, attachmentContext map[string]interface{}) string { - return testAccComputeAutoscaler_basic(itName, tpName, igmName, autoscalerName) + - testAccComputeDiskResourcePolicyAttachment_basic(diskName, policyName) + - testAccComputeNetworkEndpointGroup_networkEndpointGroup(endpointContext) + - testAccComputeRegionAutoscaler_basic(itNameRegion, tpNameRegion, igmNameRegion, autoscalerNameRegion) + - testAccComputeResourcePolicy_resourcePolicyBasicExample(policyContext) + - testAccComputeServiceAttachment_serviceAttachmentBasicExample(attachmentContext) -} - -// need to make copies of all the respective resource functions within here as *_test.go files can not be imported -// checkdestroys -func testAccCheckComputeResourcePolicyDestroyProducer(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_compute_resource_policy" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := acctest.GoogleProviderConfig(t) - - url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{ComputeBasePath}}projects/{{project}}/regions/{{region}}/resourcePolicies/{{name}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("ComputeResourcePolicy still exists at %s", url) - } - } - - return nil - } -} - -func testAccCheckComputeRegionAutoscalerDestroyProducer(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_compute_region_autoscaler" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := acctest.GoogleProviderConfig(t) - - url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{ComputeBasePath}}projects/{{project}}/regions/{{region}}/autoscalers/{{name}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("ComputeRegionAutoscaler still exists at %s", url) - } - } - - return nil - } -} - -func testAccCheckComputeNetworkEndpointGroupDestroyProducer(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_compute_network_endpoint_group" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := acctest.GoogleProviderConfig(t) - - url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{ComputeBasePath}}projects/{{project}}/zones/{{zone}}/networkEndpointGroups/{{name}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("ComputeNetworkEndpointGroup still exists at %s", url) - } - } - - return nil - } -} - -func testAccCheckComputeAutoscalerDestroyProducer(t *testing.T) func(s *terraform.State) error { - return func(s *terraform.State) error { - for name, rs := range s.RootModule().Resources { - if rs.Type != "google_compute_autoscaler" { - continue - } - if strings.HasPrefix(name, "data.") { - continue - } - - config := acctest.GoogleProviderConfig(t) - - url, err := tpgresource.ReplaceVarsForTest(config, rs, "{{ComputeBasePath}}projects/{{project}}/zones/{{zone}}/autoscalers/{{name}}") - if err != nil { - return err - } - - billingProject := "" - - if config.BillingProject != "" { - billingProject = config.BillingProject - } - - _, err = transport_tpg.SendRequest(transport_tpg.SendRequestOptions{ - Config: config, - Method: "GET", - Project: billingProject, - RawURL: url, - UserAgent: config.UserAgent, - }) - if err == nil { - return fmt.Errorf("ComputeAutoscaler still exists at %s", url) - } - } - - return nil - } -} - -// tests -func testAccComputeAutoscaler_scaffolding(itName, tpName, igmName string) string { - return fmt.Sprintf(` -data "google_compute_image" "my_image1" { - family = "debian-11" - project = "debian-cloud" -} - -resource "google_compute_instance_template" "foobar1" { - name = "%s" - machine_type = "e2-medium" - can_ip_forward = false - tags = ["foo", "bar"] - - disk { - source_image = data.google_compute_image.my_image1.self_link - auto_delete = true - boot = true - } - - network_interface { - network = "default" - } - - service_account { - scopes = ["userinfo-email", "compute-ro", "storage-ro"] - } -} - -resource "google_compute_target_pool" "foobar1" { - description = "Resource created for Terraform acceptance testing" - name = "%s" - session_affinity = "CLIENT_IP_PROTO" - region = "us-west1" -} - -resource "google_compute_instance_group_manager" "foobar1" { - description = "Terraform test instance group manager" - name = "%s" - version { - instance_template = google_compute_instance_template.foobar1.self_link - name = "primary" - } - target_pools = [google_compute_target_pool.foobar1.self_link] - base_instance_name = "foobar1" - zone = "us-west1-a" -} -`, itName, tpName, igmName) - -} - -func testAccComputeAutoscaler_basic(itName, tpName, igmName, autoscalerName string) string { - return testAccComputeAutoscaler_scaffolding(itName, tpName, igmName) + fmt.Sprintf(` -resource "google_compute_autoscaler" "foobar1" { - description = "Resource created for Terraform acceptance testing" - name = "%s" - zone = "us-west1-a" - target = google_compute_instance_group_manager.foobar1.self_link - autoscaling_policy { - max_replicas = 5 - min_replicas = 1 - cooldown_period = 60 - cpu_utilization { - target = 0.5 - } - } -} -`, autoscalerName) -} - -func testAccComputeDiskResourcePolicyAttachment_basic(diskName, policyName string) string { - return fmt.Sprintf(` -data "google_compute_image" "my_image2" { - family = "debian-11" - project = "debian-cloud" -} - -resource "google_compute_disk" "foobar2" { - name = "%s" - image = data.google_compute_image.my_image2.self_link - size = 1000 - type = "pd-extreme" - zone = "us-west1-c" - labels = { - my-label = "my-label-value" - } - provisioned_iops = 90000 -} - -resource "google_compute_resource_policy" "foobar2" { - name = "%s" - region = "us-west1" - snapshot_schedule_policy { - schedule { - daily_schedule { - days_in_cycle = 1 - start_time = "04:00" - } - } - } -} - -resource "google_compute_disk_resource_policy_attachment" "foobar2" { - name = google_compute_resource_policy.foobar2.name - disk = google_compute_disk.foobar2.name - zone = "us-west1-c" -} -`, diskName, policyName) -} - -func testAccComputeNetworkEndpointGroup_networkEndpointGroup(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_compute_network_endpoint_group" "neg3" { - name = "tf-test-my-lb-neg%{random_suffix}" - network = google_compute_network.default3.id - default_port = "90" - zone = "us-west1-a" -} - -resource "google_compute_network" "default3" { - name = "tf-test-neg-network%{random_suffix}" - auto_create_subnetworks = true -} -`, context) -} - -func testAccComputeRegionAutoscaler_scaffolding(itName, tpName, igmName string) string { - return fmt.Sprintf(` -data "google_compute_image" "my_image4" { - family = "debian-11" - project = "debian-cloud" -} - -resource "google_compute_instance_template" "foobar4" { - name = "%s" - machine_type = "e2-medium" - can_ip_forward = false - tags = ["foo", "bar"] - - disk { - source_image = data.google_compute_image.my_image4.self_link - auto_delete = true - boot = true - } - - network_interface { - network = "default" - } - - service_account { - scopes = ["userinfo-email", "compute-ro", "storage-ro"] - } -} - -resource "google_compute_target_pool" "foobar4" { - description = "Resource created for Terraform acceptance testing" - name = "%s" - session_affinity = "CLIENT_IP_PROTO" - region = "us-west1" -} - -resource "google_compute_region_instance_group_manager" "foobar4" { - description = "Terraform test instance group manager" - name = "%s" - version { - instance_template = google_compute_instance_template.foobar4.self_link - name = "primary" - } - target_pools = [google_compute_target_pool.foobar4.self_link] - base_instance_name = "tf-test-foobar4" - region = "us-west1" -} - -`, itName, tpName, igmName) -} - -func testAccComputeRegionAutoscaler_basic(itName, tpName, igmName, autoscalerName string) string { - return testAccComputeRegionAutoscaler_scaffolding(itName, tpName, igmName) + fmt.Sprintf(` -resource "google_compute_region_autoscaler" "foobar4" { - description = "Resource created for Terraform acceptance testing" - name = "%s" - region = "us-west1" - target = google_compute_region_instance_group_manager.foobar4.self_link - autoscaling_policy { - max_replicas = 5 - min_replicas = 0 - cooldown_period = 60 - cpu_utilization { - target = 0.5 - } - } -} -`, autoscalerName) -} - -func testAccComputeResourcePolicy_resourcePolicyBasicExample(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_compute_resource_policy" "foo5" { - name = "tf-test-gce-policy%{random_suffix}" - region = "us-west1" - snapshot_schedule_policy { - schedule { - daily_schedule { - days_in_cycle = 1 - start_time = "04:00" - } - } - } -} -`, context) -} - -func testAccComputeServiceAttachment_serviceAttachmentBasicExample(context map[string]interface{}) string { - return acctest.Nprintf(` -resource "google_compute_service_attachment" "psc_ilb_service_attachment6" { - name = "tf-test-my-psc-ilb%{random_suffix}" - region = "us-west2" - description = "A service attachment configured with Terraform" - - enable_proxy_protocol = true - connection_preference = "ACCEPT_AUTOMATIC" - nat_subnets = [google_compute_subnetwork.psc_ilb_nat6.id] - target_service = google_compute_forwarding_rule.psc_ilb_target_service6.id - reconcile_connections = true -} - -resource "google_compute_address" "psc_ilb_consumer_address6" { - name = "tf-test-psc-ilb-consumer-address%{random_suffix}" - region = "us-west2" - - subnetwork = "default" - address_type = "INTERNAL" -} - -resource "google_compute_forwarding_rule" "psc_ilb_consumer6" { - name = "tf-test-psc-ilb-consumer-forwarding-rule%{random_suffix}" - region = "us-west2" - - target = google_compute_service_attachment.psc_ilb_service_attachment6.id - load_balancing_scheme = "" # need to override EXTERNAL default when target is a service attachment - network = "default" - ip_address = google_compute_address.psc_ilb_consumer_address6.id -} - -resource "google_compute_forwarding_rule" "psc_ilb_target_service6" { - name = "tf-test-producer-forwarding-rule%{random_suffix}" - region = "us-west2" - - load_balancing_scheme = "INTERNAL" - backend_service = google_compute_region_backend_service.producer_service_backend6.id - all_ports = true - network = google_compute_network.psc_ilb_network6.name - subnetwork = google_compute_subnetwork.psc_ilb_producer_subnetwork6.name -} - -resource "google_compute_region_backend_service" "producer_service_backend6" { - name = "tf-test-producer-service%{random_suffix}" - region = "us-west2" - - health_checks = [google_compute_health_check.producer_service_health_check6.id] -} - -resource "google_compute_health_check" "producer_service_health_check6" { - name = "tf-test-producer-service-health-check%{random_suffix}" - - check_interval_sec = 1 - timeout_sec = 1 - tcp_health_check { - port = "80" - } -} - -resource "google_compute_network" "psc_ilb_network6" { - name = "tf-test-psc-ilb-network%{random_suffix}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "psc_ilb_producer_subnetwork6" { - name = "tf-test-psc-ilb-producer-subnetwork%{random_suffix}" - region = "us-west2" - - network = google_compute_network.psc_ilb_network6.id - ip_cidr_range = "10.0.0.0/16" -} - -resource "google_compute_subnetwork" "psc_ilb_nat6" { - name = "tf-test-psc-ilb-nat%{random_suffix}" - region = "us-west2" - - network = google_compute_network.psc_ilb_network6.id - purpose = "PRIVATE_SERVICE_CONNECT" - ip_cidr_range = "10.1.0.0/16" -} -`, context) -} diff --git a/mmv1/third_party/terraform/website/docs/r/compute_firewall_policy_rule.html.markdown b/mmv1/third_party/terraform/website/docs/r/compute_firewall_policy_rule.html.markdown deleted file mode 100644 index 741feff67645..000000000000 --- a/mmv1/third_party/terraform/website/docs/r/compute_firewall_policy_rule.html.markdown +++ /dev/null @@ -1,207 +0,0 @@ ---- -# ---------------------------------------------------------------------------- -# -# *** AUTO GENERATED CODE *** Type: DCL *** -# -# ---------------------------------------------------------------------------- -# -# This file is managed by Magic Modules (https:#github.com/GoogleCloudPlatform/magic-modules) -# and is based on the DCL (https:#github.com/GoogleCloudPlatform/declarative-resource-client-library). -# Changes will need to be made to the DCL or Magic Modules instead of here. -# -# We are not currently able to accept contributions to this file. If changes -# are required, please file an issue at https:#github.com/hashicorp/terraform-provider-google/issues/new/choose -# -# ---------------------------------------------------------------------------- -subcategory: "Compute Engine" -description: |- - Specific rules to add to a hierarchical firewall policy ---- - -# google\_compute\_firewall\_policy\_rule - -Hierarchical firewall policy rules let you create and enforce a consistent firewall policy across your organization. Rules can explicitly allow or deny connections or delegate evaluation to lower level policies. - -For more information see the [official documentation](https://cloud.google.com/vpc/docs/using-firewall-policies#create-rules) - -## Example Usage - -```hcl -resource "google_network_security_address_group" "basic_global_networksecurity_address_group" { - provider = google-beta - - name = "policy" - parent = "organizations/12345" - description = "Sample global networksecurity_address_group" - location = "global" - items = ["208.80.154.224/32"] - type = "IPV4" - capacity = 100 -} - -resource "google_compute_firewall_policy" "default" { - parent = "organizations/12345" - short_name = "my-policy" - description = "Example Resource" -} - -resource "google_compute_firewall_policy_rule" "default" { - firewall_policy = google_compute_firewall_policy.default.id - description = "Example Resource" - priority = 9000 - enable_logging = true - action = "allow" - direction = "EGRESS" - disabled = false - match { - layer4_configs { - ip_protocol = "tcp" - ports = [80, 8080] - } - dest_ip_ranges = ["11.100.0.1/32"] - dest_fqdns = ["google.com"] - dest_region_codes = ["US"] - dest_threat_intelligences = ["iplist-public-clouds"] - dest_address_groups = [google_network_security_address_group.basic_global_networksecurity_address_group.id] - } -} -``` - -## Argument Reference - -The following arguments are supported: - -* `action` - - (Required) - The Action to perform when the client connection triggers the rule. Can currently be either "allow" or "deny()" where valid values for status are 403, 404, and 502. - -* `direction` - - (Required) - The direction in which this rule applies. Possible values: INGRESS, EGRESS - -* `firewall_policy` - - (Required) - The firewall policy of the resource. - -* `match` - - (Required) - A match condition that incoming traffic is evaluated against. If it evaluates to true, the corresponding 'action' is enforced. Structure is [documented below](#nested_match). - -* `priority` - - (Required) - An integer indicating the priority of a rule in the list. The priority must be a positive value between 0 and 2147483647. Rules are evaluated from highest to lowest priority where 0 is the highest priority and 2147483647 is the lowest prority. - - - -The `match` block supports: - -* `dest_address_groups` - - (Optional) - Address groups which should be matched against the traffic destination. Maximum number of destination address groups is 10. Destination address groups is only supported in Egress rules. - -* `dest_fqdns` - - (Optional) - Domain names that will be used to match against the resolved domain name of destination of traffic. Can only be specified if DIRECTION is egress. - -* `dest_ip_ranges` - - (Optional) - CIDR IP address range. Maximum number of destination CIDR IP ranges allowed is 5000. - -* `dest_region_codes` - - (Optional) - The Unicode country codes whose IP addresses will be used to match against the source of traffic. Can only be specified if DIRECTION is egress. - -* `dest_threat_intelligences` - - (Optional) - Name of the Google Cloud Threat Intelligence list. - -* `layer4_configs` - - (Required) - Pairs of IP protocols and ports that the rule should match. Structure is [documented below](#nested_layer4_configs). - -* `src_address_groups` - - (Optional) - Address groups which should be matched against the traffic source. Maximum number of source address groups is 10. Source address groups is only supported in Ingress rules. - -* `src_fqdns` - - (Optional) - Domain names that will be used to match against the resolved domain name of source of traffic. Can only be specified if DIRECTION is ingress. - -* `src_ip_ranges` - - (Optional) - CIDR IP address range. Maximum number of source CIDR IP ranges allowed is 5000. - -* `src_region_codes` - - (Optional) - The Unicode country codes whose IP addresses will be used to match against the source of traffic. Can only be specified if DIRECTION is ingress. - -* `src_threat_intelligences` - - (Optional) - Name of the Google Cloud Threat Intelligence list. - -The `layer4_configs` block supports: - -* `ip_protocol` - - (Required) - The IP protocol to which this rule applies. The protocol type is required when creating a firewall rule. This value can either be one of the following well known protocol strings (`tcp`, `udp`, `icmp`, `esp`, `ah`, `ipip`, `sctp`), or the IP protocol number. - -* `ports` - - (Optional) - An optional list of ports to which this rule applies. This field is only applicable for UDP or TCP protocol. Each entry must be either an integer or a range. If not specified, this rule applies to connections through any port. Example inputs include: ``. - -- - - - -* `description` - - (Optional) - An optional description for this resource. - -* `disabled` - - (Optional) - Denotes whether the firewall policy rule is disabled. When set to true, the firewall policy rule is not enforced and traffic behaves as if it did not exist. If this is unspecified, the firewall policy rule will be enabled. - -* `enable_logging` - - (Optional) - Denotes whether to enable logging for a particular rule. If logging is enabled, logs will be exported to the configured export destination in Stackdriver. Logs may be exported to BigQuery or Pub/Sub. Note: you cannot enable logging on "goto_next" rules. - -* `target_resources` - - (Optional) - A list of network resource URLs to which this rule applies. This field allows you to control which network's VMs get this rule. If this field is left blank, all VMs within the organization will receive the rule. - -* `target_service_accounts` - - (Optional) - A list of service accounts indicating the sets of instances that are applied with this rule. - - - -## Attributes Reference - -In addition to the arguments listed above, the following computed attributes are exported: - -* `id` - an identifier for the resource with format `locations/global/firewallPolicies/{{firewall_policy}}/rules/{{priority}}` - -* `kind` - - Type of the resource. Always `compute#firewallPolicyRule` for firewall policy rules - -* `rule_tuple_count` - - Calculation of the complexity of a single firewall policy rule. - -## Timeouts - -This resource provides the following -[Timeouts](https://developer.hashicorp.com/terraform/plugin/sdkv2/resources/retries-and-customizable-timeouts) configuration options: configuration options: - -- `create` - Default is 20 minutes. -- `update` - Default is 20 minutes. -- `delete` - Default is 20 minutes. - -## Import - -FirewallPolicyRule can be imported using any of these accepted formats: - -``` -$ terraform import google_compute_firewall_policy_rule.default locations/global/firewallPolicies/{{firewall_policy}}/rules/{{priority}} -$ terraform import google_compute_firewall_policy_rule.default {{firewall_policy}}/{{priority}} -``` - - -