From 1b1cf168cb302b4eecc035f16d9bc1bde14162f2 Mon Sep 17 00:00:00 2001 From: Ray Date: Fri, 12 Jan 2024 13:41:18 -0800 Subject: [PATCH] Discovery Engine Data Store (#9682) * Create a configuration file product.yaml for product discoveryengine * Create resource DataStore configuration file template DataStore.yaml * Modify resource DataStore configuration file template to match the API resource's documented behavior * Add each API resource field to configuration file properties attribute * Add IAM support in Terraform to match DataStore API resource IAM support * Polish field contents in product configuration file. * Modify DataStore.yaml for LRO handling, parameters matching and readability improvement. * Add a DataStore create test. * Modify product name from DiscoveryEngine to Discoveryengine so that generated resource name would be google_discoveryengine_data_store * change to GA in doc (#9491) Co-authored-by: Edward Sun * Add value of to template into the generated tests to fix error 'fmt.Sprintf format %s reads arg #3, but call has 2 args'. * Remove Api::Product::ApiReference to match the change https://github.com/GoogleCloudPlatform/magic-modules/pull/9536 * Add var collection_id to unit test examples in resource config. * Fix argument missing error in unit tests. * Update resource create unit test case fields to contain display_name, industry_vertical, content_config. * Fix errors in acceptance tests: now acc test fails at invalid parent field of CreateDataStoreRequest. * Set collection_id as default_collection. * Remove uncessary variables under unit test example. * Fix acceptance tests issues by (1) removing iam_policy and (2) adding import_format. * Clean up configs: (1) update api version for tpg-beta to v1alpha; (2) remove fields for std Update method; (3) rename structured datastore example. * Add examples and tests for all types of DataStore. * Remove redundant field project from create test examples. * Fix lint-yaml check failure. * Add field solution_types to acceptance tests to satisfy coverage requirements. * Clean up industry_vertical enum values. * Add a new query parameter createAdvancedSiteSearch to create_url. * Add a create test for advanced site search data store. * Add IAM roles/discoveryengine.admin. * Remove role config from .ci/infra/terraform/main.tf * Remove beta version in product.yaml because both ga and beta versions have the same base_url. * Rename product from Discoveryengine to DiscoveryEngine. * Remove unnecessary configs and improve description readability. * Mark property solution_types as immutable. * Add update related settings and update test. * Add delete_url for resource's Delete method. * Remove all but one test example. * Rename test example to make it more generic. * Modify first template in update test to omit all non-required fields. * Fix the update test: update display_name not data_store_id. * Modify property description: remove brackets. --------- Co-authored-by: Jialei Chen Co-authored-by: Edward Sun <42220489+edwardmedia@users.noreply.github.com> Co-authored-by: Edward Sun Co-authored-by: jialei-chen <147877028+jialei-chen@users.noreply.github.com> --- mmv1/products/discoveryengine/DataStore.yaml | 146 ++++++++++++++++++ mmv1/products/discoveryengine/product.yaml | 22 +++ .../discoveryengine_datastore_basic.tf.erb | 9 ++ ...source_discovery_engine_data_store_test.go | 67 ++++++++ 4 files changed, 244 insertions(+) create mode 100644 mmv1/products/discoveryengine/DataStore.yaml create mode 100644 mmv1/products/discoveryengine/product.yaml create mode 100644 mmv1/templates/terraform/examples/discoveryengine_datastore_basic.tf.erb create mode 100644 mmv1/third_party/terraform/services/discoveryengine/resource_discovery_engine_data_store_test.go diff --git a/mmv1/products/discoveryengine/DataStore.yaml b/mmv1/products/discoveryengine/DataStore.yaml new file mode 100644 index 000000000000..0c6c70afff2e --- /dev/null +++ b/mmv1/products/discoveryengine/DataStore.yaml @@ -0,0 +1,146 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +name: 'DataStore' +description: | + Data store is a collection of websites and documents used to find answers for + end-user's questions in Discovery Engine (a.k.a. Vertex AI Search and + Conversation). +references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Create a search data store': 'https://cloud.google.com/generative-ai-app-builder/docs/create-data-store-es' + api: 'https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1alpha/projects.locations.collections.dataStores' + +base_url: 'projects/{{project}}/locations/{{location}}/collections/default_collection/dataStores' +self_link: 'projects/{{project}}/locations/{{location}}/collections/default_collection/dataStores/{{data_store_id}}' +create_url: 'projects/{{project}}/locations/{{location}}/collections/default_collection/dataStores?dataStoreId={{data_store_id}}&createAdvancedSiteSearch={{create_advanced_site_search}}' +update_verb: :PATCH +update_mask: true +delete_url: 'projects/{{project}}/locations/{{location}}/collections/default_collection/dataStores/{{data_store_id}}' +import_format: + [ + 'projects/{{project}}/locations/{{location}}/collections/default_collection/dataStores/{{data_store_id}}', + ] + +autogen_async: true +async: !ruby/object:Api::OpAsync + operation: !ruby/object:Api::OpAsync::Operation + path: "name" + base_url: "{{op_id}}" + wait_ms: 1000 + timeouts: !ruby/object:Api::Timeouts + insert_minutes: 60 + update_minutes: 60 + delete_minutes: 60 + result: !ruby/object:Api::OpAsync::Result + path: "response" + status: !ruby/object:Api::OpAsync::Status + path: "done" + complete: true + allowed: + - true + - false + error: !ruby/object:Api::OpAsync::Error + path: "error" + message: "message" + +examples: + - !ruby/object:Provider::Terraform::Examples + name: "discoveryengine_datastore_basic" + primary_resource_id: 'basic' + primary_resource_name: + 'fmt.Sprintf("tf_test_data_store%s", context["random_suffix"])' + vars: + data_store_id: "data-store-id" +parameters: + - !ruby/object:Api::Type::String + name: 'location' + required: true + immutable: true + url_param_only: true + description: | + The geographic location where the data store should reside. + - !ruby/object:Api::Type::String + name: 'dataStoreId' + required: true + immutable: true + url_param_only: true + description: | + The unique id of the data store. + - !ruby/object:Api::Type::Boolean + name: 'createAdvancedSiteSearch' + default_value: false + url_param_only: true + description: | + If true, an advanced data store for site search will be created. If the + data store is not configured as site search (GENERIC vertical and + PUBLIC_WEBSITE contentConfig), this flag will be ignored. + +properties: + - !ruby/object:Api::Type::String + name: 'name' + description: | + The unique full resource name of the data store. Values are of the format + `projects/{project}/locations/{location}/collections/{collection_id}/dataStores/{data_store_id}`. + This field must be a UTF-8 encoded string with a length limit of 1024 + characters. + output: true + - !ruby/object:Api::Type::String + name: 'displayName' + description: | + The display name of the data store. This field must be a UTF-8 encoded + string with a length limit of 128 characters. + required: true + - !ruby/object:Api::Type::Enum + name: 'industryVertical' + description: | + The industry vertical that the data store registers. + values: + - :GENERIC + - :MEDIA + immutable: true + required: true + - !ruby/object:Api::Type::Array + name: 'solutionTypes' + description: | + The solutions that the data store enrolls. + item_type: !ruby/object:Api::Type::Enum + name: 'solutionType' + description: | + The type of solution. + values: + - :SOLUTION_TYPE_RECOMMENDATION + - :SOLUTION_TYPE_SEARCH + - :SOLUTION_TYPE_CHAT + immutable: true + - !ruby/object:Api::Type::String + name: 'defaultSchemaId' + description: | + The id of the default Schema associated with this data store. + output: true + - !ruby/object:Api::Type::Enum + name: 'contentConfig' + description: | + The content config of the data store. + values: + - :NO_CONTENT + - :CONTENT_REQUIRED + - :PUBLIC_WEBSITE + immutable: true + required: true + - !ruby/object:Api::Type::Time + name: "createTime" + description: | + Timestamp when the DataStore was created. + output: true diff --git a/mmv1/products/discoveryengine/product.yaml b/mmv1/products/discoveryengine/product.yaml new file mode 100644 index 000000000000..d3bd3cc11270 --- /dev/null +++ b/mmv1/products/discoveryengine/product.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Product +name: DiscoveryEngine +display_name: Discovery Engine +versions: + - !ruby/object:Api::Product::Version + name: ga + base_url: https://discoveryengine.googleapis.com/v1alpha/ +scopes: + - https://www.googleapis.com/auth/cloud-platform diff --git a/mmv1/templates/terraform/examples/discoveryengine_datastore_basic.tf.erb b/mmv1/templates/terraform/examples/discoveryengine_datastore_basic.tf.erb new file mode 100644 index 000000000000..f69c1fdd320c --- /dev/null +++ b/mmv1/templates/terraform/examples/discoveryengine_datastore_basic.tf.erb @@ -0,0 +1,9 @@ +resource "google_discovery_engine_data_store" "basic" { + location = "global" + data_store_id = "<%= ctx[:vars]['data_store_id'] %>" + display_name = "tf-test-structured-datastore" + industry_vertical = "GENERIC" + content_config = "NO_CONTENT" + solution_types = ["SOLUTION_TYPE_SEARCH"] + create_advanced_site_search = false +} \ No newline at end of file diff --git a/mmv1/third_party/terraform/services/discoveryengine/resource_discovery_engine_data_store_test.go b/mmv1/third_party/terraform/services/discoveryengine/resource_discovery_engine_data_store_test.go new file mode 100644 index 000000000000..fc324b9f601d --- /dev/null +++ b/mmv1/third_party/terraform/services/discoveryengine/resource_discovery_engine_data_store_test.go @@ -0,0 +1,67 @@ +package discoveryengine_test + +import ( + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" + "testing" +) + +func TestAccDiscoveryEngineDataStore_discoveryengineDatastoreBasicExample_update(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDiscoveryEngineDataStoreDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDiscoveryEngineDataStore_discoveryengineDatastoreBasicExample_basic(context), + }, + { + ResourceName: "google_discovery_engine_data_store.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "data_store_id", "create_advanced_site_search"}, + }, + { + Config: testAccDiscoveryEngineDataStore_discoveryengineDatastoreBasicExample_update(context), + }, + { + ResourceName: "google_discovery_engine_data_store.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "data_store_id", "create_advanced_site_search"}, + }, + }, + }) +} + +func testAccDiscoveryEngineDataStore_discoveryengineDatastoreBasicExample_basic(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_discovery_engine_data_store" "basic" { + location = "global" + data_store_id = "tf-test-data-store-id%{random_suffix}" + display_name = "tf-test-structured-datastore" + industry_vertical = "GENERIC" + content_config = "NO_CONTENT" +} +`, context) +} + +func testAccDiscoveryEngineDataStore_discoveryengineDatastoreBasicExample_update(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_discovery_engine_data_store" "basic" { + location = "global" + data_store_id = "tf-test-data-store-id%{random_suffix}" + display_name = "updated-tf-test-structured-datastore" + industry_vertical = "GENERIC" + content_config = "NO_CONTENT" + solution_types = ["SOLUTION_TYPE_SEARCH"] + create_advanced_site_search = false +} +`, context) +}