diff --git a/mmv1/products/pubsub/Topic.yaml b/mmv1/products/pubsub/Topic.yaml index 3446dc4571ae..8b539c7edd93 100644 --- a/mmv1/products/pubsub/Topic.yaml +++ b/mmv1/products/pubsub/Topic.yaml @@ -83,6 +83,10 @@ examples: primary_resource_id: 'example' vars: topic_name: 'example-topic' + - name: 'pubsub_topic_ingestion_cloud_storage' + primary_resource_id: 'example' + vars: + topic_name: 'example-topic' parameters: properties: - name: 'name' @@ -166,6 +170,9 @@ properties: type: NestedObject description: | Settings for ingestion from Amazon Kinesis Data Streams. + conflicts: + - 'aws_kinesis' + - 'cloud_storage' properties: - name: 'streamArn' type: String @@ -194,3 +201,100 @@ properties: role). The `awsRoleArn` must be set up with `accounts.google.com:sub` equals to this service account number. required: true + - name: 'cloudStorage' + type: NestedObject + description: Settings for ingestion from Cloud Storage. + conflicts: + - 'aws_kinesis' + - 'cloud_storage' + properties: + - name: 'bucket' + type: String + description: | + Cloud Storage bucket. The bucket name must be without any + prefix like "gs://". See the bucket naming requirements: + https://cloud.google.com/storage/docs/buckets#naming. + required: true + - name: 'textFormat' + type: NestedObject + description: | + Configuration for reading Cloud Storage data in text format. Each line of + text as specified by the delimiter will be set to the `data` field of a + Pub/Sub message. + exactly_one_of: + - 'text_format' + - 'avro_format' + - 'pubsub_avro_format' + properties: + - name: 'delimiter' + type: String + description: | + The delimiter to use when using the 'text' format. Each line of text as + specified by the delimiter will be set to the 'data' field of a Pub/Sub + message. When unset, '\n' is used. + required: false + default_value: "\\n" + - name: 'avroFormat' + type: NestedObject + description: | + Configuration for reading Cloud Storage data in Avro binary format. The + bytes of each object will be set to the `data` field of a Pub/Sub message. + send_empty_value: true + allow_empty_object: true + exactly_one_of: + - 'text_format' + - 'avro_format' + - 'pubsub_avro_format' + properties: + # Meant to be an empty object with no properties. + [] + - name: 'pubsubAvroFormat' + type: NestedObject + description: | + Configuration for reading Cloud Storage data written via Cloud Storage + subscriptions(See https://cloud.google.com/pubsub/docs/cloudstorage). The + data and attributes fields of the originally exported Pub/Sub message + will be restored when publishing. + send_empty_value: true + allow_empty_object: true + exactly_one_of: + - 'text_format' + - 'avro_format' + - 'pubsub_avro_format' + properties: + # Meant to be an empty object with no properties. + [] + - name: 'minimumObjectCreateTime' + type: String + description: | + The timestamp set in RFC3339 text format. If set, only objects with a + larger or equal timestamp will be ingested. Unset by default, meaning + all objects will be ingested. + required: false + - name: 'matchGlob' + type: String + description: | + Glob pattern used to match objects that will be ingested. If unset, all + objects will be ingested. See the supported patterns: + https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-objects-and-prefixes-using-glob + required: false + - name: 'platformLogsSettings' + type: NestedObject + description: | + Settings for Platform Logs regarding ingestion to Pub/Sub. If unset, + no Platform Logs will be generated.' + required: false + properties: + - name: 'severity' + type: Enum + description: | + The minimum severity level of Platform Logs that will be written. If unspecified, + no Platform Logs will be written. + default_value: "SEVERITY_UNSPECIFIED" + enum_values: + - 'SEVERITY_UNSPECIFIED' + - 'DISABLED' + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' diff --git a/mmv1/templates/terraform/examples/pubsub_topic_ingestion_cloud_storage.tf.tmpl b/mmv1/templates/terraform/examples/pubsub_topic_ingestion_cloud_storage.tf.tmpl new file mode 100644 index 000000000000..064307735cd9 --- /dev/null +++ b/mmv1/templates/terraform/examples/pubsub_topic_ingestion_cloud_storage.tf.tmpl @@ -0,0 +1,18 @@ +resource "google_pubsub_topic" "{{$.PrimaryResourceId}}" { + name = "{{index $.Vars "topic_name"}}" + + # Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass. + ingestion_data_source_settings { + cloud_storage { + bucket = "test-bucket" + text_format { + delimiter = " " + } + minimum_object_create_time = "2024-01-01T00:00:00Z" + match_glob = "foo/**" + } + platform_logs_settings { + severity = "WARNING" + } + } +} diff --git a/mmv1/third_party/terraform/services/pubsub/resource_pubsub_topic_test.go b/mmv1/third_party/terraform/services/pubsub/resource_pubsub_topic_test.go index 9cce95d53b27..38d02051d76e 100644 --- a/mmv1/third_party/terraform/services/pubsub/resource_pubsub_topic_test.go +++ b/mmv1/third_party/terraform/services/pubsub/resource_pubsub_topic_test.go @@ -175,6 +175,38 @@ func TestAccPubsubTopic_kinesisIngestionUpdate(t *testing.T) { }) } +func TestAccPubsubTopic_cloudStorageIngestionUpdate(t *testing.T) { + t.Parallel() + + topic := fmt.Sprintf("tf-test-topic-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckPubsubTopicDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccPubsubTopic_updateWithCloudStorageIngestionSettings(topic), + }, + { + ResourceName: "google_pubsub_topic.foo", + ImportStateId: topic, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccPubsubTopic_updateWithUpdatedCloudStorageIngestionSettings(topic), + }, + { + ResourceName: "google_pubsub_topic.foo", + ImportStateId: topic, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func testAccPubsubTopic_update(topic, key, value string) string { return fmt.Sprintf(` resource "google_pubsub_topic" "foo" { @@ -291,3 +323,47 @@ resource "google_pubsub_topic" "foo" { } `, topic) } + +func testAccPubsubTopic_updateWithCloudStorageIngestionSettings(topic string) string { + return fmt.Sprintf(` +resource "google_pubsub_topic" "foo" { + name = "%s" + + # Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass. + ingestion_data_source_settings { + cloud_storage { + bucket = "test-bucket" + text_format { + delimiter = " " + } + minimum_object_create_time = "2024-01-01T00:00:00Z" + match_glob = "foo/**" + } + platform_logs_settings { + severity = "WARNING" + } + } +} +`, topic) +} + +func testAccPubsubTopic_updateWithUpdatedCloudStorageIngestionSettings(topic string) string { + return fmt.Sprintf(` +resource "google_pubsub_topic" "foo" { + name = "%s" + + # Outside of automated terraform-provider-google CI tests, these values must be of actual Cloud Storage resources for the test to pass. + ingestion_data_source_settings { + cloud_storage { + bucket = "updated-test-bucket" + avro_format {} + minimum_object_create_time = "2024-02-02T00:00:00Z" + match_glob = "bar/**" + } + platform_logs_settings { + severity = "ERROR" + } + } +} +`, topic) +}