diff --git a/.changelog/36646.txt b/.changelog/36646.txt new file mode 100644 index 00000000000..9539e1a81d5 --- /dev/null +++ b/.changelog/36646.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_kinesis_firehose_delivery_stream: Add `snowflake_configuration` argument +``` \ No newline at end of file diff --git a/internal/acctest/crypto.go b/internal/acctest/crypto.go index d1745a5835d..d606e4351d0 100644 --- a/internal/acctest/crypto.go +++ b/internal/acctest/crypto.go @@ -32,6 +32,12 @@ var ( tlsX509CertificateSerialNumberLimit = new(big.Int).Lsh(big.NewInt(1), 128) //nolint:gomnd ) +// TLSPEMRemoveRSAPrivateKeyEncapsulationBoundaries removes RSA private key +// pre and post encapsulation boundaries from a PEM string. +func TLSPEMRemoveRSAPrivateKeyEncapsulationBoundaries(pem string) string { + return removePEMEncapsulationBoundaries(pem, PEMBlockTypeRSAPrivateKey) +} + // TLSPEMRemovePublicKeyEncapsulationBoundaries removes public key // pre and post encapsulation boundaries from a PEM string. func TLSPEMRemovePublicKeyEncapsulationBoundaries(pem string) string { diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index 71b95012532..4262ef50cf1 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -41,6 +41,7 @@ const ( destinationTypeOpenSearch destinationType = "opensearch" destinationTypeOpenSearchServerless destinationType = "opensearchserverless" destinationTypeRedshift destinationType = "redshift" + destinationTypeSnowflake destinationType = "snowflake" destinationTypeSplunk destinationType = "splunk" ) @@ -52,6 +53,7 @@ func (destinationType) Values() []destinationType { destinationTypeOpenSearch, destinationTypeOpenSearchServerless, destinationTypeRedshift, + destinationTypeSnowflake, destinationTypeSplunk, } } @@ -977,6 +979,119 @@ func resourceDeliveryStream() *schema.Resource { }, }, }, + "snowflake_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "account_url": { + Type: schema.TypeString, + Required: true, + }, + "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), + "content_column_name": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "data_loading_option": { + Type: schema.TypeString, + Optional: true, + Default: types.SnowflakeDataLoadingOptionJsonMapping, + ValidateDiagFunc: enum.Validate[types.SnowflakeDataLoadingOption](), + }, + "database": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "key_passphrase": { + Type: schema.TypeString, + Optional: true, + Sensitive: true, + ValidateFunc: validation.StringLenBetween(7, 255), + }, + "metadata_column_name": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "private_key": { + Type: schema.TypeString, + Required: true, + Sensitive: true, + }, + "processing_configuration": processingConfigurationSchema(), + "retry_duration": { + Type: schema.TypeInt, + Optional: true, + Default: 60, + ValidateFunc: validation.IntBetween(0, 7200), + }, + "role_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: verify.ValidARN, + }, + "s3_backup_mode": { + Type: schema.TypeString, + Optional: true, + Default: types.SnowflakeS3BackupModeFailedDataOnly, + ValidateDiagFunc: enum.Validate[types.SnowflakeS3BackupMode](), + }, + "s3_configuration": s3ConfigurationSchema(), + "schema": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "snowflake_role_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "snowflake_role": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + }, + }, + DiffSuppressFunc: verify.SuppressMissingOptionalConfigurationBlock, + }, + "snowflake_vpc_configuration": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "private_link_vpce_id": { + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, + "table": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + "user": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringLenBetween(1, 255), + }, + }, + }, + }, "opensearchserverless_configuration": { Type: schema.TypeList, Optional: true, @@ -1220,6 +1335,7 @@ func resourceDeliveryStream() *schema.Resource { destinationTypeOpenSearch: "opensearch_configuration", destinationTypeOpenSearchServerless: "opensearchserverless_configuration", destinationTypeRedshift: "redshift_configuration", + destinationTypeSnowflake: "snowflake_configuration", destinationTypeSplunk: "splunk_configuration", }[destination] @@ -1277,6 +1393,10 @@ func resourceDeliveryStreamCreate(ctx context.Context, d *schema.ResourceData, m if v, ok := d.GetOk("redshift_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { input.RedshiftDestinationConfiguration = expandRedshiftDestinationConfiguration(v.([]interface{})[0].(map[string]interface{})) } + case destinationTypeSnowflake: + if v, ok := d.GetOk("snowflake_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { + input.SnowflakeDestinationConfiguration = expandSnowflakeDestinationConfiguration(v.([]interface{})[0].(map[string]interface{})) + } case destinationTypeSplunk: if v, ok := d.GetOk("splunk_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { input.SplunkDestinationConfiguration = expandSplunkDestinationConfiguration(v.([]interface{})[0].(map[string]interface{})) @@ -1398,6 +1518,13 @@ func resourceDeliveryStreamRead(ctx context.Context, d *schema.ResourceData, met if err := d.Set("redshift_configuration", flattenRedshiftDestinationDescription(destination.RedshiftDestinationDescription, configuredPassword)); err != nil { return sdkdiag.AppendErrorf(diags, "setting redshift_configuration: %s", err) } + case destination.SnowflakeDestinationDescription != nil: + d.Set("destination", destinationTypeSnowflake) + configuredKeyPassphrase := d.Get("snowflake_configuration.0.key_passphrase").(string) + configuredPrivateKey := d.Get("snowflake_configuration.0.private_key").(string) + if err := d.Set("snowflake_configuration", flattenSnowflakeDestinationDescription(destination.SnowflakeDestinationDescription, configuredKeyPassphrase, configuredPrivateKey)); err != nil { + return sdkdiag.AppendErrorf(diags, "setting snowflake_configuration: %s", err) + } case destination.SplunkDestinationDescription != nil: d.Set("destination", destinationTypeSplunk) if err := d.Set("splunk_configuration", flattenSplunkDestinationDescription(destination.SplunkDestinationDescription)); err != nil { @@ -1453,6 +1580,10 @@ func resourceDeliveryStreamUpdate(ctx context.Context, d *schema.ResourceData, m if v, ok := d.GetOk("redshift_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { input.RedshiftDestinationUpdate = expandRedshiftDestinationUpdate(v.([]interface{})[0].(map[string]interface{})) } + case destinationTypeSnowflake: + if v, ok := d.GetOk("snowflake_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { + input.SnowflakeDestinationUpdate = expandSnowflakeDestinationUpdate(v.([]interface{})[0].(map[string]interface{})) + } case destinationTypeSplunk: if v, ok := d.GetOk("splunk_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { input.SplunkDestinationUpdate = expandSplunkDestinationUpdate(v.([]interface{})[0].(map[string]interface{})) @@ -2515,6 +2646,108 @@ func expandAmazonOpenSearchServerlessDestinationUpdate(oss map[string]interface{ return update } +func expandSnowflakeDestinationConfiguration(tfMap map[string]interface{}) *types.SnowflakeDestinationConfiguration { + roleARN := tfMap["role_arn"].(string) + apiObject := &types.SnowflakeDestinationConfiguration{ + AccountUrl: aws.String(tfMap["account_url"].(string)), + Database: aws.String(tfMap["database"].(string)), + PrivateKey: aws.String(tfMap["private_key"].(string)), + RetryOptions: expandSnowflakeRetryOptions(tfMap), + RoleARN: aws.String(roleARN), + S3Configuration: expandS3DestinationConfiguration(tfMap["s3_configuration"].([]interface{})), + Schema: aws.String(tfMap["schema"].(string)), + Table: aws.String(tfMap["table"].(string)), + User: aws.String(tfMap["user"].(string)), + } + + if _, ok := tfMap["cloudwatch_logging_options"]; ok { + apiObject.CloudWatchLoggingOptions = expandCloudWatchLoggingOptions(tfMap) + } + + if v, ok := tfMap["content_column_name"]; ok && v.(string) != "" { + apiObject.ContentColumnName = aws.String(v.(string)) + } + + if v, ok := tfMap["data_loading_option"]; ok && v.(string) != "" { + apiObject.DataLoadingOption = types.SnowflakeDataLoadingOption(v.(string)) + } + + if v, ok := tfMap["key_passphrase"]; ok && v.(string) != "" { + apiObject.KeyPassphrase = aws.String(v.(string)) + } + + if v, ok := tfMap["metadata_column_name"]; ok && v.(string) != "" { + apiObject.MetaDataColumnName = aws.String(v.(string)) + } + + if _, ok := tfMap["processing_configuration"]; ok { + apiObject.ProcessingConfiguration = expandProcessingConfiguration(tfMap, destinationTypeSnowflake, roleARN) + } + + if v, ok := tfMap["s3_backup_mode"]; ok { + apiObject.S3BackupMode = types.SnowflakeS3BackupMode(v.(string)) + } + + if _, ok := tfMap["snowflake_role_configuration"]; ok { + apiObject.SnowflakeRoleConfiguration = expandSnowflakeRoleConfiguration(tfMap) + } + + if _, ok := tfMap["snowflake_vpc_configuration"]; ok { + apiObject.SnowflakeVpcConfiguration = expandSnowflakeVPCConfiguration(tfMap) + } + + return apiObject +} + +func expandSnowflakeDestinationUpdate(tfMap map[string]interface{}) *types.SnowflakeDestinationUpdate { + roleARN := tfMap["role_arn"].(string) + apiObject := &types.SnowflakeDestinationUpdate{ + AccountUrl: aws.String(tfMap["account_url"].(string)), + Database: aws.String(tfMap["database"].(string)), + PrivateKey: aws.String(tfMap["private_key"].(string)), + RetryOptions: expandSnowflakeRetryOptions(tfMap), + RoleARN: aws.String(roleARN), + S3Update: expandS3DestinationUpdate(tfMap["s3_configuration"].([]interface{})), + Schema: aws.String(tfMap["schema"].(string)), + Table: aws.String(tfMap["table"].(string)), + User: aws.String(tfMap["user"].(string)), + } + + if _, ok := tfMap["cloudwatch_logging_options"]; ok { + apiObject.CloudWatchLoggingOptions = expandCloudWatchLoggingOptions(tfMap) + } + + if v, ok := tfMap["content_column_name"]; ok && v.(string) != "" { + apiObject.ContentColumnName = aws.String(v.(string)) + } + + if v, ok := tfMap["data_loading_option"]; ok && v.(string) != "" { + apiObject.DataLoadingOption = types.SnowflakeDataLoadingOption(v.(string)) + } + + if v, ok := tfMap["key_passphrase"]; ok && v.(string) != "" { + apiObject.KeyPassphrase = aws.String(v.(string)) + } + + if v, ok := tfMap["metadata_column_name"]; ok && v.(string) != "" { + apiObject.MetaDataColumnName = aws.String(v.(string)) + } + + if _, ok := tfMap["processing_configuration"]; ok { + apiObject.ProcessingConfiguration = expandProcessingConfiguration(tfMap, destinationTypeSnowflake, roleARN) + } + + if v, ok := tfMap["s3_backup_mode"]; ok { + apiObject.S3BackupMode = types.SnowflakeS3BackupMode(v.(string)) + } + + if _, ok := tfMap["snowflake_role_configuration"]; ok { + apiObject.SnowflakeRoleConfiguration = expandSnowflakeRoleConfiguration(tfMap) + } + + return apiObject +} + func expandSplunkDestinationConfiguration(splunk map[string]interface{}) *types.SplunkDestinationConfiguration { configuration := &types.SplunkDestinationConfiguration{ HECToken: aws.String(splunk["hec_token"].(string)), @@ -2801,6 +3034,47 @@ func expandRedshiftRetryOptions(redshift map[string]interface{}) *types.Redshift return retryOptions } +func expandSnowflakeRetryOptions(tfMap map[string]interface{}) *types.SnowflakeRetryOptions { + apiObject := &types.SnowflakeRetryOptions{} + + if v, ok := tfMap["retry_duration"].(int); ok { + apiObject.DurationInSeconds = aws.Int32(int32(v)) + } + + return apiObject +} + +func expandSnowflakeRoleConfiguration(tfMap map[string]interface{}) *types.SnowflakeRoleConfiguration { + tfList := tfMap["snowflake_role_configuration"].([]interface{}) + if len(tfList) == 0 { + return nil + } + + tfMap = tfList[0].(map[string]interface{}) + + apiObject := &types.SnowflakeRoleConfiguration{ + Enabled: aws.Bool(tfMap["enabled"].(bool)), + SnowflakeRole: aws.String(tfMap["snowflake_role"].(string)), + } + + return apiObject +} + +func expandSnowflakeVPCConfiguration(tfMap map[string]interface{}) *types.SnowflakeVpcConfiguration { + tfList := tfMap["snowflake_vpc_configuration"].([]interface{}) + if len(tfList) == 0 { + return nil + } + + tfMap = tfList[0].(map[string]interface{}) + + apiObject := &types.SnowflakeVpcConfiguration{ + PrivateLinkVpceId: aws.String(tfMap["private_link_vpce_id"].(string)), + } + + return apiObject +} + func expandSplunkRetryOptions(splunk map[string]interface{}) *types.SplunkRetryOptions { retryOptions := &types.SplunkRetryOptions{} @@ -3129,6 +3403,39 @@ func flattenRedshiftDestinationDescription(description *types.RedshiftDestinatio return []map[string]interface{}{m} } +func flattenSnowflakeDestinationDescription(apiObject *types.SnowflakeDestinationDescription, configuredKeyPassphrase, configuredPrivateKey string) []map[string]interface{} { + if apiObject == nil { + return []map[string]interface{}{} + } + + roleARN := aws.ToString(apiObject.RoleARN) + tfMap := map[string]interface{}{ + "account_url": aws.ToString(apiObject.AccountUrl), + "cloudwatch_logging_options": flattenCloudWatchLoggingOptions(apiObject.CloudWatchLoggingOptions), + "content_column_name": aws.ToString(apiObject.ContentColumnName), + "data_loading_option": apiObject.DataLoadingOption, + "database": aws.ToString(apiObject.Database), + "key_passphrase": configuredKeyPassphrase, + "metadata_column_name": aws.ToString(apiObject.MetaDataColumnName), + "private_key": configuredPrivateKey, + "processing_configuration": flattenProcessingConfiguration(apiObject.ProcessingConfiguration, destinationTypeSnowflake, roleARN), + "role_arn": roleARN, + "s3_backup_mode": apiObject.S3BackupMode, + "s3_configuration": flattenS3DestinationDescription(apiObject.S3DestinationDescription), + "schema": aws.ToString(apiObject.Schema), + "snowflake_role_configuration": flattenSnowflakeRoleConfiguration(apiObject.SnowflakeRoleConfiguration), + "snowflake_vpc_configuration": flattenSnowflakeVPCConfiguration(apiObject.SnowflakeVpcConfiguration), + "table": aws.ToString(apiObject.Table), + "user": aws.ToString(apiObject.User), + } + + if apiObject.RetryOptions != nil { + tfMap["retry_duration"] = int(aws.ToInt32(apiObject.RetryOptions.DurationInSeconds)) + } + + return []map[string]interface{}{tfMap} +} + func flattenSplunkDestinationDescription(description *types.SplunkDestinationDescription) []map[string]interface{} { if description == nil { return []map[string]interface{}{} @@ -3554,6 +3861,31 @@ func flattenDocumentIDOptions(apiObject *types.DocumentIdOptions) map[string]int return tfMap } +func flattenSnowflakeRoleConfiguration(apiObject *types.SnowflakeRoleConfiguration) []map[string]interface{} { + if apiObject == nil { + return []map[string]interface{}{} + } + + m := map[string]interface{}{ + "enabled": aws.ToBool(apiObject.Enabled), + "snowflake_role": aws.ToString(apiObject.SnowflakeRole), + } + + return []map[string]interface{}{m} +} + +func flattenSnowflakeVPCConfiguration(apiObject *types.SnowflakeVpcConfiguration) []map[string]interface{} { + if apiObject == nil { + return []map[string]interface{}{} + } + + m := map[string]interface{}{ + "private_link_vpce_id": aws.ToString(apiObject.PrivateLinkVpceId), + } + + return []map[string]interface{}{m} +} + func isDeliveryStreamOptionDisabled(v interface{}) bool { tfList := v.([]interface{}) if len(tfList) == 0 || tfList[0] == nil { diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index c4f173e19b4..92810de04ee 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -90,6 +90,7 @@ func TestAccFirehoseDeliveryStream_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_arn", ""), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "splunk_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), resource.TestCheckResourceAttrSet(resourceName, "version_id"), @@ -1061,6 +1062,164 @@ func TestAccFirehoseDeliveryStream_redshiftUpdates(t *testing.T) { }) } +func TestAccFirehoseDeliveryStream_snowflakeUpdates(t *testing.T) { + ctx := acctest.Context(t) + var stream types.DeliveryStreamDescription + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + key := acctest.TLSRSAPrivateKeyPEM(t, 4096) + resourceName := "aws_kinesis_firehose_delivery_stream.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, names.FirehoseServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccDeliveryStreamConfig_snowflakeBasic(rName, key), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), + resource.TestCheckResourceAttrSet(resourceName, "arn"), + resource.TestCheckResourceAttr(resourceName, "destination", "snowflake"), + resource.TestCheckResourceAttrSet(resourceName, "destination_id"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "extended_s3_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "kinesis_source_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "opensearchserverless_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "redshift_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_arn", ""), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.account_url", fmt.Sprintf("https://%s.snowflakecomputing.com", rName)), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.log_group_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.log_stream_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.content_column_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.data_loading_option", "JSON_MAPPING"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.database", "test-db"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.key_passphrase", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.metadata_column_name", ""), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.private_key"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.#", "0"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.retry_duration", "60"), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.role_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_backup_mode", "FailedDataOnly"), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.s3_configuration.0.bucket_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.buffering_interval", "400"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.buffering_size", "10"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.log_group_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.log_stream_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.compression_format", "GZIP"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.kms_key_arn", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.prefix", ""), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.s3_configuration.0.role_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.schema", "test-schema"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.0.snowflake_role", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_vpc_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.table", "test-table"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.user", "test-usr"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + resource.TestCheckResourceAttrSet(resourceName, "version_id"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"snowflake_configuration.0.private_key"}, + }, + { + Config: testAccDeliveryStreamConfig_snowflakeUpdate(rName, key), + Check: resource.ComposeAggregateTestCheckFunc( + testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), + resource.TestCheckResourceAttrSet(resourceName, "arn"), + resource.TestCheckResourceAttr(resourceName, "destination", "snowflake"), + resource.TestCheckResourceAttrSet(resourceName, "destination_id"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "extended_s3_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "kinesis_source_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "name", rName), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "opensearchserverless_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "redshift_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_arn", ""), + resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.account_url", fmt.Sprintf("https://%s.snowflakecomputing.com", rName)), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.log_group_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.cloudwatch_logging_options.0.log_stream_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.content_column_name", "test-content"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.data_loading_option", "VARIANT_CONTENT_MAPPING"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.database", "test-db"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.key_passphrase", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.metadata_column_name", ""), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.private_key"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.0.type", "Lambda"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.0.parameters.#", "3"), + resource.TestCheckTypeSetElemNestedAttrs(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.0.parameters.*", map[string]string{ + "parameter_name": "LambdaArn", + }), + resource.TestCheckTypeSetElemNestedAttrs(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.0.parameters.*", map[string]string{ + "parameter_name": "BufferSizeInMBs", + "parameter_value": "1.1", + }), + resource.TestCheckTypeSetElemNestedAttrs(resourceName, "snowflake_configuration.0.processing_configuration.0.processors.0.parameters.*", map[string]string{ + "parameter_name": "BufferIntervalInSeconds", + "parameter_value": "70", + }), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.retry_duration", "60"), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.role_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_backup_mode", "FailedDataOnly"), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.s3_configuration.0.bucket_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.buffering_interval", "400"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.buffering_size", "10"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.enabled", "false"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.log_group_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.cloudwatch_logging_options.0.log_stream_name", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.compression_format", "GZIP"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.kms_key_arn", ""), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.s3_configuration.0.prefix", ""), + resource.TestCheckResourceAttrSet(resourceName, "snowflake_configuration.0.s3_configuration.0.role_arn"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.schema", "test-schema"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.0.enabled", "true"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_role_configuration.0.snowflake_role", "test-role"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.snowflake_vpc_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.table", "test-table"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.0.user", "test-usr"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.#", "0"), + resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), + resource.TestCheckResourceAttrSet(resourceName, "version_id"), + ), + }, + }, + }) +} + func TestAccFirehoseDeliveryStream_splunkUpdates(t *testing.T) { ctx := acctest.Context(t) var stream types.DeliveryStreamDescription @@ -1836,6 +1995,7 @@ func TestAccFirehoseDeliveryStream_openSearchServerlessUpdates(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_arn", ""), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "splunk_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), resource.TestCheckResourceAttrSet(resourceName, "version_id"), @@ -1907,6 +2067,7 @@ func TestAccFirehoseDeliveryStream_openSearchServerlessUpdates(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_arn", ""), resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), + resource.TestCheckResourceAttr(resourceName, "snowflake_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "splunk_configuration.#", "0"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), resource.TestCheckResourceAttrSet(resourceName, "version_id"), @@ -3547,6 +3708,91 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { `, rName)) } +func testAccDeliveryStreamConfig_snowflakeBasic(rName, privateKey string) string { + return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` +resource "aws_kinesis_firehose_delivery_stream" "test" { + depends_on = [aws_iam_role_policy.firehose] + name = %[1]q + destination = "snowflake" + + snowflake_configuration { + account_url = "https://%[1]s.snowflakecomputing.com" + database = "test-db" + private_key = "%[2]s" + role_arn = aws_iam_role.firehose.arn + schema = "test-schema" + table = "test-table" + user = "test-usr" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + } +} +`, rName, acctest.TLSPEMRemoveRSAPrivateKeyEncapsulationBoundaries(acctest.TLSPEMRemoveNewlines(privateKey)))) +} + +func testAccDeliveryStreamConfig_snowflakeUpdate(rName, privateKey string) string { + return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), testAccDeliveryStreamConfig_baseLambda(rName), fmt.Sprintf(` +resource "aws_kinesis_firehose_delivery_stream" "test" { + depends_on = [aws_iam_role_policy.firehose] + name = %[1]q + destination = "snowflake" + + snowflake_configuration { + account_url = "https://%[1]s.snowflakecomputing.com" + database = "test-db" + private_key = "%[2]s" + role_arn = aws_iam_role.firehose.arn + schema = "test-schema" + table = "test-table" + user = "test-usr" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + + data_loading_option = "VARIANT_CONTENT_MAPPING" + content_column_name = "test-content" + + snowflake_role_configuration { + enabled = true + snowflake_role = "test-role" + } + + processing_configuration { + enabled = false + + processors { + type = "Lambda" + + parameters { + parameter_name = "LambdaArn" + parameter_value = "${aws_lambda_function.lambda_function_test.arn}:$LATEST" + } + parameters { + parameter_name = "BufferSizeInMBs" + parameter_value = "1.1" + } + parameters { + parameter_name = "BufferIntervalInSeconds" + parameter_value = "70" + } + } + } + } +} +`, rName, acctest.TLSPEMRemoveRSAPrivateKeyEncapsulationBoundaries(acctest.TLSPEMRemoveNewlines(privateKey)))) +} + func testAccDeliveryStreamConfig_splunkBasic(rName string) string { return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` resource "aws_kinesis_firehose_delivery_stream" "test" { diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 8ce4c323076..7f242b2d883 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -8,7 +8,7 @@ description: |- # Resource: aws_kinesis_firehose_delivery_stream -Provides a Kinesis Firehose Delivery Stream resource. Amazon Kinesis Firehose is a fully managed, elastic service to easily deliver real-time data streams to destinations such as Amazon S3 and Amazon Redshift. +Provides a Kinesis Firehose Delivery Stream resource. Amazon Kinesis Firehose is a fully managed, elastic service to easily deliver real-time data streams to destinations such as Amazon S3 , Amazon Redshift and Snowflake. For more details, see the [Amazon Kinesis Firehose Documentation][1]. @@ -592,6 +592,33 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { } ``` +### Snowflake Destination + +```terraform +resource "aws_kinesis_firehose_delivery_stream" "example_snowflake_destination" { + name = "example-snowflake-destination" + destination = "snowflake" + + snowflake_configuration { + account_url = "https://example.snowflakecomputing.com" + database = "example-db" + private_key = "..." + role_arn = aws_iam_role.firehose.arn + schema = "example-schema" + table = "example-table" + user = "example-usr" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + } +} +``` + ## Argument Reference This resource supports the following arguments: @@ -603,13 +630,14 @@ This resource supports the following arguments: * `server_side_encryption` - (Optional) Encrypt at rest options. See [`server_side_encryption` block](#server_side_encryption-block) below for details. **NOTE:** Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream. -* `destination` – (Required) This is the destination to where the data is delivered. The only options are `s3` (Deprecated, use `extended_s3` instead), `extended_s3`, `redshift`, `elasticsearch`, `splunk`, `http_endpoint`, `opensearch` and `opensearchserverless`. +* `destination` – (Required) This is the destination to where the data is delivered. The only options are `s3` (Deprecated, use `extended_s3` instead), `extended_s3`, `redshift`, `elasticsearch`, `splunk`, `http_endpoint`, `opensearch`, `opensearchserverless` and `snowflake`. * `elasticsearch_configuration` - (Optional) Configuration options when `destination` is `elasticsearch`. See [`elasticsearch_configuration` block](#elasticsearch_configuration-block) below for details. * `extended_s3_configuration` - (Optional, only Required when `destination` is `extended_s3`) Enhanced configuration options for the s3 destination. See [`extended_s3_configuration` block](#extended_s3_configuration-block) below for details. * `http_endpoint_configuration` - (Optional) Configuration options when `destination` is `http_endpoint`. Requires the user to also specify an `s3_configuration` block. See [`http_endpoint_configuration` block](#http_endpoint_configuration-block) below for details. * `opensearch_configuration` - (Optional) Configuration options when `destination` is `opensearch`. See [`opensearch_configuration` block](#opensearch_configuration-block) below for details. * `opensearchserverless_configuration` - (Optional) Configuration options when `destination` is `opensearchserverless`. See [`opensearchserverless_configuration` block](#opensearchserverless_configuration-block) below for details. * `redshift_configuration` - (Optional) Configuration options when `destination` is `redshift`. Requires the user to also specify an `s3_configuration` block. See [`redshift_configuration` block](#redshift_configuration-block) below for details. +* `snowflake_configuration` - (Optional) Configuration options when `destination` is `snowflake`. See [`snowflake_configuration` block](#snowflake_configuration-block) below for details. * `splunk_configuration` - (Optional) Configuration options when `destination` is `splunk`. See [`splunk_configuration` block](#splunk_configuration-block) below for details. ### `kinesis_source_configuration` block @@ -760,6 +788,32 @@ The `http_endpoint_configuration` configuration block supports the following arg * `request_configuration` - (Optional) The request configuration. See [`request_configuration` block](#request_configuration-block) below for details. * `retry_duration` - (Optional) Total amount of seconds Firehose spends on retries. This duration starts after the initial attempt fails, It does not include the time periods during which Firehose waits for acknowledgment from the specified destination after each attempt. Valid values between `0` and `7200`. Default is `300`. +### `snowflake_configuration` block + +The `snowflake_configuration` configuration block supports the following arguments: + +* `account_url` - (Required) The URL of the Snowflake account. Format: https://[account_identifier].snowflakecomputing.com. +* `private_key` - (Required) The private key for authentication. +* `key_passphrase` - (Required) The passphrase for the private key. +* `user` - (Required) The user for authentication. +* `database` - (Required) The Snowflake database name. +* `schema` - (Required) The Snowflake schema name. +* `table` - (Required) The Snowflake table name. +* `snowflake_role_configuration` - (Optional) The configuration for Snowflake role. + * `enabled` - (Optional) Whether the Snowflake role is enabled. + * `snowflake_role` - (Optional) The Snowflake role. +* `data_loading_option` - (Optional) The data loading option. +* `metadata_column_name` - (Optional) The name of the metadata column. +* `content_column_name` - (Optional) The name of the content column. +* `snowflake_vpc_configuration` - (Optional) The VPC configuration for Snowflake. + * `private_link_vpce_id` - (Required) The VPCE ID for Firehose to privately connect with Snowflake. +* `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. See [`cloudwatch_logging_options` block](#cloudwatch_logging_options-block) below for details. +* `processing_configuration` - (Optional) The processing configuration. See [`processing_configuration` block](#processing_configuration-block) below for details. +* `role_arn` - (Required) The ARN of the IAM role. +* `retry_duration` - (Optional) After an initial failure to deliver to Snowflake, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 60s. There will be no retry if the value is 0. +* `s3_backup_mode` - (Optional) The S3 backup mode. +* `s3_configuration` - (Required) The S3 configuration. See [`s3_configuration` block](#s3_configuration-block) below for details. + ### `cloudwatch_logging_options` block The `cloudwatch_logging_options` configuration block supports the following arguments: