From 138914e33b0e0c692b8b99105fa77ddcc4fc1609 Mon Sep 17 00:00:00 2001 From: ljjulia Date: Tue, 2 Jul 2024 09:13:19 -0700 Subject: [PATCH 1/5] Disable SoftDeletePolicy when creating a bucket --- sdks/go/pkg/beam/util/gcsx/gcs.go | 15 +++++++++++++-- sdks/go/pkg/beam/util/gcsx/gcs_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go b/sdks/go/pkg/beam/util/gcsx/gcs.go index 83eeb823f4a2..9c9c5f9ca025 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs.go @@ -63,9 +63,20 @@ func Upload(ctx context.Context, client *storage.Client, project, bucket, object } -// CreateBucket creates a bucket in GCS. +// Get BucketAttrs with RetentionDuration of SoftDeletePolicy set to zero for disabling SoftDeletePolicy. +func getDisableSoftDeletePolicyBucketAttrs() *storage.BucketAttrs { + attrs := storage.BucketAttrs{ + SoftDeletePolicy: &storage.SoftDeletePolicy{ + RetentionDuration: 0, + }, + } + return &attrs +} + +// CreateBucket creates a bucket in GCS with RetentionDuration of zero to disable SoftDeletePolicy. func CreateBucket(ctx context.Context, client *storage.Client, project, bucket string) error { - return client.Bucket(bucket).Create(ctx, project, nil) + disableSoftDeletePolicyBucketAttrs := getDisableSoftDeletePolicyBucketAttrs() + return client.Bucket(bucket).Create(ctx, project, disableSoftDeletePolicyBucketAttrs) } // BucketExists returns true iff the given bucket exists. diff --git a/sdks/go/pkg/beam/util/gcsx/gcs_test.go b/sdks/go/pkg/beam/util/gcsx/gcs_test.go index 90fb4b59f2fe..463ba3ea1833 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs_test.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs_test.go @@ -96,3 +96,13 @@ func TestJoin(t *testing.T) { } } } + +func TestGetDisableSoftDeletePolicyBucketAttrs(t *testing.T) { + attrs := getDisableSoftDeletePolicyBucketAttrs() + if attrs == nil { + t.Errorf("Fail to getDisableSoftDeletePolicyBucketAttrs.") + } + if attrs != nil && attrs.SoftDeletePolicy.RetentionDuration != 0 { + t.Errorf("attrs has RetentionDuration %v which is not correct", attrs.SoftDeletePolicy.RetentionDuration) + } +} From 0580f509ce84fa4685d9016f47ba56207fe324c8 Mon Sep 17 00:00:00 2001 From: ljjulia Date: Tue, 2 Jul 2024 09:31:43 -0700 Subject: [PATCH 2/5] Disable SoftDeletePolicy when creating a bucket --- sdks/go/pkg/beam/util/gcsx/gcs.go | 6 +++--- sdks/go/pkg/beam/util/gcsx/gcs_test.go | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go b/sdks/go/pkg/beam/util/gcsx/gcs.go index 9c9c5f9ca025..d7436462964a 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs.go @@ -64,19 +64,19 @@ func Upload(ctx context.Context, client *storage.Client, project, bucket, object } // Get BucketAttrs with RetentionDuration of SoftDeletePolicy set to zero for disabling SoftDeletePolicy. -func getDisableSoftDeletePolicyBucketAttrs() *storage.BucketAttrs { +func getDisableSoftDeletePolicyBucketAttrs() storage.BucketAttrs { attrs := storage.BucketAttrs{ SoftDeletePolicy: &storage.SoftDeletePolicy{ RetentionDuration: 0, }, } - return &attrs + return attrs } // CreateBucket creates a bucket in GCS with RetentionDuration of zero to disable SoftDeletePolicy. func CreateBucket(ctx context.Context, client *storage.Client, project, bucket string) error { disableSoftDeletePolicyBucketAttrs := getDisableSoftDeletePolicyBucketAttrs() - return client.Bucket(bucket).Create(ctx, project, disableSoftDeletePolicyBucketAttrs) + return client.Bucket(bucket).Create(ctx, project, &disableSoftDeletePolicyBucketAttrs) } // BucketExists returns true iff the given bucket exists. diff --git a/sdks/go/pkg/beam/util/gcsx/gcs_test.go b/sdks/go/pkg/beam/util/gcsx/gcs_test.go index 463ba3ea1833..5bcbcc29b974 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs_test.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs_test.go @@ -99,10 +99,7 @@ func TestJoin(t *testing.T) { func TestGetDisableSoftDeletePolicyBucketAttrs(t *testing.T) { attrs := getDisableSoftDeletePolicyBucketAttrs() - if attrs == nil { - t.Errorf("Fail to getDisableSoftDeletePolicyBucketAttrs.") - } - if attrs != nil && attrs.SoftDeletePolicy.RetentionDuration != 0 { + if attrs.SoftDeletePolicy.RetentionDuration != 0 { t.Errorf("attrs has RetentionDuration %v which is not correct", attrs.SoftDeletePolicy.RetentionDuration) } } From 9c83c931e7353203a424dace1f10941d1049bb2f Mon Sep 17 00:00:00 2001 From: ljj Date: Tue, 2 Jul 2024 11:08:17 -0700 Subject: [PATCH 3/5] Update gcs.go --- sdks/go/pkg/beam/util/gcsx/gcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go b/sdks/go/pkg/beam/util/gcsx/gcs.go index d7436462964a..91e0ea1a5856 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs.go @@ -65,7 +65,7 @@ func Upload(ctx context.Context, client *storage.Client, project, bucket, object // Get BucketAttrs with RetentionDuration of SoftDeletePolicy set to zero for disabling SoftDeletePolicy. func getDisableSoftDeletePolicyBucketAttrs() storage.BucketAttrs { - attrs := storage.BucketAttrs{ + attrs := &storage.BucketAttrs{ SoftDeletePolicy: &storage.SoftDeletePolicy{ RetentionDuration: 0, }, From 97fe5e55eece2c9a2b42262786be8a1c7033471e Mon Sep 17 00:00:00 2001 From: ljj Date: Tue, 2 Jul 2024 11:10:34 -0700 Subject: [PATCH 4/5] Update gcs.go --- sdks/go/pkg/beam/util/gcsx/gcs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/go/pkg/beam/util/gcsx/gcs.go b/sdks/go/pkg/beam/util/gcsx/gcs.go index 91e0ea1a5856..1dd85924447f 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs.go @@ -64,7 +64,7 @@ func Upload(ctx context.Context, client *storage.Client, project, bucket, object } // Get BucketAttrs with RetentionDuration of SoftDeletePolicy set to zero for disabling SoftDeletePolicy. -func getDisableSoftDeletePolicyBucketAttrs() storage.BucketAttrs { +func getDisableSoftDeletePolicyBucketAttrs() *storage.BucketAttrs { attrs := &storage.BucketAttrs{ SoftDeletePolicy: &storage.SoftDeletePolicy{ RetentionDuration: 0, @@ -76,7 +76,7 @@ func getDisableSoftDeletePolicyBucketAttrs() storage.BucketAttrs { // CreateBucket creates a bucket in GCS with RetentionDuration of zero to disable SoftDeletePolicy. func CreateBucket(ctx context.Context, client *storage.Client, project, bucket string) error { disableSoftDeletePolicyBucketAttrs := getDisableSoftDeletePolicyBucketAttrs() - return client.Bucket(bucket).Create(ctx, project, &disableSoftDeletePolicyBucketAttrs) + return client.Bucket(bucket).Create(ctx, project, disableSoftDeletePolicyBucketAttrs) } // BucketExists returns true iff the given bucket exists. From c73d9ec2ccecd49fdad33154cb7225e69a951f4f Mon Sep 17 00:00:00 2001 From: ljj Date: Tue, 2 Jul 2024 11:11:26 -0700 Subject: [PATCH 5/5] Update gcs_test.go --- sdks/go/pkg/beam/util/gcsx/gcs_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/go/pkg/beam/util/gcsx/gcs_test.go b/sdks/go/pkg/beam/util/gcsx/gcs_test.go index 5bcbcc29b974..463ba3ea1833 100644 --- a/sdks/go/pkg/beam/util/gcsx/gcs_test.go +++ b/sdks/go/pkg/beam/util/gcsx/gcs_test.go @@ -99,7 +99,10 @@ func TestJoin(t *testing.T) { func TestGetDisableSoftDeletePolicyBucketAttrs(t *testing.T) { attrs := getDisableSoftDeletePolicyBucketAttrs() - if attrs.SoftDeletePolicy.RetentionDuration != 0 { + if attrs == nil { + t.Errorf("Fail to getDisableSoftDeletePolicyBucketAttrs.") + } + if attrs != nil && attrs.SoftDeletePolicy.RetentionDuration != 0 { t.Errorf("attrs has RetentionDuration %v which is not correct", attrs.SoftDeletePolicy.RetentionDuration) } }