From e71c1e2b1188fdf3cd5adc2caac7ec558a1f0250 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 16 Aug 2023 11:32:54 -0700 Subject: [PATCH 01/19] Migration of impersonate changes --- remote/remote_state_gcs.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/remote/remote_state_gcs.go b/remote/remote_state_gcs.go index be5cda40f0..3b42295f90 100644 --- a/remote/remote_state_gcs.go +++ b/remote/remote_state_gcs.go @@ -9,6 +9,8 @@ import ( "strconv" "time" + "google.golang.org/api/impersonate" + "cloud.google.com/go/storage" "github.com/gruntwork-io/terragrunt/options" "github.com/gruntwork-io/terragrunt/pkg/errors" @@ -470,9 +472,15 @@ func CreateGCSClient(gcsConfigRemote RemoteStateConfigGCS) (*storage.Client, err } if gcsConfigRemote.ImpersonateServiceAccount != "" { - opts = append(opts, option.ImpersonateCredentials( - gcsConfigRemote.ImpersonateServiceAccount, - gcsConfigRemote.ImpersonateServiceAccountDelegates...)) + ts, err := impersonate.CredentialsTokenSource(ctx, impersonate.CredentialsConfig{ + TargetPrincipal: gcsConfigRemote.ImpersonateServiceAccount, + Scopes: []string{storage.ScopeFullControl}, + Delegates: gcsConfigRemote.ImpersonateServiceAccountDelegates, + }) + if err != nil { + return nil, err + } + opts = append(opts, option.WithTokenSource(ts)) } client, err := storage.NewClient(ctx, opts...) From 297408654a2f0f1d4eaca57f7a03f20e1cdfd22e Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 16 Aug 2023 22:47:23 +0300 Subject: [PATCH 02/19] Add test for bucket creation --- test/fixture-gcs-impersonate/main.tf | 3 +++ test/fixture-gcs-impersonate/terragrunt.hcl | 16 +++++++++++++ test/integration_serial_test.go | 26 +++++++++++++++++++++ test/integration_test.go | 1 + 4 files changed, 46 insertions(+) create mode 100644 test/fixture-gcs-impersonate/main.tf create mode 100644 test/fixture-gcs-impersonate/terragrunt.hcl diff --git a/test/fixture-gcs-impersonate/main.tf b/test/fixture-gcs-impersonate/main.tf new file mode 100644 index 0000000000..2770f71737 --- /dev/null +++ b/test/fixture-gcs-impersonate/main.tf @@ -0,0 +1,3 @@ +terraform { + backend "gcs" {} +} diff --git a/test/fixture-gcs-impersonate/terragrunt.hcl b/test/fixture-gcs-impersonate/terragrunt.hcl new file mode 100644 index 0000000000..0ab02651ff --- /dev/null +++ b/test/fixture-gcs-impersonate/terragrunt.hcl @@ -0,0 +1,16 @@ +remote_state { + backend = "gcs" + + config = { + project = "__FILL_IN_PROJECT__" + location = "__FILL_IN_LOCATION__" + bucket = "__FILL_IN_BUCKET_NAME__" + impersonate_service_account = "__FILL_IN_GCP_EMAIL__" + prefix = "terraform.tfstate" + + gcs_bucket_labels = { + owner = "terragrunt_test" + name = "terraform_state_storage" + } + } +} diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index b054dc018d..c56cc9afa4 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -352,3 +352,29 @@ func TestTerragruntParallelism(t *testing.T) { }) } } +func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { + defaultCreds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + defer os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCreds) + + impersonatorKey := os.Getenv("GCLOUD_SERVICE_KEY_IMPERSONATOR") + if impersonatorKey == "" { + t.Fatalf("Required environment variable `%s` - not found", "GCLOUD_SERVICE_KEY_IMPERSONATOR") + } + tmpImpersonatorCreds := createTmpTerragruntConfigContent(t, impersonatorKey, "impersonator-key.json") + defer removeFile(t, tmpImpersonatorCreds) + os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", tmpImpersonatorCreds) + + project := os.Getenv("GOOGLE_CLOUD_PROJECT") + gcsBucketName := fmt.Sprintf("terragrunt-test-bucket-%s", strings.ToLower(uniqueId())) + + // run with impersonation + tmpTerragruntImpersonateGCSConfigPath := createTmpTerragruntGCSConfig(t, TEST_FIXTURE_GCS_IMPERSONATE_PATH, project, TERRAFORM_REMOTE_STATE_GCP_REGION, gcsBucketName, config.DefaultTerragruntConfigPath) + runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-config %s --terragrunt-working-dir %s", tmpTerragruntImpersonateGCSConfigPath, TEST_FIXTURE_GCS_IMPERSONATE_PATH)) + + // restore default credentials + + var expectedGCSLabels = map[string]string{ + "owner": "terragrunt_test", + "name": "terraform_state_storage"} + validateGCSBucketExistsAndIsLabeled(t, TERRAFORM_REMOTE_STATE_GCP_REGION, gcsBucketName, expectedGCSLabels) +} diff --git a/test/integration_test.go b/test/integration_test.go index 92c0b7da71..a9263af37d 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -162,6 +162,7 @@ const ( TEST_FIXTURE_STRCONTAINS = "fixture-strcontains" TEST_FIXTURE_INIT_CACHE = "fixture-init-cache" TEST_FIXTURE_NULL_VALUE = "fixture-null-values" + TEST_FIXTURE_GCS_IMPERSONATE_PATH = "fixture-gcs-impersonate/" TERRAFORM_BINARY = "terraform" TERRAFORM_FOLDER = ".terraform" TERRAFORM_STATE = "terraform.tfstate" From c13848a85d9f0d37fbb1b564d1f17337f2e91326 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 16 Aug 2023 23:07:10 +0300 Subject: [PATCH 03/19] Add replacement of email --- test/integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration_test.go b/test/integration_test.go index a9263af37d..e6545997cb 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4062,6 +4062,9 @@ func copyTerragruntGCSConfigAndFillPlaceholders(t *testing.T, configSrcPath stri contents = strings.Replace(contents, "__FILL_IN_LOCATION__", location, -1) contents = strings.Replace(contents, "__FILL_IN_BUCKET_NAME__", gcsBucketName, -1) + email := os.Getenv("GOOGLE_IDENTITY_EMAIL") + contents = strings.Replace(contents, "__FILL_IN_GCP_EMAIL__", email, -1) + if err := ioutil.WriteFile(configDestPath, []byte(contents), 0444); err != nil { t.Fatalf("Error writing temp Terragrunt config to %s: %v", configDestPath, err) } From 0f075435a230f570435f8850d985858374467a88 Mon Sep 17 00:00:00 2001 From: Denis O Date: Wed, 16 Aug 2023 23:29:20 +0300 Subject: [PATCH 04/19] acl item --- test/integration_serial_test.go | 3 +-- test/integration_test.go | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index c56cc9afa4..aafceb7645 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -352,6 +352,7 @@ func TestTerragruntParallelism(t *testing.T) { }) } } + func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { defaultCreds := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") defer os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", defaultCreds) @@ -371,8 +372,6 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { tmpTerragruntImpersonateGCSConfigPath := createTmpTerragruntGCSConfig(t, TEST_FIXTURE_GCS_IMPERSONATE_PATH, project, TERRAFORM_REMOTE_STATE_GCP_REGION, gcsBucketName, config.DefaultTerragruntConfigPath) runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-config %s --terragrunt-working-dir %s", tmpTerragruntImpersonateGCSConfigPath, TEST_FIXTURE_GCS_IMPERSONATE_PATH)) - // restore default credentials - var expectedGCSLabels = map[string]string{ "owner": "terragrunt_test", "name": "terraform_state_storage"} diff --git a/test/integration_test.go b/test/integration_test.go index e6545997cb..dd0b8cf816 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4370,11 +4370,15 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa // verify the bucket location ctx := context.Background() bucket := gcsClient.Bucket(bucketName) - attrs, err := bucket.Attrs(ctx) if err != nil { t.Fatal(err) } + list, err := bucket.DefaultObjectACL().List(ctx) + for _, item := range list { + t.Logf("acl item: %v", item) + t.Logf("acl item email: %v", item.Email) + } assert.Equal(t, strings.ToUpper(location), attrs.Location, "Did not find GCS bucket in expected location.") From f875ee9209ea2e9daf83eebeaf8af23d7902605c Mon Sep 17 00:00:00 2001 From: Denis O Date: Thu, 17 Aug 2023 13:12:09 -0700 Subject: [PATCH 05/19] add logging of acl --- test/integration_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration_test.go b/test/integration_test.go index dd0b8cf816..4178436907 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4380,6 +4380,9 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa t.Logf("acl item email: %v", item.Email) } + acl, err := bucket.ACL().List(ctx) + t.Logf("acl item: %v", acl) + assert.Equal(t, strings.ToUpper(location), attrs.Location, "Did not find GCS bucket in expected location.") if expectedLabels != nil { From 91eb877e0a0438b1e3ea02cf1bb664217b929441 Mon Sep 17 00:00:00 2001 From: Denis O Date: Fri, 18 Aug 2023 12:36:15 -0700 Subject: [PATCH 06/19] Testing gcp fix --- test/integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration_test.go b/test/integration_test.go index 4178436907..fc691946f3 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4374,6 +4374,7 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa if err != nil { t.Fatal(err) } + t.Logf("attrs item: %v", attrs) list, err := bucket.DefaultObjectACL().List(ctx) for _, item := range list { t.Logf("acl item: %v", item) From ae02ea703eac42a3ab265e218f4b4e35e20cf65e Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 11:48:59 -0700 Subject: [PATCH 07/19] object fetching test --- test/integration_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/integration_test.go b/test/integration_test.go index fc691946f3..393160dadc 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4376,9 +4376,10 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa } t.Logf("attrs item: %v", attrs) list, err := bucket.DefaultObjectACL().List(ctx) - for _, item := range list { - t.Logf("acl item: %v", item) - t.Logf("acl item email: %v", item.Email) + for _, rule := range list { + + t.Logf("acl item: %v", rule) + t.Logf("acl item email: %v", rule.Email) } acl, err := bucket.ACL().List(ctx) @@ -4389,6 +4390,16 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa if expectedLabels != nil { assertGCSLabels(t, expectedLabels, bucketName, gcsClient) } + + obj := gcsClient.Bucket(bucketName).Object("terraform.tfstate") + + acls, err := obj.ACL().List(ctx) + for _, acl := range acls { + fmt.Printf("Owner Entity: %s\n", acl.Entity) + fmt.Printf("Owner entity email: %s\n", acl.Email) + fmt.Printf("Owner : %v\n", acl) + } + } func assertGCSLabels(t *testing.T, expectedLabels map[string]string, bucketName string, client *storage.Client) { From 2177003e2fc5dec99d68a9836c9863ecb2136f03 Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 12:07:50 -0700 Subject: [PATCH 08/19] Testing iam printing --- test/integration_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration_test.go b/test/integration_test.go index 393160dadc..a757a5ac9a 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4391,9 +4391,16 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa assertGCSLabels(t, expectedLabels, bucketName, gcsClient) } - obj := gcsClient.Bucket(bucketName).Object("terraform.tfstate") + iam, err := gcsClient.Bucket(bucketName).IAM().Policy(ctx) + assert.NoError(t, err) + fmt.Printf("role iam : %v\n", iam) + for _, r := range iam.Roles() { + fmt.Printf("role : %v\n", r) + } + obj := gcsClient.Bucket(bucketName).Object("terraform.tfstate") acls, err := obj.ACL().List(ctx) + assert.NoError(t, err) for _, acl := range acls { fmt.Printf("Owner Entity: %s\n", acl.Entity) fmt.Printf("Owner entity email: %s\n", acl.Email) From f9a34241ea9aee51e4f987f53273a45c297dbe75 Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 13:27:18 -0700 Subject: [PATCH 09/19] testing ProjectTeam --- go.mod | 2 ++ go.sum | 3 +++ test/integration_test.go | 22 +++++----------------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 34e98179c1..87f7be8f85 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( ) require ( + cloud.google.com/go/logging v1.7.0 github.com/gruntwork-io/go-commons v0.16.2 github.com/gruntwork-io/gruntwork-cli v0.7.0 github.com/urfave/cli/v2 v2.25.5 @@ -65,6 +66,7 @@ require ( cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/longrunning v0.4.1 // indirect filippo.io/age v1.0.0 // indirect github.com/Azure/azure-sdk-for-go v63.3.0+incompatible // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index d35cccc509..0de5e1b2d3 100644 --- a/go.sum +++ b/go.sum @@ -117,7 +117,10 @@ cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= diff --git a/test/integration_test.go b/test/integration_test.go index a757a5ac9a..d763a897c9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4374,12 +4374,17 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa if err != nil { t.Fatal(err) } + t.Logf("attrs item: %v", attrs) + list, err := bucket.DefaultObjectACL().List(ctx) for _, rule := range list { t.Logf("acl item: %v", rule) + t.Logf("acl item email: %v", rule.Email) + + t.Logf("acl ProjectTeam: %v", rule.ProjectTeam) } acl, err := bucket.ACL().List(ctx) @@ -4390,23 +4395,6 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa if expectedLabels != nil { assertGCSLabels(t, expectedLabels, bucketName, gcsClient) } - - iam, err := gcsClient.Bucket(bucketName).IAM().Policy(ctx) - assert.NoError(t, err) - fmt.Printf("role iam : %v\n", iam) - for _, r := range iam.Roles() { - fmt.Printf("role : %v\n", r) - } - - obj := gcsClient.Bucket(bucketName).Object("terraform.tfstate") - acls, err := obj.ACL().List(ctx) - assert.NoError(t, err) - for _, acl := range acls { - fmt.Printf("Owner Entity: %s\n", acl.Entity) - fmt.Printf("Owner entity email: %s\n", acl.Email) - fmt.Printf("Owner : %v\n", acl) - } - } func assertGCSLabels(t *testing.T, expectedLabels map[string]string, bucketName string, client *storage.Client) { From 7530dca9e347d921e84ce980003bb433821c1f1c Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 14:00:31 -0700 Subject: [PATCH 10/19] Notifications --- test/integration_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/integration_test.go b/test/integration_test.go index d763a897c9..046790204f 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4375,6 +4375,13 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa t.Fatal(err) } + notifications, err := bucket.Notifications(ctx) + if err != nil { + t.Fatalf("Error fetching notification: %v", err) + } + + t.Logf("attrs item: %v", notifications) + t.Logf("attrs item: %v", attrs) list, err := bucket.DefaultObjectACL().List(ctx) @@ -4384,7 +4391,7 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa t.Logf("acl item email: %v", rule.Email) - t.Logf("acl ProjectTeam: %v", rule.ProjectTeam) + t.Logf("acl c: %v", rule.ProjectTeam) } acl, err := bucket.ACL().List(ctx) From a6ee1a60e04282edb4119d826d16dc0230370580 Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 14:10:38 -0700 Subject: [PATCH 11/19] add log printing --- test/integration_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/integration_test.go b/test/integration_test.go index 046790204f..22b0b3363b 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "math/rand" "net/url" "os" @@ -4397,6 +4398,22 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa acl, err := bucket.ACL().List(ctx) t.Logf("acl item: %v", acl) + it := bucket.Objects(ctx, nil) + for { + attrs, err := it.Next() + if err != nil { + break + } + if err != nil { + log.Fatalf("Error iterating through objects: %v", err) + } + fmt.Println("******") + fmt.Println(attrs.Name) + fmt.Println(attrs.Owner) + fmt.Println(attrs.Size) + fmt.Println("******") + } + assert.Equal(t, strings.ToUpper(location), attrs.Location, "Did not find GCS bucket in expected location.") if expectedLabels != nil { From 86585072ec3b08b0d092f84fbc503d858278984d Mon Sep 17 00:00:00 2001 From: Denis O Date: Sun, 20 Aug 2023 14:14:26 -0700 Subject: [PATCH 12/19] Add test output value --- test/fixture-gcs-impersonate/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/fixture-gcs-impersonate/main.tf b/test/fixture-gcs-impersonate/main.tf index 2770f71737..4d5edd98e7 100644 --- a/test/fixture-gcs-impersonate/main.tf +++ b/test/fixture-gcs-impersonate/main.tf @@ -1,3 +1,7 @@ terraform { backend "gcs" {} } + +output "value" { + value = "42" +} \ No newline at end of file From 16cc17e601782422ac9c24a5fbbce6068790a8c9 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 00:43:29 +0300 Subject: [PATCH 13/19] Add function to read objects metadata --- test/integration_serial_test.go | 4 ++++ test/integration_test.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index aafceb7645..4c18f5384e 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -376,4 +376,8 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { "owner": "terragrunt_test", "name": "terraform_state_storage"} validateGCSBucketExistsAndIsLabeled(t, TERRAFORM_REMOTE_STATE_GCP_REGION, gcsBucketName, expectedGCSLabels) + + email := os.Getenv("GOOGLE_IDENTITY_EMAIL") + attrs := gcsObjectAttrs(t, gcsBucketName, "terraform.tfstate/default.tfstate") + assert.Equal(t, email, attrs.Owner, "Identity email should match the impersonated account") } diff --git a/test/integration_test.go b/test/integration_test.go index 22b0b3363b..e4a0206f86 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4421,6 +4421,26 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa } } +// gcsObjectAttrs returns the attributes of the specified object in the bucket +func gcsObjectAttrs(t *testing.T, bucketName string, objectName string) *storage.ObjectAttrs { + remoteStateConfig := remote.RemoteStateConfigGCS{Bucket: bucketName} + + gcsClient, err := remote.CreateGCSClient(remoteStateConfig) + if err != nil { + t.Fatalf("Error creating GCS client: %v", err) + } + + ctx := context.Background() + bucket := gcsClient.Bucket(bucketName) + + handle := bucket.Object(objectName) + attrs, err := handle.Attrs(ctx) + if err != nil { + t.Fatalf("Error reading object attributes %s %v", objectName, err) + } + return attrs +} + func assertGCSLabels(t *testing.T, expectedLabels map[string]string, bucketName string, client *storage.Client) { ctx := context.Background() bucket := client.Bucket(bucketName) From 146b00396614f25ec543608e6db491a04d436729 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:02:22 +0300 Subject: [PATCH 14/19] acl email --- test/integration_serial_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 4c18f5384e..645e0a6e8a 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -379,5 +379,9 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { email := os.Getenv("GOOGLE_IDENTITY_EMAIL") attrs := gcsObjectAttrs(t, gcsBucketName, "terraform.tfstate/default.tfstate") + for _, a := range attrs.ACL { + fmt.Printf("ACL: %v\n", a) + fmt.Printf("ACL email: %v\n", a.Email) + } assert.Equal(t, email, attrs.Owner, "Identity email should match the impersonated account") } From be8362c5b3708aee09c78a739e7169ca2294fa8f Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:14:01 +0300 Subject: [PATCH 15/19] Add check for gcp acl --- test/integration_serial_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 645e0a6e8a..609e34684b 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -379,9 +379,13 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { email := os.Getenv("GOOGLE_IDENTITY_EMAIL") attrs := gcsObjectAttrs(t, gcsBucketName, "terraform.tfstate/default.tfstate") + ownerEmail := false for _, a := range attrs.ACL { fmt.Printf("ACL: %v\n", a) fmt.Printf("ACL email: %v\n", a.Email) + if (a.Role == "OWNER") && (a.Email == email) { + ownerEmail = true + } } - assert.Equal(t, email, attrs.Owner, "Identity email should match the impersonated account") + assert.True(t, ownerEmail, "Identity email should match the impersonated account") } From 86f5b6c37d16cf1f6a951e74b549a7355082bf8e Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:16:42 +0300 Subject: [PATCH 16/19] Tests cleanup --- go.mod | 2 -- go.sum | 3 --- test/integration_test.go | 39 --------------------------------------- 3 files changed, 44 deletions(-) diff --git a/go.mod b/go.mod index 87f7be8f85..34e98179c1 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( ) require ( - cloud.google.com/go/logging v1.7.0 github.com/gruntwork-io/go-commons v0.16.2 github.com/gruntwork-io/gruntwork-cli v0.7.0 github.com/urfave/cli/v2 v2.25.5 @@ -66,7 +65,6 @@ require ( cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.12.0 // indirect - cloud.google.com/go/longrunning v0.4.1 // indirect filippo.io/age v1.0.0 // indirect github.com/Azure/azure-sdk-for-go v63.3.0+incompatible // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index 0de5e1b2d3..d35cccc509 100644 --- a/go.sum +++ b/go.sum @@ -117,10 +117,7 @@ cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= -cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= diff --git a/test/integration_test.go b/test/integration_test.go index e4a0206f86..034e32e2be 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "math/rand" "net/url" "os" @@ -4376,44 +4375,6 @@ func validateGCSBucketExistsAndIsLabeled(t *testing.T, location string, bucketNa t.Fatal(err) } - notifications, err := bucket.Notifications(ctx) - if err != nil { - t.Fatalf("Error fetching notification: %v", err) - } - - t.Logf("attrs item: %v", notifications) - - t.Logf("attrs item: %v", attrs) - - list, err := bucket.DefaultObjectACL().List(ctx) - for _, rule := range list { - - t.Logf("acl item: %v", rule) - - t.Logf("acl item email: %v", rule.Email) - - t.Logf("acl c: %v", rule.ProjectTeam) - } - - acl, err := bucket.ACL().List(ctx) - t.Logf("acl item: %v", acl) - - it := bucket.Objects(ctx, nil) - for { - attrs, err := it.Next() - if err != nil { - break - } - if err != nil { - log.Fatalf("Error iterating through objects: %v", err) - } - fmt.Println("******") - fmt.Println(attrs.Name) - fmt.Println(attrs.Owner) - fmt.Println(attrs.Size) - fmt.Println("******") - } - assert.Equal(t, strings.ToUpper(location), attrs.Location, "Did not find GCS bucket in expected location.") if expectedLabels != nil { From c6c08603a2f22e216cbef9aca8b09542f6d4f772 Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:19:10 +0300 Subject: [PATCH 17/19] Tests cleanup --- test/fixture-gcs-impersonate/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixture-gcs-impersonate/main.tf b/test/fixture-gcs-impersonate/main.tf index 4d5edd98e7..f110804e36 100644 --- a/test/fixture-gcs-impersonate/main.tf +++ b/test/fixture-gcs-impersonate/main.tf @@ -4,4 +4,4 @@ terraform { output "value" { value = "42" -} \ No newline at end of file +} From fdf645e482d9586bc405e390ec0291dd2ecf524a Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:27:07 +0300 Subject: [PATCH 18/19] Removed debug prints --- test/integration_serial_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 609e34684b..5260247d04 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -381,8 +381,6 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { attrs := gcsObjectAttrs(t, gcsBucketName, "terraform.tfstate/default.tfstate") ownerEmail := false for _, a := range attrs.ACL { - fmt.Printf("ACL: %v\n", a) - fmt.Printf("ACL email: %v\n", a.Email) if (a.Role == "OWNER") && (a.Email == email) { ownerEmail = true } From 23ed8c338c59e70d99fc68328926d98485e68b1a Mon Sep 17 00:00:00 2001 From: Denis O Date: Mon, 21 Aug 2023 01:36:48 +0300 Subject: [PATCH 19/19] Added loop termination --- test/integration_serial_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go index 5260247d04..0584050079 100644 --- a/test/integration_serial_test.go +++ b/test/integration_serial_test.go @@ -383,6 +383,7 @@ func TestTerragruntWorksWithImpersonateGCSBackend(t *testing.T) { for _, a := range attrs.ACL { if (a.Role == "OWNER") && (a.Email == email) { ownerEmail = true + break } } assert.True(t, ownerEmail, "Identity email should match the impersonated account")