eks/cluster: fix OIDC CA thumbprint fetch (#172)

* eks/cluster: fix OIDC CA thumbprint fetch Fix the following panic: {"level":"info","ts":"2020-09-25T23:43:56.457-0700","caller":"cluster/cluster.go:700","msg":"updating host with port :443","host":"..."} {"level":"info","ts":"2020-09-25T23:43:56.457-0700","caller":"cluster/cluster.go:711","msg":"fetching OIDC CA thumbprint","url":"https://...:443/test/id/78FD753967D735CEFC309F15590F617F"} {"level":"warn","ts":"2020-09-25T23:44:38.734-0700","caller":"cluster/cluster.go:720","msg":"failed to fetch OIDC CA thumbprint","url":"https://...:443/test/id/78FD753967D735CEFC309F15590F617F","error":"Get \"https://...:443/test/id/78FD753967D735CEFC309F15590F617F\": local error: tls: bad record MAC"} panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x221029b] goroutine 221 [running]: github.com/aws/aws-k8s-tester/eks/cluster.(*tester).updateClusterStatus(0xc00074d4a0, 0xc00100abd0, 0x0, 0x0, 0x507acd1, 0x6) aws-k8s-tester/eks/cluster/cluster.go:725 +0xd5b github.com/aws/aws-k8s-tester/eks/cluster.(*tester).createEKS(0xc00074d4a0, 0x0, 0x0) aws-k8s-tester/eks/cluster/cluster.go:509 +0x1688 github.com/aws/aws-k8s-tester/eks/cluster.(*tester).Create(0xc00074d4a0, 0x527b490, 0xc000623a00) aws-k8s-tester/eks/cluster/cluster.go:105 +0x1de github.com/aws/aws-k8s-tester/eks.catchInterrupt.func1(0xc000661c80, 0xc001003e80) aws-k8s-tester/eks/eks.go:1682 +0x27 created by github.com/aws/aws-k8s-tester/eks.catchInterrupt aws-k8s-tester/eks/eks.go:1681 +0x7a Signed-off-by: Gyuho Lee <[email protected]> * vendor: update AWS SDK Go Signed-off-by: Gyuho Lee <[email protected]>
aws · Sep 28, 2020 · 8302fde · 8302fde
1 parent 7e547ed
commit 8302fde
Show file tree

Hide file tree

Showing 18 changed files with 379 additions and 133 deletions.
diff --git a/CHANGELOG/CHANGELOG-1.5.md b/CHANGELOG/CHANGELOG-1.5.md
@@ -3,6 +3,26 @@
 <hr>
 
 
+## [v1.5.3](https://github.com/aws/aws-k8s-tester/releases/tag/v1.5.3) (2020-09)
+
+See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.5.2...v1.5.3).
+
+### `eks`
+
+- Fix [`eks/cluster` status update panic issue](https://github.com/aws/aws-k8s-tester/pull/TODO).
+
+### Dependency
+
+- Upgrade [`github.com/aws/aws-sdk-go`](https://github.com/aws/aws-sdk-go/releases) from [`v1.34.22`](https://github.com/aws/aws-sdk-go/releases/tag/v1.34.22) to [`v1.34.32`](https://github.com/aws/aws-sdk-go/releases/tag/v1.34.32).
+
+### Go
+
+- Compile with [*Go 1.15.2*](https://golang.org/doc/devel/release.html#go1.15).
+
+
+<hr>
+
+
 ## [v1.5.2](https://github.com/aws/aws-k8s-tester/releases/tag/v1.5.2) (2020-09-12)
 
 See [code changes](https://github.com/aws/aws-k8s-tester/compare/v1.5.1...v1.5.2).

diff --git a/eks/cluster/cluster.go b/eks/cluster/cluster.go
@@ -715,16 +715,28 @@ func (ts *tester) updateClusterStatus(v wait.ClusterStatus, desired string) {
 					Proxy:           http.ProxyFromEnvironment,
 				},
 			}
-			resp, err := httpClient.Get(ts.cfg.EKSConfig.Status.ClusterOIDCIssuerURL)
-			if err != nil {
+			var resp *http.Response
+			for i := 0; i < 5; i++ {
+				resp, err = httpClient.Get(ts.cfg.EKSConfig.Status.ClusterOIDCIssuerURL)
+				if err == nil {
+					break
+				}
+				code := 0
+				if resp != nil {
+					code = resp.StatusCode
+				}
+				// TODO: parse response status code to decide retries?
 				ts.cfg.Logger.Warn("failed to fetch OIDC CA thumbprint",
 					zap.String("url", ts.cfg.EKSConfig.Status.ClusterOIDCIssuerURL),
+					zap.Int("status-code", code),
 					zap.Error(err),
 				)
+				time.Sleep(5 * time.Second)
 			}
-			defer resp.Body.Close()
-
-			if resp.TLS != nil {
+			if resp != nil && resp.Body != nil {
+				defer resp.Body.Close()
+			}
+			if resp != nil && resp.TLS != nil {
 				certs := len(resp.TLS.PeerCertificates)
 				if certs >= 1 {
 					root := resp.TLS.PeerCertificates[certs-1]

diff --git a/go.mod b/go.mod
@@ -43,7 +43,7 @@ replace (
 )
 
 require (
-	github.com/aws/aws-sdk-go v1.34.22
+	github.com/aws/aws-sdk-go v1.34.32
 	github.com/briandowns/spinner v1.11.1
 	github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
 	github.com/davecgh/go-spew v1.1.1

diff --git a/go.sum b/go.sum
@@ -102,8 +102,8 @@ github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:
 github.com/aws/aws-sdk-go v1.6.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.34.22 h1:7V2sKilVVgHqdjbW+O/xaVWYfnmuLwZdF/+6JuUh6Cw=
-github.com/aws/aws-sdk-go v1.34.22/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.34.32 h1:EHjowHEGXyLHWhcO7M7AVA+oA2c8aLE9WfRvqHwxd3A=
+github.com/aws/aws-sdk-go v1.34.32/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/bazelbuild/bazel-gazelle v0.18.2/go.mod h1:D0ehMSbS+vesFsLGiD6JXu3mVEzOlfUl8wNnq+x/9p0=
 github.com/bazelbuild/bazel-gazelle v0.19.1-0.20191105222053-70208cbdc798/go.mod h1:rPwzNHUqEzngx1iVBfO/2X2npKaT3tqPqqHW6rVsn/A=
 github.com/bazelbuild/buildtools v0.0.0-20190731111112-f720930ceb60/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
@@ -560,8 +560,10 @@ github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mo
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
-github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=

diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go