From 421f248815298adae108a35b70e353efa4dec468 Mon Sep 17 00:00:00 2001 From: Zack Brown Date: Tue, 10 Nov 2020 22:28:59 -0500 Subject: [PATCH 1/2] CheckSshConnectionWithRetry / CheckSshConnectionWithRetryE + tests --- go.mod | 4 ++-- go.sum | 21 ++++++++++++++------- modules/ssh/ssh.go | 28 ++++++++++++++++++++++++++++ modules/ssh/ssh_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index be31d6366..6bbdd82fd 100644 --- a/go.mod +++ b/go.mod @@ -38,9 +38,9 @@ require ( github.com/urfave/cli v1.22.2 github.com/zclconf/go-cty v1.2.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/net v0.0.0-20201021035429-f5854403a974 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/tools v0.0.0-20200113040837-eac381796e91 // indirect + golang.org/x/tools v0.0.0-20201110201400-7099162a900a // indirect google.golang.org/api v0.15.0 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 k8s.io/api v0.19.3 diff --git a/go.sum b/go.sum index bb106501f..06c34b174 100644 --- a/go.sum +++ b/go.sum @@ -424,7 +424,7 @@ github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6Ac github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/zclconf/go-cty v1.2.0 h1:sPHsy7ADcIZQP3vILvTjrh74ZA175TFP5vqiNK1UmlI= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.2.1 h1:vGMsygfmeCl4Xb6OA5U5XVAaQZ69FvoG7X2jUtQujb8= github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= @@ -473,6 +473,8 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,8 +498,9 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -509,8 +512,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -539,8 +543,9 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -574,16 +579,18 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191205215504-7b8c8591a921/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200113040837-eac381796e91 h1:OOkytthzFBKHY5EfEgLUabprb0LtJVkQtNxAQ02+UE4= -golang.org/x/tools v0.0.0-20200113040837-eac381796e91/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201110201400-7099162a900a h1:5E6TPwSBG74zT8xSrVc8W59K4ch4NFobVTnh2BYzHyU= +golang.org/x/tools v0.0.0-20201110201400-7099162a900a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index dfbeafed4..e6b220b9e 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -14,6 +14,7 @@ import ( "github.com/gruntwork-io/terratest/modules/files" "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/retry" "github.com/gruntwork-io/terratest/modules/testing" "github.com/hashicorp/go-multierror" "golang.org/x/crypto/ssh" @@ -199,6 +200,33 @@ func CheckSshConnectionE(t testing.TestingT, host Host) error { return err } +// CheckSshConnectionWithRetry attempts to connect via SSH until max retries has been exceeded and fails the test +// if the connection fails +func CheckSshConnectionWithRetry(t testing.TestingT, host Host, retries int, sleepBetweenRetries time.Duration, f ...func(testing.TestingT, Host) error) { + handler := CheckSshConnectionE + if f != nil { + handler = f[0] + } + err := CheckSshConnectionWithRetryE(t, host, retries, sleepBetweenRetries, handler) + if err != nil { + t.Fatal(err) + } +} + +// CheckSshConnectionWithRetryE attempts to connect via SSH until max retries has been exceeded and returns an error if +// the connection fails +func CheckSshConnectionWithRetryE(t testing.TestingT, host Host, retries int, sleepBetweenRetries time.Duration, f ...func(testing.TestingT, Host) error) error { + handler := CheckSshConnectionE + if f != nil { + handler = f[0] + } + _, err := retry.DoWithRetryE(t, fmt.Sprintf("Checking SSH connection to %s", host.Hostname), retries, sleepBetweenRetries, func() (string, error) { + return "", handler(t, host) + }) + + return err +} + // CheckSshCommand checks that you can connect via SSH to the given host and run the given command. Returns the stdout/stderr. func CheckSshCommand(t testing.TestingT, host Host, command string) string { out, err := CheckSshCommandE(t, host, command) diff --git a/modules/ssh/ssh_test.go b/modules/ssh/ssh_test.go index 147f77f5a..6f2ddd3c5 100644 --- a/modules/ssh/ssh_test.go +++ b/modules/ssh/ssh_test.go @@ -1,8 +1,11 @@ package ssh import ( + "errors" + "fmt" "testing" + grunttest "github.com/gruntwork-io/terratest/modules/testing" "github.com/stretchr/testify/assert" ) @@ -22,3 +25,27 @@ func TestHostWithCustomPort(t *testing.T) { assert.Equal(t, customPort, host.getPort(), "host.getPort() did not return the custom port number") } + +func TestCheckSSHConnectionWithRetryE(t *testing.T) { + timesCalled = 0 + host := Host{Hostname: "Host"} + assert.Nil(t, CheckSshConnectionWithRetryE(t, host, 10, 3, mockSshConnectionE)) +} + +func TestCheckSshConnectionWithRetry(t *testing.T) { + timesCalled = 0 + host := Host{Hostname: "Host"} + CheckSshConnectionWithRetry(t, host, 10, 3, mockSshConnectionE) +} + +var timesCalled int + +func mockSshConnectionE(t grunttest.TestingT, host Host) error { + timesCalled += 1 + fmt.Println() + if timesCalled >= 5 { + return nil + } else { + return errors.New(fmt.Sprintf("Called %v times", timesCalled)) + } +} From 061deaa2c125664f42f4b6e7c4ec7be7cfc2ca69 Mon Sep 17 00:00:00 2001 From: Zack Brown Date: Tue, 30 Mar 2021 14:21:40 -0400 Subject: [PATCH 2/2] Add methods for testing SSH Command * CheckSshCommandEWithRetry * CheckSshCommandWithRetry * Tests for failing on max retries * Tests for passing after some retries --- modules/ssh/ssh.go | 26 +++++++++++++++ modules/ssh/ssh_test.go | 71 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/modules/ssh/ssh.go b/modules/ssh/ssh.go index e6b220b9e..147363550 100644 --- a/modules/ssh/ssh.go +++ b/modules/ssh/ssh.go @@ -261,6 +261,32 @@ func CheckSshCommandE(t testing.TestingT, host Host, command string) (string, er return runSSHCommand(t, sshSession) } +// CheckSshCommandWithRetry checks that you can connect via SSH to the given host and run the given command until max retries have been exceeded. Returns the stdout/stderr. +func CheckSshCommandWithRetry(t testing.TestingT, host Host, command string, retries int, sleepBetweenRetries time.Duration, f ...func(testing.TestingT, Host, string) (string, error)) string { + handler := CheckSshCommandE + if f != nil { + handler = f[0] + } + out, err := CheckSshCommandWithRetryE(t, host, command, retries, sleepBetweenRetries, handler) + if err != nil { + t.Fatal(err) + } + return out +} + +// CheckSshCommandWithRetryE checks that you can connect via SSH to the given host and run the given command until max retries has been exceeded. +// It return an error if the command fails after max retries has been exceeded. + +func CheckSshCommandWithRetryE(t testing.TestingT, host Host, command string, retries int, sleepBetweenRetries time.Duration, f ...func(testing.TestingT, Host, string) (string, error)) (string, error) { + handler := CheckSshCommandE + if f != nil { + handler = f[0] + } + return retry.DoWithRetryE(t, fmt.Sprintf("Checking SSH connection to %s", host.Hostname), retries, sleepBetweenRetries, func() (string, error) { + return handler(t, host, command) + }) +} + // CheckPrivateSshConnection attempts to connect to privateHost (which is not addressable from the Internet) via a // separate publicHost (which is addressable from the Internet) and then executes "command" on privateHost and returns // its output. It is useful for checking that it's possible to SSH from a Bastion Host to a private instance. diff --git a/modules/ssh/ssh_test.go b/modules/ssh/ssh_test.go index 6f2ddd3c5..bdf41ca03 100644 --- a/modules/ssh/ssh_test.go +++ b/modules/ssh/ssh_test.go @@ -26,26 +26,87 @@ func TestHostWithCustomPort(t *testing.T) { assert.Equal(t, customPort, host.getPort(), "host.getPort() did not return the custom port number") } -func TestCheckSSHConnectionWithRetryE(t *testing.T) { +// global var for use in mock callback +var timesCalled int + +func TestCheckSshConnectionWithRetryE(t *testing.T) { + // Reset the global call count timesCalled = 0 + host := Host{Hostname: "Host"} - assert.Nil(t, CheckSshConnectionWithRetryE(t, host, 10, 3, mockSshConnectionE)) + retries := 10 + + assert.Nil(t, CheckSshConnectionWithRetryE(t, host, retries, 3, mockSshConnectionE)) +} + +func TestCheckSshConnectionWithRetryEExceedsMaxRetries(t *testing.T) { + // Reset the global call count + timesCalled = 0 + + host := Host{Hostname: "Host"} + + // Not enough retries + retries := 3 + + assert.Error(t, CheckSshConnectionWithRetryE(t, host, retries, 3, mockSshConnectionE)) } func TestCheckSshConnectionWithRetry(t *testing.T) { + // Reset the global call count timesCalled = 0 + host := Host{Hostname: "Host"} - CheckSshConnectionWithRetry(t, host, 10, 3, mockSshConnectionE) + retries := 10 + + CheckSshConnectionWithRetry(t, host, retries, 3, mockSshConnectionE) } -var timesCalled int +func TestCheckSshCommandWithRetryE(t *testing.T) { + // Reset the global call count + timesCalled = 0 + + host := Host{Hostname: "Host"} + command := "echo -n hello world" + retries := 10 + + _, err := CheckSshCommandWithRetryE(t, host, command, retries, 3, mockSshCommandE) + assert.Nil(t, err) +} + +func TestCheckSshCommandWithRetryEExceedsRetries(t *testing.T) { + // Reset the global call count + timesCalled = 0 + + host := Host{Hostname: "Host"} + command := "echo -n hello world" + + // Not enough retries + retries := 3 + + _, err := CheckSshCommandWithRetryE(t, host, command, retries, 3, mockSshCommandE) + assert.Error(t, err) +} + +func TestCheckSshCommandWithRetry(t *testing.T) { + // Reset the global call count + timesCalled = 0 + + host := Host{Hostname: "Host"} + command := "echo -n hello world" + retries := 10 + + CheckSshCommandWithRetry(t, host, command, retries, 3, mockSshCommandE) +} func mockSshConnectionE(t grunttest.TestingT, host Host) error { timesCalled += 1 - fmt.Println() if timesCalled >= 5 { return nil } else { return errors.New(fmt.Sprintf("Called %v times", timesCalled)) } } + +func mockSshCommandE(t grunttest.TestingT, host Host, command string) (string, error) { + return "", mockSshConnectionE(t, host) +}