From e667542ce2ebe3eea62a31a8d01866b2268e653b Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:06:58 +0700 Subject: [PATCH 01/10] Respect custom waitStrategy for InfluxDB (#1) fix(influxdb): Respect passed waitStrategy --- modules/influxdb/influxdb.go | 52 +++++++++++++++---------------- modules/influxdb/influxdb_test.go | 25 +++++++++++++++ 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index 609b11467b..ef781f1e00 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -34,7 +34,6 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom "INFLUXDB_HTTP_HTTPS_ENABLED": "false", "INFLUXDB_HTTP_AUTH_ENABLED": "false", }, - WaitingFor: wait.ForListeningPort("8086/tcp"), } genericContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -47,8 +46,24 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom } } - hasInitDb := false + if genericContainerReq.WaitingFor == nil { + genericContainerReq.WaitingFor = defaultWaitStrategy(genericContainerReq) + } + + container, err := testcontainers.GenericContainer(ctx, genericContainerReq) + var c *InfluxDbContainer + if container != nil { + c = &InfluxDbContainer{Container: container} + } + + if err != nil { + return c, fmt.Errorf("generic container: %w", err) + } + + return c, nil +} +func defaultWaitStrategy(genericContainerReq testcontainers.GenericContainerRequest) wait.Strategy { for _, f := range genericContainerReq.Files { if f.ContainerFilePath == "/" && strings.HasSuffix(f.HostFilePath, "docker-entrypoint-initdb.d") { // Init service in container will start influxdb, run scripts in docker-entrypoint-initdb.d and then @@ -57,39 +72,24 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom // "Open shard" which is the last thing that happens before the server is ready to accept connections. // This is probably different for InfluxDB 2.x, but that is left as an exercise for the reader. strategies := []wait.Strategy{ - genericContainerReq.WaitingFor, + wait.ForListeningPort("8086/tcp"), wait.ForLog("influxdb init process in progress..."), wait.ForLog("Server shutdown completed"), wait.ForLog("Opened shard"), } - genericContainerReq.WaitingFor = wait.ForAll(strategies...) - hasInitDb = true - break + return wait.ForAll(strategies...) } } - if !hasInitDb { - if lastIndex := strings.LastIndex(genericContainerReq.Image, ":"); lastIndex != -1 { - tag := genericContainerReq.Image[lastIndex+1:] - if tag == "latest" || tag[0] == '2' { - genericContainerReq.WaitingFor = wait.ForLog(`Listening log_id=[0-9a-zA-Z_~]+ service=tcp-listener transport=http`).AsRegexp() - } - } else { - genericContainerReq.WaitingFor = wait.ForLog("Listening for signals") + if lastIndex := strings.LastIndex(genericContainerReq.Image, ":"); lastIndex != -1 { + tag := genericContainerReq.Image[lastIndex+1:] + if tag == "latest" || tag[0] == '2' { + return wait.ForLog(`Listening log_id=[0-9a-zA-Z_~]+ service=tcp-listener transport=http`).AsRegexp() } + } else { + return wait.ForLog("Listening for signals") } - - container, err := testcontainers.GenericContainer(ctx, genericContainerReq) - var c *InfluxDbContainer - if container != nil { - c = &InfluxDbContainer{Container: container} - } - - if err != nil { - return c, fmt.Errorf("generic container: %w", err) - } - - return c, nil + return wait.ForListeningPort("8086/tcp") } func (c *InfluxDbContainer) MustConnectionUrl(ctx context.Context) string { diff --git a/modules/influxdb/influxdb_test.go b/modules/influxdb/influxdb_test.go index e04a800dc6..13860d7527 100644 --- a/modules/influxdb/influxdb_test.go +++ b/modules/influxdb/influxdb_test.go @@ -3,6 +3,7 @@ package influxdb_test import ( "context" "encoding/json" + "net/http" "path/filepath" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/influxdb" + "github.com/testcontainers/testcontainers-go/wait" ) func TestV1Container(t *testing.T) { @@ -48,6 +50,29 @@ func TestV2Container(t *testing.T) { } } +func TestWithWaitStrategy(t *testing.T) { + ctx := context.Background() + influxDbContainer, err := influxdb.Run(ctx, + "influxdb:2.7.5-alpine", + influxdb.WithDatabase("foo"), + influxdb.WithUsername("root"), + influxdb.WithPassword("password"), + testcontainers.WithWaitStrategy(wait.ForHTTP("/health"). + WithStatusCodeMatcher(func(status int) bool { + return status == http.StatusOK + })), + ) + testcontainers.CleanupContainer(t, influxDbContainer) + require.NoError(t, err) + + state, err := influxDbContainer.State(ctx) + require.NoError(t, err) + + if !state.Running { + t.Fatal("InfluxDB container is not running") + } +} + func TestWithInitDb(t *testing.T) { ctx := context.Background() influxDbContainer, err := influxdb.Run(ctx, From 3b1e1297260a23f8c533c0523bb6d0c1cc2ea8b2 Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Wed, 23 Oct 2024 07:27:04 +0700 Subject: [PATCH 02/10] Refactor test to use require for container state assertion Co-authored-by: Steven Hartland --- modules/influxdb/influxdb_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/influxdb/influxdb_test.go b/modules/influxdb/influxdb_test.go index 13860d7527..e22eb90eb8 100644 --- a/modules/influxdb/influxdb_test.go +++ b/modules/influxdb/influxdb_test.go @@ -68,9 +68,7 @@ func TestWithWaitStrategy(t *testing.T) { state, err := influxDbContainer.State(ctx) require.NoError(t, err) - if !state.Running { - t.Fatal("InfluxDB container is not running") - } + require.True(t, state.Running) } func TestWithInitDb(t *testing.T) { From 3a7ddf5b567494ffa152d0a0e987f4b28d6c9bb6 Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:16:24 +0700 Subject: [PATCH 03/10] Change default wait strategy from checking logs to health check for influxdb2 (#2) --- modules/influxdb/influxdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index ef781f1e00..d70a036e0f 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -84,7 +84,7 @@ func defaultWaitStrategy(genericContainerReq testcontainers.GenericContainerRequ if lastIndex := strings.LastIndex(genericContainerReq.Image, ":"); lastIndex != -1 { tag := genericContainerReq.Image[lastIndex+1:] if tag == "latest" || tag[0] == '2' { - return wait.ForLog(`Listening log_id=[0-9a-zA-Z_~]+ service=tcp-listener transport=http`).AsRegexp() + return wait.ForHTTP("/health") } } else { return wait.ForLog("Listening for signals") From 28c28ee31793f352c3501dcc546f0121a4fb824b Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:22:53 +0700 Subject: [PATCH 04/10] /health check waitStrategy for influx1 and influx2 --- modules/influxdb/influxdb.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index d70a036e0f..903f025ac6 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -3,6 +3,7 @@ package influxdb import ( "context" "fmt" + "io" "path" "strings" @@ -81,15 +82,14 @@ func defaultWaitStrategy(genericContainerReq testcontainers.GenericContainerRequ } } - if lastIndex := strings.LastIndex(genericContainerReq.Image, ":"); lastIndex != -1 { - tag := genericContainerReq.Image[lastIndex+1:] - if tag == "latest" || tag[0] == '2' { - return wait.ForHTTP("/health") - } - } else { - return wait.ForLog("Listening for signals") - } - return wait.ForListeningPort("8086/tcp") + return wait.ForHTTP("/health"). + WithResponseMatcher(func(body io.Reader) bool { + bs, err := io.ReadAll(body) + if err != nil { + return false + } + return strings.Contains(string(bs), "ready for queries and writes") + }) } func (c *InfluxDbContainer) MustConnectionUrl(ctx context.Context) string { From 825667cfa713629a681a6e016b3a5a060ba6ef4d Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:27:48 +0700 Subject: [PATCH 05/10] Undo health endpoint test after default strategy was changed --- modules/influxdb/influxdb_test.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/modules/influxdb/influxdb_test.go b/modules/influxdb/influxdb_test.go index e22eb90eb8..e04a800dc6 100644 --- a/modules/influxdb/influxdb_test.go +++ b/modules/influxdb/influxdb_test.go @@ -3,7 +3,6 @@ package influxdb_test import ( "context" "encoding/json" - "net/http" "path/filepath" "testing" "time" @@ -14,7 +13,6 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/modules/influxdb" - "github.com/testcontainers/testcontainers-go/wait" ) func TestV1Container(t *testing.T) { @@ -50,27 +48,6 @@ func TestV2Container(t *testing.T) { } } -func TestWithWaitStrategy(t *testing.T) { - ctx := context.Background() - influxDbContainer, err := influxdb.Run(ctx, - "influxdb:2.7.5-alpine", - influxdb.WithDatabase("foo"), - influxdb.WithUsername("root"), - influxdb.WithPassword("password"), - testcontainers.WithWaitStrategy(wait.ForHTTP("/health"). - WithStatusCodeMatcher(func(status int) bool { - return status == http.StatusOK - })), - ) - testcontainers.CleanupContainer(t, influxDbContainer) - require.NoError(t, err) - - state, err := influxDbContainer.State(ctx) - require.NoError(t, err) - - require.True(t, state.Running) -} - func TestWithInitDb(t *testing.T) { ctx := context.Background() influxDbContainer, err := influxdb.Run(ctx, From 56d1e62e7d4df6067de6dbc0000f7aff17968a22 Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:07:57 +0700 Subject: [PATCH 06/10] Update http health check to verify JSON status value (#3) --- modules/influxdb/influxdb.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index 903f025ac6..866b8a0a56 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -2,6 +2,7 @@ package influxdb import ( "context" + "encoding/json" "fmt" "io" "path" @@ -69,26 +70,29 @@ func defaultWaitStrategy(genericContainerReq testcontainers.GenericContainerRequ if f.ContainerFilePath == "/" && strings.HasSuffix(f.HostFilePath, "docker-entrypoint-initdb.d") { // Init service in container will start influxdb, run scripts in docker-entrypoint-initdb.d and then // terminate the influxdb server, followed by restart of influxdb. This is tricky to wait for, and - // in this case, we are assuming that data was added by init script, so we then look for an - // "Open shard" which is the last thing that happens before the server is ready to accept connections. + // in this case, we are assuming that data was added by init script // This is probably different for InfluxDB 2.x, but that is left as an exercise for the reader. strategies := []wait.Strategy{ - wait.ForListeningPort("8086/tcp"), - wait.ForLog("influxdb init process in progress..."), wait.ForLog("Server shutdown completed"), - wait.ForLog("Opened shard"), + waitForHttpHealth(), } return wait.ForAll(strategies...) } } + return waitForHttpHealth() +} +func waitForHttpHealth() *wait.HTTPStrategy { return wait.ForHTTP("/health"). WithResponseMatcher(func(body io.Reader) bool { - bs, err := io.ReadAll(body) - if err != nil { + decoder := json.NewDecoder(body) + r := struct { + Status string `json:"status"` + }{} + if err := decoder.Decode(&r); err != nil { return false } - return strings.Contains(string(bs), "ready for queries and writes") + return r.Status == "pass" }) } From 90ad980c08af6f9b9e19b03040e2029942affb48 Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Thu, 31 Oct 2024 07:46:56 +0700 Subject: [PATCH 07/10] Move Shutdown check to WithInitDb (#4) Move Shutdown check to WithInitDb --- modules/influxdb/influxdb.go | 76 ++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index 866b8a0a56..b2988bdd9d 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "path" - "strings" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" @@ -36,6 +35,7 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom "INFLUXDB_HTTP_HTTPS_ENABLED": "false", "INFLUXDB_HTTP_AUTH_ENABLED": "false", }, + WaitingFor: waitForHttpHealth(), } genericContainerReq := testcontainers.GenericContainerRequest{ ContainerRequest: req, @@ -48,10 +48,6 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom } } - if genericContainerReq.WaitingFor == nil { - genericContainerReq.WaitingFor = defaultWaitStrategy(genericContainerReq) - } - container, err := testcontainers.GenericContainer(ctx, genericContainerReq) var c *InfluxDbContainer if container != nil { @@ -65,37 +61,6 @@ func Run(ctx context.Context, img string, opts ...testcontainers.ContainerCustom return c, nil } -func defaultWaitStrategy(genericContainerReq testcontainers.GenericContainerRequest) wait.Strategy { - for _, f := range genericContainerReq.Files { - if f.ContainerFilePath == "/" && strings.HasSuffix(f.HostFilePath, "docker-entrypoint-initdb.d") { - // Init service in container will start influxdb, run scripts in docker-entrypoint-initdb.d and then - // terminate the influxdb server, followed by restart of influxdb. This is tricky to wait for, and - // in this case, we are assuming that data was added by init script - // This is probably different for InfluxDB 2.x, but that is left as an exercise for the reader. - strategies := []wait.Strategy{ - wait.ForLog("Server shutdown completed"), - waitForHttpHealth(), - } - return wait.ForAll(strategies...) - } - } - return waitForHttpHealth() -} - -func waitForHttpHealth() *wait.HTTPStrategy { - return wait.ForHTTP("/health"). - WithResponseMatcher(func(body io.Reader) bool { - decoder := json.NewDecoder(body) - r := struct { - Status string `json:"status"` - }{} - if err := decoder.Decode(&r); err != nil { - return false - } - return r.Status == "pass" - }) -} - func (c *InfluxDbContainer) MustConnectionUrl(ctx context.Context) string { connectionString, err := c.ConnectionUrl(ctx) if err != nil { @@ -151,9 +116,22 @@ func WithConfigFile(configFile string) testcontainers.CustomizeRequestOption { } } -// WithInitDb will copy a 'docker-entrypoint-initdb.d' directory to the container. -// The secPath is the path to the directory on the host machine. -// The directory will be copied to the root of the container. +// WithInitDb copies a 'docker-entrypoint-initdb.d' directory from the specified host path to the root of the container. +// The `srcPath` parameter should point to the directory containing initialization files on the host. +// +// Initialization Process in the Container: +// 1. The copied 'docker-entrypoint-initdb.d' directory contains scripts that initialize the database. +// 2. On container start, InfluxDB runs, executes the scripts in 'docker-entrypoint-initdb.d', and then shuts down. +// 3. The InfluxDB server restarts automatically after initialization to make the new data available. +// +// Note: This approach assumes the initialization completes on startup and the data is properly added. +// This behavior may differ in InfluxDB 2.x and may require additional handling. +// +// Parameters: +// - srcPath: The host path to the directory containing initialization scripts. +// +// Returns: +// - testcontainers.CustomizeRequestOption: An option to customize the container request. func WithInitDb(srcPath string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { cf := testcontainers.ContainerFile{ @@ -162,6 +140,26 @@ func WithInitDb(srcPath string) testcontainers.CustomizeRequestOption { FileMode: 0o755, } req.Files = append(req.Files, cf) + + strategies := []wait.Strategy{ + wait.ForLog("Server shutdown completed"), + waitForHttpHealth(), + } + req.WaitingFor = wait.ForAll(strategies...) return nil } } + +func waitForHttpHealth() *wait.HTTPStrategy { + return wait.ForHTTP("/health"). + WithResponseMatcher(func(body io.Reader) bool { + decoder := json.NewDecoder(body) + r := struct { + Status string `json:"status"` + }{} + if err := decoder.Decode(&r); err != nil { + return false + } + return r.Status == "pass" + }) +} From df78856a71238e16ac0e2e1bc7969dc0b1c882eb Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:51:20 +0700 Subject: [PATCH 08/10] Remove slice Co-authored-by: Steven Hartland --- modules/influxdb/influxdb.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index b2988bdd9d..c3f08e71fa 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -141,11 +141,10 @@ func WithInitDb(srcPath string) testcontainers.CustomizeRequestOption { } req.Files = append(req.Files, cf) - strategies := []wait.Strategy{ + req.WaitingFor = wait.ForAll( wait.ForLog("Server shutdown completed"), waitForHttpHealth(), - } - req.WaitingFor = wait.ForAll(strategies...) + ) return nil } } From e0a47e9942a55abd82d09128d0b7c42084763cc6 Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Thu, 31 Oct 2024 19:51:44 +0700 Subject: [PATCH 09/10] Simplify comment Co-authored-by: Steven Hartland --- modules/influxdb/influxdb.go | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index c3f08e71fa..1da904bd5a 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -116,22 +116,8 @@ func WithConfigFile(configFile string) testcontainers.CustomizeRequestOption { } } -// WithInitDb copies a 'docker-entrypoint-initdb.d' directory from the specified host path to the root of the container. -// The `srcPath` parameter should point to the directory containing initialization files on the host. -// -// Initialization Process in the Container: -// 1. The copied 'docker-entrypoint-initdb.d' directory contains scripts that initialize the database. -// 2. On container start, InfluxDB runs, executes the scripts in 'docker-entrypoint-initdb.d', and then shuts down. -// 3. The InfluxDB server restarts automatically after initialization to make the new data available. -// -// Note: This approach assumes the initialization completes on startup and the data is properly added. -// This behavior may differ in InfluxDB 2.x and may require additional handling. -// -// Parameters: -// - srcPath: The host path to the directory containing initialization scripts. -// -// Returns: -// - testcontainers.CustomizeRequestOption: An option to customize the container request. +// WithInitDb returns a request customizer that initialises the database using the file `docker-entrypoint-initdb.d` +// located in `srcPath`. func WithInitDb(srcPath string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { cf := testcontainers.ContainerFile{ From fe28e6e46d52f0d887aaabdd66d2489b2b440f1b Mon Sep 17 00:00:00 2001 From: Marcin Milewski <55257568+marcinmilewski93@users.noreply.github.com> Date: Fri, 1 Nov 2024 07:25:29 +0700 Subject: [PATCH 10/10] More precise WitInitDb domment --- modules/influxdb/influxdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/influxdb/influxdb.go b/modules/influxdb/influxdb.go index 1da904bd5a..4c6024d79c 100644 --- a/modules/influxdb/influxdb.go +++ b/modules/influxdb/influxdb.go @@ -117,7 +117,7 @@ func WithConfigFile(configFile string) testcontainers.CustomizeRequestOption { } // WithInitDb returns a request customizer that initialises the database using the file `docker-entrypoint-initdb.d` -// located in `srcPath`. +// located in `srcPath` directory. func WithInitDb(srcPath string) testcontainers.CustomizeRequestOption { return func(req *testcontainers.GenericContainerRequest) error { cf := testcontainers.ContainerFile{