Merge branch 'main' into fix/lifecycle-datarace

docker.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -602,19 +603,41 @@ func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath
 		return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode)
 	}
 
-	fileContent, err := os.ReadFile(hostFilePath)
+	f, err := os.Open(hostFilePath)
 	if err != nil {
 		return err
 	}
-	return c.CopyToContainer(ctx, fileContent, containerFilePath, fileMode)
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	// In Go 1.22 os.File is always an io.WriterTo. However, testcontainers
+	// currently allows Go 1.21, so we need to trick the compiler a little.
+	var file fs.File = f
+	return c.copyToContainer(ctx, func(tw io.Writer) error {
+		// Attempt optimized writeTo, implemented in linux
+		if wt, ok := file.(io.WriterTo); ok {
+			_, err := wt.WriteTo(tw)
+			return err
+		}
+		_, err := io.Copy(tw, f)
+		return err
+	}, info.Size(), containerFilePath, fileMode)
 }
 
 // CopyToContainer copies fileContent data to a file in container
 func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error {
-	buffer, err := tarFile(fileContent, containerFilePath, fileMode)
-	if err != nil {
+	return c.copyToContainer(ctx, func(tw io.Writer) error {
+		_, err := tw.Write(fileContent)
 		return err
-	}
+	}, int64(len(fileContent)), containerFilePath, fileMode)
+}
+
+func (c *DockerContainer) copyToContainer(ctx context.Context, fileContent func(tw io.Writer) error, fileContentSize int64, containerFilePath string, fileMode int64) error {
+	buffer, err := tarFile(containerFilePath, fileContent, fileContentSize, fileMode)
 
 	err = c.provider.client.CopyToContainer(ctx, c.ID, "/", buffer, types.CopyToContainerOptions{})
 	if err != nil {
@@ -1574,7 +1597,8 @@ func (p *DockerProvider) SaveImages(ctx context.Context, output string, images .
 		_ = imageReader.Close()
 	}()
 
-	_, err = io.Copy(outputFile, imageReader)
+	// Attempt optimized readFrom, implemented in linux
+	_, err = outputFile.ReadFrom(imageReader)
 	if err != nil {
 		return fmt.Errorf("writing images to output %w", err)
 	}

docker_exec_test.go

@@ -1,44 +1,16 @@
 package testcontainers
 
 import (
+	"bytes"
 	"context"
 	"io"
-	"strings"
 	"testing"
 
+	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/stretchr/testify/require"
-
 	tcexec "github.com/testcontainers/testcontainers-go/exec"
 )
 
-func TestExecWithMultiplexedResponse(t *testing.T) {
-	ctx := context.Background()
-	req := ContainerRequest{
-		Image: nginxAlpineImage,
-	}
-
-	container, err := GenericContainer(ctx, GenericContainerRequest{
-		ProviderType:     providerType,
-		ContainerRequest: req,
-		Started:          true,
-	})
-
-	require.NoError(t, err)
-	terminateContainerOnEnd(t, ctx, container)
-
-	code, reader, err := container.Exec(ctx, []string{"ls", "/usr/share/nginx"}, tcexec.Multiplexed())
-	require.NoError(t, err)
-	require.Zero(t, code)
-	require.NotNil(t, reader)
-
-	b, err := io.ReadAll(reader)
-	require.NoError(t, err)
-	require.NotNil(t, b)
-
-	str := string(b)
-	require.Equal(t, "html\n", str)
-}
-
 func TestExecWithOptions(t *testing.T) {
 	tests := []struct {
 		name string
@@ -79,7 +51,6 @@ func TestExecWithOptions(t *testing.T) {
 			}
 
 			container, err := GenericContainer(ctx, GenericContainerRequest{
-				ProviderType:     providerType,
 				ContainerRequest: req,
 				Started:          true,
 			})
@@ -106,32 +77,32 @@ func TestExecWithOptions(t *testing.T) {
 	}
 }
 
-func TestExecWithMultiplexedStderrResponse(t *testing.T) {
+func TestExecWithMultiplexedResponse(t *testing.T) {
 	ctx := context.Background()
 	req := ContainerRequest{
 		Image: nginxAlpineImage,
 	}
 
 	container, err := GenericContainer(ctx, GenericContainerRequest{
-		ProviderType:     providerType,
 		ContainerRequest: req,
 		Started:          true,
 	})
 
 	require.NoError(t, err)
 	terminateContainerOnEnd(t, ctx, container)
 
-	code, reader, err := container.Exec(ctx, []string{"ls", "/non-existing-directory"}, tcexec.Multiplexed())
+	code, reader, err := container.Exec(ctx, []string{"sh", "-c", "echo stdout; echo stderr >&2"}, tcexec.Multiplexed())
 	require.NoError(t, err)
-	require.NotZero(t, code)
+	require.Zero(t, code)
 	require.NotNil(t, reader)
 
 	b, err := io.ReadAll(reader)
 	require.NoError(t, err)
 	require.NotNil(t, b)
 
 	str := string(b)
-	require.Contains(t, str, "No such file or directory")
+	require.Contains(t, str, "stdout")
+	require.Contains(t, str, "stderr")
 }
 
 func TestExecWithNonMultiplexedResponse(t *testing.T) {
@@ -141,23 +112,27 @@ func TestExecWithNonMultiplexedResponse(t *testing.T) {
 	}
 
 	container, err := GenericContainer(ctx, GenericContainerRequest{
-		ProviderType:     providerType,
 		ContainerRequest: req,
 		Started:          true,
 	})
 
 	require.NoError(t, err)
 	terminateContainerOnEnd(t, ctx, container)
 
-	code, reader, err := container.Exec(ctx, []string{"ls", "/usr/share/nginx"})
+	code, reader, err := container.Exec(ctx, []string{"sh", "-c", "echo stdout; echo stderr >&2"})
 	require.NoError(t, err)
 	require.Zero(t, code)
 	require.NotNil(t, reader)
 
-	b, err := io.ReadAll(reader)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	written, err := stdcopy.StdCopy(&stdout, &stderr, reader)
 	require.NoError(t, err)
-	require.NotNil(t, b)
+	require.NotZero(t, written)
+	require.NotNil(t, stdout)
+	require.NotNil(t, stderr)
 
-	str := string(b)
-	require.True(t, strings.HasSuffix(str, "html\n"))
+	require.Equal(t, stdout.String(), "stdout\n")
+	require.Equal(t, stderr.String(), "stderr\n")
 }

docker_test.go

@@ -1854,7 +1854,7 @@ func TestContainerWithNoUserID(t *testing.T) {
 }
 
 func TestGetGatewayIP(t *testing.T) {
-	// When using docker-compose with DinD mode, and using host port or http wait strategy
+	// When using docker compose with DinD mode, and using host port or http wait strategy
 	// It's need to invoke GetGatewayIP for get the host
 	provider, err := providerType.GetProvider(WithLogger(TestLogger(t)))
 	if err != nil {

docs/features/common_functional_options.md

@@ -22,7 +22,7 @@ Using the `WithImageSubstitutors` options, you could define your own substitutio
 If you need to either pass additional environment variables to a container or override them, you can use `testcontainers.WithEnv` for example:
 
 ```golang
-postgres, err = postgresModule.RunContainer(ctx, testcontainers.WithEnv(map[string]string{"POSTGRES_INITDB_ARGS", "--no-sync"}))
+postgres, err = postgresModule.RunContainer(ctx, testcontainers.WithEnv(map[string]string{"POSTGRES_INITDB_ARGS": "--no-sync"}))
 ```
 
 #### WithLogConsumers

docs/features/docker_compose.md

@@ -7,7 +7,7 @@ This is intended to be useful on projects where Docker Compose is already used
 in dev or other environments to define services that an application may be
 dependent upon.
 
-## Using `docker-compose` directly
+## Using `docker compose` directly
 
 !!!warning
 	The minimal version of Go required to use this module is **1.21**.
@@ -16,9 +16,9 @@ dependent upon.
 go get github.com/testcontainers/testcontainers-go/modules/compose
 ```
 
-Because `docker-compose` v2 is implemented in Go it's possible for _Testcontainers for Go_ to
+Because `compose` v2 is implemented in Go it's possible for _Testcontainers for Go_ to
 use [`github.com/docker/compose`](https://github.com/docker/compose) directly and skip any process execution/_docker-compose-in-a-container_ scenario.
-The `ComposeStack` API exposes this variant of using `docker-compose` in an easy way.
+The `ComposeStack` API exposes this variant of using `docker compose` in an easy way.
 
 ### Basic examples
 
@@ -88,14 +88,14 @@ func TestSomethingElse(t *testing.T) {
 
 To interact with service containers after a stack was started it is possible to get an `*tc.DockerContainer` instance via the `ServiceContainer(...)` function.
 The function takes a **service name** (and a `context.Context`) and returns either a `*tc.DockerContainer` or an `error`.
-This is different to the previous `LocalDockerCompose` API where service containers were accessed via their **container name** e.g. `mysql_1` or `mysql-1` (depending on the version of `docker-compose`).
+This is different to the previous `LocalDockerCompose` API where service containers were accessed via their **container name** e.g. `mysql_1` or `mysql-1` (depending on the version of `docker compose`).
 
 Furthermore, there's the convenience function `Serices()` to get a list of all services **defined** by the current project.
 Note that not all of them need necessarily be correctly started as the information is based on the given compose files.
 
 ### Wait strategies
 
-Just like with regular test containers you can also apply wait strategies to `docker-compose` services.
+Just like with regular test containers you can also apply wait strategies to `docker compose` services.
 The `ComposeStack.WaitForService(...)` function allows you to apply a wait strategy to **a service by name**.
 All wait strategies are executed in parallel to both improve startup performance by not blocking too long and to fail
 early if something's wrong.
@@ -139,7 +139,7 @@ func TestSomethingWithWaiting(t *testing.T) {
 
 ### Compose environment
 
-`docker-compose` supports expansion based on environment variables.
+`docker compose` supports expansion based on environment variables.
 The `ComposeStack` supports this as well in two different variants:
 
 - `ComposeStack.WithEnv(m map[string]string) ComposeStack` to parameterize stacks from your test code
@@ -150,14 +150,14 @@ The `ComposeStack` supports this as well in two different variants:
 Also have a look at [ComposeStack](https://pkg.go.dev/github.com/testcontainers/testcontainers-go#ComposeStack) docs for
 further information.
 
-## Usage of `docker-compose` binary
+## Usage of the `docker compose` binary
 
-_Node:_ this API is deprecated and superseded by `ComposeStack` which takes advantage of `docker-compose` v2 being
+_Node:_ this API is deprecated and superseded by `ComposeStack` which takes advantage of `compose` v2 being
 implemented in Go as well by directly using the upstream project.
 
 You can override Testcontainers' default behaviour and make it use a
-docker-compose binary installed on the local machine. This will generally yield
-an experience that is closer to running docker-compose locally, with the caveat
+docker compose binary installed on the local machine. This will generally yield
+an experience that is closer to running docker compose locally, with the caveat
 that Docker Compose needs to be present on dev and CI machines.
 
 ### Examples

exec/processor.go

@@ -82,10 +82,6 @@ func Multiplexed() ProcessOption {
 
 		<-done
 
-		if errBuff.Bytes() != nil {
-			opts.Reader = &errBuff
-		} else {
-			opts.Reader = &outBuff
-		}
+		opts.Reader = io.MultiReader(&outBuff, &errBuff)
 	})
 }

file.go

@@ -110,7 +110,7 @@ func tarDir(src string, fileMode int64) (*bytes.Buffer, error) {
 }
 
 // tarFile compress a single file using tar + gzip algorithms
-func tarFile(fileContent []byte, basePath string, fileMode int64) (*bytes.Buffer, error) {
+func tarFile(basePath string, fileContent func(tw io.Writer) error, fileContentSize int64, fileMode int64) (*bytes.Buffer, error) {
 	buffer := &bytes.Buffer{}
 
 	zr := gzip.NewWriter(buffer)
@@ -119,12 +119,12 @@ func tarFile(fileContent []byte, basePath string, fileMode int64) (*bytes.Buffer
 	hdr := &tar.Header{
 		Name: basePath,
 		Mode: fileMode,
-		Size: int64(len(fileContent)),
+		Size: fileContentSize,
 	}
 	if err := tw.WriteHeader(hdr); err != nil {
 		return buffer, err
 	}
-	if _, err := tw.Write(fileContent); err != nil {
+	if err := fileContent(tw); err != nil {
 		return buffer, err
 	}
 

file_test.go

@@ -119,7 +119,10 @@ func Test_TarFile(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	buff, err := tarFile(b, "Docker.file", 0o755)
+	buff, err := tarFile("Docker.file", func(tw io.Writer) error {
+		_, err := tw.Write(b)
+		return err
+	}, int64(len(b)), 0o755)
 	if err != nil {
 		t.Fatal(err)
 	}

modules/compose/compose.go

@@ -155,11 +155,12 @@ func NewLocalDockerCompose(filePaths []string, identifier string, opts ...LocalD
 		opts[idx].ApplyToLocalCompose(dc.LocalDockerComposeOptions)
 	}
 
-	dc.Executable = "docker-compose"
+	dc.Executable = "docker"
 	if runtime.GOOS == "windows" {
-		dc.Executable = "docker-compose.exe"
+		dc.Executable = "docker.exe"
 	}
 
+	dc.composeSubcommand = "compose"
 	dc.ComposeFilePaths = filePaths
 
 	dc.absComposeFilePaths = make([]string, len(filePaths))

modules/compose/compose_api.go

@@ -34,7 +34,7 @@ func (f stackDownOptionFunc) applyToStackDown(do *api.DownOptions) {
 	f(do)
 }
 
-// RunServices is comparable to 'docker-compose run' as it only creates a subset of containers
+// RunServices is comparable to 'docker compose run' as it only creates a subset of containers
 // instead of all services defined by the project
 func RunServices(serviceNames ...string) StackUpOption {
 	return stackUpOptionFunc(func(o *stackUpOptions) {
@@ -231,7 +231,6 @@ func (d *dockerCompose) Up(ctx context.Context, opts ...StackUpOption) error {
 			Wait:    upOptions.Wait,
 		},
 	})
-
 	if err != nil {
 		return err
 	}