From 4d36c20c1938dc08e34a4bc510fb28b659e1759f Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 27 Mar 2024 13:26:12 +0900 Subject: [PATCH 1/3] feat: optimizes file copies to and from containers Signed-off-by: Adrian Cole --- docker.go | 28 ++++++++++++++++++++++------ file.go | 6 +++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docker.go b/docker.go index ec703ac913..1366b0c480 100644 --- a/docker.go +++ b/docker.go @@ -602,19 +602,34 @@ func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode) } - fileContent, err := os.ReadFile(hostFilePath) + file, err := os.Open(hostFilePath) if err != nil { return err } - return c.CopyToContainer(ctx, fileContent, containerFilePath, fileMode) + defer file.Close() + + info, err := file.Stat() + if err != nil { + return err + } + + return c.copyToContainer(ctx, func(tw io.Writer) error { + // Attempt optimized writeTo, implemented in linux + _, err := file.WriteTo(tw) + return err + }, info.Size(), containerFilePath, fileMode) } // CopyToContainer copies fileContent data to a file in container func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error { - buffer, err := tarFile(fileContent, containerFilePath, fileMode) - if err != nil { + return c.copyToContainer(ctx, func(tw io.Writer) error { + _, err := tw.Write(fileContent) return err - } + }, int64(len(fileContent)), containerFilePath, fileMode) +} + +func (c *DockerContainer) copyToContainer(ctx context.Context, fileContent func(tw io.Writer) error, fileContentSize int64, containerFilePath string, fileMode int64) error { + buffer, err := tarFile(containerFilePath, fileContent, fileContentSize, fileMode) err = c.provider.client.CopyToContainer(ctx, c.ID, "/", buffer, types.CopyToContainerOptions{}) if err != nil { @@ -1574,7 +1589,8 @@ func (p *DockerProvider) SaveImages(ctx context.Context, output string, images . _ = imageReader.Close() }() - _, err = io.Copy(outputFile, imageReader) + // Attempt optimized readFrom, implemented in linux + _, err = outputFile.ReadFrom(imageReader) if err != nil { return fmt.Errorf("writing images to output %w", err) } diff --git a/file.go b/file.go index 4c1f3ee32b..a6743cc9e4 100644 --- a/file.go +++ b/file.go @@ -110,7 +110,7 @@ func tarDir(src string, fileMode int64) (*bytes.Buffer, error) { } // tarFile compress a single file using tar + gzip algorithms -func tarFile(fileContent []byte, basePath string, fileMode int64) (*bytes.Buffer, error) { +func tarFile(basePath string, fileContent func(tw io.Writer) error, fileContentSize int64, fileMode int64) (*bytes.Buffer, error) { buffer := &bytes.Buffer{} zr := gzip.NewWriter(buffer) @@ -119,12 +119,12 @@ func tarFile(fileContent []byte, basePath string, fileMode int64) (*bytes.Buffer hdr := &tar.Header{ Name: basePath, Mode: fileMode, - Size: int64(len(fileContent)), + Size: fileContentSize, } if err := tw.WriteHeader(hdr); err != nil { return buffer, err } - if _, err := tw.Write(fileContent); err != nil { + if err := fileContent(tw); err != nil { return buffer, err } From 4649a1e9af5b9946d5084a94ed421fe645de4b2a Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 27 Mar 2024 13:58:33 +0900 Subject: [PATCH 2/3] drift Signed-off-by: Adrian Cole --- file_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/file_test.go b/file_test.go index c1fc9f0704..e5e660dee7 100644 --- a/file_test.go +++ b/file_test.go @@ -119,7 +119,10 @@ func Test_TarFile(t *testing.T) { t.Fatal(err) } - buff, err := tarFile(b, "Docker.file", 0o755) + buff, err := tarFile("Docker.file", func(tw io.Writer) error { + _, err := tw.Write(b) + return err + }, int64(len(b)), 0o755) if err != nil { t.Fatal(err) } From 61a8c3d69eeced1570af668f2c4ce5c6821e342f Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 27 Mar 2024 14:13:01 +0900 Subject: [PATCH 3/3] go 1.21 defense Signed-off-by: Adrian Cole --- docker.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker.go b/docker.go index 1366b0c480..c946103556 100644 --- a/docker.go +++ b/docker.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/url" "os" "path/filepath" @@ -602,20 +603,27 @@ func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode) } - file, err := os.Open(hostFilePath) + f, err := os.Open(hostFilePath) if err != nil { return err } - defer file.Close() + defer f.Close() - info, err := file.Stat() + info, err := f.Stat() if err != nil { return err } + // In Go 1.22 os.File is always an io.WriterTo. However, testcontainers + // currently allows Go 1.21, so we need to trick the compiler a little. + var file fs.File = f return c.copyToContainer(ctx, func(tw io.Writer) error { // Attempt optimized writeTo, implemented in linux - _, err := file.WriteTo(tw) + if wt, ok := file.(io.WriterTo); ok { + _, err := wt.WriteTo(tw) + return err + } + _, err := io.Copy(tw, f) return err }, info.Size(), containerFilePath, fileMode) }