[7.17] systemtest: fix elastic-agent log copying (backport #7163) (#7168

)

* systemtest: fix elastic-agent log copying (#7163)

Start copying logs as soon as the container
has started, in case container startup fails;
otherwise the container logs are lost and we
cannot tell why the container failed to start.

(cherry picked from commit 9d67ac1)

# Conflicts:
#	systemtest/cmd/runapm/main.go
#	systemtest/containers.go

* Fix merge conflicts

Co-authored-by: Andrew Wilkins <[email protected]>

systemtest/containers.go

@@ -361,6 +361,7 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
 
 	return &ElasticAgentContainer{
 		request:      req,
+		exited:       make(chan struct{}),
 		StackVersion: agentImageVersion,
 	}, nil
 }
@@ -369,6 +370,7 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
 type ElasticAgentContainer struct {
 	container testcontainers.Container
 	request   testcontainers.ContainerRequest
+	exited    chan struct{}
 
 	// StackVersion holds the stack version of the container image,
 	// e.g. 8.0.0-SNAPSHOT.
@@ -388,6 +390,14 @@ type ElasticAgentContainer struct {
 	// use for enrolling the agent with Fleet. The agent will only enroll
 	// if this is specified.
 	FleetEnrollmentToken string
+
+	// Stdout, if non-nil, holds a writer to which the container's stdout
+	// will be written.
+	Stdout io.Writer
+
+	// Stderr, if non-nil, holds a writer to which the container's stderr
+	// will be written.
+	Stderr io.Writer
 }
 
 // Start starts the container.
@@ -416,9 +426,29 @@ func (c *ElasticAgentContainer) Start() error {
 	}
 	c.container = container
 
+	// Start a goroutine to read logs, and signal when the container process has exited.
+	if c.Stdout != nil || c.Stderr != nil {
+		go func() {
+			defer close(c.exited)
+			defer cancel()
+			stdout, stderr := c.Stdout, c.Stderr
+			if stdout == nil {
+				stdout = io.Discard
+			}
+			if stderr == nil {
+				stderr = io.Discard
+			}
+			_ = c.copyLogs(stdout, stderr)
+		}()
+	}
+
 	if err := container.Start(ctx); err != nil {
-		return err
+		if err != context.Canceled {
+			return fmt.Errorf("failed to start container: %w", err)
+		}
+		return errors.New("failed to start container")
 	}
+
 	if len(c.request.ExposedPorts) > 0 {
 		hostIP, err := container.Host(ctx)
 		if err != nil {
@@ -438,6 +468,41 @@ func (c *ElasticAgentContainer) Start() error {
 	return nil
 }
 
+func (c *ElasticAgentContainer) copyLogs(stdout, stderr io.Writer) error {
+	// Wait for the container to be running (or have gone past that),
+	// or ContainerLogs will return immediately.
+	ctx := context.Background()
+	for {
+		state, err := c.container.State(ctx)
+		if err != nil {
+			return err
+		}
+		if state.Status != "created" {
+			break
+		}
+	}
+
+	docker, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return err
+	}
+	defer docker.Close()
+
+	options := types.ContainerLogsOptions{
+		ShowStdout: stdout != nil,
+		ShowStderr: stderr != nil,
+		Follow:     true,
+	}
+	rc, err := docker.ContainerLogs(ctx, c.container.GetContainerID(), options)
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+
+	_, err = stdcopy.StdCopy(stdout, stderr, rc)
+	return err
+}
+
 // Close terminates and removes the container.
 func (c *ElasticAgentContainer) Close() error {
 	if c.container == nil {
@@ -446,14 +511,14 @@ func (c *ElasticAgentContainer) Close() error {
 	return c.container.Terminate(context.Background())
 }
 
-// Logs returns an io.ReadCloser that can be used for reading the
-// container's combined stdout/stderr log. If the container has not
-// been created by Start(), Logs will return an error.
-func (c *ElasticAgentContainer) Logs(ctx context.Context) (io.ReadCloser, error) {
-	if c.container == nil {
-		return nil, errors.New("container not created")
+// Wait waits for the container process to exit, and returns its state.
+func (c *ElasticAgentContainer) Wait(ctx context.Context) (*types.ContainerState, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-c.exited:
+		return c.container.State(ctx)
 	}
-	return c.container.Logs(ctx)
 }
 
 // Exec executes a command in the container, and returns its stdout and stderr.

systemtest/fleet_test.go

@@ -18,6 +18,7 @@
 package systemtest_test
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -104,21 +105,19 @@ func newAPMIntegration(t testing.TB, vars map[string]interface{}) apmIntegration
 	_, enrollmentAPIKey := systemtest.CreateAgentPolicy(t, policyName, "default", vars)
 
 	// Enroll an elastic-agent to run the APM integration.
+	var output bytes.Buffer
 	agent, err := systemtest.NewUnstartedElasticAgentContainer()
 	require.NoError(t, err)
+	agent.Stdout = &output
+	agent.Stderr = &output
 	agent.FleetEnrollmentToken = enrollmentAPIKey.APIKey
 	t.Cleanup(func() { agent.Close() })
 	t.Cleanup(func() {
 		// Log the elastic-agent container output if the test fails.
 		if !t.Failed() {
 			return
 		}
-		if logs, err := agent.Logs(context.Background()); err == nil {
-			defer logs.Close()
-			if out, err := ioutil.ReadAll(logs); err == nil {
-				t.Logf("elastic-agent logs: %s", out)
-			}
-		}
+		t.Logf("elastic-agent logs: %s", output.String())
 	})
 
 	// Start elastic-agent with port 8200 exposed, and wait for the server to service