debug test

testing/integration/logs_ingestion_test.go

@@ -53,8 +53,11 @@ func TestLogIngestionFleetManaged(t *testing.T) {
 		Sudo:  true,
 	})
 
-	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
-	defer cancel()
+	if d, ok := t.Deadline(); ok {
+		t.Logf("test global timeout: %s", d.Sub(time.Now()))
+	} else {
+		t.Logf("test global timeout not defined")
+	}
 
 	agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
 	require.NoError(t, err)
@@ -92,6 +95,14 @@ func TestLogIngestionFleetManaged(t *testing.T) {
 		Force:          true,
 	}
 
+	ctx, cancel := testcontext.WithTimeout(t, context.Background(), 5*time.Minute)
+	defer cancel()
+	if d, ok := ctx.Deadline(); ok {
+		t.Logf("context timeout: %s", d.Sub(time.Now()))
+	} else {
+		t.Fatal("context without deadline after calling context.WithDeadline")
+	}
+
 	// 2. Install the Elastic-Agent with the policy that
 	// was just created.
 	policy, err := tools.InstallAgentWithPolicy(
@@ -111,10 +122,24 @@ func TestLogIngestionFleetManaged(t *testing.T) {
 	// 4. Ensure healthy state at startup
 	checkHealthAtStartup(t, ctx, agentFixture)
 
+	ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute)
+	defer cancel()
+	if d, ok := ctx.Deadline(); ok {
+		t.Logf("context timeout: %s", d.Sub(time.Now()))
+	} else {
+		t.Fatal("context without deadline after calling context.WithDeadline")
+	}
 	t.Run("Monitoring logs are shipped", func(t *testing.T) {
 		testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy)
 	})
 
+	ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute)
+	defer cancel()
+	if d, ok := ctx.Deadline(); ok {
+		t.Logf("context timeout: %s", d.Sub(time.Now()))
+	} else {
+		t.Fatal("context without deadline after calling context.WithDeadline")
+	}
 	t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) {
 		testFlattenedDatastreamFleetPolicy(t, ctx, info, policy)
 	})
@@ -404,7 +429,7 @@ func testFlattenedDatastreamFleetPolicy(
 	require.Eventually(
 		t,
 		ensureDocumentsInES(t, ctx, info.ESClient, dsType, dsDataset, dsNamespace, numEvents),
-		120*time.Second,
+		3*time.Minute,
 		time.Second,
 		"could not get all expected documents form ES")
 }

testing/integration/metrics_monitoring_test.go

@@ -7,6 +7,7 @@
 package integration
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"testing"
@@ -17,6 +18,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/elastic/elastic-agent-libs/kibana"
+	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	atesting "github.com/elastic/elastic-agent/pkg/testing"
 	"github.com/elastic/elastic-agent/pkg/testing/define"
 	"github.com/elastic/elastic-agent/pkg/testing/tools"
@@ -76,15 +78,53 @@ func (runner *MetricsRunner) SetupSuite() {
 
 	_, err = tools.InstallPackageFromDefaultFile(ctx, runner.info.KibanaClient, "system", "1.53.1", "system_integration_setup.json", uuid.Must(uuid.NewV4()).String(), policyResp.ID)
 	require.NoError(runner.T(), err)
-
 }
 
 func (runner *MetricsRunner) TestBeatsMetrics() {
 	UnitOutputName := "default"
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20)
 	defer cancel()
-	agentStatus, err := runner.agentFixture.ExecStatus(ctx)
-	require.NoError(runner.T(), err)
+
+	var agentStatus atesting.AgentStatusOutput
+	f := runner.agentFixture
+	stateBuff := bytes.Buffer{}
+	require.Eventuallyf(runner.T(), func() bool {
+		stateBuff.Reset()
+
+		status, err := f.ExecStatus(ctx)
+		if err != nil {
+			stateBuff.WriteString(fmt.Sprintf("failed to get agent status: %v",
+				err))
+			return false
+		}
+
+		if client.State(status.State) != client.Healthy {
+			stateBuff.WriteString(fmt.Sprintf(
+				"agent isn't healthy: %s-%s",
+				client.State(status.State), status.Message))
+			return false
+		}
+
+		if len(status.Components) == 0 {
+			stateBuff.WriteString(fmt.Sprintf(
+				"healthy but without components: agent status: %s-%s",
+				client.State(status.State), status.Message))
+			return false
+		}
+
+		for _, c := range status.Components {
+
+			state := client.State(c.State)
+			if state != client.Healthy {
+				stateBuff.WriteString(fmt.Sprintf(
+					"%s not health: %s: %s",
+					c.Name, state, c.Message))
+				return false
+			}
+		}
+
+		return true
+	}, 2*time.Minute, 15*time.Second, "agent never became healthy. Last status: %v", &stateBuff)
 
 	componentIds := []string{
 		fmt.Sprintf("system/metrics-%s", UnitOutputName),

testing/integration/monitoring_probe_reload_test.go

@@ -172,12 +172,12 @@ func (runner *MonitoringRunner) AllComponentsHealthy(ctx context.Context) {
 		}
 
 		for _, comp := range status.Components {
-			runner.T().Logf("component state: %s", comp.Message)
+			runner.T().Logf("%s current state: %s", comp.Name, comp.Message)
 			if comp.State != int(cproto.State_HEALTHY) {
 				compDebugName = comp.Name
 				allHealthy = false
 			}
 		}
 		return allHealthy
-	}, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %s", compDebugName)
+	}, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %q", compDebugName)
 }