[8.0](backport #1829) Get output files when they have a suffix

e2e/_suites/kubernetes-autodiscover/autodiscover_test.go

@@ -366,19 +366,45 @@ func (m *podsManager) waitForEventsCondition(podName string, conditionFn func(ct
 	localPath := filepath.Join(tmpDir, "events")
 	exp := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx)
 	return backoff.Retry(func() error {
-		_, err := m.kubectl.Run(ctx, "cp", "--no-preserve", containerPath, localPath)
+		err := m.copyEvents(ctx, containerPath, localPath)
 		if err != nil {
-			log.Debugf("Failed to copy events from %s to %s: %s", containerPath, localPath, err)
-			return err
+			return fmt.Errorf("failed to copy events from %s: %w", containerPath, err)
 		}
 		ok, err := conditionFn(ctx, localPath)
+		if err != nil {
+			return fmt.Errorf("events condition failed: %w", err)
+		}
 		if !ok {
 			return fmt.Errorf("events do not satisfy condition")
 		}
 		return nil
 	}, exp)
 }
 
+func (m *podsManager) copyEvents(ctx context.Context, containerPath string, localPath string) error {
+	today := time.Now().Format("20060102")
+	paths := []string{
+		containerPath,
+
+		// Format used since 8.0.
+		containerPath + "-" + today + ".ndjson",
+	}
+
+	var err error
+	var output string
+	for _, containerPath := range paths {
+		// This command always succeeds, so check if the local path has been created.
+		os.Remove(localPath)
+		output, _ = m.kubectl.Run(ctx, "cp", "--no-preserve", containerPath, localPath)
+		if _, err = os.Stat(localPath); os.IsNotExist(err) {
+			continue
+		}
+		return nil
+	}
+	log.Debugf("Failed to copy events from %s to %s: %s", containerPath, localPath, output)
+	return err
+}
+
 func (m *podsManager) getPodInstances(ctx context.Context, podName string) (instances []string, err error) {
 	span, _ := apm.StartSpanOptions(m.ctx, "Getting pod instances", "pod.instances.get", apm.SpanOptions{
 		Parent: apm.SpanFromContext(m.ctx).TraceContext(),

e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl

@@ -44,6 +44,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 apiVersion: apps/v1
 kind: DaemonSet

e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl

@@ -41,6 +41,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 # Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics
 apiVersion: apps/v1

e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl

@@ -18,6 +18,8 @@ data:
     output.file:
       path: /tmp
       filename: beats-events
+      rotate_every_kb: 100000
+      rotate_on_startup: false
 ---
 apiVersion: apps/v1
 kind: DaemonSet