Fix filebeat script tester tests (#17972) (#18090)

The tester assumes pipelines have a JSON extension. So it started failing when the
one of the pipelines was changed to YAML. I made changes to fix the test, but added
an explicit error if you try to use this tool with a YAML pipeline.

Additionally I noticed this tool does not handle templated pipelines.

(cherry picked from commit 96f66a4)

Co-authored-by: Andrew Kroh <[email protected]>

filebeat/fileset/fileset_test.go

@@ -56,7 +56,7 @@ func TestLoadManifestNginx(t *testing.T) {
 	manifest, err := fs.readManifest()
 	assert.NoError(t, err)
 	assert.Equal(t, manifest.ModuleVersion, "1.0")
-	assert.Equal(t, manifest.IngestPipeline, []string{"ingest/default.json"})
+	assert.Equal(t, manifest.IngestPipeline, []string{"ingest/pipeline.yml"})
 	assert.Equal(t, manifest.Input, "config/nginx-access.yml")
 
 	vars := manifest.Vars
@@ -189,7 +189,7 @@ func TestGetInputConfigNginx(t *testing.T) {
 	assert.True(t, cfg.HasField("pipeline"))
 	pipelineID, err := cfg.String("pipeline", -1)
 	assert.NoError(t, err)
-	assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipelineID)
+	assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipelineID)
 }
 
 func TestGetInputConfigNginxOverrides(t *testing.T) {
@@ -217,7 +217,7 @@ func TestGetInputConfigNginxOverrides(t *testing.T) {
 
 				pipelineID, err := c.String("pipeline", -1)
 				assert.NoError(t, err)
-				assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipelineID)
+				assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipelineID)
 			},
 		},
 		"pipeline": {
@@ -276,7 +276,7 @@ func TestGetPipelineNginx(t *testing.T) {
 	assert.Len(t, pipelines, 1)
 
 	pipeline := pipelines[0]
-	assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipeline.id)
+	assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipeline.id)
 	assert.Contains(t, pipeline.contents, "description")
 	assert.Contains(t, pipeline.contents, "processors")
 }

filebeat/fileset/modules_integration_test.go

@@ -115,7 +115,7 @@ func TestSetupNginx(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil)
+	status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-pipeline", "", nil, nil)
 	assert.Equal(t, 200, status)
 	status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
 	assert.Equal(t, 200, status)

filebeat/magefile.go

@@ -185,7 +185,9 @@ func IntegTest() {
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
 func GoIntegTest(ctx context.Context) error {
-	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+	return devtools.RunIntegTest("goIntegTest", func() error {
+		return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+	})
 }
 
 // PythonIntegTest executes the python system tests in the integration environment (Docker).

filebeat/scripts/tester/main.go

@@ -108,6 +108,11 @@ func main() {
 	}
 
 	for _, path := range paths {
+		// TODO: Add support for testing YAML pipelines.
+		if filepath.Ext(path) == ".yml" {
+			fmt.Fprintf(os.Stderr, "YAML pipelines are not supported by this tool. Cannot process %q.", path)
+			os.Exit(3)
+		}
 		err = testPipeline(*esURL, path, logs, *verbose, *simulateVerbose)
 		if err != nil {
 			os.Stderr.WriteString(err.Error())
@@ -185,8 +190,14 @@ func getPipelinePath(path, modulesPath string) ([]string, error) {
 		module := parts[0]
 		fileset := parts[1]
 
-		pathToPipeline := filepath.Join(modulesPath, module, fileset, "ingest", "pipeline.json")
-		_, err := os.Stat(pathToPipeline)
+		var pathToPipeline string
+		for _, ext := range []string{".json", ".yml"} {
+			pathToPipeline = filepath.Join(modulesPath, module, fileset, "ingest", "pipeline"+ext)
+			_, err = os.Stat(pathToPipeline)
+			if err == nil {
+				break
+			}
+		}
 		if err != nil {
 			return nil, fmt.Errorf("Cannot find pipeline in %s: %v %v\n", path, err, pathToPipeline)
 		}
@@ -199,8 +210,7 @@ func getPipelinePath(path, modulesPath string) ([]string, error) {
 			return nil, err
 		}
 		for _, f := range files {
-			isPipelineFile := strings.HasSuffix(f.Name(), ".json")
-			if isPipelineFile {
+			if isPipelineFileExtension(f.Name()) {
 				fullPath := filepath.Join(path, f.Name())
 				paths = append(paths, fullPath)
 			}
@@ -211,15 +221,23 @@ func getPipelinePath(path, modulesPath string) ([]string, error) {
 		return paths, nil
 	}
 
-	isPipelineFile := strings.HasSuffix(path, ".json")
-	if isPipelineFile {
+	if isPipelineFileExtension(path) {
 		return []string{path}, nil
 	}
 
 	return paths, nil
 
 }
 
+func isPipelineFileExtension(path string) bool {
+	ext := filepath.Ext(path)
+	switch strings.ToLower(ext) {
+	case ".yml", ".json":
+		return true
+	}
+	return false
+}
+
 func testPipeline(esURL, path string, logs []string, verbose, simulateVerbose bool) error {
 	pipeline, err := readPipeline(path)
 	if err != nil {

filebeat/scripts/tester/main_test.go

@@ -29,7 +29,7 @@ func TestGetPipelinePath(t *testing.T) {
 		count        int
 	}{
 		{
-			pipelinePath: "../../module/postgresql/log/ingest/pipeline.json",
+			pipelinePath: "../../module/postgresql/log/ingest/pipeline.yml",
 			count:        1,
 		},
 		{