New Filebeat subcommand: generate (#9314)

This PR exposes the existing Filebeat module generators as `generate` subcommand.
```
$ ./filebeat generate -h
Generate Filebeat modules, filesets and fields.yml

Usage:
  filebeat generate [command]

Available Commands:
  fields      Generates a new fields.yml file for fileset
  fileset     Generates a new fileset
  module      Generates a new module
```

The subcommands `module`, `fileset` and `fields` use the same code as the scripts.

I added E2E tests to validate the functionality.

Blocked by #9147 as it contains the refactorings of that PR.

filebeat/cmd/generate.go

@@ -0,0 +1,121 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/elastic/beats/filebeat/generator/fields"
+	"github.com/elastic/beats/filebeat/generator/fileset"
+	"github.com/elastic/beats/filebeat/generator/module"
+	"github.com/elastic/beats/libbeat/common/cli"
+	"github.com/elastic/beats/libbeat/paths"
+)
+
+var (
+	defaultHomePath = paths.Resolve(paths.Home, "")
+)
+
+func genGenerateCmd() *cobra.Command {
+	generateCmd := cobra.Command{
+		Use:   "generate",
+		Short: "Generate Filebeat modules, filesets and fields.yml",
+	}
+	generateCmd.AddCommand(genGenerateModuleCmd())
+	generateCmd.AddCommand(genGenerateFilesetCmd())
+	generateCmd.AddCommand(genGenerateFieldsCmd())
+
+	return &generateCmd
+}
+
+func genGenerateModuleCmd() *cobra.Command {
+	genModuleCmd := &cobra.Command{
+		Use:   "module [module]",
+		Short: "Generates a new module",
+		Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+			modulesPath, _ := cmd.Flags().GetString("modules-path")
+			esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+
+			if len(args) != 1 {
+				fmt.Fprintf(os.Stderr, "Exactly one parameter is required: module name\n")
+				os.Exit(1)
+			}
+			name := args[0]
+
+			return module.Generate(name, modulesPath, esBeatsPath)
+		}),
+	}
+
+	genModuleCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
+	genModuleCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+
+	return genModuleCmd
+}
+
+func genGenerateFilesetCmd() *cobra.Command {
+	genFilesetCmd := &cobra.Command{
+		Use:   "fileset [module] [fileset]",
+		Short: "Generates a new fileset",
+		Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+			modulesPath, _ := cmd.Flags().GetString("modules-path")
+			esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+
+			if len(args) != 2 {
+				fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n")
+				os.Exit(1)
+			}
+			moduleName := args[0]
+			filesetName := args[1]
+
+			return fileset.Generate(moduleName, filesetName, modulesPath, esBeatsPath)
+		}),
+	}
+
+	genFilesetCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
+	genFilesetCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+
+	return genFilesetCmd
+}
+
+func genGenerateFieldsCmd() *cobra.Command {
+	genFieldsCmd := &cobra.Command{
+		Use:   "fields [module] [fileset]",
+		Short: "Generates a new fields.yml file for fileset",
+		Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+			esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+			noDoc, _ := cmd.Flags().GetBool("without-documentation")
+
+			if len(args) != 2 {
+				fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n")
+				os.Exit(1)
+			}
+			moduleName := args[0]
+			filesetName := args[1]
+
+			return fields.Generate(esBeatsPath, moduleName, filesetName, noDoc)
+		}),
+	}
+
+	genFieldsCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+	genFieldsCmd.Flags().Bool("without-documentation", false, "Do not add description fields")
+
+	return genFieldsCmd
+}

filebeat/cmd/root.go

@@ -43,4 +43,5 @@ func init() {
 	RootCmd.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
+	RootCmd.AddCommand(genGenerateCmd())
 }

filebeat/tests/system/input/my-module-pipeline.json

@@ -0,0 +1,27 @@
+{
+  "description": "Pipeline for parsing PostgreSQL logs.",
+  "processors": [
+    {
+      "grok": {
+        "field": "message",
+        "ignore_missing": true,
+        "patterns": [
+            "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}:  (duration: %{NUMBER:postgresql.log.duration} ms  statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})"
+        ],
+        "pattern_definitions": {
+          "LOCALDATETIME": "[-0-9]+ %{TIME}",
+          "GREEDYDATA": "(.|\n|\t)*",
+          "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*"
+        }
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "set": {
+        "field": "error.message",
+        "value": "{{ _ingest.on_failure_message }}"
+      }
+    }
+  ]
+}

filebeat/tests/system/test_generate.py

@@ -0,0 +1,127 @@
+import os
+import shutil
+import filebeat
+
+
+class Test(filebeat.BaseTest):
+    def test_generate_module(self):
+        """
+        Test filebeat generate module my_module generates a new module
+        """
+
+        self._create_clean_test_modules()
+        exit_code = self.run_beat(
+            extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+        assert exit_code == 0
+
+        module_root = os.path.join("test_modules", "module", "my_module")
+        module_meta_root = os.path.join(module_root, "_meta")
+        self._assert_required_module_directories_are_created(module_root, module_meta_root)
+        self._assert_required_module_files_are_created_and_substitution_is_done(module_root, module_meta_root)
+
+        shutil.rmtree("test_modules")
+
+    def _assert_required_module_directories_are_created(self, module_root, module_meta_root):
+        expected_created_directories = [
+            module_root,
+            module_meta_root,
+        ]
+        for expected_dir in expected_created_directories:
+            assert os.path.isdir(expected_dir)
+
+    def _assert_required_module_files_are_created_and_substitution_is_done(self, module_root, module_meta_root):
+        expected_created_template_files = [
+            os.path.join(module_root, "module.yml"),
+            os.path.join(module_meta_root, "config.yml"),
+            os.path.join(module_meta_root, "docs.asciidoc"),
+            os.path.join(module_meta_root, "fields.yml"),
+        ]
+        for template_file in expected_created_template_files:
+            assert os.path.isfile(template_file)
+            assert '{module}' not in open(template_file)
+
+    def test_generate_fileset(self):
+        """
+        Test filebeat generate fileset my_module my_fileset generates a new fileset
+        """
+
+        self._create_clean_test_modules()
+        exit_code = self.run_beat(
+            extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+        assert exit_code == 0
+
+        exit_code = self.run_beat(
+            extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+        assert exit_code == 0
+
+        fileset_root = os.path.join("test_modules", "module", "my_module", "my_fileset")
+        self._assert_required_fileset_directories_are_created(fileset_root)
+        self._assert_required_fileset_files_are_created_and_substitution_is_done(fileset_root)
+
+        shutil.rmtree("test_modules")
+
+    def _assert_required_fileset_directories_are_created(self, fileset_root):
+        expected_created_directories = [
+            fileset_root,
+            os.path.join(fileset_root, "config"),
+            os.path.join(fileset_root, "ingest"),
+            os.path.join(fileset_root, "_meta"),
+            os.path.join(fileset_root, "test"),
+        ]
+        for expected_dir in expected_created_directories:
+            assert os.path.isdir(expected_dir)
+
+    def _assert_required_fileset_files_are_created_and_substitution_is_done(self, fileset_root):
+        expected_created_template_files = [
+            os.path.join(fileset_root, "config", "my_fileset.yml"),
+            os.path.join(fileset_root, "ingest", "pipeline.json"),
+            os.path.join(fileset_root, "manifest.yml"),
+        ]
+        for template_file in expected_created_template_files:
+            assert os.path.isfile(template_file)
+            assert '{fileset}' not in open(template_file)
+
+    def test_generate_fields_yml(self):
+        """
+        Test filebeat generate fields my_module my_fileset generates a new fields.yml for my_module/my_fileset
+        """
+
+        self._create_clean_test_modules()
+        exit_code = self.run_beat(
+            extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+        assert exit_code == 0
+
+        exit_code = self.run_beat(
+            extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+        assert exit_code == 0
+
+        test_pipeline_path = os.path.join(self.beat_path, "tests", "system", "input", "my-module-pipeline.json")
+        fileset_pipeline = os.path.join("test_modules", "module",
+                                        "my_module", "my_fileset", "ingest", "pipeline.json")
+
+        print(os.path.isdir("test_modules"))
+        print(os.path.isdir(os.path.join("test_modules", "module")))
+        print(os.path.isdir(os.path.join("test_modules", "module", "my_module")))
+        print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset")))
+        print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest")))
+        print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest")))
+        print(os.path.exists(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest", "pipeline.json")))
+        print(os.path.exists(fileset_pipeline))
+        print(os.path.exists(test_pipeline_path))
+        shutil.copyfile(test_pipeline_path, fileset_pipeline)
+
+        print(fileset_pipeline)
+        print(os.path.abspath(fileset_pipeline))
+        exit_code = self.run_beat(
+            extra_args=["generate", "fields", "my_module", "my_fileset", "-es-beats", "test_modules", "-without-documentation"])
+        assert exit_code == 0
+
+        fields_yml_path = os.path.join("test_modules", "module", "my_module", "my_fileset", "_meta", "fields.yml")
+        assert os.path.isfile(fields_yml_path)
+
+        shutil.rmtree("test_modules")
+
+    def _create_clean_test_modules(self):
+        if os.path.isdir("test_modules"):
+            shutil.rmtree("test_modules")
+        os.mkdir("test_modules")