diff --git a/README.md b/README.md index f526203bd..e6bbe418b 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,12 @@ These benchmarks allow you to benchmark any Ingest Node Pipelines defined by you For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md). +#### System Benchmarks + +These benchmarks allow you to benchmark an integration end to end. + +For details on how to configure system benchmarks for a package, review the [HOWTO guide](./docs/howto/system_benchmarking.md). + ### `elastic-package benchmark generate-corpus` _Context: package_ @@ -116,6 +122,12 @@ _Context: package_ Run pipeline benchmarks for the package. +### `elastic-package benchmark system` + +_Context: package_ + +Run system benchmarks for the package. + ### `elastic-package build` _Context: package_ diff --git a/cmd/benchmark.go b/cmd/benchmark.go index f2bcd920e..ce13fe53c 100644 --- a/cmd/benchmark.go +++ b/cmd/benchmark.go @@ -5,20 +5,23 @@ package cmd import ( + "errors" "fmt" "strings" + "time" "github.com/dustin/go-humanize" "github.com/elastic/elastic-package/internal/corpusgenerator" + "github.com/elastic/elastic-package/internal/kibana" - "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/benchrunner" - "github.com/elastic/elastic-package/internal/benchrunner/reporters/formats" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/benchrunner/reporters/outputs" - _ "github.com/elastic/elastic-package/internal/benchrunner/runners" // register all benchmark runners + "github.com/elastic/elastic-package/internal/benchrunner/runners/pipeline" + "github.com/elastic/elastic-package/internal/benchrunner/runners/system" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -39,182 +42,262 @@ const benchLongDescription = `Use this command to run benchmarks on a package. C These benchmarks allow you to benchmark any Ingest Node Pipelines defined by your packages. -For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md).` +For details on how to configure pipeline benchmarks for a package, review the [HOWTO guide](./docs/howto/pipeline_benchmarking.md). -func setupBenchmarkCommand() *cobraext.Command { - var benchTypeCmdActions []cobraext.CommandAction +#### System Benchmarks + +These benchmarks allow you to benchmark an integration end to end. +For details on how to configure system benchmarks for a package, review the [HOWTO guide](./docs/howto/system_benchmarking.md).` + +func setupBenchmarkCommand() *cobraext.Command { cmd := &cobra.Command{ Use: "benchmark", Short: "Run benchmarks for the package", Long: benchLongDescription, - RunE: func(cmd *cobra.Command, args []string) error { - cmd.Println("Run benchmarks for the package") + } + + pipelineCmd := getPipelineCommand() + cmd.AddCommand(pipelineCmd) - if len(args) > 0 { - return fmt.Errorf("unsupported benchmark type: %s", args[0]) - } + systemCmd := getSystemCommand() + cmd.AddCommand(systemCmd) - return cobraext.ComposeCommandActions(cmd, args, benchTypeCmdActions...) - }} + generateCorpusCmd := getGenerateCorpusCommand() + cmd.AddCommand(generateCorpusCmd) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func getPipelineCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "pipeline", + Short: "Run pipeline benchmarks", + Long: "Run pipeline benchmarks for the package", + RunE: pipelineCommandAction, + } cmd.Flags().BoolP(cobraext.FailOnMissingFlagName, "m", false, cobraext.FailOnMissingFlagDescription) - cmd.Flags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) + cmd.Flags().StringP(cobraext.ReportFormatFlagName, "", string(pipeline.ReportFormatHuman), cobraext.ReportFormatFlagDescription) cmd.Flags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) + cmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription) cmd.Flags().BoolP(cobraext.BenchWithTestSamplesFlagName, "", true, cobraext.BenchWithTestSamplesFlagDescription) cmd.Flags().IntP(cobraext.BenchNumTopProcsFlagName, "", 10, cobraext.BenchNumTopProcsFlagDescription) - cmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "", nil, cobraext.DataStreamsFlagDescription) - for benchType, runner := range benchrunner.BenchRunners() { - action := benchTypeCommandActionFactory(runner) - benchTypeCmdActions = append(benchTypeCmdActions, action) + return cmd +} - benchTypeCmd := &cobra.Command{ - Use: string(benchType), - Short: fmt.Sprintf("Run %s benchmarks", runner.String()), - Long: fmt.Sprintf("Run %s benchmarks for the package.", runner.String()), - RunE: action, - } +func pipelineCommandAction(cmd *cobra.Command, args []string) error { + cmd.Println("Run pipeline benchmarks for the package") - benchTypeCmd.Flags().BoolP(cobraext.FailOnMissingFlagName, "m", false, cobraext.FailOnMissingFlagDescription) - benchTypeCmd.Flags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) - benchTypeCmd.Flags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) - benchTypeCmd.Flags().BoolP(cobraext.BenchWithTestSamplesFlagName, "", true, cobraext.BenchWithTestSamplesFlagDescription) - benchTypeCmd.Flags().IntP(cobraext.BenchNumTopProcsFlagName, "", 10, cobraext.BenchNumTopProcsFlagDescription) - benchTypeCmd.Flags().StringSliceP(cobraext.DataStreamsFlagName, "d", nil, cobraext.DataStreamsFlagDescription) + failOnMissing, err := cmd.Flags().GetBool(cobraext.FailOnMissingFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName) + } - cmd.AddCommand(benchTypeCmd) + reportFormat, err := cmd.Flags().GetString(cobraext.ReportFormatFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.ReportFormatFlagName) } - generateCorpusCmd := getGenerateCorpusCommand() - cmd.AddCommand(generateCorpusCmd) + reportOutput, err := cmd.Flags().GetString(cobraext.ReportOutputFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.ReportOutputFlagName) + } - return cobraext.NewCommand(cmd, cobraext.ContextPackage) -} + useTestSamples, err := cmd.Flags().GetBool(cobraext.BenchWithTestSamplesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchWithTestSamplesFlagName) + } -func benchTypeCommandActionFactory(runner benchrunner.BenchRunner) cobraext.CommandAction { - benchType := runner.Type() - return func(cmd *cobra.Command, args []string) error { - cmd.Printf("Run %s benchmarks for the package\n", benchType) + numTopProcs, err := cmd.Flags().GetInt(cobraext.BenchNumTopProcsFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchNumTopProcsFlagName) + } - failOnMissing, err := cmd.Flags().GetBool(cobraext.FailOnMissingFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FailOnMissingFlagName) - } + packageRootPath, found, err := packages.FindPackageRoot() + if !found { + return errors.New("package root not found") + } + if err != nil { + return fmt.Errorf("locating package root failed: %w", err) + } - reportFormat, err := cmd.Flags().GetString(cobraext.ReportFormatFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.ReportFormatFlagName) - } + dataStreams, err := cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) + } - reportOutput, err := cmd.Flags().GetString(cobraext.ReportOutputFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.ReportOutputFlagName) - } + if len(dataStreams) > 0 { + common.TrimStringSlice(dataStreams) - useTestSamples, err := cmd.Flags().GetBool(cobraext.BenchWithTestSamplesFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.BenchWithTestSamplesFlagName) + if err := validateDataStreamsFlag(packageRootPath, dataStreams); err != nil { + return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) } + } - numTopProcs, err := cmd.Flags().GetInt(cobraext.BenchNumTopProcsFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.BenchNumTopProcsFlagName) - } + signal.Enable() - packageRootPath, found, err := packages.FindPackageRoot() - if !found { - return errors.New("package root not found") - } - if err != nil { - return errors.Wrap(err, "locating package root failed") - } + benchFolders, err := pipeline.FindBenchmarkFolders(packageRootPath, dataStreams) + if err != nil { + return fmt.Errorf("unable to determine benchmark folder paths: %w", err) + } - dataStreams, err := cmd.Flags().GetStringSlice(cobraext.DataStreamsFlagName) + if useTestSamples { + testFolders, err := testrunner.FindTestFolders(packageRootPath, dataStreams, testrunner.TestType(pipeline.BenchType)) if err != nil { - return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) + return fmt.Errorf("unable to determine test folder paths: %w", err) } + benchFolders = append(benchFolders, testFolders...) + } + if failOnMissing && len(benchFolders) == 0 { if len(dataStreams) > 0 { - common.TrimStringSlice(dataStreams) - - if err := validateDataStreamsFlag(packageRootPath, dataStreams); err != nil { - return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) - } + return fmt.Errorf("no pipeline benchmarks found for %s data stream(s)", strings.Join(dataStreams, ",")) } + return errors.New("no pipeline benchmarks found") + } + + esClient, err := elasticsearch.NewClient() + if err != nil { + return fmt.Errorf("can't create Elasticsearch client: %w", err) + } + err = esClient.CheckHealth(cmd.Context()) + if err != nil { + return err + } - signal.Enable() + var results []reporters.Reportable + for idx, folder := range benchFolders { + opts := pipeline.NewOptions( + pipeline.WithBenchmarkName(fmt.Sprintf("%s-%d", folder.Package, idx+1)), + pipeline.WithFolder(folder), + pipeline.WithPackageRootPath(packageRootPath), + pipeline.WithESAPI(esClient.API), + pipeline.WithNumTopProcs(numTopProcs), + pipeline.WithFormat(reportFormat), + ) + runner := pipeline.NewPipelineBenchmark(opts) + + r, err := benchrunner.Run(runner) - benchFolders, err := benchrunner.FindBenchmarkFolders(packageRootPath, dataStreams, benchType) if err != nil { - return errors.Wrap(err, "unable to determine benchmark folder paths") + return fmt.Errorf("error running package pipeline benchmarks: %w", err) } - if useTestSamples { - testFolders, err := testrunner.FindTestFolders(packageRootPath, dataStreams, testrunner.TestType(benchType)) - if err != nil { - return errors.Wrap(err, "unable to determine test folder paths") - } - benchFolders = append(benchFolders, testFolders...) - } + results = append(results, r) + } - if failOnMissing && len(benchFolders) == 0 { - if len(dataStreams) > 0 { - return fmt.Errorf("no %s benchmarks found for %s data stream(s)", benchType, strings.Join(dataStreams, ",")) - } - return fmt.Errorf("no %s benchmarks found", benchType) - } + if err != nil { + return fmt.Errorf("error running package pipeline benchmarks: %w", err) + } - esClient, err := elasticsearch.NewClient() - if err != nil { - return errors.Wrap(err, "can't create Elasticsearch client") - } - err = esClient.CheckHealth(cmd.Context()) - if err != nil { - return err + for _, report := range results { + if err := reporters.WriteReportable(reporters.Output(reportOutput), report); err != nil { + return fmt.Errorf("error writing benchmark report: %w", err) } + } - var results []*benchrunner.Result - for _, folder := range benchFolders { - r, err := benchrunner.Run(benchType, benchrunner.BenchOptions{ - Folder: folder, - PackageRootPath: packageRootPath, - API: esClient.API, - NumTopProcs: numTopProcs, - }) + return nil +} - if err != nil { - return errors.Wrapf(err, "error running package %s benchmarks", benchType) - } +func getSystemCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "system", + Short: "Run system benchmarks", + Long: "Run system benchmarks for the package", + RunE: systemCommandAction, + } - results = append(results, r) - } + cmd.Flags().StringP(cobraext.BenchNameFlagName, "", "", cobraext.BenchNameFlagDescription) + cmd.Flags().BoolP(cobraext.BenchReindexToMetricstoreFlagName, "", false, cobraext.BenchReindexToMetricstoreFlagDescription) + cmd.Flags().DurationP(cobraext.BenchMetricsIntervalFlagName, "", time.Second, cobraext.BenchMetricsIntervalFlagDescription) + cmd.Flags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) - format := benchrunner.BenchReportFormat(reportFormat) - benchReports, err := benchrunner.FormatReport(format, results) - if err != nil { - return errors.Wrap(err, "error formatting benchmark report") - } + return cmd +} - m, err := packages.ReadPackageManifestFromPackageRoot(packageRootPath) - if err != nil { - return errors.Wrapf(err, "reading package manifest failed (path: %s)", packageRootPath) - } +func systemCommandAction(cmd *cobra.Command, args []string) error { + cmd.Println("Run system benchmarks for the package") - for idx, report := range benchReports { - if err := benchrunner.WriteReport(fmt.Sprintf("%s-%d", m.Name, idx+1), benchrunner.BenchReportOutput(reportOutput), report, format); err != nil { - return errors.Wrap(err, "error writing benchmark report") - } - } + benchName, err := cmd.Flags().GetString(cobraext.BenchNameFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchNameFlagName) + } - // Check if there is any error or failure reported - for _, r := range results { - if r.ErrorMsg != "" { - return fmt.Errorf("one or more benchmarks failed: %v", r.ErrorMsg) - } - } - return nil + deferCleanup, err := cmd.Flags().GetDuration(cobraext.DeferCleanupFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.DeferCleanupFlagName) } + + metricsInterval, err := cmd.Flags().GetDuration(cobraext.BenchMetricsIntervalFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchMetricsIntervalFlagName) + } + + dataReindex, err := cmd.Flags().GetBool(cobraext.BenchReindexToMetricstoreFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.BenchReindexToMetricstoreFlagName) + } + + packageRootPath, found, err := packages.FindPackageRoot() + if !found { + return errors.New("package root not found") + } + if err != nil { + return fmt.Errorf("locating package root failed: %w", err) + } + + signal.Enable() + + esClient, err := elasticsearch.NewClient() + if err != nil { + return fmt.Errorf("can't create Elasticsearch client: %w", err) + } + err = esClient.CheckHealth(cmd.Context()) + if err != nil { + return err + } + + kc, err := kibana.NewClient() + if err != nil { + return fmt.Errorf("can't create Kibana client: %w", err) + } + + opts := system.NewOptions( + system.WithBenchmarkName(benchName), + system.WithDeferCleanup(deferCleanup), + system.WithMetricsInterval(metricsInterval), + system.WithDataReindexing(dataReindex), + system.WithPackageRootPath(packageRootPath), + system.WithESAPI(esClient.API), + system.WithKibanaClient(kc), + ) + runner := system.NewSystemBenchmark(opts) + + r, err := benchrunner.Run(runner) + if err != nil { + return fmt.Errorf("error running package system benchmarks: %w", err) + } + + multiReport, ok := r.(reporters.MultiReportable) + if !ok { + return fmt.Errorf("system benchmark is expected to return multiple reports") + } + + // human report will always be the first + human := multiReport.Split()[0] + if err := reporters.WriteReportable(reporters.Output(outputs.ReportOutputSTDOUT), human); err != nil { + return fmt.Errorf("error writing benchmark report: %w", err) + } + + // file report will always be the second + file := multiReport.Split()[1] + if err := reporters.WriteReportable(reporters.Output(outputs.ReportOutputFile), file); err != nil { + return fmt.Errorf("error writing benchmark report: %w", err) + } + + return nil } func getGenerateCorpusCommand() *cobra.Command { @@ -272,14 +355,14 @@ func generateDataStreamCorpusCommandAction(cmd *cobra.Command, _ []string) error genLibClient := corpusgenerator.NewClient(commit) generator, err := corpusgenerator.NewGenerator(genLibClient, packageName, dataSetName, totSizeInBytes) if err != nil { - return errors.Wrap(err, "can't generate benchmarks data corpus for data stream") + return fmt.Errorf("can't generate benchmarks data corpus for data stream: %w", err) } // TODO: we need a way to extract the type from the package and dataset, currently hardcode to `metrics` dataStream := fmt.Sprintf("metrics-%s.%s-default", packageName, dataSetName) err = corpusgenerator.RunGenerator(generator, dataStream, rallyTrackOutputDir) if err != nil { - return errors.Wrap(err, "can't generate benchmarks data corpus for data stream") + return fmt.Errorf("can't generate benchmarks data corpus for data stream: %w", err) } return nil diff --git a/docs/howto/system_benchmarking.md b/docs/howto/system_benchmarking.md new file mode 100644 index 000000000..2db242ad5 --- /dev/null +++ b/docs/howto/system_benchmarking.md @@ -0,0 +1,245 @@ +# HOWTO: Writing system benchmarks for a package + +## Introduction +Elastic Packages are comprised of data streams. A system benchmark exercises the end-to-end flow of data for a package's data stream — from ingesting data from the package's integration service all the way to indexing it into an Elasticsearch data stream, and retrieves performance metrics from the Elasticsearch nodes. + +## Conceptual process + +Conceptually, running a system benchmark involves the following steps: + +1. Deploy the Elastic Stack, including Elasticsearch, Kibana, and the Elastic Agent(s). This step takes time so it should typically be done once as a pre-requisite to running a system benchmark scenario. +1. Enroll the Elastic Agent(s) with Fleet (running in the Kibana instance). This step also can be done once, as a pre-requisite. +1. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package's integration service. +1. Create a benchmark policy that configures a single data stream for a single package. +1. Assign the policy to the enrolled Agent(s). +1. Metrics collections from the cluster starts. (**TODO**: record metrics from all Elastic Agents involved using the `system` integration.) +1. **TODO**: Send the collected metrics to the ES Metricstore if set. +1. Generate data if configured (it uses the [corpus-generator-rool](https://github.com/elastic/elastic-integration-corpus-generator-tool)) +1. Wait a reasonable amount of time for the Agent to collect data from the + integration service and index it into the correct Elasticsearch data stream. + This time can be pre-defined with the `benchmark_time`. In case this setting is not set + the benchmark will continue until the number of documents is not changed in the data stream. +1. Metrics collection ends and a summary report is created. +1. Delete test artifacts and tear down the instance of the package's integration service. +1. **TODO**: Optionally reindex all ingested data into the ES Metricstore for further analysis. +1. **TODO**: Optionally compare results against another benchmark run. + +## Defining a system benchmark scenario + +System benchmarks are defined at the package level. + +Optionally system benchmarks can define a configuration for deploying a package's integration service. We must define it on the package level: + +``` +/ + _dev/ + benchmark/ + system/ + deploy/ + / + +``` + +`` - a name of the supported service deployer: +* `docker` - Docker Compose + +**TODO**: support other service deployers + +### Docker Compose service deployer + +When using the Docker Compose service deployer, the `` must include a `docker-compose.yml` file. +The `docker-compose.yml` file defines the integration service(s) for the package. If your package has a logs data stream, the log files from your package's integration service must be written to a volume. + +`elastic-package` will remove orphan volumes associated to the started services +when they are stopped. Docker compose may not be able to find volumes defined in +the Dockerfile for this cleanup. In these cases, override the volume definition. + +### Benchmark scenario definition + +Next, we must define at least one configuration for the package that we +want to benchmark. There can be multiple scenarios defined for the same package. + +``` +/ + _dev/ + benchmark/ + system/ + .yml +``` + +The `.yml` files allow you to define various settings for the benchmark scenario +along with values for package and data stream-level variables. These are the available configuration options for system benchmarks. + +| Option | Type | Required | Description | +|---|---|---|---| +| package | string | | The name of the package. If omitted will pick the current package, this is to allow for future definition of benchmarks outside of the packages folders. | +| description | string | | A description for the scenario. | +| version | string | | The version of the package to benchmark. If omitted will pick the current version of the package. | +| input | string | yes | Input type to test (e.g. logfile, httpjson, etc). Defaults to the input used by the first stream in the data stream manifest. | +| vars | dictionary | | Package level variables to set (i.e. declared in `$package_root/manifest.yml`). If not specified the defaults from the manifest are used. | +| data_stream.name | string | yes | The data stream to benchmark. | +| data_stream.vars | dictionary | | Data stream level variables to set (i.e. declared in `package_root/data_stream/$data_stream/manifest.yml`). If not specified the defaults from the manifest are used. | +| warmup_time_period | duration | | Warmup time period. All data prior to this period will be ignored in the benchmark results. | +| benchmark_time_period | duration | | Amount of time the benchmark needs to run for. If set the benchmark will stop after this period even though more data is still pending to be ingested. | +| wait_for_data_timeout | duration | | Amount of time to wait for data to be present in Elasticsearch. Defaults to 10m. | +| corpora.generator.size | string | | String describing the amount of data to generate. Example: `20MiB` | +| corpora.generator.template.raw | string | | Raw template for the corpus generator. | +| corpora.generator.template.path | string | | Path to the template for the corpus generator. If a `path` is defined, it will override any `raw` template definition. | +| corpora.generator.template.type | string | | Type of the template for the corpus generator. Default `placeholder`. | +| corpora.generator.config.raw | dictionary | | Raw config for the corpus generator. | +| corpora.generator.config.path | string | | Path to the config for the corpus generator. If a `path` is defined, it will override any `raw` config definition. | +| corpora.generator.fields.raw | dictionary | | Raw fields for the corpus generator. | +| corpora.generator.fields.path | string | | Path to the fields for the corpus generator. If a `path` is defined, it will override any `raw` fields definition. | +| corpora.input_service.name | string | | Name of the input service to use (defined in the `deploy` folder). | +| corpora.input_service.signal | string | | Signal to send to the input service once the benchmark is ready to start. | + +Example: + +`logs-benchmark.yml` +```yaml +--- +description: Benchmark 100MiB of data ingested +input: filestream +vars: ~ +data_stream.name: test +data_stream.vars.paths: + - "{{SERVICE_LOGS_DIR}}/corpus-*" +warmup_time_period: 10s +corpora.generator.size: 100MiB +corpora.generator.template.path: ./logs-benchmark/template.log +corpora.generator.config.path: ./logs-benchmark/config.yml +corpora.generator.fields.path: ./logs-benchmark/fields.yml +``` + +The top-level `vars` field corresponds to package-level variables defined in the +package's `manifest.yml` file. In the above example we don't override +any of these package-level variables, so their default values, as specified in +the package's `manifest.yml` file are used. + +The `data_stream.vars` field corresponds to data stream-level variables for the +current data stream (`test` in the above example). In the above example +we override the `paths` variable. All other variables are populated with their +default values, as specified in the data stream's `manifest.yml` file. + +Notice the use of the `{{SERVICE_LOGS_DIR}}` placeholder. This also corresponds to +the `${SERVICE_LOGS_DIR}` variable that can be used the `docker-compose.yml` files. + +**The generator puts the generated data inside `{{SERVICE_LOGS_DIR}}` so they are available both +in `docker-compose.yml` and inside the Elastic Agent that is provisioned by the `stack` command.** + + +#### Placeholders + +The `SERVICE_LOGS_DIR` placeholder is not the only one available for use in a data stream's `.yml` file. The complete list of available placeholders is shown below. + +| Placeholder name | Data type | Description | +| --- | --- | --- | +| `Hostname`| string | Addressable host name of the integration service. | +| `Ports` | []int | Array of addressable ports the integration service is listening on. | +| `Port` | int | Alias for `Ports[0]`. Provided as a convenience. | +| `Logs.Folder.Agent` | string | Path to integration service's logs folder, as addressable by the Agent. | +| `SERVICE_LOGS_DIR` | string | Alias for `Logs.Folder.Agent`. Provided as a convenience. | + +Placeholders used in the `.yml` must be enclosed in `{{` and `}}` delimiters, per Go syntax. + +## Running a system benchmark + +Once the configuration is defined as described in the previous section, you are ready to run system benchmarks for a package. + +First you must deploy the Elastic Stack. + +``` +elastic-package stack up -d +``` + +For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, you must set environment variables needed for further `elastic-package` commands. + +``` +$(elastic-package stack shellinit) +``` + +Next, you must invoke the system benchmark runner. + +``` +elastic-package benchmark system --benchmark logs-benchmark -v +# ... debug output +--- Benchmark results for package: system_benchmarks - START --- +╭─────────────────────────────────────────────────────╮ +│ info │ +├──────────────┬──────────────────────────────────────┤ +│ benchmark │ logs-benchmark │ +│ description │ Benchmark 100MiB of data ingested │ +│ run ID │ d2960c04-0028-42c9-bafc-35e599563cb1 │ +│ package │ system_benchmarks │ +│ start ts (s) │ 1682320355 │ +│ end ts (s) │ 1682320355 │ +│ duration │ 2m3s │ +╰──────────────┴──────────────────────────────────────╯ +╭───────────────────────────────────────────────────────────────────────╮ +│ parameters │ +├─────────────────────────────────┬─────────────────────────────────────┤ +│ package version │ 999.999.999 │ +│ input │ filestream │ +│ data_stream.name │ test │ +│ data_stream.vars.paths │ [/tmp/service_logs/corpus-*] │ +│ warmup time period │ 10s │ +│ benchmark time period │ 0s │ +│ wait for data timeout │ 0s │ +│ corpora.generator.size │ 100MiB │ +│ corpora.generator.template.path │ ./logs-benchmark/template.log │ +│ corpora.generator.template.raw │ │ +│ corpora.generator.template.type │ │ +│ corpora.generator.config.path │ ./logs-benchmark/config.yml │ +│ corpora.generator.config.raw │ map[] │ +│ corpora.generator.fields.path │ ./logs-benchmark/fields.yml │ +│ corpora.generator.fields.raw │ map[] │ +╰─────────────────────────────────┴─────────────────────────────────────╯ +╭───────────────────────╮ +│ cluster info │ +├───────┬───────────────┤ +│ name │ elasticsearch │ +│ nodes │ 1 │ +╰───────┴───────────────╯ +╭─────────────────────────────────────────────────────────────╮ +│ data stream stats │ +├────────────────────────────┬────────────────────────────────┤ +│ data stream │ logs-system_benchmarks.test-ep │ +│ approx total docs ingested │ 410127 │ +│ backing indices │ 1 │ +│ store size bytes │ 136310570 │ +│ maximum ts (ms) │ 1682320467448 │ +╰────────────────────────────┴────────────────────────────────╯ +╭───────────────────────────────────────╮ +│ disk usage for index .ds-logs-system_ │ +│ benchmarks.test-ep-2023.04.22-000001 │ +│ (for all fields) │ +├──────────────────────────────┬────────┤ +│ total │ 99.8mb │ +│ inverted_index.total │ 31.3mb │ +│ inverted_index.stored_fields │ 35.5mb │ +│ inverted_index.doc_values │ 30mb │ +│ inverted_index.points │ 2.8mb │ +│ inverted_index.norms │ 0b │ +│ inverted_index.term_vectors │ 0b │ +│ inverted_index.knn_vectors │ 0b │ +╰──────────────────────────────┴────────╯ +╭───────────────────────────────────────────────────────────────────────────────────────────╮ +│ pipeline logs-system_benchmarks.test-999.999.999 stats in node Qa9ujRVfQuWhqEESdt6xnw │ +├───────────────────────────────────────────────┬───────────────────────────────────────────┤ +│ grok () │ Count: 407819 | Failed: 0 | Time: 16.615s │ +│ user_agent () │ Count: 407819 | Failed: 0 | Time: 768ms │ +│ pipeline (logs-system_benchmarks.test@custom) │ Count: 407819 | Failed: 0 | Time: 59ms │ +╰───────────────────────────────────────────────┴───────────────────────────────────────────╯ + +--- Benchmark results for package: system_benchmarks - END --- +Done +``` + +Finally, when you are done running the benchmark, bring down the Elastic Stack. + +``` +elastic-package stack down +``` + diff --git a/go.mod b/go.mod index 1cf01f857..a52cd9b4e 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/go-github/v32 v32.1.0 github.com/google/go-querystring v1.1.0 + github.com/google/uuid v1.3.0 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/magefile/mage v1.15.0 github.com/mholt/archiver/v3 v3.5.1 @@ -86,7 +87,6 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect diff --git a/internal/benchrunner/benchmark.go b/internal/benchrunner/benchmark.go deleted file mode 100644 index 1fc67757f..000000000 --- a/internal/benchrunner/benchmark.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package benchrunner - -import ( - "fmt" -) - -type BenchmarkResult struct { - // XMLName is a zero-length field used as an annotation for XML marshaling. - XMLName struct{} `xml:"group" json:"-"` - // Type of benchmark - Type string `xml:"type" json:"type"` - // Package of the benchmark - Package string `xml:"package" json:"package"` - // DataStream of the benchmark - DataStream string `xml:"data_stream" json:"data_stream"` - // Description of the benchmark run. - Description string `xml:"description,omitempty" json:"description,omitempty"` - // Parameters used for this benchmark. - Parameters []BenchmarkValue `xml:"parameters,omitempty" json:"parameters,omitempty"` - // Tests holds the results for the benchmark. - Tests []BenchmarkTest `xml:"test" json:"test"` -} - -// BenchmarkTest models a particular test performed during a benchmark. -type BenchmarkTest struct { - // Name of this test. - Name string `xml:"name" json:"name"` - // Detailed benchmark tests will be printed to the output but not - // included in file reports. - Detailed bool `xml:"-" json:"-"` - // Description of this test. - Description string `xml:"description,omitempty" json:"description,omitempty"` - // Parameters for this test. - Parameters []BenchmarkValue `xml:"parameters,omitempty" json:"parameters,omitempty"` - // Results of the test. - Results []BenchmarkValue `xml:"result" json:"result"` -} - -// BenchmarkValue represents a value (result or parameter) -// with an optional associated unit. -type BenchmarkValue struct { - // Name of the value. - Name string `xml:"name" json:"name"` - // Description of the value. - Description string `xml:"description,omitempty" json:"description,omitempty"` - // Unit used for this value. - Unit string `xml:"unit,omitempty" json:"unit,omitempty"` - // Value is of any type, usually string or numeric. - Value interface{} `xml:"value,omitempty" json:"value,omitempty"` -} - -// String returns a BenchmarkValue's value nicely-formatted. -func (p BenchmarkValue) String() (r string) { - if str, ok := p.Value.(fmt.Stringer); ok { - return str.String() - } - if float, ok := p.Value.(float64); ok { - r = fmt.Sprintf("%.02f", float) - } else { - r = fmt.Sprintf("%v", p.Value) - } - if p.Unit != "" { - r += p.Unit - } - return r -} diff --git a/internal/benchrunner/benchrunner.go b/internal/benchrunner/benchrunner.go deleted file mode 100644 index 5e023aaf6..000000000 --- a/internal/benchrunner/benchrunner.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package benchrunner - -import ( - "fmt" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/elastic/elastic-package/internal/elasticsearch" - "github.com/elastic/elastic-package/internal/testrunner" -) - -// BenchType represents the various supported benchmark types -type BenchType string - -// BenchOptions contains benchmark runner options. -type BenchOptions struct { - Folder testrunner.TestFolder - PackageRootPath string - API *elasticsearch.API - NumTopProcs int -} - -// BenchRunner is the interface all benchmark runners must implement. -type BenchRunner interface { - // Type returns the benchmark runner's type. - Type() BenchType - - // String returns the human-friendly name of the benchmark runner. - String() string - - // Run executes the benchmark runner. - Run(BenchOptions) (*Result, error) - - // TearDown cleans up any benchmark runner resources. It must be called - // after the benchmark runner has finished executing. - TearDown() error -} - -var runners = map[BenchType]BenchRunner{} - -// Result contains a single benchmark's results -type Result struct { - // Package to which this benchmark result belongs. - Package string - - // BenchType indicates the type of benchmark. - BenchType BenchType - - // Data stream to which this benchmark result belongs. - DataStream string - - // Time elapsed from running a benchmark case to arriving at its result. - TimeElapsed time.Duration - - // If there was an error while running the benchmark case, description - // of the error. An error is when the benchmark cannot complete execution due - // to an unexpected runtime error in the benchmark execution. - ErrorMsg string - - // Benchmark results. - Benchmark *BenchmarkResult -} - -// ResultComposer wraps a Result and provides convenience methods for -// manipulating this Result. -type ResultComposer struct { - Result - StartTime time.Time -} - -// NewResultComposer returns a new ResultComposer with the StartTime -// initialized to now. -func NewResultComposer(tr Result) *ResultComposer { - return &ResultComposer{ - Result: tr, - StartTime: time.Now(), - } -} - -// WithError sets an error on the benchmark result wrapped by ResultComposer. -func (rc *ResultComposer) WithError(err error) ([]Result, error) { - rc.TimeElapsed = time.Since(rc.StartTime) - if err == nil { - return []Result{rc.Result}, nil - } - - rc.ErrorMsg += err.Error() - return []Result{rc.Result}, err -} - -// WithSuccess marks the benchmark result wrapped by ResultComposer as successful. -func (rc *ResultComposer) WithSuccess() ([]Result, error) { - return rc.WithError(nil) -} - -// FindBenchmarkFolders finds benchmark folders for the given package and, optionally, benchmark type and data streams -func FindBenchmarkFolders(packageRootPath string, dataStreams []string, benchType BenchType) ([]testrunner.TestFolder, error) { - // Expected folder structure: - // / - // data_stream/ - // / - // _dev/ - // benchmark/ - // / - - benchTypeGlob := "*" - if benchType != "" { - benchTypeGlob = string(benchType) - } - - var paths []string - if len(dataStreams) > 0 { - sort.Strings(dataStreams) - for _, dataStream := range dataStreams { - p, err := findBenchFolderPaths(packageRootPath, dataStream, benchTypeGlob) - if err != nil { - return nil, err - } - - paths = append(paths, p...) - } - } else { - p, err := findBenchFolderPaths(packageRootPath, "*", benchTypeGlob) - if err != nil { - return nil, err - } - - paths = p - } - - sort.Strings(dataStreams) - for _, dataStream := range dataStreams { - p, err := findBenchFolderPaths(packageRootPath, dataStream, benchTypeGlob) - if err != nil { - return nil, err - } - - paths = append(paths, p...) - } - - folders := make([]testrunner.TestFolder, len(paths)) - _, pkg := filepath.Split(packageRootPath) - for idx, p := range paths { - relP := strings.TrimPrefix(p, packageRootPath) - parts := strings.Split(relP, string(filepath.Separator)) - dataStream := parts[2] - - folder := testrunner.TestFolder{ - Path: p, - Package: pkg, - DataStream: dataStream, - } - - folders[idx] = folder - } - - return folders, nil -} - -// RegisterRunner method registers the benchmark runner. -func RegisterRunner(runner BenchRunner) { - runners[runner.Type()] = runner -} - -// Run method delegates execution to the registered benchmark runner, based on the benchmark type. -func Run(benchType BenchType, options BenchOptions) (*Result, error) { - runner, defined := runners[benchType] - if !defined { - return nil, fmt.Errorf("unregistered runner benchmark: %s", benchType) - } - - result, err := runner.Run(options) - tdErr := runner.TearDown() - if err != nil { - return nil, fmt.Errorf("could not complete benchmark run: %w", err) - } - if tdErr != nil { - return result, fmt.Errorf("could not teardown benchmark runner: %w", err) - } - return result, nil -} - -// BenchRunners returns registered benchmark runners. -func BenchRunners() map[BenchType]BenchRunner { - return runners -} - -// findBenchFoldersPaths can only be called for benchmark runners that require benchmarks to be defined -// at the data stream level. -func findBenchFolderPaths(packageRootPath, dataStreamGlob, benchTypeGlob string) ([]string, error) { - benchFoldersGlob := filepath.Join(packageRootPath, "data_stream", dataStreamGlob, "_dev", "benchmark", benchTypeGlob) - paths, err := filepath.Glob(benchFoldersGlob) - if err != nil { - return nil, fmt.Errorf("error finding benchmark folders: %w", err) - } - return paths, err -} diff --git a/internal/benchrunner/report_format.go b/internal/benchrunner/report_format.go deleted file mode 100644 index 816dcaa9f..000000000 --- a/internal/benchrunner/report_format.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package benchrunner - -import "fmt" - -// BenchReportFormat represents a benchmark report format -type BenchReportFormat string - -// ReportFormatFunc defines the report formatter function. -type ReportFormatFunc func(results []*Result) ([]string, error) - -var reportFormatters = map[BenchReportFormat]ReportFormatFunc{} - -// RegisterReporterFormat registers a benchmark report formatter. -func RegisterReporterFormat(name BenchReportFormat, formatFunc ReportFormatFunc) { - reportFormatters[name] = formatFunc -} - -// FormatReport delegates formatting of benchmark results to the registered benchmark report formatter. -func FormatReport(name BenchReportFormat, results []*Result) (benchmarkReports []string, err error) { - reportFunc, defined := reportFormatters[name] - if !defined { - return nil, fmt.Errorf("unregistered benchmark report format: %s", name) - } - - return reportFunc(results) -} diff --git a/internal/benchrunner/report_output.go b/internal/benchrunner/report_output.go deleted file mode 100644 index 3f6db9b95..000000000 --- a/internal/benchrunner/report_output.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package benchrunner - -import ( - "fmt" -) - -// BenchReportOutput represents an output for a benchmark report -type BenchReportOutput string - -// ReportOutputFunc defines the report writer function. -type ReportOutputFunc func(pkg, report string, format BenchReportFormat) error - -var reportOutputs = map[BenchReportOutput]ReportOutputFunc{} - -// RegisterReporterOutput registers a benchmark report output. -func RegisterReporterOutput(name BenchReportOutput, outputFunc ReportOutputFunc) { - reportOutputs[name] = outputFunc -} - -// WriteReport delegates writing of benchmark results to the registered benchmark report output -func WriteReport(pkg string, name BenchReportOutput, report string, format BenchReportFormat) error { - outputFunc, defined := reportOutputs[name] - if !defined { - return fmt.Errorf("unregistered benchmark report output: %s", name) - } - return outputFunc(pkg, report, format) -} diff --git a/internal/benchrunner/reporters/formats/human.go b/internal/benchrunner/reporters/formats/human.go deleted file mode 100644 index 272274bf9..000000000 --- a/internal/benchrunner/reporters/formats/human.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package formats - -import ( - "strings" - - "github.com/jedib0t/go-pretty/table" - "github.com/jedib0t/go-pretty/text" - - "github.com/elastic/elastic-package/internal/benchrunner" -) - -func init() { - benchrunner.RegisterReporterFormat(ReportFormatHuman, reportHumanFormat) -} - -const ( - // ReportFormatHuman reports benchmark results in a human-readable format - ReportFormatHuman benchrunner.BenchReportFormat = "human" -) - -func reportHumanFormat(results []*benchrunner.Result) ([]string, error) { - if len(results) == 0 { - return nil, nil - } - - var benchmarks []benchrunner.BenchmarkResult - for _, r := range results { - if r.Benchmark != nil { - benchmarks = append(benchmarks, *r.Benchmark) - } - } - - benchFormatted, err := reportHumanFormatBenchmark(benchmarks) - if err != nil { - return nil, err - } - return benchFormatted, nil -} - -func reportHumanFormatBenchmark(benchmarks []benchrunner.BenchmarkResult) ([]string, error) { - var textReports []string - for _, b := range benchmarks { - var report strings.Builder - if len(b.Parameters) > 0 { - report.WriteString(renderBenchmarkTable("parameters", b.Parameters) + "\n") - } - for _, t := range b.Tests { - report.WriteString(renderBenchmarkTable(t.Name, t.Results) + "\n") - } - textReports = append(textReports, report.String()) - } - return textReports, nil -} - -func renderBenchmarkTable(title string, values []benchrunner.BenchmarkValue) string { - t := table.NewWriter() - t.SetStyle(table.StyleRounded) - t.SetTitle(title) - t.SetColumnConfigs([]table.ColumnConfig{ - { - Number: 2, - Align: text.AlignRight, - }, - }) - for _, r := range values { - t.AppendRow(table.Row{r.Name, r.String()}) - } - return t.Render() -} diff --git a/internal/benchrunner/reporters/formats/json.go b/internal/benchrunner/reporters/formats/json.go deleted file mode 100644 index 6a460d9a0..000000000 --- a/internal/benchrunner/reporters/formats/json.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package formats - -import ( - "encoding/json" - "fmt" - - "github.com/elastic/elastic-package/internal/benchrunner" -) - -func init() { - benchrunner.RegisterReporterFormat(ReportFormatJSON, jsonFormat) -} - -const ( - // ReportFormatJSON reports benchmark results in the json format - ReportFormatJSON benchrunner.BenchReportFormat = "json" -) - -func jsonFormat(results []*benchrunner.Result) ([]string, error) { - var benchmarks []*benchrunner.BenchmarkResult - for _, r := range results { - if r.Benchmark != nil { - benchmarks = append(benchmarks, r.Benchmark) - } - } - - benchFormatted, err := jsonFormatBenchmark(benchmarks) - if err != nil { - return nil, err - } - return benchFormatted, nil -} - -func jsonFormatBenchmark(benchmarks []*benchrunner.BenchmarkResult) ([]string, error) { - var reports []string - for _, b := range benchmarks { - // Filter out detailed benchmarks. These add too much information for the - // aggregated nature of the reports, creating a lot of noise in Jenkins. - var benchmarks []benchrunner.BenchmarkTest - for _, t := range b.Tests { - if !t.Detailed { - benchmarks = append(benchmarks, t) - } - } - b.Tests = benchmarks - out, err := json.MarshalIndent(b, "", " ") - if err != nil { - return nil, fmt.Errorf("unable to format benchmark results as json: %w", err) - } - reports = append(reports, string(out)) - } - return reports, nil -} diff --git a/internal/benchrunner/reporters/formats/xunit.go b/internal/benchrunner/reporters/formats/xunit.go deleted file mode 100644 index 3ddaa8bae..000000000 --- a/internal/benchrunner/reporters/formats/xunit.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. -package formats - -import ( - "encoding/xml" - "fmt" - - "github.com/elastic/elastic-package/internal/benchrunner" -) - -func init() { - benchrunner.RegisterReporterFormat(ReportFormatXUnit, reportXUnitFormat) -} - -const ( - // ReportFormatXUnit reports benchmark results in the xUnit format - ReportFormatXUnit benchrunner.BenchReportFormat = "xUnit" -) - -func reportXUnitFormat(results []*benchrunner.Result) ([]string, error) { - var benchmarks []*benchrunner.BenchmarkResult - for _, r := range results { - if r.Benchmark != nil { - benchmarks = append(benchmarks, r.Benchmark) - } - } - - benchFormatted, err := reportXUnitFormatBenchmark(benchmarks) - if err != nil { - return nil, err - } - return benchFormatted, nil -} - -func reportXUnitFormatBenchmark(benchmarks []*benchrunner.BenchmarkResult) ([]string, error) { - var reports []string - for _, b := range benchmarks { - // Filter out detailed benchmarks. These add too much information for the - // aggregated nature of xUnit reports, creating a lot of noise in Jenkins. - var benchmarks []benchrunner.BenchmarkTest - for _, t := range b.Tests { - if !t.Detailed { - benchmarks = append(benchmarks, t) - } - } - b.Tests = benchmarks - out, err := xml.MarshalIndent(b, "", " ") - if err != nil { - return nil, fmt.Errorf("unable to format benchmark results as xUnit: %w", err) - } - reports = append(reports, xml.Header+string(out)) - } - return reports, nil -} diff --git a/internal/benchrunner/reporters/outputs/file.go b/internal/benchrunner/reporters/outputs/file.go index 50724eba0..1bb827fed 100644 --- a/internal/benchrunner/reporters/outputs/file.go +++ b/internal/benchrunner/reporters/outputs/file.go @@ -9,28 +9,63 @@ import ( "fmt" "os" "path/filepath" - "time" - "github.com/elastic/elastic-package/internal/benchrunner" - "github.com/elastic/elastic-package/internal/benchrunner/reporters/formats" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/multierror" ) func init() { - benchrunner.RegisterReporterOutput(ReportOutputFile, reportToFile) + reporters.RegisterOutput(ReportOutputFile, reportToFile) } const ( // ReportOutputFile reports benchmark results to files in a folder - ReportOutputFile benchrunner.BenchReportOutput = "file" + ReportOutputFile reporters.Output = "file" ) -func reportToFile(pkg, report string, format benchrunner.BenchReportFormat) error { +func reportToFile(report reporters.Reportable) error { + multiReport, ok := report.(reporters.MultiReportable) + if !ok { + return reportSingle(report) + } + + var merr multierror.Error + for _, r := range multiReport.Split() { + reportableFile, ok := r.(reporters.ReportableFile) + if !ok { + continue + } + + if err := reportSingle(reportableFile); err != nil { + merr = append(merr, err) + } + } + + if len(merr) > 0 { + return merr + } + + return nil +} + +func reportSingle(report reporters.Reportable) error { + reportableFile, ok := report.(reporters.ReportableFile) + if !ok { + return errors.New("this output requires a reportable file") + } + dest, err := reportsDir() if err != nil { return fmt.Errorf("could not determine benchmark reports folder: %w", err) } + // If filename contains folders, be sure we create them properly + dir := filepath.Dir(reportableFile.Filename()) + if dir != reportableFile.Filename() { + dest = filepath.Join(dest, dir) + } + // Create benchmark reports folder if it doesn't exist _, err = os.Stat(dest) if err != nil && errors.Is(err, os.ErrNotExist) { @@ -39,17 +74,9 @@ func reportToFile(pkg, report string, format benchrunner.BenchReportFormat) erro } } - var ext string - switch format { - case formats.ReportFormatXUnit: - ext = "xml" - default: - ext = string(format) - } - fileName := fmt.Sprintf("%s_%d.%s", pkg, time.Now().UnixNano(), ext) - filePath := filepath.Join(dest, fileName) + filePath := filepath.Join(dest, filepath.Base(reportableFile.Filename())) - if err := os.WriteFile(filePath, []byte(report+"\n"), 0644); err != nil { + if err := os.WriteFile(filePath, append(reportableFile.Report(), byte('\n')), 0644); err != nil { return fmt.Errorf("could not write benchmark report file: %w", err) } diff --git a/internal/benchrunner/reporters/outputs/stdout.go b/internal/benchrunner/reporters/outputs/stdout.go index b0fb25bf7..84149fee4 100644 --- a/internal/benchrunner/reporters/outputs/stdout.go +++ b/internal/benchrunner/reporters/outputs/stdout.go @@ -7,22 +7,22 @@ package outputs import ( "fmt" - "github.com/elastic/elastic-package/internal/benchrunner" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" ) func init() { - benchrunner.RegisterReporterOutput(ReportOutputSTDOUT, reportToSTDOUT) + reporters.RegisterOutput(ReportOutputSTDOUT, reportToSTDOUT) } const ( // ReportOutputSTDOUT reports benchmark results to STDOUT - ReportOutputSTDOUT benchrunner.BenchReportOutput = "stdout" + ReportOutputSTDOUT reporters.Output = "stdout" ) -func reportToSTDOUT(pkg, report string, _ benchrunner.BenchReportFormat) error { - fmt.Printf("--- Benchmark results for package: %s - START ---\n", pkg) - fmt.Println(report) - fmt.Printf("--- Benchmark results for package: %s - END ---\n", pkg) +func reportToSTDOUT(report reporters.Reportable) error { + fmt.Printf("--- Benchmark results for package: %s - START ---\n", report.Package()) + fmt.Println(string(report.Report())) + fmt.Printf("--- Benchmark results for package: %s - END ---\n", report.Package()) fmt.Println("Done") return nil } diff --git a/internal/benchrunner/reporters/report_output.go b/internal/benchrunner/reporters/report_output.go new file mode 100644 index 000000000..ecb14ae06 --- /dev/null +++ b/internal/benchrunner/reporters/report_output.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package reporters + +import ( + "fmt" +) + +// Output represents an output for a benchmark report +type Output string + +// OutputFunc defines the report writer function. +type OutputFunc func(Reportable) error + +var reportOutputs = map[Output]OutputFunc{} + +// RegisterOutput registers a benchmark report output. +func RegisterOutput(name Output, outputFunc OutputFunc) { + reportOutputs[name] = outputFunc +} + +// WriteReportable delegates writing of benchmark results to the registered benchmark report output +func WriteReportable(output Output, report Reportable) error { + outputFunc, defined := reportOutputs[output] + if !defined { + return fmt.Errorf("unregistered benchmark report output: %s", output) + } + return outputFunc(report) +} diff --git a/internal/benchrunner/reporters/reportable.go b/internal/benchrunner/reporters/reportable.go new file mode 100644 index 000000000..12d03d884 --- /dev/null +++ b/internal/benchrunner/reporters/reportable.go @@ -0,0 +1,84 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package reporters + +// Reportable defines a raw report associated to a package. +type Reportable interface { + Package() string + Report() []byte +} + +var _ Reportable = &Report{} + +type Report struct { + pkg string + r []byte +} + +func NewReport(pkg string, p []byte) *Report { + return &Report{pkg: pkg, r: p} +} + +func (r *Report) Package() string { return r.pkg } + +func (r *Report) Report() []byte { return r.r } + +// Reportable file associates a report to a filename. +type ReportableFile interface { + Reportable + Filename() string +} + +var _ Reportable = &FileReport{} +var _ ReportableFile = &FileReport{} + +type FileReport struct { + pkg string + r []byte + filename string +} + +func NewFileReport(pkg, name string, p []byte) *FileReport { + return &FileReport{pkg: pkg, r: p, filename: name} +} + +func (r *FileReport) Package() string { return r.pkg } + +func (r *FileReport) Report() []byte { return r.r } + +func (r *FileReport) Filename() string { return r.filename } + +// MultiReportable defines an extended interface to ship multiple reports together. +// A call to Report() will return all reports contents combined. +type MultiReportable interface { + Reportable + Split() []Reportable +} + +var _ Reportable = &MultiReport{} +var _ MultiReportable = &MultiReport{} + +type MultiReport struct { + pkg string + reports []Reportable +} + +func NewMultiReport(pkg string, reports []Reportable) *MultiReport { + return &MultiReport{pkg: pkg, reports: reports} +} + +func (r *MultiReport) Package() string { return r.pkg } + +func (r *MultiReport) Report() []byte { + var combined []byte + for _, fr := range r.reports { + combined = append(combined, append(fr.Report(), '\n')...) + } + return combined +} + +func (r *MultiReport) Split() []Reportable { + return r.reports +} diff --git a/internal/benchrunner/runner.go b/internal/benchrunner/runner.go new file mode 100644 index 000000000..00399d14d --- /dev/null +++ b/internal/benchrunner/runner.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package benchrunner + +import ( + "errors" + "fmt" + + "github.com/elastic/elastic-package/internal/benchrunner/reporters" +) + +// Type represents the various supported benchmark types +type Type string + +type Runner interface { + SetUp() error + Run() (reporters.Reportable, error) + TearDown() error +} + +// Run method delegates execution to the benchmark runner. +func Run(runner Runner) (reporters.Reportable, error) { + if runner == nil { + return nil, errors.New("a runner is required") + } + + if err := runner.SetUp(); err != nil { + return nil, fmt.Errorf("could not set up benchmark runner: %w", err) + } + + report, err := runner.Run() + tdErr := runner.TearDown() + + if err != nil { + return nil, fmt.Errorf("could not complete benchmark run: %w", err) + } + + if tdErr != nil { + return report, fmt.Errorf("could not teardown benchmark runner: %w", tdErr) + } + + return report, nil +} diff --git a/internal/benchrunner/runners/pipeline/benchmark.go b/internal/benchrunner/runners/pipeline/benchmark.go index ff7a269e1..bf14a0f7d 100644 --- a/internal/benchrunner/runners/pipeline/benchmark.go +++ b/internal/benchrunner/runners/pipeline/benchmark.go @@ -11,11 +11,71 @@ import ( "sort" "time" - "github.com/elastic/elastic-package/internal/benchrunner" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" ) -func (r *runner) benchmarkPipeline(b *benchmark, entryPipeline string) (*benchrunner.BenchmarkResult, error) { +type BenchmarkResult struct { + // XMLName is a zero-length field used as an annotation for XML marshaling. + XMLName struct{} `xml:"group" json:"-"` + // Type of benchmark + Type string `xml:"type" json:"type"` + // Package of the benchmark + Package string `xml:"package" json:"package"` + // DataStream of the benchmark + DataStream string `xml:"data_stream" json:"data_stream"` + // Description of the benchmark run. + Description string `xml:"description,omitempty" json:"description,omitempty"` + // Parameters used for this benchmark. + Parameters []BenchmarkValue `xml:"parameters,omitempty" json:"parameters,omitempty"` + // Tests holds the results for the benchmark. + Tests []BenchmarkTest `xml:"test" json:"test"` +} + +// BenchmarkTest models a particular test performed during a benchmark. +type BenchmarkTest struct { + // Name of this test. + Name string `xml:"name" json:"name"` + // Detailed benchmark tests will be printed to the output but not + // included in file reports. + Detailed bool `xml:"-" json:"-"` + // Description of this test. + Description string `xml:"description,omitempty" json:"description,omitempty"` + // Parameters for this test. + Parameters []BenchmarkValue `xml:"parameters,omitempty" json:"parameters,omitempty"` + // Results of the test. + Results []BenchmarkValue `xml:"result" json:"result"` +} + +// BenchmarkValue represents a value (result or parameter) +// with an optional associated unit. +type BenchmarkValue struct { + // Name of the value. + Name string `xml:"name" json:"name"` + // Description of the value. + Description string `xml:"description,omitempty" json:"description,omitempty"` + // Unit used for this value. + Unit string `xml:"unit,omitempty" json:"unit,omitempty"` + // Value is of any type, usually string or numeric. + Value interface{} `xml:"value,omitempty" json:"value,omitempty"` +} + +// String returns a BenchmarkValue's value nicely-formatted. +func (p BenchmarkValue) String() (r string) { + if str, ok := p.Value.(fmt.Stringer); ok { + return str.String() + } + if float, ok := p.Value.(float64); ok { + r = fmt.Sprintf("%.02f", float) + } else { + r = fmt.Sprintf("%v", p.Value) + } + if p.Unit != "" { + r += p.Unit + } + return r +} + +func (r *runner) benchmarkPipeline(b *benchmark, entryPipeline string) (*BenchmarkResult, error) { // Run benchmark bench, err := r.benchmarkIngest(b, entryPipeline) if err != nil { @@ -40,16 +100,16 @@ func (r *runner) benchmarkPipeline(b *benchmark, entryPipeline string) (*benchru } return record.TimeInMillis * int64(time.Millisecond) / record.Count } - asPercentageOfTotalTime := func(perf processorPerformance) benchrunner.BenchmarkValue { - return benchrunner.BenchmarkValue{ + asPercentageOfTotalTime := func(perf processorPerformance) BenchmarkValue { + return BenchmarkValue{ Name: perf.key, Description: perf.key, Unit: "%", Value: time.Duration(perf.value).Seconds() * 100 / bench.elapsed.Seconds(), } } - asDuration := func(perf processorPerformance) benchrunner.BenchmarkValue { - return benchrunner.BenchmarkValue{ + asDuration := func(perf processorPerformance) BenchmarkValue { + return BenchmarkValue{ Name: perf.key, Description: perf.key, Value: time.Duration(perf.value), @@ -81,12 +141,12 @@ func (r *runner) benchmarkPipeline(b *benchmark, entryPipeline string) (*benchru } // Build result - result := &benchrunner.BenchmarkResult{ + result := &BenchmarkResult{ Type: string(BenchType), Package: r.options.Folder.Package, DataStream: r.options.Folder.DataStream, Description: fmt.Sprintf("pipeline benchmark for %s/%s", r.options.Folder.Package, r.options.Folder.DataStream), - Parameters: []benchrunner.BenchmarkValue{ + Parameters: []BenchmarkValue{ { Name: "source_doc_count", Value: len(b.events), @@ -96,10 +156,10 @@ func (r *runner) benchmarkPipeline(b *benchmark, entryPipeline string) (*benchru Value: bench.numDocs, }, }, - Tests: []benchrunner.BenchmarkTest{ + Tests: []BenchmarkTest{ { Name: "pipeline_performance", - Results: []benchrunner.BenchmarkValue{ + Results: []BenchmarkValue{ { Name: "processing_time", Description: "time elapsed in pipeline processors", @@ -154,7 +214,7 @@ type aggregation struct { type ( keyFn func(ingest.Pipeline, ingest.Processor) string valueFn func(record ingest.StatsRecord) int64 - mapFn func(processorPerformance) benchrunner.BenchmarkValue + mapFn func(processorPerformance) BenchmarkValue compareFn func(a, b processorPerformance) bool filterFn func(processorPerformance) bool ) @@ -227,11 +287,11 @@ func (agg aggregation) filter(keep filterFn) aggregation { return agg } -func (agg aggregation) collect(fn mapFn) ([]benchrunner.BenchmarkValue, error) { +func (agg aggregation) collect(fn mapFn) ([]BenchmarkValue, error) { if agg.err != nil { return nil, agg.err } - r := make([]benchrunner.BenchmarkValue, len(agg.result)) + r := make([]BenchmarkValue, len(agg.result)) for idx := range r { r[idx] = fn(agg.result[idx]) } diff --git a/internal/benchrunner/runners/pipeline/format.go b/internal/benchrunner/runners/pipeline/format.go new file mode 100644 index 000000000..639607063 --- /dev/null +++ b/internal/benchrunner/runners/pipeline/format.go @@ -0,0 +1,121 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package pipeline + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "strings" + "time" + + "github.com/jedib0t/go-pretty/table" + "github.com/jedib0t/go-pretty/text" +) + +const ( + // ReportFormatHuman reports benchmark results in a human-readable format + ReportFormatHuman Format = "human" + // ReportFormatJSON reports benchmark results in the json format + ReportFormatJSON Format = "json" + // ReportFormatXUnit reports benchmark results in the xUnit format + ReportFormatXUnit Format = "xUnit" +) + +// Format represents a benchmark report format +type Format string + +// FormatResult delegates formatting of benchmark results to the registered benchmark report formatter. +func formatResult(name Format, result *BenchmarkResult) (report []byte, err error) { + switch name { + case ReportFormatHuman: + return reportHumanFormat(result), nil + case ReportFormatJSON: + return reportJSONFormat(result) + case ReportFormatXUnit: + return reportXUnitFormat(result) + } + return nil, fmt.Errorf("unknown format: %s", name) +} + +func reportHumanFormat(b *BenchmarkResult) []byte { + var report strings.Builder + if len(b.Parameters) > 0 { + report.WriteString(renderBenchmarkTable("parameters", b.Parameters) + "\n") + } + for _, t := range b.Tests { + report.WriteString(renderBenchmarkTable(t.Name, t.Results) + "\n") + } + return []byte(report.String()) +} + +func renderBenchmarkTable(title string, values []BenchmarkValue) string { + t := table.NewWriter() + t.SetStyle(table.StyleRounded) + t.SetTitle(title) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Number: 2, + Align: text.AlignRight, + }, + }) + for _, r := range values { + t.AppendRow(table.Row{r.Name, r.String()}) + } + return t.Render() +} + +func reportJSONFormat(b *BenchmarkResult) ([]byte, error) { + // Filter out detailed benchmarks. These add too much information for the + // aggregated nature of the reports, creating a lot of noise in Jenkins. + var benchmarks []BenchmarkTest + for _, t := range b.Tests { + if !t.Detailed { + benchmarks = append(benchmarks, t) + } + } + b.Tests = benchmarks + out, err := json.MarshalIndent(b, "", " ") + if err != nil { + return nil, fmt.Errorf("unable to format benchmark results as json: %w", err) + } + return out, nil +} + +func reportXUnitFormat(b *BenchmarkResult) ([]byte, error) { + // Filter out detailed benchmarks. These add too much information for the + // aggregated nature of xUnit reports, creating a lot of noise in Jenkins. + var benchmarks []BenchmarkTest + for _, t := range b.Tests { + if !t.Detailed { + benchmarks = append(benchmarks, t) + } + } + b.Tests = benchmarks + out, err := xml.MarshalIndent(b, "", " ") + if err != nil { + return nil, fmt.Errorf("unable to format benchmark results as xUnit: %w", err) + } + return out, nil +} + +func filenameByFormat(pkg string, format Format) string { + var ext string + switch format { + default: + fallthrough + case ReportFormatJSON: + ext = "json" + case ReportFormatXUnit: + ext = "xml" + } + fileName := fmt.Sprintf( + "%s_%d.%s", + pkg, + time.Now().UnixNano(), + ext, + ) + return fileName +} diff --git a/internal/benchrunner/runners/pipeline/options.go b/internal/benchrunner/runners/pipeline/options.go new file mode 100644 index 000000000..c2699d6be --- /dev/null +++ b/internal/benchrunner/runners/pipeline/options.go @@ -0,0 +1,66 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package pipeline + +import ( + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/testrunner" +) + +// Options contains benchmark runner options. +type Options struct { + BenchName string + Folder testrunner.TestFolder + PackageRootPath string + API *elasticsearch.API + NumTopProcs int + Format Format +} + +type OptionFunc func(*Options) + +func NewOptions(fns ...OptionFunc) Options { + var opts Options + for _, fn := range fns { + fn(&opts) + } + return opts +} + +func WithFolder(f testrunner.TestFolder) OptionFunc { + return func(opts *Options) { + opts.Folder = f + } +} + +func WithPackageRootPath(path string) OptionFunc { + return func(opts *Options) { + opts.PackageRootPath = path + } +} + +func WithESAPI(api *elasticsearch.API) OptionFunc { + return func(opts *Options) { + opts.API = api + } +} + +func WithNumTopProcs(n int) OptionFunc { + return func(opts *Options) { + opts.NumTopProcs = n + } +} + +func WithFormat(format string) OptionFunc { + return func(opts *Options) { + opts.Format = Format(format) + } +} + +func WithBenchmarkName(name string) OptionFunc { + return func(opts *Options) { + opts.BenchName = name + } +} diff --git a/internal/benchrunner/runners/pipeline/runner.go b/internal/benchrunner/runners/pipeline/runner.go index 39b59c2d9..e55f293e8 100644 --- a/internal/benchrunner/runners/pipeline/runner.go +++ b/internal/benchrunner/runners/pipeline/runner.go @@ -10,41 +10,49 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" - "time" "github.com/elastic/elastic-package/internal/benchrunner" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" "github.com/elastic/elastic-package/internal/elasticsearch/ingest" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/testrunner" ) const ( // BenchType defining pipeline benchmarks. - BenchType benchrunner.BenchType = "pipeline" + BenchType benchrunner.Type = "pipeline" expectedTestResultSuffix = "-expected.json" configTestSuffixYAML = "-config.yml" ) type runner struct { - options benchrunner.BenchOptions - pipelines []ingest.Pipeline + options Options + entryPipeline string + pipelines []ingest.Pipeline } -// Type returns the type of benchmark that can be run by this benchmark runner. -func (r *runner) Type() benchrunner.BenchType { - return BenchType +func NewPipelineBenchmark(opts Options) benchrunner.Runner { + return &runner{options: opts} } -// String returns the human-friendly name of the benchmark runner. -func (r *runner) String() string { - return "pipeline" -} +func (r *runner) SetUp() error { + dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.Folder.Path) + if err != nil { + return fmt.Errorf("locating data_stream root failed: %w", err) + } + if !found { + return errors.New("data stream root not found") + } -// Run runs the pipeline benchmarks defined under the given folder -func (r *runner) Run(options benchrunner.BenchOptions) (*benchrunner.Result, error) { - r.options = options - return r.run() + r.entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.options.API, dataStreamPath) + if err != nil { + return fmt.Errorf("installing ingest pipelines failed: %w", err) + } + + return nil } // TearDown shuts down the pipeline benchmark runner. @@ -55,40 +63,96 @@ func (r *runner) TearDown() error { return nil } -func (r *runner) run() (*benchrunner.Result, error) { - dataStreamPath, found, err := packages.FindDataStreamRootForPath(r.options.Folder.Path) +// Run runs the pipeline benchmarks defined under the given folder +func (r *runner) Run() (reporters.Reportable, error) { + return r.run() +} + +func (r *runner) run() (reporters.Reportable, error) { + b, err := r.loadBenchmark() if err != nil { - return nil, fmt.Errorf("locating data_stream root failed: %w", err) + return nil, fmt.Errorf("loading benchmark failed: %w", err) } - if !found { - return nil, errors.New("data stream root not found") + + benchmark, err := r.benchmarkPipeline(b, r.entryPipeline) + if err != nil { + return nil, err } - var entryPipeline string - entryPipeline, r.pipelines, err = ingest.InstallDataStreamPipelines(r.options.API, dataStreamPath) + formattedReport, err := formatResult(r.options.Format, benchmark) if err != nil { - return nil, fmt.Errorf("installing ingest pipelines failed: %w", err) + return nil, err } - start := time.Now() - result := &benchrunner.Result{ - BenchType: BenchType + " benchmark", - Package: r.options.Folder.Package, - DataStream: r.options.Folder.DataStream, + switch r.options.Format { + case ReportFormatHuman: + return reporters.NewReport(r.options.Folder.Package, formattedReport), nil } - b, err := r.loadBenchmark() - if err != nil { - return nil, fmt.Errorf("loading benchmark failed: %w", err) + return reporters.NewFileReport( + r.options.BenchName, + filenameByFormat(r.options.BenchName, r.options.Format), + formattedReport, + ), nil +} + +// FindBenchmarkFolders finds benchmark folders for the given package and, optionally, benchmark type and data streams +func FindBenchmarkFolders(packageRootPath string, dataStreams []string) ([]testrunner.TestFolder, error) { + // Expected folder structure: + // / + // data_stream/ + // / + // _dev/ + // benchmark/ + // / + + var paths []string + if len(dataStreams) > 0 { + sort.Strings(dataStreams) + for _, dataStream := range dataStreams { + p, err := findBenchFolderPaths(packageRootPath, dataStream) + if err != nil { + return nil, err + } + + paths = append(paths, p...) + } + } else { + p, err := findBenchFolderPaths(packageRootPath, "*") + if err != nil { + return nil, err + } + + paths = p } - if result.Benchmark, err = r.benchmarkPipeline(b, entryPipeline); err != nil { - result.ErrorMsg = err.Error() + sort.Strings(dataStreams) + for _, dataStream := range dataStreams { + p, err := findBenchFolderPaths(packageRootPath, dataStream) + if err != nil { + return nil, err + } + + paths = append(paths, p...) } - result.TimeElapsed = time.Since(start) + folders := make([]testrunner.TestFolder, len(paths)) + _, pkg := filepath.Split(packageRootPath) + for idx, p := range paths { + relP := strings.TrimPrefix(p, packageRootPath) + parts := strings.Split(relP, string(filepath.Separator)) + dataStream := parts[2] + + folder := testrunner.TestFolder{ + Path: p, + Package: pkg, + DataStream: dataStream, + } + + folders[idx] = folder + } - return result, nil + return folders, nil } func (r *runner) listBenchmarkFiles() ([]string, error) { @@ -156,6 +220,13 @@ func (r *runner) loadBenchmark() (*benchmark, error) { return tc, nil } -func init() { - benchrunner.RegisterRunner(&runner{}) +// findBenchFoldersPaths can only be called for benchmark runners that require benchmarks to be defined +// at the data stream level. +func findBenchFolderPaths(packageRootPath, dataStreamGlob string) ([]string, error) { + benchFoldersGlob := filepath.Join(packageRootPath, "data_stream", dataStreamGlob, "_dev", "benchmark", "pipeline") + paths, err := filepath.Glob(benchFoldersGlob) + if err != nil { + return nil, fmt.Errorf("error finding benchmark folders: %w", err) + } + return paths, err } diff --git a/internal/benchrunner/runners/runners.go b/internal/benchrunner/runners/runners.go deleted file mode 100644 index 1fc2f6b06..000000000 --- a/internal/benchrunner/runners/runners.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package runners - -import ( - // Registered benchmark runners - _ "github.com/elastic/elastic-package/internal/benchrunner/runners/pipeline" -) diff --git a/internal/benchrunner/runners/system/metrics.go b/internal/benchrunner/runners/system/metrics.go new file mode 100644 index 000000000..40edae2c1 --- /dev/null +++ b/internal/benchrunner/runners/system/metrics.go @@ -0,0 +1,249 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package system + +import ( + "sync" + "time" + + "github.com/elastic/elastic-package/internal/benchrunner/runners/system/servicedeployer" + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/signal" +) + +type collector struct { + ctxt servicedeployer.ServiceContext + warmupD time.Duration + interval time.Duration + esapi *elasticsearch.API + datastream string + pipelinePrefix string + + stopC chan struct{} + tick *time.Ticker + + startIngestMetrics map[string]ingest.PipelineStatsMap + endIngestMetrics map[string]ingest.PipelineStatsMap + collectedMetrics []metrics + diskUsage map[string]ingest.DiskUsage + startTotalHits int + endTotalHits int +} + +type metrics struct { + ts int64 + dsMetrics *ingest.DataStreamStats + nMetrics *ingest.NodesStats +} + +type metricsSummary struct { + ClusterName string + Nodes int + RunID string + CollectionStartTs int64 + CollectionEndTs int64 + DataStreamStats *ingest.DataStreamStats + IngestPipelineStats map[string]ingest.PipelineStatsMap + DiskUsage map[string]ingest.DiskUsage + TotalHits int +} + +func newCollector( + ctxt servicedeployer.ServiceContext, + esapi *elasticsearch.API, + interval, warmup time.Duration, + datastream, pipelinePrefix string, +) *collector { + return &collector{ + ctxt: ctxt, + interval: interval, + warmupD: warmup, + esapi: esapi, + datastream: datastream, + pipelinePrefix: pipelinePrefix, + stopC: make(chan struct{}, 1), + } +} + +func (c *collector) start() { + c.tick = time.NewTicker(c.interval) + + go func() { + var once sync.Once + once.Do(c.waitUntilReady) + + defer c.tick.Stop() + + c.startIngestMetrics = c.collectIngestMetrics() + c.startTotalHits = c.collectTotalHits() + + for { + if signal.SIGINT() { + logger.Debug("SIGINT: cancel metrics collection") + c.collectMetricsPreviousToStop() + return + } + + select { + case <-c.stopC: + // last collect before stopping + c.collectMetricsPreviousToStop() + c.stopC <- struct{}{} + return + case <-c.tick.C: + c.collect() + } + } + }() +} + +func (c *collector) stop() { + c.stopC <- struct{}{} + <-c.stopC + close(c.stopC) +} + +func (c *collector) collect() { + m := metrics{ + ts: time.Now().Unix(), + } + + nstats, err := ingest.GetNodesStats(c.esapi) + if err != nil { + logger.Debug(err) + } else { + m.nMetrics = nstats + } + + dsstats, err := ingest.GetDataStreamStats(c.esapi, c.datastream) + if err != nil { + logger.Debug(err) + } else { + m.dsMetrics = dsstats + } + + c.collectedMetrics = append(c.collectedMetrics, m) +} + +func (c *collector) summarize() (*metricsSummary, error) { + sum := metricsSummary{ + RunID: c.ctxt.Bench.RunID, + IngestPipelineStats: make(map[string]ingest.PipelineStatsMap), + DiskUsage: c.diskUsage, + TotalHits: c.endTotalHits - c.startTotalHits, + } + + if len(c.collectedMetrics) > 0 { + sum.CollectionStartTs = c.collectedMetrics[0].ts + sum.CollectionEndTs = c.collectedMetrics[len(c.collectedMetrics)-1].ts + sum.DataStreamStats = c.collectedMetrics[len(c.collectedMetrics)-1].dsMetrics + sum.ClusterName = c.collectedMetrics[0].nMetrics.ClusterName + sum.Nodes = len(c.collectedMetrics[len(c.collectedMetrics)-1].nMetrics.Nodes) + } + + for node, endPStats := range c.endIngestMetrics { + startPStats, found := c.startIngestMetrics[node] + if !found { + logger.Debugf("node %s not found in initial metrics", node) + continue + } + sumStats := make(ingest.PipelineStatsMap) + for pname, endStats := range endPStats { + startStats, found := startPStats[pname] + if !found { + logger.Debugf("pipeline %s not found in node %s initial metrics", pname, node) + continue + } + sumStats[pname] = ingest.PipelineStats{ + StatsRecord: ingest.StatsRecord{ + Count: endStats.Count - startStats.Count, + Failed: endStats.TimeInMillis - startStats.TimeInMillis, + }, + Processors: make([]ingest.ProcessorStats, len(endStats.Processors)), + } + for i, endPr := range endStats.Processors { + startPr := startStats.Processors[i] + sumStats[pname].Processors[i] = ingest.ProcessorStats{ + Type: endPr.Type, + Extra: endPr.Extra, + Conditional: endPr.Conditional, + Stats: ingest.StatsRecord{ + Count: endPr.Stats.Count - startPr.Stats.Count, + Failed: endPr.Stats.Failed - startPr.Stats.Failed, + TimeInMillis: endPr.Stats.TimeInMillis - startPr.Stats.TimeInMillis, + }, + } + } + } + sum.IngestPipelineStats[node] = sumStats + } + + return &sum, nil +} + +func (c *collector) waitUntilReady() { + logger.Debug("waiting for datastream to be created...") + + waitTick := time.NewTicker(time.Second) + defer waitTick.Stop() + +readyLoop: + for { + if signal.SIGINT() { + logger.Debug("SIGINT: cancel metrics collection") + return + } + + <-waitTick.C + dsstats, err := ingest.GetDataStreamStats(c.esapi, c.datastream) + if err != nil { + logger.Debug(err) + } + if dsstats != nil { + break readyLoop + } + } + + if c.warmupD > 0 { + logger.Debugf("waiting %s for warmup period", c.warmupD) + <-time.After(c.warmupD) + } + logger.Debug("metric collection starting...") +} + +func (c *collector) collectIngestMetrics() map[string]ingest.PipelineStatsMap { + ipMetrics, err := ingest.GetPipelineStatsByPrefix(c.esapi, c.pipelinePrefix) + if err != nil { + logger.Debugf("could not get ingest pipeline metrics: %w", err) + return nil + } + return ipMetrics +} + +func (c *collector) collectDiskUsage() map[string]ingest.DiskUsage { + du, err := ingest.GetDiskUsage(c.esapi, c.datastream) + if err != nil { + logger.Debugf("could not get disk usage metrics: %w", err) + return nil + } + return du +} + +func (c *collector) collectMetricsPreviousToStop() { + c.collect() + c.endIngestMetrics = c.collectIngestMetrics() + c.diskUsage = c.collectDiskUsage() + c.endTotalHits = c.collectTotalHits() +} + +func (c *collector) collectTotalHits() int { + totalHits, err := getTotalHits(c.esapi, c.datastream) + if err != nil { + logger.Debugf("could not total hits: %w", err) + } + return totalHits +} diff --git a/internal/benchrunner/runners/system/options.go b/internal/benchrunner/runners/system/options.go new file mode 100644 index 000000000..afdb22e47 --- /dev/null +++ b/internal/benchrunner/runners/system/options.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package system + +import ( + "time" + + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" +) + +// Options contains benchmark runner options. +type Options struct { + ESAPI *elasticsearch.API + KibanaClient *kibana.Client + DeferCleanup time.Duration + MetricsInterval time.Duration + ReindexData bool + MetricstoreESURL string + BenchName string + PackageRootPath string +} + +type OptionFunc func(*Options) + +func NewOptions(fns ...OptionFunc) Options { + var opts Options + for _, fn := range fns { + fn(&opts) + } + return opts +} + +func WithESAPI(api *elasticsearch.API) OptionFunc { + return func(opts *Options) { + opts.ESAPI = api + } +} + +func WithKibanaClient(c *kibana.Client) OptionFunc { + return func(opts *Options) { + opts.KibanaClient = c + } +} + +func WithPackageRootPath(path string) OptionFunc { + return func(opts *Options) { + opts.PackageRootPath = path + } +} + +func WithBenchmarkName(name string) OptionFunc { + return func(opts *Options) { + opts.BenchName = name + } +} + +func WithDeferCleanup(d time.Duration) OptionFunc { + return func(opts *Options) { + opts.DeferCleanup = d + } +} +func WithMetricsInterval(d time.Duration) OptionFunc { + return func(opts *Options) { + opts.MetricsInterval = d + } +} +func WithDataReindexing(b bool) OptionFunc { + return func(opts *Options) { + opts.ReindexData = b + } +} diff --git a/internal/benchrunner/runners/system/report.go b/internal/benchrunner/runners/system/report.go new file mode 100644 index 000000000..5aa5430a2 --- /dev/null +++ b/internal/benchrunner/runners/system/report.go @@ -0,0 +1,223 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package system + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/jedib0t/go-pretty/table" + "github.com/jedib0t/go-pretty/text" + + "github.com/elastic/elastic-package/internal/benchrunner/reporters" + "github.com/elastic/elastic-package/internal/elasticsearch/ingest" +) + +type report struct { + Info struct { + Benchmark string + Description string + RunID string + Package string + StartTs int64 + EndTs int64 + Duration time.Duration + GeneratedCorporaFile string + } + Parameters struct { + PackageVersion string + Input string + Vars map[string]interface{} + DataStream dataStream + WarmupTimePeriod time.Duration + BenchmarkTimePeriod time.Duration + WaitForDataTimeout time.Duration + Corpora corpora + } + ClusterName string + Nodes int + DataStreamStats *ingest.DataStreamStats + IngestPipelineStats map[string]ingest.PipelineStatsMap + DiskUsage map[string]ingest.DiskUsage + TotalHits int +} + +func createReport(benchName, corporaFile string, s *scenario, sum *metricsSummary) (reporters.Reportable, error) { + r := newReport(benchName, corporaFile, s, sum) + human := reporters.NewReport(s.Package, reportHumanFormat(r)) + + jsonBytes, err := reportJSONFormat(r) + if err != nil { + return nil, fmt.Errorf("rendering JSON report: %w", err) + } + + jsonFile := reporters.NewFileReport(s.Package, fmt.Sprintf("system/%s/report.json", sum.RunID), jsonBytes) + + mr := reporters.NewMultiReport(s.Package, []reporters.Reportable{human, jsonFile}) + + return mr, nil +} + +func newReport(benchName, corporaFile string, s *scenario, sum *metricsSummary) *report { + var report report + report.Info.Benchmark = benchName + report.Info.Description = s.Description + report.Info.RunID = sum.RunID + report.Info.Package = s.Package + report.Info.StartTs = sum.CollectionStartTs + report.Info.EndTs = sum.CollectionEndTs + report.Info.Duration = time.Duration(sum.CollectionEndTs-sum.CollectionStartTs) * time.Second + report.Info.GeneratedCorporaFile = corporaFile + report.Parameters.PackageVersion = s.Version + report.Parameters.Input = s.Input + report.Parameters.Vars = s.Vars + report.Parameters.DataStream = s.DataStream + report.Parameters.WarmupTimePeriod = s.WarmupTimePeriod + report.Parameters.BenchmarkTimePeriod = s.BenchmarkTimePeriod + report.Parameters.WaitForDataTimeout = s.WaitForDataTimeout + report.Parameters.Corpora = s.Corpora + report.ClusterName = sum.ClusterName + report.Nodes = sum.Nodes + report.DataStreamStats = sum.DataStreamStats + report.IngestPipelineStats = sum.IngestPipelineStats + report.DiskUsage = sum.DiskUsage + report.TotalHits = sum.TotalHits + return &report +} + +func reportJSONFormat(r *report) ([]byte, error) { + b, err := json.MarshalIndent(r, "", "\t") + if err != nil { + return nil, err + } + return b, nil +} + +func reportHumanFormat(r *report) []byte { + var report strings.Builder + report.WriteString(renderBenchmarkTable( + "info", + "benchmark", r.Info.Benchmark, + "description", r.Info.Description, + "run ID", r.Info.RunID, + "package", r.Info.Package, + "start ts (s)", r.Info.StartTs, + "end ts (s)", r.Info.EndTs, + "duration", r.Info.Duration, + "generated corpora file", r.Info.GeneratedCorporaFile, + ) + "\n") + + pkvs := []interface{}{ + "package version", r.Parameters.PackageVersion, + "input", r.Parameters.Input, + } + + for k, v := range r.Parameters.Vars { + pkvs = append(pkvs, fmt.Sprintf("vars.%s", k), v) + } + + pkvs = append(pkvs, "data_stream.name", r.Parameters.DataStream.Name) + + for k, v := range r.Parameters.DataStream.Vars { + pkvs = append(pkvs, fmt.Sprintf("data_stream.vars.%s", k), v) + } + + pkvs = append(pkvs, + "warmup time period", r.Parameters.WarmupTimePeriod, + "benchmark time period", r.Parameters.BenchmarkTimePeriod, + "wait for data timeout", r.Parameters.WaitForDataTimeout, + ) + + if r.Parameters.Corpora.Generator != nil { + pkvs = append(pkvs, + "corpora.generator.size", r.Parameters.Corpora.Generator.Size, + "corpora.generator.template.path", r.Parameters.Corpora.Generator.Template.Path, + "corpora.generator.template.raw", r.Parameters.Corpora.Generator.Template.Raw, + "corpora.generator.template.type", r.Parameters.Corpora.Generator.Template.Type, + "corpora.generator.config.path", r.Parameters.Corpora.Generator.Config.Path, + "corpora.generator.config.raw", r.Parameters.Corpora.Generator.Config.Raw, + "corpora.generator.fields.path", r.Parameters.Corpora.Generator.Fields.Path, + "corpora.generator.fields.raw", r.Parameters.Corpora.Generator.Fields.Raw, + ) + } + + if r.Parameters.Corpora.InputService != nil { + pkvs = append(pkvs, + "corpora.input_service.name", r.Parameters.Corpora.InputService.Name, + "corpora.input_service.signal", r.Parameters.Corpora.InputService.Signal, + ) + } + + report.WriteString(renderBenchmarkTable("parameters", pkvs...) + "\n") + + report.WriteString(renderBenchmarkTable( + "cluster info", + "name", r.ClusterName, + "nodes", r.Nodes, + ) + "\n") + + report.WriteString(renderBenchmarkTable( + "data stream stats", + "data stream", r.DataStreamStats.DataStream, + "approx total docs ingested", r.TotalHits, + "backing indices", r.DataStreamStats.BackingIndices, + "store size bytes", r.DataStreamStats.StoreSizeBytes, + "maximum ts (ms)", r.DataStreamStats.MaximumTimestamp, + ) + "\n") + + for index, du := range r.DiskUsage { + adu := du.AllFields + report.WriteString(renderBenchmarkTable( + fmt.Sprintf("disk usage for index %s (for all fields)", index), + "total", adu.Total, + "inverted_index.total", adu.InvertedIndex.Total, + "inverted_index.stored_fields", adu.StoredFields, + "inverted_index.doc_values", adu.DocValues, + "inverted_index.points", adu.Points, + "inverted_index.norms", adu.Norms, + "inverted_index.term_vectors", adu.TermVectors, + "inverted_index.knn_vectors", adu.KnnVectors, + ) + "\n") + } + + for node, pStats := range r.IngestPipelineStats { + for pipeline, stats := range pStats { + var kvs []interface{} + for _, procStats := range stats.Processors { + str := fmt.Sprintf( + "Count: %d | Failed: %d | Time: %s", + procStats.Stats.Count, + procStats.Stats.Failed, + time.Duration(procStats.Stats.TimeInMillis)*time.Millisecond, + ) + kvs = append(kvs, fmt.Sprintf("%s (%s)", procStats.Type, procStats.Extra), str) + } + report.WriteString(renderBenchmarkTable( + fmt.Sprintf("pipeline %s stats in node %s", pipeline, node), + kvs..., + ) + "\n") + } + } + + return []byte(report.String()) +} + +func renderBenchmarkTable(title string, kv ...interface{}) string { + t := table.NewWriter() + t.SetStyle(table.StyleRounded) + t.SetTitle(title) + t.SetColumnConfigs([]table.ColumnConfig{ + { + Number: 2, + Align: text.AlignRight, + }, + }) + for i := 0; i < len(kv)-1; i += 2 { + t.AppendRow(table.Row{kv[i], kv[i+1]}) + } + return t.Render() +} diff --git a/internal/benchrunner/runners/system/runner.go b/internal/benchrunner/runners/system/runner.go new file mode 100644 index 000000000..9f0f00a3e --- /dev/null +++ b/internal/benchrunner/runners/system/runner.go @@ -0,0 +1,779 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package system + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib" + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields" + + "github.com/elastic/elastic-package/internal/benchrunner" + "github.com/elastic/elastic-package/internal/benchrunner/reporters" + "github.com/elastic/elastic-package/internal/benchrunner/runners/system/servicedeployer" + "github.com/elastic/elastic-package/internal/configuration/locations" + "github.com/elastic/elastic-package/internal/elasticsearch" + "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/signal" +) + +const ( + // ServiceLogsAgentDir is folder path where log files produced by the service + // are stored on the Agent container's filesystem. + ServiceLogsAgentDir = "/tmp/service_logs" + + waitForDataDefaultTimeout = 10 * time.Minute +) + +const ( + // BenchType defining system benchmark + BenchType benchrunner.Type = "system" +) + +type runner struct { + options Options + scenario *scenario + + ctxt servicedeployer.ServiceContext + benchPolicy *kibana.Policy + runtimeDataStream string + pipelinePrefix string + generator genlib.Generator + mcollector *collector + corporaFile string + + // Execution order of following handlers is defined in runner.TearDown() method. + deletePolicyHandler func() error + resetAgentPolicyHandler func() error + shutdownServiceHandler func() error + wipeDataStreamHandler func() error + clearCorporaHandler func() error +} + +func NewSystemBenchmark(opts Options) benchrunner.Runner { + return &runner{options: opts} +} + +func (r *runner) SetUp() error { + return r.setUp() +} + +// Run runs the system benchmarks defined under the given folder +func (r *runner) Run() (reporters.Reportable, error) { + return r.run() +} + +func (r *runner) TearDown() error { + if r.options.DeferCleanup > 0 { + logger.Debugf("waiting for %s before tearing down...", r.options.DeferCleanup) + signal.Sleep(r.options.DeferCleanup) + } + + var merr multierror.Error + + if r.resetAgentPolicyHandler != nil { + if err := r.resetAgentPolicyHandler(); err != nil { + merr = append(merr, err) + } + r.resetAgentPolicyHandler = nil + } + + if r.deletePolicyHandler != nil { + if err := r.deletePolicyHandler(); err != nil { + merr = append(merr, err) + } + r.deletePolicyHandler = nil + } + + if r.shutdownServiceHandler != nil { + if err := r.shutdownServiceHandler(); err != nil { + merr = append(merr, err) + } + r.shutdownServiceHandler = nil + } + + if r.wipeDataStreamHandler != nil { + if err := r.wipeDataStreamHandler(); err != nil { + merr = append(merr, err) + } + r.wipeDataStreamHandler = nil + } + + if r.clearCorporaHandler != nil { + if err := r.clearCorporaHandler(); err != nil { + merr = append(merr, err) + } + r.clearCorporaHandler = nil + } + + if len(merr) == 0 { + return nil + } + return merr +} + +func (r *runner) setUp() error { + locationManager, err := locations.NewLocationManager() + if err != nil { + return fmt.Errorf("reading service logs directory failed: %w", err) + } + + serviceLogsDir := locationManager.ServiceLogDir() + r.ctxt.Logs.Folder.Local = serviceLogsDir + r.ctxt.Logs.Folder.Agent = ServiceLogsAgentDir + r.ctxt.Bench.RunID = createRunID() + + scenario, err := readConfig(r.options.PackageRootPath, r.options.BenchName, r.ctxt) + if err != nil { + return err + } + r.scenario = scenario + + if r.scenario.Corpora.Generator != nil { + var err error + r.generator, err = r.initializeGenerator() + if err != nil { + return fmt.Errorf("can't initialize generator: %w", err) + } + } + + pkgManifest, err := packages.ReadPackageManifestFromPackageRoot(r.options.PackageRootPath) + if err != nil { + return fmt.Errorf("reading package manifest failed: %w", err) + } + + policy, err := r.createBenchmarkPolicy(pkgManifest) + if err != nil { + return err + } + r.benchPolicy = policy + + // Delete old data + logger.Debug("deleting old data in data stream...") + dataStreamManifest, err := packages.ReadDataStreamManifest( + filepath.Join( + getDataStreamPath(r.options.PackageRootPath, r.scenario.DataStream.Name), + packages.DataStreamManifestFile, + ), + ) + if err != nil { + return fmt.Errorf("reading data stream manifest failed: %w", err) + } + + r.runtimeDataStream = fmt.Sprintf( + "%s-%s.%s-%s", + dataStreamManifest.Type, + pkgManifest.Name, + dataStreamManifest.Name, + policy.Namespace, + ) + r.pipelinePrefix = fmt.Sprintf( + "%s-%s.%s-%s", + dataStreamManifest.Type, + pkgManifest.Name, + dataStreamManifest.Name, + r.scenario.Version, + ) + + r.wipeDataStreamHandler = func() error { + logger.Debugf("deleting data in data stream...") + if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + return fmt.Errorf("error deleting data in data stream: %w", err) + } + return nil + } + + if err := r.deleteDataStreamDocs(r.runtimeDataStream); err != nil { + return fmt.Errorf("error deleting old data in data stream: %s: %w", r.runtimeDataStream, err) + } + + cleared, err := waitUntilTrue(func() (bool, error) { + if signal.SIGINT() { + return true, errors.New("SIGINT: cancel clearing data") + } + + hits, err := getTotalHits(r.options.ESAPI, r.runtimeDataStream) + return hits == 0, err + }, 2*time.Minute) + if err != nil || !cleared { + if err == nil { + err = errors.New("unable to clear previous data") + } + return err + } + + return nil +} + +func (r *runner) run() (report reporters.Reportable, err error) { + var service servicedeployer.DeployedService + if r.scenario.Corpora.InputService != nil { + // Setup service. + logger.Debug("setting up service...") + serviceDeployer, err := servicedeployer.Factory(servicedeployer.FactoryOptions{ + RootPath: r.options.PackageRootPath, + }) + + if err != nil { + return nil, fmt.Errorf("could not create service runner: %w", err) + } + + r.ctxt.Name = r.scenario.Corpora.InputService.Name + service, err = serviceDeployer.SetUp(r.ctxt) + if err != nil { + return nil, fmt.Errorf("could not setup service: %w", err) + } + + r.ctxt = service.Context() + r.shutdownServiceHandler = func() error { + logger.Debug("tearing down service...") + if err := service.TearDown(); err != nil { + return fmt.Errorf("error tearing down service: %w", err) + } + + return nil + } + } + + r.startMetricsColletion() + + // if there is a generator config, generate the data + if r.generator != nil { + logger.Debugf("generating corpus data to %s...", r.ctxt.Logs.Folder.Local) + if err := r.runGenerator(r.ctxt.Logs.Folder.Local); err != nil { + return nil, fmt.Errorf("can't generate benchmarks data corpus for data stream: %w", err) + } + } + + // once data is generated, enroll agents and assign policy + if err := r.enrollAgents(); err != nil { + return nil, err + } + + // Signal to the service that the agent is ready (policy is assigned). + if r.scenario.Corpora.InputService != nil && r.scenario.Corpora.InputService.Signal != "" { + if err = service.Signal(r.scenario.Corpora.InputService.Signal); err != nil { + return nil, fmt.Errorf("failed to notify benchmark service: %w", err) + } + } + + if err := r.waitUntilBenchmarkFinishes(); err != nil { + return nil, err + } + + msum, err := r.collectAndSummarizeMetrics() + if err != nil { + return nil, fmt.Errorf("can't summarize metrics: %w", err) + } + + // TODO reindex if configured and es metricstore is set + + return createReport(r.options.BenchName, r.corporaFile, r.scenario, msum) +} + +func (r *runner) startMetricsColletion() { + // TODO send metrics to es metricstore if set + // TODO collect agent hosts metrics using system integration + r.mcollector = newCollector( + r.ctxt, + r.options.ESAPI, + r.options.MetricsInterval, + r.scenario.WarmupTimePeriod, + r.runtimeDataStream, + r.pipelinePrefix, + ) + r.mcollector.start() +} + +func (r *runner) collectAndSummarizeMetrics() (*metricsSummary, error) { + r.mcollector.stop() + sum, err := r.mcollector.summarize() + return sum, err +} + +func (r *runner) deleteDataStreamDocs(dataStream string) error { + body := strings.NewReader(`{ "query": { "match_all": {} } }`) + _, err := r.options.ESAPI.DeleteByQuery([]string{dataStream}, body) + if err != nil { + return err + } + return nil +} + +func (r *runner) createBenchmarkPolicy(pkgManifest *packages.PackageManifest) (*kibana.Policy, error) { + // Configure package (single data stream) via Ingest Manager APIs. + logger.Debug("creating benchmark policy...") + benchTime := time.Now().Format("20060102T15:04:05Z") + p := kibana.Policy{ + Name: fmt.Sprintf("ep-bench-%s-%s", r.options.BenchName, benchTime), + Description: fmt.Sprintf("policy created by elastic-package for benchmark %s", r.options.BenchName), + Namespace: "ep", + MonitoringEnabled: []string{"logs", "metrics"}, + } + + policy, err := r.options.KibanaClient.CreatePolicy(p) + if err != nil { + return nil, err + } + + packagePolicy, err := r.createPackagePolicy(pkgManifest, policy) + if err != nil { + return nil, err + } + + r.deletePolicyHandler = func() error { + var merr multierror.Error + + logger.Debug("deleting benchmark package policy...") + if err := r.options.KibanaClient.DeletePackagePolicy(*packagePolicy); err != nil { + merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err)) + } + + logger.Debug("deleting benchmark policy...") + if err := r.options.KibanaClient.DeletePolicy(*policy); err != nil { + merr = append(merr, fmt.Errorf("error cleaning up benchmark policy: %w", err)) + } + + if len(merr) > 0 { + return merr + } + + return nil + } + + return policy, nil +} + +func (r *runner) createPackagePolicy(pkgManifest *packages.PackageManifest, p *kibana.Policy) (*kibana.PackagePolicy, error) { + logger.Debug("creating package policy...") + + if r.scenario.Version == "" { + r.scenario.Version = pkgManifest.Version + } + + if r.scenario.Package == "" { + r.scenario.Package = pkgManifest.Name + } + + // TODO: add ability to define which policy template to use + pp := kibana.PackagePolicy{ + Namespace: "ep", + PolicyID: p.ID, + Vars: r.scenario.Vars, + Force: true, + Inputs: map[string]kibana.PackagePolicyInput{ + fmt.Sprintf("%s-%s", pkgManifest.PolicyTemplates[0].Name, r.scenario.Input): { + Enabled: true, + Streams: map[string]kibana.PackagePolicyStream{ + fmt.Sprintf("%s.%s", pkgManifest.Name, r.scenario.DataStream.Name): { + Enabled: true, + Vars: r.scenario.DataStream.Vars, + }, + }, + }, + }, + } + pp.Package.Name = pkgManifest.Name + pp.Package.Version = r.scenario.Version + + policy, err := r.options.KibanaClient.CreatePackagePolicy(pp) + if err != nil { + return nil, err + } + + return policy, nil +} + +func (r *runner) initializeGenerator() (genlib.Generator, error) { + totSizeInBytes, err := humanize.ParseBytes(r.scenario.Corpora.Generator.Size) + if err != nil { + return nil, err + } + + config, err := r.getGeneratorConfig() + if err != nil { + return nil, err + } + + fields, err := r.getGeneratorFields() + if err != nil { + return nil, err + } + + tpl, err := r.getGeneratorTemplate() + if err != nil { + return nil, err + } + + var generator genlib.Generator + switch r.scenario.Corpora.Generator.Template.Type { + default: + logger.Debugf("unknown generator template type %q, defaulting to \"placeholder\"", r.scenario.Corpora.Generator.Template.Type) + fallthrough + case "", "placeholder": + generator, err = genlib.NewGeneratorWithCustomTemplate(tpl, *config, fields, totSizeInBytes) + case "gotext": + generator, err = genlib.NewGeneratorWithTextTemplate(tpl, *config, fields, totSizeInBytes) + } + + if err != nil { + return nil, err + } + + return generator, nil +} + +func (r *runner) getGeneratorConfig() (*config.Config, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Config.Path != "" { + configPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Config.Path)) + configPath = os.ExpandEnv(configPath) + if _, err := os.Stat(configPath); err != nil { + return nil, fmt.Errorf("can't find config file %s: %w", configPath, err) + } + data, err = os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("can't read config file %s: %w", configPath, err) + } + } else if len(r.scenario.Corpora.Generator.Config.Raw) > 0 { + data, err = yaml.Marshal(r.scenario.Corpora.Generator.Config.Raw) + if err != nil { + return nil, fmt.Errorf("can't parse raw generator config: %w", err) + } + } + + cfg, err := config.LoadConfigFromYaml(data) + if err != nil { + return nil, fmt.Errorf("can't get generator config: %w", err) + } + + return &cfg, nil +} + +func (r *runner) getGeneratorFields() (fields.Fields, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Fields.Path != "" { + fieldsPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Fields.Path)) + fieldsPath = os.ExpandEnv(fieldsPath) + if _, err := os.Stat(fieldsPath); err != nil { + return nil, fmt.Errorf("can't find fields file %s: %w", fieldsPath, err) + } + + data, err = os.ReadFile(fieldsPath) + if err != nil { + return nil, fmt.Errorf("can't read fields file %s: %w", fieldsPath, err) + } + } else if len(r.scenario.Corpora.Generator.Fields.Raw) > 0 { + data, err = yaml.Marshal(r.scenario.Corpora.Generator.Config.Raw) + if err != nil { + return nil, fmt.Errorf("can't parse raw generator config: %w", err) + } + } + + fields, err := fields.LoadFieldsWithTemplateFromString(context.Background(), string(data)) + if err != nil { + return nil, fmt.Errorf("could not load fields yaml: %w", err) + } + + return fields, nil +} + +func (r *runner) getGeneratorTemplate() ([]byte, error) { + var ( + data []byte + err error + ) + + if r.scenario.Corpora.Generator.Template.Path != "" { + tplPath := filepath.Clean(filepath.Join(devPath, r.scenario.Corpora.Generator.Template.Path)) + tplPath = os.ExpandEnv(tplPath) + if _, err := os.Stat(tplPath); err != nil { + return nil, fmt.Errorf("can't find template file %s: %w", tplPath, err) + } + + data, err = os.ReadFile(tplPath) + if err != nil { + return nil, fmt.Errorf("can't read template file %s: %w", tplPath, err) + } + } else if len(r.scenario.Corpora.Generator.Template.Raw) > 0 { + data = []byte(r.scenario.Corpora.Generator.Template.Raw) + } + + return data, nil +} + +func (r *runner) runGenerator(destDir string) error { + state := genlib.NewGenState() + + f, err := os.CreateTemp(destDir, "corpus-*") + if err != nil { + return err + } + defer f.Close() + + buf := bytes.NewBufferString("") + var corpusDocsCount uint64 + for { + err := r.generator.Emit(state, buf) + if err == io.EOF { + break + } + + if err != nil { + return err + } + + // TODO: this should be taken care of by the corpus generator tool, once it will be done let's remove this + replacer := strings.NewReplacer("\n", "") + event := replacer.Replace(buf.String()) + if _, err = f.Write([]byte(event)); err != nil { + return err + } + + if _, err = f.Write([]byte("\n")); err != nil { + return err + } + + buf.Reset() + corpusDocsCount += 1 + } + + r.corporaFile = f.Name() + r.clearCorporaHandler = func() error { + return os.Remove(r.corporaFile) + } + + return r.generator.Close() +} + +func (r *runner) checkEnrolledAgents() ([]kibana.Agent, error) { + var agents []kibana.Agent + enrolled, err := waitUntilTrue(func() (bool, error) { + if signal.SIGINT() { + return false, errors.New("SIGINT: cancel checking enrolled agents") + } + allAgents, err := r.options.KibanaClient.ListAgents() + if err != nil { + return false, fmt.Errorf("could not list agents: %w", err) + } + + agents = filterAgents(allAgents) + if len(agents) == 0 { + return false, nil // selected agents are unavailable yet + } + + return true, nil + }, 5*time.Minute) + if err != nil { + return nil, fmt.Errorf("agent enrollment failed: %w", err) + } + if !enrolled { + return nil, errors.New("no agent enrolled in time") + } + return agents, nil +} + +func (r *runner) waitUntilBenchmarkFinishes() error { + logger.Debug("checking for all data in data stream...") + var benchTime *time.Timer + if r.scenario.BenchmarkTimePeriod > 0 { + benchTime = time.NewTimer(r.scenario.BenchmarkTimePeriod) + } + waitForDataTimeout := waitForDataDefaultTimeout + if r.scenario.WaitForDataTimeout > 0 { + waitForDataTimeout = r.scenario.WaitForDataTimeout + } + + oldHits := 0 + _, err := waitUntilTrue(func() (bool, error) { + if signal.SIGINT() { + return true, errors.New("SIGINT: cancel waiting for policy assigned") + } + + var err error + hits, err := getTotalHits(r.options.ESAPI, r.runtimeDataStream) + if hits == 0 { + return false, err + } + + ret := hits == oldHits + if hits != oldHits { + oldHits = hits + } + + if benchTime != nil { + select { + case <-benchTime.C: + return true, err + default: + return false, err + } + } + + return ret, err + }, waitForDataTimeout) + return err +} + +func (r *runner) enrollAgents() error { + agents, err := r.checkEnrolledAgents() + if err != nil { + return fmt.Errorf("can't check enrolled agents: %w", err) + } + + handlers := make([]func() error, len(agents)) + for i, agent := range agents { + origPolicy := kibana.Policy{ + ID: agent.PolicyID, + Revision: agent.PolicyRevision, + } + + // Assign policy to agent + handlers[i] = func() error { + logger.Debug("reassigning original policy back to agent...") + if err := r.options.KibanaClient.AssignPolicyToAgent(agent, origPolicy); err != nil { + return fmt.Errorf("error reassigning original policy to agent %s: %w", agent.ID, err) + } + return nil + } + + policyWithDataStream, err := r.options.KibanaClient.GetPolicy(r.benchPolicy.ID) + if err != nil { + return fmt.Errorf("could not read the policy with data stream: %w", err) + } + + logger.Debug("assigning package data stream to agent...") + if err := r.options.KibanaClient.AssignPolicyToAgent(agent, *policyWithDataStream); err != nil { + return fmt.Errorf("could not assign policy to agent: %w", err) + } + } + + r.resetAgentPolicyHandler = func() error { + var merr multierror.Error + for _, h := range handlers { + if err := h(); err != nil { + merr = append(merr, err) + } + } + if len(merr) == 0 { + return nil + } + return merr + } + + return nil +} + +func getTotalHits(esapi *elasticsearch.API, dataStream string) (int, error) { + resp, err := esapi.Count( + esapi.Count.WithIndex(dataStream), + ) + if err != nil { + return 0, fmt.Errorf("could not search data stream: %w", err) + } + defer resp.Body.Close() + + var results struct { + Count int + Error *struct { + Type string + Reason string + } + Status int + } + + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return 0, fmt.Errorf("could not decode search results response: %w", err) + } + + numHits := results.Count + if results.Error != nil { + logger.Debugf("found %d hits in %s data stream: %s: %s Status=%d", + numHits, dataStream, results.Error.Type, results.Error.Reason, results.Status) + } else { + logger.Debugf("found %d hits in %s data stream", numHits, dataStream) + } + + return numHits, nil +} + +func filterAgents(allAgents []kibana.Agent) []kibana.Agent { + var filtered []kibana.Agent + for _, agent := range allAgents { + if agent.PolicyRevision == 0 { + // For some reason Kibana doesn't always return + // a valid policy revision (eventually it will be present and valid) + continue + } + + // best effort to ignore fleet server agents + switch { + case agent.LocalMetadata.Host.Name == "docker-fleet-server", + agent.PolicyID == "fleet-server-policy", + agent.PolicyID == "Elastic Cloud agent policy": + continue + } + filtered = append(filtered, agent) + } + return filtered +} + +func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error) { + timeoutTicker := time.NewTicker(timeout) + defer timeoutTicker.Stop() + + retryTicker := time.NewTicker(5 * time.Second) + defer retryTicker.Stop() + + for { + result, err := fn() + if err != nil { + return false, err + } + if result { + return true, nil + } + + select { + case <-retryTicker.C: + continue + case <-timeoutTicker.C: + return false, nil + } + } +} + +func createRunID() string { + return uuid.New().String() +} + +func getDataStreamPath(packageRoot, dataStream string) string { + return filepath.Join(packageRoot, "data_stream", dataStream) +} diff --git a/internal/benchrunner/runners/system/scenario.go b/internal/benchrunner/runners/system/scenario.go new file mode 100644 index 000000000..be1010170 --- /dev/null +++ b/internal/benchrunner/runners/system/scenario.go @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package system + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/aymerick/raymond" + "github.com/elastic/go-ucfg" + "github.com/elastic/go-ucfg/yaml" + + "github.com/elastic/elastic-package/internal/benchrunner/runners/system/servicedeployer" +) + +const devPath = "_dev/benchmark/system" + +type scenario struct { + Package string `config:"package"` + Description string `config:"description"` + Version string `config:"version"` + Input string `config:"input"` + Vars map[string]interface{} `config:"vars"` + DataStream dataStream `config:"data_stream"` + WarmupTimePeriod time.Duration `config:"warmup_time_period"` + BenchmarkTimePeriod time.Duration `config:"benchmark_time_period"` + WaitForDataTimeout time.Duration `config:"wait_for_data_timeout"` + Corpora corpora `config:"corpora"` +} + +type dataStream struct { + Name string `config:"name"` + Vars map[string]interface{} `config:"vars"` +} + +type corpora struct { + Generator *generator `config:"generator"` + InputService *inputService `config:"input_service"` +} + +type inputService struct { + Name string `config:"name"` + Signal string `config:"signal"` +} + +type generator struct { + Size string `config:"size"` + Template corporaTemplate `config:"template"` + Config corporaConfig `config:"config"` + Fields corporaFields `config:"fields"` +} + +type corporaTemplate struct { + Raw string `config:"raw"` + Path string `config:"path"` + Type string `config:"type"` +} + +type corporaConfig struct { + Raw map[string]interface{} `config:"raw"` + Path string `config:"path"` +} + +type corporaFields struct { + Raw map[string]interface{} `config:"raw"` + Path string `config:"path"` +} + +func defaultConfig() *scenario { + return &scenario{} +} + +func readConfig(path, scenario string, ctxt servicedeployer.ServiceContext) (*scenario, error) { + configPath := filepath.Join(path, devPath, fmt.Sprintf("%s.yml", scenario)) + data, err := os.ReadFile(configPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("unable to find system benchmark configuration file: %s: %w", configPath, err) + } + return nil, fmt.Errorf("could not load system benchmark configuration file: %s: %w", configPath, err) + } + + data, err = applyContext(data, ctxt) + if err != nil { + return nil, fmt.Errorf("could not apply context to benchmark configuration file: %s: %w", configPath, err) + } + + cfg, err := yaml.NewConfig(data, ucfg.PathSep(".")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + configPath = filepath.Join(path, devPath, fmt.Sprintf("%s.yaml", scenario)) + cfg, err = yaml.NewConfigWithFile(configPath) + } + if err != nil { + return nil, fmt.Errorf("can't load scenario: %s: %w", configPath, err) + } + } + + c := defaultConfig() + if err := cfg.Unpack(c); err != nil { + return nil, fmt.Errorf("can't unpack scenario configuration: %s: %w", configPath, err) + } + return c, nil +} + +// applyContext takes the given system benchmark configuration (data) and replaces any placeholder variables in +// it with values from the given context (ctxt). The context may be populated from various sources but usually the +// most interesting context values will be set by a ServiceDeployer in its SetUp method. +func applyContext(data []byte, ctxt servicedeployer.ServiceContext) ([]byte, error) { + tmpl, err := raymond.Parse(string(data)) + if err != nil { + return data, fmt.Errorf("parsing template body failed: %w", err) + } + tmpl.RegisterHelpers(ctxt.Aliases()) + + result, err := tmpl.Exec(ctxt) + if err != nil { + return data, fmt.Errorf("could not render data with context: %w", err) + } + return []byte(result), nil +} diff --git a/internal/benchrunner/runners/system/servicedeployer/compose.go b/internal/benchrunner/runners/system/servicedeployer/compose.go new file mode 100644 index 000000000..0b3dd5e2c --- /dev/null +++ b/internal/benchrunner/runners/system/servicedeployer/compose.go @@ -0,0 +1,215 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/elastic/elastic-package/internal/builder" + "github.com/elastic/elastic-package/internal/compose" + "github.com/elastic/elastic-package/internal/docker" + "github.com/elastic/elastic-package/internal/files" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/stack" +) + +// DockerComposeServiceDeployer knows how to deploy a service defined via +// a Docker Compose file. +type DockerComposeServiceDeployer struct { + ymlPaths []string +} + +type dockerComposeDeployedService struct { + ctxt ServiceContext + + ymlPaths []string + project string +} + +// NewDockerComposeServiceDeployer returns a new instance of a DockerComposeServiceDeployer. +func NewDockerComposeServiceDeployer(ymlPaths []string) (*DockerComposeServiceDeployer, error) { + return &DockerComposeServiceDeployer{ + ymlPaths: ymlPaths, + }, nil +} + +// SetUp sets up the service and returns any relevant information. +func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedService, error) { + logger.Debug("setting up service using Docker Compose service deployer") + service := dockerComposeDeployedService{ + ymlPaths: d.ymlPaths, + project: "elastic-package-service", + } + outCtxt := inCtxt + + p, err := compose.NewProject(service.project, service.ymlPaths...) + if err != nil { + return nil, fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + // Verify the Elastic stack network + err = stack.EnsureStackNetworkUp() + if err != nil { + return nil, fmt.Errorf("elastic stack network is not ready: %w", err) + } + + // Clean service logs + err = files.RemoveContent(outCtxt.Logs.Folder.Local) + if err != nil { + return nil, fmt.Errorf("removing service logs failed: %w", err) + } + + serviceName := inCtxt.Name + opts := compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", ServiceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + ExtraArgs: []string{"--build", "-d"}, + } + err = p.Up(opts) + if err != nil { + return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err) + } + err = p.WaitForHealthy(opts) + if err != nil { + processServiceContainerLogs(p, compose.CommandOptions{ + Env: opts.Env, + }, outCtxt.Name) + return nil, fmt.Errorf("service is unhealthy: %w", err) + } + + // Build service container name + outCtxt.Hostname = p.ContainerName(serviceName) + + // Connect service network with stack network (for the purpose of metrics collection) + err = docker.ConnectToNetwork(p.ContainerName(serviceName), stack.Network()) + if err != nil { + return nil, fmt.Errorf("can't attach service container to the stack network: %w", err) + } + + logger.Debugf("adding service container %s internal ports to context", p.ContainerName(serviceName)) + serviceComposeConfig, err := p.Config(compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", ServiceLogsDirEnv, outCtxt.Logs.Folder.Local)}, + }) + if err != nil { + return nil, fmt.Errorf("could not get Docker Compose configuration for service: %w", err) + } + + s := serviceComposeConfig.Services[serviceName] + outCtxt.Ports = make([]int, len(s.Ports)) + for idx, port := range s.Ports { + outCtxt.Ports[idx] = port.InternalPort + } + + // Shortcut to first port for convenience + if len(outCtxt.Ports) > 0 { + outCtxt.Port = outCtxt.Ports[0] + } + + outCtxt.Agent.Host.NamePrefix = "docker-fleet-agent" + service.ctxt = outCtxt + return &service, nil +} + +// Signal sends a signal to the service. +func (s *dockerComposeDeployedService) Signal(signal string) error { + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", ServiceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, + ExtraArgs: []string{"-s", signal}, + } + if s.ctxt.Name != "" { + opts.Services = append(opts.Services, s.ctxt.Name) + } + + if err := p.Kill(opts); err != nil { + return fmt.Errorf("could not send %q signal: %w", signal, err) + } + return nil +} + +// TearDown tears down the service. +func (s *dockerComposeDeployedService) TearDown() error { + logger.Debugf("tearing down service using Docker Compose runner") + defer func() { + err := files.RemoveContent(s.ctxt.Logs.Folder.Local) + if err != nil { + logger.Errorf("could not remove the service logs (path: %s)", s.ctxt.Logs.Folder.Local) + } + }() + + p, err := compose.NewProject(s.project, s.ymlPaths...) + if err != nil { + return fmt.Errorf("could not create Docker Compose project for service: %w", err) + } + + opts := compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", ServiceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, + } + processServiceContainerLogs(p, opts, s.ctxt.Name) + + if err := p.Down(compose.CommandOptions{ + Env: []string{fmt.Sprintf("%s=%s", ServiceLogsDirEnv, s.ctxt.Logs.Folder.Local)}, + ExtraArgs: []string{"--volumes"}, // Remove associated volumes. + }); err != nil { + return fmt.Errorf("could not shut down service using Docker Compose: %w", err) + } + return nil +} + +// Context returns the current context for the service. +func (s *dockerComposeDeployedService) Context() ServiceContext { + return s.ctxt +} + +// SetContext sets the current context for the service. +func (s *dockerComposeDeployedService) SetContext(ctxt ServiceContext) error { + s.ctxt = ctxt + return nil +} + +func processServiceContainerLogs(p *compose.Project, opts compose.CommandOptions, serviceName string) { + content, err := p.Logs(opts) + if err != nil { + logger.Errorf("can't export service logs: %v", err) + return + } + + if len(content) == 0 { + logger.Info("service container hasn't written anything logs.") + return + } + + err = writeServiceContainerLogs(serviceName, content) + if err != nil { + logger.Errorf("can't write service container logs: %v", err) + } +} + +func writeServiceContainerLogs(serviceName string, content []byte) error { + buildDir, err := builder.BuildDirectory() + if err != nil { + return fmt.Errorf("locating build directory failed: %w", err) + } + + containerLogsDir := filepath.Join(buildDir, "container-logs") + err = os.MkdirAll(containerLogsDir, 0755) + if err != nil { + return fmt.Errorf("can't create directory for service container logs (path: %s): %w", containerLogsDir, err) + } + + containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("%s-%d.log", serviceName, time.Now().UnixNano())) + logger.Infof("Write container logs to file: %s", containerLogsFilepath) + err = os.WriteFile(containerLogsFilepath, content, 0644) + if err != nil { + return fmt.Errorf("can't write container logs to file (path: %s): %w", containerLogsFilepath, err) + } + return nil +} diff --git a/internal/benchrunner/runners/system/servicedeployer/context.go b/internal/benchrunner/runners/system/servicedeployer/context.go new file mode 100644 index 000000000..d9c0aabf5 --- /dev/null +++ b/internal/benchrunner/runners/system/servicedeployer/context.go @@ -0,0 +1,83 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +const ( + LocalCACertEnv = "LOCAL_CA_CERT" + ServiceLogsDirEnv = "SERVICE_LOGS_DIR" + BenchRunIDEnv = "BENCH_RUN_ID" +) + +// ServiceContext encapsulates context that is both available to a ServiceDeployer and +// populated by a DeployedService. The fields in ServiceContext may be used in handlebars +// templates in system benchmark configuration files, for example: {{ Hostname }}. +type ServiceContext struct { + // Name is the name of the service. + Name string + + // Hostname is the host name of the service, as addressable from + // the Agent container. + Hostname string + + // Ports is a list of ports that the service listens on, as addressable + // from the Agent container. + Ports []int + + // Port points to the first port in the list of ports. It's provided as + // a convenient shortcut as most services tend to listen on a single port. + Port int + + // Logs contains folder paths for log files produced by the service. + Logs struct { + Folder struct { + // Local contains the folder path where log files produced by + // the service are stored on the local filesystem, i.e. where + // elastic-package is running. + Local string + + // Agent contains the folder path where log files produced by + // the service are stored on the Agent container's filesystem. + Agent string + } + } + + // Bench related properties. + Bench struct { + // RunID identifies the current benchmark run. + RunID string + } + + // Agent related properties. + Agent struct { + // Host describes the machine which is running the agent. + Host struct { + // Name prefix for the host's name + NamePrefix string + } + } + + // CustomProperties store additional data used to boot up the service, e.g. AWS credentials. + CustomProperties map[string]interface{} +} + +// Aliases method returned aliases to properties of the service context. +func (sc *ServiceContext) Aliases() map[string]interface{} { + m := map[string]interface{}{ + ServiceLogsDirEnv: func() interface{} { + return sc.Logs.Folder.Agent + }, + BenchRunIDEnv: func() interface{} { + return sc.Bench.RunID + }, + } + + for k, v := range sc.CustomProperties { + var that = v + m[k] = func() interface{} { // wrap as function + return that + } + } + return m +} diff --git a/internal/benchrunner/runners/system/servicedeployer/deployed_service.go b/internal/benchrunner/runners/system/servicedeployer/deployed_service.go new file mode 100644 index 000000000..ebd1a87f9 --- /dev/null +++ b/internal/benchrunner/runners/system/servicedeployer/deployed_service.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +// DeployedService defines the interface for interacting with a service that has been deployed. +type DeployedService interface { + // TearDown implements the logic for tearing down a service. + TearDown() error + + // Signal sends a signal to the service. + Signal(signal string) error + + // Context returns the current context from the service. + Context() ServiceContext + + // SetContext sets the current context for the service. + SetContext(str ServiceContext) error +} diff --git a/internal/benchrunner/runners/system/servicedeployer/factory.go b/internal/benchrunner/runners/system/servicedeployer/factory.go new file mode 100644 index 000000000..b7b7145d8 --- /dev/null +++ b/internal/benchrunner/runners/system/servicedeployer/factory.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +const devDeployDir = "_dev/benchmark/system/deploy" + +// FactoryOptions defines options used to create an instance of a service deployer. +type FactoryOptions struct { + RootPath string +} + +// Factory chooses the appropriate service runner for the given data stream, depending +// on service configuration files defined in the package or data stream. +func Factory(options FactoryOptions) (ServiceDeployer, error) { + devDeployPath, err := FindDevDeployPath(options) + if err != nil { + return nil, fmt.Errorf("can't find \"%s\" directory: %w", devDeployDir, err) + } + + serviceDeployerName, err := findServiceDeployer(devDeployPath) + if err != nil { + return nil, fmt.Errorf("can't find any valid service deployer: %w", err) + } + + serviceDeployerPath := filepath.Join(devDeployPath, serviceDeployerName) + + switch serviceDeployerName { + case "docker": + dockerComposeYMLPath := filepath.Join(serviceDeployerPath, "docker-compose.yml") + if _, err := os.Stat(dockerComposeYMLPath); err == nil { + return NewDockerComposeServiceDeployer([]string{dockerComposeYMLPath}) + } + } + return nil, fmt.Errorf("unsupported service deployer (name: %s)", serviceDeployerName) +} + +// FindDevDeployPath function returns a path reference to the "_dev/deploy" directory. +func FindDevDeployPath(options FactoryOptions) (string, error) { + path := filepath.Join(options.RootPath, devDeployDir) + if _, err := os.Stat(path); err == nil { + return path, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("stat failed for path (path: %s): %w", path, err) + } + return "", fmt.Errorf("\"%s\" directory doesn't exist", devDeployDir) +} + +func findServiceDeployer(devDeployPath string) (string, error) { + fis, err := os.ReadDir(devDeployPath) + if err != nil { + return "", fmt.Errorf("can't read directory (path: %s): %w", devDeployDir, err) + } + + var folders []os.DirEntry + for _, fi := range fis { + if fi.IsDir() { + folders = append(folders, fi) + } + } + + if len(folders) != 1 { + return "", fmt.Errorf("expected to find only one service deployer in \"%s\"", devDeployPath) + } + return folders[0].Name(), nil +} diff --git a/internal/benchrunner/runners/system/servicedeployer/service_deployer.go b/internal/benchrunner/runners/system/servicedeployer/service_deployer.go new file mode 100644 index 000000000..5e1cb93af --- /dev/null +++ b/internal/benchrunner/runners/system/servicedeployer/service_deployer.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package servicedeployer + +// ServiceDeployer defines the interface for deploying a service. It defines methods for +// controlling the lifecycle of a service. +type ServiceDeployer interface { + // SetUp implements the logic for setting up a service. It takes a context and returns a + // ServiceHandler. + SetUp(ctxt ServiceContext) (DeployedService, error) +} diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 3a4e1203c..a7eb785ee 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -29,9 +29,18 @@ const ( AllowSnapshotFlagName = "allow-snapshot" AllowSnapshotDescription = "allow to export dashboards from a Elastic stack SNAPSHOT version" + BenchNameFlagName = "benchmark" + BenchNameFlagDescription = "name of the benchmark scenario to run" + BenchNumTopProcsFlagName = "num-top-procs" BenchNumTopProcsFlagDescription = "number of top processors to show in the benchmarks results" + BenchMetricsIntervalFlagName = "metrics-collection-interval" + BenchMetricsIntervalFlagDescription = "the interval at which metrics are collected" + + BenchReindexToMetricstoreFlagName = "reindex-to-metricstore" + BenchReindexToMetricstoreFlagDescription = "if set the documents from the benchmark will be reindexed to the metricstore for posterior analysis" + BenchReportNewPathFlagName = "new" BenchReportNewPathFlagDescription = "path of the directory containing the new benchmarks of the report" diff --git a/internal/elasticsearch/ingest/datastream.go b/internal/elasticsearch/ingest/datastream.go index e2283978c..aebd1add0 100644 --- a/internal/elasticsearch/ingest/datastream.go +++ b/internal/elasticsearch/ingest/datastream.go @@ -16,8 +16,6 @@ import ( "strings" "time" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/elasticsearch" "github.com/elastic/elastic-package/internal/packages" ) @@ -27,7 +25,7 @@ var ingestPipelineTag = regexp.MustCompile(`{{\s*IngestPipeline.+}}`) func InstallDataStreamPipelines(api *elasticsearch.API, dataStreamPath string) (string, []Pipeline, error) { dataStreamManifest, err := packages.ReadDataStreamManifest(filepath.Join(dataStreamPath, packages.DataStreamManifestFile)) if err != nil { - return "", nil, errors.Wrap(err, "reading data stream manifest failed") + return "", nil, fmt.Errorf("reading data stream manifest failed: %w", err) } nonce := time.Now().UnixNano() @@ -35,7 +33,7 @@ func InstallDataStreamPipelines(api *elasticsearch.API, dataStreamPath string) ( mainPipeline := getPipelineNameWithNonce(dataStreamManifest.GetPipelineNameOrDefault(), nonce) pipelines, err := loadIngestPipelineFiles(dataStreamPath, nonce) if err != nil { - return "", nil, errors.Wrap(err, "loading ingest pipeline files failed") + return "", nil, fmt.Errorf("loading ingest pipeline files failed: %w", err) } err = installPipelinesInElasticsearch(api, pipelines) @@ -52,7 +50,7 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er for _, pattern := range []string{"*.json", "*.yml"} { files, err := filepath.Glob(filepath.Join(elasticsearchPath, pattern)) if err != nil { - return nil, errors.Wrapf(err, "listing '%s' in '%s'", pattern, elasticsearchPath) + return nil, fmt.Errorf("listing '%s' in '%s': %w", pattern, elasticsearchPath, err) } pipelineFiles = append(pipelineFiles, files...) } @@ -61,7 +59,7 @@ func loadIngestPipelineFiles(dataStreamPath string, nonce int64) ([]Pipeline, er for _, path := range pipelineFiles { c, err := os.ReadFile(path) if err != nil { - return nil, errors.Wrapf(err, "reading ingest pipeline failed (path: %s)", path) + return nil, fmt.Errorf("reading ingest pipeline failed (path: %s): %w", path, err) } c = ingestPipelineTag.ReplaceAllFunc(c, func(found []byte) []byte { @@ -98,7 +96,8 @@ func pipelineError(err error, pipeline Pipeline, format string, args ...interfac context += ", path: " + pipeline.Path } - return errors.Wrapf(err, format+" ("+context+")", args...) + errorStr := fmt.Sprintf(format+" ("+context+")", args...) + return fmt.Errorf("%s: %w", errorStr, err) } func installPipeline(api *elasticsearch.API, pipeline Pipeline) error { diff --git a/internal/elasticsearch/ingest/datastreamstats.go b/internal/elasticsearch/ingest/datastreamstats.go new file mode 100644 index 000000000..664af9e79 --- /dev/null +++ b/internal/elasticsearch/ingest/datastreamstats.go @@ -0,0 +1,53 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ingest + +import ( + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/elastic/elastic-package/internal/elasticsearch" +) + +type DataStreamsStats struct { + DataStreams []DataStreamStats `json:"data_streams"` +} + +type DataStreamStats struct { + DataStream string `json:"data_stream"` + BackingIndices int `json:"backing_indices"` + StoreSizeBytes int `json:"store_size_bytes"` + MaximumTimestamp int `json:"maximum_timestamp"` +} + +func GetDataStreamStats(esClient *elasticsearch.API, datastream string) (*DataStreamStats, error) { + req := esClient.Indices.DataStreamsStats.WithName(datastream) + resp, err := esClient.Indices.DataStreamsStats(req) + if err != nil { + return nil, fmt.Errorf("failed call to DataStream Stats API: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read Stats API response body: %w", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected response status for DataStream Stats (%d): %s: %w", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) + } + + var statsResponse DataStreamsStats + if err = json.Unmarshal(body, &statsResponse); err != nil { + return nil, fmt.Errorf("error decoding DataStream Stats response: %w", err) + } + if len(statsResponse.DataStreams) > 0 { + return &statsResponse.DataStreams[0], nil + } + + return nil, errors.New("couldn't get DataStream stats") +} diff --git a/internal/elasticsearch/ingest/diskusage.go b/internal/elasticsearch/ingest/diskusage.go new file mode 100644 index 000000000..b6be4f644 --- /dev/null +++ b/internal/elasticsearch/ingest/diskusage.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package ingest + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/elastic/elastic-package/internal/elasticsearch" +) + +type DiskUsageStat struct { + Total string `json:"total"` + TotalInBytes int `json:"total_in_bytes"` + InvertedIndex struct { + Total string `json:"total"` + TotalInBytes int `json:"total_in_bytes"` + } `json:"inverted_index"` + StoredFields string `json:"stored_fields"` + StoredFieldsInBytes int `json:"stored_fields_in_bytes"` + DocValues string `json:"doc_values"` + DocValuesInBytes int `json:"doc_values_in_bytes"` + Points string `json:"points"` + PointsInBytes int `json:"points_in_bytes"` + Norms string `json:"norms"` + NormsInBytes int `json:"norms_in_bytes"` + TermVectors string `json:"term_vectors"` + TermVectorsInBytes int `json:"term_vectors_in_bytes"` + KnnVectors string `json:"knn_vectors"` + KnnVectorsInBytes int `json:"knn_vectors_in_bytes"` +} + +type DiskUsage struct { + StoreSize string `json:"store_size"` + StoreSizeInBytes int `json:"store_size_in_bytes"` + AllFields DiskUsageStat `json:"all_fields"` + Fields map[string]DiskUsageStat `json:"fields"` +} + +func GetDiskUsage(esClient *elasticsearch.API, datastream string) (map[string]DiskUsage, error) { + resp, err := esClient.Indices.DiskUsage(datastream, + esClient.Indices.DiskUsage.WithFlush(true), + esClient.Indices.DiskUsage.WithRunExpensiveTasks(true), + ) + if err != nil { + return nil, fmt.Errorf("DiskUsage Stats API call failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read Stats API response body: %w", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected response status for DiskUsage Stats (%d): %s: %w", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) + } + + var stats map[string]DiskUsage + if err = json.Unmarshal(body, &stats); err != nil { + return nil, fmt.Errorf("error decoding stats response: %w", err) + } + delete(stats, "_shards") + return stats, nil +} diff --git a/internal/elasticsearch/ingest/nodestats.go b/internal/elasticsearch/ingest/nodestats.go index b26282a3e..f827759aa 100644 --- a/internal/elasticsearch/ingest/nodestats.go +++ b/internal/elasticsearch/ingest/nodestats.go @@ -6,11 +6,10 @@ package ingest import ( "encoding/json" + "fmt" "io" "strings" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/elasticsearch" ) @@ -47,7 +46,7 @@ func (p wrappedProcessor) extract() (stats ProcessorStats, err error) { for k := range p { keys = append(keys, k) } - return stats, errors.Errorf("can't extract processor stats. Need a single key in the processor identifier, got %d: %v", len(p), keys) + return stats, fmt.Errorf("can't extract processor stats. Need a single key in the processor identifier, got %d: %v", len(p), keys) } // Read single entry in map. @@ -65,7 +64,7 @@ func (p wrappedProcessor) extract() (stats ProcessorStats, err error) { case "conditional": stats.Conditional = true default: - return stats, errors.Errorf("can't understand processor identifier '%s' in %+v", processorType, p) + return stats, fmt.Errorf("can't understand processor identifier '%s' in %+v", processorType, p) } stats.Type = processorType @@ -84,7 +83,7 @@ func (r pipelineStatsRecord) extract() (stats PipelineStats, err error) { } for idx, wrapped := range r.Processors { if stats.Processors[idx], err = wrapped.extract(); err != nil { - return stats, errors.Wrapf(err, "extracting processor %d", idx) + return stats, fmt.Errorf("extracting processor %d: %w", idx, err) } } return stats, nil @@ -103,32 +102,72 @@ type pipelinesStatsResponse struct { } func GetPipelineStats(esClient *elasticsearch.API, pipelines []Pipeline) (stats PipelineStatsMap, err error) { + statsResponse, err := requestPipelineStats(esClient) + if err != nil { + return nil, err + } + return getPipelineStats(statsResponse, pipelines) +} + +func GetPipelineStatsByPrefix(esClient *elasticsearch.API, pipelinePrefix string) (map[string]PipelineStatsMap, error) { + statsResponse, err := requestPipelineStats(esClient) + if err != nil { + return nil, err + } + return getPipelineStatsByPrefix(statsResponse, pipelinePrefix) +} + +func requestPipelineStats(esClient *elasticsearch.API) ([]byte, error) { statsReq := esClient.Nodes.Stats.WithFilterPath("nodes.*.ingest.pipelines") resp, err := esClient.Nodes.Stats(statsReq) if err != nil { - return nil, errors.Wrapf(err, "Node Stats API call failed") + return nil, fmt.Errorf("node stats API call failed: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return nil, errors.Wrap(err, "failed to read Stats API response body") + return nil, fmt.Errorf("failed to read Stats API response body: %w", err) } if resp.StatusCode != 200 { - return nil, errors.Wrapf(elasticsearch.NewError(body), "unexpected response status for Node Stats (%d): %s", resp.StatusCode, resp.Status()) + return nil, fmt.Errorf("unexpected response status for Node Stats (%d): %s: %w", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) } - return getPipelineStats(body, pipelines) + + return body, nil } -func getPipelineStats(body []byte, pipelines []Pipeline) (stats PipelineStatsMap, err error) { +func getPipelineStatsByPrefix(body []byte, pipelinePrefix string) (stats map[string]PipelineStatsMap, err error) { var statsResponse pipelinesStatsResponse if err = json.Unmarshal(body, &statsResponse); err != nil { - return nil, errors.Wrap(err, "error decoding Node Stats response") + return nil, fmt.Errorf("error decoding Node Stats response: %w", err) } + stats = make(map[string]PipelineStatsMap, len(statsResponse.Nodes)) + + for nid, node := range statsResponse.Nodes { + nodePStats := make(PipelineStatsMap) + for name, pStats := range node.Ingest.Pipelines { + if !strings.HasPrefix(name, pipelinePrefix) { + continue + } + if nodePStats[name], err = pStats.extract(); err != nil { + return stats, fmt.Errorf("converting pipeline %s: %w", name, err) + } + } + stats[nid] = nodePStats + } + + return stats, nil +} + +func getPipelineStats(body []byte, pipelines []Pipeline) (stats PipelineStatsMap, err error) { + var statsResponse pipelinesStatsResponse + if err = json.Unmarshal(body, &statsResponse); err != nil { + return nil, fmt.Errorf("error decoding Node Stats response: %w", err) + } if nodeCount := len(statsResponse.Nodes); nodeCount != 1 { - return nil, errors.Errorf("Need exactly one ES node in stats response (got %d)", nodeCount) + return nil, fmt.Errorf("need exactly one ES node in stats response (got %d)", nodeCount) } var nodePipelines map[string]pipelineStatsRecord for _, node := range statsResponse.Nodes { @@ -139,15 +178,274 @@ func getPipelineStats(body []byte, pipelines []Pipeline) (stats PipelineStatsMap for _, requested := range pipelines { if pStats, found := nodePipelines[requested.Name]; found { if stats[requested.Name], err = pStats.extract(); err != nil { - return stats, errors.Wrapf(err, "converting pipeline %s", requested.Name) + return stats, fmt.Errorf("converting pipeline %s: %w", requested.Name, err) } } else { missing = append(missing, requested.Name) } } if len(missing) != 0 { - return stats, errors.Errorf("Node Stats response is missing expected pipelines: %s", strings.Join(missing, ", ")) + return stats, fmt.Errorf("node stats response is missing expected pipelines: %s", strings.Join(missing, ", ")) } return stats, nil } + +type NodesStats struct { + ClusterName string `json:"cluster_name"` + Nodes map[string]NodeStats `json:"nodes"` +} + +type NodeStats struct { + Breakers map[string]struct { + LimitSizeInBytes int `json:"limit_size_in_bytes"` + LimitSize string `json:"limit_size"` + EstimatedSizeInBytes int `json:"estimated_size_in_bytes"` + EstimatedSize string `json:"estimated_size"` + Overhead float64 `json:"overhead"` + Tripped int `json:"tripped"` + } + Indices struct { + Docs struct { + Count int `json:"count"` + Deleted int `json:"deleted"` + } `json:"docs"` + ShardStats struct { + TotalCount int `json:"total_count"` + } `json:"shard_stats"` + Store struct { + SizeInBytes int `json:"size_in_bytes"` + TotalDataSetSizeInBytes int `json:"total_data_set_size_in_bytes"` + ReservedInBytes int `json:"reserved_in_bytes"` + } `json:"store"` + Indexing struct { + IndexTotal int `json:"index_total"` + IndexTimeInMillis int `json:"index_time_in_millis"` + IndexCurrent int `json:"index_current"` + IndexFailed int `json:"index_failed"` + DeleteTotal int `json:"delete_total"` + DeleteTimeInMillis int `json:"delete_time_in_millis"` + DeleteCurrent int `json:"delete_current"` + NoopUpdateTotal int `json:"noop_update_total"` + IsThrottled bool `json:"is_throttled"` + ThrottleTimeInMillis int `json:"throttle_time_in_millis"` + WriteLoad float64 `json:"write_load"` + } `json:"indexing"` + Get struct { + Total int `json:"total"` + TimeInMillis int `json:"time_in_millis"` + ExistsTotal int `json:"exists_total"` + ExistsTimeInMillis int `json:"exists_time_in_millis"` + MissingTotal int `json:"missing_total"` + MissingTimeInMillis int `json:"missing_time_in_millis"` + Current int `json:"current"` + } `json:"get"` + Search struct { + OpenContexts int `json:"open_contexts"` + QueryTotal int `json:"query_total"` + QueryTimeInMillis int `json:"query_time_in_millis"` + QueryCurrent int `json:"query_current"` + FetchTotal int `json:"fetch_total"` + FetchTimeInMillis int `json:"fetch_time_in_millis"` + FetchCurrent int `json:"fetch_current"` + ScrollTotal int `json:"scroll_total"` + ScrollTimeInMillis int `json:"scroll_time_in_millis"` + ScrollCurrent int `json:"scroll_current"` + SuggestTotal int `json:"suggest_total"` + SuggestTimeInMillis int `json:"suggest_time_in_millis"` + SuggestCurrent int `json:"suggest_current"` + } `json:"search"` + Merges struct { + Current int `json:"current"` + CurrentDocs int `json:"current_docs"` + CurrentSizeInBytes int `json:"current_size_in_bytes"` + Total int `json:"total"` + TotalTimeInMillis int `json:"total_time_in_millis"` + TotalDocs int `json:"total_docs"` + TotalSizeInBytes int `json:"total_size_in_bytes"` + TotalStoppedTimeInMillis int `json:"total_stopped_time_in_millis"` + TotalThrottledTimeInMillis int `json:"total_throttled_time_in_millis"` + TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` + } `json:"merges"` + Refresh struct { + Total int `json:"total"` + TotalTimeInMillis int `json:"total_time_in_millis"` + ExternalTotal int `json:"external_total"` + ExternalTotalTimeInMillis int `json:"external_total_time_in_millis"` + Listeners int `json:"listeners"` + } `json:"refresh"` + Flush struct { + Total int `json:"total"` + Periodic int `json:"periodic"` + TotalTimeInMillis int `json:"total_time_in_millis"` + } `json:"flush"` + Warmer struct { + Current int `json:"current"` + Total int `json:"total"` + TotalTimeInMillis int `json:"total_time_in_millis"` + } `json:"warmer"` + QueryCache struct { + MemorySizeInBytes int `json:"memory_size_in_bytes"` + TotalCount int `json:"total_count"` + HitCount int `json:"hit_count"` + MissCount int `json:"miss_count"` + CacheSize int `json:"cache_size"` + CacheCount int `json:"cache_count"` + Evictions int `json:"evictions"` + } `json:"query_cache"` + Fielddata struct { + MemorySizeInBytes int `json:"memory_size_in_bytes"` + Evictions int `json:"evictions"` + } `json:"fielddata"` + Completion struct { + SizeInBytes int `json:"size_in_bytes"` + } `json:"completion"` + Segments struct { + Count int `json:"count"` + MemoryInBytes int `json:"memory_in_bytes"` + TermsMemoryInBytes int `json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes int `json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes int `json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes int `json:"norms_memory_in_bytes"` + PointsMemoryInBytes int `json:"points_memory_in_bytes"` + DocValuesMemoryInBytes int `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes int `json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes int `json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes int `json:"fixed_bit_set_memory_in_bytes"` + MaxUnsafeAutoIDTimestamp int `json:"max_unsafe_auto_id_timestamp"` + FileSizes map[string]int `json:"file_sizes"` + } `json:"segments"` + Translog struct { + Operations int `json:"operations"` + SizeInBytes int `json:"size_in_bytes"` + UncommittedOperations int `json:"uncommitted_operations"` + UncommittedSizeInBytes int `json:"uncommitted_size_in_bytes"` + EarliestLastModifiedAge int `json:"earliest_last_modified_age"` + } `json:"translog"` + RequestCache struct { + MemorySizeInBytes int `json:"memory_size_in_bytes"` + Evictions int `json:"evictions"` + HitCount int `json:"hit_count"` + MissCount int `json:"miss_count"` + } `json:"request_cache"` + Recovery struct { + CurrentAsSource int `json:"current_as_source"` + CurrentAsTarget int `json:"current_as_target"` + ThrottleTimeInMillis int `json:"throttle_time_in_millis"` + } `json:"recovery"` + Bulk struct { + TotalOperations int64 `json:"total_operations"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + TotalSizeInBytes int64 `json:"total_size_in_bytes"` + AvgTimeInMillis int64 `json:"avg_time_in_millis"` + AvgSizeInBytes int64 `json:"avg_size_in_bytes"` + } `json:"bulk"` + Mappings struct { + TotalCount int64 `json:"total_count"` + TotalEstimatedOverheadInBytes int64 `json:"total_estimated_overhead_in_bytes"` + } `json:"mappings"` + } `json:"indices"` + JVM struct { + Mem struct { + HeapUsedInBytes int `json:"heap_used_in_bytes"` + HeapUsedPercent int `json:"heap_used_percent"` + HeapCommittedInBytes int `json:"heap_committed_in_bytes"` + HeapMaxInBytes int `json:"heap_max_in_bytes"` + NonHeapUsedInBytes int `json:"non_heap_used_in_bytes"` + NonHeapCommittedInBytes int `json:"non_heap_committed_in_bytes"` + Pools map[string]struct { + UsedInBytes int `json:"used_in_bytes"` + MaxInBytes int `json:"max_in_bytes"` + PeakUsedInBytes int `json:"peak_used_in_bytes"` + PeakMaxInBytes int `json:"peak_max_in_bytes"` + } `json:"pools"` + } `json:"mem"` + Gc struct { + Collectors map[string]struct { + CollectionCount int `json:"collection_count"` + CollectionTimeInMillis int `json:"collection_time_in_millis"` + } `json:"collectors"` + } `json:"gc"` + BufferPools map[string]struct { + Count int `json:"count"` + UsedInBytes int `json:"used_in_bytes"` + TotalCapacityInBytes int `json:"total_capacity_in_bytes"` + } `json:"buffer_pools"` + } `json:"jvm"` + OS struct { + Mem struct { + TotalInBytes int64 `json:"total_in_bytes"` + AdjustedTotalInBytes int64 `json:"adjusted_total_in_bytes"` + FreeInBytes int64 `json:"free_in_bytes"` + UsedInBytes int64 `json:"used_in_bytes"` + FreePercent int `json:"free_percent"` + UsedPercent int `json:"used_percent"` + } `json:"mem"` + } `json:"os"` + Process struct { + CPU struct { + Percent int `json:"percent"` + TotalInMillis int64 `json:"total_in_millis"` + } `json:"cpu"` + } `json:"process"` + ThreadPool map[string]struct { + Threads int `json:"threads"` + Queue int `json:"queue"` + Active int `json:"active"` + Rejected int `json:"rejected"` + Largest int `json:"largest"` + Completed int `json:"completed"` + } `json:"thread_pool"` + Transport struct { + ServerOpen int `json:"server_open"` + TotalOutboundConnections int `json:"total_outbound_connections"` + RxCount int `json:"rx_count"` + RxSizeInBytes int `json:"rx_size_in_bytes"` + TxCount int `json:"tx_count"` + TxSizeInBytes int `json:"tx_size_in_bytes"` + InboundHandlingTimeHistogram []struct { + GEMillis int `json:"ge_millis"` + LTMillis int `json:"lt_millis"` + Count int `json:"count"` + } `json:"inbound_handling_time_histogram"` + OutboundHandlingTimeHistogram []struct { + GEMillis int `json:"ge_millis"` + LTMillis int `json:"lt_millis"` + Count int `json:"count"` + } `json:"outbound_handling_time_histogram"` + } `json:"transport"` +} + +func GetNodesStats(esClient *elasticsearch.API) (*NodesStats, error) { + req := esClient.Nodes.Stats.WithFilterPath("cluster_name," + + "nodes.*.breakers," + + "nodes.*.indices," + + "nodes.*.jvm.mem," + + "nodes.*.jvm.gc," + + "nodes.*.jvm.buffer_pools," + + "nodes.*.os.mem," + + "nodes.*.process.cpu," + + "nodes.*.thread_pool," + + "nodes.*.transport", + ) + resp, err := esClient.Nodes.Stats(req) + if err != nil { + return nil, fmt.Errorf("node stats API call failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read Stats API response body: %w", err) + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected response status for Node Stats (%d): %s: %w", resp.StatusCode, resp.Status(), elasticsearch.NewError(body)) + } + + var statsResponse NodesStats + if err = json.Unmarshal(body, &statsResponse); err != nil { + return nil, fmt.Errorf("error decoding Node Stats response: %w", err) + } + return &statsResponse, nil +} diff --git a/internal/elasticsearch/ingest/pipeline.go b/internal/elasticsearch/ingest/pipeline.go index 839bd6e52..bbeeee3bc 100644 --- a/internal/elasticsearch/ingest/pipeline.go +++ b/internal/elasticsearch/ingest/pipeline.go @@ -7,11 +7,11 @@ package ingest import ( "bytes" "encoding/json" + "fmt" "io" "net/http" "strings" - "github.com/pkg/errors" "gopkg.in/yaml.v3" "github.com/elastic/elastic-package/internal/elasticsearch" @@ -59,13 +59,13 @@ func (p *Pipeline) MarshalJSON() (asJSON []byte, err error) { var node map[string]interface{} err = yaml.Unmarshal(p.Content, &node) if err != nil { - return nil, errors.Wrapf(err, "unmarshalling pipeline content failed (pipeline: %s)", p.Name) + return nil, fmt.Errorf("unmarshalling pipeline content failed (pipeline: %s): %w", p.Name, err) } if asJSON, err = json.Marshal(node); err != nil { - return nil, errors.Wrapf(err, "marshalling pipeline content failed (pipeline: %s)", p.Name) + return nil, fmt.Errorf("marshalling pipeline content failed (pipeline: %s): %w", p.Name, err) } default: - return nil, errors.Errorf("unsupported pipeline format '%s' (pipeline: %s)", p.Format, p.Name) + return nil, fmt.Errorf("unsupported pipeline format '%s' (pipeline: %s)", p.Format, p.Name) } return asJSON, nil } @@ -80,30 +80,30 @@ func SimulatePipeline(api *elasticsearch.API, pipelineName string, events []json requestBody, err := json.Marshal(&request) if err != nil { - return nil, errors.Wrap(err, "marshalling simulate request failed") + return nil, fmt.Errorf("marshalling simulate request failed: %w", err) } r, err := api.Ingest.Simulate(bytes.NewReader(requestBody), func(request *elasticsearch.IngestSimulateRequest) { request.PipelineID = pipelineName }) if err != nil { - return nil, errors.Wrapf(err, "Simulate API call failed (pipelineName: %s)", pipelineName) + return nil, fmt.Errorf("simulate API call failed (pipelineName: %s): %w", pipelineName, err) } defer r.Body.Close() body, err := io.ReadAll(r.Body) if err != nil { - return nil, errors.Wrap(err, "failed to read Simulate API response body") + return nil, fmt.Errorf("failed to read Simulate API response body: %w", err) } if r.StatusCode != http.StatusOK { - return nil, errors.Wrapf(elasticsearch.NewError(body), "unexpected response status for Simulate (%d): %s", r.StatusCode, r.Status()) + return nil, fmt.Errorf("unexpected response status for Simulate (%d): %s: %w", r.StatusCode, r.Status(), elasticsearch.NewError(body)) } var response simulatePipelineResponse err = json.Unmarshal(body, &response) if err != nil { - return nil, errors.Wrap(err, "unmarshalling simulate request failed") + return nil, fmt.Errorf("unmarshalling simulate request failed: %w", err) } processedEvents := make([]json.RawMessage, len(response.Docs)) @@ -117,7 +117,7 @@ func UninstallPipelines(api *elasticsearch.API, pipelines []Pipeline) error { for _, p := range pipelines { resp, err := api.Ingest.DeletePipeline(p.Name) if err != nil { - return errors.Wrapf(err, "DeletePipeline API call failed (pipelineName: %s)", p.Name) + return fmt.Errorf("delete pipeline API call failed (pipelineName: %s): %w", p.Name, err) } resp.Body.Close() } diff --git a/internal/elasticsearch/ingest/processors.go b/internal/elasticsearch/ingest/processors.go index 4b9ec6d8a..bab9c506e 100644 --- a/internal/elasticsearch/ingest/processors.go +++ b/internal/elasticsearch/ingest/processors.go @@ -5,7 +5,8 @@ package ingest import ( - "github.com/pkg/errors" + "fmt" + "gopkg.in/yaml.v3" ) @@ -27,9 +28,12 @@ func (p Pipeline) Processors() (procs []Processor, err error) { case "yaml", "yml", "json": procs, err = processorsFromYAML(p.Content) default: - return nil, errors.Errorf("unsupported pipeline format: %s", p.Format) + return nil, fmt.Errorf("unsupported pipeline format: %s", p.Format) + } + if err != nil { + return nil, fmt.Errorf("failure processing %s pipeline '%s': %w", p.Format, p.Filename(), err) } - return procs, errors.Wrapf(err, "failure processing %s pipeline '%s'", p.Format, p.Filename()) + return procs, nil } // extract a list of processors from a pipeline definition in YAML format. @@ -42,14 +46,14 @@ func processorsFromYAML(content []byte) (procs []Processor, err error) { } for idx, entry := range p.Processors { if entry.Kind != yaml.MappingNode || len(entry.Content) != 2 { - return nil, errors.Errorf("processor#%d is not a single-key map (kind:%v content:%d)", idx, entry.Kind, len(entry.Content)) + return nil, fmt.Errorf("processor#%d is not a single-key map (kind:%v content:%d)", idx, entry.Kind, len(entry.Content)) } var proc Processor if err := entry.Content[1].Decode(&proc); err != nil { - return nil, errors.Wrapf(err, "error decoding processor#%d configuration", idx) + return nil, fmt.Errorf("error decoding processor#%d configuration: %w", idx, err) } if err := entry.Content[0].Decode(&proc.Type); err != nil { - return nil, errors.Wrapf(err, "error decoding processor#%d type", idx) + return nil, fmt.Errorf("error decoding processor#%d type: %w", idx, err) } proc.FirstLine = entry.Line proc.LastLine = lastLine(&entry) diff --git a/internal/elasticsearch/ingest/processors_test.go b/internal/elasticsearch/ingest/processors_test.go index 30523b90c..0d3d161d3 100644 --- a/internal/elasticsearch/ingest/processors_test.go +++ b/internal/elasticsearch/ingest/processors_test.go @@ -172,17 +172,11 @@ processors: } procs, err := p.Processors() if !tt.wantErr { - if !assert.NoError(t, err) { - t.Fatal(err) - } + assert.NoError(t, err) } else { - if !assert.Error(t, err) { - t.Fatal("error expected") - } - } - if !assert.Equal(t, tt.expected, procs) { - t.Errorf("Processors() gotProcs = %v, want %v", procs, tt.expected) + assert.Error(t, err) } + assert.Equal(t, tt.expected, procs) }) } } diff --git a/internal/formatter/yaml_formatter.go b/internal/formatter/yaml_formatter.go index d60cc0437..013df9e74 100644 --- a/internal/formatter/yaml_formatter.go +++ b/internal/formatter/yaml_formatter.go @@ -16,7 +16,7 @@ import ( // The function is exposed, so it can be used by other internal packages. func YAMLFormatter(content []byte) ([]byte, bool, error) { // yaml.Unmarshal() requires `yaml.Node` to be passed instead of generic `interface{}`. - // Otherwise it can detect any comments and fields are considered as normal map. + // Otherwise it can't detect any comments and fields are considered as normal map. var node yaml.Node err := yaml.Unmarshal(content, &node) if err != nil { @@ -31,5 +31,12 @@ func YAMLFormatter(content []byte) ([]byte, bool, error) { return nil, false, errors.Wrap(err, "marshalling YAML node failed") } formatted := b.Bytes() + + prefix := []byte("---\n") + // required to preserve yaml files starting with "---" as yaml.Encoding strips them + if bytes.HasPrefix(content, prefix) && !bytes.HasPrefix(formatted, prefix) { + formatted = append(prefix, formatted...) + } + return formatted, string(content) == string(formatted), nil } diff --git a/internal/kibana/agents.go b/internal/kibana/agents.go index 909eca117..01826ae4e 100644 --- a/internal/kibana/agents.go +++ b/internal/kibana/agents.go @@ -6,12 +6,11 @@ package kibana import ( "encoding/json" + "errors" "fmt" "net/http" "time" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/signal" ) @@ -46,7 +45,7 @@ func (a *Agent) String() string { func (c *Client) ListAgents() ([]Agent, error) { statusCode, respBody, err := c.get(fmt.Sprintf("%s/agents", FleetAPI)) if err != nil { - return nil, errors.Wrap(err, "could not list agents") + return nil, fmt.Errorf("could not list agents: %w", err) } if statusCode != http.StatusOK { @@ -58,7 +57,7 @@ func (c *Client) ListAgents() ([]Agent, error) { } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert list agents (response) to JSON") + return nil, fmt.Errorf("could not convert list agents (response) to JSON: %w", err) } return resp.List, nil @@ -71,7 +70,7 @@ func (c *Client) AssignPolicyToAgent(a Agent, p Policy) error { path := fmt.Sprintf("%s/agents/%s/reassign", FleetAPI, a.ID) statusCode, respBody, err := c.put(path, []byte(reqBody)) if err != nil { - return errors.Wrap(err, "could not assign policy to agent") + return fmt.Errorf("could not assign policy to agent: %w", err) } if statusCode != http.StatusOK { @@ -80,7 +79,7 @@ func (c *Client) AssignPolicyToAgent(a Agent, p Policy) error { err = c.waitUntilPolicyAssigned(a, p) if err != nil { - return errors.Wrap(err, "error occurred while waiting for the policy to be assigned to all agents") + return fmt.Errorf("error occurred while waiting for the policy to be assigned to all agents: %w", err) } return nil } @@ -98,7 +97,7 @@ func (c *Client) waitUntilPolicyAssigned(a Agent, p Policy) error { agent, err := c.getAgent(a.ID) if err != nil { - return errors.Wrap(err, "can't get the agent") + return fmt.Errorf("can't get the agent: %w", err) } logger.Debugf("Agent data: %s", agent.String()) @@ -122,7 +121,7 @@ func (c *Client) waitUntilPolicyAssigned(a Agent, p Policy) error { func (c *Client) getAgent(agentID string) (*Agent, error) { statusCode, respBody, err := c.get(fmt.Sprintf("%s/agents/%s", FleetAPI, agentID)) if err != nil { - return nil, errors.Wrap(err, "could not list agents") + return nil, fmt.Errorf("could not list agents: %w", err) } if statusCode != http.StatusOK { @@ -133,7 +132,7 @@ func (c *Client) getAgent(agentID string) (*Agent, error) { Item Agent `json:"item"` } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert list agents (response) to JSON") + return nil, fmt.Errorf("could not convert list agents (response) to JSON: %w", err) } return &resp.Item, nil } diff --git a/internal/kibana/client.go b/internal/kibana/client.go index 24f140457..f47d9d599 100644 --- a/internal/kibana/client.go +++ b/internal/kibana/client.go @@ -13,8 +13,6 @@ import ( "net/url" "os" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/certs" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/logger" @@ -108,12 +106,12 @@ func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []b func (c *Client) newRequest(method, resourcePath string, reqBody io.Reader) (*http.Request, error) { base, err := url.Parse(c.host) if err != nil { - return nil, errors.Wrapf(err, "could not create base URL from host: %v", c.host) + return nil, fmt.Errorf("could not create base URL from host: %v: %w", c.host, err) } rel, err := url.Parse(resourcePath) if err != nil { - return nil, errors.Wrapf(err, "could not create relative URL from resource path: %v", resourcePath) + return nil, fmt.Errorf("could not create relative URL from resource path: %v: %w", resourcePath, err) } u := base.JoinPath(rel.EscapedPath()) @@ -123,7 +121,7 @@ func (c *Client) newRequest(method, resourcePath string, reqBody io.Reader) (*ht req, err := http.NewRequest(method, u.String(), reqBody) if err != nil { - return nil, errors.Wrapf(err, "could not create %v request to Kibana API resource: %s", method, resourcePath) + return nil, fmt.Errorf("could not create %v request to Kibana API resource: %s: %w", method, resourcePath, err) } req.SetBasicAuth(c.username, c.password) @@ -151,13 +149,13 @@ func (c *Client) doRequest(request *http.Request) (int, []byte, error) { resp, err := client.Do(request) if err != nil { - return 0, nil, errors.Wrap(err, "could not send request to Kibana API") + return 0, nil, fmt.Errorf("could not send request to Kibana API: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - return resp.StatusCode, nil, errors.Wrap(err, "could not read response body") + return resp.StatusCode, nil, fmt.Errorf("could not read response body: %w", err) } return resp.StatusCode, body, nil diff --git a/internal/kibana/dashboards.go b/internal/kibana/dashboards.go index d1e9313ad..9b47fccf7 100644 --- a/internal/kibana/dashboards.go +++ b/internal/kibana/dashboards.go @@ -6,11 +6,10 @@ package kibana import ( "encoding/json" + "errors" "fmt" "strings" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" @@ -35,13 +34,13 @@ func (c *Client) Export(dashboardIDs []string) ([]common.MapStr, error) { path := fmt.Sprintf("%s/dashboards/export%s", CoreAPI, query.String()) statusCode, respBody, err := c.get(path) if err != nil { - return nil, errors.Wrapf(err, "could not export dashboards; API status code = %d; response body = %s", statusCode, respBody) + return nil, fmt.Errorf("could not export dashboards; API status code = %d; response body = %s: %w", statusCode, respBody, err) } var exported exportedType err = json.Unmarshal(respBody, &exported) if err != nil { - return nil, errors.Wrapf(err, "unmarshalling response failed (body: \n%s)", respBody) + return nil, fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) } var multiErr multierror.Error @@ -57,7 +56,7 @@ func (c *Client) Export(dashboardIDs []string) ([]common.MapStr, error) { } if len(multiErr) > 0 { - return nil, errors.Wrap(multiErr, "at least Kibana object returned an error") + return nil, fmt.Errorf("at least Kibana object returned an error: %w", multiErr) } return exported.Objects, nil } diff --git a/internal/kibana/packages.go b/internal/kibana/packages.go index cdaba6195..84fc42f92 100644 --- a/internal/kibana/packages.go +++ b/internal/kibana/packages.go @@ -10,8 +10,6 @@ import ( "net/http" "os" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/packages" ) @@ -22,7 +20,7 @@ func (c *Client) InstallPackage(name, version string) ([]packages.Asset, error) statusCode, respBody, err := c.post(path, reqBody) if err != nil { - return nil, errors.Wrap(err, "could not install package") + return nil, fmt.Errorf("could not install package: %w", err) } return processResults("install", statusCode, respBody) @@ -34,7 +32,7 @@ func (c *Client) InstallZipPackage(zipFile string) ([]packages.Asset, error) { body, err := os.Open(zipFile) if err != nil { - return nil, errors.Wrap(err, "failed to read zip file") + return nil, fmt.Errorf("failed to read zip file: %w", err) } defer body.Close() @@ -46,7 +44,7 @@ func (c *Client) InstallZipPackage(zipFile string) ([]packages.Asset, error) { statusCode, respBody, err := c.doRequest(req) if err != nil { - return nil, errors.Wrap(err, "could not install zip package") + return nil, fmt.Errorf("could not install zip package: %w", err) } return processResults("zip-install", statusCode, respBody) @@ -57,7 +55,7 @@ func (c *Client) RemovePackage(name, version string) ([]packages.Asset, error) { path := fmt.Sprintf("%s/epm/packages/%s-%s", FleetAPI, name, version) statusCode, respBody, err := c.delete(path) if err != nil { - return nil, errors.Wrap(err, "could not delete package") + return nil, fmt.Errorf("could not delete package: %w", err) } return processResults("remove", statusCode, respBody) @@ -73,7 +71,7 @@ func processResults(action string, statusCode int, respBody []byte) ([]packages. } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrapf(err, "could not convert %s package (response) to JSON", action) + return nil, fmt.Errorf("could not convert %s package (response) to JSON: %w", action, err) } return resp.Assets, nil diff --git a/internal/kibana/policies.go b/internal/kibana/policies.go index 59cd10f87..ef9f56e67 100644 --- a/internal/kibana/policies.go +++ b/internal/kibana/policies.go @@ -9,30 +9,30 @@ import ( "fmt" "net/http" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/packages" ) // Policy represents an Agent Policy in Fleet. type Policy struct { - ID string `json:"id,omitempty"` - Name string `json:"name"` - Description string `json:"description"` - Namespace string `json:"namespace"` - Revision int `json:"revision,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Namespace string `json:"namespace"` + Revision int `json:"revision,omitempty"` + MonitoringEnabled []string `json:"monitoring_enabled"` + MonitoringOutputID string `json:"monitoring_output_id"` } // CreatePolicy persists the given Policy in Fleet. func (c *Client) CreatePolicy(p Policy) (*Policy, error) { reqBody, err := json.Marshal(p) if err != nil { - return nil, errors.Wrap(err, "could not convert policy (request) to JSON") + return nil, fmt.Errorf("could not convert policy (request) to JSON: %w", err) } statusCode, respBody, err := c.post(fmt.Sprintf("%s/agent_policies", FleetAPI), reqBody) if err != nil { - return nil, errors.Wrap(err, "could not create policy") + return nil, fmt.Errorf("could not create policy: %w", err) } if statusCode != http.StatusOK { @@ -44,7 +44,7 @@ func (c *Client) CreatePolicy(p Policy) (*Policy, error) { } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert policy (response) to JSON") + return nil, fmt.Errorf("could not convert policy (response) to JSON: %w", err) } return &resp.Item, nil @@ -54,7 +54,7 @@ func (c *Client) CreatePolicy(p Policy) (*Policy, error) { func (c *Client) GetPolicy(policyID string) (*Policy, error) { statusCode, respBody, err := c.get(fmt.Sprintf("%s/agent_policies/%s", FleetAPI, policyID)) if err != nil { - return nil, errors.Wrap(err, "could not get policy") + return nil, fmt.Errorf("could not get policy: %w", err) } if statusCode != http.StatusOK { @@ -66,7 +66,7 @@ func (c *Client) GetPolicy(policyID string) (*Policy, error) { } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert policy (response) to JSON") + return nil, fmt.Errorf("could not convert policy (response) to JSON: %w", err) } return &resp.Item, nil @@ -76,7 +76,7 @@ func (c *Client) GetPolicy(policyID string) (*Policy, error) { func (c *Client) GetRawPolicy(policyID string) (json.RawMessage, error) { statusCode, respBody, err := c.get(fmt.Sprintf("%s/agent_policies/%s", FleetAPI, policyID)) if err != nil { - return nil, errors.Wrap(err, "could not get policy") + return nil, fmt.Errorf("could not get policy: %w", err) } if statusCode != http.StatusOK { @@ -88,7 +88,7 @@ func (c *Client) GetRawPolicy(policyID string) (json.RawMessage, error) { } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert policy (response) to JSON") + return nil, fmt.Errorf("could not convert policy (response) to JSON: %w", err) } return resp.Item, nil @@ -109,7 +109,7 @@ func (c *Client) ListRawPolicies() ([]json.RawMessage, error) { for finished := false; !finished; finished = itemsRetrieved == resp.Total { statusCode, respBody, err := c.get(fmt.Sprintf("%s/agent_policies?full=true&page=%d", FleetAPI, currentPage)) if err != nil { - return nil, errors.Wrap(err, "could not get policies") + return nil, fmt.Errorf("could not get policies: %w", err) } if statusCode != http.StatusOK { @@ -117,7 +117,7 @@ func (c *Client) ListRawPolicies() ([]json.RawMessage, error) { } if err := json.Unmarshal(respBody, &resp); err != nil { - return nil, errors.Wrap(err, "could not convert policies (response) to JSON") + return nil, fmt.Errorf("could not convert policies (response) to JSON: %w", err) } itemsRetrieved += len(resp.Items) @@ -134,7 +134,7 @@ func (c *Client) DeletePolicy(p Policy) error { statusCode, respBody, err := c.post(fmt.Sprintf("%s/agent_policies/delete", FleetAPI), []byte(reqBody)) if err != nil { - return errors.Wrap(err, "could not delete policy") + return fmt.Errorf("could not delete policy: %w", err) } if statusCode != http.StatusOK { @@ -201,12 +201,12 @@ type PackageDataStream struct { func (c *Client) AddPackageDataStreamToPolicy(r PackageDataStream) error { reqBody, err := json.Marshal(r) if err != nil { - return errors.Wrap(err, "could not convert policy-package (request) to JSON") + return fmt.Errorf("could not convert policy-package (request) to JSON: %w", err) } statusCode, respBody, err := c.post(fmt.Sprintf("%s/package_policies", FleetAPI), reqBody) if err != nil { - return errors.Wrap(err, "could not add package to policy") + return fmt.Errorf("could not add package to policy: %w", err) } if statusCode != http.StatusOK { @@ -215,3 +215,76 @@ func (c *Client) AddPackageDataStreamToPolicy(r PackageDataStream) error { return nil } + +// PackagePolicy represents an Package Policy in Fleet. +type PackagePolicy struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Namespace string `json:"namespace"` + PolicyID string `json:"policy_id"` + Package struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"package"` + Vars map[string]interface{} `json:"vars,omitempty"` + Inputs map[string]PackagePolicyInput `json:"inputs,omitempty"` + Force bool `json:"force"` +} + +type PackagePolicyInput struct { + Enabled bool `json:"enabled"` + Streams map[string]PackagePolicyStream `json:"streams,omitempty"` +} + +type PackagePolicyStream struct { + Enabled bool `json:"enabled"` + Vars map[string]interface{} `json:"vars,omitempty"` +} + +// CreatePackagePolicy persists the given Package Policy in Fleet. +func (c *Client) CreatePackagePolicy(p PackagePolicy) (*PackagePolicy, error) { + reqBody, err := json.Marshal(p) + if err != nil { + return nil, fmt.Errorf("could not convert package policy (request) to JSON: %w", err) + } + + statusCode, respBody, err := c.post(fmt.Sprintf("%s/package_policies", FleetAPI), reqBody) + if err != nil { + return nil, fmt.Errorf("could not create package policy (req %s): %w", string(reqBody), err) + } + + if statusCode != http.StatusOK { + return nil, fmt.Errorf("could not create package policy (req %s); API status code = %d; response body = %s", string(reqBody), statusCode, respBody) + } + + // Response format for the policy is inconsistent with the creation one, + // we update the ID to avoid having a full type only to hold the response. + var resp struct { + Item struct { + ID string `json:"id"` + } `json:"item"` + } + + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("could not convert package policy (response) to JSON: %w", err) + } + + p.ID = resp.Item.ID + + return &p, nil +} + +// DeletePackagePolicy removes the given Package Policy from Fleet. +func (c *Client) DeletePackagePolicy(p PackagePolicy) error { + statusCode, respBody, err := c.delete(fmt.Sprintf("%s/package_policies/%s", FleetAPI, p.ID)) + if err != nil { + return fmt.Errorf("could not delete package policy: %w", err) + } + + if statusCode != http.StatusOK { + return fmt.Errorf("could not delete package policy; API status code = %d; response body = %s", statusCode, respBody) + } + + return nil +} diff --git a/internal/kibana/saved_objects.go b/internal/kibana/saved_objects.go index c1a35ba9b..2956bf112 100644 --- a/internal/kibana/saved_objects.go +++ b/internal/kibana/saved_objects.go @@ -10,8 +10,6 @@ import ( "sort" "strings" - "github.com/pkg/errors" - "github.com/elastic/elastic-package/internal/logger" ) @@ -65,7 +63,7 @@ func (c *Client) FindDashboards() (DashboardSavedObjects, error) { for { r, err := c.findDashboardsNextPage(page) if err != nil { - return nil, errors.Wrap(err, "can't fetch page with results") + return nil, fmt.Errorf("can't fetch page with results: %w", err) } if r.Error != "" { return nil, fmt.Errorf("%s: %s", r.Error, r.Message) @@ -94,13 +92,13 @@ func (c *Client) findDashboardsNextPage(page int) (*savedObjectsResponse, error) path := fmt.Sprintf("%s/_find?type=dashboard&fields=title&per_page=%d&page=%d", SavedObjectsAPI, findDashboardsPerPage, page) statusCode, respBody, err := c.get(path) if err != nil { - return nil, errors.Wrapf(err, "could not find dashboards; API status code = %d; response body = %s", statusCode, respBody) + return nil, fmt.Errorf("could not find dashboards; API status code = %d; response body = %s: %w", statusCode, respBody, err) } var r savedObjectsResponse err = json.Unmarshal(respBody, &r) if err != nil { - return nil, errors.Wrap(err, "unmarshalling response failed") + return nil, fmt.Errorf("unmarshalling response failed: %w", err) } return &r, nil } diff --git a/internal/kibana/status.go b/internal/kibana/status.go index 9644f4ea8..d13179f74 100644 --- a/internal/kibana/status.go +++ b/internal/kibana/status.go @@ -8,8 +8,6 @@ import ( "encoding/json" "fmt" "net/http" - - "github.com/pkg/errors" ) const SNAPSHOT_SUFFIX = "-SNAPSHOT" @@ -39,7 +37,7 @@ func (c *Client) Version() (VersionInfo, error) { var version VersionInfo statusCode, respBody, err := c.get(StatusAPI) if err != nil { - return version, errors.Wrapf(err, "could not reach status endpoint") + return version, fmt.Errorf("could not reach status endpoint: %w", err) } if statusCode != http.StatusOK { @@ -49,7 +47,7 @@ func (c *Client) Version() (VersionInfo, error) { var status statusType err = json.Unmarshal(respBody, &status) if err != nil { - return version, errors.Wrapf(err, "unmarshalling response failed (body: \n%s)", respBody) + return version, fmt.Errorf("unmarshalling response failed (body: \n%s): %w", respBody, err) } return status.Version, nil diff --git a/internal/reportgenerator/generators/generators.go b/internal/reportgenerator/generators/generators.go index 3e4bc7d41..b56d53fac 100644 --- a/internal/reportgenerator/generators/generators.go +++ b/internal/reportgenerator/generators/generators.go @@ -6,5 +6,5 @@ package runners import ( // Registered benchmark runners - _ "github.com/elastic/elastic-package/internal/reportgenerator/generators/benchmark" + _ "github.com/elastic/elastic-package/internal/reportgenerator/generators/pipelinebench" ) diff --git a/internal/reportgenerator/generators/benchmark/generator.go b/internal/reportgenerator/generators/pipelinebench/generator.go similarity index 83% rename from internal/reportgenerator/generators/benchmark/generator.go rename to internal/reportgenerator/generators/pipelinebench/generator.go index 8c4b502ac..51cba5087 100644 --- a/internal/reportgenerator/generators/benchmark/generator.go +++ b/internal/reportgenerator/generators/pipelinebench/generator.go @@ -13,7 +13,7 @@ import ( "os" "path/filepath" - "github.com/elastic/elastic-package/internal/benchrunner" + "github.com/elastic/elastic-package/internal/benchrunner/runners/pipeline" "github.com/elastic/elastic-package/internal/reportgenerator" ) @@ -79,7 +79,7 @@ func (g *generator) generate() ([]byte, error) { return nil, fmt.Errorf("reading new result: %w", err) } pkg, ds := newRes.Package, newRes.DataStream - var oldRes benchrunner.BenchmarkResult + var oldRes pipeline.BenchmarkResult if oldEntry, found := oldResults[pkg]; found { if ds, found := oldEntry[ds]; found { oldRes, err = readResult(g.options.OldPath, ds) @@ -95,7 +95,7 @@ func (g *generator) generate() ([]byte, error) { return g.markdownFormat(reports) } -func createReport(new, old benchrunner.BenchmarkResult) Report { +func createReport(new, old pipeline.BenchmarkResult) Report { var r Report r.Package, r.DataStream = new.Package, new.DataStream @@ -110,7 +110,7 @@ func createReport(new, old benchrunner.BenchmarkResult) Report { return r } -func getEPS(r benchrunner.BenchmarkResult) float64 { +func getEPS(r pipeline.BenchmarkResult) float64 { for _, test := range r.Tests { for _, res := range test.Results { if res.Name == "eps" { @@ -166,29 +166,29 @@ func listAllDirResultsAsMap(path string) (map[string]map[string]fs.DirEntry, err return m, nil } -func readResult(path string, e fs.DirEntry) (benchrunner.BenchmarkResult, error) { +func readResult(path string, e fs.DirEntry) (pipeline.BenchmarkResult, error) { fi, err := e.Info() if err != nil { - return benchrunner.BenchmarkResult{}, fmt.Errorf("getting file info failed (file: %s): %w", e.Name(), err) + return pipeline.BenchmarkResult{}, fmt.Errorf("getting file info failed (file: %s): %w", e.Name(), err) } b, err := os.ReadFile(path + string(os.PathSeparator) + fi.Name()) if err != nil { - return benchrunner.BenchmarkResult{}, fmt.Errorf("reading result contents (file: %s): %w", fi.Name(), err) + return pipeline.BenchmarkResult{}, fmt.Errorf("reading result contents (file: %s): %w", fi.Name(), err) } - var br benchrunner.BenchmarkResult + var br pipeline.BenchmarkResult switch ext := filepath.Ext(fi.Name()); ext { case ".json": if err := json.Unmarshal(b, &br); err != nil { - return benchrunner.BenchmarkResult{}, fmt.Errorf("decoding json (file: %s): %w", fi.Name(), err) + return pipeline.BenchmarkResult{}, fmt.Errorf("decoding json (file: %s): %w", fi.Name(), err) } case ".xml": if err := xml.Unmarshal(b, &br); err != nil { - return benchrunner.BenchmarkResult{}, fmt.Errorf("decoding xml (file: %s): %w", fi.Name(), err) + return pipeline.BenchmarkResult{}, fmt.Errorf("decoding xml (file: %s): %w", fi.Name(), err) } default: - return benchrunner.BenchmarkResult{}, fmt.Errorf("unsupported result format: %v", ext) + return pipeline.BenchmarkResult{}, fmt.Errorf("unsupported result format: %v", ext) } return br, nil diff --git a/internal/reportgenerator/generators/benchmark/report.go b/internal/reportgenerator/generators/pipelinebench/report.go similarity index 100% rename from internal/reportgenerator/generators/benchmark/report.go rename to internal/reportgenerator/generators/pipelinebench/report.go diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index b9ca09d4b..eb3692499 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -511,9 +511,10 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext logger.Debug("creating test policy...") testTime := time.Now().Format("20060102T15:04:05Z") p := kibana.Policy{ - Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), - Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), - Namespace: "ep", + Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), + Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), + Namespace: "ep", + MonitoringEnabled: []string{"logs", "metrics"}, } policy, err := kib.CreatePolicy(p) if err != nil { diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 6c610a0b3..2978bf213 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -69,18 +69,23 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do elastic-package install -v if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then - rm -rf "${OLDPWD}/build/benchmark-results" - elastic-package benchmark -v --report-format xUnit --report-output file --fail-on-missing - - rm -rf "${OLDPWD}/build/benchmark-results-old" - mv "${OLDPWD}/build/benchmark-results" "${OLDPWD}/build/benchmark-results-old" - - elastic-package benchmark -v --report-format json --report-output file --fail-on-missing - - elastic-package report --fail-on-missing benchmark \ - --new ${OLDPWD}/build/benchmark-results \ - --old ${OLDPWD}/build/benchmark-results-old \ - --threshold 1 --report-output-path="${OLDPWD}/build/benchreport" + if [ "${PACKAGE_UNDER_TEST:-*}" == "pipeline_benchmark" ]; then + rm -rf "${OLDPWD}/build/benchmark-results" + elastic-package benchmark pipeline -v --report-format xUnit --report-output file --fail-on-missing + + rm -rf "${OLDPWD}/build/benchmark-results-old" + mv "${OLDPWD}/build/benchmark-results" "${OLDPWD}/build/benchmark-results-old" + + elastic-package benchmark pipeline -v --report-format json --report-output file --fail-on-missing + + elastic-package report --fail-on-missing benchmark \ + --new ${OLDPWD}/build/benchmark-results \ + --old ${OLDPWD}/build/benchmark-results-old \ + --threshold 1 --report-output-path="${OLDPWD}/build/benchreport" + fi + if [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then + elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s + fi else # defer-cleanup is set to a short period to verify that the option is available elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage diff --git a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark.yml b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark.yml new file mode 100644 index 000000000..167394209 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark.yml @@ -0,0 +1,12 @@ +--- +description: Benchmark 20MiB of data ingested +input: filestream +vars: ~ +data_stream.name: testds +data_stream.vars.paths: + - "{{SERVICE_LOGS_DIR}}/corpus-*" +warmup_time_period: 10s +corpora.generator.size: 20MiB +corpora.generator.template.path: ./logs-benchmark/template.log +corpora.generator.config.path: ./logs-benchmark/config.yml +corpora.generator.fields.path: ./logs-benchmark/fields.yml diff --git a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml new file mode 100644 index 000000000..af365d3c9 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/config.yml @@ -0,0 +1,40 @@ +- name: IP + cardinality: + numerator: 1 + denominator: 100 +- name: Day + range: + min: 1 + max: 28 +- name: H + range: + min: 10 + max: 23 +- name: MS + range: + min: 10 + max: 59 +- name: Mon + enum: + - "Jan" + - "Feb" + - "Mar" + - "Apr" + - "May" + - "Jun" + - "Jul" + - "Aug" + - "Sep" + - "Oct" + - "Nov" + - "Dec" +- name: StatusCode + enum: ["200", "400", "404"] +- name: Size + range: + min: 1 + max: 1000 +- name: Port + range: + min: 8000 + max: 8080 diff --git a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml new file mode 100644 index 000000000..4ed5ea81a --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/fields.yml @@ -0,0 +1,18 @@ +- name: IP + type: ip +- name: Day + type: long +- name: Mon + type: keyword +- name: H + type: long +- name: MS + type: long +- name: StatusCode + type: keyword +- name: Size + type: long +- name: Hostname + type: keyword +- name: Port + type: long diff --git a/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log new file mode 100644 index 000000000..f38c5b861 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/_dev/benchmark/system/logs-benchmark/template.log @@ -0,0 +1 @@ +{{.IP}} - - [{{.Day}}/{{.Mon}}/2022:{{.H}}:{{.MS}}:{{.MS}} +0200] "GET /favicon.ico HTTP/1.1" {{.StatusCode}} {{.Size}} "http://{{.Hostname}}:{{.Port}}/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/benchmarks/system_benchmark/changelog.yml b/test/packages/benchmarks/system_benchmark/changelog.yml new file mode 100644 index 000000000..1ced0b8d3 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "999.999.999" + changes: + - description: initial release + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/elastic-package/pull/906 diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log b/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log new file mode 100644 index 000000000..c8c9ffe96 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/access-raw.log @@ -0,0 +1 @@ +1.2.3.4 - - [25/Oct/2016:14:49:34 +0200] "GET /favicon.ico HTTP/1.1" 404 571 "http://localhost:8080/" "skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36" \ No newline at end of file diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml b/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml new file mode 100644 index 000000000..30a2b50cf --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/_dev/benchmark/pipeline/config.yml @@ -0,0 +1 @@ +num_docs: 10000 diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs b/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs new file mode 100644 index 000000000..cc801fea2 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/filestream.yml.hbs @@ -0,0 +1,4 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs b/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs new file mode 100644 index 000000000..b4a46979e --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/agent/stream/udp.yml.hbs @@ -0,0 +1 @@ +host: "{{host}}:{{port}}" diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml b/test/packages/benchmarks/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..f39b8ee23 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,23 @@ +--- +description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent + plugins. +processors: + - grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true + - user_agent: + field: user_agent.original + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/fields/base-fields.yml b/test/packages/benchmarks/system_benchmark/data_stream/testds/fields/base-fields.yml new file mode 100644 index 000000000..0ec2cc7e0 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/fields/base-fields.yml @@ -0,0 +1,38 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: container.id + description: Unique container id. + ignore_above: 1024 + type: keyword +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.source.address + description: Source address from which the log event was read / sent from. + type: keyword +- name: log.flags + description: Flags for the log file. + type: keyword +- name: log.offset + description: Offset of the entry in the log file. + type: long +- name: tags + description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + type: keyword diff --git a/test/packages/benchmarks/system_benchmark/data_stream/testds/manifest.yml b/test/packages/benchmarks/system_benchmark/data_stream/testds/manifest.yml new file mode 100644 index 000000000..34c28ea4a --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/data_stream/testds/manifest.yml @@ -0,0 +1,36 @@ +title: Test +release: experimental +type: logs +streams: + - input: udp + title: UDP logs + enabled: false + description: Collect UDP logs + template_path: udp.yml.hbs + vars: + - name: host + type: text + title: UDP host to listen on + multi: false + required: true + show_user: true + default: localhost + - name: port + type: integer + title: UDP port to listen on + multi: false + required: true + show_user: true + default: 9511 + - input: filestream + enabled: false + title: Logs + description: Collect logs + template_path: filestream.yml.hbs + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true diff --git a/test/packages/benchmarks/system_benchmark/docs/README.md b/test/packages/benchmarks/system_benchmark/docs/README.md new file mode 100644 index 000000000..e0ef7b4a1 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/docs/README.md @@ -0,0 +1,2 @@ +# Test integration + diff --git a/test/packages/benchmarks/system_benchmark/manifest.yml b/test/packages/benchmarks/system_benchmark/manifest.yml new file mode 100644 index 000000000..cb20ca7b4 --- /dev/null +++ b/test/packages/benchmarks/system_benchmark/manifest.yml @@ -0,0 +1,23 @@ +format_version: 1.0.0 +name: system_benchmarks +title: System benchmarks +version: 999.999.999 +description: Test for system benchmark runner +categories: ["network"] +license: basic +type: integration +conditions: + kibana.version: '^8.0.0' +policy_templates: + - name: testpo + title: Test + description: Description + inputs: + - type: udp + title: Foo bar + description: Foo bar + - type: filestream + title: Collect logs + description: Collecting logs +owner: + github: elastic/integrations