Extend system benchmarks spec

spec/changelog.yml

@@ -7,6 +7,9 @@
   - description: Prepare for next version
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/509
+  - description: Improve system benchmarks definition
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/512
 - version: 2.7.0
   changes:
   - description: Add `profiling` data stream type (technical preview)

spec/integration/_dev/benchmark/scenario.spec.yml

@@ -0,0 +1,134 @@
+spec:
+  type: object
+  additionalProperties: false
+  required:
+    - data_stream
+    - input
+  properties:
+    package:
+      description: The package to benchmark.
+      type: string
+    description:
+      description: A description of the benchmark scenario.
+      type: string
+    version:
+      $ref: "../../manifest.spec.yml#/definitions/version"
+    input:
+      description: The input of the package to benchmark.
+      type: string
+    vars:
+      description: The package level variables.
+      type:
+        - object
+        - "null"
+      additionalProperties: true
+    data_stream:
+      description: Data-stream parameters.
+      type: object
+      additionalProperties: false
+      required:
+        - name
+      properties:
+        name:
+          description: The name of the data_stream to benchmark.
+          type: string
+        vars:
+          description: The data_stream specific variables.
+          type:
+            - object
+            - "null"
+          additionalProperties: true
+    warmup_time_period:
+      description: |
+        A time period as a duration string for warmup of the benchmark. 
+        All response data captured during warmup will not show up in the measurement results.
+        By default the value is 0 seconds, which means that all data captured will be taken into account for the benchmark results.
+      type: string
+      default: ""
+      example: 10s
+    benchmark_time_period:
+      description: |
+        A time period as a duration string for the duration of the benchmark.
+        By default the value is 0 seconds, which means that the benchmark will run until no more data is ingested.
+      type: string
+      default: ""
+      example: 10m
+    wait_for_data_timeout:
+      description: Timeout for waiting for metrics data during a system test.
+      type: string
+      example: 10m
+    corpora:
+      description: Corpora settings.
+      type: object
+      additionalProperties: false
+      properties:
+        generator:
+          description: Configuration for the corpus generator.
+          type: object
+          additionalProperties: false
+          required:
+            - size
+            - template
+            - config
+            - fields
+          properties:
+            size:
+              description: A string describing the amount of data to generate.
+              type: string
+              example: 10MiB
+            template:
+              description: Configuration for the generator template.
+              type: object
+              additionalProperties: false
+              properties:
+                raw:
+                  description: Raw template.
+                  type: string
+                path:
+                  description: Template path.
+                  type: string
+                type:
+                  description: The type of template.
+                  type: string
+                  enum:
+                    - ""
+                    - placeholder
+                    - gotext
+            config:
+              description: Configuration for the generator.
+              type: object
+              additionalProperties: false
+              properties:
+                raw:
+                  description: Raw config.
+                  type: object
+                  additionalProperties: true
+                path:
+                  description: Config path.
+                  type: string
+            fields:
+              description: Fields description for the generator.
+              type: object
+              additionalProperties: false
+              properties:
+                raw:
+                  description: Raw fields.
+                  type: object
+                  additionalProperties: true
+                path:
+                  description: Fields path.
+                  type: string
+        input_service:
+          description: Options for the service (found in ./deploy/docker) that will send corpora logs to the elastic agents.
+          type: object
+          additionalProperties: false
+          required:
+            - name
+          properties:
+            name:
+              description: The name of the input service.
+              type: string
+            signal:
+              description: The signal to send to the service once the benchmark is ready to start.
+              type: string
+              example: SIGHUP

spec/integration/_dev/benchmarks/spec.yml → spec/integration/_dev/benchmark/spec.yml

@@ -2,9 +2,9 @@ spec:
   additionalContents: false
   developmentFolder: true
   contents:
-    - description: Folder containing end-to-end benchmarks scenarios.
+    - description: Folder containing system benchmarks scenarios.
       type: folder
-      name: e2e
+      name: system
       required: false
       additionalContents: false
       contents:

spec/integration/_dev/spec.yml

@@ -4,9 +4,9 @@ spec:
   contents:
     - description: Folder containing resources related to package benchmarks.
       type: folder
-      name: benchmarks
+      name: benchmark
       required: false
-      $ref: "./benchmarks/spec.yml"
+      $ref: "./benchmark/spec.yml"
     - description: Folder containing resources related to building the package.
       type: folder
       name: build

test/packages/integration_benchmarks/_dev/benchmarks/e2e/deploy/docker/docker-compose.yml → test/packages/integration_benchmarks/_dev/benchmark/system/deploy/docker/docker-compose.yml

@@ -5,4 +5,4 @@ services:
     volumes:
       - ${PATH}:/sample_logs:ro
     entrypoint: /bin/bash
-    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr ${AGENT_IP}:${PORT} -p=tcp /sample_logs/log.log"
+    command: -c "/stream log --start-signal=SIGHUP --delay=5s --addr elastic-agent:3000 -p=tcp /sample_logs/log.log"

test/packages/integration_benchmarks/_dev/benchmark/system/tcp-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 100MiB of data ingested
+input: tcp
+vars: ~
+data_stream.name: test
+data_stream.vars.paths:
+  - "{{SERVICE_LOGS_DIR}}/corpus-*"
+warmup_time_period: 10s
+corpora.generator.size: 20MiB
+corpora.generator.template.raw: a template
+corpora.generator.config.path: ./20mb-logs-benchmark/config.yml
+corpora.generator.fields.path: ./20mb-logs-benchmark/fields.yml
+corpora.input_service.name: foo
+corpora.input_service.signal: SIGHUP