diff --git a/packages/logstash/_dev/build/build.yml b/packages/logstash/_dev/build/build.yml index 08d85edcf9a..36f6e3a812f 100644 --- a/packages/logstash/_dev/build/build.yml +++ b/packages/logstash/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@1.12 + reference: git@8.4 diff --git a/packages/logstash/_dev/build/docs/README.md b/packages/logstash/_dev/build/docs/README.md index 13c52d2a4a9..6e428d5b495 100644 --- a/packages/logstash/_dev/build/docs/README.md +++ b/packages/logstash/_dev/build/docs/README.md @@ -4,7 +4,7 @@ The `logstash` package collects metrics and logs of Logstash. ## Compatibility -The `logstash` package works with Logstash 7.3.0 and later +The `logstash` package works with Logstash 8.5.0 and later ## Logs diff --git a/packages/logstash/_dev/deploy/docker/.env b/packages/logstash/_dev/deploy/docker/.env new file mode 100644 index 00000000000..0130e89172e --- /dev/null +++ b/packages/logstash/_dev/deploy/docker/.env @@ -0,0 +1 @@ +ELASTIC_VERSION=8.5.0-SNAPSHOT diff --git a/packages/logstash/_dev/deploy/docker/config/log4j2.properties b/packages/logstash/_dev/deploy/docker/config/log4j2.properties new file mode 100644 index 00000000000..6c0f873d759 --- /dev/null +++ b/packages/logstash/_dev/deploy/docker/config/log4j2.properties @@ -0,0 +1,102 @@ +status = error +name = LogstashPropertiesConfig + +appender.console.type = Console +appender.console.name = plain_console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n + +appender.json_console.type = Console +appender.json_console.name = json_console +appender.json_console.layout.type = JSONLayout +appender.json_console.layout.compact = true +appender.json_console.layout.eventEol = true + +appender.rolling.type = RollingFile +appender.rolling.name = plain_rolling +appender.rolling.fileName = ${sys:ls.logs}/logstash-plain.log +appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c]%notEmpty{[%X{pipeline.id}]}%notEmpty{[%X{plugin.id}]} %m%n +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 100MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.max = 30 + +appender.json_rolling.type = RollingFile +appender.json_rolling.name = json_rolling +appender.json_rolling.fileName = ${sys:ls.logs}/logstash-json.log +appender.json_rolling.filePattern = ${sys:ls.logs}/logstash-json-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling.policies.type = Policies +appender.json_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling.policies.time.interval = 1 +appender.json_rolling.policies.time.modulate = true +appender.json_rolling.layout.type = JSONLayout +appender.json_rolling.layout.compact = true +appender.json_rolling.layout.eventEol = true +appender.json_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling.policies.size.size = 100MB +appender.json_rolling.strategy.type = DefaultRolloverStrategy +appender.json_rolling.strategy.max = 30 + +rootLogger.level = ${sys:ls.log.level} +rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console +rootLogger.appenderRef.rolling.ref = ${sys:ls.log.format}_rolling +rootLogger.appenderRef.routing.ref = pipeline_routing_appender + +# Slowlog + +appender.console_slowlog.type = Console +appender.console_slowlog.name = plain_console_slowlog +appender.console_slowlog.layout.type = PatternLayout +appender.console_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n + +appender.json_console_slowlog.type = Console +appender.json_console_slowlog.name = json_console_slowlog +appender.json_console_slowlog.layout.type = JSONLayout +appender.json_console_slowlog.layout.compact = true +appender.json_console_slowlog.layout.eventEol = true + +appender.rolling_slowlog.type = RollingFile +appender.rolling_slowlog.name = plain_rolling_slowlog +appender.rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-plain.log +appender.rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-plain-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_slowlog.policies.type = Policies +appender.rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_slowlog.policies.time.interval = 1 +appender.rolling_slowlog.policies.time.modulate = true +appender.rolling_slowlog.layout.type = PatternLayout +appender.rolling_slowlog.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n +appender.rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_slowlog.policies.size.size = 100MB +appender.rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.rolling_slowlog.strategy.max = 30 + +appender.json_rolling_slowlog.type = RollingFile +appender.json_rolling_slowlog.name = json_rolling_slowlog +appender.json_rolling_slowlog.fileName = ${sys:ls.logs}/logstash-slowlog-json.log +appender.json_rolling_slowlog.filePattern = ${sys:ls.logs}/logstash-slowlog-json-%d{yyyy-MM-dd}-%i.log.gz +appender.json_rolling_slowlog.policies.type = Policies +appender.json_rolling_slowlog.policies.time.type = TimeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.time.interval = 1 +appender.json_rolling_slowlog.policies.time.modulate = true +appender.json_rolling_slowlog.layout.type = JSONLayout +appender.json_rolling_slowlog.layout.compact = true +appender.json_rolling_slowlog.layout.eventEol = true +appender.json_rolling_slowlog.policies.size.type = SizeBasedTriggeringPolicy +appender.json_rolling_slowlog.policies.size.size = 100MB +appender.json_rolling_slowlog.strategy.type = DefaultRolloverStrategy +appender.json_rolling_slowlog.strategy.max = 30 + +logger.slowlog.name = slowlog +logger.slowlog.level = info +logger.slowlog.appenderRef.console_slowlog.ref = ${sys:ls.log.format}_console_slowlog +logger.slowlog.appenderRef.rolling_slowlog.ref = ${sys:ls.log.format}_rolling_slowlog +logger.slowlog.additivity = false + +logger.licensereader.name = logstash.licensechecker.licensereader +logger.licensereader.level = error diff --git a/packages/logstash/_dev/deploy/docker/config/logstash.yml b/packages/logstash/_dev/deploy/docker/config/logstash.yml index 90c16104430..3eebf1c1524 100644 --- a/packages/logstash/_dev/deploy/docker/config/logstash.yml +++ b/packages/logstash/_dev/deploy/docker/config/logstash.yml @@ -1,2 +1,8 @@ http.host: "0.0.0.0" config.reload.automatic: true +path.logs: /usr/share/logstash/logs +log.format: json +slowlog.threshold.warn: 1nanos +slowlog.threshold.info: 1nanos +slowlog.threshold.debug: 1nanos +slowlog.threshold.trace: 1nanos diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml index 2ba386e1d29..ee35207e259 100644 --- a/packages/logstash/_dev/deploy/docker/docker-compose.yml +++ b/packages/logstash/_dev/deploy/docker/docker-compose.yml @@ -1,9 +1,10 @@ version: '2.3' services: logstash: - image: "docker.elastic.co/logstash/logstash:8.5.0-SNAPSHOT" + image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION}" volumes: - "./pipeline:/usr/share/logstash/pipeline" - "./config:/usr/share/logstash/config" + - ${SERVICE_LOGS_DIR}/logstash:/usr/share/logstash/logs ports: - "127.0.0.1:9600:9600" diff --git a/packages/logstash/_dev/deploy/docker/pipeline/persisted-queue.conf b/packages/logstash/_dev/deploy/docker/pipeline/persisted-queue.conf index 38dfb247cfa..ec7fc90bc8f 100644 --- a/packages/logstash/_dev/deploy/docker/pipeline/persisted-queue.conf +++ b/packages/logstash/_dev/deploy/docker/pipeline/persisted-queue.conf @@ -4,6 +4,13 @@ input { } } +filter { + sleep { + time => 1 + every => 10 + } +} + output { elasticsearch { hosts => ["https://elasticsearch:9200"] diff --git a/packages/logstash/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/logstash/data_stream/log/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..019f30a7668 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +dynamic_fields: + event.ingested: ".*" + event.created: ".*" diff --git a/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log new file mode 100644 index 00000000000..b8522965495 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log @@ -0,0 +1,4 @@ +{"level":"WARN","loggerName":"logstash.outputs.elasticsearch","timeMillis":1663084834955,"thread":"[pipeline-with-memory-queue]-pipeline-manager","logEvent":{"message":"Restored connection to ES instance","url":"https://elastic:xxxxxx@elasticsearch:9200/"}} +{"level":"INFO","loggerName":"logstash.outputs.elasticsearch","timeMillis":1663084834958,"thread":"[pipeline-with-persisted-queue]-pipeline-manager","logEvent":{"message":"Elasticsearch version determined (8.5.0-SNAPSHOT)","es_version":8}} +{"level":"WARN","loggerName":"logstash.outputs.elasticsearch","timeMillis":1663084834961,"thread":"[pipeline-with-persisted-queue]-pipeline-manager","logEvent":{"message":"Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type","es_version":8}} +{"level":"INFO","loggerName":"logstash.outputs.elasticsearch","timeMillis":1663084834963,"thread":"[pipeline-with-memory-queue]-pipeline-manager","logEvent":{"message":"Elasticsearch version determined (8.5.0-SNAPSHOT)","es_version":8}} diff --git a/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json new file mode 100644 index 00000000000..3cd47c1b958 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-json.log-expected.json @@ -0,0 +1,92 @@ +{ + "expected": [ + { + "@timestamp": "2022-09-13T16:00:34.955Z", + "event": { + "created": "2022-09-13T16:00:34.955Z", + "ingested": "2022-09-20T13:49:39.087356123Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "log": { + "log_event": { + "url": "https://elastic:xxxxxx@elasticsearch:9200/" + }, + "module": "logstash.outputs.elasticsearch", + "thread": "[pipeline-with-memory-queue]-pipeline-manager" + } + }, + "message": "Restored connection to ES instance" + }, + { + "@timestamp": "2022-09-13T16:00:34.958Z", + "event": { + "created": "2022-09-13T16:00:34.958Z", + "ingested": "2022-09-20T13:49:39.087384421Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "log_event": { + "es_version": 8 + }, + "module": "logstash.outputs.elasticsearch", + "thread": "[pipeline-with-persisted-queue]-pipeline-manager" + } + }, + "message": "Elasticsearch version determined (8.5.0-SNAPSHOT)" + }, + { + "@timestamp": "2022-09-13T16:00:34.961Z", + "event": { + "created": "2022-09-13T16:00:34.961Z", + "ingested": "2022-09-20T13:49:39.087390447Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "log": { + "log_event": { + "es_version": 8 + }, + "module": "logstash.outputs.elasticsearch", + "thread": "[pipeline-with-persisted-queue]-pipeline-manager" + } + }, + "message": "Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type" + }, + { + "@timestamp": "2022-09-13T16:00:34.963Z", + "event": { + "created": "2022-09-13T16:00:34.963Z", + "ingested": "2022-09-20T13:49:39.087395138Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "log_event": { + "es_version": 8 + }, + "module": "logstash.outputs.elasticsearch", + "thread": "[pipeline-with-memory-queue]-pipeline-manager" + } + }, + "message": "Elasticsearch version determined (8.5.0-SNAPSHOT)" + } + ] +} \ No newline at end of file diff --git a/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log new file mode 100644 index 00000000000..56633ef80a0 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log @@ -0,0 +1,4 @@ +[2022-09-14T09:31:20,934][INFO ][logstash.javapipeline ][standalone-pipeline] Pipeline started {"pipeline.id"=>"standalone-pipeline"} +[2022-09-14T09:31:20,934][INFO ][logstash.javapipeline ][pipeline-with-memory-queue] Pipeline started {"pipeline.id"=>"pipeline-with-memory-queue"} +[2022-09-14T09:31:20,936][INFO ][logstash.javapipeline ][pipeline-with-persisted-queue] Pipeline started {"pipeline.id"=>"pipeline-with-persisted-queue"} +[2022-09-14T09:31:20,946][INFO ][logstash.agent ] Pipelines running {:count=>3, :running_pipelines=>[:"pipeline-with-memory-queue", :"standalone-pipeline", :"pipeline-with-persisted-queue"], :non_running_pipelines=>[]} diff --git a/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log-expected.json b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log-expected.json new file mode 100644 index 00000000000..1af0da681c9 --- /dev/null +++ b/packages/logstash/data_stream/log/_dev/test/pipeline/test-log-plain.log-expected.json @@ -0,0 +1,79 @@ +{ + "expected": [ + { + "@timestamp": "2022-09-14T09:31:20.934Z", + "event": { + "created": "2022-09-14T09:31:20.934Z", + "ingested": "2022-09-20T13:49:39.150272446Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "module": "logstash.javapipeline", + "pipeline_id": "standalone-pipeline" + } + }, + "message": "Pipeline started {\"pipeline.id\"=\u003e\"standalone-pipeline\"}" + }, + { + "@timestamp": "2022-09-14T09:31:20.934Z", + "event": { + "created": "2022-09-14T09:31:20.934Z", + "ingested": "2022-09-20T13:49:39.150301050Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "module": "logstash.javapipeline", + "pipeline_id": "pipeline-with-memory-queue" + } + }, + "message": "Pipeline started {\"pipeline.id\"=\u003e\"pipeline-with-memory-queue\"}" + }, + { + "@timestamp": "2022-09-14T09:31:20.936Z", + "event": { + "created": "2022-09-14T09:31:20.936Z", + "ingested": "2022-09-20T13:49:39.150307033Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "module": "logstash.javapipeline", + "pipeline_id": "pipeline-with-persisted-queue" + } + }, + "message": "Pipeline started {\"pipeline.id\"=\u003e\"pipeline-with-persisted-queue\"}" + }, + { + "@timestamp": "2022-09-14T09:31:20.946Z", + "event": { + "created": "2022-09-14T09:31:20.946Z", + "ingested": "2022-09-20T13:49:39.150342613Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "INFO" + }, + "logstash": { + "log": { + "module": "logstash.agent" + } + }, + "message": "Pipelines running {:count=\u003e3, :running_pipelines=\u003e[:\"pipeline-with-memory-queue\", :\"standalone-pipeline\", :\"pipeline-with-persisted-queue\"], :non_running_pipelines=\u003e[]}" + } + ] +} \ No newline at end of file diff --git a/packages/logstash/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/logstash/data_stream/log/elasticsearch/ingest_pipeline/default.yml index c5746efa498..e8b84a57585 100644 --- a/packages/logstash/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/logstash/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created - grok: field: message patterns: @@ -19,6 +16,9 @@ processors: - pipeline: if: ctx.first_char == '{' name: '{{ IngestPipeline "pipeline-json" }}' +- set: + copy_from: "@timestamp" + field: event.created - remove: field: - first_char diff --git a/packages/logstash/data_stream/log/fields/ecs.yml b/packages/logstash/data_stream/log/fields/ecs.yml index ea28bbb52ad..0e2e0c4e4a1 100644 --- a/packages/logstash/data_stream/log/fields/ecs.yml +++ b/packages/logstash/data_stream/log/fields/ecs.yml @@ -4,3 +4,7 @@ external: ecs - name: log.level external: ecs +- name: ecs.version + external: ecs +- name: log.file.path + external: ecs diff --git a/packages/logstash/data_stream/log/fields/fields.yml b/packages/logstash/data_stream/log/fields/fields.yml index fa72f6aeb50..bbbc32c3f0a 100644 --- a/packages/logstash/data_stream/log/fields/fields.yml +++ b/packages/logstash/data_stream/log/fields/fields.yml @@ -1,3 +1,7 @@ +- name: input.type + type: keyword +- name: log.offset + type: long - name: logstash.log title: Logstash type: group @@ -13,13 +17,9 @@ multi_fields: - name: text type: text - - name: log_event - type: object - description: | - key and value debugging information. - - name: log_event.action - type: keyword - name: pipeline_id type: keyword + - name: log_event + type: flattened description: | - The ID of the pipeline. + key and value debugging information. diff --git a/packages/logstash/data_stream/log/manifest.yml b/packages/logstash/data_stream/log/manifest.yml index f3d0deb8fb2..b7555a51377 100644 --- a/packages/logstash/data_stream/log/manifest.yml +++ b/packages/logstash/data_stream/log/manifest.yml @@ -11,7 +11,6 @@ streams: required: true show_user: true default: - - /var/log/logstash/logstash-plain*.log - /var/log/logstash/logstash-json*.log template_path: log.yml.hbs title: Logstash logs diff --git a/packages/logstash/data_stream/log/sample_event.json b/packages/logstash/data_stream/log/sample_event.json new file mode 100644 index 00000000000..7746611f36f --- /dev/null +++ b/packages/logstash/data_stream/log/sample_event.json @@ -0,0 +1,70 @@ +{ + "@timestamp": "2022-09-13T19:08:48.030Z", + "agent": { + "ephemeral_id": "c028b260-d373-4eab-978c-08bc42b9dc7c", + "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.5.0" + }, + "data_stream": { + "dataset": "logstash.log", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.10.0" + }, + "elastic_agent": { + "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "snapshot": true, + "version": "8.5.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.log", + "ingested": "2022-09-13T19:08:58Z", + "kind": "event", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "8127511256f0493fa1abf625ca3e0609", + "ip": [ + "172.20.0.8" + ], + "mac": [ + "02-42-AC-14-00-08" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/logstash/logstash-json.log" + }, + "level": "INFO", + "offset": 0 + }, + "logstash": { + "log": { + "log_event": {}, + "module": "logstash.runner", + "thread": "main" + } + }, + "message": "Log4j configuration path used is: /usr/share/logstash/config/log4j2.properties" +} \ No newline at end of file diff --git a/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-common-config.yml b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..019f30a7668 --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +dynamic_fields: + event.ingested: ".*" + event.created: ".*" diff --git a/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log new file mode 100644 index 00000000000..8c86cc3546a --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log @@ -0,0 +1 @@ +{"level":"WARN","loggerName":"slowlog.logstash.filters.sleep","timeMillis":1663093597055,"thread":"[pipeline-with-persisted-queue]>worker5","logEvent":{"message":"event processing time","plugin_params":{"every":10,"time":1,"id":"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019"},"took_in_nanos":15000,"took_in_millis":0,"event":"{\"hostname\":\"66b70fb1a96b\",\"sequence\":430370,\"message\":\"Hello world!\",\"@version\":\"1\",\"@timestamp\":\"2022-09-13T18:26:26.574073600Z\",\"thread_number\":0}"}} diff --git a/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log-expected.json b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log-expected.json new file mode 100644 index 00000000000..d55565ef70e --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-json.log-expected.json @@ -0,0 +1,32 @@ +{ + "expected": [ + { + "@timestamp": "2022-09-13T18:26:37.055Z", + "event": { + "created": "2022-09-13T18:26:37.055Z", + "duration": 15000, + "ingested": "2022-09-20T13:49:39.545814980Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "slowlog": { + "event": "{\"hostname\":\"66b70fb1a96b\",\"sequence\":430370,\"message\":\"Hello world!\",\"@version\":\"1\",\"@timestamp\":\"2022-09-13T18:26:26.574073600Z\",\"thread_number\":0}", + "module": "slowlog.logstash.filters.sleep", + "plugin_name": "sleep", + "plugin_params_object": { + "every": 10, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "time": 1 + }, + "plugin_type": "filters", + "thread": "[pipeline-with-persisted-queue]\u003eworker5", + "took_in_millis": 0 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log new file mode 100644 index 00000000000..647e5d2aa9d --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log @@ -0,0 +1,3 @@ +[2022-09-14T09:36:53,767][WARN ][slowlog.logstash.filters.sleep] event processing time {:plugin_params=>{"every"=>10, "time"=>1, "id"=>"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019"}, :took_in_nanos=>10477, :took_in_millis=>0, :event=>"{\"hostname\":\"4f8082baaffe\",\"thread_number\":0,\"message\":\"Hello world!\",\"sequence\":16207,\"@timestamp\":\"2022-09-14T09:36:44.849789316Z\",\"@version\":\"1\"}"} +[2022-09-14T09:36:53,767][WARN ][slowlog.logstash.filters.sleep] event processing time {:plugin_params=>{"every"=>10, "time"=>1, "id"=>"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019"}, :took_in_nanos=>9171, :took_in_millis=>0, :event=>"{\"hostname\":\"4f8082baaffe\",\"thread_number\":0,\"message\":\"Hello world!\",\"sequence\":16208,\"@timestamp\":\"2022-09-14T09:36:44.870011465Z\",\"@version\":\"1\"}"} +[2022-09-14T09:36:53,790][WARN ][slowlog.logstash.filters.sleep] event processing time {:plugin_params=>{"every"=>10, "time"=>1, "id"=>"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019"}, :took_in_nanos=>1000166192, :took_in_millis=>1000, :event=>"{\"hostname\":\"4f8082baaffe\",\"thread_number\":0,\"message\":\"Hello world!\",\"sequence\":15999,\"@timestamp\":\"2022-09-14T09:36:40.690458954Z\",\"@version\":\"1\"}"} diff --git a/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log-expected.json b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log-expected.json new file mode 100644 index 00000000000..f318df78bb2 --- /dev/null +++ b/packages/logstash/data_stream/slowlog/_dev/test/pipeline/test-slowlog-plain.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "@timestamp": "2022-09-14T09:36:53.767Z", + "event": { + "created": "2022-09-14T09:36:53.767Z", + "duration": 10477, + "ingested": "2022-09-20T13:49:39.594195981Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "slowlog": { + "event": "\"{\\\"hostname\\\":\\\"4f8082baaffe\\\",\\\"thread_number\\\":0,\\\"message\\\":\\\"Hello world!\\\",\\\"sequence\\\":16207,\\\"@timestamp\\\":\\\"2022-09-14T09:36:44.849789316Z\\\",\\\"@version\\\":\\\"1\\\"}\"", + "module": "slowlog.logstash.filters.sleep", + "plugin_name": "sleep", + "plugin_params": "{\"every\"=\u003e10, \"time\"=\u003e1, \"id\"=\u003e\"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019\"}", + "plugin_type": "filters", + "took_in_millis": 0 + } + } + }, + { + "@timestamp": "2022-09-14T09:36:53.767Z", + "event": { + "created": "2022-09-14T09:36:53.767Z", + "duration": 9171, + "ingested": "2022-09-20T13:49:39.594219649Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "slowlog": { + "event": "\"{\\\"hostname\\\":\\\"4f8082baaffe\\\",\\\"thread_number\\\":0,\\\"message\\\":\\\"Hello world!\\\",\\\"sequence\\\":16208,\\\"@timestamp\\\":\\\"2022-09-14T09:36:44.870011465Z\\\",\\\"@version\\\":\\\"1\\\"}\"", + "module": "slowlog.logstash.filters.sleep", + "plugin_name": "sleep", + "plugin_params": "{\"every\"=\u003e10, \"time\"=\u003e1, \"id\"=\u003e\"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019\"}", + "plugin_type": "filters", + "took_in_millis": 0 + } + } + }, + { + "@timestamp": "2022-09-14T09:36:53.790Z", + "event": { + "created": "2022-09-14T09:36:53.790Z", + "duration": 1000166192, + "ingested": "2022-09-20T13:49:39.594224827Z", + "kind": "event", + "type": "info" + }, + "log": { + "level": "WARN" + }, + "logstash": { + "slowlog": { + "event": "\"{\\\"hostname\\\":\\\"4f8082baaffe\\\",\\\"thread_number\\\":0,\\\"message\\\":\\\"Hello world!\\\",\\\"sequence\\\":15999,\\\"@timestamp\\\":\\\"2022-09-14T09:36:40.690458954Z\\\",\\\"@version\\\":\\\"1\\\"}\"", + "module": "slowlog.logstash.filters.sleep", + "plugin_name": "sleep", + "plugin_params": "{\"every\"=\u003e10, \"time\"=\u003e1, \"id\"=\u003e\"e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019\"}", + "plugin_type": "filters", + "took_in_millis": 1000 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/logstash/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml b/packages/logstash/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml index d9173e9fa76..efda0a018d4 100644 --- a/packages/logstash/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/logstash/data_stream/slowlog/elasticsearch/ingest_pipeline/default.yml @@ -4,9 +4,6 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created - grok: field: message patterns: @@ -19,6 +16,9 @@ processors: - pipeline: if: ctx.first_char == '{' name: '{{ IngestPipeline "pipeline-json" }}' +- set: + copy_from: "@timestamp" + field: event.created - remove: field: - first_char diff --git a/packages/logstash/data_stream/slowlog/fields/ecs.yml b/packages/logstash/data_stream/slowlog/fields/ecs.yml index 2b6139cc81f..38a8e35999d 100644 --- a/packages/logstash/data_stream/slowlog/fields/ecs.yml +++ b/packages/logstash/data_stream/slowlog/fields/ecs.yml @@ -4,3 +4,7 @@ external: ecs - name: log.level external: ecs +- name: ecs.version + external: ecs +- name: log.file.path + external: ecs diff --git a/packages/logstash/data_stream/slowlog/fields/fields.yml b/packages/logstash/data_stream/slowlog/fields/fields.yml index bc0621cbe3e..92db45ee741 100644 --- a/packages/logstash/data_stream/slowlog/fields/fields.yml +++ b/packages/logstash/data_stream/slowlog/fields/fields.yml @@ -1,3 +1,7 @@ +- name: input.type + type: keyword +- name: log.offset + type: long - name: logstash.slowlog type: group fields: @@ -39,6 +43,6 @@ - name: text type: text - name: plugin_params_object - type: object + type: flattened description: | key -> value of the configuration used by the plugin. diff --git a/packages/logstash/data_stream/slowlog/manifest.yml b/packages/logstash/data_stream/slowlog/manifest.yml index 50daf67d421..056a2648e21 100644 --- a/packages/logstash/data_stream/slowlog/manifest.yml +++ b/packages/logstash/data_stream/slowlog/manifest.yml @@ -11,7 +11,6 @@ streams: required: true show_user: true default: - - /var/log/logstash/logstash-slowlog-plain*.log - /var/log/logstash/logstash-slowlog-json*.log template_path: log.yml.hbs title: Logstash slowlog logs diff --git a/packages/logstash/data_stream/slowlog/sample_event.json b/packages/logstash/data_stream/slowlog/sample_event.json new file mode 100644 index 00000000000..d8e1aca69b8 --- /dev/null +++ b/packages/logstash/data_stream/slowlog/sample_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2022-09-13T19:09:45.759Z", + "agent": { + "ephemeral_id": "f3343539-9dd4-4db6-9284-e5d738fd2228", + "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.5.0" + }, + "data_stream": { + "dataset": "logstash.slowlog", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.10.0" + }, + "elastic_agent": { + "id": "aeed8481-2d37-45f5-989f-daa7c2173ca0", + "snapshot": true, + "version": "8.5.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "logstash.slowlog", + "duration": 417800, + "ingested": "2022-09-13T19:09:55Z", + "kind": "event", + "type": "info" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "8127511256f0493fa1abf625ca3e0609", + "ip": [ + "172.20.0.8" + ], + "mac": [ + "02-42-AC-14-00-08" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/logstash/logstash-slowlog-json.log" + }, + "level": "WARN", + "offset": 0 + }, + "logstash": { + "slowlog": { + "event": "{\"hostname\":\"128aab07d8dc\",\"message\":\"Hello world!\",\"sequence\":0,\"@timestamp\":\"2022-09-13T19:09:43.252725100Z\",\"thread_number\":0,\"@version\":\"1\"}", + "module": "slowlog.logstash.filters.sleep", + "plugin_name": "sleep", + "plugin_params_object": { + "every": 10, + "id": "e24d45cc4f3bb9981356480856120ed5f68127abbc3af7f47e7bca32460e5019", + "time": 1 + }, + "plugin_type": "filters", + "thread": "[pipeline-with-persisted-queue]\u003eworker6", + "took_in_millis": 0 + } + } +} \ No newline at end of file diff --git a/packages/logstash/docs/README.md b/packages/logstash/docs/README.md index 0cc26e824a7..6c42d1bbe2c 100644 --- a/packages/logstash/docs/README.md +++ b/packages/logstash/docs/README.md @@ -4,7 +4,7 @@ The `logstash` package collects metrics and logs of Logstash. ## Compatibility -The `logstash` package works with Logstash 7.3.0 and later +The `logstash` package works with Logstash 8.5.0 and later ## Logs diff --git a/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Log-Dashboard.json b/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Log-Dashboard.json index d5279df522c..515fda82c7a 100644 --- a/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Log-Dashboard.json +++ b/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Log-Dashboard.json @@ -3,91 +3,19 @@ "description": "Overview of Logstash logs", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true}" }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 12, - "i": "2", - "w": 24, - "x": 24, - "y": 0 - }, - "panelIndex": "2", - "panelRefName": "panel_2", - "type": "visualization", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 12, - "i": "3", - "w": 24, - "x": 0, - "y": 0 - }, - "panelIndex": "3", - "panelRefName": "panel_3", - "type": "visualization", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "columns": [ - "log.level", - "logstash.log.integration", - "message", - "source" - ], - "enhancements": {}, - "sort": [ - "@timestamp", - "desc" - ] - }, - "gridData": { - "h": 40, - "i": "4", - "w": 48, - "x": 0, - "y": 12 - }, - "panelIndex": "4", - "panelRefName": "panel_4", - "type": "search", - "version": "7.3.0" - } - ], + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"h\":12,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"h\":12,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"type\":\"search\",\"gridData\":{\"h\":40,\"i\":\"4\",\"w\":48,\"x\":0,\"y\":12},\"panelIndex\":\"4\",\"embeddableConfig\":{\"columns\":[\"log.level\",\"logstash.log.integration\",\"message\",\"source\"],\"enhancements\":{},\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_4\"}]", "timeRestore": false, "title": "[Logs Logstash] Logstash Logs", "version": 1 }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-Logs-Logstash-Log-Dashboard", "migrationVersion": { - "dashboard": "7.14.0" + "dashboard": "8.5.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6", @@ -103,7 +31,19 @@ "id": "logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6", "name": "4:panel_4", "type": "search" + }, + { + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "dashboard" + "type": "dashboard", + "updated_at": "2022-09-14T17:40:15.454Z", + "version": "WzE0MjIsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Slowlog-Dashboard.json b/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Slowlog-Dashboard.json index 2f2defe98ad..32a4bd8f9fd 100644 --- a/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Slowlog-Dashboard.json +++ b/packages/logstash/kibana/dashboard/logstash-Logs-Logstash-Slowlog-Dashboard.json @@ -3,117 +3,19 @@ "description": "Overview of Logstash Slowlogs", "hits": 0, "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "highlightAll": true, - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } + "searchSourceJSON": "{\"filter\":[],\"highlightAll\":true,\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true}" }, - "optionsJSON": { - "darkTheme": false - }, - "panelsJSON": [ - { - "embeddableConfig": { - "columns": [ - "log.level", - "logstash.slowlog.plugin_type", - "logstash.slowlog.plugin_name", - "logstash.slowlog.message", - "logstash.slowlog.plugin_params", - "logstash.slowlog.execution_time_ns" - ], - "enhancements": {}, - "sort": [ - "@timestamp", - "desc" - ] - }, - "gridData": { - "h": 36, - "i": "1", - "w": 48, - "x": 0, - "y": 24 - }, - "panelIndex": "1", - "panelRefName": "panel_1", - "type": "search", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 12, - "i": "2", - "w": 24, - "x": 24, - "y": 0 - }, - "panelIndex": "2", - "panelRefName": "panel_2", - "type": "visualization", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "enhancements": {} - }, - "gridData": { - "h": 12, - "i": "3", - "w": 24, - "x": 0, - "y": 0 - }, - "panelIndex": "3", - "panelRefName": "panel_3", - "type": "visualization", - "version": "7.3.0" - }, - { - "embeddableConfig": { - "enhancements": {}, - "vis": { - "params": { - "sort": { - "columnIndex": 3, - "direction": null - } - } - } - }, - "gridData": { - "h": 12, - "i": "4", - "w": 48, - "x": 0, - "y": 12 - }, - "panelIndex": "4", - "panelRefName": "panel_4", - "type": "visualization", - "version": "7.3.0" - } - ], + "optionsJSON": "{\"darkTheme\":false}", + "panelsJSON": "[{\"version\":\"7.3.0\",\"type\":\"search\",\"gridData\":{\"h\":36,\"i\":\"1\",\"w\":48,\"x\":0,\"y\":24},\"panelIndex\":\"1\",\"embeddableConfig\":{\"columns\":[\"log.level\",\"logstash.slowlog.plugin_type\",\"logstash.slowlog.plugin_name\",\"logstash.slowlog.message\",\"logstash.slowlog.plugin_params\",\"logstash.slowlog.execution_time_ns\"],\"enhancements\":{},\"sort\":[\"@timestamp\",\"desc\"]},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"h\":12,\"i\":\"2\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"h\":12,\"i\":\"3\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"h\":12,\"i\":\"4\",\"w\":48,\"x\":0,\"y\":12},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{},\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":null}}}},\"panelRefName\":\"panel_4\"}]", "timeRestore": false, "title": "[Logs Logstash] Slowlogs", "version": 1 }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-Logs-Logstash-Slowlog-Dashboard", "migrationVersion": { - "dashboard": "7.14.0" + "dashboard": "8.5.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6", @@ -134,7 +36,19 @@ "id": "logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6", "name": "4:panel_4", "type": "visualization" + }, + { + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "dashboard" + "type": "dashboard", + "updated_at": "2022-09-14T17:40:15.454Z", + "version": "WzE0MjMsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/search/logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6.json b/packages/logstash/kibana/search/logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6.json index 5088460612b..ed9c2e5efdf 100644 --- a/packages/logstash/kibana/search/logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6.json +++ b/packages/logstash/kibana/search/logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6.json @@ -9,64 +9,12 @@ "logstash.slowlog.execution_time_ns" ], "description": "", + "grid": {}, + "hideChart": false, "hits": 0, + "isTextBasedQuery": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "query", - "negate": false, - "type": "custom", - "value": "{\"prefix\":{\"data_stream.dataset\":\"logstash.\"}}" - }, - "query": { - "prefix": { - "data_stream.dataset": "logstash." - } - } - }, - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", - "key": "fileset.name", - "negate": false, - "params": { - "query": "slowlog", - "type": "phrase" - }, - "type": "phrase", - "value": "slowlog" - }, - "query": { - "match": { - "fileset.name": { - "query": "slowlog", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } + "searchSourceJSON": "{\"highlightAll\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"data_stream.dataset\",\"params\":{\"query\":\"logstash.slowlog\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"data_stream.dataset\":\"logstash.slowlog\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ @@ -74,17 +22,15 @@ "asc" ] ], + "timeRestore": false, "title": "Slow logs [Logs Logstash]", "version": 1 }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6", "migrationVersion": { - "search": "7.9.3" + "search": "8.0.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logs-*", @@ -97,10 +43,17 @@ "type": "index-pattern" }, { - "id": "logs-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", - "type": "index-pattern" + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "search" + "type": "search", + "updated_at": "2022-09-14T14:45:48.992Z", + "version": "WzExMDQsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/search/logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6.json b/packages/logstash/kibana/search/logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6.json index def307a881c..a04b2b8a5d2 100644 --- a/packages/logstash/kibana/search/logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6.json +++ b/packages/logstash/kibana/search/logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6.json @@ -7,64 +7,12 @@ "source" ], "description": "", + "grid": {}, + "hideChart": false, "hits": 0, + "isTextBasedQuery": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "query", - "negate": false, - "type": "custom", - "value": "{\"prefix\":{\"data_stream.dataset\":\"logstash.\"}}" - }, - "query": { - "prefix": { - "data_stream.dataset": "logstash." - } - } - }, - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", - "key": "fileset.name", - "negate": false, - "params": { - "query": "log", - "type": "phrase" - }, - "type": "phrase", - "value": "log" - }, - "query": { - "match": { - "fileset.name": { - "query": "log", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", - "query": { - "language": "kuery", - "query": "" - }, - "version": true - } + "searchSourceJSON": "{\"highlightAll\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"version\":true,\"filter\":[{\"meta\":{\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"data_stream.dataset\",\"params\":{\"query\":\"logstash.log\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"data_stream.dataset\":\"logstash.log\"}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" }, "sort": [ [ @@ -72,17 +20,15 @@ "desc" ] ], + "timeRestore": false, "title": "logs [Logs Logstash]", "version": 1 }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6", "migrationVersion": { - "search": "7.9.3" + "search": "8.0.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logs-*", @@ -95,10 +41,17 @@ "type": "index-pattern" }, { - "id": "logs-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[1].meta.index", - "type": "index-pattern" + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "search" + "type": "search", + "updated_at": "2022-09-14T14:45:44.856Z", + "version": "WzEwODUsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/visualization/logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6.json b/packages/logstash/kibana/visualization/logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6.json index 2380d226196..59ffa4124b8 100644 --- a/packages/logstash/kibana/visualization/logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6.json +++ b/packages/logstash/kibana/visualization/logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6.json @@ -2,70 +2,37 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"kuery\",\"query\":\"\"}}" }, "savedSearchRefName": "search_0", "title": "Logs Severity [Logs Logstash]", - "uiStateJSON": {}, + "uiStateJSON": "{}", "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "field": "log.level", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "segment", - "type": "terms" - } - ], - "params": { - "addLegend": true, - "addTooltip": true, - "distinctColors": true, - "isDonut": true, - "legendPosition": "right", - "palette": { - "name": "kibana_palette", - "type": "palette" - }, - "type": "pie" - }, - "title": "Logs Severity [Logs Logstash]", - "type": "pie" - } + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"segment\",\"type\":\"terms\"}],\"params\":{\"addTooltip\":true,\"distinctColors\":true,\"isDonut\":true,\"legendPosition\":\"right\",\"palette\":{\"name\":\"kibana_palette\",\"type\":\"palette\"},\"type\":\"pie\",\"legendDisplay\":\"show\",\"legendSize\":\"auto\"},\"title\":\"Logs Severity [Logs Logstash]\",\"type\":\"pie\"}" }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-0b1dace0-cbdb-11e7-9852-73e0a9df1bb6", "migrationVersion": { - "visualization": "7.14.0" + "visualization": "8.5.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6", "name": "search_0", "type": "search" + }, + { + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "visualization" + "type": "visualization", + "updated_at": "2022-09-14T17:40:15.454Z", + "version": "WzE0MjQsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/visualization/logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6.json b/packages/logstash/kibana/visualization/logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6.json index 523522a39e6..b657e7e09c2 100644 --- a/packages/logstash/kibana/visualization/logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6.json +++ b/packages/logstash/kibana/visualization/logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6.json @@ -2,116 +2,37 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"kuery\",\"query\":\"\"}}" }, "savedSearchRefName": "search_0", "title": "Slowest plugins [Logs Logstash]", - "uiStateJSON": { - "vis": { - "params": { - "sort": { - "columnIndex": 3, - "direction": null - } - } - } - }, + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":null}}}}", "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "6", - "params": { - "customLabel": "Average", - "field": "logstash.slowlog.took_in_millis" - }, - "schema": "metric", - "type": "avg" - }, - { - "enabled": true, - "id": "5", - "params": { - "customLabel": "Min", - "field": "logstash.slowlog.took_in_millis" - }, - "schema": "metric", - "type": "min" - }, - { - "enabled": true, - "id": "8", - "params": { - "customLabel": "Plugin Name", - "field": "logstash.slowlog.plugin_name", - "order": "desc", - "orderBy": "5", - "size": 5 - }, - "schema": "bucket", - "type": "terms" - }, - { - "enabled": true, - "id": "7", - "params": { - "customLabel": "Max", - "field": "logstash.slowlog.took_in_millis" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "9", - "params": { - "customLabel": "Plugin Type", - "field": "logstash.slowlog.plugin_type", - "order": "desc", - "orderBy": "5", - "size": 5 - }, - "schema": "bucket", - "type": "terms" - } - ], - "params": { - "perPage": 10, - "showMeticsAtAllLevels": false, - "showPartialRows": false, - "showToolbar": true, - "showTotal": false, - "sort": { - "columnIndex": null, - "direction": null - }, - "totalFunc": "sum" - }, - "title": "Slowest plugins [Logs Logstash]", - "type": "table" - } + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"6\",\"params\":{\"customLabel\":\"Average\",\"field\":\"logstash.slowlog.took_in_millis\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"5\",\"params\":{\"customLabel\":\"Min\",\"field\":\"logstash.slowlog.took_in_millis\"},\"schema\":\"metric\",\"type\":\"min\"},{\"enabled\":true,\"id\":\"8\",\"params\":{\"customLabel\":\"Plugin Name\",\"field\":\"logstash.slowlog.plugin_name\",\"order\":\"desc\",\"orderBy\":\"5\",\"size\":5},\"schema\":\"bucket\",\"type\":\"terms\"},{\"enabled\":true,\"id\":\"7\",\"params\":{\"customLabel\":\"Max\",\"field\":\"logstash.slowlog.took_in_millis\"},\"schema\":\"metric\",\"type\":\"max\"},{\"enabled\":true,\"id\":\"9\",\"params\":{\"customLabel\":\"Plugin Type\",\"field\":\"logstash.slowlog.plugin_type\",\"order\":\"desc\",\"orderBy\":\"5\",\"size\":5},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showToolbar\":true,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Slowest plugins [Logs Logstash]\",\"type\":\"table\"}" }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-b3315630-cbdf-11e7-9852-73e0a9df1bb6", "migrationVersion": { - "visualization": "7.14.0" + "visualization": "8.5.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logstash-742e45d0-cbdd-11e7-9852-73e0a9df1bb6", "name": "search_0", "type": "search" + }, + { + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "visualization" + "type": "visualization", + "updated_at": "2022-09-14T17:40:15.454Z", + "version": "WzE0MjUsMV0=" } \ No newline at end of file diff --git a/packages/logstash/kibana/visualization/logstash-e90b7240-cbda-11e7-9852-73e0a9df1bb6.json b/packages/logstash/kibana/visualization/logstash-e90b7240-cbda-11e7-9852-73e0a9df1bb6.json index 6dc1c6f4b56..4e7b8329127 100644 --- a/packages/logstash/kibana/visualization/logstash-e90b7240-cbda-11e7-9852-73e0a9df1bb6.json +++ b/packages/logstash/kibana/visualization/logstash-e90b7240-cbda-11e7-9852-73e0a9df1bb6.json @@ -2,145 +2,37 @@ "attributes": { "description": "", "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } + "searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"kuery\",\"query\":\"\"}}" }, "savedSearchRefName": "search_0", "title": "logs over time [Logs Logstash]", - "uiStateJSON": {}, + "uiStateJSON": "{}", "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "field": "log.level", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "group", - "type": "terms" - } - ], - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "filter": true, - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" - } - ], - "detailedTooltip": true, - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } - }, - "isVislibVis": true, - "legendPosition": "right", - "palette": { - "name": "kibana_palette", - "type": "palette" - }, - "seriesParams": [ - { - "data": { - "id": "1", - "label": "Count" - }, - "drawLinesBetweenPoints": true, - "mode": "stacked", - "show": "true", - "showCircles": true, - "type": "histogram", - "valueAxis": "ValueAxis-1" - } - ], - "times": [], - "type": "histogram", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "mode": "normal", - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "Count" - }, - "type": "value" - } - ] - }, - "title": "logs over time [Logs Logstash]", - "type": "histogram" - } + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"detailedTooltip\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"isVislibVis\":true,\"legendPosition\":\"right\",\"palette\":{\"name\":\"kibana_palette\",\"type\":\"palette\"},\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"legendSize\":\"auto\"},\"title\":\"logs over time [Logs Logstash]\",\"type\":\"histogram\"}" }, - "coreMigrationVersion": "7.15.0", + "coreMigrationVersion": "8.5.0", "id": "logstash-e90b7240-cbda-11e7-9852-73e0a9df1bb6", "migrationVersion": { - "visualization": "7.14.0" + "visualization": "8.5.0" }, - "namespaces": [ - "default" - ], "references": [ { "id": "logstash-cfaba090-cbda-11e7-9852-73e0a9df1bb6", "name": "search_0", "type": "search" + }, + { + "id": "managed", + "name": "tag-ref-managed", + "type": "tag" + }, + { + "id": "logstash", + "name": "tag-ref-logstash", + "type": "tag" } ], - "type": "visualization" + "type": "visualization", + "updated_at": "2022-09-14T17:40:15.454Z", + "version": "WzE0MjYsMV0=" } \ No newline at end of file