[logstash] Add missing monitoring cpu aliases (#5431)

* add missing monitoring aliases * bump image version * bump version * use 8.5.0 * update description and titles * bump variant and generate sample events * use 8.5.0 * generate sample_event * build docs * changelog * bump variant to 8.6.0
elastic · Mar 7, 2023 · b72fe2e · b72fe2e
1 parent 69aa2ac
commit b72fe2e
Show file tree

Hide file tree

Showing 12 changed files with 339 additions and 307 deletions.
diff --git a/packages/logstash/_dev/deploy/docker/docker-compose.yml b/packages/logstash/_dev/deploy/docker/docker-compose.yml
@@ -2,7 +2,7 @@ version: "2.3"
 services:
   logstash:
     user: root
-    image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.5.0-SNAPSHOT}"
+    image: "docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.6.0}"
     volumes:
       - "./pipeline:/usr/share/logstash/pipeline"
       - "./config:/usr/share/logstash/config"

diff --git a/packages/logstash/_dev/deploy/variants.yml b/packages/logstash/_dev/deploy/variants.yml
@@ -1,4 +1,4 @@
 variants:
-  logstash_8.5.0:
-    ELASTIC_VERSION: 8.5.0-SNAPSHOT
-default: logstash_8.5.0
+  logstash_8.6.0:
+    ELASTIC_VERSION: 8.6.0
+default: logstash_8.6.0
diff --git a/packages/logstash/changelog.yml b/packages/logstash/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.2-preview1"
+  changes:
+    - description: Add missing node_stats cpu aliases
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/5431
 - version: "2.2.1-preview1"
   changes:
     - description: Add period variable to define polling frequency

diff --git a/packages/logstash/data_stream/log/sample_event.json b/packages/logstash/data_stream/log/sample_event.json
@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2022-10-11T14:03:32.641Z",
+    "@timestamp": "2023-03-02T15:56:09.250Z",
     "agent": {
-        "ephemeral_id": "e1088945-f8e4-466b-83fd-f636ffbf9bdf",
-        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
+        "ephemeral_id": "350b18f5-b2c5-44c2-9968-316905d2cfa0",
+        "id": "3cc85092-54dc-4b58-8726-5e9458167f42",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.5.0"
@@ -16,34 +16,34 @@
         "version": "1.10.0"
     },
     "elastic_agent": {
-        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
-        "snapshot": true,
+        "id": "3cc85092-54dc-4b58-8726-5e9458167f42",
+        "snapshot": false,
         "version": "8.5.0"
     },
     "event": {
         "agent_id_status": "verified",
-        "created": "2022-10-11T14:03:32.641Z",
+        "created": "2023-03-02T15:56:09.250Z",
         "dataset": "logstash.log",
-        "ingested": "2022-10-11T14:03:44Z",
+        "ingested": "2023-03-02T15:56:12Z",
         "kind": "event",
         "type": "info"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": false,
+        "containerized": true,
         "hostname": "docker-fleet-agent",
-        "id": "b6bc6723e51b43959ce07f0c3105c72d",
+        "id": "66392b0697b84641af8006d87aeb89f1",
         "ip": [
-            "192.168.0.7"
+            "192.168.224.7"
         ],
         "mac": [
-            "02-42-C0-A8-00-07"
+            "02-42-C0-A8-E0-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.124-linuxkit",
+            "kernel": "5.10.47-linuxkit",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -55,17 +55,17 @@
     },
     "log": {
         "file": {
-            "path": "/tmp/service_logs/logstash/logstash-json.log"
+            "path": "/tmp/service_logs/logstash-json.log"
         },
-        "level": "INFO",
+        "level": "WARN",
         "offset": 0
     },
     "logstash": {
         "log": {
             "log_event": {},
-            "module": "logstash.runner",
+            "module": "deprecation.logstash.runner",
             "thread": "main"
         }
     },
-    "message": "Log4j configuration path used is: /usr/share/logstash/config/log4j2.properties"
+    "message": "NOTICE: Running Logstash as superuser is not recommended and won't be allowed in the future. Set 'allow_superuser' to 'false' to avoid startup errors in future releases."
 }
diff --git a/packages/logstash/data_stream/node/sample_event.json b/packages/logstash/data_stream/node/sample_event.json
@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2022-10-11T14:04:44.089Z",
+    "@timestamp": "2023-03-02T15:57:03.999Z",
     "agent": {
-        "ephemeral_id": "1a1ca75b-a20f-4ae4-82a9-4e269c855a5d",
-        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
+        "ephemeral_id": "16f2dd63-454b-4699-a8c8-2a748bd044b8",
+        "id": "3cc85092-54dc-4b58-8726-5e9458167f42",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
         "version": "8.5.0"
@@ -16,33 +16,33 @@
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
-        "snapshot": true,
+        "id": "3cc85092-54dc-4b58-8726-5e9458167f42",
+        "snapshot": false,
         "version": "8.5.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "logstash.stack_monitoring.node",
-        "duration": 131377542,
-        "ingested": "2022-10-11T14:04:45Z",
+        "duration": 69490100,
+        "ingested": "2023-03-02T15:57:05Z",
         "module": "logstash"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": false,
+        "containerized": true,
         "hostname": "docker-fleet-agent",
-        "id": "b6bc6723e51b43959ce07f0c3105c72d",
+        "id": "66392b0697b84641af8006d87aeb89f1",
         "ip": [
-            "192.168.0.7"
+            "192.168.224.7"
         ],
         "mac": [
-            "02-42-C0-A8-00-07"
+            "02-42-C0-A8-E0-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.124-linuxkit",
+            "kernel": "5.10.47-linuxkit",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -51,23 +51,23 @@
     },
     "logstash": {
         "cluster": {
-            "id": "U8DCOXCFQHWlaKczNT4LNQ"
+            "id": "0toa26-cTzmqx0WD40-4XQ"
         },
         "elasticsearch": {
             "cluster": {
-                "id": "U8DCOXCFQHWlaKczNT4LNQ"
+                "id": "0toa26-cTzmqx0WD40-4XQ"
             }
         },
         "node": {
-            "host": "17a6005cfeaa",
-            "id": "7d7ee953-cf82-4d1d-91e0-1714346531de",
+            "host": "45730b5f8c3d",
+            "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a",
             "jvm": {
                 "version": "17.0.4"
             },
             "state": {
                 "pipeline": {
                     "batch_size": 125,
-                    "ephemeral_id": "3d2aff1f-dde1-4c56-9560-14f3b092f894",
+                    "ephemeral_id": "472cf082-aa15-41ca-9ed1-62d03afbadd0",
                     "hash": "d83c53e142e85177df0f039e5b9f4575b858e9cfdd51c2c60b1a9e8d5f9b1aaa",
                     "id": "pipeline-with-persisted-queue",
                     "representation": {
@@ -150,7 +150,7 @@
                         "type": "lir",
                         "version": "0.0.0"
                     },
-                    "workers": 7
+                    "workers": 10
                 }
             },
             "version": "8.5.0"
@@ -164,9 +164,9 @@
         "pid": 1
     },
     "service": {
-        "address": "http://elastic-package-service-logstash-1:9600/_node",
-        "hostname": "17a6005cfeaa",
-        "id": "7d7ee953-cf82-4d1d-91e0-1714346531de",
+        "address": "http://elastic-package-service_logstash_1:9600/_node",
+        "hostname": "45730b5f8c3d",
+        "id": "2e17cd45-ecb8-4358-a420-b867f2e32b7a",
         "name": "logstash",
         "type": "logstash",
         "version": "8.5.0"

diff --git a/packages/logstash/data_stream/node_stats/fields/fields.yml b/packages/logstash/data_stream/node_stats/fields/fields.yml
@@ -123,7 +123,7 @@
                   type: group
                   fields:
                     - name: cpuacct
-                      type: object
+                      type: group
                       fields:
                         - name: control_group
                           type: text
@@ -137,7 +137,7 @@
                         - name: cfs_quota_micros
                           type: long
                         - name: stat
-                          type: object
+                          type: group
                           fields:
                             - name: number_of_elapsed_periods
                               type: long

diff --git a/packages/logstash/data_stream/node_stats/fields/package-fields.yml b/packages/logstash/data_stream/node_stats/fields/package-fields.yml
@@ -84,11 +84,29 @@
           type: group
           fields:
             - name: cpuacct
-              type: object
+              type: group
               fields:
                 - name: usage_nanos
                   type: alias
                   path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos
+            - name: cpu
+              type: group
+              fields:
+                - name: cfs_quota_micros
+                  path: logstash.node.stats.os.cgroup.cpu.cfs_quota_micros
+                  type: alias
+                - name: stat
+                  type: group
+                  fields:
+                    - name: number_of_elapsed_periods
+                      path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods
+                      type: alias
+                    - name: number_of_times_throttled
+                      path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled
+                      type: alias
+                    - name: time_throttled_nanos
+                      path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos
+                      type: alias
     - name: process.cpu.percent
       type: alias
       path: logstash.node.stats.process.cpu.percent

diff --git a/packages/logstash/data_stream/node_stats/manifest.yml b/packages/logstash/data_stream/node_stats/manifest.yml
@@ -8,8 +8,8 @@ elasticsearch:
       dynamic: false
 streams:
   - input: logstash/metrics
-    title: Logstash node stats
-    description: Collect Logstash node stats
+    title: Logstash node stats metrics
+    description: Collect Logstash node stats metrics
     vars:
       - name: period
         type: text