From 0cad2c01abb31bc7a2283bb5d74679a91d31a157 Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Wed, 11 Dec 2024 16:42:19 +0200 Subject: [PATCH 1/7] Support Podman metrics --- packages/docker/changelog.yml | 5 +++++ .../data_stream/cpu/agent/stream/stream.yml.hbs | 3 ++- .../data_stream/memory/agent/stream/stream.yml.hbs | 3 ++- packages/docker/manifest.yml | 14 ++++++++++++-- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 03c88f04f6c..fde2af2c227 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 2.12.0 + changes: + - description: Support for podman metrics collection. + type: enhancement + link: https://github.com/elastic/integrations/pull/11196 - version: 2.11.0 changes: - description: Bump package-spec version to 3.2.2 to run on Serverless and stack version 9.0. diff --git a/packages/docker/data_stream/cpu/agent/stream/stream.yml.hbs b/packages/docker/data_stream/cpu/agent/stream/stream.yml.hbs index 202ea776e55..a1551ad1934 100644 --- a/packages/docker/data_stream/cpu/agent/stream/stream.yml.hbs +++ b/packages/docker/data_stream/cpu/agent/stream/stream.yml.hbs @@ -4,4 +4,5 @@ hosts: - {{this}} {{/each}} period: {{period}} -labels.dedot: {{labels.dedot}} \ No newline at end of file +labels.dedot: {{labels.dedot}} +podman: {{podman}} \ No newline at end of file diff --git a/packages/docker/data_stream/memory/agent/stream/stream.yml.hbs b/packages/docker/data_stream/memory/agent/stream/stream.yml.hbs index 7d3580d9fa9..4a9b238ba31 100644 --- a/packages/docker/data_stream/memory/agent/stream/stream.yml.hbs +++ b/packages/docker/data_stream/memory/agent/stream/stream.yml.hbs @@ -4,4 +4,5 @@ hosts: - {{this}} {{/each}} period: {{period}} -labels.dedot: {{labels.dedot}} \ No newline at end of file +labels.dedot: {{labels.dedot}} +podman: {{podman}} \ No newline at end of file diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index ef3189161d7..67503a2acd2 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.11.0 +version: 2.12.0 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: @@ -19,7 +19,7 @@ categories: - containers conditions: kibana: - version: ^8.8.0 + version: "^8.16.2" policy_templates: - name: docker title: Docker logs and metrics @@ -28,6 +28,16 @@ policy_templates: - type: docker/metrics title: Collect Docker metrics description: Collecting container, cpu, diskio, event, healthcheck, image, info, memory and network metrics from Docker instances + vars: + - name: podman + type: bool + title: Podman + multi: false + required: false + show_user: true + default: false + description: > + If True, Docker integration is used to collect metrics for Podman - type: filestream title: Collect Docker container logs description: Collecting docker container logs From 42d6a9b7013c7a5d66baf14058ac669dc5b16f14 Mon Sep 17 00:00:00 2001 From: Michael Katsoulis Date: Mon, 23 Dec 2024 15:18:31 +0200 Subject: [PATCH 2/7] Update changelog.yml --- packages/docker/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 1061c378597..e644b05657a 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Support for podman metrics collection. type: enhancement - link: https://github.com/elastic/integrations/pull/11196 + link: https://github.com/elastic/integrations/pull/12073 - version: 2.12.0 changes: - description: Use ecs definition of the 'event.dataset' field. From 54d0b527acd69adf978422789e12a3e1a7d9e5e0 Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Mon, 23 Dec 2024 16:01:24 +0200 Subject: [PATCH 3/7] Update docker package readme --- packages/docker/changelog.yml | 2 +- packages/docker/docs/README.md | 7 +++++-- packages/docker/manifest.yml | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index e644b05657a..9fd1a75e795 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: 2.13.0 +- version: 2.13.1 changes: - description: Support for podman metrics collection. type: enhancement diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index b17bd9a0bd0..bf9e2a667ac 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -7,8 +7,9 @@ The `container_logs` data stream for containers' logs collection is enabled by d ## Compatibility -The Docker module is currently tested on Linux and Mac with the community -edition engine, versions 1.11 and 17.09.0-ce. +The Docker module is currently tested on Linux and Mac with the community edition engine, versions 1.11 and 17.09.0-ce. It is not tested on Windows, but it should also work there. + +The Docker module supports collection of metrics from Podman’s Docker-compatible API. It has been tested on Linux and Mac with Podman Rest API v2.0.0 and above. ## Running from within Docker @@ -52,6 +53,8 @@ Docker API already takes up to 2 seconds. Specifying less than 3 seconds will result in requests that timeout, and no data will be reported for those requests. +In the case of Podman, the configuration parameter podman should be switched to true. This enables streaming of container stats output, which allows for more accurate CPU percentage calculations when using Podman. + ## Metrics ### Container diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index 49397e78c0b..ab5d8bc8a4d 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.13.0 +version: 2.13.1 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: @@ -19,7 +19,7 @@ categories: - containers conditions: kibana: - version: "^8.16.2" + version: ^8.16.2 policy_templates: - name: docker title: Docker logs and metrics From 1169077d79fda85007fd422729437d476ce1d10f Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Mon, 23 Dec 2024 16:06:24 +0200 Subject: [PATCH 4/7] Add readme --- packages/docker/changelog.yml | 2 +- packages/docker/docs/README.md | 7 ++----- packages/docker/manifest.yml | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 9fd1a75e795..b08bfc77dee 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: 2.13.1 +- version: 2.13.2 changes: - description: Support for podman metrics collection. type: enhancement diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index bf9e2a667ac..b17bd9a0bd0 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -7,9 +7,8 @@ The `container_logs` data stream for containers' logs collection is enabled by d ## Compatibility -The Docker module is currently tested on Linux and Mac with the community edition engine, versions 1.11 and 17.09.0-ce. It is not tested on Windows, but it should also work there. - -The Docker module supports collection of metrics from Podman’s Docker-compatible API. It has been tested on Linux and Mac with Podman Rest API v2.0.0 and above. +The Docker module is currently tested on Linux and Mac with the community +edition engine, versions 1.11 and 17.09.0-ce. ## Running from within Docker @@ -53,8 +52,6 @@ Docker API already takes up to 2 seconds. Specifying less than 3 seconds will result in requests that timeout, and no data will be reported for those requests. -In the case of Podman, the configuration parameter podman should be switched to true. This enables streaming of container stats output, which allows for more accurate CPU percentage calculations when using Podman. - ## Metrics ### Container diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index ab5d8bc8a4d..248c44c18bb 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.13.1 +version: 2.13.2 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: From 5bcc6e4338ebb1a502632f7dad36328675923a84 Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Mon, 23 Dec 2024 16:11:54 +0200 Subject: [PATCH 5/7] Add readme --- packages/docker/_dev/build/docs/README.md | 7 +++++-- packages/docker/changelog.yml | 2 +- packages/docker/docs/README.md | 7 +++++-- packages/docker/manifest.yml | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/docker/_dev/build/docs/README.md b/packages/docker/_dev/build/docs/README.md index ce7d464d400..070bb87ade8 100644 --- a/packages/docker/_dev/build/docs/README.md +++ b/packages/docker/_dev/build/docs/README.md @@ -7,8 +7,9 @@ The `container_logs` data stream for containers' logs collection is enabled by d ## Compatibility -The Docker module is currently tested on Linux and Mac with the community -edition engine, versions 1.11 and 17.09.0-ce. +The Docker module is currently tested on Linux and Mac with the community edition engine, versions 1.11 and 17.09.0-ce. It is not tested on Windows, but it should also work there. + +The Docker module supports collection of metrics from Podman’s Docker-compatible API. It has been tested on Linux and Mac with Podman Rest API v2.0.0 and above. ## Running from within Docker @@ -52,6 +53,8 @@ Docker API already takes up to 2 seconds. Specifying less than 3 seconds will result in requests that timeout, and no data will be reported for those requests. +In the case of Podman, the configuration parameter podman should be switched to true. This enables streaming of container stats output, which allows for more accurate CPU percentage calculations when using Podman. + ## Metrics ### Container diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index b08bfc77dee..691412a146e 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: 2.13.2 +- version: 2.13.3 changes: - description: Support for podman metrics collection. type: enhancement diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index b17bd9a0bd0..bf9e2a667ac 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -7,8 +7,9 @@ The `container_logs` data stream for containers' logs collection is enabled by d ## Compatibility -The Docker module is currently tested on Linux and Mac with the community -edition engine, versions 1.11 and 17.09.0-ce. +The Docker module is currently tested on Linux and Mac with the community edition engine, versions 1.11 and 17.09.0-ce. It is not tested on Windows, but it should also work there. + +The Docker module supports collection of metrics from Podman’s Docker-compatible API. It has been tested on Linux and Mac with Podman Rest API v2.0.0 and above. ## Running from within Docker @@ -52,6 +53,8 @@ Docker API already takes up to 2 seconds. Specifying less than 3 seconds will result in requests that timeout, and no data will be reported for those requests. +In the case of Podman, the configuration parameter podman should be switched to true. This enables streaming of container stats output, which allows for more accurate CPU percentage calculations when using Podman. + ## Metrics ### Container diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index 248c44c18bb..a1025f26db1 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.13.2 +version: 2.13.3 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: From 382faf0dfe7c11cd6bda96fbcf6048a4bf8d4a5c Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Mon, 23 Dec 2024 16:56:27 +0200 Subject: [PATCH 6/7] Add docker.in,* fields --- packages/docker/changelog.yml | 2 +- .../data_stream/network/fields/fields.yml | 44 +++++++++++++++++++ packages/docker/manifest.yml | 2 +- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/docker/changelog.yml b/packages/docker/changelog.yml index 691412a146e..e644b05657a 100644 --- a/packages/docker/changelog.yml +++ b/packages/docker/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: 2.13.3 +- version: 2.13.0 changes: - description: Support for podman metrics collection. type: enhancement diff --git a/packages/docker/data_stream/network/fields/fields.yml b/packages/docker/data_stream/network/fields/fields.yml index fc5b1dcad5a..57f0540c375 100644 --- a/packages/docker/data_stream/network/fields/fields.yml +++ b/packages/docker/data_stream/network/fields/fields.yml @@ -12,6 +12,50 @@ dimension: true description: | Network interface name. + - name: in + type: group + description: > + Incoming network stats per second. + fields: + - name: bytes + type: long + format: bytes + description: > + Incoming bytes per seconds. + - name: dropped + type: scaled_float + description: > + Dropped incoming packets per second. + - name: errors + type: long + description: > + Errors on incoming packets per second. + - name: packets + type: long + description: > + Incoming packets per second. + - name: out + type: group + description: > + Outgoing network stats per second. + fields: + - name: bytes + type: long + format: bytes + description: > + Outgoing bytes per second. + - name: dropped + type: scaled_float + description: > + Dropped outgoing packets per second. + - name: errors + type: long + description: > + Errors on outgoing packets per second. + - name: packets + type: long + description: > + Outgoing packets per second. - name: inbound type: group fields: diff --git a/packages/docker/manifest.yml b/packages/docker/manifest.yml index a1025f26db1..8e4faa0641d 100644 --- a/packages/docker/manifest.yml +++ b/packages/docker/manifest.yml @@ -1,6 +1,6 @@ name: docker title: Docker -version: 2.13.3 +version: 2.13.0 description: Collect metrics and logs from Docker instances with Elastic Agent. type: integration icons: From a1b0a87380ad944473940575086a107811c07707 Mon Sep 17 00:00:00 2001 From: Michalis Katsoulis Date: Mon, 23 Dec 2024 16:56:59 +0200 Subject: [PATCH 7/7] Add docker.in,* fields definitions --- packages/docker/docs/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/docker/docs/README.md b/packages/docker/docs/README.md index bf9e2a667ac..b2024b89782 100644 --- a/packages/docker/docs/README.md +++ b/packages/docker/docs/README.md @@ -1018,11 +1018,19 @@ The Docker `network` data stream collects network metrics. | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | | docker.container.labels.\* | Container labels | object | | +| docker.network.in.bytes | Incoming bytes per seconds. | long | | +| docker.network.in.dropped | Dropped incoming packets per second. | scaled_float | | +| docker.network.in.errors | Errors on incoming packets per second. | long | | +| docker.network.in.packets | Incoming packets per second. | long | | | docker.network.inbound.bytes | Total number of incoming bytes. | long | counter | | docker.network.inbound.dropped | Total number of dropped incoming packets. | long | counter | | docker.network.inbound.errors | Total errors on incoming packets. | long | counter | | docker.network.inbound.packets | Total number of incoming packets. | long | counter | | docker.network.interface | Network interface name. | keyword | | +| docker.network.out.bytes | Outgoing bytes per second. | long | | +| docker.network.out.dropped | Dropped outgoing packets per second. | scaled_float | | +| docker.network.out.errors | Errors on outgoing packets per second. | long | | +| docker.network.out.packets | Outgoing packets per second. | long | | | docker.network.outbound.bytes | Total number of outgoing bytes. | long | counter | | docker.network.outbound.dropped | Total number of dropped outgoing packets. | long | counter | | docker.network.outbound.errors | Total errors on outgoing packets. | long | counter |