From 3c678779f023e2bd0ccf095e1475623a5680a531 Mon Sep 17 00:00:00 2001 From: StrongestNumber9 <16169054+StrongestNumber9@users.noreply.github.com> Date: Mon, 22 May 2023 14:12:05 -0400 Subject: [PATCH 1/4] Adds prometheus metrics endpoint --- .gitignore | 1 + Dockerfile | 2 +- etc/config.json | 3 + example/combined.yaml | 7 ++- example/config/config.json | 3 + pom.xml | 30 ++++++++++ .../teragrep/k8s_01/KubernetesLogReader.java | 2 + .../teragrep/k8s_01/PrometheusMetrics.java | 60 +++++++++++++++++++ .../com/teragrep/k8s_01/config/AppConfig.java | 6 ++ .../k8s_01/config/AppConfigMetrics.java | 33 ++++++++++ 10 files changed, 144 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java create mode 100644 src/main/java/com/teragrep/k8s_01/config/AppConfigMetrics.java diff --git a/.gitignore b/.gitignore index 3a855b4..a787546 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ target/** +rpm/target/** .idea/** dependency-reduced-pom.xml var/ diff --git a/Dockerfile b/Dockerfile index b713e25..e49e880 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM rockylinux:8 COPY rpm/target/rpm/com.teragrep-k8s_01/RPMS/noarch/com.teragrep-k8s_01-*.rpm /rpm/ -RUN dnf -y install jq java-1.8.0-headless /rpm/*.rpm && yum clean all +RUN dnf -y install jq /rpm/*.rpm && yum clean all VOLUME /opt/teragrep/k8s_01/var VOLUME /opt/teragrep/k8s_01/etc WORKDIR /opt/teragrep/k8s_01 diff --git a/etc/config.json b/etc/config.json index 5f5b57d..96e32fd 100644 --- a/etc/config.json +++ b/etc/config.json @@ -1,4 +1,7 @@ { + "metrics": { + "port": 12345 + }, "kubernetes": { "logdir": "/var/log/containers", "url": "https://127.0.0.1:8443", diff --git a/example/combined.yaml b/example/combined.yaml index 2eb925a..c1711fd 100644 --- a/example/combined.yaml +++ b/example/combined.yaml @@ -36,6 +36,9 @@ apiVersion: v1 data: config.json: | { + "metrics": { + "port": 12345 + }, "kubernetes": { "logdir": "/var/log/containers", "url": "https://127.0.0.1:8443", @@ -98,7 +101,7 @@ data: kind: ConfigMap metadata: - name: app-config-42gthtbf4f + name: app-config-td2t2mhm2c --- apiVersion: v1 kind: Secret @@ -221,7 +224,7 @@ spec: terminationGracePeriodSeconds: 0 volumes: - configMap: - name: app-config-42gthtbf4f + name: app-config-td2t2mhm2c name: app-config - hostPath: path: /var/log/containers diff --git a/example/config/config.json b/example/config/config.json index 264c49e..9d7cf49 100644 --- a/example/config/config.json +++ b/example/config/config.json @@ -1,4 +1,7 @@ { + "metrics": { + "port": 12345 + }, "kubernetes": { "logdir": "/var/log/containers", "url": "https://127.0.0.1:8443", diff --git a/pom.xml b/pom.xml index 4a10dba..ea7b9f5 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,8 @@ 0.0.1 -SNAPSHOT + 0.16.0 + 9.4.51.v20230217 @@ -130,6 +132,34 @@ 2.0.7 + + + io.prometheus + simpleclient + ${prometheus-simpleclient.version} + + + io.prometheus + simpleclient_dropwizard + ${prometheus-simpleclient.version} + + + io.prometheus + simpleclient_servlet + ${prometheus-simpleclient.version} + + + io.prometheus + simpleclient_hotspot + ${prometheus-simpleclient.version} + + + + org.eclipse.jetty + jetty-servlet + ${prometheus-jettyservlet.version} + + diff --git a/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java b/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java index 9780a34..7df6529 100644 --- a/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java +++ b/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java @@ -64,6 +64,7 @@ public static void main(String[] args) throws IOException { return; } KubernetesCachingAPIClient cacheClient = new KubernetesCachingAPIClient(appConfig.getKubernetes()); + PrometheusMetrics prometheusMetrics = new PrometheusMetrics(appConfig.getMetrics().getPort()); // Pool of Relp output threads to be shared by every consumer BlockingQueue relpOutputPool = new LinkedBlockingDeque<>(appConfig.getRelp().getOutputThreads()); @@ -171,5 +172,6 @@ public static void main(String[] args) throws IOException { throw new RuntimeException(e); } } + prometheusMetrics.close(); } } diff --git a/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java new file mode 100644 index 0000000..7871e57 --- /dev/null +++ b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java @@ -0,0 +1,60 @@ +/* + Kubernetes log forwarder k8s_01 + Copyright (C) 2023 Suomen Kanuuna Oy + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.teragrep.k8s_01; + +import io.prometheus.client.exporter.MetricsServlet; +import io.prometheus.client.hotspot.DefaultExports; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PrometheusMetrics { + private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusMetrics.class); + Server jettyServer; + public PrometheusMetrics(int port) { + LOGGER.info("Starting prometheus metrics server on port {}", port); + // prometheus-exporter + jettyServer = new Server(port); + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + jettyServer.setHandler(context); + + MetricsServlet metricsServlet = new MetricsServlet(); + ServletHolder servletHolder = new ServletHolder(metricsServlet); + context.addServlet(servletHolder, "/metrics"); + // Add metrics about CPU, JVM memory etc. + DefaultExports.initialize(); + // Start the webserver. + try { + jettyServer.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void close() { + LOGGER.info("Closing prometheus metrics server"); + try { + jettyServer.stop(); + } catch (Exception e) { + LOGGER.error("Failed to stop jettyServer:", e); + } + } +} diff --git a/src/main/java/com/teragrep/k8s_01/config/AppConfig.java b/src/main/java/com/teragrep/k8s_01/config/AppConfig.java index 2542f42..f55a192 100644 --- a/src/main/java/com/teragrep/k8s_01/config/AppConfig.java +++ b/src/main/java/com/teragrep/k8s_01/config/AppConfig.java @@ -22,6 +22,12 @@ /* POJO representing the main config.json */ public class AppConfig { private AppConfigKubernetes kubernetes; + + public AppConfigMetrics getMetrics() { + return metrics; + } + + private AppConfigMetrics metrics; private AppConfigRelp relp; public AppConfigKubernetes getKubernetes() { diff --git a/src/main/java/com/teragrep/k8s_01/config/AppConfigMetrics.java b/src/main/java/com/teragrep/k8s_01/config/AppConfigMetrics.java new file mode 100644 index 0000000..1b80966 --- /dev/null +++ b/src/main/java/com/teragrep/k8s_01/config/AppConfigMetrics.java @@ -0,0 +1,33 @@ +/* + Kubernetes log forwarder k8s_01 + Copyright (C) 2023 Suomen Kanuuna Oy + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package com.teragrep.k8s_01.config; + +import com.google.gson.Gson; + +public class AppConfigMetrics { + public int getPort() { + return port; + } + + private int port; + + @Override + public String toString() { + return new Gson().toJson(this); + } +} From 441c9f55ac22771f657c93822de834cebfe4817a Mon Sep 17 00:00:00 2001 From: StrongestNumber9 <16169054+StrongestNumber9@users.noreply.github.com> Date: Tue, 23 May 2023 11:18:37 -0400 Subject: [PATCH 2/4] Adds some metrics using dropwizard tools --- pom.xml | 17 +++++++++ .../teragrep/k8s_01/PrometheusMetrics.java | 35 +++++++++++++++++++ .../java/com/teragrep/k8s_01/RelpOutput.java | 34 +++++++++++++++++- 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea7b9f5..b98cc30 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ 0.16.0 9.4.51.v20230217 + 4.2.18 @@ -159,6 +160,22 @@ jetty-servlet ${prometheus-jettyservlet.version} + + + io.dropwizard.metrics + metrics-core + ${dropwizard-metrics.version} + + + io.dropwizard.metrics + metrics-jmx + ${dropwizard-metrics.version} + + + io.dropwizard.metrics + metrics-jvm + ${dropwizard-metrics.version} + diff --git a/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java index 7871e57..9b3a5c9 100644 --- a/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java +++ b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java @@ -17,6 +17,11 @@ package com.teragrep.k8s_01; +import com.codahale.metrics.*; +import com.codahale.metrics.jmx.JmxReporter; +import com.codahale.metrics.jvm.*; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.dropwizard.DropwizardExports; import io.prometheus.client.exporter.MetricsServlet; import io.prometheus.client.hotspot.DefaultExports; import org.eclipse.jetty.server.Server; @@ -25,6 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.codahale.metrics.MetricRegistry.name; + public class PrometheusMetrics { private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusMetrics.class); Server jettyServer; @@ -39,6 +46,7 @@ public PrometheusMetrics(int port) { MetricsServlet metricsServlet = new MetricsServlet(); ServletHolder servletHolder = new ServletHolder(metricsServlet); context.addServlet(servletHolder, "/metrics"); + setupDropWizard(); // Add metrics about CPU, JVM memory etc. DefaultExports.initialize(); // Start the webserver. @@ -49,6 +57,33 @@ public PrometheusMetrics(int port) { } } + static void setupDropWizard() { + MetricRegistry metricRegistry = new MetricRegistry(); + + // Totals + metricRegistry.register(name("total", "reconnects"), new Counter()); + metricRegistry.register(name("total", "connections"), new Counter()); + + // Throughput meters + metricRegistry.register(name("throughput", "bytes"), new Meter(new SlidingTimeWindowMovingAverages())); + metricRegistry.register(name("throughput", "records"), new Meter(new SlidingTimeWindowMovingAverages())); + metricRegistry.register(name("throughput", "errors"), new Meter(new SlidingTimeWindowMovingAverages())); + + // Misc + metricRegistry.register(name("jvm", "vm"), new JvmAttributeGaugeSet()); + metricRegistry.register(name("jvm", "memory"), new MemoryUsageGaugeSet()); + metricRegistry.register(name("jvm", "threads"), new ThreadStatesGaugeSet()); + metricRegistry.register(name("jvm", "gc"), new GarbageCollectorMetricSet()); + SharedMetricRegistries.add("default", metricRegistry); + + // Add to Prometheus metrics + CollectorRegistry.defaultRegistry.register(new DropwizardExports(metricRegistry)); + + // Enable JMX listener + JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry).build(); + jmxReporter.start(); + } + public void close() { LOGGER.info("Closing prometheus metrics server"); try { diff --git a/src/main/java/com/teragrep/k8s_01/RelpOutput.java b/src/main/java/com/teragrep/k8s_01/RelpOutput.java index 04fd5be..7d000c3 100644 --- a/src/main/java/com/teragrep/k8s_01/RelpOutput.java +++ b/src/main/java/com/teragrep/k8s_01/RelpOutput.java @@ -18,6 +18,7 @@ package com.teragrep.k8s_01; import com.cloudbees.syslog.SyslogMessage; +import com.codahale.metrics.*; import com.teragrep.k8s_01.config.AppConfigRelp; import com.teragrep.rlp_01.RelpBatch; import com.teragrep.rlp_01.RelpConnection; @@ -28,12 +29,18 @@ import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeoutException; +import static com.codahale.metrics.MetricRegistry.name; + public class RelpOutput { private static final Logger LOGGER = LoggerFactory.getLogger(RelpOutput.class); private final RelpConnection relpConnection; private final AppConfigRelp relpConfig; private final int id; - + Counter totalReconnects; + Counter totalConnections; + Meter throughputBytes; + Meter throughputRecords; + Meter throughputErrors; RelpOutput(AppConfigRelp appConfigRelp, int threadId) { relpConfig = appConfigRelp; id = threadId; @@ -55,6 +62,7 @@ public class RelpOutput { relpConnection.setConnectionTimeout(relpConfig.getConnectionTimeout()); relpConnection.setReadTimeout(relpConfig.getReadTimeout()); relpConnection.setWriteTimeout(relpConfig.getWriteTimeout()); + loadMetrics(); connect(); } @@ -71,14 +79,18 @@ private void connect() { ); } connected = relpConnection.connect(relpConfig.getTarget(), relpConfig.getPort()); + totalConnections.inc(); } catch (IOException | TimeoutException e) { LOGGER.error( "[#{}] Can't connect to Relp server:", getId(), e ); + throughputErrors.mark(); + totalConnections.dec(); } if (!connected) { + totalReconnects.inc(); try { LOGGER.info( "[#{}] Attempting to reconnect in {}ms.", @@ -88,6 +100,7 @@ private void connect() { Thread.sleep(relpConfig.getReconnectInterval()); } catch (InterruptedException e) { e.printStackTrace(); + throughputErrors.mark(); } } } @@ -99,6 +112,7 @@ public void disconnect() { getId() ); try { + totalConnections.dec(); relpConnection.disconnect(); } catch (IOException | TimeoutException e) { LOGGER.debug( @@ -106,6 +120,7 @@ public void disconnect() { getId() ); relpConnection.tearDown(); + throughputErrors.mark(); throw new RuntimeException(e); } } @@ -145,6 +160,7 @@ public void send(SyslogMessage syslogMessage) { getId(), e ); + throughputErrors.mark(); } // Check if everything has been sent, retry and reconnect if not. if (!batch.verifyTransactionAll()) { @@ -154,9 +170,12 @@ public void send(SyslogMessage syslogMessage) { ); batch.retryAllFailed(); relpConnection.tearDown(); + totalConnections.dec(); connect(); } else { allSent = true; + throughputBytes.mark(syslogMessage.toRfc5424SyslogMessage().getBytes(StandardCharsets.UTF_8).length); + throughputRecords.mark(); } } } @@ -164,4 +183,17 @@ public void send(SyslogMessage syslogMessage) { public int getId() { return id; } + + private void loadMetrics() { + // All registered through PrometheusMetrics + MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate("default"); + // Throughput + throughputBytes = metricRegistry.meter(name("throughput", "bytes")); + throughputRecords = metricRegistry.meter(name("throughput", "records")); + throughputErrors = metricRegistry.meter(name("throughput", "errors")); + + // Totals + totalConnections = metricRegistry.counter(name("total", "connections")); + totalReconnects = metricRegistry.counter(name("total", "reconnects")); + } } From c02a038cb883389a595f576032e703e5b3016927 Mon Sep 17 00:00:00 2001 From: StrongestNumber9 <16169054+StrongestNumber9@users.noreply.github.com> Date: Tue, 13 Jun 2023 13:11:48 +0300 Subject: [PATCH 3/4] Make metricsregistry to not be singleton --- .../teragrep/k8s_01/KubernetesLogReader.java | 5 +-- .../java/com/teragrep/k8s_01/RelpOutput.java | 34 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java b/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java index 7df6529..acb36be 100644 --- a/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java +++ b/src/main/java/com/teragrep/k8s_01/KubernetesLogReader.java @@ -17,6 +17,7 @@ package com.teragrep.k8s_01; +import com.codahale.metrics.MetricRegistry; import com.google.gson.Gson; import com.google.gson.JsonParseException; import com.teragrep.k8s_01.config.AppConfig; @@ -35,7 +36,7 @@ public class KubernetesLogReader { private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesLogReader.class); - + private static final MetricRegistry metricRegistry = new MetricRegistry(); static Gson gson = new Gson(); public static void main(String[] args) throws IOException { AppConfig appConfig; @@ -80,7 +81,7 @@ public static void main(String[] args) throws IOException { "Adding RelpOutput thread #{}", i ); - relpOutputPool.put(new RelpOutput(appConfig.getRelp(), i)); + relpOutputPool.put(new RelpOutput(appConfig.getRelp(), i, metricRegistry)); } catch (InterruptedException e) { throw new RuntimeException(e); } diff --git a/src/main/java/com/teragrep/k8s_01/RelpOutput.java b/src/main/java/com/teragrep/k8s_01/RelpOutput.java index 7d000c3..97bfe54 100644 --- a/src/main/java/com/teragrep/k8s_01/RelpOutput.java +++ b/src/main/java/com/teragrep/k8s_01/RelpOutput.java @@ -36,12 +36,12 @@ public class RelpOutput { private final RelpConnection relpConnection; private final AppConfigRelp relpConfig; private final int id; - Counter totalReconnects; - Counter totalConnections; - Meter throughputBytes; - Meter throughputRecords; - Meter throughputErrors; - RelpOutput(AppConfigRelp appConfigRelp, int threadId) { + private final Counter totalReconnects; + private final Counter totalConnections; + private final Meter throughputBytes; + private final Meter throughputRecords; + private final Meter throughputErrors; + RelpOutput(AppConfigRelp appConfigRelp, int threadId, MetricRegistry metricRegistry) { relpConfig = appConfigRelp; id = threadId; if(LOGGER.isDebugEnabled()) { @@ -62,7 +62,14 @@ public class RelpOutput { relpConnection.setConnectionTimeout(relpConfig.getConnectionTimeout()); relpConnection.setReadTimeout(relpConfig.getReadTimeout()); relpConnection.setWriteTimeout(relpConfig.getWriteTimeout()); - loadMetrics(); + // Throughput + throughputBytes = metricRegistry.meter(name("throughput", "bytes")); + throughputRecords = metricRegistry.meter(name("throughput", "records")); + throughputErrors = metricRegistry.meter(name("throughput", "errors")); + + // Totals + totalConnections = metricRegistry.counter(name("total", "connections")); + totalReconnects = metricRegistry.counter(name("total", "reconnects")); connect(); } @@ -183,17 +190,4 @@ public void send(SyslogMessage syslogMessage) { public int getId() { return id; } - - private void loadMetrics() { - // All registered through PrometheusMetrics - MetricRegistry metricRegistry = SharedMetricRegistries.getOrCreate("default"); - // Throughput - throughputBytes = metricRegistry.meter(name("throughput", "bytes")); - throughputRecords = metricRegistry.meter(name("throughput", "records")); - throughputErrors = metricRegistry.meter(name("throughput", "errors")); - - // Totals - totalConnections = metricRegistry.counter(name("total", "connections")); - totalReconnects = metricRegistry.counter(name("total", "reconnects")); - } } From 211673ceb0940eeed1ee02b30c44a1ca856a7bf8 Mon Sep 17 00:00:00 2001 From: StrongestNumber9 <16169054+StrongestNumber9@users.noreply.github.com> Date: Tue, 13 Jun 2023 13:19:00 +0300 Subject: [PATCH 4/4] Private final jettyServer --- src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java index 9b3a5c9..6544cce 100644 --- a/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java +++ b/src/main/java/com/teragrep/k8s_01/PrometheusMetrics.java @@ -34,7 +34,7 @@ public class PrometheusMetrics { private static final Logger LOGGER = LoggerFactory.getLogger(PrometheusMetrics.class); - Server jettyServer; + private final Server jettyServer; public PrometheusMetrics(int port) { LOGGER.info("Starting prometheus metrics server on port {}", port); // prometheus-exporter