diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/ExecutorCollector.java b/src/main/java/org/jenkinsci/plugins/prometheus/ExecutorCollector.java index 67559ef88..137376fab 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/ExecutorCollector.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/ExecutorCollector.java @@ -30,13 +30,13 @@ public List collect() { List> collectors = new ArrayList<>(); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_AVAILABLE_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_BUSY_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_CONNECTING_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_DEFINED_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_IDLE_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_ONLINE_GAUGE, labelNameArray, prefix)); - collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_QUEUE_LENGTH_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_AVAILABLE_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_BUSY_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_CONNECTING_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_DEFINED_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_IDLE_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_ONLINE_GAUGE, labelNameArray, prefix)); + collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_QUEUE_LENGTH_GAUGE, labelNameArray, prefix)); LOGGER.debug("getting load statistics for Executors"); diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java b/src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java index a3e055816..19f11df26 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java @@ -32,6 +32,7 @@ public class JobCollector extends Collector { private MetricCollector, ? extends Collector> nbBuildsGauge; private MetricCollector, ? extends Collector> buildDiscardGauge; private MetricCollector, ? extends Collector> currentRunDurationGauge; + private MetricCollector, ? extends Collector> logUpdatedGauge; private static class BuildMetrics { @@ -129,6 +130,8 @@ public List collect() { currentRunDurationGauge = factory.createJobCollector(CollectorType.CURRENT_RUN_DURATION_GAUGE, labelBaseNameArray); + logUpdatedGauge = factory.createJobCollector(CollectorType.JOB_LOG_UPDATED_GAUGE, labelBaseNameArray); + if (PrometheusConfiguration.get().isPerBuildMetrics()) { labelNameArray = Arrays.copyOf(labelNameArray, labelNameArray.length + 1); labelNameArray[labelNameArray.length - 1] = "number"; @@ -164,6 +167,7 @@ public List collect() { addSamples(samples, nbBuildsGauge.collect(), "Adding [{}] samples from gauge ({})"); addSamples(samples, buildDiscardGauge.collect(), "Adding [{}] samples from gauge ({})"); addSamples(samples, currentRunDurationGauge.collect(), "Adding [{}] samples from gauge ({})"); + addSamples(samples, logUpdatedGauge.collect(), "Adding [{}] samples from gauge ({})"); addSamples(samples, lastBuildMetrics); if (PrometheusConfiguration.get().isPerBuildMetrics()) { addSamples(samples, perBuildMetrics); @@ -217,6 +221,8 @@ protected void appendJobMetrics(Job job) { jobHealthScoreGauge.calculateMetric(job, baseLabelValueArray); buildDiscardGauge.calculateMetric(job, baseLabelValueArray); currentRunDurationGauge.calculateMetric(job, baseLabelValueArray); + logUpdatedGauge.calculateMetric(job, baseLabelValueArray); + processRun(job, lastBuild, baseLabelValueArray, lastBuildMetrics); Run run = lastBuild; diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/NodeCollector.java b/src/main/java/org/jenkinsci/plugins/prometheus/NodeCollector.java new file mode 100644 index 000000000..5134aca7c --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/prometheus/NodeCollector.java @@ -0,0 +1,54 @@ +package org.jenkinsci.plugins.prometheus; + +import hudson.model.Computer; +import hudson.model.Executor; +import hudson.model.Node; +import io.prometheus.client.Collector; +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.prometheus.collectors.CollectorFactory; +import org.jenkinsci.plugins.prometheus.collectors.CollectorType; +import org.jenkinsci.plugins.prometheus.collectors.MetricCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class NodeCollector extends Collector { + + private static final Logger LOGGER = LoggerFactory.getLogger(NodeCollector.class); + + @Override + public List collect() { + LOGGER.debug("Collecting node metrics for prometheus"); + String[] labelNameArray = {"computerName"}; + + CollectorFactory factory = new CollectorFactory(); + + MetricCollector likelyStuckCollector = factory.createExecutorStatisticsCollector(CollectorType.EXECUTOR_LIKELY_STUCK_GAUGE, labelNameArray); + + List> collectors = List.of(likelyStuckCollector); + + List computers = Jenkins.get().getNodes().parallelStream() + .map(Node::toComputer) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + for (Computer computer : computers) { + String computerName = computer.getName(); + List executors = computer.getExecutors(); + if (executors != null) { + for (Executor ex : executors) { + likelyStuckCollector.calculateMetric(ex, new String[]{computerName}); + } + } + } + + return collectors.stream() + .map(MetricCollector::collect) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorFactory.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorFactory.java index 756dd396a..54b68d62e 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorFactory.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorFactory.java @@ -2,6 +2,7 @@ import com.cloudbees.simplediskusage.DiskItem; import com.cloudbees.simplediskusage.JobDiskItem; +import hudson.model.Executor; import hudson.model.Job; import hudson.model.LoadStatistics; import hudson.model.Run; @@ -13,7 +14,7 @@ import org.jenkinsci.plugins.prometheus.collectors.executors.ExecutorCollectorFactory; import org.jenkinsci.plugins.prometheus.collectors.jenkins.JenkinsCollectorFactory; import org.jenkinsci.plugins.prometheus.collectors.jobs.JobCollectorFactory; -import org.json.Cookie; +import org.jenkinsci.plugins.prometheus.collectors.nodes.NodeCollectorFactory; import java.nio.file.FileStore; @@ -27,6 +28,8 @@ public class CollectorFactory { private final CoverageCollectorFactory coverageCollectorFactory; + private final NodeCollectorFactory nodeCollectorFactory; + public CollectorFactory() { buildCollectorFactory = new BuildCollectorFactory(); jobCollectorFactory = new JobCollectorFactory(); @@ -34,6 +37,7 @@ public CollectorFactory() { executorCollectorFactory = new ExecutorCollectorFactory(); diskCollectorFactory = new DiskCollectorFactory(); coverageCollectorFactory = new CoverageCollectorFactory(); + nodeCollectorFactory = new NodeCollectorFactory(); } public MetricCollector, ? extends Collector> createCoverageRunCollector(CollectorType type, String[] labelNames) { @@ -48,11 +52,15 @@ public CollectorFactory() { return jobCollectorFactory.createCollector(type, labelNames); } + public MetricCollector createExecutorStatisticsCollector(CollectorType type, String[] labelNames) { + return nodeCollectorFactory.createExecutorCollector(type, labelNames); + } + public MetricCollector createJenkinsCollector(CollectorType type, String[] labelNames) { return jenkinsCollectorFactory.createCollector(type, labelNames); } - public MetricCollector createExecutorCollector(CollectorType type, String[] labelNames, String prefix) { + public MetricCollector createLoadStatisticsCollector(CollectorType type, String[] labelNames, String prefix) { return executorCollectorFactory.createCollector(type, labelNames, prefix); } diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorType.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorType.java index fd19083e3..0e2c8d560 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorType.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/CollectorType.java @@ -52,9 +52,8 @@ public enum CollectorType { COVERAGE_FILE_MISSED("coverage_file_missed"), COVERAGE_FILE_TOTAL("coverage_file_total"), - - - ; + JOB_LOG_UPDATED_GAUGE("job_log_updated"), + EXECUTOR_LIKELY_STUCK_GAUGE("likely_stuck"); private final String name; diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorFactory.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorFactory.java index 059742785..276e354a8 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorFactory.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorFactory.java @@ -23,6 +23,8 @@ public JobCollectorFactory() { return saveBuildCollector(new BuildDiscardGauge(labelNames, namespace, subsystem)); case CURRENT_RUN_DURATION_GAUGE: return saveBuildCollector(new CurrentRunDurationGauge(labelNames, namespace, subsystem)); + case JOB_LOG_UPDATED_GAUGE: + return saveBuildCollector(new LogUpdatedGauge(labelNames, namespace, subsystem)); default: return new NoOpMetricCollector<>(); } diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGauge.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGauge.java new file mode 100644 index 000000000..bdf0bf0ee --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGauge.java @@ -0,0 +1,38 @@ +package org.jenkinsci.plugins.prometheus.collectors.jobs; + +import hudson.model.Job; +import io.prometheus.client.Gauge; +import io.prometheus.client.SimpleCollector; +import org.jenkinsci.plugins.prometheus.collectors.CollectorType; +import org.jenkinsci.plugins.prometheus.collectors.builds.BuildsMetricCollector; + +public class LogUpdatedGauge extends BuildsMetricCollector, Gauge> { + + protected LogUpdatedGauge(String[] labelNames, String namespace, String subsystem) { + super(labelNames, namespace, subsystem); + } + + @Override + protected CollectorType getCollectorType() { + return CollectorType.JOB_LOG_UPDATED_GAUGE; + } + + @Override + protected String getHelpText() { + return "Provides a hint if a job is still logging. Maybe not 100% accurate - but a good hint."; + } + + @Override + protected SimpleCollector.Builder getCollectorBuilder() { + return Gauge.build(); + } + + @Override + public void calculateMetric(Job jenkinsObject, String[] labelValues) { + + if (jenkinsObject != null) { + boolean logUpdated = jenkinsObject.isLogUpdated(); + this.collector.labels(labelValues).set(logUpdated ? 1.0 : 0.0); + } + } +} diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/ExecutorLikelyStuckGauge.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/ExecutorLikelyStuckGauge.java new file mode 100644 index 000000000..beda30dc4 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/ExecutorLikelyStuckGauge.java @@ -0,0 +1,39 @@ +package org.jenkinsci.plugins.prometheus.collectors.nodes; + +import hudson.model.Executor; +import io.prometheus.client.Gauge; +import io.prometheus.client.SimpleCollector; +import org.jenkinsci.plugins.prometheus.collectors.BaseMetricCollector; +import org.jenkinsci.plugins.prometheus.collectors.CollectorType; + +public class ExecutorLikelyStuckGauge extends BaseMetricCollector { + + protected ExecutorLikelyStuckGauge(String[] labelNames, String namespace, String subsystem) { + super(labelNames, namespace, subsystem); + } + + @Override + protected CollectorType getCollectorType() { + return CollectorType.EXECUTOR_LIKELY_STUCK_GAUGE; + } + + @Override + protected String getHelpText() { + return "Returns an indication if an executor of a node is likely stuck"; + } + + @Override + protected SimpleCollector.Builder getCollectorBuilder() { + return Gauge.build(); + } + + @Override + public void calculateMetric(Executor executor, String[] labelValues) { + if (executor == null) { + return; + } + boolean likelyStuck = executor.isLikelyStuck(); + collector.labels(labelValues).set(likelyStuck ? 1.0 : 0.0); + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/NodeCollectorFactory.java b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/NodeCollectorFactory.java new file mode 100644 index 000000000..e5fc19698 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/prometheus/collectors/nodes/NodeCollectorFactory.java @@ -0,0 +1,23 @@ +package org.jenkinsci.plugins.prometheus.collectors.nodes; + +import hudson.model.Executor; +import org.jenkinsci.plugins.prometheus.collectors.BaseCollectorFactory; +import org.jenkinsci.plugins.prometheus.collectors.CollectorType; +import org.jenkinsci.plugins.prometheus.collectors.MetricCollector; +import org.jenkinsci.plugins.prometheus.collectors.NoOpMetricCollector; + +import java.util.stream.Collector; + +public class NodeCollectorFactory extends BaseCollectorFactory { + + public NodeCollectorFactory(){super();} + + public MetricCollector createExecutorCollector(CollectorType type, String[] labelNames) { + switch (type) { + case EXECUTOR_LIKELY_STUCK_GAUGE: + return saveBuildCollector(new ExecutorLikelyStuckGauge(labelNames, namespace, subsystem)); + default: + return new NoOpMetricCollector<>(); + } + } +} diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java b/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java index 9024c9c3d..6c4498735 100644 --- a/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java +++ b/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java @@ -31,6 +31,7 @@ public DefaultPrometheusMetrics() { collectorRegistry.register(new DropwizardExports(Metrics.metricRegistry(), new JenkinsNodeBuildsSampleBuilder())); collectorRegistry.register(new DiskUsageCollector()); collectorRegistry.register(new ExecutorCollector()); + collectorRegistry.register(new NodeCollector()); collectorRegistry.register(new CodeCoverageCollector()); // other collectors from other plugins diff --git a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/BuildDiscardGaugeTest.java b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/BuildDiscardGaugeTest.java index 464d62abf..8f663e964 100644 --- a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/BuildDiscardGaugeTest.java +++ b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/BuildDiscardGaugeTest.java @@ -10,7 +10,7 @@ public class BuildDiscardGaugeTest extends JobCollectorTest { - @Override + @Test void testCollectResult() { when(job.getBuildDiscarder()).thenReturn(null); diff --git a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/CurrentRunDurationGaugeTest.java b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/CurrentRunDurationGaugeTest.java index e9f066f5f..b710530c5 100644 --- a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/CurrentRunDurationGaugeTest.java +++ b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/CurrentRunDurationGaugeTest.java @@ -19,7 +19,6 @@ public class CurrentRunDurationGaugeTest extends JobCollectorTest { @Mock Run currentRun; - @Override @Test public void testCollectResult() { when(currentRun.isBuilding()).thenReturn(true); diff --git a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/HealthScoreGaugeTest.java b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/HealthScoreGaugeTest.java index 2793a27df..8f344d065 100644 --- a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/HealthScoreGaugeTest.java +++ b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/HealthScoreGaugeTest.java @@ -10,7 +10,6 @@ public class HealthScoreGaugeTest extends JobCollectorTest { - @Override @Test public void testCollectResult() { diff --git a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorTest.java b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorTest.java index e9f729adc..6632adf35 100644 --- a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorTest.java +++ b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/JobCollectorTest.java @@ -14,7 +14,4 @@ public abstract class JobCollectorTest extends CollectorTest { protected Job job; - abstract void testCollectResult(); - - } diff --git a/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGaugeTest.java b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGaugeTest.java new file mode 100644 index 000000000..d9a2e512d --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGaugeTest.java @@ -0,0 +1,55 @@ +package org.jenkinsci.plugins.prometheus.collectors.jobs; + +import io.prometheus.client.Collector; +import org.junit.Test; + +import java.util.List; + +import static org.mockito.Mockito.when; + +public class LogUpdatedGaugeTest extends JobCollectorTest { + + + @Test + public void testBasicAttributes() { + when(job.isLogUpdated()).thenReturn(true); + + LogUpdatedGauge sut = new LogUpdatedGauge(new String[]{"jenkins_job", "repo"}, "default", "jenkins"); + + sut.calculateMetric(job, new String[]{"job1", "NA"}); + List collect = sut.collect(); + + validateMetricFamilySampleListSize(collect, 1); + + Collector.MetricFamilySamples samples = collect.get(0); + validateNames(samples, new String[]{"default_jenkins_builds_job_log_updated"}); + validateMetricFamilySampleSize(samples, 1); + + } + + @Test + public void testLogIsUpdatedReturnsOne() { + + when(job.isLogUpdated()).thenReturn(true); + + LogUpdatedGauge sut = new LogUpdatedGauge(new String[]{"jenkins_job", "repo"}, "default", "jenkins"); + + sut.calculateMetric(job, new String[]{"job1", "NA"}); + List collect = sut.collect(); + Collector.MetricFamilySamples samples = collect.get(0); + validateValue(samples.samples.get(0), 1.0); + } + + @Test + public void testLogIsNotUpdatedReturnsZero() { + + when(job.isLogUpdated()).thenReturn(false); + + LogUpdatedGauge sut = new LogUpdatedGauge(new String[]{"jenkins_job", "repo"}, "default", "jenkins"); + + sut.calculateMetric(job, new String[]{"job1", "NA"}); + List collect = sut.collect(); + Collector.MetricFamilySamples samples = collect.get(0); + validateValue(samples.samples.get(0), 0.0); + } +} \ No newline at end of file