Closes #1285 - Agents regularly report status to config server

components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatus.java

@@ -4,6 +4,7 @@
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import rocks.inspectit.ocelot.commons.models.health.AgentHealth;
 
 import java.util.Date;
 import java.util.Map;
@@ -42,4 +43,9 @@ public class AgentStatus {
      * The branch of which the mapping delivered to the agent originates.
      */
     private String sourceBranch;
+
+    /**
+     * The health status of the agent.
+     */
+    private AgentHealth health;
 }

components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentstatus/AgentStatusManager.java

@@ -3,9 +3,11 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 import rocks.inspectit.ocelot.agentconfiguration.AgentConfiguration;
+import rocks.inspectit.ocelot.commons.models.health.AgentHealth;
 import rocks.inspectit.ocelot.config.model.InspectitServerSettings;
 
 import javax.annotation.PostConstruct;
@@ -19,8 +21,14 @@
  * This is useful for detecting which agents are active.
  */
 @Component
+@Slf4j
 public class AgentStatusManager {
 
+    /**
+     * Name of the agent health header.
+     */
+    private static final String HEADER_AGENT_HEALTH = "x-ocelot-health";
+
     @Autowired
     @VisibleForTesting
     InspectitServerSettings config;
@@ -52,9 +60,11 @@ public void reset() {
      */
     public void notifyAgentConfigurationFetched(Map<String, String> agentAttributes, Map<String, String> headers, AgentConfiguration resultConfiguration) {
         AgentMetaInformation metaInformation = AgentMetaInformation.of(headers);
+        AgentHealth agentHealth = AgentHealth.valueOf(headers.getOrDefault(HEADER_AGENT_HEALTH, "OK"));
 
         AgentStatus agentStatus = AgentStatus.builder()
                 .metaInformation(metaInformation)
+                .health(agentHealth)
                 .attributes(agentAttributes)
                 .lastConfigFetch(new Date())
                 .mappingName(resultConfiguration == null ? null : resultConfiguration.getMapping().getName())
@@ -70,9 +80,18 @@ public void notifyAgentConfigurationFetched(Map<String, String> agentAttributes,
             statusKey = agentAttributes;
         }
 
+        logHealthIfChanged(statusKey, agentHealth);
         attributesToAgentStatusCache.put(statusKey, agentStatus);
     }
 
+    private void logHealthIfChanged(Object statusKey, AgentHealth agentHealth) {
+        AgentStatus lastStatus = attributesToAgentStatusCache.getIfPresent(statusKey);
+
+        if (lastStatus != null && lastStatus.getHealth() != agentHealth) {
+            log.info("Health of agent {} changed to {}.", statusKey, agentHealth);
+        }
+    }
+
     /**
      * @return a collection of all agent statuses since {@link #reset()} was called.
      */

inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/commons/models/health/AgentHealth.java

@@ -0,0 +1,72 @@
+package rocks.inspectit.ocelot.commons.models.health;
+
+import ch.qos.logback.classic.Level;
+import lombok.RequiredArgsConstructor;
+
+/**
+ * Represents the health status of an individual agent.
+ */
+@RequiredArgsConstructor
+public enum AgentHealth {
+
+    OK("the agent is working properly"),
+
+    WARNING("the agent has warning messages"),
+
+    ERROR("the agent has encountered errors");
+
+    private final String description;
+
+    /**
+     * Decides whether this health status is more severe or equal to the passed one.
+     * {@code null} is always considered less severe.
+     *
+     * @param other The health to compare with
+     *
+     * @return {@code true} if both health status are equal or this is more severe than other; {@code false} otherwise.
+     */
+    public boolean isMoreSevereOrEqualTo(AgentHealth other) {
+        return other != null ? compareTo(other) >= 0 : true;
+    }
+
+    /**
+     * Compares multiple health status and returns the one that is most severe.
+     *
+     * @param status The array of health status to compare (may contain {@code null})
+     *
+     * @return That health of the passed ones that is more severe. {@code null} if none is passed.
+     */
+    public static AgentHealth mostSevere(AgentHealth... status) {
+        AgentHealth max = null;
+
+        for (AgentHealth curr : status) {
+            if (curr != null && curr.isMoreSevereOrEqualTo(max)) {
+                max = curr;
+            }
+        }
+
+        return max;
+    }
+
+    /**
+     * Determines the agent health based on the level of a log event that occurred (e.g., WARN level corresponds with WARNING).
+     *
+     * @param logLevel The log level that occurred
+     *
+     * @return The agent health that corresponds to the log level.
+     */
+    public static AgentHealth fromLogLevel(Level logLevel) {
+        if (logLevel.isGreaterOrEqual(Level.ERROR)) {
+            return ERROR;
+        } else if (logLevel.isGreaterOrEqual(Level.WARN)) {
+            return WARNING;
+        } else {
+            return OK;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return name() + " (" + description + ")";
+    }
+}

inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/AgentHealthSettings.java

@@ -0,0 +1,29 @@
+package rocks.inspectit.ocelot.config.model.selfmonitoring;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+
+import javax.validation.constraints.AssertFalse;
+import java.time.Duration;
+
+/**
+ * Defines the settings for the agent status.
+ */
+@Data
+@NoArgsConstructor
+public class AgentHealthSettings {
+
+    /**
+     * The period during which a non-ok and non-instrumentation-related status is valid.
+     * Status changes due to instrumentation errors are valid until the next re-instrumentation.
+     */
+    @NonNull
+    private Duration validityPeriod;
+
+    @AssertFalse(message = "The specified period should not be negative!")
+    public boolean isNegativeDuration() {
+        return validityPeriod != null && validityPeriod.isNegative();
+    }
+
+}

inspectit-ocelot-config/src/main/java/rocks/inspectit/ocelot/config/model/selfmonitoring/SelfMonitoringSettings.java

@@ -20,4 +20,10 @@ public class SelfMonitoringSettings {
     @Valid
     private ActionMetricsSettings actionMetrics;
 
+    /**
+     * Settings for {@link rocks.inspectit.ocelot.core.selfmonitoring.AgentStatusManager}
+     */
+    @Valid
+    private AgentHealthSettings agentHealth;
+
 }

inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/default/self-monitoring.yml

@@ -9,6 +9,12 @@ inspectit:
       # if true, the execution time (duration in ms) per action will be recorded
       enabled: false
 
+    # settings regarding the update and notification frequency of the agent health
+    agent-health:
+      # defines how long a non-ok and non-instrumentation-related health status is valid
+      # health changes due to instrumentation errors are valid until the next re-instrumentation
+      validity-period: 1h
+
   # definitions of existing self-monitoring metrics
   metrics:
     definitions:
@@ -43,6 +49,15 @@ inspectit:
             aggregation: SUM
             tags: {"level": true}
 
+      '[inspectit/self/health]':
+        enabled: ${inspectit.self-monitoring.enabled}
+        type: LONG
+        unit: health
+        description: "current health status of the agent (0 = OK, 1 = WARNING, 2 = ERROR)"
+        views:
+          '[inspectit/self/health]':
+            aggregation: LAST_VALUE
+
       '[inspectit/self/action/execution-time]':
         enabled: ${inspectit.self-monitoring.action-metrics.enabled}
         type: LONG

inspectit-ocelot-config/src/main/resources/rocks/inspectit/ocelot/config/fallback/fallback.yml

@@ -20,6 +20,8 @@ inspectit:
 
   self-monitoring:
     enabled: false
+    agent-status:
+      validity-period: 1h
 
   log-preloading:
     enabled: false

inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/InspectitEnvironment.java

@@ -142,6 +142,7 @@ public InspectitEnvironment(ConfigurableApplicationContext ctx, Optional<String>
      *                          neither the property sources nor {@link #currentConfig} will change during the execution of propertiesUpdater.
      */
     public synchronized void updatePropertySources(Consumer<MutablePropertySources> propertiesUpdater) {
+        eventDrain.publishEvent(new PropertySourcesReloadEvent(this));
         propertiesUpdater.accept(getPropertySources());
         InspectitConfig oldConfig = currentConfig;
 

inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/PropertySourcesReloadEvent.java

@@ -0,0 +1,14 @@
+package rocks.inspectit.ocelot.core.config;
+
+import org.springframework.context.ApplicationEvent;
+
+/**
+ * This event is fired whenever the set of {@link org.springframework.context.annotation.PropertySource}s is to be reloaded.
+ * In contrast to {@link PropertySourcesChangedEvent}, this event fires <b>before</b> the reload process.
+ */
+public class PropertySourcesReloadEvent extends ApplicationEvent {
+
+    PropertySourcesReloadEvent(Object source) {
+        super(source);
+    }
+}

inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpConfigurationPoller.java

@@ -2,10 +2,12 @@
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.event.EventListener;
 import org.springframework.stereotype.Service;
 import rocks.inspectit.ocelot.config.model.InspectitConfig;
 import rocks.inspectit.ocelot.config.model.config.HttpConfigSettings;
 import rocks.inspectit.ocelot.core.config.InspectitEnvironment;
+import rocks.inspectit.ocelot.core.selfmonitoring.event.AgentHealthChangedEvent;
 import rocks.inspectit.ocelot.core.service.DynamicallyActivatableService;
 
 import java.util.concurrent.ScheduledExecutorService;
@@ -80,4 +82,11 @@ public void run() {
             });
         }
     }
+
+    @EventListener
+    void agentHealthChanged(AgentHealthChangedEvent event) {
+        if (currentState != null) {
+            currentState.updateAgentHealth(event.getNewHealth());
+        }
+    }
 }

inspectit-ocelot-core/src/main/java/rocks/inspectit/ocelot/core/config/propertysources/http/HttpPropertySourceState.java

@@ -1,6 +1,7 @@
 package rocks.inspectit.ocelot.core.config.propertysources.http;
 
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.Header;
@@ -16,7 +17,7 @@
 import org.springframework.core.env.PropertiesPropertySource;
 import org.springframework.core.env.PropertySource;
 import rocks.inspectit.ocelot.bootstrap.AgentManager;
-import rocks.inspectit.ocelot.bootstrap.IAgent;
+import rocks.inspectit.ocelot.commons.models.health.AgentHealth;
 import rocks.inspectit.ocelot.config.model.config.HttpConfigSettings;
 import rocks.inspectit.ocelot.core.config.util.PropertyUtils;
 
@@ -95,6 +96,8 @@ public class HttpPropertySourceState {
     @Getter
     private boolean firstFileWriteAttemptSuccessful = true;
 
+    private AgentHealth agentHealth = AgentHealth.OK;
+
     /**
      * Constructor.
      *
@@ -104,7 +107,7 @@ public class HttpPropertySourceState {
     public HttpPropertySourceState(String name, HttpConfigSettings currentSettings) {
         this.name = name;
         this.currentSettings = currentSettings;
-        this.errorCounter = 0;
+        errorCounter = 0;
         //ensure that currentPropertySource is never null, even if the initial fetching fails
         currentPropertySource = new PropertiesPropertySource(name, new Properties());
     }
@@ -115,6 +118,7 @@ public HttpPropertySourceState(String name, HttpConfigSettings currentSettings)
      * that the configuration has not be changed the property source will not be updated!
      *
      * @param fallBackToFile if true, the configured persisted configuration will be loaded in case of an error
+     *
      * @return returns true if a new property source has been created, otherwise false.
      */
     public boolean update(boolean fallBackToFile) {
@@ -132,11 +136,21 @@ public boolean update(boolean fallBackToFile) {
         return false;
     }
 
+    /**
+     * Updates the agent health to be sent with the request for agent configuration.
+     *
+     * @param newHealth The new agent health
+     */
+    public void updateAgentHealth(@NonNull AgentHealth newHealth) {
+        agentHealth = newHealth;
+    }
+
     /**
      * Parse the given properties string into an instance of {@link Properties}. The string can be represented as a JSON
      * or YAML document.
      *
      * @param rawProperties the properties in a String representation
+     *
      * @return the parsed {@link Properties} object
      */
     private Properties parseProperties(String rawProperties) {
@@ -243,6 +257,7 @@ private void setAgentMetaHeaders(HttpGet httpGet) {
         httpGet.setHeader(META_HEADER_PREFIX + "VM-NAME", runtime.getVmName());
         httpGet.setHeader(META_HEADER_PREFIX + "VM-VENDOR", runtime.getVmVendor());
         httpGet.setHeader(META_HEADER_PREFIX + "START-TIME", String.valueOf(runtime.getStartTime()));
+        httpGet.setHeader(META_HEADER_PREFIX + "HEALTH", agentHealth.name());
     }
 
     /**
@@ -263,11 +278,14 @@ private void logFetchError(String message, Exception exception) {
      * Builds the request URI by combining the base URI with the configured attributes.
      *
      * @return the resulting URI
+     *
      * @throws URISyntaxException if the base URI is malformed
      */
     public URI getEffectiveRequestUri() throws URISyntaxException {
         URIBuilder uriBuilder = new URIBuilder(currentSettings.getUrl().toURI());
-        currentSettings.getAttributes().entrySet().stream()
+        currentSettings.getAttributes()
+                .entrySet()
+                .stream()
                 .filter(pair -> !StringUtils.isEmpty(pair.getValue()))
                 .forEach(pair -> uriBuilder.setParameter(pair.getKey(), pair.getValue()));
         return uriBuilder.build();
@@ -278,7 +296,9 @@ public URI getEffectiveRequestUri() throws URISyntaxException {
      * If the response contains a 'Last-Modified' header, its value will be stored.
      *
      * @param response the HTTP response object
+     *
      * @return the response body or null in case server sends 304 (not modified)
+     *
      * @throws IOException if an error occurs reading the input stream or if the server returned an unexpected status code
      */
     private String processHttpResponse(HttpResponse response) throws IOException {
@@ -362,5 +382,4 @@ private String readPersistenceFile() {
         return null;
     }
 
-
 }