Sidecar support for tags and collectors with multiple configurations

graylog2-server/src/main/java/org/graylog/plugins/sidecar/common/SidecarPluginConfiguration.java

@@ -33,7 +33,7 @@ public class SidecarPluginConfiguration implements PluginConfigBean {
     private Duration cacheTime = Duration.hours(1L);
 
     @Parameter(value = PREFIX + "cache_max_size", validator = PositiveIntegerValidator.class)
-    private int cacheMaxSize = 100;
+    private int cacheMaxSize = 5000;
 
     public Duration getCacheTime() {
         return cacheTime;

graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java

@@ -79,8 +79,8 @@ public void upgrade() {
                         "output.logstash:\n" +
                         "   hosts: [\"192.168.1.1:5044\"]\n" +
                         "path:\n" +
-                        "  data: /var/lib/graylog-sidecar/collectors/filebeat/data\n" +
-                        "  logs: /var/lib/graylog-sidecar/collectors/filebeat/log"
+                        "  data: ${sidecar.spoolDir}/data\n" +
+                        "  logs: ${sidecar.spoolDir}/log"
         );
         ensureCollector(
                 "filebeat",
@@ -98,8 +98,8 @@ public void upgrade() {
                         "output.logstash:\n" +
                         "   hosts: [\"192.168.1.1:5044\"]\n" +
                         "path:\n" +
-                        "  data: /var/lib/graylog-sidecar/collectors/filebeat/data\n" +
-                        "  logs: /var/lib/graylog-sidecar/collectors/filebeat/log"
+                        "  data: ${sidecar.spoolDir}/data\n" +
+                        "  logs: ${sidecar.spoolDir}/log"
         );
         ensureCollector(
                 "filebeat",
@@ -117,8 +117,8 @@ public void upgrade() {
                         "output.logstash:\n" +
                         "   hosts: [\"192.168.1.1:5044\"]\n" +
                         "path:\n" +
-                        "  data: /var/lib/graylog-sidecar/collectors/filebeat/data\n" +
-                        "  logs: /var/lib/graylog-sidecar/collectors/filebeat/log"
+                        "  data: ${sidecar.spoolDir}/data\n" +
+                        "  logs: ${sidecar.spoolDir}/log"
         );
         ensureCollector(
                 "winlogbeat",
@@ -131,8 +131,8 @@ public void upgrade() {
                         "output.logstash:\n" +
                         "   hosts: [\"192.168.1.1:5044\"]\n" +
                         "path:\n" +
-                        "  data: C:\\Program Files\\Graylog\\sidecar\\cache\\winlogbeat\\data\n" +
-                        "  logs: C:\\Program Files\\Graylog\\sidecar\\logs\n" +
+                        "  data: ${sidecar.spoolDir}\\data\n" +
+                        "  logs: ${sidecar.spoolDir}\\logs\n" +
                         "tags:\n" +
                         " - windows\n" +
                         "winlogbeat:\n" +
@@ -164,9 +164,9 @@ public void upgrade() {
                         "Group nxlog\n" +
                         "\n" +
                         "Moduledir /usr/lib/nxlog/modules\n" +
-                        "CacheDir /var/spool/nxlog/data\n" +
-                        "PidFile /var/run/nxlog/nxlog.pid\n" +
-                        "LogFile /var/log/nxlog/nxlog.log\n" +
+                        "CacheDir ${sidecar.spoolDir}/data\n" +
+                        "PidFile ${sidecar.spoolDir}/nxlog.pid\n" +
+                        "LogFile ${sidecar.spoolDir}/nxlog.log\n" +
                         "LogLevel INFO\n" +
                         "\n" +
                         "\n" +
@@ -217,7 +217,7 @@ public void upgrade() {
                 "C:\\Program Files (x86)\\nxlog\\nxlog.exe",
                 "-c \"%s\"",
                 "-v -f -c \"%s\"",
-                "define ROOT C:\\Program Files (x86)\\nxlog\n" +
+                "define ROOT ${sidecar.spoolDir}\\nxlog\n" +
                         "\n" +
                         "Moduledir %ROOT%\\modules\n" +
                         "CacheDir %ROOT%\\data\n" +
@@ -302,8 +302,8 @@ public void upgrade() {
                         "output.logstash:\n" +
                         "   hosts: [\"192.168.1.1:5044\"]\n" +
                         "path:\n" +
-                        "  data: C:\\Program Files\\Graylog\\sidecar\\cache\\filebeat\\data\n" +
-                        "  logs: C:\\Program Files\\Graylog\\sidecar\\logs\n" +
+                        "  data: ${sidecar.spoolDir}\\data\n" +
+                        "  logs: ${sidecar.spoolDir}\\logs\n" +
                         "tags:\n" +
                         " - windows\n" +
                         "filebeat.inputs:\n" +

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/CollectorStatus.java

@@ -21,6 +21,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.auto.value.AutoValue;
 
+import javax.annotation.Nullable;
+
 @AutoValue
 @JsonAutoDetect
 public abstract class CollectorStatus {
@@ -36,11 +38,16 @@ public abstract class CollectorStatus {
     @JsonProperty("verbose_message")
     public abstract String verboseMessage();
 
+    @JsonProperty("configuration_id")
+    @Nullable
+    public abstract String configurationId();
+
     @JsonCreator
     public static CollectorStatus create(@JsonProperty("collector_id") String collectorId,
                                          @JsonProperty("status") int status,
                                          @JsonProperty("message") String message,
-                                         @JsonProperty("verbose_message") String verboseMessage) {
-        return new AutoValue_CollectorStatus(collectorId, status, message, verboseMessage);
+                                         @JsonProperty("verbose_message") String verboseMessage,
+                                         @JsonProperty("configuration_id") @Nullable String configurationId) {
+        return new AutoValue_CollectorStatus(collectorId, status, message, verboseMessage, configurationId);
     }
-}
+}

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Configuration.java

@@ -25,6 +25,7 @@
 import org.mongojack.ObjectId;
 
 import javax.annotation.Nullable;
+import java.util.Set;
 
 @AutoValue
 @WithBeanGetter
@@ -35,6 +36,7 @@ public abstract class Configuration {
     public static final String FIELD_NAME = "name";
     public static final String FIELD_COLOR = "color";
     public static final String FIELD_TEMPLATE = "template";
+    public static final String FIELD_TAGS = "tags";
 
     @Id
     @ObjectId
@@ -54,30 +56,37 @@ public abstract class Configuration {
     @JsonProperty(FIELD_TEMPLATE)
     public abstract String template();
 
+    @JsonProperty(FIELD_TAGS)
+    public abstract Set<String> tags();
+
     @JsonCreator
     public static Configuration create(@JsonProperty(FIELD_ID) String id,
                                        @JsonProperty(FIELD_COLLECTOR_ID) String collectorId,
                                        @JsonProperty(FIELD_NAME) String name,
                                        @JsonProperty(FIELD_COLOR) String color,
-                                       @JsonProperty(FIELD_TEMPLATE) String template) {
+                                       @JsonProperty(FIELD_TEMPLATE) String template,
+                                       @JsonProperty(FIELD_TAGS) @Nullable Set<String> tags) {
         return builder()
                 .id(id)
                 .collectorId(collectorId)
                 .name(name)
                 .color(color)
                 .template(template)
+                .tags(tags == null ? Set.of() : tags)
                 .build();
     }
 
-    public static Configuration create(String collectorId,
-                                       String name,
-                                       String color,
-                                       String template) {
+    public static Configuration createWithoutId(String collectorId,
+                                                String name,
+                                                String color,
+                                                String template,
+                                                Set<String> tags) {
         return create(new org.bson.types.ObjectId().toHexString(),
                 collectorId,
                 name,
                 color,
-                template);
+                template,
+                tags);
     }
 
     public static Builder builder() {
@@ -98,6 +107,8 @@ public abstract static class Builder {
 
         public abstract Builder template(String template);
 
+        public abstract Builder tags(Set<String> tags);
+
         public abstract Configuration build();
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/ConfigurationSummary.java

@@ -22,6 +22,8 @@
 import org.mongojack.Id;
 import org.mongojack.ObjectId;
 
+import java.util.Set;
+
 @AutoValue
 public abstract class ConfigurationSummary {
     @JsonProperty("id")
@@ -38,20 +40,25 @@ public abstract class ConfigurationSummary {
     @JsonProperty("color")
     public abstract String color();
 
+    @JsonProperty("tags")
+    public abstract Set<String> tags();
+
     @JsonCreator
     public static ConfigurationSummary create(@JsonProperty("id") @Id @ObjectId String id,
                                               @JsonProperty("name") String name,
                                               @JsonProperty("collector_id") String collectorId,
-                                              @JsonProperty("color") String color) {
-        return new AutoValue_ConfigurationSummary(id, name, collectorId, color);
+                                              @JsonProperty("color") String color,
+                                              @JsonProperty("tags") Set<String> tags) {
+        return new AutoValue_ConfigurationSummary(id, name, collectorId, color, tags);
     }
 
     public static ConfigurationSummary create(Configuration configuration) {
         return create(
                 configuration.id(),
                 configuration.name(),
                 configuration.collectorId(),
-                configuration.color());
+                configuration.color(),
+                configuration.tags());
     }
 
 }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/NodeDetails.java

@@ -18,16 +18,19 @@
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.auto.value.AutoValue;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
 import javax.validation.constraints.Size;
 import java.util.List;
+import java.util.Set;
 
 @AutoValue
 @JsonAutoDetect
+@JsonIgnoreProperties(ignoreUnknown = true)
 public abstract class NodeDetails {
     @JsonProperty("operating_system")
     @NotNull
@@ -50,12 +53,21 @@ public abstract class NodeDetails {
     @Nullable
     public abstract CollectorStatusList statusList();
 
+    @JsonProperty("tags")
+    public abstract Set<String> tags();
+
+    @JsonProperty("collector_configuration_directory")
+    @Nullable
+    public abstract String collectorConfigurationDirectory();
+
     @JsonCreator
     public static NodeDetails create(@JsonProperty("operating_system") String operatingSystem,
                                      @JsonProperty("ip") @Nullable String ip,
                                      @JsonProperty("metrics") @Nullable NodeMetrics metrics,
                                      @JsonProperty("log_file_list") @Nullable List<NodeLogFile> logFileList,
-                                     @JsonProperty("status") @Nullable CollectorStatusList statusList) {
-        return new AutoValue_NodeDetails(operatingSystem, ip, metrics, logFileList, statusList);
+                                     @JsonProperty("status") @Nullable CollectorStatusList statusList,
+                                     @JsonProperty("tags") @Nullable Set<String> tags,
+                                     @JsonProperty("collector_configuration_directory") @Nullable String configDir) {
+        return new AutoValue_NodeDetails(operatingSystem, ip, metrics, logFileList, statusList, tags == null ? Set.of() : tags, configDir);
     }
 }

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Sidecar.java

@@ -92,7 +92,6 @@ public static Status fromString(String statusString) {
     public abstract NodeDetails nodeDetails();
 
     @JsonProperty
-    @Nullable
     public abstract List<ConfigurationAssignment> assignments();
 
     @JsonProperty
@@ -133,7 +132,7 @@ public static Sidecar create(@JsonProperty(FIELD_ID) @Id @ObjectId String id,
                 .nodeId(nodeId)
                 .nodeName(nodeName)
                 .nodeDetails(nodeDetails)
-                .assignments(assignments)
+                .assignments(assignments == null ? List.of() : assignments)
                 .sidecarVersion(sidecarVersion)
                 .lastSeen(lastSeen)
                 .build();
@@ -151,6 +150,7 @@ public static Sidecar create(@JsonProperty(FIELD_NODE_ID) String nodeId,
                 .nodeDetails(nodeDetails)
                 .sidecarVersion(sidecarVersion)
                 .lastSeen(DateTime.now(DateTimeZone.UTC))
+                .assignments(List.of())
                 .build();
     }
 

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/requests/BulkActionRequest.java

@@ -33,8 +33,8 @@ public abstract class BulkActionRequest {
     public abstract List<String> collectorIds();
 
     @JsonCreator
-    public static BulkActionRequest create(@JsonProperty("sidecar_id") String collectorId,
+    public static BulkActionRequest create(@JsonProperty("sidecar_id") String sidecarId,
                                            @JsonProperty("collector_ids") List<String> collectorIds) {
-        return new AutoValue_BulkActionRequest(collectorId, collectorIds);
+        return new AutoValue_BulkActionRequest(sidecarId, collectorIds);
     }
-}
+}

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/requests/ConfigurationAssignment.java

@@ -21,6 +21,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.auto.value.AutoValue;
 
+import javax.annotation.Nullable;
+import java.util.Set;
+
 @AutoValue
 @JsonAutoDetect
 public abstract class ConfigurationAssignment {
@@ -30,9 +33,13 @@ public abstract class ConfigurationAssignment {
     @JsonProperty
     public abstract String configurationId();
 
+    @JsonProperty
+    public abstract Set<String> assignedFromTags();
+
     @JsonCreator
     public static ConfigurationAssignment create(@JsonProperty("collector_id") String collectorId,
-                                                 @JsonProperty("configuration_id") String configurationId) {
-        return new AutoValue_ConfigurationAssignment(collectorId, configurationId);
+                                                 @JsonProperty("configuration_id") String configurationId,
+                                                 @JsonProperty("assigned_from_tags") @Nullable Set<String> assignedFromTags) {
+        return new AutoValue_ConfigurationAssignment(collectorId, configurationId, assignedFromTags == null ? Set.of() : assignedFromTags);
     }
 }