Remove Sidecar collector configuration path (#5278)

* Remove Sidecar collector configuration path

Instead of a GUI option, we let the sidecar autogenerate the path
to ".../generated/<collector_name>.conf"

Remove existing configuration_path values in the migration
to simplify upgrading to this change.

Fixes #5265

* Restrict collector names to path safe characters

We use the collector name in the configuration path.
Avoid problems by restricting it to a characters that need no escape
handling.

* Perform validatons all collector requests

and fix annotation on copy route

* Remove debug printf

graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java

@@ -16,8 +16,14 @@
  */
 package org.graylog.plugins.sidecar.migrations;
 
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.result.UpdateResult;
+import org.bson.Document;
+import org.bson.types.ObjectId;
 import org.graylog.plugins.sidecar.rest.models.Collector;
 import org.graylog.plugins.sidecar.services.CollectorService;
+import org.graylog2.database.MongoConnection;
 import org.graylog2.migrations.Migration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,14 +32,19 @@
 import javax.inject.Inject;
 import java.time.ZonedDateTime;
 
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.exists;
+
 public class V20180212165000_AddDefaultCollectors extends Migration {
     private static final Logger LOG = LoggerFactory.getLogger(V20180212165000_AddDefaultCollectors.class);
 
     private final CollectorService collectorService;
+    private final MongoCollection<Document> collection;
 
     @Inject
-    public V20180212165000_AddDefaultCollectors(CollectorService collectorService) {
+    public V20180212165000_AddDefaultCollectors(CollectorService collectorService, MongoConnection mongoConnection) {
         this.collectorService = collectorService;
+        this.collection = mongoConnection.getMongoDatabase().getCollection(CollectorService.COLLECTION_NAME);
     }
 
     @Override
@@ -43,6 +54,9 @@ public ZonedDateTime createdAt() {
 
     @Override
     public void upgrade() {
+
+        removeConfigPath();
+
         final String beatsPreambel =
                 "# Needed for Graylog\n" +
                 "fields_under_root: true\n" +
@@ -54,7 +68,6 @@ public void upgrade() {
                 "exec",
                 "linux",
                 "/usr/lib/graylog-sidecar/filebeat",
-                "/var/lib/graylog-sidecar/generated/filebeat.yml",
                 "-c  %s",
                 "test config -c %s",
                 beatsPreambel +
@@ -74,7 +87,6 @@ public void upgrade() {
                 "svc",
                 "windows",
                 "C:\\Program Files\\Graylog\\sidecar\\winlogbeat.exe",
-                "C:\\Program Files\\Graylog\\sidecar\\generated\\winlogbeat.yml",
                 "-c \"%s\"",
                 "test config -c \"%s\"",
                 beatsPreambel +
@@ -96,19 +108,31 @@ public void upgrade() {
                 "exec",
                 "linux",
                 "/usr/lib/graylog-sidecar/nxlog",
-                "/var/lib/graylog-sidecar/generated/nxlog.conf",
                 "-f -c %s",
                 "-v -c %s",
                 ""
         );
     }
 
+    private void removeConfigPath() {
+        final FindIterable<Document> documentsWithConfigPath = collection.find(exists("configuration_path"));
+        for (Document document : documentsWithConfigPath) {
+            final ObjectId objectId = document.getObjectId("_id");
+            document.remove("configuration_path");
+            final UpdateResult updateResult = collection.replaceOne(eq("_id", objectId), document);
+            if (updateResult.wasAcknowledged()) {
+                LOG.debug("Successfully updated document with ID <{}>", objectId);
+            } else {
+                LOG.error("Failed to update document with ID <{}>", objectId);
+            }
+        }
+    }
+
     @Nullable
     private String ensureCollector(String collectorName,
                                    String serviceType,
                                    String nodeOperatingSystem,
                                    String executablePath,
-                                   String configurationPath,
                                    String executeParameters,
                                    String validationCommand,
                                    String defaultTemplate) {
@@ -129,7 +153,6 @@ private String ensureCollector(String collectorName,
                     serviceType,
                     nodeOperatingSystem,
                     executablePath,
-                    configurationPath,
                     executeParameters,
                     validationCommand,
                     defaultTemplate

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java

@@ -16,9 +16,7 @@
  */
 package org.graylog.plugins.sidecar.rest.models;
 
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import com.google.auto.value.AutoValue;
 import org.mongojack.Id;
 import org.mongojack.ObjectId;
@@ -33,7 +31,6 @@ public abstract class Collector {
     public static final String FIELD_SERVICE_TYPE = "service_type";
     public static final String FIELD_NODE_OPERATING_SYSTEM = "node_operating_system";
     public static final String FIELD_EXECUTABLE_PATH = "executable_path";
-    public static final String FIELD_CONFIGURATION_PATH = "configuration_path";
     public static final String FIELD_EXECUTE_PARAMETERS = "execute_parameters";
     public static final String FIELD_VALIDATION_PARAMETERS = "validation_parameters";
     public static final String FIELD_DEFAULT_TEMPLATE = "default_template";
@@ -57,9 +54,6 @@ public abstract class Collector {
     @JsonProperty(FIELD_EXECUTABLE_PATH)
     public abstract String executablePath();
 
-    @JsonProperty(FIELD_CONFIGURATION_PATH)
-    public abstract String configurationPath();
-
     @JsonProperty(FIELD_EXECUTE_PARAMETERS)
     @Nullable
     public abstract String executeParameters();
@@ -84,7 +78,6 @@ public abstract static class Builder {
         public abstract Builder serviceType(String serviceType);
         public abstract Builder nodeOperatingSystem(String nodeOperatingSystem);
         public abstract Builder executablePath(String executablePath);
-        public abstract Builder configurationPath(String configurationPath);
         public abstract Builder executeParameters(String executeParameters);
         public abstract Builder validationParameters(String validationParameters);
         public abstract Builder defaultTemplate(String defaultTemplate);
@@ -97,7 +90,6 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                                    @JsonProperty(FIELD_SERVICE_TYPE) String serviceType,
                                    @JsonProperty(FIELD_NODE_OPERATING_SYSTEM) String nodeOperatingSystem,
                                    @JsonProperty(FIELD_EXECUTABLE_PATH) String executablePath,
-                                   @JsonProperty(FIELD_CONFIGURATION_PATH) String configurationPath,
                                    @JsonProperty(FIELD_EXECUTE_PARAMETERS) @Nullable String executeParameters,
                                    @JsonProperty(FIELD_VALIDATION_PARAMETERS) @Nullable String validationParameters,
                                    @JsonProperty(FIELD_DEFAULT_TEMPLATE) String defaultTemplate) {
@@ -107,7 +99,6 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                 .serviceType(serviceType)
                 .nodeOperatingSystem(nodeOperatingSystem)
                 .executablePath(executablePath)
-                .configurationPath(configurationPath)
                 .executeParameters(executeParameters)
                 .validationParameters(validationParameters)
                 .defaultTemplate(defaultTemplate)

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/CollectorResource.java

@@ -35,6 +35,7 @@
 import org.graylog.plugins.sidecar.services.EtagService;
 import org.graylog2.audit.jersey.AuditEvent;
 import org.graylog2.database.PaginatedList;
+import org.graylog2.plugin.database.ValidationException;
 import org.graylog2.plugin.rest.PluginRestResource;
 import org.graylog2.search.SearchQuery;
 import org.graylog2.search.SearchQueryField;
@@ -44,24 +45,8 @@
 import javax.inject.Inject;
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.NotFoundException;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.CacheControl;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.EntityTag;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
+import javax.ws.rs.*;
+import javax.ws.rs.core.*;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -180,7 +165,11 @@ public CollectorSummaryResponse listSummary(@ApiParam(name = "page") @QueryParam
     @ApiOperation(value = "Create a new collector")
     @AuditEvent(type = SidecarAuditEventTypes.COLLECTOR_CREATE)
     public Collector createCollector(@ApiParam(name = "JSON body", required = true)
-                                     @Valid @NotNull Collector request) {
+                                     @Valid @NotNull Collector request) throws ValidationException {
+        final ValidationResponse validation = validate(request.name(), null);
+        if (validation.error()) {
+            throw new ValidationException(validation.errorMessage());
+        }
         etagService.invalidateAll();
         Collector collector = collectorService.fromRequest(request);
         return collectorService.save(collector);
@@ -195,7 +184,11 @@ public Collector createCollector(@ApiParam(name = "JSON body", required = true)
     public Collector updateCollector(@ApiParam(name = "id", required = true)
                                      @PathParam("id") String id,
                                      @ApiParam(name = "JSON body", required = true)
-                                     @Valid @NotNull Collector request) {
+                                     @Valid @NotNull Collector request) throws ValidationException {
+        final ValidationResponse validation = validate(request.name(), id);
+        if (validation.error()) {
+            throw new ValidationException(validation.errorMessage());
+        }
         etagService.invalidateAll();
         Collector collector = collectorService.fromRequest(id, request);
         return collectorService.save(collector);
@@ -208,7 +201,12 @@ public Collector updateCollector(@ApiParam(name = "id", required = true)
     @AuditEvent(type = SidecarAuditEventTypes.COLLECTOR_CLONE)
     public Response copyCollector(@ApiParam(name = "id", required = true)
                                   @PathParam("id") String id,
+                                  @ApiParam(name = "name", required = true)
                                   @PathParam("name") String name) throws NotFoundException {
+        final ValidationResponse validation = validate(name, null);
+        if (validation.error()) {
+            return Response.status(Response.Status.BAD_REQUEST).entity(validation).build();
+        }
         etagService.invalidateAll();
         final Collector collector = collectorService.copy(id, name);
         collectorService.save(collector);
@@ -246,6 +244,13 @@ public Response deleteCollector(@ApiParam(name = "id", required = true)
     public ValidationResponse validateCollector(
             @ApiParam(name = "name", required = true) @QueryParam("name") String name,
             @ApiParam(name = "id", required = false) @QueryParam("id") String id) {
+        return validate(name, id);
+    }
+
+    private ValidationResponse validate(String name, String id) {
+        if (!name.matches("^[A-Za-z0-9_.-]+$")) {
+            return ValidationResponse.create(true, "Collector name can only contain the following characters: A-Z,a-z,0-9,_,-,.");
+        }
         final Collector collector;
         if (id == null) {
             collector = collectorService.findByName(name);

graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java

@@ -35,7 +35,7 @@
 
 @Singleton
 public class CollectorService extends PaginatedDbService<Collector> {
-    private static final String COLLECTION_NAME = "sidecar_collectors";
+    public static final String COLLECTION_NAME = "sidecar_collectors";
 
     @Inject
     public CollectorService(MongoConnection mongoConnection,
@@ -90,7 +90,6 @@ public Collector fromRequest(Collector request) {
                 request.serviceType(),
                 request.nodeOperatingSystem(),
                 request.executablePath(),
-                request.configurationPath(),
                 request.executeParameters(),
                 request.validationParameters(),
                 request.defaultTemplate());

graylog2-server/src/main/java/org/graylog2/contentpacks/facades/SidecarCollectorFacade.java

@@ -65,7 +65,6 @@ public EntityWithConstraints exportNativeEntity(Collector collector) {
                 ValueReference.of(collector.serviceType()),
                 ValueReference.of(collector.nodeOperatingSystem()),
                 ValueReference.of(collector.executablePath()),
-                ValueReference.of(collector.configurationPath()),
                 ValueReference.of(collector.executeParameters()),
                 ValueReference.of(collector.validationParameters()),
                 ValueReference.of(collector.defaultTemplate())
@@ -100,7 +99,6 @@ private NativeEntity<Collector> decode(EntityV1 entity, Map<String, ValueReferen
                 .serviceType(collectorEntity.serviceType().asString(parameters))
                 .nodeOperatingSystem(collectorEntity.nodeOperatingSystem().asString(parameters))
                 .executablePath(collectorEntity.executablePath().asString(parameters))
-                .configurationPath(collectorEntity.configurationPath().asString(parameters))
                 .executeParameters(collectorEntity.executeParameters().asString(parameters))
                 .validationParameters(collectorEntity.validationParameters().asString(parameters))
                 .defaultTemplate(collectorEntity.defaultTemplate().asString(parameters))

graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SidecarCollectorEntity.java

@@ -39,9 +39,6 @@ public abstract class SidecarCollectorEntity {
     @JsonProperty("executable_path")
     public abstract ValueReference executablePath();
 
-    @JsonProperty("configuration_path")
-    public abstract ValueReference configurationPath();
-
     @JsonProperty("execute_parameters")
     public abstract ValueReference executeParameters();
 
@@ -56,15 +53,13 @@ public static SidecarCollectorEntity create(@JsonProperty("name") ValueReference
                                          @JsonProperty("service_type") ValueReference serviceType,
                                          @JsonProperty("node_operating_system") ValueReference nodeOperatingSystem,
                                          @JsonProperty("executable_path") ValueReference executablePath,
-                                         @JsonProperty("configuration_path") ValueReference configurationPath,
                                          @JsonProperty("execute_parameters") ValueReference executeParameters,
                                          @JsonProperty("validation_parameters") ValueReference validationParameters,
                                          @JsonProperty("default_template") ValueReference defaultTemplate) {
         return new AutoValue_SidecarCollectorEntity(name,
                 serviceType,
                 nodeOperatingSystem,
                 executablePath,
-                configurationPath,
                 executeParameters,
                 validationParameters,
                 defaultTemplate);

graylog2-server/src/test/java/org/graylog2/contentpacks/facades/SidecarCollectorFacadeTest.java

@@ -82,7 +82,6 @@ public void exportNativeEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -105,7 +104,6 @@ public void exportEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -124,7 +122,6 @@ public void createNativeEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -154,7 +151,6 @@ public void findExisting() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -234,7 +230,6 @@ public void resolveEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))

graylog2-server/src/test/resources/org/graylog2/contentpacks/sidecar_collectors.json

@@ -8,7 +8,6 @@
       "service_type": "exec",
       "node_operating_system": "linux",
       "executable_path": "/usr/lib/graylog-sidecar/filebeat",
-      "configuration_path": "/var/lib/graylog-sidecar/generated/filebeat.yml",
       "execute_parameters": "-c %s",
       "validation_parameters": "test config -c %s",
       "default_template": ""
@@ -21,7 +20,6 @@
       "service_type": "svc",
       "node_operating_system": "windows",
       "executable_path": "C:\\Program Files\\Graylog\\sidecar\\winlogbeat.exe",
-      "configuration_path": "C:\\Program Files\\Graylog\\sidecar\\generated\\winlogbeat.yml",
       "execute_parameters": "-c \"%s\"",
       "validation_parameters": "test config -c \"%s\"",
       "default_template": ""
@@ -34,7 +32,6 @@
       "service_type": "exec",
       "node_operating_system": "linux",
       "executable_path": "/usr/lib/graylog-sidecar/nxlog",
-      "configuration_path": "/var/lib/graylog-sidecar/generated/nxlog.conf",
       "execute_parameters": "-f -c %s",
       "validation_parameters": "-v -c %s",
       "default_template": ""