Remove Sidecar collector configuration path

Instead of a GUI option, we let the sidecar autogenerate the path
to ".../generated/<collector_name>.conf"

Remove existing configuration_path values in the migration
to simplify upgrading to this change.

Fixes #5265

graylog2-server/src/main/java/org/graylog/plugins/sidecar/migrations/V20180212165000_AddDefaultCollectors.java

@@ -16,8 +16,14 @@
  */
 package org.graylog.plugins.sidecar.migrations;
 
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.result.UpdateResult;
+import org.bson.Document;
+import org.bson.types.ObjectId;
 import org.graylog.plugins.sidecar.rest.models.Collector;
 import org.graylog.plugins.sidecar.services.CollectorService;
+import org.graylog2.database.MongoConnection;
 import org.graylog2.migrations.Migration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -26,14 +32,19 @@
 import javax.inject.Inject;
 import java.time.ZonedDateTime;
 
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.exists;
+
 public class V20180212165000_AddDefaultCollectors extends Migration {
     private static final Logger LOG = LoggerFactory.getLogger(V20180212165000_AddDefaultCollectors.class);
 
     private final CollectorService collectorService;
+    private final MongoCollection<Document> collection;
 
     @Inject
-    public V20180212165000_AddDefaultCollectors(CollectorService collectorService) {
+    public V20180212165000_AddDefaultCollectors(CollectorService collectorService, MongoConnection mongoConnection) {
         this.collectorService = collectorService;
+        this.collection = mongoConnection.getMongoDatabase().getCollection(CollectorService.COLLECTION_NAME);
     }
 
     @Override
@@ -43,6 +54,9 @@ public ZonedDateTime createdAt() {
 
     @Override
     public void upgrade() {
+
+        removeConfigPath();
+
         final String beatsPreambel =
                 "# Needed for Graylog\n" +
                 "fields_under_root: true\n" +
@@ -54,7 +68,6 @@ public void upgrade() {
                 "exec",
                 "linux",
                 "/usr/lib/graylog-sidecar/filebeat",
-                "/var/lib/graylog-sidecar/generated/filebeat.yml",
                 "-c  %s",
                 "test config -c %s",
                 beatsPreambel +
@@ -74,7 +87,6 @@ public void upgrade() {
                 "svc",
                 "windows",
                 "C:\\Program Files\\Graylog\\sidecar\\winlogbeat.exe",
-                "C:\\Program Files\\Graylog\\sidecar\\generated\\winlogbeat.yml",
                 "-c \"%s\"",
                 "test config -c \"%s\"",
                 beatsPreambel +
@@ -96,19 +108,32 @@ public void upgrade() {
                 "exec",
                 "linux",
                 "/usr/lib/graylog-sidecar/nxlog",
-                "/var/lib/graylog-sidecar/generated/nxlog.conf",
                 "-f -c %s",
                 "-v -c %s",
                 ""
         );
     }
 
+    private void removeConfigPath() {
+        final FindIterable<Document> documentsWithConfigPath = collection.find(exists("configuration_path"));
+        for (Document document : documentsWithConfigPath) {
+            final ObjectId objectId = document.getObjectId("_id");
+            System.out.print( document);
+            document.remove("configuration_path");
+            final UpdateResult updateResult = collection.replaceOne(eq("_id", objectId), document);
+            if (updateResult.wasAcknowledged()) {
+                LOG.debug("Successfully updated document with ID <{}>", objectId);
+            } else {
+                LOG.error("Failed to update document with ID <{}>", objectId);
+            }
+        }
+    }
+
     @Nullable
     private String ensureCollector(String collectorName,
                                    String serviceType,
                                    String nodeOperatingSystem,
                                    String executablePath,
-                                   String configurationPath,
                                    String executeParameters,
                                    String validationCommand,
                                    String defaultTemplate) {
@@ -129,7 +154,6 @@ private String ensureCollector(String collectorName,
                     serviceType,
                     nodeOperatingSystem,
                     executablePath,
-                    configurationPath,
                     executeParameters,
                     validationCommand,
                     defaultTemplate

graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/models/Collector.java

@@ -16,9 +16,7 @@
  */
 package org.graylog.plugins.sidecar.rest.models;
 
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.*;
 import com.google.auto.value.AutoValue;
 import org.mongojack.Id;
 import org.mongojack.ObjectId;
@@ -33,7 +31,6 @@ public abstract class Collector {
     public static final String FIELD_SERVICE_TYPE = "service_type";
     public static final String FIELD_NODE_OPERATING_SYSTEM = "node_operating_system";
     public static final String FIELD_EXECUTABLE_PATH = "executable_path";
-    public static final String FIELD_CONFIGURATION_PATH = "configuration_path";
     public static final String FIELD_EXECUTE_PARAMETERS = "execute_parameters";
     public static final String FIELD_VALIDATION_PARAMETERS = "validation_parameters";
     public static final String FIELD_DEFAULT_TEMPLATE = "default_template";
@@ -57,9 +54,6 @@ public abstract class Collector {
     @JsonProperty(FIELD_EXECUTABLE_PATH)
     public abstract String executablePath();
 
-    @JsonProperty(FIELD_CONFIGURATION_PATH)
-    public abstract String configurationPath();
-
     @JsonProperty(FIELD_EXECUTE_PARAMETERS)
     @Nullable
     public abstract String executeParameters();
@@ -84,7 +78,6 @@ public abstract static class Builder {
         public abstract Builder serviceType(String serviceType);
         public abstract Builder nodeOperatingSystem(String nodeOperatingSystem);
         public abstract Builder executablePath(String executablePath);
-        public abstract Builder configurationPath(String configurationPath);
         public abstract Builder executeParameters(String executeParameters);
         public abstract Builder validationParameters(String validationParameters);
         public abstract Builder defaultTemplate(String defaultTemplate);
@@ -97,7 +90,6 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                                    @JsonProperty(FIELD_SERVICE_TYPE) String serviceType,
                                    @JsonProperty(FIELD_NODE_OPERATING_SYSTEM) String nodeOperatingSystem,
                                    @JsonProperty(FIELD_EXECUTABLE_PATH) String executablePath,
-                                   @JsonProperty(FIELD_CONFIGURATION_PATH) String configurationPath,
                                    @JsonProperty(FIELD_EXECUTE_PARAMETERS) @Nullable String executeParameters,
                                    @JsonProperty(FIELD_VALIDATION_PARAMETERS) @Nullable String validationParameters,
                                    @JsonProperty(FIELD_DEFAULT_TEMPLATE) String defaultTemplate) {
@@ -107,7 +99,6 @@ public static Collector create(@JsonProperty(FIELD_ID) @Nullable String id,
                 .serviceType(serviceType)
                 .nodeOperatingSystem(nodeOperatingSystem)
                 .executablePath(executablePath)
-                .configurationPath(configurationPath)
                 .executeParameters(executeParameters)
                 .validationParameters(validationParameters)
                 .defaultTemplate(defaultTemplate)

graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/CollectorService.java

@@ -35,7 +35,7 @@
 
 @Singleton
 public class CollectorService extends PaginatedDbService<Collector> {
-    private static final String COLLECTION_NAME = "sidecar_collectors";
+    public static final String COLLECTION_NAME = "sidecar_collectors";
 
     @Inject
     public CollectorService(MongoConnection mongoConnection,
@@ -90,7 +90,6 @@ public Collector fromRequest(Collector request) {
                 request.serviceType(),
                 request.nodeOperatingSystem(),
                 request.executablePath(),
-                request.configurationPath(),
                 request.executeParameters(),
                 request.validationParameters(),
                 request.defaultTemplate());

graylog2-server/src/main/java/org/graylog2/contentpacks/facades/SidecarCollectorFacade.java

@@ -65,7 +65,6 @@ public EntityWithConstraints exportNativeEntity(Collector collector) {
                 ValueReference.of(collector.serviceType()),
                 ValueReference.of(collector.nodeOperatingSystem()),
                 ValueReference.of(collector.executablePath()),
-                ValueReference.of(collector.configurationPath()),
                 ValueReference.of(collector.executeParameters()),
                 ValueReference.of(collector.validationParameters()),
                 ValueReference.of(collector.defaultTemplate())
@@ -100,7 +99,6 @@ private NativeEntity<Collector> decode(EntityV1 entity, Map<String, ValueReferen
                 .serviceType(collectorEntity.serviceType().asString(parameters))
                 .nodeOperatingSystem(collectorEntity.nodeOperatingSystem().asString(parameters))
                 .executablePath(collectorEntity.executablePath().asString(parameters))
-                .configurationPath(collectorEntity.configurationPath().asString(parameters))
                 .executeParameters(collectorEntity.executeParameters().asString(parameters))
                 .validationParameters(collectorEntity.validationParameters().asString(parameters))
                 .defaultTemplate(collectorEntity.defaultTemplate().asString(parameters))

graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SidecarCollectorEntity.java

@@ -39,9 +39,6 @@ public abstract class SidecarCollectorEntity {
     @JsonProperty("executable_path")
     public abstract ValueReference executablePath();
 
-    @JsonProperty("configuration_path")
-    public abstract ValueReference configurationPath();
-
     @JsonProperty("execute_parameters")
     public abstract ValueReference executeParameters();
 
@@ -56,15 +53,13 @@ public static SidecarCollectorEntity create(@JsonProperty("name") ValueReference
                                          @JsonProperty("service_type") ValueReference serviceType,
                                          @JsonProperty("node_operating_system") ValueReference nodeOperatingSystem,
                                          @JsonProperty("executable_path") ValueReference executablePath,
-                                         @JsonProperty("configuration_path") ValueReference configurationPath,
                                          @JsonProperty("execute_parameters") ValueReference executeParameters,
                                          @JsonProperty("validation_parameters") ValueReference validationParameters,
                                          @JsonProperty("default_template") ValueReference defaultTemplate) {
         return new AutoValue_SidecarCollectorEntity(name,
                 serviceType,
                 nodeOperatingSystem,
                 executablePath,
-                configurationPath,
                 executeParameters,
                 validationParameters,
                 defaultTemplate);

graylog2-server/src/test/java/org/graylog2/contentpacks/facades/SidecarCollectorFacadeTest.java

@@ -82,7 +82,6 @@ public void exportNativeEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -105,7 +104,6 @@ public void exportEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -124,7 +122,6 @@ public void createNativeEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -154,7 +151,6 @@ public void findExisting() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))
@@ -234,7 +230,6 @@ public void resolveEntity() {
                         ValueReference.of("exec"),
                         ValueReference.of("linux"),
                         ValueReference.of("/usr/lib/graylog-sidecar/filebeat"),
-                        ValueReference.of("/var/lib/graylog-sidecar/generated/filebeat.yml"),
                         ValueReference.of("-c %s"),
                         ValueReference.of("test config -c %s"),
                         ValueReference.of("")), JsonNode.class))

graylog2-server/src/test/resources/org/graylog2/contentpacks/sidecar_collectors.json

@@ -8,7 +8,6 @@
       "service_type": "exec",
       "node_operating_system": "linux",
       "executable_path": "/usr/lib/graylog-sidecar/filebeat",
-      "configuration_path": "/var/lib/graylog-sidecar/generated/filebeat.yml",
       "execute_parameters": "-c %s",
       "validation_parameters": "test config -c %s",
       "default_template": ""
@@ -21,7 +20,6 @@
       "service_type": "svc",
       "node_operating_system": "windows",
       "executable_path": "C:\\Program Files\\Graylog\\sidecar\\winlogbeat.exe",
-      "configuration_path": "C:\\Program Files\\Graylog\\sidecar\\generated\\winlogbeat.yml",
       "execute_parameters": "-c \"%s\"",
       "validation_parameters": "test config -c \"%s\"",
       "default_template": ""
@@ -34,7 +32,6 @@
       "service_type": "exec",
       "node_operating_system": "linux",
       "executable_path": "/usr/lib/graylog-sidecar/nxlog",
-      "configuration_path": "/var/lib/graylog-sidecar/generated/nxlog.conf",
       "execute_parameters": "-f -c %s",
       "validation_parameters": "-v -c %s",
       "default_template": ""

graylog2-web-interface/src/components/sidecars/configuration-forms/CollectorForm.jsx

@@ -43,7 +43,6 @@ const CollectorForm = createReactClass({
         name: this.props.collector.name,
         service_type: this.props.collector.service_type,
         node_operating_system: this.props.collector.node_operating_system,
-        configuration_path: this.props.collector.configuration_path,
         executable_path: this.props.collector.executable_path,
         execute_parameters: this.props.collector.execute_parameters,
         validation_parameters: this.props.collector.validation_parameters,
@@ -164,14 +163,6 @@ const CollectorForm = createReactClass({
               <HelpBlock>Choose the operating system this collector is meant for.</HelpBlock>
             </FormGroup>
 
-            <Input type="text"
-                   id="configurationFilePath"
-                   label="Configuration Path"
-                   onChange={this._onInputChange('configuration_path')}
-                   help="Path to the collector configuration file"
-                   value={this.state.formData.configuration_path}
-                   required />
-
             <Input type="text"
                    id="executablePath"
                    label="Executable Path"
@@ -184,14 +175,14 @@ const CollectorForm = createReactClass({
                    id="executeParameters"
                    label="Execute Parameters"
                    onChange={this._onInputChange('execute_parameters')}
-                   help="Parameters the collector is started with. %s will be replaced by the path to the configuration file."
+                   help={<span>Parameters the collector is started with.<strong> %s will be replaced by the path to the configuration file.</strong></span>}
                    value={executeParameters} />
 
             <Input type="text"
                    id="validationParameters"
                    label="Parameters for Configuration Validation"
                    onChange={this._onInputChange('validation_parameters')}
-                   help="Parameters that validate the configuration file. %s will be replaced by the path to the configuration file."
+                   help={<span>Parameters that validate the configuration file. <strong> %s will be replaced by the path to the configuration file.</strong></span>}
                    value={validationParameters} />
 
             <FormGroup controlId="defaultTemplate">