Merge branch 'master' into dependabot/maven/org.apache.logging.log4j-…

…log4j-bom-2.24.3

changelog/unreleased/issue-20955.toml

@@ -0,0 +1,5 @@
+type = "fixed"
+message = "Fix unescaped double quotes in map and collection typed fields in Custom HTTP Notification JSON body."
+
+issues = ["20955"]
+pulls = ["21167"]

full-backend-tests/src/test/java/org/graylog2/inputs/InputCreationIT.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.inputs;
+
+import org.graylog.testing.completebackend.Lifecycle;
+import org.graylog.testing.completebackend.apis.GraylogApis;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest;
+import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration;
+
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS)
+public class InputCreationIT {
+
+    private final GraylogApis apis;
+
+    public InputCreationIT(GraylogApis apis) {
+        this.apis = apis;
+    }
+
+    @ContainerMatrixTest
+    void testHttpRandomInputCreation() {
+        String inputId = apis.inputs().createGlobalInput("testInput",
+                "org.graylog2.inputs.random.FakeHttpMessageInput",
+                Map.of("sleep", 30,
+                        "sleep_deviation", 30,
+                        "source", "example.org"));
+        apis.inputs().getInput(inputId)
+                .assertThat().body("title", equalTo("testInput"));
+        apis.waitFor(() ->
+                        apis.inputs().getInputState(inputId)
+                                .extract().body().jsonPath().get("state")
+                                .equals("RUNNING"),
+                "Timed out waiting for HTTP Random Message Input to become available");
+        apis.inputs().deleteInput(inputId);
+    }
+
+    /**
+     * Test to make sure configuration encryption serialization/deserialization works
+     */
+    @ContainerMatrixTest
+    void testFailingAwsCloudTrailInputCreation() {
+        String inputId = apis.inputs().createGlobalInput("testInput",
+                "org.graylog.aws.inputs.cloudtrail.CloudTrailInput",
+                Map.of("aws_sqs_region", "us-east-1",
+                        "aws_s3_region", "us-east-1",
+                        "aws_sqs_queue_name", "invalid-queue-no-messages-read",
+                        "aws_access_key", "invalid-access-key",
+                        "aws_secret_key", "invalid-secret-key"));
+        apis.inputs().getInput(inputId)
+                .assertThat().body("attributes.aws_access_key", equalTo("invalid-access-key"));
+        apis.waitFor(() ->
+                        apis.inputs().getInputState(inputId)
+                                .extract().body().jsonPath().get("state")
+                                .equals("FAILING"),
+                "Timed out waiting for AWS CloudTrail Input to reach failing state");
+        apis.inputs().deleteInput(inputId);
+    }
+}

graylog-project-parent/pom.xml

@@ -614,7 +614,7 @@
                                 <plugin>
                                     <groupId>com.mebigfatguy.fb-contrib</groupId>
                                     <artifactId>fb-contrib</artifactId>
-                                    <version>7.6.8</version>
+                                    <version>7.6.9</version>
                                 </plugin>
                             </plugins>
                         </configuration>

graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java

@@ -50,6 +50,11 @@ default boolean withNodeIdFile() {
         return true;
     }
 
+    @Override
+    default boolean withInputs() {
+        return false;
+    }
+
     @Override
     default Set<ServerStatus.Capability> withCapabilities() {
         return Set.of(ServerStatus.Capability.SERVER);

graylog2-server/src/main/java/org/graylog2/Configuration.java

@@ -670,4 +670,9 @@ private static int defaultNumberOfOutputBufferProcessors() {
     public boolean withPlugins() {
         return true;
     }
+
+    @Override
+    public boolean withInputs() {
+        return true;
+    }
 }

graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java

@@ -51,6 +51,11 @@ public interface GraylogNodeConfiguration {
      */
     boolean withNodeIdFile();
 
+    /**
+     * Will only bind an InputConfigurationDeserializerModifier stub if there are no inputs configured
+     */
+    boolean withInputs();
+
     /**
      * Provides the {@link ServerStatus.Capability} to be used by ServerStatusBindings.
      */

graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java

@@ -78,7 +78,9 @@ protected void configure() {
         install(new ServerStatusBindings(configuration.withCapabilities()));
 
         bind(EncryptedValueService.class).asEagerSingleton();
-        bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig());
+        if (!configuration.withInputs()) {
+            bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig());
+        }
     }
 
     public Set<Object> getConfigurationBeans() {

graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java

@@ -23,6 +23,8 @@
 import jakarta.inject.Singleton;
 import org.apache.commons.lang.StringEscapeUtils;
 
+import java.util.Collection;
+import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
 
@@ -34,7 +36,11 @@ public class JsonSafeEngineProvider implements Provider<Engine> {
     public JsonSafeEngineProvider() {
         engine = Engine.createEngine();
         engine.registerRenderer(String.class, new JsonSafeRenderer());
+        engine.registerRenderer(Map.class, new JsonSafeMapRenderer());
+        engine.registerRenderer(Iterable.class, new JsonSafeIterableRenderer());
+        engine.registerRenderer(Collection.class, new JsonSafeCollectionRenderer());
     }
+
     @Override
     public Engine get() {
         return engine;
@@ -52,4 +58,55 @@ public String render(String s, Locale locale, Map<String, Object> map) {
             return StringEscapeUtils.escapeJava(s).replace("/", "\\/");
         }
     }
+
+    @SuppressWarnings("rawtypes")
+    private static class JsonSafeMapRenderer implements Renderer<Map> {
+
+        @Override
+        public String render(Map map, Locale locale, Map<String, Object> map2) {
+            final String renderedResult;
+
+            if (map.isEmpty()) {
+                renderedResult = "";
+            } else if (map.size() == 1) {
+                renderedResult = map.values().iterator().next().toString();
+            } else {
+                renderedResult = map.toString();
+            }
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+        }
+    }
+
+    private static class JsonSafeIterableRenderer implements Renderer<Iterable> {
+
+        @Override
+        public String render(Iterable iterable, Locale locale, Map<String, Object> model) {
+            final String renderedResult;
+
+            final Iterator<?> iterator = iterable.iterator();
+            renderedResult = iterator.hasNext() ? iterator.next().toString() : "";
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+
+        }
+
+    }
+
+    private static class JsonSafeCollectionRenderer implements Renderer<Collection> {
+
+        @Override
+        public String render(Collection collection, Locale locale, Map<String, Object> model) {
+            final String renderedResult;
+
+            if (collection.isEmpty()) {
+                renderedResult = "";
+            } else if (collection.size() == 1) {
+                renderedResult = collection.iterator().next().toString();
+            } else {
+                renderedResult = collection.toString();
+            }
+            return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/");
+
+        }
+
+    }
 }

graylog2-server/src/test/java/org/graylog/events/notifications/types/HTTPEventNotificationV2Test.java

@@ -16,7 +16,6 @@
  */
 package org.graylog.events.notifications.types;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.floreysoft.jmte.Engine;
 import com.google.common.collect.ImmutableList;
 import org.graylog.events.configuration.EventsConfigurationProvider;
@@ -38,7 +37,8 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mock;
 
-import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -74,21 +74,63 @@ void setUp() {
     }
 
     @Test
-    public void testEscapedQuotesInBacklog() throws UnsupportedEncodingException, JsonProcessingException {
+    public void testEscapedQuotesInBacklog() {
         Map<String, Object> model = Map.of(
                 "event_definition_title", "<<Test Event Title>>",
-                "event", Map.of("message", "Event Message & Whatnot"),
-                "backlog", createBacklog()
+                "backlog", createBacklog(),
+                "event", createEvent()
         );
         String bodyTemplate = "${if backlog}{\"backlog\": [${foreach backlog message}{ \"title\": \"Message\", \"value\": \"${message.message}\" }${if last_message}${else},${end}${end}]}${end}";
         String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model);
         assertThat(body).contains("\"value\": \"Message with \\\"Double Quotes\\\"");
     }
 
+    @Test
+    public void testEscapedQuotesInEventFields() {
+        Map<String, Object> model = Map.of(
+                "event_definition_title", "<<Test Event Title>>",
+                "backlog", createBacklog(),
+                "event", createEvent()
+        );
+        String bodyTemplate = "{\n" +
+                "    \"message\": \"${event.message}\\\\n\\\\n${event.fields}\",\n" +
+                "    \"title\": \"${event_definition_title}\"\n" +
+                "}";
+        String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model);
+        assertThat(body).contains("\\\"bad_field\\\"");
+    }
+
+    @Test
+    public void testEscapedQuotesInList() {
+        Map<String, Object> model = Map.of(
+                "event_definition_title", "<<Test Event Title>>",
+                "backlog", createBacklog(),
+                "event", createEvent()
+        );
+        String bodyTemplate = "{\n" +
+                "    \"message\": \"${event.message}\\\\n\\\\n${event.list_field}\",\n" +
+                "    \"title\": \"${event_definition_title}\"\n" +
+                "}";
+        String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model);
+        assertThat(body).contains("\\\"list_value1\\\"");
+    }
+
     private ImmutableList<MessageSummary> createBacklog() {
         Message message = new TestMessageFactory().createMessage("Message with \"Double Quotes\"", "Unit Test", DateTime.now(DateTimeZone.UTC));
         MessageSummary summary = new MessageSummary("index1", message);
         return ImmutableList.of(summary);
     }
 
+    private Map<String, Object> createEvent() {
+        final Map<String, Object> event = new HashMap<>();
+        final Map<String, Object> fields = Map.of(
+                "field1", "\"bad_field\"",
+                "field2", "A somehow \"worse\" field!"
+        );
+        event.put("message", "Event Message & Whatnot");
+        event.put("fields", fields);
+        event.put("list_field", List.of("\"list_value1\"", "\"list_value2\""));
+        return event;
+    }
+
 }

graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/GraylogApis.java

@@ -31,6 +31,7 @@
 import org.graylog.plugins.views.search.searchtypes.pivot.Pivot;
 import org.graylog.testing.completebackend.GraylogBackend;
 import org.graylog.testing.completebackend.apis.inputs.GelfInputApi;
+import org.graylog.testing.completebackend.apis.inputs.Inputs;
 import org.graylog.testing.completebackend.apis.inputs.PortBoundGelfInputApi;
 import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange;
 import org.graylog2.shared.bindings.providers.ObjectMapperProvider;
@@ -67,6 +68,7 @@ public class GraylogApis implements GraylogRestApi {
     private final EventDefinitions eventDefinitions;
     private final Dashboards dashboards;
     private final Pipelines pipelines;
+    private final Inputs inputs;
 
     public GraylogApis(GraylogBackend backend) {
         this.backend = backend;
@@ -83,6 +85,7 @@ public GraylogApis(GraylogBackend backend) {
         this.eventDefinitions = new EventDefinitions(this);
         this.dashboards = new Dashboards(this);
         this.pipelines = new Pipelines(this);
+        this.inputs = new Inputs(this);
     }
 
     public RequestSpecification requestSpecification() {
@@ -156,6 +159,10 @@ public Pipelines pipelines() {
         return pipelines;
     }
 
+    public Inputs inputs() {
+        return inputs;
+    }
+
     protected RequestSpecification prefix(final Users.User user) {
         return given()
                 .config(withGraylogBackendFailureConfig())