diff --git a/changelog/unreleased/issue-20955.toml b/changelog/unreleased/issue-20955.toml new file mode 100644 index 000000000000..7f3232612b68 --- /dev/null +++ b/changelog/unreleased/issue-20955.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Fix unescaped double quotes in map and collection typed fields in Custom HTTP Notification JSON body." + +issues = ["20955"] +pulls = ["21167"] diff --git a/full-backend-tests/src/test/java/org/graylog2/inputs/InputCreationIT.java b/full-backend-tests/src/test/java/org/graylog2/inputs/InputCreationIT.java new file mode 100644 index 000000000000..5669863fb60b --- /dev/null +++ b/full-backend-tests/src/test/java/org/graylog2/inputs/InputCreationIT.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.inputs; + +import org.graylog.testing.completebackend.Lifecycle; +import org.graylog.testing.completebackend.apis.GraylogApis; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTest; +import org.graylog.testing.containermatrix.annotations.ContainerMatrixTestsConfiguration; + +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; + +@ContainerMatrixTestsConfiguration(serverLifecycle = Lifecycle.CLASS) +public class InputCreationIT { + + private final GraylogApis apis; + + public InputCreationIT(GraylogApis apis) { + this.apis = apis; + } + + @ContainerMatrixTest + void testHttpRandomInputCreation() { + String inputId = apis.inputs().createGlobalInput("testInput", + "org.graylog2.inputs.random.FakeHttpMessageInput", + Map.of("sleep", 30, + "sleep_deviation", 30, + "source", "example.org")); + apis.inputs().getInput(inputId) + .assertThat().body("title", equalTo("testInput")); + apis.waitFor(() -> + apis.inputs().getInputState(inputId) + .extract().body().jsonPath().get("state") + .equals("RUNNING"), + "Timed out waiting for HTTP Random Message Input to become available"); + apis.inputs().deleteInput(inputId); + } + + /** + * Test to make sure configuration encryption serialization/deserialization works + */ + @ContainerMatrixTest + void testFailingAwsCloudTrailInputCreation() { + String inputId = apis.inputs().createGlobalInput("testInput", + "org.graylog.aws.inputs.cloudtrail.CloudTrailInput", + Map.of("aws_sqs_region", "us-east-1", + "aws_s3_region", "us-east-1", + "aws_sqs_queue_name", "invalid-queue-no-messages-read", + "aws_access_key", "invalid-access-key", + "aws_secret_key", "invalid-secret-key")); + apis.inputs().getInput(inputId) + .assertThat().body("attributes.aws_access_key", equalTo("invalid-access-key")); + apis.waitFor(() -> + apis.inputs().getInputState(inputId) + .extract().body().jsonPath().get("state") + .equals("FAILING"), + "Timed out waiting for AWS CloudTrail Input to reach failing state"); + apis.inputs().deleteInput(inputId); + } +} diff --git a/graylog-project-parent/pom.xml b/graylog-project-parent/pom.xml index 685deb620893..b0aa1635483c 100644 --- a/graylog-project-parent/pom.xml +++ b/graylog-project-parent/pom.xml @@ -614,7 +614,7 @@ com.mebigfatguy.fb-contrib fb-contrib - 7.6.8 + 7.6.9 diff --git a/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java b/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java index 9b2666b98524..cf62d22f66fd 100644 --- a/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/CommonNodeConfiguration.java @@ -50,6 +50,11 @@ default boolean withNodeIdFile() { return true; } + @Override + default boolean withInputs() { + return false; + } + @Override default Set withCapabilities() { return Set.of(ServerStatus.Capability.SERVER); diff --git a/graylog2-server/src/main/java/org/graylog2/Configuration.java b/graylog2-server/src/main/java/org/graylog2/Configuration.java index 2ac5763422bc..a617e63c7261 100644 --- a/graylog2-server/src/main/java/org/graylog2/Configuration.java +++ b/graylog2-server/src/main/java/org/graylog2/Configuration.java @@ -670,4 +670,9 @@ private static int defaultNumberOfOutputBufferProcessors() { public boolean withPlugins() { return true; } + + @Override + public boolean withInputs() { + return true; + } } diff --git a/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java b/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java index 81022704c13f..1207284da0aa 100644 --- a/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java +++ b/graylog2-server/src/main/java/org/graylog2/GraylogNodeConfiguration.java @@ -51,6 +51,11 @@ public interface GraylogNodeConfiguration { */ boolean withNodeIdFile(); + /** + * Will only bind an InputConfigurationDeserializerModifier stub if there are no inputs configured + */ + boolean withInputs(); + /** * Provides the {@link ServerStatus.Capability} to be used by ServerStatusBindings. */ diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java b/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java index 4ed076768e4a..c6f4699e7289 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/GraylogNodeModule.java @@ -78,7 +78,9 @@ protected void configure() { install(new ServerStatusBindings(configuration.withCapabilities())); bind(EncryptedValueService.class).asEagerSingleton(); - bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig()); + if (!configuration.withInputs()) { + bind(InputConfigurationBeanDeserializerModifier.class).toInstance(InputConfigurationBeanDeserializerModifier.withoutConfig()); + } } public Set getConfigurationBeans() { diff --git a/graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java b/graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java index b616c662a75e..9abfbd42d5d3 100644 --- a/graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java +++ b/graylog2-server/src/main/java/org/graylog2/bindings/providers/JsonSafeEngineProvider.java @@ -23,6 +23,8 @@ import jakarta.inject.Singleton; import org.apache.commons.lang.StringEscapeUtils; +import java.util.Collection; +import java.util.Iterator; import java.util.Locale; import java.util.Map; @@ -34,7 +36,11 @@ public class JsonSafeEngineProvider implements Provider { public JsonSafeEngineProvider() { engine = Engine.createEngine(); engine.registerRenderer(String.class, new JsonSafeRenderer()); + engine.registerRenderer(Map.class, new JsonSafeMapRenderer()); + engine.registerRenderer(Iterable.class, new JsonSafeIterableRenderer()); + engine.registerRenderer(Collection.class, new JsonSafeCollectionRenderer()); } + @Override public Engine get() { return engine; @@ -52,4 +58,55 @@ public String render(String s, Locale locale, Map map) { return StringEscapeUtils.escapeJava(s).replace("/", "\\/"); } } + + @SuppressWarnings("rawtypes") + private static class JsonSafeMapRenderer implements Renderer { + + @Override + public String render(Map map, Locale locale, Map map2) { + final String renderedResult; + + if (map.isEmpty()) { + renderedResult = ""; + } else if (map.size() == 1) { + renderedResult = map.values().iterator().next().toString(); + } else { + renderedResult = map.toString(); + } + return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/"); + } + } + + private static class JsonSafeIterableRenderer implements Renderer { + + @Override + public String render(Iterable iterable, Locale locale, Map model) { + final String renderedResult; + + final Iterator iterator = iterable.iterator(); + renderedResult = iterator.hasNext() ? iterator.next().toString() : ""; + return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/"); + + } + + } + + private static class JsonSafeCollectionRenderer implements Renderer { + + @Override + public String render(Collection collection, Locale locale, Map model) { + final String renderedResult; + + if (collection.isEmpty()) { + renderedResult = ""; + } else if (collection.size() == 1) { + renderedResult = collection.iterator().next().toString(); + } else { + renderedResult = collection.toString(); + } + return StringEscapeUtils.escapeJava(renderedResult).replace("/", "\\/"); + + } + + } } diff --git a/graylog2-server/src/test/java/org/graylog/events/notifications/types/HTTPEventNotificationV2Test.java b/graylog2-server/src/test/java/org/graylog/events/notifications/types/HTTPEventNotificationV2Test.java index a01d2b7a0877..241090c93781 100644 --- a/graylog2-server/src/test/java/org/graylog/events/notifications/types/HTTPEventNotificationV2Test.java +++ b/graylog2-server/src/test/java/org/graylog/events/notifications/types/HTTPEventNotificationV2Test.java @@ -16,7 +16,6 @@ */ package org.graylog.events.notifications.types; -import com.fasterxml.jackson.core.JsonProcessingException; import com.floreysoft.jmte.Engine; import com.google.common.collect.ImmutableList; import org.graylog.events.configuration.EventsConfigurationProvider; @@ -38,7 +37,8 @@ import org.junit.jupiter.api.Test; import org.mockito.Mock; -import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -74,21 +74,63 @@ void setUp() { } @Test - public void testEscapedQuotesInBacklog() throws UnsupportedEncodingException, JsonProcessingException { + public void testEscapedQuotesInBacklog() { Map model = Map.of( "event_definition_title", "<>", - "event", Map.of("message", "Event Message & Whatnot"), - "backlog", createBacklog() + "backlog", createBacklog(), + "event", createEvent() ); String bodyTemplate = "${if backlog}{\"backlog\": [${foreach backlog message}{ \"title\": \"Message\", \"value\": \"${message.message}\" }${if last_message}${else},${end}${end}]}${end}"; String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model); assertThat(body).contains("\"value\": \"Message with \\\"Double Quotes\\\""); } + @Test + public void testEscapedQuotesInEventFields() { + Map model = Map.of( + "event_definition_title", "<>", + "backlog", createBacklog(), + "event", createEvent() + ); + String bodyTemplate = "{\n" + + " \"message\": \"${event.message}\\\\n\\\\n${event.fields}\",\n" + + " \"title\": \"${event_definition_title}\"\n" + + "}"; + String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model); + assertThat(body).contains("\\\"bad_field\\\""); + } + + @Test + public void testEscapedQuotesInList() { + Map model = Map.of( + "event_definition_title", "<>", + "backlog", createBacklog(), + "event", createEvent() + ); + String bodyTemplate = "{\n" + + " \"message\": \"${event.message}\\\\n\\\\n${event.list_field}\",\n" + + " \"title\": \"${event_definition_title}\"\n" + + "}"; + String body = notification.transformBody(bodyTemplate, HTTPEventNotificationConfigV2.ContentType.JSON, model); + assertThat(body).contains("\\\"list_value1\\\""); + } + private ImmutableList createBacklog() { Message message = new TestMessageFactory().createMessage("Message with \"Double Quotes\"", "Unit Test", DateTime.now(DateTimeZone.UTC)); MessageSummary summary = new MessageSummary("index1", message); return ImmutableList.of(summary); } + private Map createEvent() { + final Map event = new HashMap<>(); + final Map fields = Map.of( + "field1", "\"bad_field\"", + "field2", "A somehow \"worse\" field!" + ); + event.put("message", "Event Message & Whatnot"); + event.put("fields", fields); + event.put("list_field", List.of("\"list_value1\"", "\"list_value2\"")); + return event; + } + } diff --git a/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/GraylogApis.java b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/GraylogApis.java index 95761a8a8cce..fabf4efa55db 100644 --- a/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/GraylogApis.java +++ b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/GraylogApis.java @@ -31,6 +31,7 @@ import org.graylog.plugins.views.search.searchtypes.pivot.Pivot; import org.graylog.testing.completebackend.GraylogBackend; import org.graylog.testing.completebackend.apis.inputs.GelfInputApi; +import org.graylog.testing.completebackend.apis.inputs.Inputs; import org.graylog.testing.completebackend.apis.inputs.PortBoundGelfInputApi; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; import org.graylog2.shared.bindings.providers.ObjectMapperProvider; @@ -67,6 +68,7 @@ public class GraylogApis implements GraylogRestApi { private final EventDefinitions eventDefinitions; private final Dashboards dashboards; private final Pipelines pipelines; + private final Inputs inputs; public GraylogApis(GraylogBackend backend) { this.backend = backend; @@ -83,6 +85,7 @@ public GraylogApis(GraylogBackend backend) { this.eventDefinitions = new EventDefinitions(this); this.dashboards = new Dashboards(this); this.pipelines = new Pipelines(this); + this.inputs = new Inputs(this); } public RequestSpecification requestSpecification() { @@ -156,6 +159,10 @@ public Pipelines pipelines() { return pipelines; } + public Inputs inputs() { + return inputs; + } + protected RequestSpecification prefix(final Users.User user) { return given() .config(withGraylogBackendFailureConfig()) diff --git a/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/inputs/Inputs.java b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/inputs/Inputs.java new file mode 100644 index 000000000000..9721852f8311 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog/testing/completebackend/apis/inputs/Inputs.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog.testing.completebackend.apis.inputs; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.restassured.response.ValidatableResponse; +import org.graylog.testing.completebackend.apis.GraylogApis; +import org.graylog.testing.completebackend.apis.GraylogRestApi; + +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.notNullValue; + +public class Inputs implements GraylogRestApi { + + private final GraylogApis api; + + public Inputs(GraylogApis api) { + this.api = api; + } + + record CreateInputRequest(@JsonProperty("title") String title, + @JsonProperty("type") String type, + @JsonProperty("global") boolean global, + @JsonProperty("configuration") Map configuration) {} + + public String createGlobalInput(String title, String type, Map configuration) { + final CreateInputRequest body = new CreateInputRequest(title, type, true, configuration); + return given(). + spec(api.requestSpecification()) + .when() + .body(body) + .post("/system/inputs") + .then() + .log().ifError() + .statusCode(201) + .assertThat().body("id", notNullValue()) + .extract().body().jsonPath().getString("id"); + } + + public ValidatableResponse getInput(String inputId) { + return given() + .spec(api.requestSpecification()) + .when() + .get("/system/inputs/" + inputId) + .then() + .log().ifError() + .statusCode(200); + } + + public ValidatableResponse getInputState(String inputId) { + return given() + .spec(api.requestSpecification()) + .when() + .get("/system/inputstates/" + inputId) + .then() + .log().ifError() + .statusCode(200); + } + + public ValidatableResponse deleteInput(String inputId) { + return given() + .spec(api.requestSpecification()) + .when() + .delete("/system/inputs/" + inputId) + .then() + .log().ifError() + .statusCode(204); + } +} diff --git a/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java b/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java index 07454907a6ee..24d9f49a3e30 100644 --- a/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java +++ b/graylog2-server/src/test/java/org/graylog2/commands/MinimalNodeCommandTest.java @@ -123,6 +123,11 @@ public String getSystemPropertyPrefix() { public boolean isMessageRecordingsEnabled() { return false; } + + @Override + public boolean withInputs() { + return false; + } } } diff --git a/graylog2-web-interface/packages/eslint-config-graylog/index.js b/graylog2-web-interface/packages/eslint-config-graylog/index.js index 22dd46e1e9f5..6c8e22bb97df 100644 --- a/graylog2-web-interface/packages/eslint-config-graylog/index.js +++ b/graylog2-web-interface/packages/eslint-config-graylog/index.js @@ -105,6 +105,7 @@ module.exports = { position: 'after', }], 'newlines-between': 'always', + pathGroupsExcludedImportTypes: ['builtin'], }], 'sort-imports': 'off', // disabled in favor of 'import/order' 'jsx-a11y/label-has-associated-control': ['error', { assert: 'either' }], diff --git a/graylog2-web-interface/packages/stylelint-config-graylog/package.json b/graylog2-web-interface/packages/stylelint-config-graylog/package.json index 9aaa073ec4f1..7de062dc73f1 100644 --- a/graylog2-web-interface/packages/stylelint-config-graylog/package.json +++ b/graylog2-web-interface/packages/stylelint-config-graylog/package.json @@ -14,7 +14,7 @@ "license": "SSPL-1.0", "dependencies": { "postcss-styled-syntax": "0.7.0", - "stylelint": "16.11.0", + "stylelint": "16.12.0", "stylelint-config-recommended": "14.0.1", "stylelint-config-standard": "36.0.1", "stylelint-config-styled-components": "0.1.1" diff --git a/pom.xml b/pom.xml index 27f6785374ac..780c8386f295 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 2.15.0-1 3.0.0 - 5.23.0 + 5.24.0 4.13.2 2.1.7 4.5.14 @@ -103,7 +103,7 @@ 2.6.0 2.2.0 1.79 - 1.15.10 + 1.15.11 3.1.8 0.0.1.10 4.8.179 @@ -123,11 +123,11 @@ 0.1.9-graylog-3 1.69.0 2.0.0 - 33.3.1-jre + 33.4.0-jre 7.0.0 2.2.2 3.0 - 8.0.1.Final + 8.0.2.Final 3.1.1 2.17.2 0.15.0 @@ -194,7 +194,7 @@ 3.17.5 1.5 4.13.2 - 5.11.3 + 5.11.4 5.14.2 5.5.0 1.19.0