Dep. check for ECS changes to User Agent processor (#38362)

The User Agent ingest processor is changing to align with ECS. Users need to be made aware that any pipelines using this processor will behave differently in 7.0.
elastic · Feb 11, 2019 · 0786e76 · 0786e76
1 parent eeb3176
commit 0786e76
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 2 deletions.
diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -343,7 +343,7 @@ public String getType() {
         return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor));
     }
 
-    static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
+    public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) {
         IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE);
         Map<String, PipelineConfiguration> pipelines;
         if (currentIngestMetadata != null) {

diff --git a/...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.ingest.PipelineConfiguration;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class ClusterDeprecationChecks {
+
+    @SuppressWarnings("unchecked")
+    static DeprecationIssue checkUserAgentPipelines(ClusterState state) {
+        List<PipelineConfiguration> pipelines = IngestService.getPipelines(state);
+
+        List<String> pipelinesWithDeprecatedEcsConfig = pipelines.stream()
+            .filter(Objects::nonNull)
+            .filter(pipeline -> {
+                Map<String, Object> pipelineConfig = pipeline.getConfigAsMap();
+
+                List<Map<String, Map<String, Object>>> processors =
+                    (List<Map<String, Map<String, Object>>>) pipelineConfig.get("processors");
+                return processors.stream()
+                    .filter(Objects::nonNull)
+                    .filter(processor -> processor.containsKey("user_agent"))
+                    .map(processor -> processor.get("user_agent"))
+                    .anyMatch(processorConfig ->
+                        false == ConfigurationUtils.readBooleanProperty(null, null, processorConfig, "ecs", false));
+            })
+            .map(PipelineConfiguration::getId)
+            .sorted() // Make the warning consistent for testing purposes
+            .collect(Collectors.toList());
+        if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) {
+            return new DeprecationIssue(DeprecationIssue.Level.WARNING,
+                "User-Agent ingest plugin will use ECS-formatted output",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
+                    "#ingest-user-agent-ecs-always",
+                "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " will change to using ECS output format in 7.0");
+        }
+        return null;
+
+    }
+}
diff --git a/...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
@@ -31,7 +31,10 @@ private DeprecationChecks() {
     }
 
     static List<Function<ClusterState, DeprecationIssue>> CLUSTER_SETTINGS_CHECKS =
-        Collections.emptyList();
+        Collections.unmodifiableList(Arrays.asList(
+            ClusterDeprecationChecks::checkUserAgentPipelines
+        ));
+
 
     static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS =
         Collections.unmodifiableList(Arrays.asList(

diff --git a/...tion/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/...tion/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.deprecation;
+
+import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.ingest.IngestService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS;
+
+public class ClusterDeprecationChecksTests extends ESTestCase {
+
+    public void testUserAgentEcsCheck() {
+        PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\",\n" +
+                "        \"ecs\" : false\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+        PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\"\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+        PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true",
+            new BytesArray("{\n" +
+                "  \"description\" : \"This has ecs set to false\",\n" +
+                "  \"processors\" : [\n" +
+                "    {\n" +
+                "      \"user_agent\" : {\n" +
+                "        \"field\" : \"agent\",\n" +
+                "        \"ecs\" : true\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}"), XContentType.JSON);
+
+        ClusterState state = ClusterState.builder(new ClusterName("test")).build();
+        state = IngestService.innerPut(ecsTrueRequest, state);
+        state = IngestService.innerPut(ecsFalseRequest, state);
+        state = IngestService.innerPut(ecsNullRequest, state);
+
+        final ClusterState finalState = state;
+        List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState));
+
+        DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING,
+            "User-Agent ingest plugin will use ECS-formatted output",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
+                "#ingest-user-agent-ecs-always",
+            "Ingest pipelines [ecs_false, ecs_null] will change to using ECS output format in 7.0");
+        assertEquals(singletonList(expected), issues);
+    }
+}