add json-processor support for non-map json types

The Json Processor originally only supported parsing field values into Maps even though the JSON spec specifies that strings, null-values, numbers, booleans, and arrays are also valid JSON types. This commit enables parsing these values now. response to #25972.
elastic · Nov 9, 2017 · 55ec9ba · 55ec9ba
1 parent d22fd4e
commit 55ec9ba
Show file tree

Hide file tree

Showing 4 changed files with 163 additions and 16 deletions.
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
@@ -1580,6 +1580,8 @@ Converts a JSON string into a structured JSON object.
 | `add_to_root`  | no        | false    | Flag that forces the serialized json to be injected into the top level of the document. `target_field` must not be set when this option is chosen.
 |======
 
+All JSON-supported types will be parsed (null, boolean, number, array, object, string).
+
 Suppose you provide this configuration of the `json` processor:
 
 [source,js]

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -19,14 +19,24 @@
 
 package org.elasticsearch.ingest.common;
 
+import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.json.JsonXContentParser;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
+import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
@@ -64,17 +74,36 @@ boolean isAddToRoot() {
 
     @Override
     public void execute(IngestDocument document) throws Exception {
-        String stringValue = document.getFieldValue(field, String.class);
-        try {
-            Map<String, Object> mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false);
-            if (addToRoot) {
-                for (Map.Entry<String, Object> entry : mapValue.entrySet()) {
+        Object fieldValue = document.getFieldValue(field, Object.class);
+        BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
+        try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, bytesRef)) {
+            XContentParser.Token token = parser.nextToken();
+            Object value = null;
+            if (token == XContentParser.Token.VALUE_NULL) {
+                value = null;
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                value = parser.text();
+            } else if (token == XContentParser.Token.VALUE_NUMBER) {
+                value = parser.numberValue();
+            } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
+                value = parser.booleanValue();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                value = parser.map();
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                value = parser.list();
+            } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+                throw new IllegalArgumentException("cannot read binary value");
+            }
+            if (addToRoot && (value instanceof Map)) {
+                for (Map.Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
                     document.setFieldValue(entry.getKey(), entry.getValue());
                 }
+            } else if (addToRoot) {
+                throw new IllegalArgumentException("cannot add non-map fields to root of document");
             } else {
-                document.setFieldValue(targetField, mapValue);
+                document.setFieldValue(targetField, value);
             }
-        } catch (ElasticsearchParseException e) {
+        } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
     }

diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -21,15 +21,19 @@
 
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class JsonProcessorTests extends ESTestCase {
@@ -44,7 +48,7 @@ public void testExecute() throws Exception {
 
         Map<String, Object> randomJsonMap = RandomDocumentPicks.randomSource(random());
         XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
-        String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
+        String randomJson = XContentHelper.convertToJson(builder.bytes(), false, XContentType.JSON);
         document.put(randomField, randomJson);
 
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@@ -53,16 +57,84 @@ public void testExecute() throws Exception {
         assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
     }
 
-    public void testInvalidJson() {
+    public void testInvalidValue() {
         JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
         Map<String, Object> document = new HashMap<>();
-        document.put("field", "invalid json");
+        document.put("field", "blah blah");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
         Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
-        assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token"
-                + " 'invalid': was expecting ('true', 'false' or 'null')\n"
-                + " at [Source: invalid json; line: 1, column: 8]"));
+        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
+            "was expecting ('true', 'false' or 'null')"));
+    }
+
+    public void testByteArray() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        document.put("field", new byte[] { 0, 1 });
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
+    }
+
+    public void testNull() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        document.put("field", null);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertNull(ingestDocument.getFieldValue("target_field", Object.class));
+    }
+
+    public void testBoolean() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        boolean value = true;
+        document.put("field", value);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+    }
+
+    public void testInteger() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        int value = 3;
+        document.put("field", value);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+    }
+
+    public void testDouble() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        double value = 3.0;
+        document.put("field", value);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+    }
+
+    public void testString() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        String value = "hello world";
+        document.put("field", "\"" + value + "\"");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+    }
+
+    public void testArray() throws Exception {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+        Map<String, Object> document = new HashMap<>();
+        List<Boolean> value = Arrays.asList(true, true, false);
+        document.put("field", value.toString());
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        jsonProcessor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
     }
 
     public void testFieldMissing() {
@@ -96,4 +168,13 @@ public void testAddToRoot() throws Exception {
 
         assertIngestDocument(ingestDocument, expectedIngestDocument);
     }
+
+    public void testAddBoolToRoot() {
+        JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", true);
+        Map<String, Object> document = new HashMap<>();
+        document.put("field", true);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+        assertThat(exception.getMessage(), containsString("cannot add non-map fields to root of document"));
+    }
 }
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml
@@ -15,7 +15,32 @@ teardown:
             "processors": [
               {
                 "json" : {
-                  "field" : "foo"
+                  "field" : "foo_object"
+                }
+              },
+              {
+                "json" : {
+                  "field" : "foo_array"
+                }
+              },
+              {
+                "json" : {
+                  "field" : "foo_null"
+                }
+              },
+              {
+                "json" : {
+                  "field" : "foo_string"
+                }
+              },
+              {
+                "json" : {
+                  "field" : "foo_number"
+                }
+              },
+              {
+                "json" : {
+                  "field" : "foo_boolean"
                 }
               }
             ]
@@ -29,12 +54,22 @@ teardown:
         id: 1
         pipeline: "1"
         body: {
-          foo: "{\"hello\": \"world\"}"
+          foo_object: "{\"hello\": \"world\"}",
+          foo_array: "[1, 2, 3]",
+          foo_null: null,
+          foo_string: "\"bla bla\"",
+          foo_number: 3,
+          foo_boolean: "true"
         }
 
   - do:
       get:
         index: test
         type: test
         id: 1
-  - match: { _source.foo.hello: "world" }
+  - match: { _source.foo_object.hello: "world" }
+  - match: { _source.foo_array.0: 1 }
+  - match: { _source.foo_string: "bla bla" }
+  - match: { _source.foo_number: 3 }
+  - is_true:  _source.foo_boolean
+  - is_false: _source.foo_null