From 55ec9ba0fcbd2f0d2cfe62dc77ecc506209c09e8 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 9 Nov 2017 14:32:04 -0800 Subject: [PATCH] add json-processor support for non-map json types The Json Processor originally only supported parsing field values into Maps even though the JSON spec specifies that strings, null-values, numbers, booleans, and arrays are also valid JSON types. This commit enables parsing these values now. response to #25972. --- docs/reference/ingest/ingest-node.asciidoc | 2 + .../ingest/common/JsonProcessor.java | 43 +++++++-- .../ingest/common/JsonProcessorTests.java | 93 +++++++++++++++++-- .../rest-api-spec/test/ingest/140_json.yml | 41 +++++++- 4 files changed, 163 insertions(+), 16 deletions(-) diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 720a180934324..54d1a00d335e4 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1580,6 +1580,8 @@ Converts a JSON string into a structured JSON object. | `add_to_root` | no | false | Flag that forces the serialized json to be injected into the top level of the document. `target_field` must not be set when this option is chosen. |====== +All JSON-supported types will be parsed (null, boolean, number, array, object, string). + Suppose you provide this configuration of the `json` processor: [source,js] diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index d4c7efe0d5092..8ca31787b5aaa 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -19,14 +19,24 @@ package org.elasticsearch.ingest.common; +import com.fasterxml.jackson.core.JsonParseException; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.json.JsonXContentParser; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import java.io.IOException; import java.util.Map; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -64,17 +74,36 @@ boolean isAddToRoot() { @Override public void execute(IngestDocument document) throws Exception { - String stringValue = document.getFieldValue(field, String.class); - try { - Map mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false); - if (addToRoot) { - for (Map.Entry entry : mapValue.entrySet()) { + Object fieldValue = document.getFieldValue(field, Object.class); + BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString()); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, bytesRef)) { + XContentParser.Token token = parser.nextToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token == XContentParser.Token.VALUE_STRING) { + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.map(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.list(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + throw new IllegalArgumentException("cannot read binary value"); + } + if (addToRoot && (value instanceof Map)) { + for (Map.Entry entry : ((Map) value).entrySet()) { document.setFieldValue(entry.getKey(), entry.getValue()); } + } else if (addToRoot) { + throw new IllegalArgumentException("cannot add non-map fields to root of document"); } else { - document.setFieldValue(targetField, mapValue); + document.setFieldValue(targetField, value); } - } catch (ElasticsearchParseException e) { + } catch (IOException e) { throw new IllegalArgumentException(e); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java index 66ffb87e8da14..ef17935962d0e 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java @@ -21,15 +21,19 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class JsonProcessorTests extends ESTestCase { @@ -44,7 +48,7 @@ public void testExecute() throws Exception { Map randomJsonMap = RandomDocumentPicks.randomSource(random()); XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap); - String randomJson = XContentHelper.convertToJson(builder.bytes(), false); + String randomJson = XContentHelper.convertToJson(builder.bytes(), false, XContentType.JSON); document.put(randomField, randomJson); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); @@ -53,16 +57,84 @@ public void testExecute() throws Exception { assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified); } - public void testInvalidJson() { + public void testInvalidValue() { JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); Map document = new HashMap<>(); - document.put("field", "invalid json"); + document.put("field", "blah blah"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument)); - assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token" - + " 'invalid': was expecting ('true', 'false' or 'null')\n" - + " at [Source: invalid json; line: 1, column: 8]")); + assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " + + "was expecting ('true', 'false' or 'null')")); + } + + public void testByteArray() { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + document.put("field", new byte[] { 0, 1 }); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument)); + assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')")); + } + + public void testNull() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + document.put("field", null); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertNull(ingestDocument.getFieldValue("target_field", Object.class)); + } + + public void testBoolean() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + boolean value = true; + document.put("field", value); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value)); + } + + public void testInteger() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + int value = 3; + document.put("field", value); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value)); + } + + public void testDouble() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + double value = 3.0; + document.put("field", value); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value)); + } + + public void testString() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + String value = "hello world"; + document.put("field", "\"" + value + "\""); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value)); + } + + public void testArray() throws Exception { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false); + Map document = new HashMap<>(); + List value = Arrays.asList(true, true, false); + document.put("field", value.toString()); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + jsonProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value)); } public void testFieldMissing() { @@ -96,4 +168,13 @@ public void testAddToRoot() throws Exception { assertIngestDocument(ingestDocument, expectedIngestDocument); } + + public void testAddBoolToRoot() { + JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", true); + Map document = new HashMap<>(); + document.put("field", true); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument)); + assertThat(exception.getMessage(), containsString("cannot add non-map fields to root of document")); + } } diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml index 3d9f6a97c081a..81761ba509e10 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml @@ -15,7 +15,32 @@ teardown: "processors": [ { "json" : { - "field" : "foo" + "field" : "foo_object" + } + }, + { + "json" : { + "field" : "foo_array" + } + }, + { + "json" : { + "field" : "foo_null" + } + }, + { + "json" : { + "field" : "foo_string" + } + }, + { + "json" : { + "field" : "foo_number" + } + }, + { + "json" : { + "field" : "foo_boolean" } } ] @@ -29,7 +54,12 @@ teardown: id: 1 pipeline: "1" body: { - foo: "{\"hello\": \"world\"}" + foo_object: "{\"hello\": \"world\"}", + foo_array: "[1, 2, 3]", + foo_null: null, + foo_string: "\"bla bla\"", + foo_number: 3, + foo_boolean: "true" } - do: @@ -37,4 +67,9 @@ teardown: index: test type: test id: 1 - - match: { _source.foo.hello: "world" } + - match: { _source.foo_object.hello: "world" } + - match: { _source.foo_array.0: 1 } + - match: { _source.foo_string: "bla bla" } + - match: { _source.foo_number: 3 } + - is_true: _source.foo_boolean + - is_false: _source.foo_null