From 06ae3ca88aafc24e8a6b6471d3b42ff074b74b2f Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Fri, 20 Dec 2024 11:09:01 -0600 Subject: [PATCH] Modify transformationPlugins contract to return Object to support One to Many transformations (#1206) Signed-off-by: Andre Kurait --- .../bulkload/common/BulkDocSection.java | 2 +- .../bulkload/common/DocumentReindexer.java | 3 +- .../TransformerToIJsonTransformerAdapter.java | 12 ++++---- .../common/DocumentReindexerTest.java | 2 +- .../replay/ResultsToLogsConsumer.java | 2 +- ...ttpRequestPreliminaryTransformHandler.java | 13 +++++++-- .../http/NettyJsonBodyConvertHandler.java | 15 ++++++++-- .../HttpJsonTransformingConsumerTest.java | 14 ++++----- ...sformerForDocumentTypeRemovalProvider.java | 8 +++-- .../transform/JsonJMESPathPredicate.java | 3 +- .../transform/JsonJMESPathTransformer.java | 2 +- .../JsonJMESPathTransformerProviderTest.java | 2 +- .../transform/JinjavaTransformer.java | 29 +++++++++++++------ .../transform/PreservesProcessor.java | 2 +- .../transform/FeatureMaskingTest.java | 18 ++++++------ .../transform/JinjavaTransformerTest.java | 6 ++-- .../migrations/transform/RouteTest.java | 8 ++--- .../JinjavaTransformerProviderTest.java | 2 +- .../transform/JsonJoltTransformer.java | 2 +- .../migrations/transform/IJsonPredicate.java | 3 +- .../transform/IJsonTransformer.java | 4 +-- .../transform/JsonCompositeTransformer.java | 5 ++-- .../transform/JsonConditionalTransformer.java | 4 +-- .../transform/NoopTransformerProvider.java | 4 +-- .../transform/TransformationLoader.java | 10 ++++--- .../transform/replay/JsonTransformerTest.java | 6 ++-- .../replay/MultipleJMESPathScriptsTest.java | 2 +- .../replay/MultipleJoltScriptsTest.java | 18 ++++++------ .../replay/TransformationLoaderTest.java | 13 +++++---- .../TypeMappingsSanitizationTransformer.java | 4 +-- ...peMappingsSanitizationCreateIndexTest.java | 4 +-- ...peMappingsSanitizationDocBackfillTest.java | 2 +- ...ppingsSanitizationTransformerBulkTest.java | 2 +- .../TypeMappingsSanitizationProviderTest.java | 8 ++--- 34 files changed, 130 insertions(+), 104 deletions(-) diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java index 60ae69877..f5d0eaad6 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java @@ -75,7 +75,7 @@ public static String convertToBulkRequestBody(Collection bulkSec } } - public static BulkDocSection fromMap(Map map) { + public static BulkDocSection fromMap(Object map) { BulkIndex bulkIndex = OBJECT_MAPPER.convertValue(map, BulkIndex.class); return new BulkDocSection(bulkIndex); } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/DocumentReindexer.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/DocumentReindexer.java index eb69123a2..38cff0fac 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/DocumentReindexer.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/DocumentReindexer.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.bulkload.common; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.function.Predicate; @@ -57,7 +56,7 @@ Mono reindexDocsInParallelBatches(Flux docs, String indexN BulkDocSection transformDocument(RfsLuceneDocument doc, String indexName) { var original = new BulkDocSection(doc.id, indexName, doc.type, doc.source, doc.routing); if (transformer != null) { - final Map transformedDoc = transformer.transformJson(original.toMap()); + final Object transformedDoc = transformer.transformJson(original.toMap()); return BulkDocSection.fromMap(transformedDoc); } return BulkDocSection.fromMap(original.toMap()); diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java index 841fe356d..23ba722df 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java @@ -41,7 +41,7 @@ public TransformerToIJsonTransformerAdapter(IJsonTransformer transformer) { this(transformer, LoggerFactory.getLogger(OUTPUT_TRANSFORMATION_JSON_LOGGER)); } - private void logTransformation(Map before, Map after) { + private void logTransformation(Map before, Object after) { if (transformerLogger.isInfoEnabled()) { try { var transformationTuple = toTransformationMap(before, after); @@ -55,7 +55,7 @@ private void logTransformation(Map before, Map a } } - private Map toTransformationMap(Map before, Map after) { + private Map toTransformationMap(Map before, Object after) { var transformationMap = new LinkedHashMap(); transformationMap.put("before", before); transformationMap.put("after", after); @@ -69,7 +69,7 @@ private static Map objectNodeToMap(Object node) { } @SneakyThrows - private static String printMap(Map map) { + private static String printMap(Object map) { return MAPPER.writeValueAsString(map); } @@ -77,7 +77,7 @@ private static String printMap(Map map) { private MigrationItem transformMigrationItem(MigrationItem migrationItem) { // Keep untouched original for logging final Map originalMap = MAPPER.convertValue(migrationItem, Map.class); - var transformedMigrationItem = transformer.transformJson(MAPPER.convertValue(migrationItem, Map.class)); + Object transformedMigrationItem = transformer.transformJson(MAPPER.convertValue(migrationItem, Map.class)); logTransformation(originalMap, transformedMigrationItem); return MAPPER.convertValue(transformedMigrationItem, MigrationItem.class); } @@ -100,7 +100,7 @@ void updateTemplates(Collection transformedItems, Objec public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { var inputJson = objectNodeToMap(globalData.toObjectNode()); log.atInfo().setMessage("BeforeJsonGlobal: {}").addArgument(() -> printMap(inputJson)).log(); - var afterJson = transformer.transformJson(inputJson); + Object afterJson = transformer.transformJson(inputJson); log.atInfo().setMessage("AfterJsonGlobal: {}").addArgument(() -> printMap(afterJson)).log(); @@ -154,7 +154,7 @@ public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { public IndexMetadata transformIndexMetadata(IndexMetadata indexData) { final Map originalInput = MAPPER.convertValue(indexData, Map.class); final Map inputJson = MAPPER.convertValue(indexData, Map.class); - var afterJson = transformer.transformJson(inputJson); + Object afterJson = transformer.transformJson(inputJson); logTransformation(originalInput, afterJson); return MAPPER.convertValue(inputJson, IndexMetadata.class); } diff --git a/RFS/src/test/java/org/opensearch/migrations/bulkload/common/DocumentReindexerTest.java b/RFS/src/test/java/org/opensearch/migrations/bulkload/common/DocumentReindexerTest.java index 0d2ae23a7..a8f3fb565 100644 --- a/RFS/src/test/java/org/opensearch/migrations/bulkload/common/DocumentReindexerTest.java +++ b/RFS/src/test/java/org/opensearch/migrations/bulkload/common/DocumentReindexerTest.java @@ -125,7 +125,7 @@ void reindex_shouldBufferByTransformedSize() throws JsonProcessingException { // Set up the transformer that replaces the sourceDoc from the document var repalcedSourceDoc = Map.of("simpleKey", "simpleValue"); IJsonTransformer transformer = originalJson -> { - originalJson.put("source", repalcedSourceDoc); + ((Map) originalJson).put("source", repalcedSourceDoc); return originalJson; }; int numDocs = 5; diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java index 528e89b93..c8805d87c 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java @@ -142,7 +142,7 @@ public void accept(SourceTargetCaptureTuple tuple, ParsedHttpMessagesAsDicts par if (tupleLogger.isInfoEnabled()) { try { var originalTuple = toJSONObject(tuple, parsedMessages); - var transformedTuple = tupleTransformer.transformJson(originalTuple); + Object transformedTuple = tupleTransformer.transformJson(originalTuple); var tupleString = PLAIN_MAPPER.writeValueAsString(transformedTuple); tupleLogger.atInfo().setMessage("{}").addArgument(tupleString).log(); } catch (Exception e) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java index 89acd028f..98ab3e31f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java @@ -117,10 +117,19 @@ public static HttpJsonRequestWithFaultingPayload transform( assert httpJsonMessage.containsKey("payload"); - var returnedObject = transformer.transformJson(httpJsonMessage); + Object returnedObject = transformer.transformJson(httpJsonMessage); + if(!(returnedObject instanceof Map)) { + throw new TransformationException( + new IllegalArgumentException("Returned object from transformation not map, instead was " + + returnedObject.getClass().getName()) + ); + } + @SuppressWarnings("unchecked") + var returnedObjectMap = (Map) returnedObject; + if (returnedObject != httpJsonMessage) { - httpJsonMessage = new HttpJsonRequestWithFaultingPayload(returnedObject); + httpJsonMessage = new HttpJsonRequestWithFaultingPayload(returnedObjectMap); } if (originalHttpJsonMessage != httpJsonMessage) { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java index 2361a18e7..4f765520c 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyConvertHandler.java @@ -1,5 +1,7 @@ package org.opensearch.migrations.replay.datahandlers.http; +import java.util.Map; + import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; import org.opensearch.migrations.transform.IJsonTransformer; import org.opensearch.migrations.transform.JsonKeysForHttpMessage; @@ -49,10 +51,19 @@ public static HttpJsonRequestWithFaultingPayload transform( var protectionArtifacts = protectByteBufInHttpMessage(httpJsonMessage); - var returnedObject = transformer.transformJson(httpJsonMessage); + Object returnedObject = transformer.transformJson(httpJsonMessage); + if(!(returnedObject instanceof Map)) { + throw new TransformationException( + new IllegalArgumentException("Returned object from transformation not map, instead was " + + returnedObject.getClass().getName()) + ); + } + @SuppressWarnings("unchecked") + var returnedObjectMap = (Map) returnedObject; + if (returnedObject != httpJsonMessage) { - httpJsonMessage = new HttpJsonRequestWithFaultingPayload(returnedObject); + httpJsonMessage = new HttpJsonRequestWithFaultingPayload(returnedObjectMap); } unProtectByteBufInHttpMessage(httpJsonMessage, protectionArtifacts); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index 7babdb434..b72e115af 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -210,11 +210,11 @@ public void testPartialBodyIsPassedThrough() throws Exception { var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var complexTransformer = new JsonCompositeTransformer(new IJsonTransformer() { @Override - public Map transformJson(Map incomingJson) { - var payload = (Map) incomingJson.get("payload"); + public Object transformJson(Object incomingJson) { + var payload = (Map) ((Map) incomingJson).get("payload"); Assertions.assertNull(payload.get(JsonKeysForHttpMessage.INLINED_NDJSON_BODIES_DOCUMENT_KEY)); Assertions.assertNull(payload.get(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY)); - ((Map) incomingJson.get("headers")) + ((Map) ((Map) incomingJson).get("headers")) .put("extraKey", "extraValue"); // just walk everything - that's enough to touch the payload and throw walkMaps(incomingJson); @@ -261,11 +261,11 @@ public void testNewlineDelimitedJsonBodyIsHandled() throws Exception { final var dummyAggregatedResponse = new AggregatedRawResponse(null, 19, Duration.ZERO, List.of(), null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var sizeCalculatingTransformer = new JsonCompositeTransformer(incomingJson -> { - var payload = (Map) incomingJson.get("payload"); + var payload = (Map) ((Map) incomingJson).get("payload"); Assertions.assertNull(payload.get(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY)); Assertions.assertNull(payload.get(JsonKeysForHttpMessage.INLINED_BINARY_BODY_DOCUMENT_KEY)); var list = (List) payload.get(JsonKeysForHttpMessage.INLINED_NDJSON_BODIES_DOCUMENT_KEY); - ((Map) incomingJson.get("headers")) + ((Map) ((Map) incomingJson).get("headers")) .put("listSize", ""+list.size()); return incomingJson; }); @@ -289,13 +289,13 @@ public void testPartialNewlineDelimitedJsonBodyIsHandled() throws Exception { final var dummyAggregatedResponse = new AggregatedRawResponse(null, 19, Duration.ZERO, List.of(), null); var testPacketCapture = new TestCapturePacketToHttpHandler(Duration.ofMillis(100), dummyAggregatedResponse); var sizeCalculatingTransformer = new JsonCompositeTransformer(incomingJson -> { - var payload = (Map) incomingJson.get("payload"); + var payload = (Map) ((Map) incomingJson).get("payload"); Assertions.assertFalse(payload.containsKey(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY)); Assertions.assertFalse(payload.containsKey(JsonKeysForHttpMessage.INLINED_BINARY_BODY_DOCUMENT_KEY)); Assertions.assertNotNull(payload.get(JsonKeysForHttpMessage.INLINED_TEXT_BODY_DOCUMENT_KEY)); var list = (List) payload.get(JsonKeysForHttpMessage.INLINED_NDJSON_BODIES_DOCUMENT_KEY); var leftoverString = (String) payload.get(JsonKeysForHttpMessage.INLINED_TEXT_BODY_DOCUMENT_KEY); - var headers = (Map) incomingJson.get("headers"); + var headers = (Map) ((Map) incomingJson).get("headers"); headers.put("listSize", "" + list.size()); headers.put("leftover", "" + leftoverString.getBytes(StandardCharsets.UTF_8).length); return incomingJson; diff --git a/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java b/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java index df8f9d083..c12421536 100644 --- a/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java +++ b/transformation/src/main/java/org/opensearch/migrations/transformation/JsonTransformerForDocumentTypeRemovalProvider.java @@ -17,9 +17,11 @@ public IJsonTransformer createTransformer(Object jsonConfig) { private static class Transformer implements IJsonTransformer { @Override @SuppressWarnings("unchecked") - public Map transformJson(Map incomingJson) { - if (incomingJson.containsKey("index")) { - ((Map) incomingJson.get("index")).remove("_type"); + public Object transformJson(Object incomingJson) { + @SuppressWarnings("unchecked") + var incomingJsonMap = ((Map) incomingJson); + if (incomingJsonMap.containsKey("index")) { + ((Map) ((Map) incomingJson).get("index")).remove("_type"); } return incomingJson; } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java index 0e7ff4b46..cc598e5a9 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathPredicate.java @@ -1,6 +1,5 @@ package org.opensearch.migrations.transform; -import java.util.Map; import io.burt.jmespath.BaseRuntime; import io.burt.jmespath.Expression; @@ -16,7 +15,7 @@ public JsonJMESPathPredicate(BaseRuntime runtime, String script) { } @Override - public boolean test(Map incomingJson) { + public boolean test(Object incomingJson) { var output = expression.search(incomingJson); log.atDebug().setMessage("output={}").addArgument(output).log(); return (Boolean) output; diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformer.java index 3778e4163..186dd8a19 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJMESPathTransformer.java @@ -16,7 +16,7 @@ public JsonJMESPathTransformer(BaseRuntime runtime, String script) { } @Override - public Map transformJson(Map incomingJson) { + public Object transformJson(Object incomingJson) { var output = expression.search(incomingJson); log.info("output=" + output); return (Map) output; diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java index 912fd3c9e..0ab088e4b 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java @@ -77,7 +77,7 @@ public void testSimpleTransform() throws JsonProcessingException { var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); var transformer = new JsonJMESPathTransformerProvider().createTransformer(Map.of( "script", EXCISE_TYPE_EXPRESSION_STRING)); - var transformedDocument = transformer.transformJson(documentJson); + Object transformedDocument = transformer.transformJson(documentJson); var outputStr = emitJson(mapper, transformedDocument); final String TEST_OUTPUT_REQUEST = "{\n" diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java index 2a5254799..2a4c0bc58 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java @@ -1,11 +1,10 @@ package org.opensearch.migrations.transform; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; -import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import org.opensearch.migrations.transform.jinjava.DynamicMacroFunction; import org.opensearch.migrations.transform.jinjava.JavaRegexCaptureFilter; @@ -30,16 +29,16 @@ public class JinjavaTransformer implements IJsonTransformer { public static final String REGEX_REPLACEMENT_CONVERSION_PATTERNS = "regex_replacement_conversion_patterns"; protected final Jinjava jinjava; - protected final Function, Map> createContextWithSourceFunction; + protected final Function> createContextWithSourceFunction; private final String templateStr; public JinjavaTransformer(String templateString, - UnaryOperator> contextProviderFromSource) { + Function> contextProviderFromSource) { this(templateString, contextProviderFromSource, new JinjavaConfig()); } public JinjavaTransformer(String templateString, - UnaryOperator> contextProviderFromSource, + Function> contextProviderFromSource, @NonNull JinjavaConfig jinjavaConfig) { this(templateString, contextProviderFromSource, @@ -48,7 +47,7 @@ public JinjavaTransformer(String templateString, } public JinjavaTransformer(String templateString, - UnaryOperator> createContextWithSource, + Function> createContextWithSource, ResourceLocator resourceLocator, List> regexReplacementConversionPatterns) { @@ -77,10 +76,22 @@ public JinjavaTransformer(String templateString, @SneakyThrows @Override - public Map transformJson(Map incomingJson) { + @SuppressWarnings("unchecked") + public Object transformJson(Object incomingJson) { var resultStr = jinjava.render(templateStr, createContextWithSourceFunction.apply(incomingJson)); log.atDebug().setMessage("output from jinjava... {}").addArgument(resultStr).log(); - var parsedObj = (Map) objectMapper.readValue(resultStr, LinkedHashMap.class); - return PreservesProcessor.doFinalSubstitutions(incomingJson, parsedObj); + Object parsedObj = objectMapper.readValue(resultStr, Object.class); + if (parsedObj instanceof Map) { + return PreservesProcessor.doFinalSubstitutions((Map) incomingJson, (Map) parsedObj); + } else if (parsedObj instanceof List) { + log.atDebug().setMessage("Received List from jinjava, processing preserves for {} maps.") + .addArgument(((List) parsedObj).size()).log(); + List> listOfMaps = (List>) parsedObj; + return listOfMaps.stream().map( json -> + PreservesProcessor.doFinalSubstitutions((Map) incomingJson, json) + ).collect(Collectors.toList()); + } else { + throw new IllegalArgumentException("Unexpected data format: " + parsedObj.getClass().getName()); + } } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java index a5d346c27..d5445c7c7 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java @@ -31,7 +31,7 @@ private static void processPreserves(Map source, Map source, Map target, - String directiveKey, boolean forced) { + String directiveKey, boolean forced) { Object directive = target.remove(directiveKey); if (directive == null) { return; diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java index 80a01b102..7c35a3172 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java @@ -13,7 +13,7 @@ @Slf4j public class FeatureMaskingTest { private static final ObjectMapper objectMapper = new ObjectMapper(); - public Map transformForMask(Map incomingFlags) throws IOException { + public Object transformForMask(Map incomingFlags) throws IOException { var incomingFlagStr = objectMapper.writeValueAsString(incomingFlags); log.atInfo().setMessage("incoming map as string: {}").addArgument(incomingFlagStr).log(); var flags = FeatureFlags.parseJson(incomingFlagStr); @@ -39,44 +39,44 @@ public Map transformForMask(Map incomingFlags) t public void testFalseFlag() throws Exception { Assertions.assertEquals( "false,false,false,false", - transformForMask(Map.of("testFlag", false)).get("enabledFlags")); + ((Map) transformForMask(Map.of("testFlag", false))).get("enabledFlags")); } @Test public void testTrueFlag() throws Exception { Assertions.assertEquals( "true,false,false,false", - transformForMask(Map.of("testFlag", true)).get("enabledFlags")); + ((Map) transformForMask(Map.of("testFlag", true))).get("enabledFlags")); } @Test public void testLongerFalseFlag() throws Exception { Assertions.assertEquals( "false,false,false,false", - transformForMask(Map.of("testFlag", false)).get("enabledFlags")); + ((Map) transformForMask(Map.of("testFlag", false))).get("enabledFlags")); } @Test public void testLongerTrueFlag() throws Exception { Assertions.assertEquals( "true,false,false,false", - transformForMask(Map.of( + ((Map) transformForMask(Map.of( "testFlag", Map.of("notPresent", false)) - ).get("enabledFlags")); + )).get("enabledFlags")); } @Test public void testImplicitTrueFlag() throws Exception { Assertions.assertEquals( "true,true,false,false", - transformForMask(Map.of( + ((Map) transformForMask(Map.of( "testFlag", Map.of("t1", true)) - ).get("enabledFlags")); + )).get("enabledFlags")); } @Test public void testNullFeatures() throws Exception { Assertions.assertEquals( - "true,true,true,true", transformForMask(null).get("enabledFlags")); + "true,true,true,true", ((Map) transformForMask(null)).get("enabledFlags")); } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java index c45733eaf..195747755 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java @@ -87,7 +87,7 @@ public void testTypeMappingSample() throws Exception { new JinjavaConfig(null, Map.of("hello", "{%- macro hello() -%} hi {%- endmacro -%}\n"))); - var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, Map.class)); + Object resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, Map.class)); Assertions.assertEquals(JsonNormalizer.fromString(testString.replace("indexA/type2/", "indexA_2/_doc/")), JsonNormalizer.fromObject(resultObj)); } @@ -101,7 +101,7 @@ public void testCustomScript() throws Exception { new JinjavaConfig(null, Map.of("hello", "{%- macro hello() -%}{\"hi\": \"world\"}{%- endmacro -%}\n"))); - var resultObj = indexTypeMappingRewriter.transformJson(Map.of()); + Object resultObj = indexTypeMappingRewriter.transformJson(Map.of()); var resultStr = OBJECT_MAPPER.writeValueAsString(resultObj); Assertions.assertEquals("{\"hi\":\"world\"}", resultStr); } @@ -121,7 +121,7 @@ public void debugLoggingWorks() throws Exception { new JinjavaConfig(null, Map.of("hello", "{%- macro hello() -%}{\"hi\": \"world\"}{%- endmacro -%}\n"))); - var resultObj = indexTypeMappingRewriter.transformJson(Map.of()); + Object resultObj = indexTypeMappingRewriter.transformJson(Map.of()); var resultStr = OBJECT_MAPPER.writeValueAsString(resultObj); Assertions.assertEquals("{}", resultStr); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java index e287bcbba..028e77f7f 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java @@ -14,7 +14,7 @@ public class RouteTest { private static final ObjectMapper objectMapper = new ObjectMapper(); - public Map doRouting(Map flags, Map inputDoc) { + public Object doRouting(Map flags, Map inputDoc) { log.atInfo().setMessage("parsed flags: {}").addArgument(flags).log(); final var template = "" + "{%- macro doDefault(ignored_input) -%}" + @@ -65,16 +65,16 @@ public void test() throws IOException { "data2" )); { - var resultMap = doRouting(null, docA); + var resultMap = (Map) doRouting(null, docA); Assertions.assertEquals(1, resultMap.size()); Assertions.assertEquals("_and more!", resultMap.get("matchedVal")); } { - var resultMap = doRouting(flagAOff, docA); + var resultMap = (Map) doRouting(flagAOff, docA); Assertions.assertTrue(resultMap.isEmpty()); } { - var resultMap = doRouting(flagAOff, docB); + var resultMap = (Map) doRouting(flagAOff, docB); Assertions.assertEquals("{\"label\":\"B-hive\",\"stuff\":[\"data2\",\"data1\"]}", objectMapper.writeValueAsString(new TreeMap<>(resultMap))); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java index 8687bf75d..0475456c5 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java @@ -77,7 +77,7 @@ public void testSimpleTransform() throws JsonProcessingException { var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); var transformer = new JsonJinjavaTransformerProvider().createTransformer(Map.of( "template", EXCISE_TYPE_EXPRESSION_STRING)); - var transformedDocument = transformer.transformJson(documentJson); + Object transformedDocument = transformer.transformJson(documentJson); var outputStr = emitJson(mapper, transformedDocument); final String TEST_OUTPUT_REQUEST = "{\n" diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java index 4343ea320..513ee82b6 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJoltMessageTransformer/src/main/java/org/opensearch/migrations/transform/JsonJoltTransformer.java @@ -18,7 +18,7 @@ public static JsonJoltTransformBuilder newBuilder() { } @Override - public Map transformJson(Map incomingJson) { + public Object transformJson(Object incomingJson) { return (Map) this.spec.transform(incomingJson); } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java index eae81f022..a74f1b943 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonPredicate.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.transform; -import java.util.Map; import java.util.function.Predicate; -public interface IJsonPredicate extends Predicate> { +public interface IJsonPredicate extends Predicate { } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java index a1b4c5ee3..fa35a862a 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformer.java @@ -1,11 +1,9 @@ package org.opensearch.migrations.transform; -import java.util.Map; - /** * This is a simple interface to convert a JSON object (String, Map, or Array) into another * JSON object. Any changes to datastructures, nesting, order, etc should be intentional. */ public interface IJsonTransformer { - Map transformJson(Map incomingJson); + Object transformJson(Object incomingJson); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java index 0cdf40a0d..5880890fb 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonCompositeTransformer.java @@ -1,7 +1,6 @@ package org.opensearch.migrations.transform; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicReference; public class JsonCompositeTransformer implements IJsonTransformer { @@ -12,8 +11,8 @@ public JsonCompositeTransformer(IJsonTransformer... jsonTransformers) { } @Override - public Map transformJson(Map incomingJson) { - var lastOutput = new AtomicReference<>(incomingJson); + public Object transformJson(Object incomingJson) { + AtomicReference lastOutput = new AtomicReference<>(incomingJson); jsonTransformerList.forEach(t -> lastOutput.set(t.transformJson(lastOutput.get()))); return lastOutput.get(); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java index 816281985..e9ba87dac 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformer.java @@ -1,7 +1,5 @@ package org.opensearch.migrations.transform; -import java.util.Map; - public class JsonConditionalTransformer implements IJsonTransformer { IJsonPredicate jsonPredicate; IJsonTransformer jsonTransformer; @@ -12,7 +10,7 @@ public JsonConditionalTransformer(IJsonPredicate jsonPredicate, IJsonTransformer } @Override - public Map transformJson(Map incomingJson) { + public Object transformJson(Object incomingJson) { if (jsonPredicate.test(incomingJson)) { return jsonTransformer.transformJson(incomingJson); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/NoopTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/NoopTransformerProvider.java index a8ddfc399..d0471cf57 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/NoopTransformerProvider.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/NoopTransformerProvider.java @@ -1,11 +1,9 @@ package org.opensearch.migrations.transform; -import java.util.Map; - public class NoopTransformerProvider implements IJsonTransformerProvider { private static class NoopTransformer implements IJsonTransformer { @Override - public Map transformJson(Map incomingJson) { + public Object transformJson(Object incomingJson) { return incomingJson; } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java index 1f30ae7ad..d7db86024 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java @@ -125,8 +125,9 @@ private static class UserAgentTransformer implements IJsonTransformer { private final String userAgent; @Override - public Map transformJson(Map incomingJson) { - var headers = (Map) incomingJson.get(JsonKeysForHttpMessage.HEADERS_KEY); + public Object transformJson(Object incomingJson) { + @SuppressWarnings("unchecked") + var headers = (Map) ((Map) incomingJson).get(JsonKeysForHttpMessage.HEADERS_KEY); var oldVal = headers.get(USER_AGENT); if (oldVal != null) { if (oldVal instanceof List) { @@ -146,8 +147,9 @@ private static class HostTransformer implements IJsonTransformer { private final String newHostName; @Override - public Map transformJson(Map incomingJson) { - var headers = (Map) incomingJson.get(JsonKeysForHttpMessage.HEADERS_KEY); + public Object transformJson(Object incomingJson) { + @SuppressWarnings("unchecked") + var headers = (Map) ((Map) incomingJson).get(JsonKeysForHttpMessage.HEADERS_KEY); if (headers != null) { headers.replace("host", newHostName); } else { diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java index 5a894d4cb..acb312a16 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java @@ -54,7 +54,7 @@ public void testSimpleTransform() throws JsonProcessingException { var transformer = JsonJoltTransformer.newBuilder() .addCannedOperation(JsonJoltTransformBuilder.CANNED_OPERATION.PASS_THRU) .build(); - var transformedDocument = transformer.transformJson(documentJson); + Object transformedDocument = transformer.transformJson(documentJson); var finalOutputStr = emitJson(transformedDocument); Assertions.assertEquals(TEST_DOCUMENT, finalOutputStr); @@ -65,7 +65,7 @@ public void testHttpTransform() throws IOException { var testResourceName = "parsed/post_formUrlEncoded_withFixedLength.json"; final var documentJson = parseSampleRequestFromResource(testResourceName); var transformer = JsonJoltTransformer.newBuilder().addHostSwitchOperation(DUMMY_HOSTNAME_TEST_STRING).build(); - var transformedDocument = transformer.transformJson(documentJson); + Object transformedDocument = transformer.transformJson(documentJson); String transformedJsonOutputStr = emitJson(transformedDocument); log.atInfo().setMessage("transformed json document: {}").addArgument(transformedJsonOutputStr).log(); Assertions.assertTrue(transformedJsonOutputStr.contains(DUMMY_HOSTNAME_TEST_STRING)); @@ -123,7 +123,7 @@ static String emitJson(ObjectMapper mapper, Object transformedDocument) throws J public void testSimpleTransformJMES() throws JsonProcessingException { var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); var transformer = new JsonJMESPathTransformer(new JcfRuntime(), EXCISE_TYPE_EXPRESSION_STRING); - var transformedDocument = transformer.transformJson(documentJson); + Object transformedDocument = transformer.transformJson(documentJson); var outputStr = emitJson(mapper, transformedDocument); final String TEST_OUTPUT_REQUEST = "{\n" diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java index fca6ecff8..726ba5cb9 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java @@ -30,7 +30,7 @@ public void testTwoScripts() throws Exception { aggregateScriptString ); var origDoc = JsonTransformerTest.parseStringAsJson(mapper, JsonTransformerTest.TEST_INPUT_REQUEST); - var newDoc = toNewHostTransformer.transformJson(origDoc); + Object newDoc = toNewHostTransformer.transformJson(origDoc); final String TEST_OUTPUT_REQUEST = "{\n" + " \"method\": \"PUT\",\n" diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java index a346370b4..53d4c0a25 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJoltScriptsTest.java @@ -29,9 +29,9 @@ public void testAddGzip() throws Exception { ); var origDocStr = SampleContents.loadSampleJsonRequestAsString(); var origDoc = parseAsMap(origDocStr); - var newDoc = toNewHostTransformer.transformJson(origDoc); - Assertions.assertEquals("testhostname", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); - Assertions.assertEquals("gzip", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)).get("content-encoding")); + Object newDoc = toNewHostTransformer.transformJson(origDoc); + Assertions.assertEquals("testhostname", ((Map) ((Map) newDoc).get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); + Assertions.assertEquals("gzip", ((Map) ((Map) newDoc).get(JsonKeysForHttpMessage.HEADERS_KEY)).get("content-encoding")); } @Test @@ -53,9 +53,9 @@ public void testAddGzipAndCustom() throws Exception { ); var origDocStr = SampleContents.loadSampleJsonRequestAsString(); var origDoc = parseAsMap(origDocStr); - var newDoc = toNewHostTransformer.transformJson(origDoc); - Assertions.assertEquals("testhostname", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); - var headers = (Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY); + Object newDoc = toNewHostTransformer.transformJson(origDoc); + Assertions.assertEquals("testhostname", ((Map) ((Map) newDoc).get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); + var headers = (Map) ((Map) newDoc).get(JsonKeysForHttpMessage.HEADERS_KEY); Assertions.assertEquals("gzip", headers.get("content-encoding")); Assertions.assertEquals("newValue", headers.get("newHeader")); } @@ -136,14 +136,14 @@ public void testExciseWhenPresent() throws Exception { "}"; var expectedDocStr = "{\"method\":\"PUT\",\"protocol\":\"HTTP/1.0\",\"URI\":\"/oldStyleIndex/moreStuff\",\"headers\":{\"host\":\"testhostname\"},\"payload\":{\"inlinedJsonBody\":{\"top\":{\"properties\":{\"field1\":{\"type\":\"text\"},\"field2\":{\"type\":\"keyword\"}}}}}}"; var origDoc = parseAsMap(origDocStr); - var newDoc = excisingTransformer.transformJson(origDoc); + Object newDoc = excisingTransformer.transformJson(origDoc); var newAsStr = mapper.writeValueAsString(newDoc); Assertions.assertEquals(expectedDocStr, newAsStr); - var secondPassDoc = excisingTransformer.transformJson(newDoc); + Object secondPassDoc = excisingTransformer.transformJson(newDoc); var secondPassDocAsStr = mapper.writeValueAsString(secondPassDoc); Assertions.assertEquals(expectedDocStr, secondPassDocAsStr); - Assertions.assertEquals("testhostname", ((Map) newDoc.get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); + Assertions.assertEquals("testhostname", ((Map) ((Map) newDoc).get(JsonKeysForHttpMessage.HEADERS_KEY)).get("host")); } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java index e0519de63..e50af4dad 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java @@ -23,9 +23,9 @@ public void testTransformationLoader() throws Exception { var toOldHostTransformer = new TransformationLoader().getTransformerFactoryLoaderWithNewHostName("localhost"); var origDoc = parseAsMap(SampleContents.loadSampleJsonRequestAsString()); var origDocStr = mapper.writeValueAsString(origDoc); - var outputWithNewHostname = toNewHostTransformer.transformJson(origDoc); + Object outputWithNewHostname = toNewHostTransformer.transformJson(origDoc); var docWithNewHostnameStr = mapper.writeValueAsString(outputWithNewHostname); - var outputWithOldHostname = toOldHostTransformer.transformJson(outputWithNewHostname); + Object outputWithOldHostname = toOldHostTransformer.transformJson(outputWithNewHostname); var docWithOldHostnameStr = mapper.writeValueAsString(outputWithOldHostname); Assertions.assertEquals(origDocStr, docWithOldHostnameStr); Assertions.assertNotEquals(origDocStr, docWithNewHostnameStr); @@ -39,7 +39,7 @@ public void testThatSimpleNoopTransformerLoads() throws Exception { "NoopTransformerProvider" ); var origDoc = parseAsMap(SampleContents.loadSampleJsonRequestAsString()); - var output = noopTransformer.transformJson(origDoc); + Object output = noopTransformer.transformJson(origDoc); Assertions.assertEquals(mapper.writeValueAsString(origDoc), mapper.writeValueAsString(output)); } @@ -72,13 +72,14 @@ public void testThatNoConfigMeansNoThrow() throws Exception { + "}\n"; @Test + @SuppressWarnings("unchecked") public void testUserAgentAppends() throws Exception { var userAgentTransformer = new TransformationLoader().getTransformerFactoryLoader("localhost", "tester", null); var origDoc = parseAsMap(TEST_INPUT_REQUEST); - var pass1 = userAgentTransformer.transformJson(origDoc); - var pass2 = userAgentTransformer.transformJson(pass1); - var finalUserAgentInHeaders = ((Map) pass2.get("headers")).get("user-agent"); + Object pass1 = userAgentTransformer.transformJson(origDoc); + Object pass2 = userAgentTransformer.transformJson(pass1); + var finalUserAgentInHeaders = ((Map) ((Map) pass2).get("headers")).get("user-agent"); Assertions.assertEquals("tester; tester", finalUserAgentInHeaders); } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java index 0ea6d409e..6add2579a 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java @@ -5,7 +5,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.UnaryOperator; +import java.util.function.Function; import org.opensearch.migrations.transform.jinjava.JinjavaConfig; import org.opensearch.migrations.transform.typemappings.SourceProperties; @@ -37,7 +37,7 @@ public TypeMappingsSanitizationTransformer( Optional.ofNullable(jinjavaSettings).orElse(new JinjavaConfig())); } - private static UnaryOperator> + private static Function> makeSourceWrapperFunction(SourceProperties sourceProperties, Map featureFlagsIncoming, Map> indexMappingsIncoming, diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java index 8186411fb..71141449d 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java @@ -138,7 +138,7 @@ public void testCreateIndexWithoutTypeButWithMappings() throws Exception{ " }\n" + "}"; var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); - var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); } @@ -161,7 +161,7 @@ public void testCreateIndexWithoutType() throws Exception{ " }\n" + "}"; var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); - var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java index 7318d25c9..24104b343 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java @@ -28,7 +28,7 @@ public void test() throws Exception { var indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(null, null); - var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); log.atInfo().setMessage("resultStr = {}").addArgument(() -> { try { return OBJECT_MAPPER.writeValueAsString(resultObj); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java index 2b715504a..bd2d7241d 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java @@ -111,7 +111,7 @@ public void testBulkRequest() throws Exception { "}"; - var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); log.atInfo().setMessage("resultStr = {}").addArgument(() -> { try { return OBJECT_MAPPER.writeValueAsString(resultObj); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java index d2500e656..49afa39d7 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java @@ -73,12 +73,12 @@ public void testSimpleTransform() throws JsonProcessingException { Map inputMap = OBJECT_MAPPER.readValue(TEST_INPUT_REQUEST, new TypeReference<>() { }); { - var transformedDocument = provider.createTransformer(config).transformJson(inputMap); + Object transformedDocument = provider.createTransformer(config).transformJson(inputMap); Assertions.assertEquals(JsonNormalizer.fromString(EXPECTED), JsonNormalizer.fromObject(transformedDocument)); } { - var resultFromNullConfig = provider.createTransformer(null).transformJson(inputMap); + Object resultFromNullConfig = provider.createTransformer(null).transformJson(inputMap); Assertions.assertEquals( JsonNormalizer.fromString( EXPECTED.replace( @@ -97,7 +97,7 @@ public void testMappingWithoutTypesAndLatestSourceInfoDoesNothing() throws Excep Map.of("major", (Object) 6, "minor", (Object) 10))); var transformer = new TypeMappingSanitizationTransformerProvider().createTransformer(fullTransformerConfig); - var resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); } @@ -110,7 +110,7 @@ public void testTypeMappingsWithSourcePropertiesWorks() throws Exception { "minor", (Object) 10)), "regex_index_mappings", List.of(List.of("", "", ""))); var transformer = new TypeMappingSanitizationTransformerProvider().createTransformer(fullTransformerConfig); - var resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Object resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); }