diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4c6e1c36e..8a2ae206d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -161,6 +161,7 @@ jobs: uses: ScribeMD/docker-cache@0.5.0 with: key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} + read-only: true - name: Start Docker Solution run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace env: diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java index 8ee1fb9c9..862ff1bfb 100644 --- a/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java +++ b/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java @@ -10,7 +10,8 @@ public interface IFieldCreator { ObjectMapper mapper = new ObjectMapper(); default ObjectNode createField(ElasticsearchType type) { - return mapper.createObjectNode().put("type", type.getValue()); + String typeString = type.getValue(); + return mapper.createObjectNode().put("type", typeString); } default ObjectNode fieldGeoPoint() { return createField(ElasticsearchType.GEO_POINT); } diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java index c20543040..6f2cf6d64 100644 --- a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java +++ b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java @@ -83,8 +83,9 @@ public Stream createDocs(int numDocs) { return IntStream.range(0, numDocs) .mapToObj(i -> { var random = new Random(i); + long randomTime = randomTime(currentTime, random); return mapper.createObjectNode() - .put("@timestamp", randomTime(currentTime, random)) + .put("@timestamp", randomTime) .put("clientip", randomIpAddress(random)) .put("request", randomRequest(random)) .put("status", randomStatus(random)) diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java index 038202a43..3655c3818 100644 --- a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java +++ b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java @@ -104,8 +104,9 @@ private ArrayNode randomAnswers(ObjectMapper mapper, long timeFrom, Random rando var numAnswers = random.nextInt(5) + 1; for (int i = 0; i < numAnswers; i++) { + long randomTime = randomTime(timeFrom, random); var answer = mapper.createObjectNode() - .put("date", randomTime(timeFrom, random)) + .put("date", randomTime) .put("user", randomUser(random)); answers.add(answer); } diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java index 61ddbb5fd..3aa06fb21 100644 --- a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java +++ b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java @@ -100,21 +100,29 @@ public Stream createDocs(int numDocs) { return IntStream.range(0, numDocs) .mapToObj(i -> { var random = new Random(i); + double totalAmount = randomDouble(random, 5.0, 50.0); + String pickupTime = randomTimeISOString(currentTime, random); + String dropOffTime = randomTimeISOString(currentTime, random); + double tolls = randomDouble(random, 0.0, 5.0); + double fare = randomDouble(random, 5.0, 50.0); + double extra = randomDouble(random, 0.0, 1.0); + double tripDistance = randomDouble(random, 0.5, 20.0); + double tip = randomDouble(random, 0.0, 15.0); return mapper.createObjectNode() - .put("total_amount", randomDouble(random, 5.0, 50.0)) + .put("total_amount", totalAmount) .put("improvement_surcharge", 0.3) .set("pickup_location", randomLocationInNyc(random)) - .put("pickup_datetime", randomTimeISOString(currentTime, random)) + .put("pickup_datetime", pickupTime) .put("trip_type", randomTripType(random)) - .put("dropoff_datetime", randomTimeISOString(currentTime, random)) + .put("dropoff_datetime", dropOffTime) .put("rate_code_id", "1") - .put("tolls_amount", randomDouble(random, 0.0, 5.0)) + .put("tolls_amount", tolls) .set("dropoff_location", randomLocationInNyc(random)) .put("passenger_count", random.nextInt(4) + 1) - .put("fare_amount", randomDouble(random, 5.0, 50.0)) - .put("extra", randomDouble(random, 0.0, 1.0)) - .put("trip_distance", randomDouble(random, 0.5, 20.0)) - .put("tip_amount", randomDouble(random, 0.0, 15.0)) + .put("fare_amount", fare) + .put("extra", extra) + .put("trip_distance", tripDistance) + .put("tip_amount", tip) .put("store_and_fwd_flag", randomStoreAndFwdFlag(random)) .put("payment_type", randomPaymentType(random)) .put("mta_tax", 0.5) diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle index f74fc5b62..c95c6362d 100644 --- a/DocumentsFromSnapshotMigration/build.gradle +++ b/DocumentsFromSnapshotMigration/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation project(":RFS") implementation project(":transformation") implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') - runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') implementation group: 'org.apache.logging.log4j', name: 'log4j-api' implementation group: 'org.apache.logging.log4j', name: 'log4j-core' diff --git a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java index c58fdcadf..a463461af 100644 --- a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java +++ b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java @@ -100,9 +100,9 @@ private void migrationDocumentsWithClusters( sourceClusterOperations.createDocument(indexName, "222", "{\"author\":\"Tobias Funke\"}"); - sourceClusterOperations.createDocument(indexName, "223", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1"); - sourceClusterOperations.createDocument(indexName, "224", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1"); - sourceClusterOperations.createDocument(indexName, "225", "{\"author\":\"Tobias Funke\", \"category\": \"tech\"}", "2"); + sourceClusterOperations.createDocument(indexName, "223", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1", null); + sourceClusterOperations.createDocument(indexName, "224", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1", null); + sourceClusterOperations.createDocument(indexName, "225", "{\"author\":\"Tobias Funke\", \"category\": \"tech\"}", "2", null); // === ACTION: Take a snapshot === var snapshotName = "my_snap"; diff --git a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java index 49fe04e97..df097f3bb 100644 --- a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java +++ b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java @@ -39,16 +39,13 @@ public class ParallelDocumentMigrationsTest extends SourceTestBase { public static Stream makeDocumentMigrationArgs() { var numWorkersList = List.of(1, 3, 40); var compressionEnabledList = List.of(true, false); - return SupportedClusters.targets().stream() - .flatMap( - targetImage -> numWorkersList.stream() + return numWorkersList.stream() .flatMap(numWorkers -> compressionEnabledList.stream().map(compression -> Arguments.of( numWorkers, - targetImage, + SearchClusterContainer.OS_V2_14_0, compression )) - ) - ); + ); } @ParameterizedTest @@ -70,8 +67,8 @@ public void testDocumentMigration( var osTargetContainer = new SearchClusterContainer(targetVersion); ) { CompletableFuture.allOf( - CompletableFuture.runAsync(() -> esSourceContainer.start(), executorService), - CompletableFuture.runAsync(() -> osTargetContainer.start(), executorService) + CompletableFuture.runAsync(esSourceContainer::start, executorService), + CompletableFuture.runAsync(osTargetContainer::start, executorService) ).join(); // Populate the source cluster with data diff --git a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java index 8a6e19987..b5a9b7db5 100644 --- a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java +++ b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.util.BytesRef; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; @@ -36,6 +37,7 @@ import static org.mockito.Mockito.when; @Slf4j +@Disabled("https://opensearch.atlassian.net/browse/MIGRATIONS-2254") public class PerformanceVerificationTest { @Test diff --git a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java index c36203c29..67130ced2 100644 --- a/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java +++ b/DocumentsFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java @@ -60,10 +60,10 @@ public class SourceTestBase { public static final long TOLERABLE_CLIENT_SERVER_CLOCK_DIFFERENCE_SECONDS = 3600; protected static Object[] makeParamsForBase(SearchClusterContainer.ContainerVersion baseSourceImage) { - return new Object[] { + return new Object[]{ baseSourceImage, GENERATOR_BASE_IMAGE, - new String[] { "/root/runTestBenchmarks.sh", "--endpoint", "http://" + SOURCE_SERVER_ALIAS + ":9200/" } }; + new String[]{"/root/runTestBenchmarks.sh", "--endpoint", "http://" + SOURCE_SERVER_ALIAS + ":9200/"}}; } @NotNull @@ -138,7 +138,7 @@ public static int migrateDocumentsSequentially( Version version, boolean compressionEnabled ) { - for (int runNumber = 1;; ++runNumber) { + for (int runNumber = 1; ; ++runNumber) { try { var workResult = migrateDocumentsWithOneWorker( sourceRepo, @@ -182,7 +182,8 @@ public Flux readDocuments(int startDoc) { } } - static class LeasePastError extends Error {} + static class LeasePastError extends Error { + } @SneakyThrows public static DocumentsRunner.CompletionStatus migrateDocumentsWithOneWorker( diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle index dc66ccb5a..53096741f 100644 --- a/MetadataMigration/build.gradle +++ b/MetadataMigration/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter' testImplementation group: 'org.hamcrest', name: 'hamcrest' testImplementation group: 'org.testcontainers', name: 'testcontainers' + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine' } diff --git a/MetadataMigration/docs/DESIGN.md b/MetadataMigration/docs/DESIGN.md index 412570026..c6b4010e0 100644 --- a/MetadataMigration/docs/DESIGN.md +++ b/MetadataMigration/docs/DESIGN.md @@ -142,17 +142,17 @@ Migration Candidates: Transformations: Index: - ERROR - IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "Multiple mapping types are not supported"" + ERROR - IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "No multi type resolution behavior declared, specify --multi-type-behavior to process"" Index Template: - ERROR - IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "Multiple mapping types are not supported" + ERROR - IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "No multi type resolution behavior declared, specify --multi-type-behavior to process" DEBUG - 6 transformations did not apply, add --`full` to see all results Result: 2 migration issues detected Issues: - IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "Multiple mapping types are not supported"" - IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "Multiple mapping types are not supported" + IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "No multi type resolution behavior declared, specify --multi-type-behavior to process"" + IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "No multi type resolution behavior declared, specify --multi-type-behavior to process" ``` ### Exclude incompatible rolling logs indices diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java index 01653a428..78079b75b 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java @@ -4,8 +4,11 @@ import org.opensearch.migrations.bulkload.common.http.ConnectionContext; import org.opensearch.migrations.bulkload.models.DataFilterArgs; +import org.opensearch.migrations.bulkload.transformers.MetadataTransformerParams; import org.opensearch.migrations.transform.TransformerParams; +import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval; +import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; import lombok.Getter; @@ -56,10 +59,20 @@ public class MigrateOrEvaluateArgs { public Version sourceVersion = null; @ParametersDelegate - public TransformerParams metadataTransformationParams = new MetadataTransformerParams(); + public MetadataTransformationParams metadataTransformationParams = new MetadataTransformationParams(); + + @ParametersDelegate + public TransformerParams metadataCustomTransformationParams = new MetadataCustomTransformationParams(); + + @Getter + public static class MetadataTransformationParams implements MetadataTransformerParams { + @Parameter(names = {"--multi-type-behavior"}, description = "Define behavior for resolving multi type mappings.") + public IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.NONE; + } @Getter - public static class MetadataTransformerParams implements TransformerParams { + public static class MetadataCustomTransformationParams implements TransformerParams { + public String getTransformerConfigParameterArgPrefix() { return ""; } @@ -89,4 +102,11 @@ public String getTransformerConfigParameterArgPrefix() { description = "Path to the JSON configuration file of metadata transformers.") private String transformerConfigFile; } + + static class MultiTypeResolutionBehaviorConverter implements IStringConverter { + @Override + public IndexMappingTypeRemoval.MultiTypeResolutionBehavior convert(String value) { + return IndexMappingTypeRemoval.MultiTypeResolutionBehavior.valueOf(value.toUpperCase()); + } + } } diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java index 8a2dd863f..55cc8febd 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java @@ -55,7 +55,7 @@ protected Clusters createClusters() { } protected Transformer getCustomTransformer() { - var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataTransformationParams); + var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataCustomTransformationParams); if (transformerConfig != null) { log.atInfo().setMessage("Metadata Transformations config string: {}") .addArgument(transformerConfig).log(); @@ -72,7 +72,8 @@ protected Transformer selectTransformer(Clusters clusters) { var versionTransformer = TransformFunctions.getTransformer( clusters.getSource().getVersion(), clusters.getTarget().getVersion(), - arguments.minNumberOfReplicas + arguments.minNumberOfReplicas, + arguments.metadataTransformationParams ); var customTransformer = getCustomTransformer(); var compositeTransformer = new CompositeTransformer(customTransformer, versionTransformer); diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java new file mode 100644 index 000000000..03451edf8 --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java @@ -0,0 +1,144 @@ +package org.opensearch.migrations; + +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; +import org.opensearch.migrations.bulkload.common.OpenSearchClient; +import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; +import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; +import org.opensearch.migrations.bulkload.http.ClusterOperations; +import org.opensearch.migrations.bulkload.worker.SnapshotRunner; +import org.opensearch.migrations.commands.MigrationItemResult; +import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; +import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.io.TempDir; + +/** + * Base test class providing shared setup and utility methods for migration tests. + */ +@Slf4j +abstract class BaseMigrationTest { + + @TempDir + protected Path localDirectory; + + @Getter + protected SearchClusterContainer sourceCluster; + @Getter + protected SearchClusterContainer targetCluster; + + protected ClusterOperations sourceOperations; + protected ClusterOperations targetOperations; + + /** + * Starts the source and target clusters. + */ + protected void startClusters() { + CompletableFuture.allOf( + CompletableFuture.runAsync(sourceCluster::start), + CompletableFuture.runAsync(targetCluster::start) + ).join(); + + sourceOperations = new ClusterOperations(sourceCluster.getUrl()); + targetOperations = new ClusterOperations(targetCluster.getUrl()); + } + + /** + * Sets up a snapshot repository and takes a snapshot of the source cluster. + * + * @param snapshotName Name of the snapshot. + * @return The name of the created snapshot. + */ + @SneakyThrows + protected String createSnapshot(String snapshotName) { + var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); + var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() + .host(sourceCluster.getUrl()) + .insecure(true) + .build() + .toConnectionContext()); + var snapshotCreator = new org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator( + snapshotName, + sourceClient, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + List.of(), + snapshotContext.createSnapshotCreateContext() + ); + org.opensearch.migrations.bulkload.worker.SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + sourceCluster.copySnapshotData(localDirectory.toString()); + return snapshotName; + } + + /** + * Prepares migration arguments for snapshot-based migrations. + * + * @param snapshotName Name of the snapshot. + * @return Prepared migration arguments. + */ + protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) { + var arguments = new MigrateOrEvaluateArgs(); + arguments.fileSystemRepoPath = localDirectory.toString(); + arguments.snapshotName = snapshotName; + arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + arguments.targetArgs.host = targetCluster.getUrl(); + return arguments; + } + + /** + * Executes the migration command (either migrate or evaluate). + * + * @param arguments Migration arguments. + * @param command The migration command to execute. + * @return The result of the migration. + */ + protected MigrationItemResult executeMigration(MigrateOrEvaluateArgs arguments, MetadataCommands command) { + var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); + var metadata = new MetadataMigration(); + + if (MetadataCommands.MIGRATE.equals(command)) { + return metadata.migrate(arguments).execute(metadataContext); + } else { + return metadata.evaluate(arguments).execute(metadataContext); + } + } + + /** + * Creates an OpenSearch client for the given cluster. + * + * @param cluster The cluster container. + * @return An OpenSearch client. + */ + protected OpenSearchClient createClient(SearchClusterContainer cluster) { + return new OpenSearchClient(ConnectionContextTestParams.builder() + .host(cluster.getUrl()) + .insecure(true) + .build() + .toConnectionContext()); + } + + protected SnapshotTestContext createSnapshotContext() { + return SnapshotTestContext.factory().noOtelTracking(); + } + + protected FileSystemSnapshotCreator createSnapshotCreator(String snapshotName, OpenSearchClient client, SnapshotTestContext context) { + return new FileSystemSnapshotCreator( + snapshotName, + client, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + List.of(), + context.createSnapshotCreateContext() + ); + } + + @SneakyThrows + protected void runSnapshotAndCopyData(FileSystemSnapshotCreator snapshotCreator, SearchClusterContainer cluster) { + SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + cluster.copySnapshotData(localDirectory.toString()); + } +} diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java index e3c90d22d..44fa5b34c 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java @@ -1,21 +1,12 @@ package org.opensearch.migrations; -import java.io.File; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; import org.opensearch.migrations.bulkload.SupportedClusters; -import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; -import org.opensearch.migrations.bulkload.common.OpenSearchClient; -import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; -import org.opensearch.migrations.bulkload.http.ClusterOperations; import org.opensearch.migrations.bulkload.models.DataFilterArgs; -import org.opensearch.migrations.bulkload.worker.SnapshotRunner; import org.opensearch.migrations.commands.MigrationItemResult; -import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; -import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; import org.opensearch.migrations.transform.TransformerParams; import lombok.Builder; @@ -23,7 +14,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -37,10 +27,7 @@ */ @Tag("isolatedTest") @Slf4j -class CustomTransformationTest { - - @TempDir - private File localDirectory; +class CustomTransformationTest extends BaseMigrationTest { private static Stream scenarios() { // Define scenarios with different source and target cluster versions @@ -60,23 +47,17 @@ void customTransformationMetadataMigration( final var sourceCluster = new SearchClusterContainer(sourceVersion); final var targetCluster = new SearchClusterContainer(targetVersion) ) { - performCustomTransformationTest(sourceCluster, targetCluster); + this.sourceCluster = sourceCluster; + this.targetCluster = targetCluster; + performCustomTransformationTest(); } } @SneakyThrows - private void performCustomTransformationTest( - final SearchClusterContainer sourceCluster, - final SearchClusterContainer targetCluster - ) { - // Start both source and target clusters asynchronously - CompletableFuture.allOf( - CompletableFuture.runAsync(sourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ).join(); + private void performCustomTransformationTest() { + startClusters(); - var sourceOperations = new ClusterOperations(sourceCluster.getUrl()); - var targetOperations = new ClusterOperations(targetCluster.getUrl()); + var newComponentCompatible = sourceCluster.getContainerVersion().getVersion().getMajor() >= 7; // Test data var originalIndexName = "test_index"; @@ -93,28 +74,22 @@ private void performCustomTransformationTest( var legacyTemplatePattern = "legacy_*"; sourceOperations.createLegacyTemplate(legacyTemplateName, legacyTemplatePattern); - // Create index template + // Create index template and component template if compatible var indexTemplateName = "index_template"; var indexTemplatePattern = "index*"; - - // Create component template var componentTemplateName = "component_template"; - var componentTemplateMode = "mode_value"; // Replace with actual mode if applicable - boolean newComponentCompatible = sourceCluster.getContainerVersion().getVersion().getMajor() >= 7; if (newComponentCompatible) { sourceOperations.createIndexTemplate(indexTemplateName, "dummy", indexTemplatePattern); - - var componentTemplateAdditionalParam = "additional_param"; // Replace with actual param if applicable - sourceOperations.createComponentTemplate(componentTemplateName, indexTemplateName, componentTemplateAdditionalParam, "index*"); + sourceOperations.createComponentTemplate(componentTemplateName, indexTemplateName, "additional_param", "index*"); } - // Create index that matches the templates + // Create indices that match the templates var legacyIndexName = "legacy_index"; var indexIndexName = "index_index"; sourceOperations.createIndex(legacyIndexName); sourceOperations.createIndex(indexIndexName); - // Define custom transformations for index, legacy, and component templates + // Define custom transformations String customTransformationJson = "[\n" + " {\n" + " \"JsonConditionalTransformerProvider\": [\n" + @@ -183,48 +158,23 @@ private void performCustomTransformationTest( " }\n" + "]"; - var arguments = new MigrateOrEvaluateArgs(); + var snapshotName = createSnapshot("custom_transformation_snap"); + var arguments = prepareSnapshotMigrationArgs(snapshotName); - // Use SnapshotImage as the transfer medium - var snapshotName = "custom_transformation_snap"; - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(sourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - sourceCluster.copySnapshotData(localDirectory.toString()); - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = snapshotName; - arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); - - arguments.targetArgs.host = targetCluster.getUrl(); - - // Set up data filters to include only the test index and templates + // Set up data filters var dataFilterArgs = new DataFilterArgs(); dataFilterArgs.indexAllowlist = List.of(originalIndexName, legacyIndexName, indexIndexName, transformedIndexName); dataFilterArgs.indexTemplateAllowlist = List.of(indexTemplateName, legacyTemplateName, "transformed_legacy_template", "transformed_index_template"); dataFilterArgs.componentTemplateAllowlist = List.of(componentTemplateName, "transformed_component_template"); arguments.dataFilterArgs = dataFilterArgs; - // Specify the custom transformer configuration - arguments.metadataTransformationParams = TestTransformationParams.builder() + // Set up transformation parameters + arguments.metadataCustomTransformationParams = TestCustomTransformationParams.builder() .transformerConfig(customTransformationJson) .build(); - // Execute the migration with the custom transformation - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); - - MigrationItemResult result = metadata.migrate(arguments).execute(metadataContext); + // Execute migration + MigrationItemResult result = executeMigration(arguments, MetadataCommands.MIGRATE); // Verify the migration result log.info(result.asCliOutput()); @@ -239,31 +189,26 @@ private void performCustomTransformationTest( res = targetOperations.get("/" + originalIndexName); assertThat(res.getKey(), equalTo(404)); - // Verify that the transformed legacy template exists on the target cluster + // Verify templates res = targetOperations.get("/_template/transformed_legacy_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_legacy_template")); - // Verify that the original legacy template does not exist on the target cluster - res = targetOperations.get("/_template/" + legacyTemplateName); + res = targetOperations.get("/_template/" + "legacy_template"); assertThat(res.getKey(), equalTo(404)); if (newComponentCompatible) { - // Verify that the transformed index template exists on the target cluster res = targetOperations.get("/_index_template/transformed_index_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_index_template")); - // Verify that the original index template does not exist on the target cluster res = targetOperations.get("/_index_template/" + indexTemplateName); assertThat(res.getKey(), equalTo(404)); - // Verify that the transformed component template exists on the target cluster res = targetOperations.get("/_component_template/transformed_component_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_component_template")); - // Verify that the original component template does not exist on the target cluster res = targetOperations.get("/_component_template/" + componentTemplateName); assertThat(res.getKey(), equalTo(404)); } @@ -271,7 +216,7 @@ private void performCustomTransformationTest( @Data @Builder - private static class TestTransformationParams implements TransformerParams { + private static class TestCustomTransformationParams implements TransformerParams { @Builder.Default private String transformerConfigParameterArgPrefix = ""; private String transformerConfigEncoded; diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java index 183f24acf..5cc7be88b 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java @@ -1,30 +1,22 @@ package org.opensearch.migrations; -import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.migrations.bulkload.SupportedClusters; -import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; -import org.opensearch.migrations.bulkload.common.OpenSearchClient; -import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; -import org.opensearch.migrations.bulkload.framework.SearchClusterContainer.ContainerVersion; -import org.opensearch.migrations.bulkload.http.ClusterOperations; import org.opensearch.migrations.bulkload.models.DataFilterArgs; -import org.opensearch.migrations.bulkload.worker.SnapshotRunner; import org.opensearch.migrations.commands.MigrationItemResult; import org.opensearch.migrations.metadata.CreationResult; -import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; -import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -37,50 +29,48 @@ import static org.hamcrest.Matchers.containsInAnyOrder; /** - * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster + * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster. */ @Tag("isolatedTest") @Slf4j -class EndToEndTest { - - @TempDir - private File localDirectory; +class EndToEndTest extends BaseMigrationTest { private static Stream scenarios() { return SupportedClusters.sources().stream() - .flatMap(sourceCluster -> { - // Determine applicable template types based on source version - List templateTypes = Stream.concat( - Stream.of(TemplateType.Legacy), - (sourceCluster.getVersion().getMajor() >= 7 - ? Stream.of(TemplateType.Index, TemplateType.IndexAndComponent) - : Stream.empty())) - .collect(Collectors.toList()); - - return SupportedClusters.targets().stream() - .flatMap(targetCluster -> templateTypes.stream().flatMap(templateType -> { - // Generate arguments for both HTTP and SnapshotImage transfer mediums - Stream httpArgs = Arrays.stream(MetadataCommands.values()) - .map(command -> Arguments.of(sourceCluster, targetCluster, TransferMedium.Http, command, templateType)); - - Stream snapshotArgs = Stream.of( - Arguments.of(sourceCluster, targetCluster, TransferMedium.SnapshotImage, MetadataCommands.MIGRATE, templateType) - ); - - return Stream.concat(httpArgs, snapshotArgs); - })); - }); + .flatMap(sourceCluster -> { + // Determine applicable template types based on source version + List templateTypes = Stream.concat( + Stream.of(TemplateType.Legacy), + (sourceCluster.getVersion().getMajor() >= 7 + ? Stream.of(TemplateType.Index, TemplateType.IndexAndComponent) + : Stream.empty())) + .collect(Collectors.toList()); + + return SupportedClusters.targets().stream() + .flatMap(targetCluster -> Arrays.stream(TransferMedium.values()) + .map(transferMedium -> Arguments.of( + sourceCluster, + targetCluster, + transferMedium, + templateTypes))) + .collect(Collectors.toList()).stream(); + }); } - @ParameterizedTest(name = "From version {0} to version {1}, Command {2}, Medium of transfer {3}, and Template Type {4}") + @ParameterizedTest(name = "From version {0} to version {1}, Medium {2}, Command {3}, Template Type {4}") @MethodSource(value = "scenarios") - void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium, - MetadataCommands command, TemplateType templateType) { + void metadataCommand(SearchClusterContainer.ContainerVersion sourceVersion, + SearchClusterContainer.ContainerVersion targetVersion, + TransferMedium medium, + List templateTypes) { try ( final var sourceCluster = new SearchClusterContainer(sourceVersion); final var targetCluster = new SearchClusterContainer(targetVersion) ) { - metadataCommandOnClusters(sourceCluster, targetCluster, medium, command, templateType); + this.sourceCluster = sourceCluster; + this.targetCluster = targetCluster; + metadataCommandOnClusters(medium, MetadataCommands.EVALUATE, templateTypes); + metadataCommandOnClusters(medium, MetadataCommands.MIGRATE, templateTypes); } } @@ -96,179 +86,167 @@ private enum TemplateType { } @SneakyThrows - private void metadataCommandOnClusters( - final SearchClusterContainer sourceCluster, - final SearchClusterContainer targetCluster, - final TransferMedium medium, - final MetadataCommands command, - final TemplateType templateType - ) { - // ACTION: Set up the source/target clusters - var bothClustersStarted = CompletableFuture.allOf( - CompletableFuture.runAsync(sourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ); - bothClustersStarted.join(); + private void metadataCommandOnClusters(TransferMedium medium, + MetadataCommands command, + List templateTypes) { + startClusters(); var testData = new TestData(); - var sourceClusterOperations = new ClusterOperations(sourceCluster.getUrl()); - if (templateType == TemplateType.Legacy) { - sourceClusterOperations.createLegacyTemplate(testData.indexTemplateName, "blog*"); - } else if (templateType == TemplateType.Index) { - sourceClusterOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*"); - } else if (templateType == TemplateType.IndexAndComponent) { - sourceClusterOperations.createComponentTemplate(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*"); - } - // Creates a document that uses the template - sourceClusterOperations.createDocument(testData.blogIndexName, "222", "{\"author\":\"Tobias Funke\"}"); - sourceClusterOperations.createDocument(testData.movieIndexName,"123", "{\"title\":\"This is spinal tap\"}"); - sourceClusterOperations.createDocument(testData.indexThatAlreadyExists, "doc66", "{}"); + for (TemplateType templateType : templateTypes) { + String uniqueSuffix = templateType.name().toLowerCase(); + String templateName = testData.indexTemplateName + "_" + uniqueSuffix; + String indexPattern = "blog_" + uniqueSuffix + "_*"; + String fieldName = "author_" + uniqueSuffix; + + if (templateType == TemplateType.Legacy) { + sourceOperations.createLegacyTemplate(templateName, indexPattern); + testData.aliasNames.add("alias_legacy"); + } else if (templateType == TemplateType.Index) { + sourceOperations.createIndexTemplate(templateName, fieldName, indexPattern); + testData.aliasNames.add("alias_index"); + } else if (templateType == TemplateType.IndexAndComponent) { + String componentTemplateName = testData.compoTemplateName + "_" + uniqueSuffix; + sourceOperations.createComponentTemplate(componentTemplateName, templateName, fieldName, indexPattern); + testData.aliasNames.add("alias_component"); + testData.componentTemplateNames.add(componentTemplateName); + } + testData.templateNames.add(templateName); + + // Create documents that use the templates + String blogIndexName = "blog_" + uniqueSuffix + "_2023"; + sourceOperations.createDocument(blogIndexName, "222", "{\"" + fieldName + "\":\"Tobias Funke\"}"); + testData.blogIndexNames.add(blogIndexName); + } - sourceClusterOperations.createAlias(testData.aliasName, "movies*"); + sourceOperations.createDocument(testData.movieIndexName, "123", "{\"title\":\"This is Spinal Tap\"}"); + sourceOperations.createDocument(testData.indexThatAlreadyExists, "doc66", "{}"); - var aliasName = "movies-alias"; - sourceClusterOperations.createAlias(aliasName, "movies*"); + sourceOperations.createAlias(testData.aliasName, "movies*"); + testData.aliasNames.add(testData.aliasName); var arguments = new MigrateOrEvaluateArgs(); switch (medium) { case SnapshotImage: - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var snapshotName = "my_snap"; - log.info("Source cluster {}", sourceCluster.getUrl()); - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(sourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - sourceCluster.copySnapshotData(localDirectory.toString()); - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = snapshotName; - arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + var snapshotName = createSnapshot("my_snap_" + command.name().toLowerCase()); + arguments = prepareSnapshotMigrationArgs(snapshotName); break; - + case Http: arguments.sourceArgs.host = sourceCluster.getUrl(); + arguments.targetArgs.host = targetCluster.getUrl(); break; } - arguments.targetArgs.host = targetCluster.getUrl(); - + // Set up data filters var dataFilterArgs = new DataFilterArgs(); - dataFilterArgs.indexAllowlist = List.of(); - dataFilterArgs.componentTemplateAllowlist = List.of(testData.compoTemplateName); - dataFilterArgs.indexTemplateAllowlist = List.of(testData.indexTemplateName); + dataFilterArgs.indexAllowlist = Stream.concat(testData.blogIndexNames.stream(), + Stream.of(testData.movieIndexName, testData.indexThatAlreadyExists)).collect(Collectors.toList()); + dataFilterArgs.componentTemplateAllowlist = testData.componentTemplateNames; + dataFilterArgs.indexTemplateAllowlist = testData.templateNames; arguments.dataFilterArgs = dataFilterArgs; - var targetClusterOperations = new ClusterOperations(targetCluster.getUrl()); - targetClusterOperations.createDocument(testData.indexThatAlreadyExists, "doc77", "{}"); - - // ACTION: Migrate the templates - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); - - MigrationItemResult result; - if (MetadataCommands.MIGRATE.equals(command)) { - result = metadata.migrate(arguments).execute(metadataContext); - } else { - result = metadata.evaluate(arguments).execute(metadataContext); - } + targetOperations.createDocument(testData.indexThatAlreadyExists, "doc77", "{}"); + + // Execute migration + MigrationItemResult result = executeMigration(arguments, command); - verifyCommandResults(result, templateType, testData); + verifyCommandResults(result, templateTypes, testData); - verifyTargetCluster(targetClusterOperations, command, templateType, testData); + verifyTargetCluster(command, templateTypes, testData); } private static class TestData { final String compoTemplateName = "simple_component_template"; final String indexTemplateName = "simple_index_template"; final String aliasInTemplate = "alias1"; - final String blogIndexName = "blog_2023"; final String movieIndexName = "movies_2023"; final String aliasName = "movies-alias"; final String indexThatAlreadyExists = "already-exists"; + final List blogIndexNames = new ArrayList<>(); + final List templateNames = new ArrayList<>(); + final List componentTemplateNames = new ArrayList<>(); + final List aliasNames = new ArrayList<>(); } - private void verifyCommandResults( - MigrationItemResult result, - TemplateType templateType, - TestData testData) { + private void verifyCommandResults(MigrationItemResult result, + List templateTypes, + TestData testData) { log.info(result.asCliOutput()); assertThat(result.getExitCode(), equalTo(0)); var migratedItems = result.getItems(); - assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), containsInAnyOrder(testData.indexTemplateName)); - assertThat(getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of())); - assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName)); - assertThat(getNames(getFailedResultsByType(migratedItems.getIndexes(), CreationResult.CreationFailureType.ALREADY_EXISTS)), containsInAnyOrder(testData.indexThatAlreadyExists)); - assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); - + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), + containsInAnyOrder(testData.templateNames.toArray(new String[0]))); + assertThat(getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), + containsInAnyOrder(testData.componentTemplateNames.toArray(new String[0]))); + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), + containsInAnyOrder(Stream.concat(testData.blogIndexNames.stream(), + Stream.of(testData.movieIndexName)).toArray())); + assertThat(getNames(getFailedResultsByType(migratedItems.getIndexes(), + CreationResult.CreationFailureType.ALREADY_EXISTS)), + containsInAnyOrder(testData.indexThatAlreadyExists)); + assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), + containsInAnyOrder(testData.aliasNames.toArray(new String[0]))); } private List getSuccessfulResults(List results) { return results.stream() - .filter(CreationResult::wasSuccessful) - .collect(Collectors.toList()); + .filter(CreationResult::wasSuccessful) + .collect(Collectors.toList()); } private List getFailedResultsByType(List results, CreationResult.CreationFailureType failureType) { return results.stream() - .filter(r -> failureType.equals(r.getFailureType())) - .collect(Collectors.toList()); + .filter(r -> failureType.equals(r.getFailureType())) + .collect(Collectors.toList()); } private List getNames(List items) { - return items.stream().map(r -> r.getName()).collect(Collectors.toList()); + return items.stream().map(CreationResult::getName).collect(Collectors.toList()); } - private void verifyTargetCluster( - ClusterOperations targetClusterOperations, - MetadataCommands command, - TemplateType templateType, - TestData testData - ) { + private void verifyTargetCluster(MetadataCommands command, + List templateTypes, + TestData testData) { var expectUpdatesOnTarget = MetadataCommands.MIGRATE.equals(command); - // If the command was migrate, the target cluster should have the items, if not they + // If the command was migrate, the target cluster should have the items, if not they shouldn't var verifyResponseCode = expectUpdatesOnTarget ? equalTo(200) : equalTo(404); - // Check that the index was migrated - var res = targetClusterOperations.get("/" + testData.blogIndexName); - assertThat(res.getValue(), res.getKey(), verifyResponseCode); + // Check that the indices were migrated + for (String blogIndexName : testData.blogIndexNames) { + var res = targetOperations.get("/" + blogIndexName); + assertThat(res.getValue(), res.getKey(), verifyResponseCode); + } - res = targetClusterOperations.get("/" + testData.movieIndexName); + var res = targetOperations.get("/" + testData.movieIndexName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); - res = targetClusterOperations.get("/" + testData.aliasName); + res = targetOperations.get("/" + testData.aliasName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); if (expectUpdatesOnTarget) { assertThat(res.getValue(), containsString(testData.movieIndexName)); } - res = targetClusterOperations.get("/_aliases"); + res = targetOperations.get("/_aliases"); assertThat(res.getValue(), res.getKey(), equalTo(200)); - var verifyAliasWasListed = allOf(containsString(testData.aliasInTemplate), containsString(testData.aliasName)); + @SuppressWarnings("unchecked") + var verifyAliasWasListed = allOf( + testData.aliasNames.stream() + .map(Matchers::containsString) + .toArray(Matcher[]::new) + ); assertThat(res.getValue(), expectUpdatesOnTarget ? verifyAliasWasListed : not(verifyAliasWasListed)); // Check that the templates were migrated - if (templateType.equals(TemplateType.Legacy)) { - res = targetClusterOperations.get("/_template/" + testData.indexTemplateName); - assertThat(res.getValue(), res.getKey(), verifyResponseCode); - } else if(templateType.equals(TemplateType.Index) || templateType.equals(TemplateType.IndexAndComponent)) { - res = targetClusterOperations.get("/_index_template/" + testData.indexTemplateName); - assertThat(res.getValue(), res.getKey(), verifyResponseCode); - if (templateType.equals(TemplateType.IndexAndComponent)) { - var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]"); - assertThat(res.getValue(), expectUpdatesOnTarget ? verifyBodyHasComponentTemplate : not(verifyBodyHasComponentTemplate)); + for (String templateName : testData.templateNames) { + if (templateName.contains("legacy")) { + res = targetOperations.get("/_template/" + templateName); + } else { + res = targetOperations.get("/_index_template/" + templateName); } + assertThat(res.getValue(), res.getKey(), verifyResponseCode); } } } diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java new file mode 100644 index 000000000..6473be7fc --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java @@ -0,0 +1,159 @@ +package org.opensearch.migrations; + +import java.util.List; + +import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; +import org.opensearch.migrations.bulkload.http.ClusterOperations; +import org.opensearch.migrations.bulkload.models.DataFilterArgs; +import org.opensearch.migrations.commands.MigrationItemResult; +import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.migrations.bulkload.framework.SearchClusterContainer.ES_V6_8_23; + +/** + * Test class to verify custom transformations during metadata migrations. + */ +@Tag("isolatedTest") +@Slf4j +class MultiTypeMappingTransformationTest extends BaseMigrationTest { + + @SneakyThrows + @Test + public void multiTypeTransformationTest_union() { + var es5Repo = "es5"; + var snapshotName = "es5-created-index"; + var originalIndexName = "test_index"; + + try ( + final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16) + ) { + indexCreatedCluster.start(); + + var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl()); + + // Create index and add documents on the source cluster + indexCreatedOperations.createIndex(originalIndexName); + indexCreatedOperations.createDocument(originalIndexName, "1", "{\"field1\":\"My Name\"}", null, "type1"); + indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"string\", \"field2\":123}", null, "type2"); + indexCreatedOperations.createDocument(originalIndexName, "3", "{\"field3\":1.1}", null, "type3"); + + indexCreatedOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); + indexCreatedOperations.takeSnapshot(es5Repo, snapshotName, originalIndexName); + indexCreatedCluster.copySnapshotData(localDirectory.toString()); + } + + try ( + final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23); + final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0) + ) { + this.sourceCluster = upgradedSourceCluster; + this.targetCluster = targetCluster; + + startClusters(); + + upgradedSourceCluster.putSnapshotData(localDirectory.toString()); + + var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl()); + + // Register snapshot repository and restore snapshot in ES 6 cluster + upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); + upgradedSourceOperations.restoreSnapshot(es5Repo, snapshotName); + + // Verify index exists on upgraded cluster + var checkIndexUpgraded = upgradedSourceOperations.get("/" + originalIndexName); + assertThat(checkIndexUpgraded.getKey(), equalTo(200)); + assertThat(checkIndexUpgraded.getValue(), containsString(originalIndexName)); + + var updatedSnapshotName = createSnapshot("union-snapshot"); + var arguments = prepareSnapshotMigrationArgs(updatedSnapshotName); + + // Set up data filters + var dataFilterArgs = new DataFilterArgs(); + dataFilterArgs.indexAllowlist = List.of(originalIndexName); + arguments.dataFilterArgs = dataFilterArgs; + + // Use union method for multi-type mappings + arguments.metadataTransformationParams.multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION; + + // Execute migration + MigrationItemResult result = executeMigration(arguments, MetadataCommands.MIGRATE); + + // Verify the migration result + log.info(result.asCliOutput()); + assertThat(result.getExitCode(), equalTo(0)); + + assertThat(result.getItems().getIndexes().size(), equalTo(1)); + var actualCreationResult = result.getItems().getIndexes().get(0); + assertThat(actualCreationResult.getException(), equalTo(null)); + assertThat(actualCreationResult.getName(), equalTo(originalIndexName)); + assertThat(actualCreationResult.getFailureType(), equalTo(null)); + + // Verify that the transformed index exists on the target cluster + var res = targetOperations.get("/" + originalIndexName); + assertThat(res.getKey(), equalTo(200)); + assertThat(res.getValue(), containsString(originalIndexName)); + + // Fetch the index mapping from the target cluster + var mappingResponse = targetOperations.get("/" + originalIndexName + "/_mapping"); + assertThat(mappingResponse.getKey(), equalTo(200)); + + // Parse the mapping response + var mapper = new ObjectMapper(); + var mappingJson = mapper.readTree(mappingResponse.getValue()); + + // Navigate to the properties of the index mapping + JsonNode properties = mappingJson.path(originalIndexName).path("mappings").path("properties"); + + // Assert that both field1 and field2 are present + assertThat(properties.get("field1").get("type").asText(), equalTo("text")); + assertThat(properties.get("field2").get("type").asText(), equalTo("long")); + assertThat(properties.get("field3").get("type").asText(), equalTo("float")); + } + } + + @Test + public void es5_doesNotAllow_multiTypeConflicts() { + try ( + final var es5 = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16) + ) { + es5.start(); + + var clusterOperations = new ClusterOperations(es5.getUrl()); + + var originalIndexName = "test_index"; + String body = "{" + + " \"settings\": {" + + " \"index\": {" + + " \"number_of_shards\": 5," + + " \"number_of_replicas\": 0" + + " }" + + " }," + + " \"mappings\": {" + + " \"type1\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"float\" }" + + " }" + + " }," + + " \"type2\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"long\" }" + + " }" + + " }" + + " }" + + "}"; + var res = clusterOperations.put("/" + originalIndexName, body); + assertThat(res.getKey(), equalTo(400)); + assertThat(res.getValue(), containsString("mapper [field1] cannot be changed from type [long] to [float]")); + } + } +} diff --git a/RFS/build.gradle b/RFS/build.gradle index aa24dd409..e40d7f6ae 100644 --- a/RFS/build.gradle +++ b/RFS/build.gradle @@ -21,7 +21,9 @@ dependencies { implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') + runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJinjavaTransformerProvider') runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') + runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') implementation group: 'org.jcommander', name: 'jcommander' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' @@ -62,7 +64,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter' testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') - testRuntimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + testRuntimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine' diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java index e0fdfede1..63676f7f4 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/BulkDocSection.java @@ -103,7 +103,7 @@ public String asBulkIndexString() { @SuppressWarnings("unchecked") public Map toMap() { - return (Map) OBJECT_MAPPER.convertValue(bulkIndex, Map.class); + return OBJECT_MAPPER.convertValue(bulkIndex, Map.class); } /** diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java index a019b871c..8b8898c36 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/ComponentTemplate.java @@ -6,8 +6,8 @@ @NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson public class ComponentTemplate extends MigrationItem { - public static final String TYPE = "component_template"; + public static final String TYPE_NAME = "component_template"; public ComponentTemplate(final String name, final ObjectNode body) { - super(TYPE, name, body); + super(TYPE_NAME, name, body); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java index cef345ad0..1620c2a81 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/Index.java @@ -6,8 +6,8 @@ @NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson public class Index extends MigrationItem { - public final static String TYPE = "index"; + public static final String TYPE_NAME = "index"; public Index(String name, ObjectNode body) { - super(TYPE, name, body); + super(TYPE_NAME, name, body); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java index 792e9d53e..9327b2225 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/IndexTemplate.java @@ -6,8 +6,8 @@ @NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson public class IndexTemplate extends MigrationItem { - public static final String TYPE = "index_template"; + public static final String TYPE_NAME = "index_template"; public IndexTemplate(final String name, final ObjectNode body) { - super(TYPE, name, body); + super(TYPE_NAME, name, body); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java index 7edfa763a..0a392c2a1 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/LegacyTemplate.java @@ -6,8 +6,8 @@ @NoArgsConstructor(force = true, access = AccessLevel.PRIVATE) // For Jackson public class LegacyTemplate extends MigrationItem { - public final static String TYPE = "template"; + public static final String TYPE_NAME = "template"; public LegacyTemplate(final String name, final ObjectNode body) { - super(TYPE, name, body); + super(TYPE_NAME, name, body); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java index dccd2cea1..e4a77aa16 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/models/MigrationItem.java @@ -9,17 +9,17 @@ @NoArgsConstructor(force = true, access = AccessLevel.PROTECTED) // For Jackson @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ - @JsonSubTypes.Type(value = Index.class, name = Index.TYPE), - @JsonSubTypes.Type(value = LegacyTemplate.class, name = LegacyTemplate.TYPE), - @JsonSubTypes.Type(value = IndexTemplate.class, name = IndexTemplate.TYPE), - @JsonSubTypes.Type(value = ComponentTemplate.class, name = ComponentTemplate.TYPE) + @JsonSubTypes.Type(value = Index.class, name = Index.TYPE_NAME), + @JsonSubTypes.Type(value = LegacyTemplate.class, name = LegacyTemplate.TYPE_NAME), + @JsonSubTypes.Type(value = IndexTemplate.class, name = IndexTemplate.TYPE_NAME), + @JsonSubTypes.Type(value = ComponentTemplate.class, name = ComponentTemplate.TYPE_NAME) }) public abstract class MigrationItem { public final String type; public final String name; public final ObjectNode body; - public MigrationItem(final String type, final String name, final ObjectNode body) { + protected MigrationItem(final String type, final String name, final ObjectNode body) { this.type = type; this.name = name; this.body = body; diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/IndexTransformationException.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/IndexTransformationException.java index aa2b35aa2..e31c2a894 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/IndexTransformationException.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/IndexTransformationException.java @@ -4,6 +4,6 @@ public class IndexTransformationException extends RfsException { public IndexTransformationException(String indexName, Throwable cause) { - super("Transformation for index index '" + indexName + "' failed.", cause); + super("Transformation for index index '" + indexName + "' failed due to " + cause.getMessage(), cause); } } diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/MetadataTransformerParams.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/MetadataTransformerParams.java new file mode 100644 index 000000000..83d179baa --- /dev/null +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/MetadataTransformerParams.java @@ -0,0 +1,7 @@ +package org.opensearch.migrations.bulkload.transformers; + +import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval; + +public interface MetadataTransformerParams { + IndexMappingTypeRemoval.MultiTypeResolutionBehavior getMultiTypeResolutionBehavior(); +} diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformFunctions.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformFunctions.java index 147a0e1d4..81e0986c8 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformFunctions.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformFunctions.java @@ -20,11 +20,12 @@ private TransformFunctions() {} public static Transformer getTransformer( Version sourceVersion, Version targetVersion, - int dimensionality + int dimensionality, + MetadataTransformerParams metadataTransformerParams ) { if (VersionMatchers.isOS_2_X.or(VersionMatchers.isOS_1_X).test(targetVersion)) { if (VersionMatchers.isES_6_X.test(sourceVersion)) { - return new Transformer_ES_6_8_to_OS_2_11(dimensionality); + return new Transformer_ES_6_8_to_OS_2_11(dimensionality, metadataTransformerParams); } if (VersionMatchers.equalOrGreaterThanES_7_10.test(sourceVersion)) { return new Transformer_ES_7_10_OS_2_11(dimensionality); diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java index eaf08cada..841fe356d 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/TransformerToIJsonTransformerAdapter.java @@ -28,7 +28,7 @@ @Slf4j public class TransformerToIJsonTransformerAdapter implements Transformer { public static final String OUTPUT_TRANSFORMATION_JSON_LOGGER = "OutputTransformationJsonLogger"; - private final static ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper(); private final IJsonTransformer transformer; private final Logger transformerLogger; @@ -65,7 +65,7 @@ private Map toTransformationMap(Map before, Map< @SuppressWarnings("unchecked") private static Map objectNodeToMap(Object node) { - return (Map) MAPPER.convertValue(node, Map.class); + return MAPPER.convertValue(node, Map.class); } @SneakyThrows @@ -124,22 +124,19 @@ public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { ) .map(this::transformMigrationItem).collect(Collectors.toList()); - var transformedLegacy = transformedTemplates.stream().filter( - item -> item instanceof LegacyTemplate - ) - .map(item -> (LegacyTemplate) item) + var transformedLegacy = transformedTemplates.stream() + .filter(LegacyTemplate.class::isInstance) + .map(LegacyTemplate.class::cast) .collect(Collectors.toList()); - var transformedIndex = transformedTemplates.stream().filter( - item -> item instanceof IndexTemplate - ) - .map(item -> (IndexTemplate) item) + var transformedIndex = transformedTemplates.stream() + .filter(IndexTemplate.class::isInstance) + .map(IndexTemplate.class::cast) .collect(Collectors.toList()); - var transformedComponent = transformedTemplates.stream().filter( - item -> item instanceof ComponentTemplate - ) - .map(item -> (ComponentTemplate) item) + var transformedComponent = transformedTemplates.stream() + .filter(ComponentTemplate.class::isInstance) + .map(ComponentTemplate.class::cast) .collect(Collectors.toList()); assert transformedLegacy.size() + transformedIndex.size() + transformedComponent.size() == transformedTemplates.size(); diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/Transformer_ES_6_8_to_OS_2_11.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/Transformer_ES_6_8_to_OS_2_11.java index 452dac09e..014172cc6 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/Transformer_ES_6_8_to_OS_2_11.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/transformers/Transformer_ES_6_8_to_OS_2_11.java @@ -18,13 +18,19 @@ public class Transformer_ES_6_8_to_OS_2_11 implements Transformer { private static final ObjectMapper mapper = new ObjectMapper(); - private final List> indexTransformations = List.of(new IndexMappingTypeRemoval()); - private final List> indexTemplateTransformations = List.of(new IndexMappingTypeRemoval()); + private final List> indexTransformations; + private final List> indexTemplateTransformations; private final int awarenessAttributeDimensionality; - public Transformer_ES_6_8_to_OS_2_11(int awarenessAttributeDimensionality) { + public Transformer_ES_6_8_to_OS_2_11(int awarenessAttributeDimensionality, MetadataTransformerParams params) { this.awarenessAttributeDimensionality = awarenessAttributeDimensionality; + this.indexTransformations = List.of(new IndexMappingTypeRemoval( + params.getMultiTypeResolutionBehavior() + )); + this.indexTemplateTransformations = List.of(new IndexMappingTypeRemoval( + params.getMultiTypeResolutionBehavior() + )); } @Override @@ -37,7 +43,18 @@ public GlobalMetadata transformGlobalMetadata(GlobalMetadata globalData) { var templates = mapper.createObjectNode(); templatesRoot.fields().forEachRemaining(template -> { var templateCopy = (ObjectNode) template.getValue().deepCopy(); - var indexTemplate = (Index) () -> templateCopy; + var indexTemplate = new Index() { + @Override + public String getName() { + return template.getKey(); + } + + @Override + public ObjectNode getRawJson() { + return templateCopy; + } + }; + try { transformIndex(indexTemplate, IndexType.TEMPLATE); templates.set(template.getKey(), indexTemplate.getRawJson()); diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11.java index 2bd08a023..bdb3efc39 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11.java @@ -118,12 +118,12 @@ private List createTemplates( return List.of(); } - var templatesToCreate = getAllTemplates(templates, templateType); + var templatesToCreate = getAllTemplates(templates); return processTemplateCreation(templatesToCreate, templateType, templateAllowlist, mode, context); } - Map getAllTemplates(ObjectNode templates, TemplateTypes templateType) { + Map getAllTemplates(ObjectNode templates) { var templatesToCreate = new HashMap(); templates.fieldNames().forEachRemaining(templateName -> { @@ -143,7 +143,7 @@ private List processTemplateCreation( ) { var skipCreation = FilterScheme.filterByAllowList(templateAllowList).negate(); - return templatesToCreate.entrySet().stream().map((kvp) -> { + return templatesToCreate.entrySet().stream().map(kvp -> { var templateName = kvp.getKey(); var templateBody = kvp.getValue(); var creationResult = CreationResult.builder().name(templateName); diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java index 832da1243..8dd42d968 100644 --- a/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java +++ b/RFS/src/main/java/org/opensearch/migrations/bulkload/worker/IndexRunner.java @@ -65,10 +65,10 @@ private CreationResult createIndex(String indexName, MigrationMode mode, ICreate try { indexMetadata = transformer.transformIndexMetadata(indexMetadata); return indexCreator.create(indexMetadata, mode, context); - } catch (Throwable t) { + } catch (Exception e) { return CreationResult.builder() .name(indexName) - .exception(new IndexTransformationException(indexName, t)) + .exception(new IndexTransformationException(indexName, e)) .failureType(CreationFailureType.UNABLE_TO_TRANSFORM_FAILURE) .build(); } diff --git a/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java b/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java index add9f2be5..d9721a880 100644 --- a/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java @@ -53,7 +53,7 @@ public void setUp() throws Exception { // Configure operations and rfs implementation operations = new ClusterOperations(cluster.getUrl()); - operations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR); + operations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, "test-repo"); srfs = new SimpleRestoreFromSnapshot_ES_7_10(); } @@ -72,7 +72,8 @@ public void SingleSnapshot_SingleDocument() throws Exception { operations.createDocument(indexName, document1Id, document1Body); final var snapshotName = "snapshot-1"; - operations.takeSnapshot(snapshotName, indexName); + final var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); @@ -110,7 +111,8 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio operations.createDocument(indexName, document1Id, document1Body); operations.deleteDocument(indexName, document1Id); final var snapshotName = "snapshot-delete-item"; - operations.takeSnapshot(snapshotName, indexName); + var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); @@ -145,7 +147,8 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception operations.createDocument(indexName, document1Id, document1BodyUpdated); final var snapshotName = "snapshot-delete-item"; - operations.takeSnapshot(snapshotName, indexName); + final var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); diff --git a/RFS/src/test/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11Test.java b/RFS/src/test/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11Test.java index 93eab276b..1a606bde9 100644 --- a/RFS/src/test/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11Test.java +++ b/RFS/src/test/java/org/opensearch/migrations/bulkload/version_os_2_11/GlobalMetadataCreator_OS_2_11Test.java @@ -7,7 +7,6 @@ import org.opensearch.migrations.MigrationMode; import org.opensearch.migrations.bulkload.common.OpenSearchClient; import org.opensearch.migrations.bulkload.models.GlobalMetadata; -import org.opensearch.migrations.bulkload.version_os_2_11.GlobalMetadataCreator_OS_2_11.TemplateTypes; import org.opensearch.migrations.metadata.CreationResult; import org.opensearch.migrations.metadata.CreationResult.CreationFailureType; import org.opensearch.migrations.metadata.tracing.IMetadataMigrationContexts.IClusterMetadataContext; @@ -22,7 +21,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -46,15 +44,18 @@ void testCreate() { doReturn(filledOptional).when(client).createIndexTemplate(any(), any(), any()); doReturn(filledOptional).when(client).createLegacyTemplate(any(), any(), any()); - var creator = spy(new GlobalMetadataCreator_OS_2_11(client, List.of("lit1"), List.of(), null)); - doReturn(Map.of("lit1", obj, "lit2", obj, ".lits", obj)).when(creator).getAllTemplates(any(), eq(TemplateTypes.LEGACY_INDEX_TEMPLATE)); - doReturn(Map.of("it1", obj, ".its", obj)).when(creator).getAllTemplates(any(), eq(TemplateTypes.INDEX_TEMPLATE)); - doReturn(Map.of("ct1", obj, ".cts", obj)).when(creator).getAllTemplates(any(), eq(TemplateTypes.COMPONENT_TEMPLATE)); - var globalMetadata = mock(GlobalMetadata.class); - doReturn(obj).when(globalMetadata).getComponentTemplates(); - doReturn(obj).when(globalMetadata).getIndexTemplates(); - doReturn(obj).when(globalMetadata).getTemplates(); + var componentTemplates = mapper.createObjectNode().put("type", "component"); + var indexTemplates = mapper.createObjectNode().put("type", "index"); + var legacyTemplates = mapper.createObjectNode().put("type", "legacy"); + doReturn(componentTemplates).when(globalMetadata).getComponentTemplates(); + doReturn(indexTemplates).when(globalMetadata).getIndexTemplates(); + doReturn(legacyTemplates).when(globalMetadata).getTemplates(); + + var creator = spy(new GlobalMetadataCreator_OS_2_11(client, List.of("lit1"), List.of(), null)); + doReturn(Map.of("lit1", obj, "lit2", obj, ".lits", obj)).when(creator).getAllTemplates(legacyTemplates); + doReturn(Map.of("it1", obj, ".its", obj)).when(creator).getAllTemplates(indexTemplates); + doReturn(Map.of("ct1", obj, ".cts", obj)).when(creator).getAllTemplates(componentTemplates); var results = creator.create(globalMetadata, MigrationMode.PERFORM, context); assertThat(results.fatalIssueCount(), equalTo(0L)); diff --git a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java index cbf485584..787232ad9 100644 --- a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java +++ b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java @@ -14,6 +14,7 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; /** * Containerized version of Elasticsearch cluster @@ -21,19 +22,25 @@ @Slf4j public class SearchClusterContainer extends GenericContainer { public static final String CLUSTER_SNAPSHOT_DIR = "/tmp/snapshots"; - public static final ContainerVersion ES_V7_10_2 = new ElasticsearchVersion( - "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2", - Version.fromString("ES 7.10.2") - ); public static final ContainerVersion ES_V7_17 = new ElasticsearchVersion( "docker.elastic.co/elasticsearch/elasticsearch:7.17.22", Version.fromString("ES 7.17.22") ); - public static final ContainerVersion ES_V6_8_23 = new ElasticsearchVersion( + public static final ContainerVersion ES_V7_10_2 = new ElasticsearchOssVersion( + "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2", + Version.fromString("ES 7.10.2") + ); + public static final ContainerVersion ES_V6_8_23 = new ElasticsearchOssVersion( "docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23", Version.fromString("ES 6.8.23") ); + public static final ContainerVersion ES_V5_6_16 = new ElasticsearchVersion( + "docker.elastic.co/elasticsearch/elasticsearch:5.6.16", + Version.fromString("ES 5.6.16") + ); + + public static final ContainerVersion OS_V1_3_16 = new OpenSearchVersion( "opensearchproject/opensearch:1.3.16", Version.fromString("OS 1.3.16") @@ -44,12 +51,18 @@ public class SearchClusterContainer extends GenericContainer().putAll(BASE.getEnvVariables()) + .put("xpack.security.enabled", "false") + .build()), + ELASTICSEARCH_OSS( + new ImmutableMap.Builder().putAll(BASE.getEnvVariables()) + .build()), OPENSEARCH( - new ImmutableMap.Builder().putAll(ELASTICSEARCH.getEnvVariables()) + new ImmutableMap.Builder().putAll(BASE.getEnvVariables()) .put("plugins.security.disabled", "true") .put("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "SecurityIsDisabled123$%^") .build() @@ -100,6 +113,16 @@ public void copySnapshotData(final String directory) { } } + public void putSnapshotData(final String directory) { + try { + this.copyFileToContainer(MountableFile.forHostPath(directory), CLUSTER_SNAPSHOT_DIR); + this.execInContainer("chown", "-R", "elasticsearch:elasticsearch", CLUSTER_SNAPSHOT_DIR); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + public void start() { log.info("Starting container version:" + containerVersion.version); super.start(); @@ -135,6 +158,12 @@ public ContainerVersion(final String imageName, final Version version, INITIALIZ } + public static class ElasticsearchOssVersion extends ContainerVersion { + public ElasticsearchOssVersion(String imageName, Version version) { + super(imageName, version, INITIALIZATION_FLAVOR.ELASTICSEARCH_OSS); + } + } + public static class ElasticsearchVersion extends ContainerVersion { public ElasticsearchVersion(String imageName, Version version) { super(imageName, version, INITIALIZATION_FLAVOR.ELASTICSEARCH); diff --git a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java index edc297410..ca87c7ab7 100644 --- a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java +++ b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Optional; import lombok.SneakyThrows; import org.apache.hc.client5.http.classic.methods.HttpDelete; @@ -31,7 +32,7 @@ public ClusterOperations(final String clusterUrl) { httpClient = HttpClients.createDefault(); } - public void createSnapshotRepository(final String repoPath) throws IOException { + public void createSnapshotRepository(final String repoPath, final String repoName) throws IOException { // Create snapshot repository final var repositoryJson = "{\n" + " \"type\": \"fs\",\n" @@ -43,7 +44,7 @@ public void createSnapshotRepository(final String repoPath) throws IOException { + " }\n" + "}"; - final var createRepoRequest = new HttpPut(clusterUrl + "/_snapshot/test-repo"); + final var createRepoRequest = new HttpPut(clusterUrl + "/_snapshot/" + repoName); createRepoRequest.setEntity(new StringEntity(repositoryJson)); createRepoRequest.setHeader("Content-Type", "application/json"); @@ -53,19 +54,23 @@ public void createSnapshotRepository(final String repoPath) throws IOException { } @SneakyThrows - public void createDocument(final String index, final String docId, final String body) { - var indexDocumentRequest = new HttpPut(clusterUrl + "/" + index + "/_doc/" + docId); - indexDocumentRequest.setEntity(new StringEntity(body)); - indexDocumentRequest.setHeader("Content-Type", "application/json"); + public void restoreSnapshot(final String repository, final String snapshotName) { + var restoreRequest = new HttpPost(clusterUrl + "/_snapshot/" + repository + "/" + snapshotName + "/_restore"+ "?wait_for_completion=true"); + restoreRequest.setHeader("Content-Type", "application/json"); + restoreRequest.setEntity(new StringEntity("{}")); - try (var response = httpClient.execute(indexDocumentRequest)) { - assertThat(response.getCode(), anyOf(equalTo(201), equalTo(200))); + try (var response = httpClient.execute(restoreRequest)) { + assertThat(response.getCode(), anyOf(equalTo(200), equalTo(202))); } } + public void createDocument(final String index, final String docId, final String body) { + createDocument(index, docId, body, null, "_doc"); + } + @SneakyThrows - public void createDocument(final String index, final String docId, final String body, String routing) { - var indexDocumentRequest = new HttpPut(clusterUrl + "/" + index + "/_doc/" + docId + "?routing=" + routing); + public void createDocument(final String index, final String docId, final String body, String routing, String type) { + var indexDocumentRequest = new HttpPut(clusterUrl + "/" + index + "/" + Optional.ofNullable(type).orElse("_doc") + "/" + docId + "?routing=" + routing); indexDocumentRequest.setEntity(new StringEntity(body)); indexDocumentRequest.setHeader("Content-Type", "application/json"); @@ -82,6 +87,19 @@ public void deleteDocument(final String index, final String docId) throws IOExce } } + public void createIndexWithMappings(final String index, final String mappings) { + var body = "{" + + " \"settings\": {" + + " \"index\": {" + + " \"number_of_shards\": 5," + + " \"number_of_replicas\": 0" + + " }" + + " }," + + " \"mappings\": " + mappings + + "}"; + createIndex(index, body); + } + public void createIndex(final String index) { var body = "{" + // " \"settings\": {" + // @@ -106,6 +124,19 @@ public void createIndex(final String index, final String body) { } } + @SneakyThrows + public Map.Entry put(final String path, final String body) { + final var putRequest = new HttpPut(clusterUrl + path); + if (body != null) { + putRequest.setEntity(new StringEntity(body)); + putRequest.setHeader("Content-Type", "application/json"); + } + try (var response = httpClient.execute(putRequest)) { + var responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + return Map.entry(response.getCode(), responseBody); + } + } + @SneakyThrows public Map.Entry get(final String path) { final var getRequest = new HttpGet(clusterUrl + path); @@ -116,7 +147,7 @@ public Map.Entry get(final String path) { } } - public void takeSnapshot(final String snapshotName, final String indexPattern) throws IOException { + public void takeSnapshot(final String repoName, final String snapshotName, final String indexPattern) throws IOException { final var snapshotJson = "{\n" + " \"indices\": \"" + indexPattern @@ -126,7 +157,7 @@ public void takeSnapshot(final String snapshotName, final String indexPattern) t + "}"; final var createSnapshotRequest = new HttpPut( - clusterUrl + "/_snapshot/test-repo/" + snapshotName + "?wait_for_completion=true" + clusterUrl + "/_snapshot/" + repoName + "/" + snapshotName + "?wait_for_completion=true" ); createSnapshotRequest.setEntity(new StringEntity(snapshotJson)); createSnapshotRequest.setHeader("Content-Type", "application/json"); @@ -149,7 +180,7 @@ public void createLegacyTemplate(final String templateName, final String pattern " \"number_of_shards\": 1\r\n" + // " },\r\n" + // " \"aliases\": {\r\n" + // - " \"alias1\": {}\r\n" + // + " \"alias_legacy\": {}\r\n" + // " },\r\n" + // " \"mappings\": {\r\n" + // " \"_doc\": {\r\n" + // @@ -208,7 +239,7 @@ public void createComponentTemplate( + " }" + " }," + " \"aliases\": {" - + " \"alias1\": {}" + + " \"alias_component\": {}" + " }" + "}," + "\"version\": 1" @@ -277,7 +308,7 @@ public void createIndexTemplate( + " }" + " }," + " \"aliases\": {" - + " \"alias1\": {}" + + " \"alias_index\": {}" + " }" + "}"; diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 3a3212b83..dc637f7a6 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -78,8 +78,8 @@ services: condition: service_started opensearchtarget: condition: service_started - command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer --speedup-factor 2 https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id logging-group-default --otelCollectorEndpoint http://otel-collector:4317" #--transformer-config-base64 W3sgIkpzb25Kb2x0VHJhbnNmb3JtZXJQcm92aWRlciI6ClsKICB7CiAgICAic2NyaXB0IjogewogICAgICAib3BlcmF0aW9uIjogInNoaWZ0IiwKICAgICAgInNwZWMiOiB7CiAgICAgICAgIm1ldGhvZCI6ICJtZXRob2QiLAogICAgICAgICJVUkkiOiAiVVJJIiwKICAgICAgICAiaGVhZGVycyI6ICJoZWFkZXJzIiwKICAgICAgICAicGF5bG9hZCI6IHsKICAgICAgICAgICJpbmxpbmVkSnNvbkJvZHkiOiB7CiAgICAgICAgICAgICJ0b3AiOiB7CiAgICAgICAgICAgICAgInRhZ1RvRXhjaXNlIjogewogICAgICAgICAgICAgICAgIioiOiAicGF5bG9hZC5pbmxpbmVkSnNvbkJvZHkudG9wLiYiIAogICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgIioiOiAicGF5bG9hZC5pbmxpbmVkSnNvbkJvZHkudG9wLiYiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAiKiI6ICJwYXlsb2FkLmlubGluZWRKc29uQm9keS4mIgogICAgICAgICAgfQogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sIAogewogICAic2NyaXB0IjogewogICAgICJvcGVyYXRpb24iOiAibW9kaWZ5LW92ZXJ3cml0ZS1iZXRhIiwKICAgICAic3BlYyI6IHsKICAgICAgICJVUkkiOiAiPXNwbGl0KCcvZXh0cmFUaGluZ1RvUmVtb3ZlJyxAKDEsJikpIgogICAgIH0KICB9CiB9LAogewogICAic2NyaXB0IjogewogICAgICJvcGVyYXRpb24iOiAibW9kaWZ5LW92ZXJ3cml0ZS1iZXRhIiwKICAgICAic3BlYyI6IHsKICAgICAgICJVUkkiOiAiPWpvaW4oJycsQCgxLCYpKSIKICAgICB9CiAgfQogfQpdCn1dCg==" - +# command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer --speedup-factor 2 https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id logging-group-default --otelCollectorEndpoint http://otel-collector:4317 " + command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer --speedup-factor 2 https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id logging-group-default --otelCollectorEndpoint http://otel-collector:4317 --transformer-config '[{\"TypeMappingSanitizationTransformerProvider\":{\"sourceProperties\":{\"version\":{\"major\":7,\"minor\":10}}}}]'" opensearchtarget: image: 'opensearchproject/opensearch:2.15.0' environment: diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/replay.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/replay.py index 2cefe0d52..3b085a81c 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/replay.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/replay.py @@ -40,7 +40,8 @@ def scale(replayer: Replayer, units: int, *args, **kwargs) -> CommandResult[str] return replayer.scale(units, *args, **kwargs) -@handle_replay_errors() +@handle_replay_errors( + on_success=lambda status: (ExitCode.SUCCESS, f"{status[0]}\n{status[1]}")) def status(replayer: Replayer, *args, **kwargs) -> CommandResult[str]: logger.info("Getting replayer status") return replayer.get_status(*args, **kwargs) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py index f055c5c6b..c4d37f225 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py @@ -191,7 +191,11 @@ def parse_tuple(line: str, line_no: int) -> dict: return initial_tuple for component in SINGLE_COMPONENTS: - tuple_component = TupleComponent(component, initial_tuple[component], line_no, is_bulk_path) + if component in initial_tuple: + tuple_component = TupleComponent(component, initial_tuple[component], line_no, is_bulk_path) + else: + logger.info(f"`{component}` was not present on line {line_no}. Skipping component.") + continue processed_tuple = tuple_component.b64decode().decode_utf8().parse_as_json() final_value = processed_tuple.final_value @@ -201,6 +205,9 @@ def parse_tuple(line: str, line_no: int) -> dict: logger.error(processed_tuple.error) for component in LIST_COMPONENTS: + if component not in initial_tuple: + logger.info(f"`{component}` was not present on line {line_no}. Skipping component.") + continue for i, item in enumerate(initial_tuple[component]): tuple_component = TupleComponent(f"{component} item {i}", item, line_no, is_bulk_path) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_missing_component.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_missing_component.json new file mode 100644 index 000000000..58759cf7c --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_missing_component.json @@ -0,0 +1,32 @@ +{ + "sourceRequest": { + "Request-URI": "/_snapshot/migration_assistant_repo/rfs-snapshot", + "Method": "DELETE", + "HTTP-Version": "HTTP/1.1", + "payload": { + "inlinedTextBody": "" + } + }, + "targetRequest": { + "Request-URI": "/_snapshot/migration_assistant_repo/rfs-snapshot", + "Method": "DELETE", + "payload": { + "inlinedTextBody": "" + } + }, + "targetResponses": [ + { + "Date": ["Tue, 10 Dec 2024 21:07:03 GMT"], + "Content-Type": ["application/json; charset=UTF-8"], + "Status-Code": 404, + "Reason-Phrase": "Not Found", + "response_time_ms": 49, + "payload": { + "inlinedJsonBody": {} + } + } + ], + "connectionId": "024ad2fffe460aed-00000007-00000ac1-61aca27669436ce2-fdb9a98a.1563", + "numRequests": 4, + "numErrors": 0 +} diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py index 7f24cc49e..30bbd6237 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py @@ -15,6 +15,7 @@ VALID_TUPLE_GZIPPED_CHUNKED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked.json" VALID_TUPLE_GZIPPED_CHUNKED_PARSED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked_parsed.json" INVALID_TUPLE = TEST_DATA_DIRECTORY / "invalid_tuple.json" +VALID_TUPLE_MISSING_COMPONENT = TEST_DATA_DIRECTORY / "valid_tuple_missing_component.json" def test_get_element_with_regex_succeeds(): @@ -161,3 +162,14 @@ def test_parse_tuple_with_malformed_bodies(caplog): assert json.loads(tuple_) == parsed # Values weren't changed if they couldn't be interpreted assert "Body value of sourceResponse on line 0 could not be decoded to utf-8" in caplog.text assert "Body value of targetResponses item 0 on line 0 should be a json, but could not be parsed" in caplog.text + + +def test_parse_tuple_with_missing_component(): + with open(VALID_TUPLE_MISSING_COMPONENT, 'r') as f: + tuple_ = f.read() + + assert 'sourceResponse' not in json.loads(tuple_) + parsed = parse_tuple(tuple_, 0) + + assert 'sourceResponse' not in parsed + assert json.loads(tuple_).keys() == parsed.keys() diff --git a/TrafficCapture/trafficReplayer/README.md b/TrafficCapture/trafficReplayer/README.md index de789a3fd..0562844c1 100644 --- a/TrafficCapture/trafficReplayer/README.md +++ b/TrafficCapture/trafficReplayer/README.md @@ -145,7 +145,7 @@ transform to add GZIP encoding and another to apply a new header would be config ``` To run only one transformer without any configuration, the `--transformer-config` argument can simply -be set to the name of the transformer (e.g. 'JsonTransformerForOpenSearch23PlusTargetTransformerProvider', +be set to the name of the transformer (e.g. 'TypeMappingSanitizationTransformerProvider', without quotes or any json surrounding it). The user can also specify a file to read the transformations from using the `--transformer-config-file`. Users can @@ -153,7 +153,7 @@ also pass the script as an argument via `--transformer-config-base64`. Each of is mutually exclusive. Some simple transformations are included to change headers to add compression or to force an HTTP message payload to -be chunked. Another transformer, [JsonTypeMappingTransformer.java](../../transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java), +be chunked. Another transformer, [TypeMappingSanitizationTransformer.java](../../transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java), is a work-in-progress to excise type mapping references from URIs and message payloads since versions of OpenSource greater than 2.3 do not support them. diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 39d2a59ee..0bf34d63c 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -22,8 +22,9 @@ dependencies { implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') + runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJinjavaTransformerProvider') runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') - runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') implementation group: 'org.jcommander', name: 'jcommander' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' @@ -59,7 +60,7 @@ dependencies { testImplementation testFixtures(project(path: ':coreUtilities')) testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') - testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api' diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java index e2e807818..c01925b46 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/HttpByteBufFormatter.java @@ -197,7 +197,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } else { content.release(); } - } else if (msg instanceof HttpMessage) { + } + if (msg instanceof HttpMessage) { // this & HttpContent are interfaces & 'Full' messages implement both message = (HttpMessage) msg; } if (msg instanceof LastHttpContent) { @@ -206,16 +207,16 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } var finalMsg = (message instanceof HttpRequest) ? new DefaultFullHttpRequest(message.protocolVersion(), - ((HttpRequest) message).method(), - ((HttpRequest) message).uri(), - aggregatedContents, - message.headers(), - ((LastHttpContent) msg).trailingHeaders()) + ((HttpRequest) message).method(), + ((HttpRequest) message).uri(), + aggregatedContents, + message.headers(), + ((LastHttpContent) msg).trailingHeaders()) : new DefaultFullHttpResponse(message.protocolVersion(), - ((HttpResponse)message).status(), - aggregatedContents, - message.headers(), - ((LastHttpContent) msg).trailingHeaders()); + ((HttpResponse)message).status(), + aggregatedContents, + message.headers(), + ((LastHttpContent) msg).trailingHeaders()); super.channelRead(ctx, finalMsg); } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java index 4a0f87609..ae598fd77 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java @@ -42,6 +42,10 @@ public class ParsedHttpMessagesAsDicts { public static final String STATUS_CODE_KEY = "Status-Code"; public static final String RESPONSE_TIME_MS_KEY = "response_time_ms"; public static final String EXCEPTION_KEY_STRING = "Exception"; + public static final String REQUEST_URI_KEY = "Request-URI"; + public static final String METHOD_KEY = "Method"; + public static final String HTTP_VERSION_KEY = "HTTP-Version"; + public static final String PAYLOAD_KEY = "payload"; public final Optional> sourceRequestOp; public final Optional> sourceResponseOp; @@ -183,15 +187,15 @@ private static Map convertRequest( var message = (HttpJsonRequestWithFaultingPayload) messageHolder.get(); if (message != null) { var map = new LinkedHashMap<>(message.headers()); - map.put("Request-URI", message.path()); - map.put("Method", message.method()); - map.put("HTTP-Version", message.protocol()); + map.put(REQUEST_URI_KEY, message.path()); + map.put(METHOD_KEY, message.method()); + map.put(HTTP_VERSION_KEY, message.protocol()); context.setMethod(message.method()); context.setEndpoint(message.path()); context.setHttpVersion(message.protocol()); encodeBinaryPayloadIfExists(message); if (!message.payload().isEmpty()) { - map.put("payload", message.payload()); + map.put(PAYLOAD_KEY, message.payload()); } return map; } else { @@ -223,14 +227,14 @@ private static Map convertResponse( var message = (HttpJsonResponseWithFaultingPayload) messageHolder.get(); if (message != null) { var map = new LinkedHashMap<>(message.headers()); - map.put("HTTP-Version", message.protocol()); + map.put(HTTP_VERSION_KEY, message.protocol()); map.put(STATUS_CODE_KEY, Integer.parseInt(message.code())); map.put("Reason-Phrase", message.reason()); map.put(RESPONSE_TIME_MS_KEY, latency.toMillis()); context.setHttpVersion(message.protocol()); encodeBinaryPayloadIfExists(message); if (!message.payload().isEmpty()) { - map.put("payload", message.payload()); + map.put(PAYLOAD_KEY, message.payload()); } return map; } else { diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java index 4ef0c825a..528e89b93 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/ResultsToLogsConsumer.java @@ -161,6 +161,8 @@ public static String getTransactionSummaryStringPreamble() { .add("SOURCE_STATUS_CODE/TARGET_STATUS_CODE...") .add("SOURCE_RESPONSE_SIZE_BYTES/TARGET_RESPONSE_SIZE_BYTES...") .add("SOURCE_LATENCY_MS/TARGET_LATENCY_MS...") + .add("METHOD...") + .add("URI...") .toString(); } @@ -218,6 +220,16 @@ public static String toTransactionSummaryString( transformStreamToString(parsed.targetResponseList.stream(), r -> "" + r.get(ParsedHttpMessagesAsDicts.RESPONSE_TIME_MS_KEY)) ) + // method + .add( + parsed.sourceRequestOp + .map(r -> (String) r.get(ParsedHttpMessagesAsDicts.METHOD_KEY)) + .orElse(MISSING_STR)) + // uri + .add( + parsed.sourceRequestOp + .map(r -> (String) r.get(ParsedHttpMessagesAsDicts.REQUEST_URI_KEY)) + .orElse(MISSING_STR)) .toString(); } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java index 3c57dbb6b..cb7e5025d 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/JsonAccumulator.java @@ -116,10 +116,10 @@ public Object getNextTopLevelObject() throws IOException { pushCompletedValue(parser.getText()); break; case VALUE_NUMBER_INT: - pushCompletedValue(parser.getIntValue()); + pushCompletedValue(parser.getLongValue()); break; case VALUE_NUMBER_FLOAT: - pushCompletedValue(parser.getFloatValue()); + pushCompletedValue(parser.getDoubleValue()); break; case NOT_AVAILABLE: // pipeline stall - need more data diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java index ff8c3f665..9b8abb459 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/PayloadAccessFaultingMap.java @@ -32,8 +32,10 @@ public class PayloadAccessFaultingMap extends AbstractMap { @Getter @Setter private boolean disableThrowingPayloadNotLoaded; + private boolean payloadWasAccessed; public PayloadAccessFaultingMap(StrictCaseInsensitiveHttpHeadersMap headers) { + disableThrowingPayloadNotLoaded = true; underlyingMap = new TreeMap<>(); isJson = Optional.ofNullable(headers.get("content-type")) .map(list -> list.stream().anyMatch(s -> s.startsWith("application/json"))) @@ -51,19 +53,19 @@ public Iterator> iterator() { return new Iterator<>() { @Override public boolean hasNext() { - throw PayloadNotLoadedException.getInstance(); + throw makeFault(); } @Override public Map.Entry next() { - throw PayloadNotLoadedException.getInstance(); + throw makeFault(); } }; } @Override public int size() { - throw PayloadNotLoadedException.getInstance(); + throw makeFault(); } }; } else { @@ -80,8 +82,21 @@ public Object put(String key, Object value) { public Object get(Object key) { var value = super.get(key); if (value == null && !disableThrowingPayloadNotLoaded) { - throw PayloadNotLoadedException.getInstance(); + throw makeFault(); } return value; } + + public boolean missingPayloadWasAccessed() { + return payloadWasAccessed; + } + + public void resetMissingPayloadWasAccessed() { + payloadWasAccessed = false; + } + + private PayloadNotLoadedException makeFault() throws PayloadNotLoadedException { + payloadWasAccessed = true; + return PayloadNotLoadedException.getInstance(); + } } diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java index 3d74c42ca..89acd028f 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyDecodedHttpRequestPreliminaryTransformHandler.java @@ -7,7 +7,6 @@ import java.util.Optional; import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; -import org.opensearch.migrations.replay.datahandlers.PayloadNotLoadedException; import org.opensearch.migrations.replay.tracing.IReplayContexts; import org.opensearch.migrations.transform.IAuthTransformer; import org.opensearch.migrations.transform.IJsonTransformer; @@ -67,25 +66,38 @@ public void channelRead(@NonNull ChannelHandlerContext ctx, @NonNull Object msg) IAuthTransformer authTransformer = requestPipelineOrchestrator.authTransfomerFactory.getAuthTransformer( httpJsonMessage ); + HttpJsonRequestWithFaultingPayload transformedMessage = null; + final var payloadMap = (PayloadAccessFaultingMap) httpJsonMessage.payload(); try { + payloadMap.setDisableThrowingPayloadNotLoaded(false); + transformedMessage = transform(transformer, httpJsonMessage); + } catch (Exception e) { + var payload = (PayloadAccessFaultingMap) httpJsonMessage.payload(); + if (payload.missingPayloadWasAccessed()) { + payload.resetMissingPayloadWasAccessed(); + log.atDebug().setMessage("The transforms for this message require payload manipulation, " + + "all content handlers are being loaded.").log(); + // make a fresh message and its headers + requestPipelineOrchestrator.addJsonParsingHandlers( + ctx, + transformer, + getAuthTransformerAsStreamingTransformer(authTransformer) + ); + ctx.fireChannelRead(handleAuthHeaders(httpJsonMessage, authTransformer)); + } else{ + throw new TransformationException(e); + } + } finally { + payloadMap.setDisableThrowingPayloadNotLoaded(true); + } + + if (transformedMessage != null) { handlePayloadNeutralTransformationOrThrow( ctx, originalHttpJsonMessage, - transform(transformer, httpJsonMessage), + transformedMessage, authTransformer ); - } catch (PayloadNotLoadedException pnle) { - log.debug( - "The transforms for this message require payload manipulation, " - + "all content handlers are being loaded." - ); - // make a fresh message and its headers - requestPipelineOrchestrator.addJsonParsingHandlers( - ctx, - transformer, - getAuthTransformerAsStreamingTransformer(authTransformer) - ); - ctx.fireChannelRead(handleAuthHeaders(httpJsonMessage, authTransformer)); } } else if (msg instanceof HttpContent) { ctx.fireChannelRead(msg); @@ -148,26 +160,26 @@ private void handlePayloadNeutralTransformationOrThrow( } else if (headerFieldIsIdentical("content-encoding", originalRequest, httpJsonMessage) && headerFieldIsIdentical("transfer-encoding", originalRequest, httpJsonMessage)) { - log.info( - diagnosticLabel - + "There were changes to the headers that require the message to be reformatted " - + "but the payload doesn't need to be transformed." - ); - // By adding the baseline handlers and removing this and previous handlers in reverse order, - // we will cause the upstream handlers to flush their in-progress accumulated ByteBufs downstream - // to be processed accordingly - requestPipelineOrchestrator.addBaselineHandlers(pipeline); - ctx.fireChannelRead(httpJsonMessage); - RequestPipelineOrchestrator.removeThisAndPreviousHandlers(pipeline, this); - } else { - log.info( - diagnosticLabel - + "New headers have been specified that require the payload stream to be " - + "reformatted. Setting up the processing pipeline to parse and reformat the request payload." - ); - requestPipelineOrchestrator.addContentRepackingHandlers(ctx, streamingAuthTransformer); - ctx.fireChannelRead(httpJsonMessage); - } + log.info( + diagnosticLabel + + "There were changes to the headers that require the message to be reformatted " + + "but the payload doesn't need to be transformed." + ); + // By adding the baseline handlers and removing this and previous handlers in reverse order, + // we will cause the upstream handlers to flush their in-progress accumulated ByteBufs downstream + // to be processed accordingly + requestPipelineOrchestrator.addBaselineHandlers(pipeline); + ctx.fireChannelRead(httpJsonMessage); + RequestPipelineOrchestrator.removeThisAndPreviousHandlers(pipeline, this); + } else { + log.info( + diagnosticLabel + + "New headers have been specified that require the payload stream to be " + + "reformatted. Setting up the processing pipeline to parse and reformat the request payload." + ); + requestPipelineOrchestrator.addContentRepackingHandlers(ctx, streamingAuthTransformer); + ctx.fireChannelRead(httpJsonMessage); + } } private static HttpJsonRequestWithFaultingPayload handleAuthHeaders( diff --git a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java index 754ac2a4e..35ed26b68 100644 --- a/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java +++ b/TrafficCapture/trafficReplayer/src/main/java/org/opensearch/migrations/replay/datahandlers/http/NettyJsonBodyAccumulateHandler.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Predicate; import org.opensearch.migrations.replay.datahandlers.JsonAccumulator; import org.opensearch.migrations.replay.tracing.IReplayContexts; @@ -24,6 +25,7 @@ import io.netty.util.ReferenceCountUtil; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.slf4j.event.Level; /** * This accumulates HttpContent messages through a JsonAccumulator and eventually fires off a @@ -93,7 +95,10 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } } } catch (JacksonException e) { - log.atInfo().setCause(e).setMessage("Error parsing json body. " + + log.atLevel(hasRequestContentTypeMatching(capturedHttpJsonMessage, + // a JacksonException for non-json data doesn't need to be surfaced to a user + v -> v.startsWith("application/json")) ? Level.INFO : Level.TRACE) + .setCause(e).setMessage("Error parsing json body. " + "Will pass all payload bytes directly as a ByteBuf within the payload map").log(); jsonWasInvalid = true; parsedJsonObjects.clear(); @@ -123,7 +128,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception var leftoverBody = accumulatedBody.slice(jsonBodyByteLength, accumulatedBody.readableBytes() - jsonBodyByteLength); - if (jsonBodyByteLength == 0 && isRequestContentTypeNotText(capturedHttpJsonMessage)) { + if (jsonBodyByteLength == 0 && + hasRequestContentTypeMatching(capturedHttpJsonMessage, v -> !v.startsWith("text/"))) + { context.onPayloadSetBinary(); capturedHttpJsonMessage.payload() .put(JsonKeysForHttpMessage.INLINED_BINARY_BODY_DOCUMENT_KEY, @@ -157,12 +164,13 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception } } - private boolean isRequestContentTypeNotText(HttpJsonMessageWithFaultingPayload message) { + private boolean hasRequestContentTypeMatching(HttpJsonMessageWithFaultingPayload message, + Predicate contentTypeFilter) { // ContentType not text if specified and has a value with / and that value does not start with text/ return Optional.ofNullable(capturedHttpJsonMessage.headers().insensitiveGet(HttpHeaderNames.CONTENT_TYPE.toString())) .map(s -> s.stream() .filter(v -> v.contains("/")) - .filter(v -> !v.startsWith("text/")) + .filter(contentTypeFilter) .count() > 1 ) .orElse(false); diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadNotFoundTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadNotFoundTest.java new file mode 100644 index 000000000..f2e1e7467 --- /dev/null +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/PayloadNotFoundTest.java @@ -0,0 +1,39 @@ +package org.opensearch.migrations.replay; + + +import org.opensearch.migrations.replay.datahandlers.PayloadAccessFaultingMap; +import org.opensearch.migrations.replay.datahandlers.http.HttpJsonRequestWithFaultingPayload; +import org.opensearch.migrations.replay.datahandlers.http.ListKeyAdaptingCaseInsensitiveHeadersMap; +import org.opensearch.migrations.replay.datahandlers.http.StrictCaseInsensitiveHttpHeadersMap; +import org.opensearch.migrations.transform.TransformationLoader; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PayloadNotFoundTest { + @Test + public void testTransformsPropagateExceptionProperly() throws JsonProcessingException { + final HttpJsonRequestWithFaultingPayload FAULTING_MAP = new HttpJsonRequestWithFaultingPayload(); + FAULTING_MAP.setMethod("PUT"); + FAULTING_MAP.setPath("/_bulk"); + FAULTING_MAP.setHeaders(new ListKeyAdaptingCaseInsensitiveHeadersMap(new StrictCaseInsensitiveHttpHeadersMap())); + FAULTING_MAP.headers().put("Content-Type", "application/json"); + var payloadMap = new PayloadAccessFaultingMap(FAULTING_MAP.headers().asStrictMap()); + FAULTING_MAP.setPayloadFaultMap(payloadMap); + payloadMap.setDisableThrowingPayloadNotLoaded(false); + final String EXPECTED = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/_bulk\",\n" + + " \"headers\": {\n" + + " \"Content-Type\": \"application/json\"\n" + + " }\n" + + "}\n"; + + var transformer = new TransformationLoader().getTransformerFactoryLoader("newhost", null, + "[{\"TypeMappingSanitizationTransformerProvider\":\"\"}]"); + var e = Assertions.assertThrows(Exception.class, + () -> transformer.transformJson(FAULTING_MAP)); + Assertions.assertTrue(((PayloadAccessFaultingMap)FAULTING_MAP.payload()).missingPayloadWasAccessed()); + } +} diff --git a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java index 8dcf6ba49..7babdb434 100644 --- a/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java +++ b/TrafficCapture/trafficReplayer/src/test/java/org/opensearch/migrations/replay/datahandlers/http/HttpJsonTransformingConsumerTest.java @@ -25,6 +25,7 @@ import org.opensearch.migrations.transform.TransformationLoader; import org.opensearch.migrations.utils.TrackedFuture; +import com.fasterxml.jackson.databind.ObjectMapper; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.util.ReferenceCountUtil; @@ -331,8 +332,11 @@ public void testMalformedPayload_andThrowingTransformation_IsPassedThrough() thr new TransformationLoader().getTransformerFactoryLoader( HOST_NAME, null, - "[{\"JsonTransformerForOpenSearch23PlusTargetTransformerProvider\":\"\"}]" - ), + new ObjectMapper().writeValueAsString(List.of( + Map.of("JsonJinjavaTransformerProvider", Map.of( + "template", "{%- throw \"intentional exception\" -%}" + )) + ))), null, testPacketCapture, rootContext.getTestConnectionRequestContext(0) @@ -362,10 +366,7 @@ public void testMalformedPayload_andThrowingTransformation_IsPassedThrough() thr ); var outputAndResult = finalizationFuture.get(); Assertions.assertInstanceOf(TransformationException.class, - TrackedFuture.unwindPossibleCompletionException(outputAndResult.transformationStatus.getException()), - "It's acceptable for now that the OpenSearch upgrade transformation can't handle non-json " + - "content. If that Transform wants to handle this on its own, we'll need to use another transform " + - "configuration so that it throws and we can do this test."); + TrackedFuture.unwindPossibleCompletionException(outputAndResult.transformationStatus.getException())); var combinedOutputBuf = outputAndResult.transformedOutput.getResponseAsByteBuf(); Assertions.assertTrue(combinedOutputBuf.readableBytes() == 0); combinedOutputBuf.release(); diff --git a/commonDependencyVersionConstraints/build.gradle b/commonDependencyVersionConstraints/build.gradle index f785f7e23..2bfc4a187 100644 --- a/commonDependencyVersionConstraints/build.gradle +++ b/commonDependencyVersionConstraints/build.gradle @@ -55,6 +55,8 @@ dependencies { def jackson = '2.16.2' api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jackson + api group: 'com.fasterxml.jackson.core', name: 'jackson-dataformat-yaml', version: jackson + api group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jackson api group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-smile', version: jackson def jupiter = '5.10.2' diff --git a/sonar-project.properties b/sonar-project.properties index 501b15e64..d2116e118 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -58,7 +58,7 @@ sonar.sourceEncoding=UTF-8 sonar.issue.ignore.multicriteria = \ p1, \ ts1, ts2, ts3, ts4, ts5, ts6, ts7, ts8, ts9, ts10, \ - j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, \ + j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, \ autoclose, \ comp1, comp2, comp3, comp4, \ loop1, loop2, loop3, loop4, loop5, \ @@ -168,6 +168,14 @@ sonar.issue.ignore.multicriteria.j11.resourceKey = **/*.java sonar.issue.ignore.multicriteria.j12.ruleKey = java:S1452 sonar.issue.ignore.multicriteria.j12.resourceKey = **/*.java +# Ignore Set the credentials provider explicitly on this builder, false positive. +sonar.issue.ignore.multicriteria.j13.ruleKey = java:S6242 +sonar.issue.ignore.multicriteria.j13.resourceKey = **/*.java + +# Ignore Set the region explicitly on this builder, false positive. +sonar.issue.ignore.multicriteria.j14.ruleKey = java:S6241 +sonar.issue.ignore.multicriteria.j14.resourceKey = **/*.java + # "Use try-with-resources or close this" diff --git a/testHelperFixtures/src/testFixtures/java/org/opensearch/migrations/testutils/JsonNormalizer.java b/testHelperFixtures/src/testFixtures/java/org/opensearch/migrations/testutils/JsonNormalizer.java new file mode 100644 index 000000000..b84219354 --- /dev/null +++ b/testHelperFixtures/src/testFixtures/java/org/opensearch/migrations/testutils/JsonNormalizer.java @@ -0,0 +1,23 @@ +package org.opensearch.migrations.testutils; + +import java.util.SortedMap; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.SneakyThrows; + +public class JsonNormalizer { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(SerializationFeature.INDENT_OUTPUT, true); + + @SneakyThrows + public static String fromString(String input) { + return OBJECT_MAPPER.writeValueAsString(OBJECT_MAPPER.readValue(input, SortedMap.class)); + } + + @SneakyThrows + public static String fromObject(Object obj) { + return fromString(OBJECT_MAPPER.writeValueAsString(obj)); + } +} diff --git a/transformation/src/main/java/org/opensearch/migrations/transformation/entity/Index.java b/transformation/src/main/java/org/opensearch/migrations/transformation/entity/Index.java index af37a0ebb..ffb22bf02 100644 --- a/transformation/src/main/java/org/opensearch/migrations/transformation/entity/Index.java +++ b/transformation/src/main/java/org/opensearch/migrations/transformation/entity/Index.java @@ -3,4 +3,6 @@ /** * Represents an Index object for transformation */ -public interface Index extends Entity {} +public interface Index extends Entity { + String getName(); +} diff --git a/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java b/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java index 833d1f13d..9610e0ab1 100644 --- a/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java +++ b/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java @@ -1,13 +1,15 @@ package org.opensearch.migrations.transformation.rules; -import java.util.Map.Entry; - import org.opensearch.migrations.transformation.CanApplyResult; import org.opensearch.migrations.transformation.CanApplyResult.Unsupported; import org.opensearch.migrations.transformation.TransformationRule; import org.opensearch.migrations.transformation.entity.Index; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * Supports transformation of the Index Mapping types that were changed from mutliple types to a single type between ES 6 to ES 7 @@ -37,9 +39,24 @@ * } * } */ +@Slf4j +@AllArgsConstructor public class IndexMappingTypeRemoval implements TransformationRule { + public enum MultiTypeResolutionBehavior { + NONE, + UNION, + SPLIT + } + public static final String PROPERTIES_KEY = "properties"; public static final String MAPPINGS_KEY = "mappings"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + public final MultiTypeResolutionBehavior multiTypeResolutionBehavior; + + // Default with NONE + public IndexMappingTypeRemoval() { + this(MultiTypeResolutionBehavior.NONE); + } @Override public CanApplyResult canApply(final Index index) { @@ -49,20 +66,25 @@ public CanApplyResult canApply(final Index index) { return CanApplyResult.NO; } - - // Detect unsupported multiple type mappings: - // 1. 
{"mappings": [{ "foo": {...} }, { "bar": {...} }]}
- // 2. 
{"mappings": [{ "foo": {...}, "bar": {...}  }]}
- if (mappingNode.isArray() && (mappingNode.size() > 1 || mappingNode.get(0).size() > 1)) { - return new Unsupported("Multiple mapping types are not supported"); - } - // Check for absence of intermediate type node // 1. 
{"mappings": {"properties": {...} }}
if (mappingNode.isObject() && mappingNode.get("properties") != null) { return CanApplyResult.NO; } + // Detect multiple type mappings: + // 1. 
{"mappings": [{ "foo": {...} }, { "bar": {...} }]}
+ // 2. 
{"mappings": [{ "foo": {...}, "bar": {...}  }]}
+ if (mappingNode.isArray() && (mappingNode.size() > 1 || mappingNode.get(0).size() > 1)) { + if (MultiTypeResolutionBehavior.NONE.equals(multiTypeResolutionBehavior)) { + return new Unsupported("No multi type resolution behavior declared, specify --multi-type-behavior to process"); + } + if (MultiTypeResolutionBehavior.SPLIT.equals(multiTypeResolutionBehavior)) { + return new Unsupported("Split on multiple mapping types is not supported"); + } + // Support UNION + } + // There is a type under mappings // 1. 
{ "mappings": [{ "foo": {...} }] }
return CanApplyResult.YES; @@ -77,14 +99,49 @@ public boolean applyTransformation(final Index index) { final var mappingsNode = index.getRawJson().get(MAPPINGS_KEY); // Handle array case if (mappingsNode.isArray()) { - final var mappingsInnerNode = (ObjectNode) mappingsNode.get(0); - - final var typeName = mappingsInnerNode.properties().stream().map(Entry::getKey).findFirst().orElseThrow(); - final var typeNode = mappingsInnerNode.get(typeName); - - mappingsInnerNode.remove(typeName); - typeNode.fields().forEachRemaining(node -> mappingsInnerNode.set(node.getKey(), node.getValue())); - index.getRawJson().set(MAPPINGS_KEY, mappingsInnerNode); + final var resolvedMappingsNode = MAPPER.createObjectNode(); + if (mappingsNode.size() < 2) { + final var mappingsInnerNode = (ObjectNode) mappingsNode.get(0); + var properties = mappingsInnerNode.fields().next().getValue().get(PROPERTIES_KEY); + resolvedMappingsNode.set(PROPERTIES_KEY, properties); + } else if (MultiTypeResolutionBehavior.UNION.equals(multiTypeResolutionBehavior)) { + var resolvedProperties = resolvedMappingsNode.withObjectProperty(PROPERTIES_KEY); + var mappings = (ArrayNode) mappingsNode; + mappings.forEach( + typeNodeEntry -> { + var typeNode = typeNodeEntry.properties().stream().findFirst().orElseThrow(); + var type = typeNode.getKey(); + var node = typeNode.getValue(); + var properties = node.get(PROPERTIES_KEY); + properties.properties().forEach(propertyEntry -> { + var fieldName = propertyEntry.getKey(); + var fieldType = propertyEntry.getValue(); + + if (resolvedProperties.has(fieldName)) { + var existingFieldType = resolvedProperties.get(fieldName); + if (!existingFieldType.equals(fieldType)) { + log.atWarn().setMessage("Conflict during type union with index: {}\n" + + "field: {}\n" + + "existingFieldType: {}\n" + + "type: {}\n" + + "secondFieldType: {}") + .addArgument(index.getName()) + .addArgument(fieldName) + .addArgument(existingFieldType) + .addArgument(type) + .addArgument(fieldType) + .log(); + throw new IllegalArgumentException("Conflicting definitions for property during union " + + fieldName + " (" + existingFieldType + " and " + fieldType + ")" ); + } + } else { + resolvedProperties.set(fieldName, fieldType); + } + }); + } + ); + } + index.getRawJson().set(MAPPINGS_KEY, resolvedMappingsNode); } if (mappingsNode.isObject()) { @@ -99,7 +156,6 @@ public boolean applyTransformation(final Index index) { } mappingsObjectNode.remove(typeNode.getKey()); } - return true; } } diff --git a/transformation/src/test/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemovalTest.java b/transformation/src/test/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemovalTest.java index 6262dd2c9..5bbfa33b3 100644 --- a/transformation/src/test/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemovalTest.java +++ b/transformation/src/test/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemovalTest.java @@ -11,6 +11,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.mockito.Mockito; import static org.hamcrest.CoreMatchers.instanceOf; @@ -18,6 +20,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; @Slf4j @@ -62,7 +66,7 @@ public class IndexMappingTypeRemovalTest { defaultMappingProperties + // " }\n" + // "}],\n" - ); + ); private final BiFunction mutlipleMappingsWithSingleTypes = ( typeName1, @@ -78,7 +82,23 @@ public class IndexMappingTypeRemovalTest { defaultMappingProperties + // " }\n" + // "}],\n" - ); + ); + + private final BiFunction conflictingMappingWithMultipleTypes = (typeName1, typeName2) -> indexSettingJson( + "\"mappings\": [{\n" + + " \"" + typeName1 + "\": {\n" + + " \"properties\": {\n" + + " \"age\": { \"type\": \"integer\" }\n" + + " }\n" + + " }},{\n" + + " \"" + typeName2 + "\": {\n" + + " \"properties\": {\n" + + " \"age\": { \"type\": \"text\" }\n" + + " }\n" + + " }\n" + + "}],\n" + ); + public ObjectNode indexSettingJson(final String mappingSection) { try { @@ -99,14 +119,21 @@ public ObjectNode indexSettingJson(final String mappingSection) { } private CanApplyResult canApply(final ObjectNode indexJson) { - var transformer = new IndexMappingTypeRemoval(); + return canApply(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.NONE, indexJson); + } + + private CanApplyResult canApply(final IndexMappingTypeRemoval.MultiTypeResolutionBehavior behavior, final ObjectNode indexJson) { + var transformer = new IndexMappingTypeRemoval(behavior); var index = mock(Index.class); Mockito.when(index.getRawJson()).thenReturn(indexJson); return transformer.canApply(index); } - private boolean applyTransformation(final ObjectNode indexJson) { - var transformer = new IndexMappingTypeRemoval(); + return applyTransformation(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.NONE, indexJson); + } + + private boolean applyTransformation(final IndexMappingTypeRemoval.MultiTypeResolutionBehavior behavior, final ObjectNode indexJson) { + var transformer = new IndexMappingTypeRemoval(behavior); var index = mock(Index.class); Mockito.when(index.getRawJson()).thenReturn(indexJson); @@ -201,37 +228,102 @@ void testApplyTransformation_customTypes() { assertThat(indexJson.toPrettyString(), not(containsString(typeName))); } - @Test - void testApplyTransformation_twoCustomTypes() { + @ParameterizedTest + @CsvSource({ + "SPLIT, 'Split on multiple mapping types is not supported'", + "NONE, 'No multi type resolution behavior declared, specify --multi-type-behavior to process'" + }) + void testApplyTransformation_twoCustomTypes(String resolutionBehavior, String expectedReason) { // Setup var originalJson = mappingWithMutlipleTypes.apply("t1", "t2"); var indexJson = originalJson.deepCopy(); + var behavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.valueOf(resolutionBehavior); + // Action - var wasChanged = applyTransformation(indexJson); - var canApply = canApply(originalJson); + var wasChanged = applyTransformation(behavior, indexJson); + var canApply = canApply(behavior, originalJson); assertThat(canApply, instanceOf(Unsupported.class)); - assertThat(((Unsupported) canApply).getReason(), equalTo("Multiple mapping types are not supported")); + assertThat(((Unsupported) canApply).getReason(), equalTo(expectedReason)); // Verification assertThat(wasChanged, equalTo(false)); assertThat(originalJson.toPrettyString(), equalTo(indexJson.toPrettyString())); } - @Test - void testApplyTransformation_twoMappingEntries() { + + @ParameterizedTest + @CsvSource({ + "SPLIT, 'Split on multiple mapping types is not supported'", + "NONE, 'No multi type resolution behavior declared, specify --multi-type-behavior to process'" + }) + void testApplyTransformation_twoMappingEntries(String resolutionBehavior, String expectedReason) { // Setup var originalJson = mutlipleMappingsWithSingleTypes.apply("t1", "t2"); var indexJson = originalJson.deepCopy(); + var behavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.valueOf(resolutionBehavior); // Action - var wasChanged = applyTransformation(indexJson); - var canApply = canApply(originalJson); + var wasChanged = applyTransformation(behavior, indexJson); + var canApply = canApply(behavior, originalJson); assertThat(canApply, instanceOf(Unsupported.class)); - assertThat(((Unsupported) canApply).getReason(), equalTo("Multiple mapping types are not supported")); + assertThat(((Unsupported) canApply).getReason(), equalTo(expectedReason)); // Verification assertThat(wasChanged, equalTo(false)); assertThat(originalJson.toPrettyString(), equalTo(indexJson.toPrettyString())); } + + // Helper method to create Index with specified raw JSON + private Index createMockIndex(ObjectNode indexJson) { + var index = mock(Index.class); + Mockito.when(index.getRawJson()).thenReturn(indexJson); + return index; + } + + // Helper method to apply transformation with a specified transformer + private boolean applyTransformation(final ObjectNode indexJson, IndexMappingTypeRemoval transformer) { + var index = createMockIndex(indexJson); + log.atInfo().setMessage("Original\n{}").addArgument(() -> indexJson.toPrettyString()).log(); + var wasChanged = transformer.applyTransformation(index); + log.atInfo().setMessage("After{}\n{}") + .addArgument(wasChanged ? " *Changed* " : "") + .addArgument(() -> indexJson.toPrettyString()) + .log(); + return wasChanged; + } + + @Test + void testApplyTransformation_multiTypeUnion_noConflicts() { + // Setup + var originalJson = mappingWithMutlipleTypes.apply("type1", "type2"); + var indexJson = originalJson.deepCopy(); + var transformer = new IndexMappingTypeRemoval(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION); + + // Action + var wasChanged = applyTransformation(indexJson, transformer); + var canApply = transformer.canApply(createMockIndex(originalJson)); + + // Verification + assertThat(canApply, equalTo(CanApplyResult.YES)); + assertThat(wasChanged, equalTo(true)); + + // Check that the "mappings" node has "properties" with merged fields from both types + var propertiesNode = indexJson.get("mappings").get("properties"); + assertThat(propertiesNode, notNullValue()); + // Assuming both types have "age" property from defaultMappingProperties + assertThat(propertiesNode.has("age"), equalTo(true)); + } + + @Test + void testApplyTransformation_multiTypeUnion_withConflicts() { + // Setup + var originalJson = conflictingMappingWithMultipleTypes.apply("type1", "type2"); + var indexJson = originalJson.deepCopy(); + var transformer = new IndexMappingTypeRemoval(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION); + + // Action & Verification + var exception = assertThrows(IllegalArgumentException.class, () -> applyTransformation(indexJson, transformer)); + assertThat(exception.getMessage(), containsString("Conflicting definitions for property during union age")); + } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java similarity index 93% rename from transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java rename to transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java index 2885678ab..912fd3c9e 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonTransformerTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJMESPathMessageTransformerProvider/src/test/java/org/opensearch/migrations/replay/JsonJMESPathTransformerProviderTest.java @@ -4,19 +4,18 @@ import java.util.Map; import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; -import org.opensearch.migrations.transform.JsonJMESPathTransformer; +import org.opensearch.migrations.transform.JsonJMESPathTransformerProvider; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import io.burt.jmespath.jcf.JcfRuntime; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @Slf4j @WrapWithNettyLeakDetection(disableLeakChecks = true) -class JsonTransformerTest { +class JsonJMESPathTransformerProviderTest { static final String TEST_INPUT_REQUEST = "{\n" + " \"method\": \"PUT\",\n" + " \"URI\": \"/oldStyleIndex\",\n" @@ -56,7 +55,7 @@ class JsonTransformerTest { static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>() { }; - public JsonTransformerTest() { + public JsonJMESPathTransformerProviderTest() { mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); } @@ -76,7 +75,8 @@ static String emitJson(ObjectMapper mapper, Object transformedDocument) throws J @Test public void testSimpleTransform() throws JsonProcessingException { var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); - var transformer = new JsonJMESPathTransformer(new JcfRuntime(), EXCISE_TYPE_EXPRESSION_STRING); + var transformer = new JsonJMESPathTransformerProvider().createTransformer(Map.of( + "script", EXCISE_TYPE_EXPRESSION_STRING)); var transformedDocument = transformer.transformJson(documentJson); var outputStr = emitJson(mapper, transformedDocument); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/README.md b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/README.md new file mode 100644 index 000000000..abb81c5e2 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/README.md @@ -0,0 +1,124 @@ +This transformer converts routes for various requests (see below) to indices that used +[multi-type mappings](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html) (configured from ES 5.x +and earlier clusters) to work with newer versions of Elasticsearch and OpenSearch. + +## Usage Prior to Elasticsearch 6 + +Let's start with a sample index definition (adapted from the Elasticsearch documentation) with two type mappings and +documents for each of them. +``` +PUT activity +{ + "mappings": { + "user": { + "properties": { + "name": { "type": "text" }, + "user_name": { "type": "keyword" }, + "email": { "type": "keyword" } + } + }, + "post": { + "properties": { + "content": { "type": "text" }, + "user_name": { "type": "keyword" }, + "post_at": { "type": "date" } + } + } + } +} + +PUT activity/user/someuser +{ + "name": "Some User", + "user_name": "user", + "email": "user@example.com" +} + +PUT activity/post/1 +{ + "user_name": "user", + "tweeted_at": "2024-11-13T00:00:00Z", + "content": "change is inevitable" +} + +GET activity/post/_search +{ + "query": { + "match": { + "user_name": "user" + } + } +} +``` + +## Routing data to new indices + +The structure of the documents will need to change. Some options are to use separate indices, drop some of the types +to make an index single-purpose, or to create an index that's the union of all the types' fields. + +With a simple mapping directive, we can define each of these three behaviors. The following yaml shows how to map +documents into two different indices named users and posts: +``` +activity: + user: new_users + post: new_posts +``` + +To drop one, just leave it out: +``` +activity: + user: only_users +``` + +To merge them together, use the same value: +``` +activity: + user: any_activity + post: any_activity +``` + +Any indices that are NOT specified won't be modified - all additions, changes, and queries on those other indices not +specified at the root level will remain untouched. To remove ALL the activity for a given index, specify and empty +index at the top level +``` +activity: {} +``` + +## Final Results + +``` +PUT any_activity +{ + "mappings": { + "properties": { + "type": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "user_name": { + "type": "keyword" + }, + "email": { + "type": "keyword" + }, + "content": { + "type": "text" + }, + "tweeted_at": { + "type": "date" + } + } + } +} + +PUT any_activity/_doc/someuser +{ + "name": "Some User", + "user_name": "user", + "email": "user@example.com" +} + + +``` \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/build.gradle similarity index 68% rename from transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle rename to transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/build.gradle index 4c9912f46..5a83cfa2f 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/build.gradle +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/build.gradle @@ -1,17 +1,20 @@ plugins { + id 'org.opensearch.migrations.java-library-conventions' id 'io.freefair.lombok' } dependencies { implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + api group: 'com.hubspot.jinjava', name: 'jinjava', version: "2.7.3" + + implementation group: 'com.google.guava', name: 'guava' + testImplementation project(':TrafficCapture:trafficReplayer') + testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders') testImplementation testFixtures(project(path: ':testHelperFixtures')) testImplementation testFixtures(project(path: ':TrafficCapture:trafficReplayer')) - testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' - testImplementation group: 'com.google.guava', name: 'guava' - testImplementation group: 'io.netty', name: 'netty-all' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api' testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params' testImplementation group: 'org.slf4j', name: 'slf4j-api' diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java new file mode 100644 index 000000000..2a5254799 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/JinjavaTransformer.java @@ -0,0 +1,86 @@ +package org.opensearch.migrations.transform; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.UnaryOperator; + +import org.opensearch.migrations.transform.jinjava.DynamicMacroFunction; +import org.opensearch.migrations.transform.jinjava.JavaRegexCaptureFilter; +import org.opensearch.migrations.transform.jinjava.JavaRegexReplaceFilter; +import org.opensearch.migrations.transform.jinjava.JinjavaConfig; +import org.opensearch.migrations.transform.jinjava.LogFunction; +import org.opensearch.migrations.transform.jinjava.NameMappingClasspathResourceLocator; +import org.opensearch.migrations.transform.jinjava.ThrowTag; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; +import com.hubspot.jinjava.loader.ResourceLocator; +import lombok.NonNull; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JinjavaTransformer implements IJsonTransformer { + + protected static final ObjectMapper objectMapper = new ObjectMapper(); + public static final String REGEX_REPLACEMENT_CONVERSION_PATTERNS = "regex_replacement_conversion_patterns"; + + protected final Jinjava jinjava; + protected final Function, Map> createContextWithSourceFunction; + private final String templateStr; + + public JinjavaTransformer(String templateString, + UnaryOperator> contextProviderFromSource) { + this(templateString, contextProviderFromSource, new JinjavaConfig()); + } + + public JinjavaTransformer(String templateString, + UnaryOperator> contextProviderFromSource, + @NonNull JinjavaConfig jinjavaConfig) { + this(templateString, + contextProviderFromSource, + new NameMappingClasspathResourceLocator(jinjavaConfig.getNamedScripts()), + jinjavaConfig.getRegexReplacementConversionPatterns()); + } + + public JinjavaTransformer(String templateString, + UnaryOperator> createContextWithSource, + ResourceLocator resourceLocator, + List> regexReplacementConversionPatterns) + { + jinjava = new Jinjava(); + this.createContextWithSourceFunction = createContextWithSource; + jinjava.setResourceLocator(resourceLocator); + jinjava.getGlobalContext().registerFilter(new JavaRegexCaptureFilter()); + jinjava.getGlobalContext().registerFilter(new JavaRegexReplaceFilter()); + + jinjava.getGlobalContext().registerFunction( + new ELFunctionDefinition("", "invoke_macro", DynamicMacroFunction.class, "invokeMacro", + String.class, Object[].class)); + jinjava.getGlobalContext().registerFunction( + new ELFunctionDefinition("", "log_value_and_return", LogFunction.class, "logValueAndReturn", + String.class, Object.class, Object.class)); + jinjava.getGlobalContext().registerFunction( + new ELFunctionDefinition("", "log_value", LogFunction.class, "logValue", + String.class, Object.class)); + + jinjava.getGlobalContext().registerTag(new ThrowTag()); + jinjava.getGlobalContext().put(REGEX_REPLACEMENT_CONVERSION_PATTERNS, + Optional.ofNullable(regexReplacementConversionPatterns) + .orElse(JavaRegexReplaceFilter.DEFAULT_REGEX_REPLACE_FILTER)); + this.templateStr = templateString; + } + + @SneakyThrows + @Override + public Map transformJson(Map incomingJson) { + var resultStr = jinjava.render(templateStr, createContextWithSourceFunction.apply(incomingJson)); + log.atDebug().setMessage("output from jinjava... {}").addArgument(resultStr).log(); + var parsedObj = (Map) objectMapper.readValue(resultStr, LinkedHashMap.class); + return PreservesProcessor.doFinalSubstitutions(incomingJson, parsedObj); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java new file mode 100644 index 000000000..a5d346c27 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/PreservesProcessor.java @@ -0,0 +1,56 @@ +package org.opensearch.migrations.transform; + +import java.util.List; +import java.util.Map; + +public class PreservesProcessor { + private static final String PRESERVE_KEY = "preserve"; + private static final String PRESERVE_WHEN_MISSING_KEY = "preserveWhenMissing"; + + private PreservesProcessor() {} + + @SuppressWarnings("unchecked") + public static Map doFinalSubstitutions(Map incomingJson, Map parsedObj) { + processPreserves(incomingJson, parsedObj); + + processPreserves( + (Map) incomingJson.get(JsonKeysForHttpMessage.PAYLOAD_KEY), + (Map) parsedObj.get(JsonKeysForHttpMessage.PAYLOAD_KEY) + ); + + return parsedObj; + } + + private static void processPreserves(Map source, Map target) { + if (target == null || source == null) { + return; + } + + copyValues(source, target, PRESERVE_KEY, true); + copyValues(source, target, PRESERVE_WHEN_MISSING_KEY, false); + } + + private static void copyValues(Map source, Map target, + String directiveKey, boolean forced) { + Object directive = target.remove(directiveKey); + if (directive == null) { + return; + } + + if (directive.equals("*")) { + source.forEach((key, value) -> { + if (forced || !target.containsKey(key)) { + target.put(key, value); + } + }); + } else if (directive instanceof List) { + ((List) directive).forEach(key -> { + if (source.containsKey(key) && + (forced || !target.containsKey(key))) + { + target.put(key, source.get(key)); + } + }); + } + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/DynamicMacroFunction.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/DynamicMacroFunction.java new file mode 100644 index 000000000..e91e6d0b2 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/DynamicMacroFunction.java @@ -0,0 +1,62 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.hubspot.jinjava.interpret.Context; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.fn.MacroFunction; + +public class DynamicMacroFunction { + + private DynamicMacroFunction() {} + + /** + * Called from templates through the registration in the JinjavaTransformer class + */ + public static Object invokeMacro(String macroName, Object... args) { + JinjavaInterpreter interpreter = JinjavaInterpreter.getCurrent(); + + var macro = getMacroFromContext(interpreter.getContext(), macroName); + if (macro == null) { + throw new IllegalArgumentException("Could not find argument name " + macroName); + } + + Context macroContext = new Context(interpreter.getContext()); + int argCount = Math.min(args.length, macro.getArguments().size()); + + for (int i = 0; i < argCount; i++) { + String argName = macro.getArguments().get(i); + macroContext.put(argName, args[i]); + } + + var argsMap = new HashMap(); + + var paramNames = macro.getArguments(); + Map defaults = macro.getDefaults(); + + for (int i = 0; i < paramNames.size(); i++) { + String paramName = paramNames.get(i); + if (i < args.length) { + argsMap.put(paramName, args[i]); + } else if (defaults.containsKey(paramName)) { + argsMap.put(paramName, defaults.get(paramName)); + } else { + throw new IllegalArgumentException("Missing argument for macro: " + paramName); + } + } + + return macro.doEvaluate(argsMap, Map.of(), List.of()); + } + + private static MacroFunction getMacroFromContext(Context context, String macroName) { + if (context == null) { + return null; + } + return context.getLocalMacro(macroName) + .or(() -> Optional.ofNullable(context.getGlobalMacro(macroName))) + .orElseGet(() -> getMacroFromContext(context.getParent(), macroName)); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexCaptureFilter.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexCaptureFilter.java new file mode 100644 index 000000000..f48c9955d --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexCaptureFilter.java @@ -0,0 +1,53 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.util.HashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.common.base.Function; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.filter.Filter; +import lombok.SneakyThrows; + +public class JavaRegexCaptureFilter implements Filter { + + private static LoadingCache regexCache = + CacheBuilder.newBuilder().build(CacheLoader.from((Function)Pattern::compile)); + + @SneakyThrows + private static Pattern getCompiledPattern(String pattern) { + return regexCache.get(pattern); + } + + @Override + public String getName() { + return "regex_capture"; + } + + @Override + public Object filter(Object inputObject, JinjavaInterpreter interpreter, String... args) { + if (inputObject == null || args.length < 1) { + return null; + } + + String input = inputObject.toString(); + String pattern = args[0]; + + try { + Matcher matcher = getCompiledPattern(pattern).matcher(input); + if (matcher.find()) { + var groups = new HashMap<>(); + for (int i = 0; i <= matcher.groupCount(); i++) { + groups.put("group" + i, matcher.group(i)); + } + return groups; + } + } catch (Exception e) { + return null; + } + return null; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilter.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilter.java new file mode 100644 index 000000000..035c5a001 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilter.java @@ -0,0 +1,90 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.opensearch.migrations.transform.JinjavaTransformer; + +import com.google.common.base.Function; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.filter.Filter; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JavaRegexReplaceFilter implements Filter { + + public static final List> JAVA_REGEX_REPLACE_FILTER = List.of(); + public static final List> PYTHONESQUE_REGEX_REPLACE_FILTER = List.of( + Map.entry("(\\$)", "\\\\\\$"), + Map.entry("((?:\\\\\\\\)*)(\\\\)(?=\\d)", "\\$")); + public static final List> DEFAULT_REGEX_REPLACE_FILTER = PYTHONESQUE_REGEX_REPLACE_FILTER; + + private static final LoadingCache regexCache = + CacheBuilder.newBuilder().build(CacheLoader.from((Function)Pattern::compile)); + + @SneakyThrows + private static Pattern getCompiledPattern(String pattern) { + return regexCache.get(pattern); + } + + @AllArgsConstructor + @EqualsAndHashCode + private static class ReplacementAndTransform { + String replacement; + List> substitutions; + } + + private static final LoadingCache replacementCache = + CacheBuilder.newBuilder().build(CacheLoader.from(rat -> { + var r = rat.replacement; + if (rat.substitutions != null) { + for (var kvp : rat.substitutions) { + r = r.replaceAll(kvp.getKey(), kvp.getValue()); + } + } + return r; + })); + + + @Override + public String getName() { + return "regex_replace"; + } + + @Override + public Object filter(Object inputObject, JinjavaInterpreter interpreter, String... args) { + if (inputObject == null || args.length < 2) { + return null; + } + + String input = inputObject.toString(); + String pattern = args[0]; + String replacement = args[1]; + + String rewritten = null; + try { + Matcher matcher = getCompiledPattern(pattern).matcher(input); + rewritten = replacementCache.get( + new ReplacementAndTransform(replacement, + Optional.ofNullable(interpreter) + .flatMap(ji->Optional.ofNullable(ji.getContext())) + .flatMap(c-> Optional.ofNullable((List>) + c.get(JinjavaTransformer.REGEX_REPLACEMENT_CONVERSION_PATTERNS))) + .orElse(DEFAULT_REGEX_REPLACE_FILTER))); + var rval = matcher.replaceAll(rewritten); + log.atTrace().setMessage("replaced value {} with {}").addArgument(input).addArgument(rval).log(); + return rval; + } catch (Exception e) { + throw new RegexReplaceException(e, input, pattern, replacement, rewritten); + } + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JinjavaConfig.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JinjavaConfig.java new file mode 100644 index 000000000..e8ddf58f6 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/JinjavaConfig.java @@ -0,0 +1,20 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class JinjavaConfig { + @JsonProperty("regexReplacementConversionPatterns") + private List> regexReplacementConversionPatterns; + + @JsonProperty("regexReplacementConversionPatterns") + Map namedScripts; +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/LogFunction.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/LogFunction.java new file mode 100644 index 000000000..4f8ea6c5b --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/LogFunction.java @@ -0,0 +1,28 @@ +package org.opensearch.migrations.transform.jinjava; + +import lombok.extern.slf4j.Slf4j; +import org.slf4j.event.Level; + +@Slf4j +public class LogFunction { + + /** + * Called from templates through the registration in the JinjavaTransformer class + */ + public static Object logValueAndReturn(String levelStr, Object valueToLog, Object valueToReturn) { + Level level; + try { + level = Level.valueOf(levelStr); + } catch (IllegalArgumentException e) { + log.atError().setMessage("Could not parse the level as it was passed in, so using ERROR. Level={}") + .addArgument(levelStr).log(); + level = Level.ERROR; + } + log.atLevel(level).setMessage("{}").addArgument(valueToLog).log(); + return valueToReturn; + } + + public static void logValue(String level, Object valueToLog) { + logValueAndReturn(level, valueToLog, null); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/NameMappingClasspathResourceLocator.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/NameMappingClasspathResourceLocator.java new file mode 100644 index 000000000..e69fcc2ff --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/NameMappingClasspathResourceLocator.java @@ -0,0 +1,80 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.io.Resources; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.loader.ClasspathResourceLocator; +import com.hubspot.jinjava.loader.ResourceNotFoundException; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NameMappingClasspathResourceLocator extends ClasspathResourceLocator { + final Map overrideResourceMap; + + @AllArgsConstructor + @Getter + @EqualsAndHashCode + private static class ResourceCacheKey { + private String fullName; + private Charset encoding; + } + + private final LoadingCache resourceCache = CacheBuilder.newBuilder() + .build(new CacheLoader<>() { + @Override + public String load(ResourceCacheKey key) throws IOException { + try { + String versionedName = getDefaultVersion("jinjava/" + key.getFullName()); + return Resources.toString(Resources.getResource(versionedName), key.getEncoding()); + } catch (IllegalArgumentException e) { + throw new ResourceNotFoundException("Couldn't find resource: " + key.getFullName()); + } + } + }); + + public NameMappingClasspathResourceLocator(Map overrideResourceMap) { + this.overrideResourceMap = Optional.ofNullable(overrideResourceMap).orElse(Map.of()); + } + + private static String getDefaultVersion(final String fullName) throws IOException { + try { + var versionFile = fullName + "/defaultVersion"; + var versionLines = Resources.readLines(Resources.getResource(versionFile), StandardCharsets.UTF_8).stream() + .filter(s->!s.isEmpty()) + .collect(Collectors.toList()); + if (versionLines.size() == 1) { + return fullName + "/" + versionLines.get(0).trim(); + } + throw new IllegalStateException("Expected defaultVersion resource to contain a single line with a name"); + } catch (IllegalArgumentException e) { + log.atTrace().setCause(e).setMessage("Caught ResourceNotFoundException, but this is expected").log(); + } + return fullName; + } + + @Override + public String getString(String fullName, Charset encoding, JinjavaInterpreter interpreter) throws IOException { + var overrideResource = overrideResourceMap.get(fullName); + if (overrideResource != null) { + return overrideResource; + } + try { + return resourceCache.get(new ResourceCacheKey(fullName, encoding)); + } catch (ExecutionException e) { + throw new IOException("Failed to get resource content named `" + fullName + "`from cache", e); + } + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexReplaceException.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexReplaceException.java new file mode 100644 index 000000000..7931e820d --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexReplaceException.java @@ -0,0 +1,31 @@ +package org.opensearch.migrations.transform.jinjava; + +import java.util.StringJoiner; + +import lombok.Getter; + +@Getter +public class RegexReplaceException extends RuntimeException { + final String input; + final String pattern; + final String replacement; + final String rewrittenReplacement; + + public RegexReplaceException(Throwable cause, String input, String pattern, String replacement, String rewrittenReplacement) { + super(cause); + this.input = input; + this.pattern = pattern; + this.replacement = replacement; + this.rewrittenReplacement = rewrittenReplacement; + } + + @Override + public String getMessage() { + return super.getMessage() + + new StringJoiner(", ", "{", "}") + .add("input='" + input + "'") + .add("pattern='" + pattern + "'") + .add("replacement='" + replacement + "'") + .add("rewrittenReplacement='" + rewrittenReplacement + "'"); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/ThrowTag.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/ThrowTag.java new file mode 100644 index 000000000..b4e314181 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/ThrowTag.java @@ -0,0 +1,46 @@ +package org.opensearch.migrations.transform.jinjava; + +import com.hubspot.jinjava.doc.annotations.JinjavaDoc; +import com.hubspot.jinjava.doc.annotations.JinjavaParam; +import com.hubspot.jinjava.doc.annotations.JinjavaSnippet; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.tag.Tag; +import com.hubspot.jinjava.tree.TagNode; + +@JinjavaDoc( + value = "Throws a runtime exception with the specified message", + params = { + @JinjavaParam(value = "message", type = "string", desc = "The error message to throw") + }, + snippets = { + @JinjavaSnippet( + code = "{% throw 'Invalid input provided' %}" + ) + } +) + +public class ThrowTag implements Tag { + private static final String TAG_NAME = "throw"; + + @Override + public String getName() { + return TAG_NAME; + } + + @Override + public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) { + String message = interpreter.render(tagNode.getHelpers().trim()); + throw new JinjavaThrowTagException(message); + } + + public static class JinjavaThrowTagException extends RuntimeException { + public JinjavaThrowTagException(String message) { + super(message); + } + } + + @Override + public String getEndTagName() { + return null; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/featureEnabled.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/featureEnabled.j2 new file mode 100644 index 000000000..16c7f8d0f --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/featureEnabled.j2 @@ -0,0 +1,21 @@ +{%- macro is_enabled(features, path) -%} + {%- if features == None -%} + true + {%- else -%} + {%- set ns = namespace(value=features) -%} + {%- for key in (path | split('.')) -%} + {%- if ns.value is mapping and key in ns.value -%} + {%- set ns.value = ns.value[key] -%} + {%- else -%} + {%- set ns.value = None -%} + {%- endif -%} + {%- endfor -%} + {%- if ns.value is boolean and ns.value -%} + true + {%- elif ns.value is mapping and ns.value.get('enabled') is boolean -%} + {{- ns.value.get('enabled') -}} + {%- else -%} + false + {%- endif -%} + {%- endif -%} +{%- endmacro -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route.j2 new file mode 100644 index 000000000..f1d606d41 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/resources/jinjava/common/route.j2 @@ -0,0 +1,22 @@ +{%- import "common/featureEnabled.j2" as fscope -%} + +{%- macro route(input, field_to_match, feature_flags, default_action, routes) -%} + {%- set ns = namespace(result=none, matched=false) -%} + {%- for pattern, action_fn, feature_name_param in routes if not ns.matched -%} + {%- set feature_name = feature_name_param | default(action_fn) -%} + {%- if not ns.matched -%} {# we haven't found a match yet, otherwise skip the rest #} + {%- set match = field_to_match | regex_capture(pattern) -%} + {%- if match is not none -%} + {%- set ns.matched = true -%} + {%- if fscope.is_enabled(feature_flags, feature_name) -%} + {%- set ns.result = invoke_macro(action_fn, match, input) -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + {%- if ns.result is none -%} + {{- invoke_macro(default_action, input) -}} + {%- else -%} + {{- ns.result -}} + {%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java new file mode 100644 index 000000000..80a01b102 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/FeatureMaskingTest.java @@ -0,0 +1,82 @@ +package org.opensearch.migrations.transform; + +import java.io.IOException; +import java.util.Map; + +import org.opensearch.migrations.transform.flags.FeatureFlags; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; + +@Slf4j +public class FeatureMaskingTest { + private static final ObjectMapper objectMapper = new ObjectMapper(); + public Map transformForMask(Map incomingFlags) throws IOException { + var incomingFlagStr = objectMapper.writeValueAsString(incomingFlags); + log.atInfo().setMessage("incoming map as string: {}").addArgument(incomingFlagStr).log(); + var flags = FeatureFlags.parseJson(incomingFlagStr); + log.atInfo().setMessage("parsed flags: {}").addArgument(flags == null ? "[NULL]" : flags.writeJson()).log(); + final var template = "" + + "{%- import \"common/featureEnabled.j2\" as fscope -%}" + + " { " + + "{%- set ns = namespace(debug_info=['list: ']) -%}" + + "{%- set ns.debug_info = ['list: '] -%}" + + "\"enabledFlags\": \"" + + "{{- fscope.is_enabled(features,'testFlag') -}}," + + "{{- fscope.is_enabled(features,'testFlag.t1') -}}," + + "{{- fscope.is_enabled(features,'testFlag.t2') -}}," + + "{{- fscope.is_enabled(features,'nextTestFlag.n1') -}}" + + "\"" + + "}"; + var transformed = new JinjavaTransformer(template, + src -> flags == null ? Map.of() : Map.of("features", flags)); + return transformed.transformJson(Map.of()); + } + + @Test + public void testFalseFlag() throws Exception { + Assertions.assertEquals( + "false,false,false,false", + transformForMask(Map.of("testFlag", false)).get("enabledFlags")); + } + + @Test + public void testTrueFlag() throws Exception { + Assertions.assertEquals( + "true,false,false,false", + transformForMask(Map.of("testFlag", true)).get("enabledFlags")); + } + + @Test + public void testLongerFalseFlag() throws Exception { + Assertions.assertEquals( + "false,false,false,false", + transformForMask(Map.of("testFlag", false)).get("enabledFlags")); + } + + @Test + public void testLongerTrueFlag() throws Exception { + Assertions.assertEquals( + "true,false,false,false", + transformForMask(Map.of( + "testFlag", Map.of("notPresent", false)) + ).get("enabledFlags")); + } + + @Test + public void testImplicitTrueFlag() throws Exception { + Assertions.assertEquals( + "true,true,false,false", + transformForMask(Map.of( + "testFlag", Map.of("t1", true)) + ).get("enabledFlags")); + } + + @Test + public void testNullFeatures() throws Exception { + Assertions.assertEquals( + "true,true,true,true", transformForMask(null).get("enabledFlags")); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java new file mode 100644 index 000000000..c45733eaf --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/JinjavaTransformerTest.java @@ -0,0 +1,133 @@ +package org.opensearch.migrations.transform; + +import java.util.Map; +import java.util.stream.Collectors; + +import org.opensearch.migrations.testutils.CloseableLogSetup; +import org.opensearch.migrations.testutils.JsonNormalizer; +import org.opensearch.migrations.transform.jinjava.JinjavaConfig; +import org.opensearch.migrations.transform.jinjava.LogFunction; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; + + +@Slf4j +class JinjavaTransformerTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final String INDEX_TYPE_MAPPING_SAMPLE_TEMPLATE = "" + + "{# First, parse the URI to check if it matches the pattern we want to transform #}\n" + + "{% set uri_parts = request.uri.split('/') %}\n" + + "{% set is_type_request = uri_parts | length == 2 %}\n" + + "{% set is_doc_request = uri_parts | length == 3 %}\n" + + "\n" + + "{# If this is a document request, check if we need to transform it based on mapping #}\n" + + "{% if is_doc_request and uri_parts[0] in index_mappings and uri_parts[1] in index_mappings[uri_parts[0]] %}\n" + + " {# This is a document request that needs transformation #}\n" + + " {\n" + + " \"verb\": \"{{ request.verb }}\",\n" + + " \"uri\": \"{{ index_mappings[uri_parts[0]][uri_parts[1]] }}/_doc/{{ uri_parts[2] }}\",\n" + + " \"body\": {{ request.body | tojson }}\n" + + " }\n" + + "{% elif is_type_request and uri_parts[0] in index_mappings %}\n" + + " {# This is an index creation request that needs transformation #}\n" + + " {\n" + + " \"verb\": \"{{ request.verb }}\",\n" + + " \"uri\": \"{{ index_mappings[uri_parts[0]][uri_parts[1]] }}\",\n" + + " \"body\": {\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"type\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " {%- for type_name, type_props in request.body.mappings.items() %}\n" + + " {%- for prop_name, prop_def in type_props.properties.items() %}\n" + + " ,\n" + + " \"{{ prop_name }}\": {{ prop_def | tojson }}\n" + + " {%- endfor %}\n" + + " {%- endfor %}\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "{% else %}\n" + + " {# Pass through any requests that don't match our transformation patterns #}\n" + + " {{ request | tojson }}\n" + + "{% endif %}"; + + @Test + public void testTypeMappingSample() throws Exception { + var testString = + "{\n" + + " \"verb\": \"PUT\",\n" + + " \"uri\": \"indexA/type2/someuser\",\n" + + " \"body\": {\n" + + " \"name\": \"Some User\",\n" + + " \"user_name\": \"user\",\n" + + " \"email\": \"user@example.com\"\n" + + " }\n" + + "}"; + var indexMappings = Map.of( + "indexA", Map.of( + "type1", "indexA_1", + "type2", "indexA_2"), + "indexB", Map.of( + "type1", "indexB", + "type2", "indexB"), + "indexC", Map.of( + "type2", "indexC")); + var indexTypeMappingRewriter = new JinjavaTransformer(INDEX_TYPE_MAPPING_SAMPLE_TEMPLATE, + request -> Map.of( + "index_mappings", indexMappings, + "request", request), + new JinjavaConfig(null, + Map.of("hello", "{%- macro hello() -%} hi {%- endmacro -%}\n"))); + + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, Map.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString.replace("indexA/type2/", "indexA_2/_doc/")), + JsonNormalizer.fromObject(resultObj)); + } + + @Test + public void testCustomScript() throws Exception { + var indexTypeMappingRewriter = new JinjavaTransformer("" + + "{%- include \"hello\" -%}" + + "{{invoke_macro('hello')}}", + request -> Map.of("request", request), + new JinjavaConfig(null, + Map.of("hello", "{%- macro hello() -%}{\"hi\": \"world\"}{%- endmacro -%}\n"))); + + var resultObj = indexTypeMappingRewriter.transformJson(Map.of()); + var resultStr = OBJECT_MAPPER.writeValueAsString(resultObj); + Assertions.assertEquals("{\"hi\":\"world\"}", resultStr); + } + + @Test + public void debugLoggingWorks() throws Exception { + try (var closeableLogSetup = new CloseableLogSetup(LogFunction.class.getName())) { + final String FIRST_LOG_VAL = "LOGGED_VALUE=16"; + final String SECOND_LOG_VAL = "next one"; + final String THIRD_LOG_VAL = "LAST"; + + var indexTypeMappingRewriter = new JinjavaTransformer("" + + "{{ log_value_and_return('ERROR', log_value_and_return('ERROR', '" + FIRST_LOG_VAL + "', '" + SECOND_LOG_VAL + "'), '') }}" + + "{{ log_value('ERROR', '" + THIRD_LOG_VAL + "') }} " + + "{}", + request -> Map.of("request", request), + new JinjavaConfig(null, + Map.of("hello", "{%- macro hello() -%}{\"hi\": \"world\"}{%- endmacro -%}\n"))); + + var resultObj = indexTypeMappingRewriter.transformJson(Map.of()); + var resultStr = OBJECT_MAPPER.writeValueAsString(resultObj); + Assertions.assertEquals("{}", resultStr); + + var logEvents = closeableLogSetup.getLogEvents(); + Assertions.assertEquals(String.join("\n", new String[]{FIRST_LOG_VAL, SECOND_LOG_VAL, THIRD_LOG_VAL}), + logEvents.stream().collect(Collectors.joining("\n"))); + } + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java new file mode 100644 index 000000000..e287bcbba --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/RouteTest.java @@ -0,0 +1,82 @@ +package org.opensearch.migrations.transform; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +@Slf4j +public class RouteTest { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public Map doRouting(Map flags, Map inputDoc) { + log.atInfo().setMessage("parsed flags: {}").addArgument(flags).log(); + final var template = "" + + "{%- macro doDefault(ignored_input) -%}" + + " {}" + + "{%- endmacro -%}\n" + + + "{% macro echoFirstMatch(matches, input) %}\n" + + " { \"matchedVal\": \"{{ matches['group1'] }}\"}" + + "{% endmacro %}" + + "{% macro echoFirstMatchAgain(matches, input) %}\n" + + " { \"again\": \"{{ matches['group1'] }}\"}" + + "{% endmacro %}" + + "{% macro switchStuff(matches, input) %}\n" + + " {% set swapped_list = [input.stuff[1], input.stuff[0]] %}" + + " {% set input = input + {'stuff': swapped_list} %}" + + " {{ input | tojson }} " + + "{% endmacro %}" + + + "{%- import \"common/route.j2\" as rscope -%}" + + "{{- rscope.route(source, source.label, flags, 'doDefault'," + + " [" + + " ('Thing_A(.*)', 'echoFirstMatch', 'matchA')," + + " ('Thing_A(.*)', 'echoFirstMatchAgain', 'matchA')," + // make sure that we don't get duplicate results + " ('B(.*)', 'switchStuff', 'matchB')" + + " ])" + + "-}}"; + + var transformed = new JinjavaTransformer(template, + src -> flags == null ? Map.of("source", inputDoc) : Map.of("source", inputDoc, "flags", flags)); + return transformed.transformJson(inputDoc); + } + + @Test + public void test() throws IOException { + var flagAOff = Map.of( + "matchA", false, + "matchB", (Object) true); + var docA = Map.of( + "label", "Thing_A_and more!", + "stuff", Map.of( + "inner1", "data1", + "inner2", "data2" + )); + var docB = Map.of( + "label", "B-hive", + "stuff", List.of( + "data1", + "data2" + )); + { + var resultMap = doRouting(null, docA); + Assertions.assertEquals(1, resultMap.size()); + Assertions.assertEquals("_and more!", resultMap.get("matchedVal")); + } + { + var resultMap = doRouting(flagAOff, docA); + Assertions.assertTrue(resultMap.isEmpty()); + } + { + var resultMap = doRouting(flagAOff, docB); + Assertions.assertEquals("{\"label\":\"B-hive\",\"stuff\":[\"data2\",\"data1\"]}", + objectMapper.writeValueAsString(new TreeMap<>(resultMap))); + } + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilterTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilterTest.java new file mode 100644 index 000000000..0b310bf8e --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/test/java/org/opensearch/migrations/transform/jinjava/JavaRegexReplaceFilterTest.java @@ -0,0 +1,29 @@ +package org.opensearch.migrations.transform.jinjava; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class JavaRegexReplaceFilterTest { + JavaRegexReplaceFilter replaceFilter = new JavaRegexReplaceFilter(); + + String makeReplacementFromKnownMatch(String replacementPattern) { + final var source = "Known Pattern 1234, and a note."; + final var capturingPattern = "(([^ \\d]*) )*(\\d*)[^\\d]*(no.e)\\."; + return (String) replaceFilter.filter(source, null, capturingPattern, replacementPattern); + } + + @Test + public void test() { + Assertions.assertEquals("somethingNew", makeReplacementFromKnownMatch("somethingNew")); + Assertions.assertEquals("Pattern$amount", makeReplacementFromKnownMatch("\\2$amount")); + Assertions.assertEquals("Pattern$1", makeReplacementFromKnownMatch("\\2$1")); + + // other things to try +// Assertions.assertEquals("Pattern\\$$1", makeReplacementFromKnownMatch("\\2\\$$\\1")); +// Assertions.assertEquals("$1\\$amount", "\\1$amount", replacement); +// "\\\\1$50", // -> \\1\$50 +// "\\\\\\1$total$", // -> \\$1\$total\$ +// "cost$1$price$2" // -> cost$1\$price$2 + + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/build.gradle b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/build.gradle new file mode 100644 index 000000000..4bd547e57 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/build.gradle @@ -0,0 +1,33 @@ +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + +plugins { + id 'io.freefair.lombok' +} + +dependencies { + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJinjavaTransformer') + + implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.4' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + + testImplementation project(':TrafficCapture:trafficReplayer') + testImplementation testFixtures(project(path: ':testHelperFixtures')) + testImplementation testFixtures(project(path: ':TrafficCapture:trafficReplayer')) + + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + testImplementation group: 'io.netty', name: 'netty-all' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params' + testImplementation group: 'org.slf4j', name: 'slf4j-api' + testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine' +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJinjavaTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJinjavaTransformerProvider.java new file mode 100644 index 000000000..c579e8efc --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonJinjavaTransformerProvider.java @@ -0,0 +1,55 @@ +package org.opensearch.migrations.transform; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import org.opensearch.migrations.transform.jinjava.JinjavaConfig; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.map.CompositeMap; + + +public class JsonJinjavaTransformerProvider implements IJsonTransformerProvider { + + public static final String REQUEST_KEY = "request"; + public static final String TEMPLATE_KEY = "template"; + public static final String JINJAVA_CONFIG_KEY = "jinjavaConfig"; + + public final static ObjectMapper mapper = new ObjectMapper(); + + @Override + public IJsonTransformer createTransformer(Object jsonConfig) { + if (!(jsonConfig instanceof Map)) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + var config = (Map) jsonConfig; + if (config.containsKey(REQUEST_KEY)) { + throw new IllegalArgumentException(REQUEST_KEY + " was already present in the incoming configuration. " + + getConfigUsageStr()); + } + if (!config.containsKey(TEMPLATE_KEY)) { + throw new IllegalArgumentException(TEMPLATE_KEY + " was not present in the incoming configuration. " + + getConfigUsageStr()); + } + + var immutableBaseConfig = Collections.unmodifiableMap(config); + try { + var templateString = (String) config.get(TEMPLATE_KEY); + return new JinjavaTransformer(templateString, + source -> new CompositeMap<>(Map.of(REQUEST_KEY, source), immutableBaseConfig), + Optional.ofNullable(config.get(JINJAVA_CONFIG_KEY)).map(jinjavaConfig -> + mapper.convertValue(jinjavaConfig, JinjavaConfig.class)).orElse(new JinjavaConfig())); + } catch (ClassCastException e) { + throw new IllegalArgumentException(getConfigUsageStr(), e); + } + } + + private String getConfigUsageStr() { + return this.getClass().getName() + " expects the incoming configuration to be a Map " + + "with a '" + TEMPLATE_KEY + "' key that specifies the Jinjava template. " + + "The key '" + REQUEST_KEY + "' must not be specified so that it can be used to pass the source document " + + "into the template. " + + "The other top-level keys will be passed directly to the template."; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 000000000..8d65ec87b --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.JsonJinjavaTransformerProvider \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java new file mode 100644 index 000000000..8687bf75d --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/java/org/opensearch/migrations/replay/JinjavaTransformerProviderTest.java @@ -0,0 +1,107 @@ +package org.opensearch.migrations.replay; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.transform.JsonJinjavaTransformerProvider; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +@Slf4j +@WrapWithNettyLeakDetection(disableLeakChecks = true) +class JinjavaTransformerProviderTest { + static final String TEST_INPUT_REQUEST = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/oldStyleIndex\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": {\n" + + " \"oldType\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}\n"; + + static final String EXCISE_TYPE_EXPRESSION_STRING = "{\n" + + " \"method\": \"{{ request.method }}\",\n" + + " \"URI\": \"{{ request.URI }}\",\n" + + " \"headers\": {{ request.headers | tojson }},\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": {{ request.payload.inlinedJsonBody.mappings.oldType | tojson }}\n" + + " }\n" + + " }\n" + + "}"; + + ObjectMapper mapper = new ObjectMapper(); + static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>() { + }; + + public JinjavaTransformerProviderTest() { + mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); + } + + static Map parseStringAsJson(ObjectMapper mapper, String jsonStr) throws JsonProcessingException { + return mapper.readValue(jsonStr, TYPE_REFERENCE_FOR_MAP_TYPE); + } + + static String normalize(ObjectMapper mapper, String input) throws JsonProcessingException { + return mapper.writeValueAsString(mapper.readTree(input)); + } + + static String emitJson(ObjectMapper mapper, Object transformedDocument) throws JsonProcessingException { + mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); // optional + return mapper.writeValueAsString(transformedDocument); + } + + @Test + public void testSimpleTransform() throws JsonProcessingException { + var documentJson = parseStringAsJson(mapper, TEST_INPUT_REQUEST); + var transformer = new JsonJinjavaTransformerProvider().createTransformer(Map.of( + "template", EXCISE_TYPE_EXPRESSION_STRING)); + var transformedDocument = transformer.transformJson(documentJson); + var outputStr = emitJson(mapper, transformedDocument); + + final String TEST_OUTPUT_REQUEST = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/oldStyleIndex\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"field2\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Assertions.assertEquals(normalize(mapper, TEST_OUTPUT_REQUEST), normalize(mapper, outputStr)); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/log4j2.properties b/transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/resources/log4j2.properties similarity index 100% rename from transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/log4j2.properties rename to transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformerProvider/src/test/resources/log4j2.properties diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java index c406b15d9..402eea157 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/IJsonTransformerProvider.java @@ -7,6 +7,7 @@ public interface IJsonTransformerProvider { * Create a new transformer from the given configuration. This transformer * will be used repeatedly and concurrently from different threads to modify * messages. + * * @param jsonConfig is a List, Map, String, or null that should be used to configure the * IJsonTransformer that is being created * @return diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java index 65f0a5894..3ab644e30 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/java/org/opensearch/migrations/transform/JsonKeysForHttpMessage.java @@ -12,6 +12,21 @@ private JsonKeysForHttpMessage() {} public static final String PROTOCOL_KEY = "protocol"; public static final String HEADERS_KEY = "headers"; public static final String PAYLOAD_KEY = "payload"; + + /** + *

This key is valid at the top level and as a direct within a "payload" value. + * After a transformation has completed, these objects will be excised and replaced with + * the original nodes from the original document.

+ * + *

For example. the following will cause the original payload to be preserved.

+ * `"preserve": [ "payload" ]` + *

The following will cause the original headers and payload to be preserved.

+ * `"preserve": [ "headers", "payload" ]` + * + *

Notice that any already existing items will be replaced.

+ */ + public static final String PRESERVE_KEY = "preserve"; + /** * This is the key under the 'payload' object whose value is the parsed json from the HTTP message payload. * Notice that there aren't yet other ways to access the payload contents. If the content-type was not json, @@ -19,11 +34,13 @@ private JsonKeysForHttpMessage() {} */ public static final String INLINED_JSON_BODY_DOCUMENT_KEY = "inlinedJsonBody"; /** - * for the type application + * Like INLINED_JSON_BODY_DOCUMENT_KEY, this key is used directly under the 'payload' object. Its value + * will be a list of json documents that represent the lines of the original ndjson payload, when present. */ public static final String INLINED_NDJSON_BODIES_DOCUMENT_KEY = "inlinedJsonSequenceBodies"; /** - * This maps to a ByteBuf that is owned by the caller. + * Like INLINED_JSON_BODY_DOCUMENT_KEY, this key is used directly under the 'payload' object. Its value + * maps to a ByteBuf that is owned by the caller. * Any consumers should retain if they need to access it later. This may be UTF8, UTF16 encoded, or something else. */ public static final String INLINED_BINARY_BODY_DOCUMENT_KEY = "inlinedBinaryBody"; diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/resources/META-INF/services/org.opensearch.migrations.transform.ITextTransformerProvider b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerInterface/src/main/resources/META-INF/services/org.opensearch.migrations.transform.ITextTransformerProvider new file mode 100644 index 000000000..e69de29bb diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle index 86f260503..d0ba8e706 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/build.gradle @@ -31,7 +31,7 @@ dependencies { testImplementation testFixtures(project(path: ':testHelperFixtures')) testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformerProvider') testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformerProvider') - testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider') + testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider') testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJoltMessageTransformer') testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJMESPathMessageTransformer') diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java index c410b9595..93d6c3d00 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/JsonConditionalTransformerProvider.java @@ -9,9 +9,6 @@ @Slf4j public class JsonConditionalTransformerProvider implements IJsonTransformerProvider { - public JsonConditionalTransformerProvider() { - } - @Override @SneakyThrows public IJsonTransformer createTransformer(Object jsonConfig) { diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java index 136815c7b..1f30ae7ad 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/TransformationLoader.java @@ -148,7 +148,11 @@ private static class HostTransformer implements IJsonTransformer { @Override public Map transformJson(Map incomingJson) { var headers = (Map) incomingJson.get(JsonKeysForHttpMessage.HEADERS_KEY); - headers.replace("host", newHostName); + if (headers != null) { + headers.replace("host", newHostName); + } else { + log.atDebug().setMessage("Host header is null in incoming message: {}").addArgument(incomingJson).log(); + } return incomingJson; } } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlags.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlags.java new file mode 100644 index 000000000..d415c18f7 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlags.java @@ -0,0 +1,38 @@ +package org.opensearch.migrations.transform.flags; + +import java.io.IOException; +import java.util.HashMap; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@JsonDeserialize(using = FeatureFlagsDeserializer.class) +public class FeatureFlags extends HashMap { + + public static final String ENABLED_KEY = "enabled"; + + // Static ObjectMappers for JSON and YAML + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + + // Parsing methods + public static FeatureFlags parseJson(String contents) throws IOException { + return jsonMapper.readValue(contents, FeatureFlags.class); + } + + public String writeJson() throws IOException { + return jsonMapper.writeValueAsString(this); + } + + @Override + public String toString() { + return "FeatureFlags{" + + "enabled=" + get(ENABLED_KEY) + + ", features=" + super.toString() + + '}'; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlagsDeserializer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlagsDeserializer.java new file mode 100644 index 000000000..47220c1cb --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/main/java/org/opensearch/migrations/transform/flags/FeatureFlagsDeserializer.java @@ -0,0 +1,50 @@ +package org.opensearch.migrations.transform.flags; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FeatureFlagsDeserializer extends StdDeserializer { + + public FeatureFlagsDeserializer() { + this(null); + } + + public FeatureFlagsDeserializer(Class vc) { + super(vc); + } + + @Override + public FeatureFlags deserialize(JsonParser p, DeserializationContext ctx) throws IOException { + FeatureFlags featureFlags = new FeatureFlags(); + JsonNode node = p.getCodec().readTree(p); + + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + final var key = entry.getKey(); + final var value = entry.getValue(); + if (value.isObject()) { + JsonParser valueParser = value.traverse(); + valueParser.setCodec(p.getCodec()); + featureFlags.put(key, ctx.readValue(valueParser, FeatureFlags.class)); + } else { + featureFlags.put(key, Map.of(FeatureFlags.ENABLED_KEY, value.booleanValue())); + } + } + // If 'enabled' is not explicitly set, default to true + if (!featureFlags.containsKey(FeatureFlags.ENABLED_KEY)) { + featureFlags.put(FeatureFlags.ENABLED_KEY, true); + } else if (!(featureFlags.get(FeatureFlags.ENABLED_KEY) instanceof Boolean)) { + throw new IllegalArgumentException("enabled key must map to a boolean value"); + } + return featureFlags; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java index 3434c31e8..5a894d4cb 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/JsonTransformerTest.java @@ -43,7 +43,6 @@ private Map parseSampleRequestFromResource(String path) { } private String emitJson(Object transformedDocument) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_COMMENTS, true); // optional return mapper.writeValueAsString(transformedDocument); } diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java index 088015583..fca6ecff8 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/MultipleJMESPathScriptsTest.java @@ -1,11 +1,9 @@ package org.opensearch.migrations.transform.replay; -import java.util.Map; import java.util.StringJoiner; import org.opensearch.migrations.transform.TransformationLoader; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -18,11 +16,6 @@ public class MultipleJMESPathScriptsTest { + "\\\"headers\\\": {\\\"host\\\": \\\"localhost\\\"},\\n \\\"payload\\\": payload\\n}"; private static final ObjectMapper mapper = new ObjectMapper(); - private static Map parseAsMap(String contents) throws Exception { - return mapper.readValue(contents.getBytes(), new TypeReference<>() { - }); - } - @Test public void testTwoScripts() throws Exception { var aggregateScriptJoiner = new StringJoiner(",\n", "[", "]"); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java index 47e93b320..e0519de63 100644 --- a/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonMessageTransformerLoaders/src/test/java/org/opensearch/migrations/transform/replay/TransformationLoaderTest.java @@ -44,10 +44,11 @@ public void testThatSimpleNoopTransformerLoads() throws Exception { } @Test - public void testMisconfiguration() throws Exception { + public void testMisconfiguration() { + var transformLoader = new TransformationLoader(); Assertions.assertThrows( IllegalArgumentException.class, - () -> new TransformationLoader().getTransformerFactoryLoader("localhost", null, "Not right") + () -> transformLoader.getTransformerFactoryLoader("localhost", null, "Not right") ); } @@ -61,7 +62,7 @@ public void testThatNoConfigMeansNoThrow() throws Exception { Assertions.assertNotNull(transformer.transformJson(origDoc)); } - final String TEST_INPUT_REQUEST = "{\n" + static final String TEST_INPUT_REQUEST = "{\n" + " \"method\": \"PUT\",\n" + " \"URI\": \"/oldStyleIndex\",\n" + " \"headers\": {\n" @@ -75,9 +76,7 @@ public void testUserAgentAppends() throws Exception { var userAgentTransformer = new TransformationLoader().getTransformerFactoryLoader("localhost", "tester", null); var origDoc = parseAsMap(TEST_INPUT_REQUEST); - var origDocStr = mapper.writeValueAsString(origDoc); var pass1 = userAgentTransformer.transformJson(origDoc); - var pass1DocStr = mapper.writeValueAsString(origDoc); var pass2 = userAgentTransformer.transformJson(pass1); var finalUserAgentInHeaders = ((Map) pass2.get("headers")).get("user-agent"); Assertions.assertEquals("tester; tester", finalUserAgentInHeaders); diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/README.md b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/README.md new file mode 100644 index 000000000..b47b75e54 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/README.md @@ -0,0 +1,155 @@ +This transformer converts routes for various requests (see below) to indices that used +[multi-type mappings](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html) (configured from ES 5.x +and earlier clusters) to work with newer versions of Elasticsearch and OpenSearch. +See "[Removal of Type Mappings](https://www.elastic.co/guide/en/elasticsearch/reference/7.10/removal-of-types.html)" +to understand how type mappings are treated differently through different versions of Elasticsearch. + +## Usage Prior to Elasticsearch 6 + +Let's start with a sample index definition (adapted from the Elasticsearch documentation) with two type mappings and +documents for each of them. +``` +PUT activity +{ + "mappings": { + "user": { + "properties": { + "name": { "type": "text" }, + "user_name": { "type": "keyword" }, + "email": { "type": "keyword" } + } + }, + "post": { + "properties": { + "content": { "type": "text" }, + "user_name": { "type": "keyword" }, + "post_at": { "type": "date" } + } + } + } +} + +PUT activity/user/someuser +{ + "name": "Some User", + "user_name": "user", + "email": "user@example.com" +} + +PUT activity/post/1 +{ + "user_name": "user", + "tweeted_at": "2024-11-13T00:00:00Z", + "content": "change is inevitable" +} + +GET activity/post/_search +{ + "query": { + "match": { + "user_name": "user" + } + } +} +``` + +## Routing data to new indices + +The structure of the documents and indices need to change. Options are to use separate indices, drop some of +the types to make an index single-purpose, or to create an index that's the union of all the types' fields. + +Specific instances of those behaviors can be expressed via a map (or dictionary) or indices to types to target indices. +The following sample json shows how to map documents from the 'activity' index into two different indices +('users' and 'posts'): +``` +{ + "activity": { + "user: "new_users", + "post": "new_posts" + } +``` + +To drop the 'post' type, just leave it out: +``` +{ + "activity": { + "user": "only_users" + } +} +``` + +To merge types into a single index, use the same value: +``` +{ + "activity": { + "user": "any_activity", + "post": "any_activity", + } +} +``` + +To remove ALL the activity for a given index, specify an index with no children types. +``` +{ + "activity": {} +} +``` + +Those regex rules take precedence **after** the static mappings specified above. + +In addition to static source/target mappings, users can specify source and type pairs as regex patterns and +use captured groups in the target index name. +Any source _indices_ that are NOT specified in the maps will be processed through the regex route rules. +The regex rules are only applied if the source index doesn't match a key in the static route map + +Regex replace rules are evaluated by concatenating the source index and source types into a single string. +The pattern components are also concatenated into a corresponding match string. +The replacement value will replace the _matched_ part of the source index + typename and replace it with the +specified value. +If that specified value contains (numerical) backreferences, those will pull from the captured groups of the +concatenated pattern. +The concatenated pattern is the index pattern followed by the type pattern, meaning that the groups in the index are +numbered from 1 and the type pattern group numbers start after all the groups from the index. + +Missing types from a specified index will be removed. +When the regex pattern isn't defined `["(.*)", "(.*)", "\\1_\\2"]` is used to map each type into its own isolated +index, preserving all data and its separation. + +For more details about regexes, see the [Python](https://docs.python.org/3/library/re.html#re.sub) or +[Java](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) documentation. +This transform uses python-style backreferences (`'`\1`) for replacement patterns. +Notice that regexes can NOT be specified in the index-type map. +They can _only_ be used via the list, which will be evaluated in the order of the list until a match is found. + +The following sample shows how indices that start with 'time-' will be migrated and every other index and type not +already matched will be dropped. +``` +[ + ["time-(.*)", "(cpu)", "time-\\1-\\2"] +] +``` + +The following example preserves all other non-matched items, +merging all types into a single index with the same name as the source index. +``` +[ + ["time-(.*)", "(cpu)", "time-\\1-\\2"], + ["", ".*", ""] +] +``` + +For more examples, compare the following cases. +Though note that anything matched by the static maps shown above will block any of these rules from being evaluated. + +| Regex Entry | Source Index | Source Type | Target Index | PUT Doc URL | Bulk Index Command | +|--------------------------------------------------------------------------------|-------------|-------------|-------------------|--------------------------------|----------------------------------------------------------------| +| `[["time-(.*)", "(cpu)", "time-\\1-\\2"]]` | time-nov11 | cpu | time-nov11-cpu | /time-nov11-cpu/_doc/doc512 | `{"index": {"_index": "time-nov11-cpu", "_id": "doc512" }}` | +| `[["time-(.*)", "(cpu)", "time-\\1-\\2"]]` | logs | access | [DELETED] | [DELETED] | [DELETED] | +| `[["time-(.*)", "(cpu)", "time-\\1-\\2"],`
` ["(.*)", "(.*)", "\\1-\\2"]]` | logs | access | logs_access | /logs_access/_doc/doc513 | `{"index": {"_index": "logs_access", "_id": "doc513" }}` | +| `[["time-(.*)", "(cpu)", "time-\\1-\\2"],`
`[["", ".*", ""]]` | everything | widgets | everything | /everything/_doc/doc514 | `{"index": {"_index": "everything", "_id": "doc514" }}` | +| `[["time-(.*)", "(cpu)", "time-\\1-\\2"],`
`[["", ".*", ""]]` | everything | sprockets | everything | /everything/_doc/doc515 | `{"index": {"_index": "everything", "_id": "doc515" }}` | +| `[["time-(.*)", "(.*)-(cpu)", "\\2-\\3-\\1"]]` | time-nov11 | host123-cpu | host123-cpu-nov11 | /host123-cpu-nov11/_doc/doc512 | `{"index": {"_index": "host123-cpu-nov11", "_id": "doc512" }}` | +| `[["", ".*", ""]]` | metadata | users | metadata | /metadata/_doc/doc516 | `{"index": {"_index": "metadata", "_id": "doc516" }}` | +| `[[".*", ".*", "leftovers"]]` | logs | access | leftovers | /leftovers/_doc/doc517 | `{"index": {"_index": "leftovers", "_id": "doc517" }}` | +| `[[".*", ".*", "leftovers"]]` | permissions | access | leftovers | /leftovers/_doc/doc517 | `{"index": {"_index": "leftovers", "_id": "doc517" }}` | + diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/build.gradle b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/build.gradle new file mode 100644 index 000000000..ccec1a158 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/build.gradle @@ -0,0 +1,30 @@ +plugins { + id 'io.freefair.lombok' + id 'java-library' + id 'java-test-fixtures' +} + +dependencies { + api project(':transformation:transformationPlugins:jsonMessageTransformers:jsonJinjavaTransformer') + + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + implementation group: 'com.google.guava', name: 'guava' + + testFixturesImplementation project(':TrafficCapture:trafficReplayer') + testFixturesImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + + testImplementation project(':TrafficCapture:trafficReplayer') + testImplementation testFixtures(project(path: ':testHelperFixtures')) + testImplementation testFixtures(project(path: ':TrafficCapture:trafficReplayer')) + + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + testImplementation group: 'com.google.guava', name: 'guava' + testImplementation group: 'org.hamcrest', name: 'hamcrest' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params' + testImplementation group: 'org.slf4j', name: 'slf4j-api' + testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine' +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java new file mode 100644 index 000000000..0ea6d409e --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java @@ -0,0 +1,64 @@ +package org.opensearch.migrations.transform; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.UnaryOperator; + +import org.opensearch.migrations.transform.jinjava.JinjavaConfig; +import org.opensearch.migrations.transform.typemappings.SourceProperties; + +import com.google.common.io.Resources; + +public class TypeMappingsSanitizationTransformer extends JinjavaTransformer { + + public static final String ENTRYPOINT_JINJA_TEMPLATE = "jinjava/typeMappings/transformByTypeOfSourceInput.j2"; + + public TypeMappingsSanitizationTransformer( + Map> indexMappings, + List> regexIndexMappings) + throws IOException { + this(indexMappings, regexIndexMappings, null, null, null); + } + + public TypeMappingsSanitizationTransformer( + Map> indexMappings, + List> regexIndexMappings, + SourceProperties sourceProperties, + Map featureFlags, + JinjavaConfig jinjavaSettings) + throws IOException + { + super( + makeTemplate(), + makeSourceWrapperFunction(sourceProperties, featureFlags, indexMappings, regexIndexMappings), + Optional.ofNullable(jinjavaSettings).orElse(new JinjavaConfig())); + } + + private static UnaryOperator> + makeSourceWrapperFunction(SourceProperties sourceProperties, + Map featureFlagsIncoming, + Map> indexMappingsIncoming, + List> regexIndexMappingsIncoming) + { + var featureFlags = featureFlagsIncoming != null ? featureFlagsIncoming : Map.of(); + var indexMappings = indexMappingsIncoming != null ? indexMappingsIncoming : Map.of(); + // By NOT including a backreference, we're a bit more efficient, but it also lets us be agnostic to what + // types of patterns are being used. + // This regex says, match the type part and reduce it to nothing, leave the index part untouched. + var regexIndexMappings = Optional.ofNullable(regexIndexMappingsIncoming) + .orElseGet(() -> (indexMappingsIncoming == null ? List.of(List.of("(.*)", "(.*)", "\\1_\\2")) : List.of())); + + return incomingJson -> Map.of("source_document", incomingJson, + "index_mappings", indexMappings, + "regex_index_mappings", regexIndexMappings, + "featureFlags", featureFlags, + "source_properties", sourceProperties == null ? Map.of() : sourceProperties); + } + + private static String makeTemplate() throws IOException { + return Resources.toString(Resources.getResource(ENTRYPOINT_JINJA_TEMPLATE), StandardCharsets.UTF_8); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/typemappings/SourceProperties.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/typemappings/SourceProperties.java new file mode 100644 index 000000000..28c21f9ca --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/typemappings/SourceProperties.java @@ -0,0 +1,21 @@ +package org.opensearch.migrations.transform.typemappings; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SourceProperties { + private String type; + private Version version; + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Version { + private int major; + private int minor; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/documentBackfillItems.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/documentBackfillItems.j2 new file mode 100644 index 000000000..d8744d738 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/documentBackfillItems.j2 @@ -0,0 +1,19 @@ +{# see https://github.com/opensearch-project/opensearch-migrations/pull/1110 for the format of these messages #} +{%- include "typeMappings/rewriteBulkRequest.j2" -%} +{%- import "typeMappings/rewriteIndexForTarget.j2" as transidx -%} + +{%- set parameters = source_document.index -%} + +{%- set type_name = parameters['_type'] -%} +{%- if type_name -%} + {%- set target_index = transidx.convert_source_index_to_target(parameters['_index'], type_name, input_map.index_mappings, input_map.regex_index_mappings) if type_name -%} + {%- if target_index -%} + { + {{ rewrite_index_parameters(parameters, target_index) }}, + "preserve": ["source"] + } + {%- endif -%} +{%- else -%} + {%- import "typeMappings/preserveAll.j2" as preserve -%} + {{- preserve.make_keep_json() -}} +{%- endif -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/httpRequests.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/httpRequests.j2 new file mode 100644 index 000000000..145bdcb38 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/httpRequests.j2 @@ -0,0 +1,21 @@ +{%- import "common/route.j2" as rscope -%} +{%- import "typeMappings/preserveAll.j2" as preserve -%} +{%- include "typeMappings/rewriteDocumentRequest.j2" -%} +{%- include "typeMappings/rewriteBulkRequest.j2" -%} +{%- include "typeMappings/rewriteCreateIndexRequest.j2" -%} + + +{%- set source_and_mappings = { + 'request': source_document, + 'index_mappings': index_mappings, + 'regex_index_mappings': regex_index_mappings, + 'properties': source_properties} +-%} +{{- rscope.route(source_and_mappings, source_document.method + " " + source_document.URI, flags, 'preserve.make_keep_json', + [ + ('(?:PUT|POST) /([^/]*)/([^/]*)/(.*)', 'rewrite_doc_request', 'rewrite_add_request_to_strip_types'), + ( 'GET /((?!\\.\\.$)[^-_+\\p{Lu}\\\\/*?\\\"<>|,# ][^\\p{Lu}\\\\/*?\\\"<>|,# ]*)/((?!\\.\\.$)[^-_+\\p{Lu}\\\\/*?\\\"<>|,# ][^\\p{Lu}\\\\/*?\\\"<>|,# ]*)/([^/]+)$', 'rewrite_doc_request', 'rewrite_get_request_to_strip_types'), + ('(?:PUT|POST) /_bulk', 'rewrite_bulk', 'rewrite_bulk'), + ('(?:PUT|POST) /([^/]*)', 'rewrite_create_index', 'rewrite_create_index') + ]) +-}} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/makeNoop.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/makeNoop.j2 new file mode 100644 index 000000000..96ba372b0 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/makeNoop.j2 @@ -0,0 +1,3 @@ +{%- macro make_request() -%} + { "method": "GET", "URI": "/", "protocol": "HTTP/1.0" } +{%- endmacro -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/preserveAll.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/preserveAll.j2 new file mode 100644 index 000000000..6de7ba27e --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/preserveAll.j2 @@ -0,0 +1,3 @@ +{%- macro make_keep_json() -%} + { "preserve": "*" } +{%- endmacro -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteBulkRequest.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteBulkRequest.j2 new file mode 100644 index 000000000..b0acfd8a7 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteBulkRequest.j2 @@ -0,0 +1,103 @@ +{%- include "typeMappings/rewriteIndexForTarget.j2" -%} +{%- import "typeMappings/rewriteIndexForTarget.j2" as transidx -%} + +{%- macro retarget_command_parameters(parameters, target_index) -%} + {%- set ns = namespace(new_params={}) -%} + {%- for key, value in parameters.items() -%} + {%- if key != '_type' and key != '_index' -%} + {%- set inner_json = value | tojson -%} + {%- set jsonblob = ("{\"" + key + "\":" + inner_json + "}") | fromjson -%} + {%- set ns.new_params = ns.new_params + jsonblob -%} + {%- endif -%} + {%- endfor -%} + {%- set index_json = target_index | tojson -%} + {%- set index_blob = ("{\"_index\":" + index_json + "}") | fromjson -%} + {{- (ns.new_params + index_blob) | tojson -}} +{%- endmacro -%} + +{%- macro get_create() -%}create{% endmacro %} +{%- macro get_index() -%}index{% endmacro %} +{%- macro get_update() -%}update{% endmacro %} + +{%- macro rewrite_command_parameters(command, parameters, target_index) -%} + {%- if target_index -%} + "{{ invoke_macro("get_"+command) }}": {{ retarget_command_parameters(parameters, target_index) }} + {%- endif -%} +{%- endmacro -%} + +{%- macro rewrite_index_parameters(parameters, target_index) -%} + {{ rewrite_command_parameters('index', parameters, target_index) }} +{%- endmacro -%} + +{%- macro rewrite_command(command, parameters, target_index, doc) -%} + {%- if target_index -%} + { {{ rewrite_command_parameters(command, parameters, target_index) }} }, + {{ doc | tojson }} + {%- endif -%} +{%- endmacro -%} + +{%- macro run_delete(parameters, target_index) -%} + {%- if target_index -%} + { "delete": {{ retarget_command_parameters(parameters, target_index) }} } + {%- endif -%} +{%- endmacro -%} + +{%- macro rewrite_bulk_for_default_source_index(uri_match, input_map, source_index) -%} +{ + "preserveWhenMissing": "*", + "payload": { + "inlinedJsonSequenceBodies": [ + {%- set operation_types = ['delete', 'update', 'index', 'create'] -%} + {%- set operations = input_map.request.payload.inlinedJsonSequenceBodies -%} + {%- set loopcontrol = namespace(skipnext=false) -%} + {%- for item in operations -%} + {%- set operation = namespace(type=None, output=None, num_written=0) -%} + {%- if not loopcontrol.skipnext -%} + {%- for type in operation_types -%} + {%- if item is mapping and type in item -%} + {%- set operation.type = type -%} + {%- endif -%} + {%- endfor -%} + {%- if not operation -%} + {%- throw "No valid operation type was found for item {{ item }}" -%} + {% endif %} + {% else %} + {%- set loopcontrol.skipnext = false -%} + {%- endif -%} + + {%- if operation.type -%} + {%- set parameters = item[operation.type] -%} + {%- set type_name = parameters['_type'] -%} + {%- set target_index = transidx.convert_source_index_to_target(parameters['_index'], type_name, input_map.index_mappings, input_map.regex_index_mappings) if type_name -%} + {%- if operation.type == 'delete' -%} + {%- if type_name -%} + {%- set operation.output = run_delete(parameters, target_index) -%} + {%- else -%} + {%- set operation.output = (item | tojson) -%} + {%- endif -%} + {%- else -%} + {%- if loop.index < operations|length -%} + {%- if type_name -%} + {%- set operation.output = rewrite_command(operation.type, parameters, target_index, operations[loop.index]) -%} + {%- else -%} + {%- set operation.output = (item | tojson) ~ "," ~ (operations[loop.index] | tojson) -%} + {%- endif -%} + {%- set loopcontrol.skipnext = true -%} + {%- else -%} + {%- throw "Handle case where there's no next item but one was expected for item {{ item }}" -%} + {%- endif -%} + {%- endif -%} + {% if (operation.output | string | trim | length) > 0 %} + {%- set loopcontrol.num_written = loopcontrol.num_written + 1 -%} + {{ "," if loopcontrol.num_written > 1 else "" }} + {{ operation.output }} + {% endif %} + {%- endif -%} + {%- endfor -%} + ] + } +} +{%- endmacro -%} +{%- macro rewrite_bulk(match, input_map) -%} + {{ rewrite_bulk_for_default_source_index(match, input_map, none) }} +{%- endmacro -%} \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteCreateIndexRequest.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteCreateIndexRequest.j2 new file mode 100644 index 000000000..63f37fc76 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteCreateIndexRequest.j2 @@ -0,0 +1,97 @@ +{%- import "typeMappings/makeNoop.j2" as noop -%} +{%- import "typeMappings/preserveAll.j2" as preserve -%} + +{%- macro rewrite_create_index_as_unioned_excise(source_index_name, target_index_name, input_map) -%} + {%- set source_input_types = input_map.index_mappings[source_index_name] -%} + {%- set source_type_name = source_input_types.keys() | first() -%} + { + "preserveWhenMissing": "*", + "method": "{{ input_map.request.method }}", + "URI": "/{{ target_index_name }}", + "payload": { + "inlinedJsonBody": { + {%- for key, value in input_map.request.payload.inlinedJsonBody.items() -%} + {%- if key != "mappings" -%} + "{{ key }}": {{ value | tojson }}, + {%- endif -%} + {%- endfor -%} + "mappings": { + "properties": { + {%- set ns = namespace(combined_props={"type": "keyword"}) -%} + {%- for source_type_name in source_input_types.keys() -%} + {%- set type_props = input_map.request.payload.inlinedJsonBody.mappings.get(source_type_name) -%} + {%- for prop_name, prop_def in type_props.properties.items() -%} + {%- if prop_name in ns.combined_props -%} + {%- if ns.combined_props[prop_name] != prop_def -%} + {%- throw "Conflicting definitions for property {{ prop_name }} ({{ ns.combined_props[prop_name] }} and {{ prop_def }})" -%} + {%- endif -%} + {%- else -%} + {%- set body = prop_def | tojson -%} + {%- set jsonblob = ("{\"" + prop_name + "\":" + body + "}") | fromjson -%} + {%- set ns.combined_props = ns.combined_props + jsonblob -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + + {%- for prop_name, prop_def in ns.combined_props.items() -%} + "{{- prop_name -}}": {{- prop_def | tojson -}}, + {%- endfor -%} + + "type": { "type": "keyword" } + } + } + } + } + } +{%- endmacro -%} + +{%- macro uses_type_names(input_map) -%} + {%- set uri_flag_match = input_map.request.URI | regex_capture("[?&]include_type_name=([^&#]*)") -%} + {%- if uri_flag_match -%} + {{- uri_flag_match.group1 | lower -}} + {%- else -%} + {%- set major_version = ((input_map.properties | default({})).version | default({})).major -%} + {%- if major_version >= 7-%} + false + {%- elif major_version <= 6 -%} + true + {%- else -%} + {%- throw "include_type_name was not set on the incoming URI." + + "The template needs to know what version the original request was targeted for " + + "in order to properly understand the semantics and what was intended. " + + "Without that, this transformation cannot map the request " + + "to an unambiguous request for the target" -%} + {%- endif -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro rewrite_create_index(match, input_map) -%} + {%- set orig_mappings = input_map.request.payload.inlinedJsonBody.mappings -%} + + {%- if orig_mappings and uses_type_names(input_map).trim() == 'true' -%} + {%- set source_index_name = match.group1 | regex_replace("[?].*", "") -%} + + {%- set ns = namespace(accum_target_indices=[]) -%} + {%- for source_type, mapping in orig_mappings.items() -%} + {%- set target_index = convert_source_index_to_target(source_index_name, source_type, + input_map.index_mappings, + input_map.regex_index_mappings) | trim -%} + {%- if target_index -%} + {%- set ns.accum_target_indices = ns.accum_target_indices + [target_index] -%} + {%- endif -%} + {%- endfor -%} + + {%- set target_indices = ns.accum_target_indices | unique() -%} + {%- set num_target_mappings = target_indices | length -%} + {%- if num_target_mappings == 0 -%} + {{- noop.make_request() -}} + {%- elif num_target_mappings == 1 -%} + {{- rewrite_create_index_as_unioned_excise(source_index_name, (target_indices | first), input_map) -}} + {%- else -%} + {%- throw "Cannot specify multiple indices to create with a single request and cannot yet " + + "represent multiple requests with the request format." -%} + {%- endif -%} + {%- else -%} + {{- preserve.make_keep_json() -}} + {%- endif -%} +{%- endmacro -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteDocumentRequest.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteDocumentRequest.j2 new file mode 100644 index 000000000..97475c5be --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteDocumentRequest.j2 @@ -0,0 +1,15 @@ +{%- import "typeMappings/makeNoop.j2" as noop -%} +{%- import "typeMappings/rewriteIndexForTarget.j2" as transidx -%} + +{%- macro rewrite_doc_request(match, input_map) -%} + {%- set target_index = transidx.convert_source_index_to_target(match.group1, match.group2, input_map.index_mappings, input_map.regex_index_mappings) -%} + {%- if target_index is none -%} + {{- noop.make_request() -}} + {%- else -%} + { + "method": "{{ input_map.request.method }}", + "URI": "/{{ target_index }}/_doc/{{ match.group3 }}", + "preserveWhenMissing": "*" + } + {%- endif -%} +{%- endmacro -%} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteIndexForTarget.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteIndexForTarget.j2 new file mode 100644 index 000000000..150f6d1a9 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteIndexForTarget.j2 @@ -0,0 +1,28 @@ +{%- macro convert_source_index_to_target_via_regex(source_index, source_type, regex_index_mappings) -%} + {%- set ns = namespace(target_index=none) -%} + {%- for idx_regex, type_regex, target_idx_pattern in regex_index_mappings -%} + {%- if ns.target_index is none -%} + {%- set conjoined_source = source_index + "/" + source_type -%} + {%- set conjoined_regex = idx_regex + "/" + type_regex -%} + {%- set didMatch = conjoined_source | regex_capture(conjoined_regex) -%} + {%- if didMatch is not none -%} +{# conjoined_source = {{ conjoined_source }} conjoined_regex {{ conjoined_regex }} target_idx_pattern = {{ target_idx_pattern }}#} + {%- set ns.target_index = conjoined_source | regex_replace(conjoined_regex, target_idx_pattern) -%} + {%- endif -%} + {%- endif -%} + {%- endfor -%} + {{- ns.target_index -}} +{%- endmacro -%} + +{%- macro convert_source_index_to_target(source_index, source_type, index_mappings, regex_index_mappings) -%} + {%- if source_type == "_doc" -%} + {{- source_index -}} + {%- else -%} + {%- set ns = namespace(target_index=none) -%} + {%- set ns.target_index = (index_mappings[source_index] | default({}))[source_type] -%} + {%- if ns.target_index is none -%} + {%- set ns.target_index = convert_source_index_to_target_via_regex(source_index, source_type, regex_index_mappings) -%} + {%- endif -%} + {{- ns.target_index -}} + {%- endif -%} +{%- endmacro -%} \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/transformByTypeOfSourceInput.j2 b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/transformByTypeOfSourceInput.j2 new file mode 100644 index 000000000..df26b6b26 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/transformByTypeOfSourceInput.j2 @@ -0,0 +1,12 @@ +{%- if not source_document -%} + {%- throw "No source_document was defined - nothing to transform!" -%} +{%- endif -%} + +{%- if ("method" in source_document and "URI" in source_document) -%} + {%- include "typeMappings/httpRequests.j2" -%} +{%- elif ("index" in source_document and "source" in source_document) -%} + {%- include "typeMappings/documentBackfillItems.j2" -%} +{%- else -%} + {%- import "typeMappings/preserveAll.j2" as preserve -%} + {{- preserve.make_keep_json() -}} +{%- endif -%} \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java new file mode 100644 index 000000000..8186411fb --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationCreateIndexTest.java @@ -0,0 +1,168 @@ +package org.opensearch.migrations.transform; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.opensearch.migrations.testutils.JsonNormalizer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TypeMappingsSanitizationCreateIndexTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static TypeMappingsSanitizationTransformer makeIndexTypeMappingRewriter() throws Exception { + var indexMappings = Map.of( + "indexa", Map.of( + "type1", "indexa_1", + "type2", "indexa_2", + "user", "a_user"), + "indexb", Map.of( + "type1", "indexb", + "type2", "indexb"), + "socialTypes", Map.of( + "tweet", "communal", + "user", "communal")); + var regexIndexMappings = List.of( + List.of("time-(.*)", "(.*)", "time-\\1-\\2")); + return new TypeMappingsSanitizationTransformer(indexMappings, regexIndexMappings); + } + + private static String makeMultiTypePutIndexRequest(String indexName, Boolean includeTypeName) { + return TestRequestBuilder.makePutIndexRequest(indexName, true, includeTypeName); + } + + @Test + public void testPutSingleTypeToMissingTarget() throws Exception { + final String index = "indexb"; // has multiple indices for its types + var testString = TestRequestBuilder.makePutIndexRequest(index, false, true); + var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); + var result = (Map) + indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + var expected = "{ \"method\": \"GET\", \"URI\": \"/\", \"protocol\" : \"HTTP/1.0\" }"; + Assertions.assertEquals(JsonNormalizer.fromString(expected), + JsonNormalizer.fromObject(result)); + } + + @Test + public void testPutSingleTypeIndex() throws Exception { + final String index = "indexa"; // has multiple indices for its types + var testString = TestRequestBuilder.makePutIndexRequest(index, false, true); + var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); + var result = (Map) + indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + var expectedString = "{\n" + + " \"URI\" : \"/a_user\",\n" + + " \"method\" : \"PUT\",\n" + + " \"protocol\" : \"HTTP/1.1\",\n" + + " \"payload\" : {\n" + + " \"inlinedJsonBody\" : {\n" + + " \"mappings\" : {\n" + + " \"properties\" : {\n" + + " \"email\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"name\" : {\n" + + " \"type\" : \"text\"\n" + + " },\n" + + " \"type\" : {\n" + + " \"type\" : \"keyword\"\n" + + " },\n" + + " \"user_name\" : {\n" + + " \"type\" : \"keyword\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"settings\" : {\n" + + " \"number_of_shards\" : 1\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + Assertions.assertEquals(JsonNormalizer.fromString(expectedString), + JsonNormalizer.fromObject(result)); + } + + @Test + public void testMultiTypeIndexMerged() throws Exception { + final String index = "socialTypes"; + var testString = makeMultiTypePutIndexRequest(index, true); + var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); + var result = (Map) + indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + var expected = OBJECT_MAPPER.readTree(makeMultiTypePutIndexRequest(index, null)); + var mappings = ((ObjectNode) expected.path(JsonKeysForHttpMessage.PAYLOAD_KEY) + .path(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY) + .path("mappings")); + mappings.remove("following"); + var newProperties = new HashMap(); + newProperties.put("type", Map.of("type", "keyword")); + var user = mappings.remove("user"); + user.path("properties").fields().forEachRemaining(e -> newProperties.put(e.getKey(), e.getValue())); + var tweet = mappings.remove("tweet"); + tweet.path("properties").fields().forEachRemaining(e -> newProperties.put(e.getKey(), e.getValue())); + mappings.set("properties", OBJECT_MAPPER.valueToTree(newProperties)); + ((ObjectNode)expected).put(JsonKeysForHttpMessage.URI_KEY, "/communal"); + Assertions.assertEquals(JsonNormalizer.fromObject(expected), JsonNormalizer.fromObject(result)); + } + + @Test + public void testCreateIndexWithoutTypeButWithMappings() throws Exception{ + var uri = TestRequestBuilder.formatCreateIndexUri("geonames", false);; + var testString = "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"" + uri + "\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\"," + + " \"" + JsonKeysForHttpMessage.HEADERS_KEY + "\": {\n" + + " \"Host\": \"capture-proxy:9200\"\n" + + " }," + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": {\n" + + " \"settings\": {\n" + + " \"index\": {\n" + + " \"number_of_shards\": 3, \n" + + " \"number_of_replicas\": 2 \n" + + " }\n" + + " }," + + " \"mappings\": {" + + " \"properties\": {\n" + + " \"field1\": { \"type\": \"text\" }\n" + + " }" + + " }\n" + + " }\n" + + " }\n" + + "}"; + var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); + } + + @Test + public void testCreateIndexWithoutType() throws Exception{ + var testString = "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/geonames\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\"," + + " \"" + JsonKeysForHttpMessage.HEADERS_KEY + "\": {\n" + + " \"Host\": \"capture-proxy:9200\"\n" + + " }," + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"settings\": {\n" + + " \"index\": {\n" + + " \"number_of_shards\": 3, \n" + + " \"number_of_replicas\": 2 \n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + var indexTypeMappingRewriter = makeIndexTypeMappingRewriter(); + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); + } + +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java new file mode 100644 index 000000000..7318d25c9 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java @@ -0,0 +1,41 @@ +package org.opensearch.migrations.transform; + +import java.util.LinkedHashMap; + +import org.opensearch.migrations.testutils.JsonNormalizer; + +import lombok.Lombok; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; + +@Slf4j +public class TypeMappingsSanitizationDocBackfillTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Test + public void test() throws Exception { + var testString = "{\n" + + " \"index\": { \"_index\": \"performance\", \"_type\": \"network\", \"_id\": \"1\" },\n" + + " \"source\": { \"field1\": \"value1\" }\n" + + "}"; + + var expectedString = "{\n" + + " \"index\": { \"_index\": \"performance_network\", \"_id\": \"1\" },\n" + + " \"source\": { \"field1\": \"value1\" }\n" + + "}"; + + + var indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(null, null); + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + log.atInfo().setMessage("resultStr = {}").addArgument(() -> { + try { + return OBJECT_MAPPER.writeValueAsString(resultObj); + } catch (Exception e) { + throw Lombok.sneakyThrow(e); + } + }).log(); + Assertions.assertEquals(JsonNormalizer.fromString(expectedString), JsonNormalizer.fromObject(resultObj)); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java new file mode 100644 index 000000000..2b715504a --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java @@ -0,0 +1,125 @@ +package org.opensearch.migrations.transform; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.opensearch.migrations.testutils.JsonNormalizer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +@Slf4j +public class TypeMappingsSanitizationTransformerBulkTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static TypeMappingsSanitizationTransformer indexTypeMappingRewriter; + @BeforeAll + static void initialize() throws IOException { + var indexMappings = Map.of( + "indexa", Map.of( + "type1", "indexa_1", + "type2", "indexa_2"), + "indexb", Map.of( + "type1", "indexb", + "type2", "indexb"), + "indexc", Map.of( + "type2", "indexc")); + var regexIndexMappings = List.of( + List.of("time-(.*)", "(.*)", "time-\\1-\\2")); + indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(indexMappings, regexIndexMappings); + } + + @Test + public void testBulkRequest() throws Exception { + var testString = + "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/_bulk\",\n" + + " \"" + JsonKeysForHttpMessage.HEADERS_KEY + "\": {},\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_NDJSON_BODIES_DOCUMENT_KEY + "\": [\n" + + "{ \"index\": { \"_index\": \"indexa\", \"_type\": \"type1\", \"_id\": \"1\" } },\n" + + "{ \"field1\": \"value1\" },\n" + + + "{ \"index\": { \"_index\": \"indexa\", \"_type\": \"typeDontMap\", \"_id\": \"1\" } },\n" + + "{ \"field1\": \"value9\" },\n" + + + "{ \"delete\": { \"_index\": \"test\", \"_type\": \"type1\", \"_id\": \"2\" } },\n" + + + "{ \"delete\": { \"_index\": \"time-January_1970\", \"_type\": \"cpu\", \"_id\": \"8\" } },\n" + + + "{ \"create\": { \"_index\": \"indexc\", \"_type\": \"type1\", \"_id\": \"3\" } },\n" + + "{ \"field1\": \"value3\" },\n" + + + "{ \"create\": { \"_index\": \"indexc\", \"_type\": \"type2\", \"_id\": \"14\" } },\n" + + "{ \"field14\": \"value14\" },\n" + + + "{ \"update\": {\"_id\": \"1\", \"_type\": \"type1\", \"_index\": \"indexb\"} },\n" + + "{ \"doc\": {\"field2\": \"value2\"} },\n" + + + "{ \"update\": {\"_id\": \"1\", \"_type\": \"type2\", \"_index\": \"indexb\"} },\n" + + "{ \"doc\": {\"field10\": \"value10\"} },\n" + + + "{ \"update\": {\"_id\": \"1\", \"_type\": \"type3\", \"_index\": \"indexb\"} },\n" + + "{ \"doc\": {\"field11\": \"value11\"} },\n" + + + "{ \"delete\": {\"_id\": \"12\", \"_index\": \"index_without_typemappings\"} },\n" + + + "{ \"update\": {\"_id\": \"13\", \"_index\": \"index_without_typemappings\"} },\n" + + "{ \"doc\": {\"field13\": \"value11\"} }\n" + + + " ]\n" + + " }\n" + + "}"; + + var expectedString = + "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/_bulk\",\n" + + " \"" + JsonKeysForHttpMessage.HEADERS_KEY + "\": {},\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_NDJSON_BODIES_DOCUMENT_KEY + "\": [\n" + + "{ \"index\": { \"_index\": \"indexa_1\", \"_id\": \"1\" } },\n" + + "{ \"field1\": \"value1\" },\n" + + + "{ \"delete\": { \"_index\": \"time-January_1970-cpu\", \"_id\": \"8\" } },\n" + + + "{ \"create\": { \"_index\": \"indexc\", \"_id\": \"14\" } },\n" + + "{ \"field14\": \"value14\" },\n" + + + "{ \"update\": {\"_id\": \"1\", \"_index\": \"indexb\"} },\n" + + "{ \"doc\": {\"field2\": \"value2\"} },\n" + + + "{ \"update\": {\"_id\": \"1\", \"_index\": \"indexb\"} },\n" + + "{ \"doc\": {\"field10\": \"value10\"} },\n" + + + "{ \"delete\": {\"_id\": \"12\", \"_index\": \"index_without_typemappings\"} },\n" + + + "{ \"update\": {\"_id\": \"13\", \"_index\": \"index_without_typemappings\"} },\n" + + "{ \"doc\": {\"field13\": \"value11\"} }\n" + + + " ]\n" + + " }\n" + + "}"; + + + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + log.atInfo().setMessage("resultStr = {}").addArgument(() -> { + try { + return OBJECT_MAPPER.writeValueAsString(resultObj); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }).log(); + Assertions.assertEquals(JsonNormalizer.fromString(expectedString), JsonNormalizer.fromObject(resultObj)); + } + +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerTest.java new file mode 100644 index 000000000..d0a738283 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerTest.java @@ -0,0 +1,105 @@ +package org.opensearch.migrations.transform; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.opensearch.migrations.testutils.JsonNormalizer; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@Slf4j +class TypeMappingsSanitizationTransformerTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static TypeMappingsSanitizationTransformer indexTypeMappingRewriter; + @BeforeAll + static void initialize() throws IOException { + var indexMappings = Map.of( + "indexa", Map.of( + "type1", "indexa_1", + "type2", "indexa_2"), + "indexb", Map.of( + "type1", "indexb", + "type2", "indexb"), + "socialTypes", Map.of( + "tweet", "communal", + "user", "communal")); + var regexIndexMappings = List.of( + List.of("time-(.*)", "(.*)", "time-\\1-\\2")); + indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(indexMappings, regexIndexMappings); + } + + + @Test + public void testPutDoc() throws Exception { + var testString = + "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/indexa/type2/someuser\",\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": {" + + " \"name\": \"Some User\",\n" + + " \"user_name\": \"user\",\n" + + " \"email\": \"user@example.com\"\n" + + " }\n" + + " }\n" + + "}"; + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString.replace("indexa/type2/", "indexa_2/_doc/")), + JsonNormalizer.fromObject(resultObj)); + } + + @Test + public void testPutDocRegex() throws Exception { + var testString = + "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/time-nov11/cpu/doc2\",\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": {" + + " \"name\": \"Some User\",\n" + + " \"user_name\": \"user\",\n" + + " \"email\": \"user@example.com\"\n" + + " }\n" + + " }\n" + + "}"; + var expectedString = "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\":\"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/time-nov11-cpu/_doc/doc2\",\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\":{" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\":{" + + " \"name\":\"Some User\"," + + " \"user_name\":\"user\"," + + " \"email\":\"user@example.com\"" + + " }" + + " }" + + "}"; + var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + log.atInfo().setMessage("resultStr = {}").setMessage(OBJECT_MAPPER.writeValueAsString(resultObj)).log(); + Assertions.assertEquals(JsonNormalizer.fromString(expectedString), + JsonNormalizer.fromObject(resultObj)); + } + + @ParameterizedTest + @ValueSource(strings = {"status", "_cat/indices", "_cat/indices/nov-*"} ) + public void testDefaultActionPreservesRequest(String uri) throws Exception { + final String bespokeRequest = "" + + "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"GET\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/" + uri + "\"" + + "}"; + var transformedResult = indexTypeMappingRewriter.transformJson( + OBJECT_MAPPER.readValue(bespokeRequest, new TypeReference<>(){})); + Assertions.assertEquals(JsonNormalizer.fromString(bespokeRequest), + JsonNormalizer.fromObject(transformedResult)); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/testFixtures/java/org/opensearch/migrations/transform/TestRequestBuilder.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/testFixtures/java/org/opensearch/migrations/transform/TestRequestBuilder.java new file mode 100644 index 000000000..025aa6ed0 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/testFixtures/java/org/opensearch/migrations/transform/TestRequestBuilder.java @@ -0,0 +1,54 @@ +package org.opensearch.migrations.transform; + +import java.util.Optional; + +import lombok.NonNull; + +public class TestRequestBuilder { + public static String makePutIndexRequest(String indexName, Boolean useMultiple, Boolean includeTypeName) { + var uri = formatCreateIndexUri(indexName, includeTypeName); + return "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"" + uri + "\",\n" + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": " + + "{\n" + + " \"settings\" : {\n" + + " \"number_of_shards\" : 1\n" + + " }," + + " \"mappings\": {\n" + + " \"user\": {\n" + + " \"properties\": {\n" + + " \"name\": { \"type\": \"text\" },\n" + + " \"user_name\": { \"type\": \"keyword\" },\n" + + " \"email\": { \"type\": \"keyword\" }\n" + + " }\n" + + " }" + + (useMultiple ? ",\n" + + " \"tweet\": {\n" + + " \"properties\": {\n" + + " \"content\": { \"type\": \"text\" },\n" + + " \"user_name\": { \"type\": \"keyword\" },\n" + + " \"tweeted_at\": { \"type\": \"date\" }\n" + + " }\n" + + " },\n" + + " \"following\": {\n" + + " \"properties\": {\n" + + " \"count\": { \"type\": \"integer\" },\n" + + " \"followers\": { \"type\": \"string\" }\n" + + " }\n" + + " }\n" + : "") + + " }\n" + + "}" + + "\n" + + " }\n" + + "}"; + } + + public static @NonNull String formatCreateIndexUri(String indexName, Boolean includeTypeName) { + return "/" + indexName + + Optional.ofNullable(includeTypeName).map(b -> "?include_type_name=" + b).orElse(""); + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/README.md b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/README.md new file mode 100644 index 000000000..9adf06e4d --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/README.md @@ -0,0 +1,38 @@ +# Configuration Routing + +See the [README for the Type Mappings Sanitization Transformer](../jsonTypeMappingsSanitizationTransformer/README.md) +for specifics of how rules are evaluated. + +This package loads a [Type Mappings Sanitization Transformer](../jsonTypeMappingsSanitizationTransformer/src/main/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformer.java) +for that appropriate application variant (currently only for the REPLAYER) along with the configuration passed into +the provider. This [Provider](./src/main/java/org/opensearch/migrations/transform/TypeMappingSanitizationTransformerProvider.java) +pulls the values for keys `featureFlags`, `staticMappings`, and `regexMappings` from the incoming configuration map +object so that the Type Mappings Sanitization Transformer can adjust requests for specific type mappings into the +appropriate target index. + +The following example will load a Transformer to rewrite types as per the static mappings shown in the second key-value +(staticMappings), or if not present, will then default to the mappings in regexMappings. Note that regexMappings will +only be checked if there are no entries in staticMappings. +staticMappings index names (top-level key) and keys to their children maps will be evaluated literally, not as patterns. +Patterns are ONLY supported via regexMappings. + +``` +{ + "staticMappings": { + "indexA": { + "type2": "indexA_2", + "type1": "indexA_1" + }, + "indexB": { + "type2": "indexB", + "type1": "indexB" + }, + "indexC": { + "type2": "indexC" + } + }, + "regexMappings": [ + [ "(time*)", "(type*)", "\\1_And_\\2" ] + ] +} +``` \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/build.gradle b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/build.gradle new file mode 100644 index 000000000..672949366 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/build.gradle @@ -0,0 +1,35 @@ +buildscript { + dependencies { + classpath 'org.junit.platform:junit-platform-gradle-plugin:1.0.1' + } +} + +plugins { + id 'io.freefair.lombok' +} + +dependencies { + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerInterface') + implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformer') + + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + + testImplementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformer') + testImplementation testFixtures(project(path:':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformer')) + testImplementation project(':coreUtilities') + testImplementation testFixtures(project(path: ':coreUtilities')) + testImplementation testFixtures(project(path: ':testHelperFixtures')) + testImplementation testFixtures(project(path: ':TrafficCapture:trafficReplayer')) + + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' + testImplementation group: 'io.netty', name: 'netty-all' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-api' + testImplementation group: 'org.junit.jupiter', name:'junit-jupiter-params' + testImplementation group: 'org.slf4j', name: 'slf4j-api' + testRuntimeOnly group:'org.junit.jupiter', name:'junit-jupiter-engine' +} + +tasks.named('test') { + useJUnitPlatform() +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/java/org/opensearch/migrations/transform/TypeMappingSanitizationTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/java/org/opensearch/migrations/transform/TypeMappingSanitizationTransformerProvider.java new file mode 100644 index 000000000..d5dc23796 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/java/org/opensearch/migrations/transform/TypeMappingSanitizationTransformerProvider.java @@ -0,0 +1,64 @@ +package org.opensearch.migrations.transform; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.opensearch.migrations.transform.jinjava.JinjavaConfig; +import org.opensearch.migrations.transform.typemappings.SourceProperties; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +public class TypeMappingSanitizationTransformerProvider implements IJsonTransformerProvider { + + public static final String FEATURE_FLAGS = "featureFlags"; + public static final String STATIC_MAPPINGS = "staticMappings"; + public static final String REGEX_MAPPINGS = "regexMappings"; + + public static final String JINJAVA_CONFIG_KEY = "jinjavaConfig"; + public static final String SOURCE_PROPERTIES_KEY = "sourceProperties"; + + public final static ObjectMapper mapper = new ObjectMapper(); + + @SneakyThrows + @Override + public IJsonTransformer createTransformer(Object jsonConfig) { + try { + if ((jsonConfig == null) || + (jsonConfig instanceof String && ((String) jsonConfig).isEmpty())) { + return new TypeMappingsSanitizationTransformer(null, null, null, null, null); + } else if (!(jsonConfig instanceof Map)) { + throw new IllegalArgumentException(getConfigUsageStr()); + } + + var config = (Map) jsonConfig; + return new TypeMappingsSanitizationTransformer( + (Map>) config.get(STATIC_MAPPINGS), + (List>) config.get(REGEX_MAPPINGS), + Optional.ofNullable(config.get(SOURCE_PROPERTIES_KEY)).map(m -> + mapper.convertValue(m, SourceProperties.class)).orElse(null), + (Map) config.get(FEATURE_FLAGS), + Optional.ofNullable(config.get(JINJAVA_CONFIG_KEY)).map(m -> + mapper.convertValue(m, JinjavaConfig.class)).orElse(null)); + } catch (ClassCastException e) { + throw new IllegalArgumentException(getConfigUsageStr(), e); + } + } + + private String getConfigUsageStr() { + return this.getClass().getName() + " " + + "expects the incoming configuration to be a Map, " + + "with values '" + STATIC_MAPPINGS + "', '" + REGEX_MAPPINGS + "', and '" + FEATURE_FLAGS + "'. " + + "The value of " + STATIC_MAPPINGS + " should be a two-level map where the top-level key is the name " + + "of a source index and that key's dictionary maps each sub-type to a specific target index. " + + REGEX_MAPPINGS + " (List<[List>]) matches index names and sub-types to a target pattern. " + + "The patterns are matched in ascending order, finding the first match. " + + "The items within each top-level " + REGEX_MAPPINGS + " element are [indexRegex, typeRegex, targetPattern]." + + " The targetPattern can contain backreferences ('\\1'...) to refer to captured groups from the regex. " + + "Finally, the " + FEATURE_FLAGS + " is an arbitrarily deep map with boolean leaves. " + + "The " + FEATURE_FLAGS + " map is optional. " + + "When present, it can disable some types of transformations, " + + "such as when they may not be applicable for a given migration."; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider new file mode 100644 index 000000000..fb490338a --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider @@ -0,0 +1 @@ +org.opensearch.migrations.transform.TypeMappingSanitizationTransformerProvider \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java new file mode 100644 index 000000000..d2500e656 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/java/org/opensearch/migrations/replay/TypeMappingsSanitizationProviderTest.java @@ -0,0 +1,165 @@ +package org.opensearch.migrations.replay; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +import org.opensearch.migrations.testutils.JsonNormalizer; +import org.opensearch.migrations.testutils.WrapWithNettyLeakDetection; +import org.opensearch.migrations.transform.JsonKeysForHttpMessage; +import org.opensearch.migrations.transform.TestRequestBuilder; +import org.opensearch.migrations.transform.TypeMappingSanitizationTransformerProvider; +import org.opensearch.migrations.transform.jinjava.ThrowTag; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.hubspot.jinjava.interpret.FatalTemplateErrorsException; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +@Slf4j +@WrapWithNettyLeakDetection(disableLeakChecks = true) +public class TypeMappingsSanitizationProviderTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Test + public void testSimpleTransform() throws JsonProcessingException { + var config = Map.of("staticMappings", + Map.of( + "indexa", Map.of( + "type1", "indexa_1", + "type2", "indexa_2"), + "indexb", Map.of( + "type1", "indexb", + "type2", "indexb"), + "indexc", Map.of( + "type2", "indexc")), + "regexMappings", List.of(List.of("(time.*)", "(type.*)", "\\1_And_\\2"))); + final String TEST_INPUT_REQUEST = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/indexa/type2/someuser\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"name\": \"Some User\",\n" + + " \"user_name\": \"user\",\n" + + " \"email\": \"user@example.com\"\n" + + " }\n" + + " }\n" + + "}\n"; + final String EXPECTED = "{\n" + + " \"method\": \"PUT\",\n" + + " \"URI\": \"/indexa_2/_doc/someuser\",\n" + + " \"headers\": {\n" + + " \"host\": \"127.0.0.1\"\n" + + " },\n" + + " \"payload\": {\n" + + " \"inlinedJsonBody\": {\n" + + " \"name\": \"Some User\",\n" + + " \"user_name\": \"user\",\n" + + " \"email\": \"user@example.com\"\n" + + " }\n" + + " }\n" + + "}\n"; + + var provider = new TypeMappingSanitizationTransformerProvider(); + Map inputMap = OBJECT_MAPPER.readValue(TEST_INPUT_REQUEST, new TypeReference<>() { + }); + { + var transformedDocument = provider.createTransformer(config).transformJson(inputMap); + Assertions.assertEquals(JsonNormalizer.fromString(EXPECTED), + JsonNormalizer.fromObject(transformedDocument)); + } + { + var resultFromNullConfig = provider.createTransformer(null).transformJson(inputMap); + Assertions.assertEquals( + JsonNormalizer.fromString( + EXPECTED.replace( + "/indexa_2/_doc/someuser", + "/indexa_type2/_doc/someuser")), + JsonNormalizer.fromObject(resultFromNullConfig)); + } + } + + @Test + public void testMappingWithoutTypesAndLatestSourceInfoDoesNothing() throws Exception { + var testString = TestRequestBuilder.makePutIndexRequest("commingled_docs", true, false); + var fullTransformerConfig = + Map.of("sourceProperties", + Map.of("version", + Map.of("major", (Object) 6, + "minor", (Object) 10))); + var transformer = new TypeMappingSanitizationTransformerProvider().createTransformer(fullTransformerConfig); + var resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); + } + + @Test + public void testTypeMappingsWithSourcePropertiesWorks() throws Exception { + var testString = TestRequestBuilder.makePutIndexRequest("commingled_docs", true, false); + var fullTransformerConfig = + Map.of("sourceProperties", Map.of("version", + Map.of("major", (Object) 5, + "minor", (Object) 10)), + "regex_index_mappings", List.of(List.of("", "", ""))); + var transformer = new TypeMappingSanitizationTransformerProvider().createTransformer(fullTransformerConfig); + var resultObj = transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class)); + Assertions.assertEquals(JsonNormalizer.fromString(testString), JsonNormalizer.fromObject(resultObj)); + } + + @Test + public void testMappingsButNoSourcePropertiesThrows() throws Exception { + var testString = makeCreateIndexRequestWithoutTypes(); + var noopString = "{\n" + + " \"URI\" : \"/\",\n" + + " \"method\" : \"GET\"\n" + + "}"; + var transformer = new TypeMappingSanitizationTransformerProvider().createTransformer(null); + var thrownException = + Assertions.assertThrows(FatalTemplateErrorsException.class, () -> + transformer.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class))); + Assertions.assertNotNull( + findCausalException(thrownException.getErrors().iterator().next().getException(), + e->e==null || e instanceof ThrowTag.JinjavaThrowTagException)); + } + + private static @NonNull String makeCreateIndexRequestWithoutTypes() { + return "{\n" + + " \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" + + " \"" + JsonKeysForHttpMessage.URI_KEY + "\": \"/geonames\",\n" + + " \"" + JsonKeysForHttpMessage.PROTOCOL_KEY + "\": \"HTTP/1.1\"," + + " \"" + JsonKeysForHttpMessage.HEADERS_KEY + "\": {\n" + + " \"Host\": \"capture-proxy:9200\"\n" + + " }," + + " \"" + JsonKeysForHttpMessage.PAYLOAD_KEY + "\": {\n" + + " \"" + JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY + "\": {\n" + + " \"settings\": {\n" + + " \"index\": {\n" + + " \"number_of_shards\": 3, \n" + + " \"number_of_replicas\": 2 \n" + + " }\n" + + " }," + + " \"mappings\": {" + + " \"properties\": {\n" + + " \"field1\": { \"type\": \"text\" }\n" + + " }" + + " }\n" + + " }\n" + + " }\n" + + "}"; + } + + public static Throwable findCausalException(Throwable t, Predicate p) { + while (!p.test(t)) { + t = t.getCause(); + } + return t; + } +} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/resources/log4j2.properties b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/resources/log4j2.properties new file mode 100644 index 000000000..6adca47b5 --- /dev/null +++ b/transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformerProvider/src/test/resources/log4j2.properties @@ -0,0 +1,18 @@ +status = WARN + +property.ownedPackagesLogLevel=${sys:migrationLogLevel:-INFO} + +appender.console.type = Console +appender.console.name = Console +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = Console + +# Allow customization of owned package logs +logger.rfs.name = org.opensearch.migrations.bulkload +logger.rfs.level = ${ownedPackagesLogLevel} +logger.migration.name = org.opensearch.migrations +logger.migration.level = ${ownedPackagesLogLevel} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java deleted file mode 100644 index 14f60b499..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTransformerForOpenSearch23PlusTargetTransformerProvider.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.opensearch.migrations.transform; - -public class JsonTransformerForOpenSearch23PlusTargetTransformerProvider implements IJsonTransformerProvider { - @Override - public IJsonTransformer createTransformer(Object jsonConfig) { - return new JsonTypeMappingTransformer(); - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java deleted file mode 100644 index b141b5045..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/java/org/opensearch/migrations/transform/JsonTypeMappingTransformer.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.opensearch.migrations.transform; - -import java.util.Map; -import java.util.regex.Pattern; - -/** - * This is an experimental JsonTransformer that is meant to perform basic URI and payload transformations - * to excise index type mappings for relevant operations. - */ -public class JsonTypeMappingTransformer implements IJsonTransformer { - /** - * This is used to match a URI of the form /INDEX/TYPE/foo... so that it can be - * transformed into /INDEX/foo... - */ - static final Pattern TYPED_OPERATION_URI_PATTERN_WITH_SIDE_CAPTURES = Pattern.compile( - "^(\\/[^\\/]*)\\/[^\\/]*(\\/[^\\/]*)$" - ); - - /** - * This is used to match a URI of the form /foo... - */ - static final Pattern SINGLE_LEVEL_OPERATION_PATTERN_WITH_CAPTURE = Pattern.compile("^(\\/[^\\/]*)$"); - public static final String SEARCH_URI_COMPONENT = "/_search"; - public static final String DOC_URI_COMPONENT = "/_doc"; - public static final String MAPPINGS_KEYNAME = "mappings"; - - @Override - public Map transformJson(Map incomingJson) { - return transformHttpMessage(incomingJson); - } - - private Map transformHttpMessage(Map httpMsg) { - var incomingMethod = httpMsg.get(JsonKeysForHttpMessage.METHOD_KEY); - if ("GET".equals(incomingMethod)) { - processGet(httpMsg); - } else if ("PUT".equals(incomingMethod)) { - processPut(httpMsg); - } - return httpMsg; - } - - private void processGet(Map httpMsg) { - var incomingUri = (String) httpMsg.get(JsonKeysForHttpMessage.URI_KEY); - var matchedUri = TYPED_OPERATION_URI_PATTERN_WITH_SIDE_CAPTURES.matcher(incomingUri); - if (matchedUri.matches()) { - var operationStr = matchedUri.group(2); - if (operationStr.equals(SEARCH_URI_COMPONENT)) { - httpMsg.put(JsonKeysForHttpMessage.URI_KEY, matchedUri.group(1) + operationStr); - } - } - } - - private void processPut(Map httpMsg) { - final var uriStr = (String) httpMsg.get(JsonKeysForHttpMessage.URI_KEY); - var matchedTriple = TYPED_OPERATION_URI_PATTERN_WITH_SIDE_CAPTURES.matcher(uriStr); - if (matchedTriple.matches()) { - // TODO: Add support for multiple type mappings per index (something possible with - // versions before ES7) - httpMsg.put( - JsonKeysForHttpMessage.URI_KEY, - matchedTriple.group(1) + DOC_URI_COMPONENT + matchedTriple.group(2) - ); - return; - } - var matchedSingle = SINGLE_LEVEL_OPERATION_PATTERN_WITH_CAPTURE.matcher(uriStr); - if (matchedSingle.matches()) { - var topPayloadElement = (Map) ((Map) httpMsg.get( - JsonKeysForHttpMessage.PAYLOAD_KEY - )).get(JsonKeysForHttpMessage.INLINED_JSON_BODY_DOCUMENT_KEY); - var mappingsValue = (Map) topPayloadElement.get(MAPPINGS_KEYNAME); - if (mappingsValue != null) { - exciseMappingsType(topPayloadElement, mappingsValue); - } - } - } - - private void exciseMappingsType(Map mappingsParent, Map mappingsValue) { - var firstMappingOp = mappingsValue.entrySet().stream().findFirst(); - firstMappingOp.ifPresent(firstMapping -> mappingsParent.put(MAPPINGS_KEYNAME, firstMapping.getValue())); - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider deleted file mode 100644 index 34d314de6..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/main/resources/META-INF/services/org.opensearch.migrations.transform.IJsonTransformerProvider +++ /dev/null @@ -1 +0,0 @@ -org.opensearch.migrations.transform.JsonTransformerForOpenSearch23PlusTargetTransformerProvider \ No newline at end of file diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java deleted file mode 100644 index d732b031f..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/java/org/opensearch/migrations/transform/TypeMappingsExcisionTest.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.opensearch.migrations.transform; - -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.opensearch.migrations.replay.datahandlers.JsonAccumulator; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.io.CharStreams; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class TypeMappingsExcisionTest { - - static final TypeReference> TYPE_REFERENCE_FOR_MAP_TYPE = new TypeReference<>() { - }; - - static ObjectMapper objectMapper = new ObjectMapper(); - - static InputStream getInputStreamForTypeMappingResource(String resourceName) { - return TypeMappingsExcisionTest.class.getResourceAsStream("/sampleJsonDocuments/typeMappings/" + resourceName); - } - - @Test - public void removesTypeMappingsFrom_indexCreation() throws Exception { - var json = parseJsonFromResourceName("put_index_input.txt"); - transformAndVerifyResult(json, "put_index_output.txt"); - } - - @Test - public void removesTypeMappingsFrom_documentPut() throws Exception { - var json = parseJsonFromResourceName("put_document_input.txt"); - transformAndVerifyResult(json, "put_document_output.txt"); - } - - @Test - public void removesTypeMappingsFrom_queryGet() throws Exception { - var json = parseJsonFromResourceName("get_query_input.txt"); - transformAndVerifyResult(json, "get_query_output.txt"); - } - - private static Map parseJsonFromResourceName(String resourceName) throws Exception { - var jsonAccumulator = new JsonAccumulator(); - try ( - var resourceStream = getInputStreamForTypeMappingResource(resourceName); - var isr = new InputStreamReader(resourceStream, StandardCharsets.UTF_8) - ) { - var expectedBytes = CharStreams.toString(isr).getBytes(StandardCharsets.UTF_8); - return (Map) jsonAccumulator.consumeByteBufferForSingleObject(ByteBuffer.wrap(expectedBytes)); - } - } - - private static void transformAndVerifyResult(Map json, String expectedValueSource) - throws Exception { - var jsonTransformer = getJsonTransformer(); - json = jsonTransformer.transformJson(json); - var jsonAsStr = objectMapper.writeValueAsString(json); - Object expectedObject = parseJsonFromResourceName(expectedValueSource); - var expectedValue = objectMapper.writeValueAsString(expectedObject); - Assertions.assertEquals(expectedValue, jsonAsStr); - } - - static IJsonTransformer getJsonTransformer() { - return new JsonTypeMappingTransformer(); - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_input.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_input.txt deleted file mode 100644 index 14318a64b..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_input.txt +++ /dev/null @@ -1,9 +0,0 @@ -{ - "method": "GET", - "URI": "/oldStyleIndex/oldType/_search", - "payload": { - "inlinedJsonBody": { - "query": { "match": { "field1": "VALUE_1" } } - } - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_output.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_output.txt deleted file mode 100644 index 33ba1bc3a..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/get_query_output.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "method": "GET", - "URI": "/oldStyleIndex/_search", - "payload": { - "inlinedJsonBody": { - "query": { "match": { "field1": "VALUE_1" } - } - } - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_input.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_input.txt deleted file mode 100644 index 1e169fcfd..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_input.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "method": "PUT", - "URI": "/oldStyleIndex/oldType/1", - "payload": { - "inlinedJsonBody": { - "field1": "VALUE1", - "field2": "VALUE2" - } - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_output.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_output.txt deleted file mode 100644 index 24902fc0a..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_document_output.txt +++ /dev/null @@ -1,10 +0,0 @@ -{ - "method": "PUT", - "URI": "/oldStyleIndex/_doc/1", - "payload": { - "inlinedJsonBody": { - "field1": "VALUE1", - "field2": "VALUE2" - } - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_input.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_input.txt deleted file mode 100644 index beed65ea3..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_input.txt +++ /dev/null @@ -1,20 +0,0 @@ -{ - "method": "PUT", - "URI": "/oldStyleIndex", - "payload": { - "inlinedJsonBody": { - "mappings": { - "oldType": { - "properties": { - "field1": { - "type": "text" - }, - "field2": { - "type": "keyword" - } - } - } - } - } - } -} diff --git a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_output.txt b/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_output.txt deleted file mode 100644 index 3ae3b4b2f..000000000 --- a/transformation/transformationPlugins/jsonMessageTransformers/openSearch23PlusTargetTransformerProvider/src/test/resources/sampleJsonDocuments/typeMappings/put_index_output.txt +++ /dev/null @@ -1,18 +0,0 @@ -{ - "method": "PUT", - "URI": "/oldStyleIndex", - "payload": { - "inlinedJsonBody": { - "mappings": { - "properties": { - "field1": { - "type": "text" - }, - "field2": { - "type": "keyword" - } - } - } - } - } -}