Merge remote-tracking branch 'upstream/main' into MIGRATIONS-2128

Signed-off-by: Andre Kurait <[email protected]>
AndreKurait · Dec 12, 2024 · 77f0244 · 77f0244
2 parents bfce5e1 + ac5dd29
commit 77f0244
Show file tree

Hide file tree

Showing 127 changed files with 3,880 additions and 715 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -161,6 +161,7 @@ jobs:
         uses: ScribeMD/[email protected]
         with:
           key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
+          read-only: true
       - name: Start Docker Solution
         run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
         env:

diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/IFieldCreator.java
@@ -10,7 +10,8 @@ public interface IFieldCreator {
     ObjectMapper mapper = new ObjectMapper();
 
     default ObjectNode createField(ElasticsearchType type) {
-        return mapper.createObjectNode().put("type", type.getValue());
+        String typeString = type.getValue();
+        return mapper.createObjectNode().put("type", typeString);
     }
 
     default ObjectNode fieldGeoPoint() { return createField(ElasticsearchType.GEO_POINT); }

diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/HttpLogs.java
@@ -83,8 +83,9 @@ public Stream<ObjectNode> createDocs(int numDocs) {
         return IntStream.range(0, numDocs)
             .mapToObj(i -> {
                 var random = new Random(i);
+                long randomTime = randomTime(currentTime, random);
                 return mapper.createObjectNode()
-                    .put("@timestamp", randomTime(currentTime, random))
+                    .put("@timestamp", randomTime)
                     .put("clientip", randomIpAddress(random))
                     .put("request", randomRequest(random))
                     .put("status", randomStatus(random))

diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/Nested.java
@@ -104,8 +104,9 @@ private ArrayNode randomAnswers(ObjectMapper mapper, long timeFrom, Random rando
         var numAnswers = random.nextInt(5) + 1;
 
         for (int i = 0; i < numAnswers; i++) {
+            long randomTime = randomTime(timeFrom, random);
             var answer = mapper.createObjectNode()
-                .put("date", randomTime(timeFrom, random))
+                .put("date", randomTime)
                 .put("user", randomUser(random));
             answers.add(answer);
         }

diff --git a/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java b/DataGenerator/src/main/java/org/opensearch/migrations/data/workloads/NycTaxis.java
@@ -100,21 +100,29 @@ public Stream<ObjectNode> createDocs(int numDocs) {
         return IntStream.range(0, numDocs)
             .mapToObj(i -> {
                 var random = new Random(i);
+                double totalAmount = randomDouble(random, 5.0, 50.0);
+                String pickupTime = randomTimeISOString(currentTime, random);
+                String dropOffTime = randomTimeISOString(currentTime, random);
+                double tolls = randomDouble(random, 0.0, 5.0);
+                double fare = randomDouble(random, 5.0, 50.0);
+                double extra = randomDouble(random, 0.0, 1.0);
+                double tripDistance = randomDouble(random, 0.5, 20.0);
+                double tip = randomDouble(random, 0.0, 15.0);
                 return mapper.createObjectNode()
-                .<ObjectNode>put("total_amount", randomDouble(random, 5.0, 50.0))
+                .<ObjectNode>put("total_amount", totalAmount)
                 .<ObjectNode>put("improvement_surcharge", 0.3)
                 .<ObjectNode>set("pickup_location", randomLocationInNyc(random))
-                .<ObjectNode>put("pickup_datetime", randomTimeISOString(currentTime, random))
+                .<ObjectNode>put("pickup_datetime", pickupTime)
                 .<ObjectNode>put("trip_type", randomTripType(random))
-                .<ObjectNode>put("dropoff_datetime", randomTimeISOString(currentTime, random))
+                .<ObjectNode>put("dropoff_datetime", dropOffTime)
                 .<ObjectNode>put("rate_code_id", "1")
-                .<ObjectNode>put("tolls_amount", randomDouble(random, 0.0, 5.0))
+                .<ObjectNode>put("tolls_amount", tolls)
                 .<ObjectNode>set("dropoff_location", randomLocationInNyc(random))
                 .<ObjectNode>put("passenger_count", random.nextInt(4) + 1)
-                .<ObjectNode>put("fare_amount", randomDouble(random, 5.0, 50.0))
-                .<ObjectNode>put("extra", randomDouble(random, 0.0, 1.0))
-                .<ObjectNode>put("trip_distance", randomDouble(random, 0.5, 20.0))
-                .<ObjectNode>put("tip_amount", randomDouble(random, 0.0, 15.0))
+                .<ObjectNode>put("fare_amount", fare)
+                .<ObjectNode>put("extra", extra)
+                .<ObjectNode>put("trip_distance", tripDistance)
+                .<ObjectNode>put("tip_amount", tip)
                 .<ObjectNode>put("store_and_fwd_flag", randomStoreAndFwdFlag(random))
                 .<ObjectNode>put("payment_type", randomPaymentType(random))
                 .<ObjectNode>put("mta_tax", 0.5)

diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle
@@ -30,7 +30,7 @@ dependencies {
     implementation project(":RFS")
     implementation project(":transformation")
     implementation project(':transformation:transformationPlugins:jsonMessageTransformers:jsonMessageTransformerLoaders')
-    runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:openSearch23PlusTargetTransformerProvider')
+    runtimeOnly project(':transformation:transformationPlugins:jsonMessageTransformers:jsonTypeMappingsSanitizationTransformerProvider')
 
     implementation group: 'org.apache.logging.log4j', name: 'log4j-api'
     implementation group: 'org.apache.logging.log4j', name: 'log4j-core'

diff --git a/...sFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java b/...sFromSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/EndToEndTest.java
@@ -100,9 +100,9 @@ private void migrationDocumentsWithClusters(
 
 
             sourceClusterOperations.createDocument(indexName, "222", "{\"author\":\"Tobias Funke\"}");
-            sourceClusterOperations.createDocument(indexName, "223", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1");
-            sourceClusterOperations.createDocument(indexName, "224", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1");
-            sourceClusterOperations.createDocument(indexName, "225", "{\"author\":\"Tobias Funke\", \"category\": \"tech\"}", "2");
+            sourceClusterOperations.createDocument(indexName, "223", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1", null);
+            sourceClusterOperations.createDocument(indexName, "224", "{\"author\":\"Tobias Funke\", \"category\": \"cooking\"}", "1", null);
+            sourceClusterOperations.createDocument(indexName, "225", "{\"author\":\"Tobias Funke\", \"category\": \"tech\"}", "2", null);
 
             // === ACTION: Take a snapshot ===
             var snapshotName = "my_snap";

diff --git a/...tion/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java b/...tion/src/test/java/org/opensearch/migrations/bulkload/ParallelDocumentMigrationsTest.java
@@ -39,16 +39,13 @@ public class ParallelDocumentMigrationsTest extends SourceTestBase {
     public static Stream<Arguments> makeDocumentMigrationArgs() {
         var numWorkersList = List.of(1, 3, 40);
         var compressionEnabledList = List.of(true, false);
-        return SupportedClusters.targets().stream()
-            .flatMap(
-                targetImage -> numWorkersList.stream()
+        return numWorkersList.stream()
                     .flatMap(numWorkers -> compressionEnabledList.stream().map(compression -> Arguments.of(
                             numWorkers,
-                            targetImage,
+                            SearchClusterContainer.OS_V2_14_0,
                             compression
                         ))
-                    )
-            );
+                    );
     }
 
     @ParameterizedTest
@@ -70,8 +67,8 @@ public void testDocumentMigration(
             var osTargetContainer = new SearchClusterContainer(targetVersion);
         ) {
             CompletableFuture.allOf(
-                CompletableFuture.runAsync(() ->  esSourceContainer.start(), executorService),
-                CompletableFuture.runAsync(() ->  osTargetContainer.start(), executorService)
+                CompletableFuture.runAsync(esSourceContainer::start, executorService),
+                CompletableFuture.runAsync(osTargetContainer::start, executorService)
             ).join();
 
             // Populate the source cluster with data

diff --git a/...gration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java b/...gration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import reactor.core.publisher.Mono;
@@ -36,6 +37,7 @@
 import static org.mockito.Mockito.when;
 
 @Slf4j
+@Disabled("https://opensearch.atlassian.net/browse/MIGRATIONS-2254")
 public class PerformanceVerificationTest {
 
     @Test

diff --git a/...romSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java b/...romSnapshotMigration/src/test/java/org/opensearch/migrations/bulkload/SourceTestBase.java
@@ -60,10 +60,10 @@ public class SourceTestBase {
     public static final long TOLERABLE_CLIENT_SERVER_CLOCK_DIFFERENCE_SECONDS = 3600;
 
     protected static Object[] makeParamsForBase(SearchClusterContainer.ContainerVersion baseSourceImage) {
-        return new Object[] {
+        return new Object[]{
             baseSourceImage,
             GENERATOR_BASE_IMAGE,
-            new String[] { "/root/runTestBenchmarks.sh", "--endpoint", "http://" + SOURCE_SERVER_ALIAS + ":9200/" } };
+            new String[]{"/root/runTestBenchmarks.sh", "--endpoint", "http://" + SOURCE_SERVER_ALIAS + ":9200/"}};
     }
 
     @NotNull
@@ -138,7 +138,7 @@ public static int migrateDocumentsSequentially(
         Version version,
         boolean compressionEnabled
     ) {
-        for (int runNumber = 1;; ++runNumber) {
+        for (int runNumber = 1; ; ++runNumber) {
             try {
                 var workResult = migrateDocumentsWithOneWorker(
                     sourceRepo,
@@ -182,7 +182,8 @@ public Flux<RfsLuceneDocument> readDocuments(int startDoc) {
         }
     }
 
-    static class LeasePastError extends Error {}
+    static class LeasePastError extends Error {
+    }
 
     @SneakyThrows
     public static DocumentsRunner.CompletionStatus migrateDocumentsWithOneWorker(

diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle
@@ -27,6 +27,7 @@ dependencies {
     testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter'
     testImplementation group: 'org.hamcrest', name: 'hamcrest'
     testImplementation group: 'org.testcontainers', name: 'testcontainers'
+    testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind'
 
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine'
 }

diff --git a/MetadataMigration/docs/DESIGN.md b/MetadataMigration/docs/DESIGN.md
@@ -142,17 +142,17 @@ Migration Candidates:
 
 Transformations:
    Index:
-      ERROR - IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "Multiple mapping types are not supported""
+      ERROR - IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "No multi type resolution behavior declared, specify --multi-type-behavior to process""
    Index Template:
-      ERROR - IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "Multiple mapping types are not supported"
+      ERROR - IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "No multi type resolution behavior declared, specify --multi-type-behavior to process"
    DEBUG - 6 transformations did not apply, add --`full` to see all results
 
 Result:
    2 migration issues detected
 
 Issues:
-   IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "Multiple mapping types are not supported""
-   IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "Multiple mapping types are not supported"
+   IndexMappingTypeRemoval is Unsupported on Index `logs-181998` "No multi type resolution behavior declared, specify --multi-type-behavior to process""
+   IndexMappingTypeRemoval is Unsupported on Index Template `daily_logs` "No multi type resolution behavior declared, specify --multi-type-behavior to process"
 ```
 ### Exclude incompatible rolling logs indices
 

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java
@@ -4,8 +4,11 @@
 
 import org.opensearch.migrations.bulkload.common.http.ConnectionContext;
 import org.opensearch.migrations.bulkload.models.DataFilterArgs;
+import org.opensearch.migrations.bulkload.transformers.MetadataTransformerParams;
 import org.opensearch.migrations.transform.TransformerParams;
+import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval;
 
+import com.beust.jcommander.IStringConverter;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParametersDelegate;
 import lombok.Getter;
@@ -56,10 +59,20 @@ public class MigrateOrEvaluateArgs {
     public Version sourceVersion = null;
 
     @ParametersDelegate
-    public TransformerParams metadataTransformationParams = new MetadataTransformerParams();
+    public MetadataTransformationParams metadataTransformationParams = new MetadataTransformationParams();
+
+    @ParametersDelegate
+    public TransformerParams metadataCustomTransformationParams = new MetadataCustomTransformationParams();
+
+    @Getter
+    public static class MetadataTransformationParams implements MetadataTransformerParams {
+        @Parameter(names = {"--multi-type-behavior"}, description = "Define behavior for resolving multi type mappings.")
+        public IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.NONE;
+    }
 
     @Getter
-    public static class MetadataTransformerParams implements TransformerParams {
+    public static class MetadataCustomTransformationParams implements TransformerParams {
+
         public String getTransformerConfigParameterArgPrefix() {
             return "";
         }
@@ -89,4 +102,11 @@ public String getTransformerConfigParameterArgPrefix() {
                 description = "Path to the JSON configuration file of metadata transformers.")
         private String transformerConfigFile;
     }
+
+    static class MultiTypeResolutionBehaviorConverter implements IStringConverter<IndexMappingTypeRemoval.MultiTypeResolutionBehavior> {
+        @Override
+        public IndexMappingTypeRemoval.MultiTypeResolutionBehavior convert(String value) {
+            return IndexMappingTypeRemoval.MultiTypeResolutionBehavior.valueOf(value.toUpperCase());
+        }
+    }
 }
diff --git a/...dataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java b/...dataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java
@@ -55,7 +55,7 @@ protected Clusters createClusters() {
     }
 
     protected Transformer getCustomTransformer() {
-        var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataTransformationParams);
+        var transformerConfig = TransformerConfigUtils.getTransformerConfig(arguments.metadataCustomTransformationParams);
         if (transformerConfig != null) {
             log.atInfo().setMessage("Metadata Transformations config string: {}")
                     .addArgument(transformerConfig).log();
@@ -72,7 +72,8 @@ protected Transformer selectTransformer(Clusters clusters) {
         var versionTransformer = TransformFunctions.getTransformer(
             clusters.getSource().getVersion(),
             clusters.getTarget().getVersion(),
-            arguments.minNumberOfReplicas
+            arguments.minNumberOfReplicas,
+            arguments.metadataTransformationParams
         );
         var customTransformer = getCustomTransformer();
         var compositeTransformer = new CompositeTransformer(customTransformer, versionTransformer);