From 1e81a14e3f84dd73ad3251bf04b953194e6cece2 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Tue, 10 Dec 2024 13:05:01 -0600 Subject: [PATCH] Simplify E2E tests in metadata Signed-off-by: Andre Kurait --- MetadataMigration/build.gradle | 1 + .../migrations/MigrateOrEvaluateArgs.java | 6 +- .../migrations/BaseMigrationTest.java | 144 +++++++++ .../migrations/CustomTransformationTest.java | 101 ++---- .../opensearch/migrations/EndToEndTest.java | 175 ++++------- .../MultiTypeMappingTransformationTest.java | 295 ++++-------------- .../integration/SnapshotStateTest.java | 9 +- .../bulkload/http/ClusterOperations.java | 17 +- .../rules/IndexMappingTypeRemoval.java | 1 + 9 files changed, 316 insertions(+), 433 deletions(-) create mode 100644 MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java diff --git a/MetadataMigration/build.gradle b/MetadataMigration/build.gradle index dc66ccb5a..53096741f 100644 --- a/MetadataMigration/build.gradle +++ b/MetadataMigration/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter' testImplementation group: 'org.hamcrest', name: 'hamcrest' testImplementation group: 'org.testcontainers', name: 'testcontainers' + testImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind' testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine' } diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java index 287b05fee..78079b75b 100644 --- a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java +++ b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java @@ -59,15 +59,15 @@ public class MigrateOrEvaluateArgs { public Version sourceVersion = null; @ParametersDelegate - public MetadataTransformerParams metadataTransformationParams = new MetadataTransformationParams(); + public MetadataTransformationParams metadataTransformationParams = new MetadataTransformationParams(); @ParametersDelegate public TransformerParams metadataCustomTransformationParams = new MetadataCustomTransformationParams(); @Getter public static class MetadataTransformationParams implements MetadataTransformerParams { - @Parameter(names = {"--multi-type-behavior"}, description = "How to resolve multi type mappings e.g SPLIT or UNION.", converter = MultiTypeResolutionBehaviorConverter.class) - public IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.SPLIT; + @Parameter(names = {"--multi-type-behavior"}, description = "Define behavior for resolving multi type mappings.") + public IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.NONE; } @Getter diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java new file mode 100644 index 000000000..b8236de7a --- /dev/null +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java @@ -0,0 +1,144 @@ +package org.opensearch.migrations; + +import java.io.File; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; +import org.opensearch.migrations.bulkload.common.OpenSearchClient; +import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; +import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; +import org.opensearch.migrations.bulkload.http.ClusterOperations; +import org.opensearch.migrations.bulkload.worker.SnapshotRunner; +import org.opensearch.migrations.commands.MigrationItemResult; +import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; +import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; + +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.io.TempDir; + +/** + * Base test class providing shared setup and utility methods for migration tests. + */ +@Slf4j +abstract class BaseMigrationTest { + + @TempDir + protected File localDirectory; + + @Getter + protected SearchClusterContainer sourceCluster; + @Getter + protected SearchClusterContainer targetCluster; + + protected ClusterOperations sourceOperations; + protected ClusterOperations targetOperations; + + /** + * Starts the source and target clusters. + */ + protected void startClusters() { + CompletableFuture.allOf( + CompletableFuture.runAsync(sourceCluster::start), + CompletableFuture.runAsync(targetCluster::start) + ).join(); + + sourceOperations = new ClusterOperations(sourceCluster.getUrl()); + targetOperations = new ClusterOperations(targetCluster.getUrl()); + } + + /** + * Sets up a snapshot repository and takes a snapshot of the source cluster. + * + * @param snapshotName Name of the snapshot. + * @return The name of the created snapshot. + */ + @SneakyThrows + protected String createSnapshot(String snapshotName) { + var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); + var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() + .host(sourceCluster.getUrl()) + .insecure(true) + .build() + .toConnectionContext()); + var snapshotCreator = new org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator( + snapshotName, + sourceClient, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + List.of(), + snapshotContext.createSnapshotCreateContext() + ); + org.opensearch.migrations.bulkload.worker.SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + sourceCluster.copySnapshotData(localDirectory.toString()); + return snapshotName; + } + + /** + * Prepares migration arguments for snapshot-based migrations. + * + * @param snapshotName Name of the snapshot. + * @return Prepared migration arguments. + */ + protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) { + var arguments = new MigrateOrEvaluateArgs(); + arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); + arguments.snapshotName = snapshotName; + arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + arguments.targetArgs.host = targetCluster.getUrl(); + return arguments; + } + + /** + * Executes the migration command (either migrate or evaluate). + * + * @param arguments Migration arguments. + * @param command The migration command to execute. + * @return The result of the migration. + */ + protected MigrationItemResult executeMigration(MigrateOrEvaluateArgs arguments, MetadataCommands command) { + var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); + var metadata = new MetadataMigration(); + + if (MetadataCommands.MIGRATE.equals(command)) { + return metadata.migrate(arguments).execute(metadataContext); + } else { + return metadata.evaluate(arguments).execute(metadataContext); + } + } + + /** + * Creates an OpenSearch client for the given cluster. + * + * @param cluster The cluster container. + * @return An OpenSearch client. + */ + protected OpenSearchClient createClient(SearchClusterContainer cluster) { + return new OpenSearchClient(ConnectionContextTestParams.builder() + .host(cluster.getUrl()) + .insecure(true) + .build() + .toConnectionContext()); + } + + protected SnapshotTestContext createSnapshotContext() { + return SnapshotTestContext.factory().noOtelTracking(); + } + + protected FileSystemSnapshotCreator createSnapshotCreator(String snapshotName, OpenSearchClient client, SnapshotTestContext context) { + return new FileSystemSnapshotCreator( + snapshotName, + client, + SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, + List.of(), + context.createSnapshotCreateContext() + ); + } + + @SneakyThrows + protected void runSnapshotAndCopyData(FileSystemSnapshotCreator snapshotCreator, SearchClusterContainer cluster) { + SnapshotRunner.runAndWaitForCompletion(snapshotCreator); + cluster.copySnapshotData(localDirectory.toString()); + } +} diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java index 94e695185..def6fdabf 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/CustomTransformationTest.java @@ -1,22 +1,12 @@ package org.opensearch.migrations; -import java.io.File; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; import org.opensearch.migrations.bulkload.SupportedClusters; -import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; -import org.opensearch.migrations.bulkload.common.OpenSearchClient; -import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; -import org.opensearch.migrations.bulkload.http.ClusterOperations; import org.opensearch.migrations.bulkload.models.DataFilterArgs; -import org.opensearch.migrations.bulkload.transformers.MetadataTransformerParams; -import org.opensearch.migrations.bulkload.worker.SnapshotRunner; import org.opensearch.migrations.commands.MigrationItemResult; -import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; -import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; import org.opensearch.migrations.transform.TransformerParams; import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval; @@ -25,7 +15,6 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -39,10 +28,7 @@ */ @Tag("isolatedTest") @Slf4j -class CustomTransformationTest { - - @TempDir - private File localDirectory; +class CustomTransformationTest extends BaseMigrationTest { private static Stream scenarios() { // Define scenarios with different source and target cluster versions @@ -62,23 +48,17 @@ void customTransformationMetadataMigration( final var sourceCluster = new SearchClusterContainer(sourceVersion); final var targetCluster = new SearchClusterContainer(targetVersion) ) { - performCustomTransformationTest(sourceCluster, targetCluster); + this.sourceCluster = sourceCluster; + this.targetCluster = targetCluster; + performCustomTransformationTest(); } } @SneakyThrows - private void performCustomTransformationTest( - final SearchClusterContainer sourceCluster, - final SearchClusterContainer targetCluster - ) { - // Start both source and target clusters asynchronously - CompletableFuture.allOf( - CompletableFuture.runAsync(sourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ).join(); + private void performCustomTransformationTest() { + startClusters(); - var sourceOperations = new ClusterOperations(sourceCluster.getUrl()); - var targetOperations = new ClusterOperations(targetCluster.getUrl()); + var newComponentCompatible = sourceCluster.getContainerVersion().getVersion().getMajor() >= 7; // Test data var originalIndexName = "test_index"; @@ -95,28 +75,22 @@ private void performCustomTransformationTest( var legacyTemplatePattern = "legacy_*"; sourceOperations.createLegacyTemplate(legacyTemplateName, legacyTemplatePattern); - // Create index template + // Create index template and component template if compatible var indexTemplateName = "index_template"; var indexTemplatePattern = "index*"; - - // Create component template var componentTemplateName = "component_template"; - var componentTemplateMode = "mode_value"; // Replace with actual mode if applicable - boolean newComponentCompatible = sourceCluster.getContainerVersion().getVersion().getMajor() >= 7; if (newComponentCompatible) { sourceOperations.createIndexTemplate(indexTemplateName, "dummy", indexTemplatePattern); - - var componentTemplateAdditionalParam = "additional_param"; // Replace with actual param if applicable - sourceOperations.createComponentTemplate(componentTemplateName, indexTemplateName, componentTemplateAdditionalParam, "index*"); + sourceOperations.createComponentTemplate(componentTemplateName, indexTemplateName, "additional_param", "index*"); } - // Create index that matches the templates + // Create indices that match the templates var legacyIndexName = "legacy_index"; var indexIndexName = "index_index"; sourceOperations.createIndex(legacyIndexName); sourceOperations.createIndex(indexIndexName); - // Define custom transformations for index, legacy, and component templates + // Define custom transformations String customTransformationJson = "[\n" + " {\n" + " \"JsonConditionalTransformerProvider\": [\n" + @@ -185,52 +159,23 @@ private void performCustomTransformationTest( " }\n" + "]"; - var arguments = new MigrateOrEvaluateArgs(); - - // Use SnapshotImage as the transfer medium - var snapshotName = "custom_transformation_snap"; - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(sourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - sourceCluster.copySnapshotData(localDirectory.toString()); - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = snapshotName; - arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + var snapshotName = createSnapshot("custom_transformation_snap"); + var arguments = prepareSnapshotMigrationArgs(snapshotName); - arguments.targetArgs.host = targetCluster.getUrl(); - - // Set up data filters to include only the test index and templates + // Set up data filters var dataFilterArgs = new DataFilterArgs(); dataFilterArgs.indexAllowlist = List.of(originalIndexName, legacyIndexName, indexIndexName, transformedIndexName); dataFilterArgs.indexTemplateAllowlist = List.of(indexTemplateName, legacyTemplateName, "transformed_legacy_template", "transformed_index_template"); dataFilterArgs.componentTemplateAllowlist = List.of(componentTemplateName, "transformed_component_template"); arguments.dataFilterArgs = dataFilterArgs; - // Use split for multi type mappings resolution - arguments.metadataTransformationParams = TestMetadataTransformationParams.builder() - .multiTypeResolutionBehavior(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.SPLIT) - .build(); - // Specify the custom transformer configuration + // Set up transformation parameters arguments.metadataCustomTransformationParams = TestCustomTransformationParams.builder() .transformerConfig(customTransformationJson) .build(); - // Execute the migration with the custom transformation - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); - - MigrationItemResult result = metadata.migrate(arguments).execute(metadataContext); + // Execute migration + MigrationItemResult result = executeMigration(arguments, MetadataCommands.MIGRATE); // Verify the migration result log.info(result.asCliOutput()); @@ -245,31 +190,26 @@ private void performCustomTransformationTest( res = targetOperations.get("/" + originalIndexName); assertThat(res.getKey(), equalTo(404)); - // Verify that the transformed legacy template exists on the target cluster + // Verify templates res = targetOperations.get("/_template/transformed_legacy_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_legacy_template")); - // Verify that the original legacy template does not exist on the target cluster - res = targetOperations.get("/_template/" + legacyTemplateName); + res = targetOperations.get("/_template/" + "legacy_template"); assertThat(res.getKey(), equalTo(404)); if (newComponentCompatible) { - // Verify that the transformed index template exists on the target cluster res = targetOperations.get("/_index_template/transformed_index_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_index_template")); - // Verify that the original index template does not exist on the target cluster res = targetOperations.get("/_index_template/" + indexTemplateName); assertThat(res.getKey(), equalTo(404)); - // Verify that the transformed component template exists on the target cluster res = targetOperations.get("/_component_template/transformed_component_template"); assertThat(res.getKey(), equalTo(200)); assertThat(res.getValue(), containsString("transformed_component_template")); - // Verify that the original component template does not exist on the target cluster res = targetOperations.get("/_component_template/" + componentTemplateName); assertThat(res.getKey(), equalTo(404)); } @@ -287,8 +227,7 @@ private static class TestCustomTransformationParams implements TransformerParams @Data @Builder - private static class TestMetadataTransformationParams implements MetadataTransformerParams { + private static class TestMetadataTransformationParams implements org.opensearch.migrations.bulkload.transformers.MetadataTransformerParams { private IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior; } - } diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java index 183f24acf..96dd511a3 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/EndToEndTest.java @@ -1,50 +1,33 @@ package org.opensearch.migrations; -import java.io.File; import java.util.Arrays; import java.util.List; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.migrations.bulkload.SupportedClusters; -import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; -import org.opensearch.migrations.bulkload.common.OpenSearchClient; -import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; -import org.opensearch.migrations.bulkload.framework.SearchClusterContainer.ContainerVersion; -import org.opensearch.migrations.bulkload.http.ClusterOperations; import org.opensearch.migrations.bulkload.models.DataFilterArgs; -import org.opensearch.migrations.bulkload.worker.SnapshotRunner; import org.opensearch.migrations.commands.MigrationItemResult; import org.opensearch.migrations.metadata.CreationResult; -import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; -import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; /** - * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster + * Tests focused on setting up whole source clusters, performing a migration, and validation on the target cluster. */ @Tag("isolatedTest") @Slf4j -class EndToEndTest { - - @TempDir - private File localDirectory; +class EndToEndTest extends BaseMigrationTest { private static Stream scenarios() { return SupportedClusters.sources().stream() @@ -72,15 +55,20 @@ private static Stream scenarios() { }); } - @ParameterizedTest(name = "From version {0} to version {1}, Command {2}, Medium of transfer {3}, and Template Type {4}") + @ParameterizedTest(name = "From version {0} to version {1}, Medium {2}, Command {3}, Template Type {4}") @MethodSource(value = "scenarios") - void metadataCommand(ContainerVersion sourceVersion, ContainerVersion targetVersion, TransferMedium medium, - MetadataCommands command, TemplateType templateType) { + void metadataCommand(SearchClusterContainer.ContainerVersion sourceVersion, + SearchClusterContainer.ContainerVersion targetVersion, + TransferMedium medium, + MetadataCommands command, + TemplateType templateType) { try ( - final var sourceCluster = new SearchClusterContainer(sourceVersion); - final var targetCluster = new SearchClusterContainer(targetVersion) + final var sourceCluster = new SearchClusterContainer(sourceVersion); + final var targetCluster = new SearchClusterContainer(targetVersion) ) { - metadataCommandOnClusters(sourceCluster, targetCluster, medium, command, templateType); + this.sourceCluster = sourceCluster; + this.targetCluster = targetCluster; + metadataCommandOnClusters(medium, command, templateType); } } @@ -96,96 +84,57 @@ private enum TemplateType { } @SneakyThrows - private void metadataCommandOnClusters( - final SearchClusterContainer sourceCluster, - final SearchClusterContainer targetCluster, - final TransferMedium medium, - final MetadataCommands command, - final TemplateType templateType - ) { - // ACTION: Set up the source/target clusters - var bothClustersStarted = CompletableFuture.allOf( - CompletableFuture.runAsync(sourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ); - bothClustersStarted.join(); + private void metadataCommandOnClusters(TransferMedium medium, + MetadataCommands command, + TemplateType templateType) { + startClusters(); var testData = new TestData(); - var sourceClusterOperations = new ClusterOperations(sourceCluster.getUrl()); + if (templateType == TemplateType.Legacy) { - sourceClusterOperations.createLegacyTemplate(testData.indexTemplateName, "blog*"); + sourceOperations.createLegacyTemplate(testData.indexTemplateName, "blog*"); } else if (templateType == TemplateType.Index) { - sourceClusterOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*"); + sourceOperations.createIndexTemplate(testData.indexTemplateName, "author", "blog*"); } else if (templateType == TemplateType.IndexAndComponent) { - sourceClusterOperations.createComponentTemplate(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*"); + sourceOperations.createComponentTemplate(testData.compoTemplateName, testData.indexTemplateName, "author", "blog*"); } // Creates a document that uses the template - sourceClusterOperations.createDocument(testData.blogIndexName, "222", "{\"author\":\"Tobias Funke\"}"); - sourceClusterOperations.createDocument(testData.movieIndexName,"123", "{\"title\":\"This is spinal tap\"}"); - sourceClusterOperations.createDocument(testData.indexThatAlreadyExists, "doc66", "{}"); + sourceOperations.createDocument(testData.blogIndexName, "222", "{\"author\":\"Tobias Funke\"}"); + sourceOperations.createDocument(testData.movieIndexName, "123", "{\"title\":\"This is Spinal Tap\"}"); + sourceOperations.createDocument(testData.indexThatAlreadyExists, "doc66", "{}"); - sourceClusterOperations.createAlias(testData.aliasName, "movies*"); - - var aliasName = "movies-alias"; - sourceClusterOperations.createAlias(aliasName, "movies*"); + sourceOperations.createAlias(testData.aliasName, "movies*"); var arguments = new MigrateOrEvaluateArgs(); switch (medium) { case SnapshotImage: - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var snapshotName = "my_snap"; - log.info("Source cluster {}", sourceCluster.getUrl()); - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(sourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - sourceCluster.copySnapshotData(localDirectory.toString()); - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = snapshotName; - arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); + var snapshotName = createSnapshot("my_snap"); + arguments = prepareSnapshotMigrationArgs(snapshotName); break; - + case Http: arguments.sourceArgs.host = sourceCluster.getUrl(); + arguments.targetArgs.host = targetCluster.getUrl(); break; } - arguments.targetArgs.host = targetCluster.getUrl(); - + // Set up data filters var dataFilterArgs = new DataFilterArgs(); dataFilterArgs.indexAllowlist = List.of(); dataFilterArgs.componentTemplateAllowlist = List.of(testData.compoTemplateName); dataFilterArgs.indexTemplateAllowlist = List.of(testData.indexTemplateName); arguments.dataFilterArgs = dataFilterArgs; - var targetClusterOperations = new ClusterOperations(targetCluster.getUrl()); - targetClusterOperations.createDocument(testData.indexThatAlreadyExists, "doc77", "{}"); - - // ACTION: Migrate the templates - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); - - MigrationItemResult result; - if (MetadataCommands.MIGRATE.equals(command)) { - result = metadata.migrate(arguments).execute(metadataContext); - } else { - result = metadata.evaluate(arguments).execute(metadataContext); - } + targetOperations.createDocument(testData.indexThatAlreadyExists, "doc77", "{}"); + + // Execute migration + MigrationItemResult result = executeMigration(arguments, command); verifyCommandResults(result, templateType, testData); - verifyTargetCluster(targetClusterOperations, command, templateType, testData); + verifyTargetCluster(command, templateType, testData); } private static class TestData { @@ -198,20 +147,25 @@ private static class TestData { final String indexThatAlreadyExists = "already-exists"; } - private void verifyCommandResults( - MigrationItemResult result, - TemplateType templateType, - TestData testData) { + private void verifyCommandResults(MigrationItemResult result, + TemplateType templateType, + TestData testData) { log.info(result.asCliOutput()); assertThat(result.getExitCode(), equalTo(0)); var migratedItems = result.getItems(); - assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), containsInAnyOrder(testData.indexTemplateName)); - assertThat(getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of())); - assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), containsInAnyOrder(testData.blogIndexName, testData.movieIndexName)); - assertThat(getNames(getFailedResultsByType(migratedItems.getIndexes(), CreationResult.CreationFailureType.ALREADY_EXISTS)), containsInAnyOrder(testData.indexThatAlreadyExists)); - assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); - + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexTemplates())), + containsInAnyOrder(testData.indexTemplateName)); + assertThat( + getNames(getSuccessfulResults(migratedItems.getComponentTemplates())), + equalTo(templateType.equals(TemplateType.IndexAndComponent) ? List.of(testData.compoTemplateName) : List.of()) + ); + assertThat(getNames(getSuccessfulResults(migratedItems.getIndexes())), + containsInAnyOrder(testData.blogIndexName, testData.movieIndexName)); + assertThat(getNames(getFailedResultsByType(migratedItems.getIndexes(), CreationResult.CreationFailureType.ALREADY_EXISTS)), + containsInAnyOrder(testData.indexThatAlreadyExists)); + assertThat(getNames(getSuccessfulResults(migratedItems.getAliases())), + containsInAnyOrder(testData.aliasInTemplate, testData.aliasName)); } private List getSuccessfulResults(List results) { @@ -227,43 +181,40 @@ private List getFailedResultsByType(List results } private List getNames(List items) { - return items.stream().map(r -> r.getName()).collect(Collectors.toList()); + return items.stream().map(CreationResult::getName).collect(Collectors.toList()); } - private void verifyTargetCluster( - ClusterOperations targetClusterOperations, - MetadataCommands command, - TemplateType templateType, - TestData testData - ) { + private void verifyTargetCluster(MetadataCommands command, + TemplateType templateType, + TestData testData) { var expectUpdatesOnTarget = MetadataCommands.MIGRATE.equals(command); - // If the command was migrate, the target cluster should have the items, if not they + // If the command was migrate, the target cluster should have the items, if not they shouldn't var verifyResponseCode = expectUpdatesOnTarget ? equalTo(200) : equalTo(404); // Check that the index was migrated - var res = targetClusterOperations.get("/" + testData.blogIndexName); + var res = targetOperations.get("/" + testData.blogIndexName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); - res = targetClusterOperations.get("/" + testData.movieIndexName); + res = targetOperations.get("/" + testData.movieIndexName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); - res = targetClusterOperations.get("/" + testData.aliasName); + res = targetOperations.get("/" + testData.aliasName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); if (expectUpdatesOnTarget) { assertThat(res.getValue(), containsString(testData.movieIndexName)); } - res = targetClusterOperations.get("/_aliases"); + res = targetOperations.get("/_aliases"); assertThat(res.getValue(), res.getKey(), equalTo(200)); var verifyAliasWasListed = allOf(containsString(testData.aliasInTemplate), containsString(testData.aliasName)); assertThat(res.getValue(), expectUpdatesOnTarget ? verifyAliasWasListed : not(verifyAliasWasListed)); // Check that the templates were migrated if (templateType.equals(TemplateType.Legacy)) { - res = targetClusterOperations.get("/_template/" + testData.indexTemplateName); + res = targetOperations.get("/_template/" + testData.indexTemplateName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); - } else if(templateType.equals(TemplateType.Index) || templateType.equals(TemplateType.IndexAndComponent)) { - res = targetClusterOperations.get("/_index_template/" + testData.indexTemplateName); + } else if (templateType.equals(TemplateType.Index) || templateType.equals(TemplateType.IndexAndComponent)) { + res = targetOperations.get("/_index_template/" + testData.indexTemplateName); assertThat(res.getValue(), res.getKey(), verifyResponseCode); if (templateType.equals(TemplateType.IndexAndComponent)) { var verifyBodyHasComponentTemplate = containsString("composed_of\":[\"" + testData.compoTemplateName + "\"]"); diff --git a/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java index 12467af24..e0093e633 100644 --- a/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java +++ b/MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java @@ -1,38 +1,22 @@ package org.opensearch.migrations; -import java.io.File; import java.util.List; -import java.util.concurrent.CompletableFuture; -import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; -import org.opensearch.migrations.bulkload.common.OpenSearchClient; -import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; import org.opensearch.migrations.bulkload.http.ClusterOperations; import org.opensearch.migrations.bulkload.models.DataFilterArgs; -import org.opensearch.migrations.bulkload.transformers.IndexTransformationException; -import org.opensearch.migrations.bulkload.transformers.MetadataTransformerParams; -import org.opensearch.migrations.bulkload.worker.SnapshotRunner; import org.opensearch.migrations.commands.MigrationItemResult; -import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; -import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; import org.opensearch.migrations.transformation.rules.IndexMappingTypeRemoval; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.testcontainers.containers.BindMode; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.migrations.bulkload.framework.SearchClusterContainer.ES_V6_8_23; @@ -41,118 +25,74 @@ */ @Tag("isolatedTest") @Slf4j -class MultiTypeMappingTransformationTest { - - @TempDir - private File localDirectory; +class MultiTypeMappingTransformationTest extends BaseMigrationTest { @SneakyThrows @Test public void multiTypeTransformationTest_union() { try ( - final SearchClusterContainer indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_13); - final SearchClusterContainer upgradedSourceCluster = new SearchClusterContainer(SearchClusterContainer.ES_V6_8_23) - .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, BindMode.READ_WRITE); - final SearchClusterContainer targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)) { - CompletableFuture.allOf( - CompletableFuture.runAsync(indexCreatedCluster::start), - CompletableFuture.runAsync(upgradedSourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ).join(); + final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_13) + .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR); + final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23) + .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR); + final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0) + ) { + indexCreatedCluster.start(); + + this.sourceCluster = upgradedSourceCluster; + this.targetCluster = targetCluster; + + startClusters(); var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl()); var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl()); - var targetOperations = new ClusterOperations(targetCluster.getUrl()); - // Test data var originalIndexName = "test_index"; - // Create index and add a document on the source cluster + // Create index and add documents on the source cluster indexCreatedOperations.createIndex(originalIndexName); indexCreatedOperations.createDocument(originalIndexName, "1", "{\"field1\":\"My Name\"}", null, "type1"); indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"string\", \"field2\":123}", null, "type2"); indexCreatedOperations.createDocument(originalIndexName, "3", "{\"field3\":1.1}", null, "type3"); - var arguments = new MigrateOrEvaluateArgs(); - - // Use SnapshotImage as the transfer medium - var snapshotName = "initial-setup-snapshot"; - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(indexCreatedCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - - // Get snapshot for ES 5 - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - indexCreatedCluster.copySnapshotData(localDirectory.toString()); - - // Snapshot is automatically visible due to container mount - - // Register snapshot repository - upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, snapshotCreator.getRepoName()); + var snapshotName = "es5-created-index"; + var es5Repo = "es5"; + indexCreatedOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); + indexCreatedOperations.takeSnapshot(es5Repo, snapshotName, originalIndexName); - // Restore snapshot - upgradedSourceOperations.restoreSnapshot(snapshotCreator.getRepoName(), snapshotCreator.getSnapshotName()); + // Register snapshot repository and restore snapshot in ES 6 cluster + upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo); + upgradedSourceOperations.restoreSnapshot(es5Repo, snapshotName); - // Verify that the index exists on the upgraded cluster + // Verify index exists on upgraded cluster var checkIndexUpgraded = upgradedSourceOperations.get("/" + originalIndexName); assertThat(checkIndexUpgraded.getKey(), equalTo(200)); assertThat(checkIndexUpgraded.getValue(), containsString(originalIndexName)); + var updatedSnapshotName = createSnapshot("union-snapshot"); + var arguments = prepareSnapshotMigrationArgs(updatedSnapshotName); - upgradedSourceOperations.deleteAllSnapshotsAndRepository(snapshotCreator.getRepoName()); - - // Use SnapshotImage as the transfer medium - var updatedSnapshotName = "union-snapshot"; - var upgradedClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(upgradedSourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var updatedSnapshotCreator = new FileSystemSnapshotCreator( - updatedSnapshotName, - upgradedClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - - // Get snapshot for ES 6 - SnapshotRunner.runAndWaitForCompletion(updatedSnapshotCreator); - - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = updatedSnapshotCreator.getSnapshotName(); - arguments.sourceVersion = ES_V6_8_23.getVersion(); - arguments.targetArgs.host = targetCluster.getUrl(); - // Set up data filters to include only the test index and templates + // Set up data filters var dataFilterArgs = new DataFilterArgs(); dataFilterArgs.indexAllowlist = List.of(originalIndexName); arguments.dataFilterArgs = dataFilterArgs; - // Use split for multi type mappings resolution - arguments.metadataTransformationParams = TestMetadataTransformationParams.builder() - .multiTypeResolutionBehavior(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION) - .build(); - - // Execute the migration with the custom transformation - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); + // Use union method for multi-type mappings + arguments.metadataTransformationParams.multiTypeResolutionBehavior = IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION; - MigrationItemResult result = metadata.migrate(arguments).execute(metadataContext); + // Execute migration + MigrationItemResult result = executeMigration(arguments, MetadataCommands.MIGRATE); // Verify the migration result log.info(result.asCliOutput()); assertThat(result.getExitCode(), equalTo(0)); + assertThat(result.getItems().getIndexes().size(), equalTo(1)); + var actualCreationResult = result.getItems().getIndexes().get(0); + assertThat(actualCreationResult.getException(), equalTo(null)); + assertThat(actualCreationResult.getName(), equalTo(originalIndexName)); + assertThat(actualCreationResult.getFailureType(), equalTo(null)); + // Verify that the transformed index exists on the target cluster var res = targetOperations.get("/" + originalIndexName); assertThat(res.getKey(), equalTo(200)); @@ -176,145 +116,38 @@ public void multiTypeTransformationTest_union() { } } - @SneakyThrows - @Disabled("This test is disabled because ES5 does not allow multiple types with the same field name and different types") @Test - public void multiTypeTransformationTest_union_withConflicts() { + public void es5_doesNotAllow_multiTypeConflicts() { try ( - final SearchClusterContainer indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_13); - final SearchClusterContainer upgradedSourceCluster = new SearchClusterContainer(SearchClusterContainer.ES_V6_8_23) - .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, BindMode.READ_WRITE); - final SearchClusterContainer targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)) { - CompletableFuture.allOf( - CompletableFuture.runAsync(indexCreatedCluster::start), - CompletableFuture.runAsync(upgradedSourceCluster::start), - CompletableFuture.runAsync(targetCluster::start) - ).join(); + final var es5 = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_13) + ) { + es5.start(); - var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl()); - var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl()); - var targetOperations = new ClusterOperations(targetCluster.getUrl()); + var clusterOperations = new ClusterOperations(es5.getUrl()); - // Test data var originalIndexName = "test_index"; - - var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(indexCreatedCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - - var mappings = "{" + - " \"type1\": {" + - " \"properties\": {" + - " \"field1\": { \"type\": \"float\" }" + - " }" + - " }," + - " \"type2\": {" + - " \"properties\": {" + - " \"field1\": { \"type\": \"long\" }" + - " }" + - " }" + - "}"; - - // Create index and add a document on the source cluster - indexCreatedOperations.createIndexWithMappings(originalIndexName, mappings); - sourceClient.refresh(null); - indexCreatedOperations.createDocument(originalIndexName, "1", "{\"field1\":1.1}", null, "type1"); - sourceClient.refresh(null); - indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"My Name\"}", null, "type2"); - - var arguments = new MigrateOrEvaluateArgs(); - - // Use SnapshotImage as the transfer medium - var snapshotName = "initial-setup-snapshot"; - var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); - var snapshotCreator = new FileSystemSnapshotCreator( - snapshotName, - sourceClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - - // Get snapshot for ES 5 - SnapshotRunner.runAndWaitForCompletion(snapshotCreator); - indexCreatedCluster.copySnapshotData(localDirectory.toString()); - - // Snapshot is automatically visible due to container mount - - // Register snapshot repository - upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, snapshotCreator.getRepoName()); - - // Restore snapshot - upgradedSourceOperations.restoreSnapshot(snapshotCreator.getRepoName(), snapshotCreator.getSnapshotName()); - - // Verify that the index exists on the upgraded cluster - var checkIndexUpgraded = upgradedSourceOperations.get("/" + originalIndexName); - assertThat(checkIndexUpgraded.getKey(), equalTo(200)); - assertThat(checkIndexUpgraded.getValue(), containsString(originalIndexName)); - - - upgradedSourceOperations.deleteAllSnapshotsAndRepository(snapshotCreator.getRepoName()); - - // Use SnapshotImage as the transfer medium - var updatedSnapshotName = "union-snapshot"; - var upgradedClient = new OpenSearchClient(ConnectionContextTestParams.builder() - .host(upgradedSourceCluster.getUrl()) - .insecure(true) - .build() - .toConnectionContext()); - var updatedSnapshotCreator = new FileSystemSnapshotCreator( - updatedSnapshotName, - upgradedClient, - SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, - List.of(), - snapshotContext.createSnapshotCreateContext() - ); - - // Get snapshot for ES 6 - SnapshotRunner.runAndWaitForCompletion(updatedSnapshotCreator); - - arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); - arguments.snapshotName = updatedSnapshotCreator.getSnapshotName(); - arguments.sourceVersion = ES_V6_8_23.getVersion(); - arguments.targetArgs.host = targetCluster.getUrl(); - // Set up data filters to include only the test index and templates - var dataFilterArgs = new DataFilterArgs(); - dataFilterArgs.indexAllowlist = List.of(originalIndexName); - arguments.dataFilterArgs = dataFilterArgs; - - // Use split for multi type mappings resolution - arguments.metadataTransformationParams = TestMetadataTransformationParams.builder() - .multiTypeResolutionBehavior(IndexMappingTypeRemoval.MultiTypeResolutionBehavior.UNION) - .build(); - - // Execute the migration with the custom transformation - var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); - var metadata = new MetadataMigration(); - - MigrationItemResult result = metadata.migrate(arguments).execute(metadataContext); - - // Verify the migration result - log.info(result.asCliOutput()); - assertThat(result.getExitCode(), equalTo(0)); - assertThat(result.getItems().getIndexes().size(), equalTo(1)); - var actualCreationResult = result.getItems().getIndexes().get(0); - assertThat(actualCreationResult.getException(), instanceOf(IndexTransformationException.class)); - assertThat(actualCreationResult.getName(), equalTo(originalIndexName)); - assertThat(actualCreationResult.getException().getMessage(), equalTo("test")); - - // Verify that the transformed index exists on the target cluster - var res = targetOperations.get("/" + originalIndexName); - assertThat(res.getKey(), equalTo(404)); - + String body = "{" + + " \"settings\": {" + + " \"index\": {" + + " \"number_of_shards\": 5," + + " \"number_of_replicas\": 0" + + " }" + + " }," + + " \"mappings\": {" + + " \"type1\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"float\" }" + + " }" + + " }," + + " \"type2\": {" + + " \"properties\": {" + + " \"field1\": { \"type\": \"long\" }" + + " }" + + " }" + + " }" + + "}"; + var res = clusterOperations.attemptCreateIndex(originalIndexName, body); + assertThat(res, containsString("mapper [field1] cannot be changed from type [long] to [float]")); } } - - @Data - @Builder - private static class TestMetadataTransformationParams implements MetadataTransformerParams { - private IndexMappingTypeRemoval.MultiTypeResolutionBehavior multiTypeResolutionBehavior; - } - } diff --git a/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java b/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java index a465c3ab0..a6fc88ccf 100644 --- a/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java +++ b/RFS/src/test/java/org/opensearch/migrations/bulkload/integration/SnapshotStateTest.java @@ -72,7 +72,8 @@ public void SingleSnapshot_SingleDocument() throws Exception { operations.createDocument(indexName, document1Id, document1Body); final var snapshotName = "snapshot-1"; - operations.takeSnapshot(snapshotName, indexName); + final var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); @@ -110,7 +111,8 @@ public void SingleSnapshot_SingleDocument_Then_DeletedDocument() throws Exceptio operations.createDocument(indexName, document1Id, document1Body); operations.deleteDocument(indexName, document1Id); final var snapshotName = "snapshot-delete-item"; - operations.takeSnapshot(snapshotName, indexName); + var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); @@ -145,7 +147,8 @@ public void SingleSnapshot_SingleDocument_Then_UpdateDocument() throws Exception operations.createDocument(indexName, document1Id, document1BodyUpdated); final var snapshotName = "snapshot-delete-item"; - operations.takeSnapshot(snapshotName, indexName); + final var repoName = "test-repo"; + operations.takeSnapshot(repoName, snapshotName, indexName); final File snapshotCopy = new File(localDirectory + "/snapshotCopy"); cluster.copySnapshotData(snapshotCopy.getAbsolutePath()); diff --git a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java index 180907dc9..63048f695 100644 --- a/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java +++ b/RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/http/ClusterOperations.java @@ -177,6 +177,17 @@ public void createIndex(final String index, final String body) { } } + @SneakyThrows + public String attemptCreateIndex(final String index, final String body) { + var createIndexRequest = new HttpPut(clusterUrl + "/" + index); + createIndexRequest.setEntity(new StringEntity(body)); + createIndexRequest.setHeader("Content-Type", "application/json"); + + try (var response = httpClient.execute(createIndexRequest)) { + return EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + } + } + @SneakyThrows public Map.Entry get(final String path) { final var getRequest = new HttpGet(clusterUrl + path); @@ -187,17 +198,17 @@ public Map.Entry get(final String path) { } } - public void takeSnapshot(final String snapshotName, final String indexPattern) throws IOException { + public void takeSnapshot(final String repoName, final String snapshotName, final String indexPattern) throws IOException { final var snapshotJson = "{\n" + " \"indices\": \"" + indexPattern + "\",\n" + " \"ignore_unavailable\": true,\n" - + " \"include_global_state\": false\n" + + " \"include_global_state\": true\n" + "}"; final var createSnapshotRequest = new HttpPut( - clusterUrl + "/_snapshot/test-repo/" + snapshotName + "?wait_for_completion=true" + clusterUrl + "/_snapshot/" + repoName + "/" + snapshotName + "?wait_for_completion=true" ); createSnapshotRequest.setEntity(new StringEntity(snapshotJson)); createSnapshotRequest.setHeader("Content-Type", "application/json"); diff --git a/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java b/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java index 503f8edc2..a4366bffb 100644 --- a/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java +++ b/transformation/src/main/java/org/opensearch/migrations/transformation/rules/IndexMappingTypeRemoval.java @@ -43,6 +43,7 @@ @AllArgsConstructor public class IndexMappingTypeRemoval implements TransformationRule { public enum MultiTypeResolutionBehavior { + NONE, UNION, SPLIT }