Merge pull request opensearch-project#1195 from AndreKurait/EnableMul…

…tiTypeGHA

Enable MultiTypeMappingTransformationTest

.github/workflows/CI.yml

@@ -111,44 +111,6 @@ jobs:
         with:
           gradle-version: ${{ env.gradle-version }}
           gradle-home-cache-cleanup: true
-      - name: Generate Cache Key from Dockerfiles
-        id: generate_cache_key
-        run: |
-          files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \))
-          file_contents=$(cat $files)
-          key=$(echo "${file_contents}" | sha1sum | awk '{print $1}')
-          echo "key=${key}" >> "$GITHUB_OUTPUT"
-      - name: Cache Docker Images
-        uses: ScribeMD/[email protected]
-        with:
-          key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
-      - name: Pre pull images
-        run: |
-          pull_if_not_present() {
-            local image="$1"
-            if docker image inspect "$image" > /dev/null 2>&1; then
-              echo "Image '$image' already exists locally."
-            else
-              echo "Pulling image '$image'..."
-              docker pull "$image"
-            fi
-          }
-          images=(
-            "opensearchproject/opensearch:1.3.16"
-            "opensearchproject/opensearch:2.14.0"
-            "docker.elastic.co/elasticsearch/elasticsearch:7.17.22"
-            "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2"
-            "docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23"
-            "docker.elastic.co/elasticsearch/elasticsearch:5.6.16"
-            "httpd:alpine"
-            "confluentinc/cp-kafka:7.5.0"
-            "ghcr.io/shopify/toxiproxy:latest"
-            "amazonlinux:2023"
-            "alpine:3.16"
-          )
-          for image in "${images[@]}"; do
-            pull_if_not_present "$image"
-          done
       - name: Run Gradle Build
         run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
         env:

MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java

@@ -1,6 +1,6 @@
 package org.opensearch.migrations;
 
-import java.io.File;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
@@ -26,7 +26,7 @@
 abstract class BaseMigrationTest {
 
     @TempDir
-    protected File localDirectory;
+    protected Path localDirectory;
 
     @Getter
     protected SearchClusterContainer sourceCluster;
@@ -83,7 +83,7 @@ protected String createSnapshot(String snapshotName) {
      */
     protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) {
         var arguments = new MigrateOrEvaluateArgs();
-        arguments.fileSystemRepoPath = localDirectory.getAbsolutePath();
+        arguments.fileSystemRepoPath = localDirectory.toString();
         arguments.snapshotName = snapshotName;
         arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion();
         arguments.targetArgs.host = targetCluster.getUrl();

MetadataMigration/src/test/java/org/opensearch/migrations/MultiTypeMappingTransformationTest.java

@@ -12,10 +12,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.testcontainers.containers.BindMode;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -31,38 +29,41 @@ class MultiTypeMappingTransformationTest extends BaseMigrationTest {
 
     @SneakyThrows
     @Test
-    @Disabled("TODO, fix in GHA")
     public void multiTypeTransformationTest_union() {
+        var es5Repo = "es5";
+        var snapshotName = "es5-created-index";
+        var originalIndexName = "test_index";
+
         try (
-                final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16);
-                final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23)
-                    .withFileSystemBind(localDirectory.getAbsolutePath(), SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, BindMode.READ_WRITE);
-                final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)
+            final var indexCreatedCluster = new SearchClusterContainer(SearchClusterContainer.ES_V5_6_16)
         ) {
             indexCreatedCluster.start();
 
-            this.sourceCluster = upgradedSourceCluster;
-            this.targetCluster = targetCluster;
-
-            startClusters();
-
             var indexCreatedOperations = new ClusterOperations(indexCreatedCluster.getUrl());
-            var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl());
-
-            var originalIndexName = "test_index";
 
             // Create index and add documents on the source cluster
             indexCreatedOperations.createIndex(originalIndexName);
             indexCreatedOperations.createDocument(originalIndexName, "1", "{\"field1\":\"My Name\"}", null, "type1");
             indexCreatedOperations.createDocument(originalIndexName, "2", "{\"field1\":\"string\", \"field2\":123}", null, "type2");
             indexCreatedOperations.createDocument(originalIndexName, "3", "{\"field3\":1.1}", null, "type3");
 
-            var snapshotName = "es5-created-index";
-            var es5Repo = "es5";
             indexCreatedOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo);
             indexCreatedOperations.takeSnapshot(es5Repo, snapshotName, originalIndexName);
+            indexCreatedCluster.copySnapshotData(localDirectory.toString());
+        }
 
-            indexCreatedCluster.copySnapshotData(localDirectory.getAbsolutePath());
+        try (
+            final var upgradedSourceCluster = new SearchClusterContainer(ES_V6_8_23);
+            final var targetCluster = new SearchClusterContainer(SearchClusterContainer.OS_V2_14_0)
+        ) {
+            this.sourceCluster = upgradedSourceCluster;
+            this.targetCluster = targetCluster;
+
+            startClusters();
+
+            upgradedSourceCluster.putSnapshotData(localDirectory.toString());
+
+            var upgradedSourceOperations = new ClusterOperations(upgradedSourceCluster.getUrl());
 
             // Register snapshot repository and restore snapshot in ES 6 cluster
             upgradedSourceOperations.createSnapshotRepository(SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, es5Repo);
@@ -131,25 +132,25 @@ public void es5_doesNotAllow_multiTypeConflicts() {
 
             var originalIndexName = "test_index";
             String body = "{" +
-                    "  \"settings\": {" +
-                    "    \"index\": {" +
-                    "      \"number_of_shards\": 5," +
-                    "      \"number_of_replicas\": 0" +
-                    "    }" +
-                    "  }," +
-                    "  \"mappings\": {" +
-                    "    \"type1\": {" +
-                    "      \"properties\": {" +
-                    "        \"field1\": { \"type\": \"float\" }" +
-                    "      }" +
-                    "    }," +
-                    "    \"type2\": {" +
-                    "      \"properties\": {" +
-                    "        \"field1\": { \"type\": \"long\" }" +
-                    "      }" +
-                    "    }" +
-                    "  }" +
-                    "}";
+                "  \"settings\": {" +
+                "    \"index\": {" +
+                "      \"number_of_shards\": 5," +
+                "      \"number_of_replicas\": 0" +
+                "    }" +
+                "  }," +
+                "  \"mappings\": {" +
+                "    \"type1\": {" +
+                "      \"properties\": {" +
+                "        \"field1\": { \"type\": \"float\" }" +
+                "      }" +
+                "    }," +
+                "    \"type2\": {" +
+                "      \"properties\": {" +
+                "        \"field1\": { \"type\": \"long\" }" +
+                "      }" +
+                "    }" +
+                "  }" +
+                "}";
             var res = clusterOperations.put("/" + originalIndexName, body);
             assertThat(res.getKey(), equalTo(400));
             assertThat(res.getValue(), containsString("mapper [field1] cannot be changed from type [long] to [float]"));

RFS/src/testFixtures/java/org/opensearch/migrations/bulkload/framework/SearchClusterContainer.java

@@ -14,6 +14,7 @@
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
 
 /**
  * Containerized version of Elasticsearch cluster
@@ -112,6 +113,16 @@ public void copySnapshotData(final String directory) {
         }
     }
 
+    public void putSnapshotData(final String directory) {
+        try {
+            this.copyFileToContainer(MountableFile.forHostPath(directory), CLUSTER_SNAPSHOT_DIR);
+            this.execInContainer("chown", "-R", "elasticsearch:elasticsearch", CLUSTER_SNAPSHOT_DIR);
+        } catch (final Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
     public void start() {
         log.info("Starting container version:" + containerVersion.version);
         super.start();