-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into more-java-sonar
- Loading branch information
Showing
113 changed files
with
3,861 additions
and
674 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -111,6 +111,44 @@ jobs: | |
| with: | ||
| gradle-version: ${{ env.gradle-version }} | ||
| gradle-home-cache-cleanup: true | ||
| - name: Generate Cache Key from Dockerfiles | ||
| id: generate_cache_key | ||
| run: | | ||
| files=$(find . -type f \( -name 'docker-compose.yml' -o -name 'Dockerfile' \)) | ||
| file_contents=$(cat $files) | ||
| key=$(echo "${file_contents}" | sha1sum | awk '{print $1}') | ||
| echo "key=${key}" >> "$GITHUB_OUTPUT" | ||
| - name: Cache Docker Images | ||
| uses: ScribeMD/[email protected] | ||
| with: | ||
| key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} | ||
| - name: Pre pull images | ||
| run: | | ||
| pull_if_not_present() { | ||
| local image="$1" | ||
| if docker image inspect "$image" > /dev/null 2>&1; then | ||
| echo "Image '$image' already exists locally." | ||
| else | ||
| echo "Pulling image '$image'..." | ||
| docker pull "$image" | ||
| fi | ||
| } | ||
| images=( | ||
| "opensearchproject/opensearch:1.3.16" | ||
| "opensearchproject/opensearch:2.14.0" | ||
| "docker.elastic.co/elasticsearch/elasticsearch:7.17.22" | ||
| "docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2" | ||
| "docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23" | ||
| "docker.elastic.co/elasticsearch/elasticsearch:5.6.16" | ||
| "httpd:alpine" | ||
| "confluentinc/cp-kafka:7.5.0" | ||
| "ghcr.io/shopify/toxiproxy:latest" | ||
| "amazonlinux:2023" | ||
| "alpine:3.16" | ||
| ) | ||
| for image in "${images[@]}"; do | ||
| pull_if_not_present "$image" | ||
| done | ||
| - name: Run Gradle Build | ||
| run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace | ||
| env: | ||
|
|
@@ -161,6 +199,7 @@ jobs: | |
| uses: ScribeMD/[email protected] | ||
| with: | ||
| key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }} | ||
| read-only: true | ||
| - name: Start Docker Solution | ||
| run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace | ||
| env: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
144 changes: 144 additions & 0 deletions
144
MetadataMigration/src/test/java/org/opensearch/migrations/BaseMigrationTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package org.opensearch.migrations; | ||
|
|
||
| import java.io.File; | ||
| import java.util.List; | ||
| import java.util.concurrent.CompletableFuture; | ||
|
|
||
| import org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator; | ||
| import org.opensearch.migrations.bulkload.common.OpenSearchClient; | ||
| import org.opensearch.migrations.bulkload.common.http.ConnectionContextTestParams; | ||
| import org.opensearch.migrations.bulkload.framework.SearchClusterContainer; | ||
| import org.opensearch.migrations.bulkload.http.ClusterOperations; | ||
| import org.opensearch.migrations.bulkload.worker.SnapshotRunner; | ||
| import org.opensearch.migrations.commands.MigrationItemResult; | ||
| import org.opensearch.migrations.metadata.tracing.MetadataMigrationTestContext; | ||
| import org.opensearch.migrations.snapshot.creation.tracing.SnapshotTestContext; | ||
|
|
||
| import lombok.Getter; | ||
| import lombok.SneakyThrows; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.junit.jupiter.api.io.TempDir; | ||
|
|
||
| /** | ||
| * Base test class providing shared setup and utility methods for migration tests. | ||
| */ | ||
| @Slf4j | ||
| abstract class BaseMigrationTest { | ||
|
|
||
| @TempDir | ||
| protected File localDirectory; | ||
|
|
||
| @Getter | ||
| protected SearchClusterContainer sourceCluster; | ||
| @Getter | ||
| protected SearchClusterContainer targetCluster; | ||
|
|
||
| protected ClusterOperations sourceOperations; | ||
| protected ClusterOperations targetOperations; | ||
|
|
||
| /** | ||
| * Starts the source and target clusters. | ||
| */ | ||
| protected void startClusters() { | ||
| CompletableFuture.allOf( | ||
| CompletableFuture.runAsync(sourceCluster::start), | ||
| CompletableFuture.runAsync(targetCluster::start) | ||
| ).join(); | ||
|
|
||
| sourceOperations = new ClusterOperations(sourceCluster.getUrl()); | ||
| targetOperations = new ClusterOperations(targetCluster.getUrl()); | ||
| } | ||
|
|
||
| /** | ||
| * Sets up a snapshot repository and takes a snapshot of the source cluster. | ||
| * | ||
| * @param snapshotName Name of the snapshot. | ||
| * @return The name of the created snapshot. | ||
| */ | ||
| @SneakyThrows | ||
| protected String createSnapshot(String snapshotName) { | ||
| var snapshotContext = SnapshotTestContext.factory().noOtelTracking(); | ||
| var sourceClient = new OpenSearchClient(ConnectionContextTestParams.builder() | ||
| .host(sourceCluster.getUrl()) | ||
| .insecure(true) | ||
| .build() | ||
| .toConnectionContext()); | ||
| var snapshotCreator = new org.opensearch.migrations.bulkload.common.FileSystemSnapshotCreator( | ||
| snapshotName, | ||
| sourceClient, | ||
| SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, | ||
| List.of(), | ||
| snapshotContext.createSnapshotCreateContext() | ||
| ); | ||
| org.opensearch.migrations.bulkload.worker.SnapshotRunner.runAndWaitForCompletion(snapshotCreator); | ||
| sourceCluster.copySnapshotData(localDirectory.toString()); | ||
| return snapshotName; | ||
| } | ||
|
|
||
| /** | ||
| * Prepares migration arguments for snapshot-based migrations. | ||
| * | ||
| * @param snapshotName Name of the snapshot. | ||
| * @return Prepared migration arguments. | ||
| */ | ||
| protected MigrateOrEvaluateArgs prepareSnapshotMigrationArgs(String snapshotName) { | ||
| var arguments = new MigrateOrEvaluateArgs(); | ||
| arguments.fileSystemRepoPath = localDirectory.getAbsolutePath(); | ||
| arguments.snapshotName = snapshotName; | ||
| arguments.sourceVersion = sourceCluster.getContainerVersion().getVersion(); | ||
| arguments.targetArgs.host = targetCluster.getUrl(); | ||
| return arguments; | ||
| } | ||
|
|
||
| /** | ||
| * Executes the migration command (either migrate or evaluate). | ||
| * | ||
| * @param arguments Migration arguments. | ||
| * @param command The migration command to execute. | ||
| * @return The result of the migration. | ||
| */ | ||
| protected MigrationItemResult executeMigration(MigrateOrEvaluateArgs arguments, MetadataCommands command) { | ||
| var metadataContext = MetadataMigrationTestContext.factory().noOtelTracking(); | ||
| var metadata = new MetadataMigration(); | ||
|
|
||
| if (MetadataCommands.MIGRATE.equals(command)) { | ||
| return metadata.migrate(arguments).execute(metadataContext); | ||
| } else { | ||
| return metadata.evaluate(arguments).execute(metadataContext); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Creates an OpenSearch client for the given cluster. | ||
| * | ||
| * @param cluster The cluster container. | ||
| * @return An OpenSearch client. | ||
| */ | ||
| protected OpenSearchClient createClient(SearchClusterContainer cluster) { | ||
| return new OpenSearchClient(ConnectionContextTestParams.builder() | ||
| .host(cluster.getUrl()) | ||
| .insecure(true) | ||
| .build() | ||
| .toConnectionContext()); | ||
| } | ||
|
|
||
| protected SnapshotTestContext createSnapshotContext() { | ||
| return SnapshotTestContext.factory().noOtelTracking(); | ||
| } | ||
|
|
||
| protected FileSystemSnapshotCreator createSnapshotCreator(String snapshotName, OpenSearchClient client, SnapshotTestContext context) { | ||
| return new FileSystemSnapshotCreator( | ||
| snapshotName, | ||
| client, | ||
| SearchClusterContainer.CLUSTER_SNAPSHOT_DIR, | ||
| List.of(), | ||
| context.createSnapshotCreateContext() | ||
| ); | ||
| } | ||
|
|
||
| @SneakyThrows | ||
| protected void runSnapshotAndCopyData(FileSystemSnapshotCreator snapshotCreator, SearchClusterContainer cluster) { | ||
| SnapshotRunner.runAndWaitForCompletion(snapshotCreator); | ||
| cluster.copySnapshotData(localDirectory.toString()); | ||
| } | ||
| } |
Oops, something went wrong.