Make RFS runnable as Container (#550)

This changes introduces a Dockerfile for RFS that is linked to the gradle build process such that a user can make changes to RFS code or dependencies and simply execute ./gradlew buildDockerImages from the RFS directory and create a Docker image with the latest changes. A user could alternatively execute ./gradlew composeUp and create a docker environment with source/target/RFS containers using latest changes. Note: The gradle link is in place for docker compose but more work is needed here to limit manual steps a user needs to take to test RFS in this environment. Along with this a few command line arguments were incorporated into RFS Signed-off-by: Tanner Lewis <[email protected]>
opensearch-project · Apr 5, 2024 · 48c43ea · 48c43ea
1 parent 605057c
commit 48c43ea
Show file tree

Hide file tree

Showing 9 changed files with 207 additions and 15 deletions.
diff --git a/RFS/.gitignore b/RFS/.gitignore
@@ -0,0 +1,5 @@
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# Ignore Gradle build output directory
+build
diff --git a/RFS/README.md b/RFS/README.md
@@ -1,5 +1,20 @@
 # reindex-from-snapshot
 
+### Table of Contents (Generated)
+- [reindex-from-snapshot](#reindex-from-snapshot)
+    - [How to run](#how-to-run)
+        - [Using a local snapshot directory](#using-a-local-snapshot-directory)
+        - [Using an existing S3 snapshot](#using-an-existing-s3-snapshot)
+        - [Using a source cluster](#using-a-source-cluster)
+        - [Using Docker](#using-docker)
+        - [Handling auth](#handling-auth)
+    - [How to set up an ES 6.8 Source Cluster w/ an attached debugger](#how-to-set-up-an-es-68-source-cluster-w-an-attached-debugger)
+    - [How to set up an ES 7.10 Source Cluster running in Docker](#how-to-set-up-an-es-710-source-cluster-running-in-docker)
+        - [Providing AWS permissions for S3 snapshot creation](#providing-aws-permissions-for-s3-snapshot-creation)
+        - [Setting up the Cluster w/ some sample docs](#setting-up-the-cluster-w-some-sample-docs)
+    - [How to set up an OS 2.11 Target Cluster](#how-to-set-up-an-os-211-target-cluster)
+
+
 ## How to run
 
 RFS provides a number of different options for running it.  We'll look at some of them below.  
@@ -35,7 +50,7 @@ gradle build
 gradle run --args='-n global_state_snapshot --s3-local-dir /tmp/s3_files --s3-repo-uri $S3_REPO_URI --s3-region $S3_REGION -l /tmp/lucene_files --target-host $TARGET_HOST --target-username $TARGET_USERNAME --target-password $TARGET_PASSWORD -s es_6_8 -t os_2_11 --movement-type everything'
 ```
 
-### Using an source cluster
+### Using a source cluster
 
 In this scenario, you have a source cluster, and don't yet have a snapshot.  RFS will need to first make a snapshot of your source cluster, send it to S3, and then begin reindexing.  In this scenario, you'll supply the source cluster-related args (`--source-host`, `--source-username`, `--source-password`) and the S3-related args (`--s3-local-dir`, `--s3-repo-uri`, `--s3-region`), but not the `--snapshot-dir` one.
 
@@ -56,6 +71,37 @@ gradle build
 gradle run --args='-n global_state_snapshot --source-host $SOURCE_HOST --source-username $SOURCE_USERNAME --source-password $SOURCE_PASSWORD --s3-local-dir /tmp/s3_files --s3-repo-uri $S3_REPO_URI --s3-region $S3_REGION -l /tmp/lucene_files --target-host $TARGET_HOST --target-username $TARGET_USERNAME --target-password $TARGET_PASSWORD -s es_6_8 -t os_2_11 --movement-type everything'
 ```
 
+### Using Docker
+RFS has support for packaging its java application as a Docker image by using the Dockerfile located in the `RFS/docker` directory. This support is directly built into Gradle as well so that a user can perform the below action, and generate a fresh Docker image (`migrations/reindex_from_snapshot:latest`) with the latest local code changes available.
+```shell
+./gradlew buildDockerImages
+```
+Also built into this Docker/Gradle support is the ability to spin up a testing RFS environment using Docker compose. This compose file can be seen [here](./docker/docker-compose.yml) and includes the RFS container, a source cluster container, and a target cluster container.
+
+This environment can be spun up with the Gradle command
+```shell
+./gradlew composeUp
+```
+And deleted with the Gradle command
+```shell
+./gradlew composeDown
+```
+
+After the Docker compose containers are created the elasticsearch/opensearch source and target clusters can be interacted with like normal. For RFS testing, a user should load templates/indices/documents into the source cluster and take a snapshot before kicking off RFS.
+```shell
+# To check indices on the source cluster
+curl https://localhost:19200/_cat/indices?v --insecure -u admin:admin
+
+# To check indices on the target cluster
+curl https://localhost:29200/_cat/indices?v --insecure -u admin:admin
+```
+
+Once the user is ready, they can kick off the RFS migration by running the RFS java command with the proper arguments on the RFS container like below:
+```shell
+docker exec -it rfs-compose-reindex-from-snapshot-1 sh -c "/rfs-app/runJavaWithClasspath.sh com.rfs.ReindexFromSnapshot --snapshot-name <snapshot-name> --snapshot-dir <snapshot-dir> --lucene-dir '/lucene' --target-host https://opensearchtarget:9200 --target-username admin --target-password admin --source-version es_7_10 --target-version os_2_11"
+```
+
+
 ### Handling auth
 
 RFS currently supports both basic auth (username/password) and no auth for both the source and target clusters.  To use the no-auth approach, just neglect the username/password arguments.

diff --git a/RFS/build.gradle b/RFS/build.gradle
@@ -1,10 +1,14 @@
 plugins {
     id 'application'
     id 'java'
+    id "com.avast.gradle.docker-compose" version "0.17.4"
+    id 'com.bmuschko.docker-remote-api'
 }
 
-sourceCompatibility = '11'
-targetCompatibility = '11'
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+java.targetCompatibility = JavaVersion.VERSION_11
 
 repositories {
     mavenCentral()
@@ -29,7 +33,44 @@ application {
     mainClassName = 'com.rfs.ReindexFromSnapshot'
 }
 
-task demoPrintOutSnapshot(type: JavaExec) {
+// Cleanup additional docker build directory
+clean.doFirst {
+    delete project.file("./docker/build")
+}
+
+task demoPrintOutSnapshot (type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
-    main = 'com.rfs.DemoPrintOutSnapshot'
-}
+    mainClass = 'com.rfs.DemoPrintOutSnapshot'
+}
+
+task copyDockerRuntimeJars (type: Copy) {
+    description = 'Copy runtime JARs and app jar to docker build directory'
+
+    // Define the destination directory
+    def buildDir = project.file("./docker/build/runtimeJars")
+    into buildDir
+
+    // Add all the required runtime JARs to be copied
+    from configurations.runtimeClasspath
+    from tasks.named('jar')
+    include '*.jar'
+}
+
+// ./gradlew composeUp
+// ./gradlew composeDown
+dockerCompose {
+    useComposeFiles = ['docker/docker-compose.yml']
+    projectName = 'rfs-compose'
+}
+
+// ./gradlew buildDockerImages
+task buildDockerImages (type: DockerBuildImage) {
+    dependsOn copyDockerRuntimeJars
+    def dockerImageName = "reindex_from_snapshot"
+    inputDir = project.file("./docker")
+    images.add("migrations/${dockerImageName}:${version}")
+    images.add("migrations/${dockerImageName}:latest")
+}
+
+tasks.getByName('composeUp')
+        .dependsOn(tasks.getByName('buildDockerImages'))
diff --git a/RFS/buildSrc/build.gradle b/RFS/buildSrc/build.gradle
@@ -0,0 +1,27 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ */
+
+plugins {
+    // Support convention plugins written in Groovy. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build.
+    id 'groovy-gradle-plugin'
+    id 'org.owasp.dependencycheck' version '8.2.1'
+}
+
+repositories {
+    // Use the plugin portal to apply community plugins in convention plugins.
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'com.bmuschko:gradle-docker-plugin:9.3.1'
+}
+
+tasks.withType(Tar){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
+
+tasks.withType(Zip){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
diff --git a/RFS/buildSrc/settings.gradle b/RFS/buildSrc/settings.gradle
@@ -0,0 +1,7 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This settings file is used to specify which projects to include in your build-logic build.
+ */
+
+rootProject.name = 'buildSrc'
diff --git a/RFS/docker/Dockerfile b/RFS/docker/Dockerfile
@@ -0,0 +1,8 @@
+# Using same base image as other Java containers in this repo
+FROM openjdk:11-jre
+# Requires Gradle to genearte runtime jars initially
+COPY ./build/runtimeJars /rfs-app/jars
+WORKDIR /rfs-app
+RUN printf "#!/bin/sh\njava -XX:MaxRAMPercentage=80.0 -cp /rfs-app/jars/*:. \"\$@\" " > /rfs-app/runJavaWithClasspath.sh
+RUN chmod +x /rfs-app/runJavaWithClasspath.sh
+CMD ["tail", "-f", "/dev/null"]
diff --git a/RFS/docker/docker-compose.yml b/RFS/docker/docker-compose.yml
@@ -0,0 +1,38 @@
+version: '3.7'
+services:
+
+  # TODO: Sync gradle for RFS and C&R
+  # Temporarily this requires the user to first build the elasticsearch image from the TrafficCapture gradle project
+  # directory using a command like ./gradlew buildDockerImages
+  elasticsearchsource:
+    image: 'migrations/elasticsearch_searchguard:latest'
+    networks:
+      - migrations
+    environment:
+      - discovery.type=single-node
+    ports:
+      - '19200:9200'
+
+  # TODO: Add example command users can run to kickoff RFS after data is loaded on source
+  reindex-from-snapshot:
+    image: 'migrations/reindex_from_snapshot:latest'
+    depends_on:
+      elasticsearchsource:
+        condition: service_started
+      opensearchtarget:
+        condition: service_started
+    networks:
+      - migrations
+
+  opensearchtarget:
+    image: 'opensearchproject/opensearch:2.11.1'
+    environment:
+      - discovery.type=single-node
+    networks:
+      - migrations
+    ports:
+      - "29200:9200"
+
+networks:
+  migrations:
+    driver: bridge
diff --git a/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java b/RFS/src/main/java/com/rfs/ReindexFromSnapshot.java
@@ -3,7 +3,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
@@ -68,18 +70,27 @@ public static class Args {
         @Parameter(names = {"--movement-type"}, description = "What you want to move - everything, metadata, or data.  Default: 'everything'", required = false, converter = MovementType.ArgsConverter.class)
         public MovementType movementType = MovementType.EVERYTHING;
 
+        @Parameter(names = {"--template-whitelist"}, description = "List of template names to migrate. Note: For ES 6.8 this refers to legacy templates and for ES 7.10 this is index templates (e.g. 'posts_index_template1, posts_index_template2')", required = false)
+        public List<String> templateWhitelist;
+
+        @Parameter(names = {"--component-template-whitelist"}, description = "List of component template names to migrate (e.g. 'posts_template1, posts_template2')", required = false)
+        public List<String> componentTemplateWhitelist;
+
+        @Parameter(names = {"--enable-persistent-run"}, description = "If enabled, the java process will continue in an idle mode after the migration is completed.  Default: false", arity=0, required = false)
+        public boolean enablePersistentRun;
+
         @Parameter(names = {"--log-level"}, description = "What log level you want.  Default: 'info'", required = false, converter = Logging.ArgsConverter.class)
         public Level logLevel = Level.INFO;
     }
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws InterruptedException {
         // Grab out args
         Args arguments = new Args();
         JCommander.newBuilder()
             .addObject(arguments)
             .build()
             .parse(args);
-        
+
         String snapshotName = arguments.snapshotName;
         Path snapshotDirPath = (arguments.snapshotDirPath != null) ? Paths.get(arguments.snapshotDirPath) : null;
         Path s3LocalDirPath = (arguments.s3LocalDirPath != null) ? Paths.get(arguments.s3LocalDirPath) : null;
@@ -94,6 +105,8 @@ public static void main(String[] args) {
         String targetPass = arguments.targetPass;
         ClusterVersion sourceVersion = arguments.sourceVersion;
         ClusterVersion targetVersion = arguments.targetVersion;
+        List<String> templateWhitelist = arguments.templateWhitelist;
+        List<String> componentTemplateWhitelist = arguments.componentTemplateWhitelist;
         MovementType movementType = arguments.movementType;
         Level logLevel = arguments.logLevel;
 
@@ -103,8 +116,6 @@ public static void main(String[] args) {
         ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
         // Should probably be passed in as an arguments
-        String[] templateWhitelist = {"posts_index_template"};
-        String[] componentTemplateWhitelist = {"posts_template"};
         int awarenessAttributeDimensionality = 3; // https://opensearch.org/docs/2.11/api-reference/cluster-api/cluster-awareness/
 
         // Sanity checks
@@ -235,7 +246,7 @@ public static void main(String[] args) {
                 if (sourceVersion == ClusterVersion.ES_6_8) {
                     ObjectNode root = globalMetadata.toObjectNode();
                     ObjectNode transformedRoot = transformer.transformGlobalMetadata(root);                    
-                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, new String[0], templateWhitelist);              
+                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, Collections.emptyList(), templateWhitelist);
                 } else if (sourceVersion == ClusterVersion.ES_7_10) {
                     ObjectNode root = globalMetadata.toObjectNode();
                     ObjectNode transformedRoot = transformer.transformGlobalMetadata(root);                    
@@ -338,5 +349,14 @@ public static void main(String[] args) {
         } catch (Exception e) {
             e.printStackTrace();
         }
+
+        // Optional temporary persistent runtime flag to continue Java process after steps have completed. This should get
+        // replaced as this app develops and becomes aware of determining work to be completed
+        if (arguments.enablePersistentRun) {
+            while (true) {
+                logger.info("Process is in idle mode, to retry migration please restart this app.");
+                Thread.sleep(TimeUnit.MINUTES.toMillis(5));
+            }
+        }
     }
 }
diff --git a/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java b/RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java
@@ -14,7 +14,7 @@
 public class GlobalMetadataCreator_OS_2_11 {
     private static final Logger logger = LogManager.getLogger(GlobalMetadataCreator_OS_2_11.class);
 
-    public static void create(ObjectNode root, ConnectionDetails connectionDetails, String[] componentTemplateWhitelist, String[] indexTemplateWhitelist) throws Exception {
+    public static void create(ObjectNode root, ConnectionDetails connectionDetails, List<String> componentTemplateWhitelist, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Global Metadata");
 
         GlobalMetadataData_OS_2_11 globalMetadata = new GlobalMetadataData_OS_2_11(root);
@@ -23,7 +23,7 @@ public static void create(ObjectNode root, ConnectionDetails connectionDetails,
         createIndexTemplates(globalMetadata, connectionDetails, indexTemplateWhitelist);
     }
 
-    public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Legacy Templates");
         ObjectNode templates = globalMetadata.getTemplates();
 
@@ -59,7 +59,7 @@ public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, Co
         }
     }
 
-    public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Component Templates");
         ObjectNode templates = globalMetadata.getComponentTemplates();
 
@@ -95,7 +95,7 @@ public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMet
         }
     }
 
-    public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Index Templates");
         ObjectNode templates = globalMetadata.getIndexTemplates();