Make RFS runnable as Container

.gitignore

@@ -8,6 +8,4 @@ __pycache__
 .python-version
 logs
 test/opensearch-cluster-cdk/
-RFS/.gradle/
-RFS/build/
 opensearch-cluster-cdk/

RFS/.gitignore

@@ -0,0 +1,5 @@
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# Ignore Gradle build output directory
+build

RFS/build.gradle

@@ -1,10 +1,14 @@
 plugins {
     id 'application'
     id 'java'
+    id "com.avast.gradle.docker-compose" version "0.17.4"
+    id 'com.bmuschko.docker-remote-api'
 }
 
-sourceCompatibility = '11'
-targetCompatibility = '11'
+import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+
+java.sourceCompatibility = JavaVersion.VERSION_11
+java.targetCompatibility = JavaVersion.VERSION_11
 
 repositories {
     mavenCentral()
@@ -29,7 +33,44 @@ application {
     mainClassName = 'com.rfs.ReindexFromSnapshot'
 }
 
-task demoPrintOutSnapshot(type: JavaExec) {
+// Cleanup additional docker build directory
+clean.doFirst {
+    delete project.file("./docker/build")
+}
+
+task demoPrintOutSnapshot (type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
-    main = 'com.rfs.DemoPrintOutSnapshot'
-}
+    mainClass = 'com.rfs.DemoPrintOutSnapshot'
+}
+
+task copyDockerRuntimeJars (type: Copy) {
+    description = 'Copy runtime JARs and app jar to docker build directory'
+
+    // Define the destination directory
+    def buildDir = project.file("./docker/build/runtimeJars")
+    into buildDir
+
+    // Add all the required runtime JARs to be copied
+    from configurations.runtimeClasspath
+    from tasks.named('jar')
+    include '*.jar'
+}
+
+// ./gradlew composeUp
+// ./gradlew composeDown
+dockerCompose {
+    useComposeFiles = ['docker/docker-compose.yml']
+    projectName = 'rfs-compose'
+}
+
+// ./gradlew buildDockerImages
+task buildDockerImages (type: DockerBuildImage) {
+    dependsOn copyDockerRuntimeJars
+    def dockerImageName = "reindex_from_snapshot"
+    inputDir = project.file("./docker")
+    images.add("migrations/${dockerImageName}:${version}")
+    images.add("migrations/${dockerImageName}:latest")
+}
+
+tasks.getByName('composeUp')
+        .dependsOn(tasks.getByName('buildDockerImages'))

RFS/buildSrc/build.gradle

@@ -0,0 +1,27 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ */
+
+plugins {
+    // Support convention plugins written in Groovy. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build.
+    id 'groovy-gradle-plugin'
+    id 'org.owasp.dependencycheck' version '8.2.1'
+}
+
+repositories {
+    // Use the plugin portal to apply community plugins in convention plugins.
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'com.bmuschko:gradle-docker-plugin:9.3.1'
+}
+
+tasks.withType(Tar){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
+
+tasks.withType(Zip){
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}

RFS/buildSrc/settings.gradle

@@ -0,0 +1,7 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This settings file is used to specify which projects to include in your build-logic build.
+ */
+
+rootProject.name = 'buildSrc'

RFS/docker/Dockerfile

@@ -0,0 +1,21 @@
+# Build jars on container
+#FROM amazoncorretto:11-alpine-jdk AS builder
+#COPY /gradle /rfs-build/gradle
+#COPY build.gradle /rfs-build
+#COPY gradlew /rfs-build
+#COPY settings.gradle /rfs-build
+#COPY gradle.properties /rfs-build
+#COPY /src /rfs-build/src
+#COPY /buildSrc /rfs-build/buildSrc
+#WORKDIR /rfs-build
+#RUN ./gradlew copyRuntimeJars
+
+# Using same base image as other Java containers in this repo
+FROM openjdk:11-jre
+#COPY --from=builder /rfs-build/build/runtimeJars /rfs-app/jars
+# Requires Gradle to genearte runtime jars initially
+COPY ./build/runtimeJars /rfs-app/jars
+WORKDIR /rfs-app
+RUN printf "#!/bin/sh\njava -XX:MaxRAMPercentage=80.0 -cp /rfs-app/jars/*:. \"\$@\" " > /rfs-app/runJavaWithClasspath.sh
+RUN chmod +x /rfs-app/runJavaWithClasspath.sh
+CMD ["tail", "-f", "/dev/null"]

RFS/docker/docker-compose.yml

@@ -0,0 +1,39 @@
+version: '3.7'
+services:
+
+  # TODO: Sync gradle for RFS and C&R
+  # Temporarily this requires the user to first build the elasticsearch image from the TrafficCapture gradle project
+  # directory using a command like ./gradlew buildDockerImages
+  elasticsearchsource:
+    image: 'migrations/elasticsearch_searchguard:latest'
+    networks:
+      - migrations
+    environment:
+      - discovery.type=single-node
+    ports:
+      - '19200:9200'
+
+  # TODO: Add example command users can run to kickoff RFS after data is loaded on source
+  reindex-from-snapshot:
+    image: 'migrations/reindex_from_snapshot:latest'
+    depends_on:
+      elasticsearchsource:
+        condition: service_started
+      opensearchtarget:
+        condition: service_started
+    networks:
+      - migrations
+
+  opensearchtarget:
+    image: 'opensearchproject/opensearch:2.11.1'
+    environment:
+      - discovery.type=single-node
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!
+    networks:
+      - migrations
+    ports:
+      - "29200:9200"
+
+networks:
+  migrations:
+    driver: bridge

RFS/src/main/java/com/rfs/ReindexFromSnapshot.java

@@ -3,7 +3,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
@@ -59,18 +61,27 @@ public static class Args {
         @Parameter(names = {"--movement-type"}, description = "What you want to move - everything, metadata, or data.  Default: 'everything'", required = false, converter = MovementType.ArgsConverter.class)
         public MovementType movementType = MovementType.EVERYTHING;
 
+        @Parameter(names = {"--index-template-whitelist"}, description = "List of index template names to migrate (e.g. 'posts_index_template1, posts_index_template2')", required = false)
+        public List<String> indexTemplateWhitelist;
+
+        @Parameter(names = {"--component-template-whitelist"}, description = "List of component template names to migrate (e.g. 'posts_template1, posts_template2')", required = false)
+        public List<String> componentTemplateWhitelist;
+
+        @Parameter(names = {"--enable-persistent-run"}, description = "If enabled, the java process will continue in an idle mode after the migration is completed.  Default: false", arity=0, required = false)
+        public boolean enablePersistentRun;
+
         @Parameter(names = {"--log-level"}, description = "What log level you want.  Default: 'info'", required = false, converter = Logging.ArgsConverter.class)
         public Level logLevel = Level.INFO;
     }
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws InterruptedException {
         // Grab out args
         Args arguments = new Args();
         JCommander.newBuilder()
             .addObject(arguments)
             .build()
             .parse(args);
-        
+
         String snapshotName = arguments.snapshotName;
         Path snapshotDirPath = (arguments.snapshotDirPath != null) ? Paths.get(arguments.snapshotDirPath) : null;
         Path s3LocalDirPath = (arguments.s3LocalDirPath != null) ? Paths.get(arguments.s3LocalDirPath) : null;
@@ -82,6 +93,8 @@ public static void main(String[] args) {
         String targetPass = arguments.targetPass;
         ClusterVersion sourceVersion = arguments.sourceVersion;
         ClusterVersion targetVersion = arguments.targetVersion;
+        List<String> indexTemplateWhitelist = arguments.indexTemplateWhitelist;
+        List<String> componentTemplateWhitelist = arguments.componentTemplateWhitelist;
         MovementType movementType = arguments.movementType;
         Level logLevel = arguments.logLevel;
 
@@ -90,8 +103,6 @@ public static void main(String[] args) {
         ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
         // Should probably be passed in as an arguments
-        String[] templateWhitelist = {"posts_index_template"};
-        String[] componentTemplateWhitelist = {"posts_template"};
         int awarenessAttributeDimensionality = 3; // https://opensearch.org/docs/2.11/api-reference/cluster-api/cluster-awareness/
 
         // Sanity checks
@@ -191,11 +202,11 @@ public static void main(String[] args) {
                 if (sourceVersion == ClusterVersion.ES_6_8) {
                     ObjectNode root = globalMetadata.toObjectNode();
                     ObjectNode transformedRoot = transformer.transformGlobalMetadata(root);                    
-                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, new String[0], templateWhitelist);              
+                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, Collections.emptyList(), indexTemplateWhitelist);
                 } else if (sourceVersion == ClusterVersion.ES_7_10) {
                     ObjectNode root = globalMetadata.toObjectNode();
                     ObjectNode transformedRoot = transformer.transformGlobalMetadata(root);                    
-                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, componentTemplateWhitelist, templateWhitelist);
+                    GlobalMetadataCreator_OS_2_11.create(transformedRoot, targetConnection, componentTemplateWhitelist, indexTemplateWhitelist);
                 }
             }
 
@@ -294,5 +305,14 @@ public static void main(String[] args) {
         } catch (Exception e) {
             e.printStackTrace();
         }
+
+        // Optional temporary persistent runtime flag to continue Java process after steps have completed. This should get
+        // replaced as this app develops and becomes aware of determining work to be completed
+        if (arguments.enablePersistentRun) {
+            while (true) {
+                logger.info("Process is in idle mode, to retry migration please restart this app.");
+                Thread.sleep(TimeUnit.MINUTES.toMillis(5));
+            }
+        }
     }
 }

RFS/src/main/java/com/rfs/version_os_2_11/GlobalMetadataCreator_OS_2_11.java

@@ -14,7 +14,7 @@
 public class GlobalMetadataCreator_OS_2_11 {
     private static final Logger logger = LogManager.getLogger(GlobalMetadataCreator_OS_2_11.class);
 
-    public static void create(ObjectNode root, ConnectionDetails connectionDetails, String[] componentTemplateWhitelist, String[] indexTemplateWhitelist) throws Exception {
+    public static void create(ObjectNode root, ConnectionDetails connectionDetails, List<String> componentTemplateWhitelist, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Global Metadata");
 
         GlobalMetadataData_OS_2_11 globalMetadata = new GlobalMetadataData_OS_2_11(root);
@@ -23,7 +23,7 @@ public static void create(ObjectNode root, ConnectionDetails connectionDetails,
         createIndexTemplates(globalMetadata, connectionDetails, indexTemplateWhitelist);
     }
 
-    public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Legacy Templates");
         ObjectNode templates = globalMetadata.getTemplates();
 
@@ -59,7 +59,7 @@ public static void createTemplates(GlobalMetadataData_OS_2_11 globalMetadata, Co
         }
     }
 
-    public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Component Templates");
         ObjectNode templates = globalMetadata.getComponentTemplates();
 
@@ -95,7 +95,7 @@ public static void createComponentTemplates(GlobalMetadataData_OS_2_11 globalMet
         }
     }
 
-    public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, String[] indexTemplateWhitelist) throws Exception {
+    public static void createIndexTemplates(GlobalMetadataData_OS_2_11 globalMetadata, ConnectionDetails connectionDetails, List<String> indexTemplateWhitelist) throws Exception {
         logger.info("Setting Index Templates");
         ObjectNode templates = globalMetadata.getIndexTemplates();