Update metadata transformations based on PR feedback

Signed-off-by: Andre Kurait <[email protected]>
AndreKurait · Nov 22, 2024 · d6643c2 · d6643c2
1 parent 488c664
commit d6643c2
Show file tree

Hide file tree

Showing 8 changed files with 337 additions and 62 deletions.
diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java b/MetadataMigration/src/main/java/org/opensearch/migrations/MigrateOrEvaluateArgs.java
@@ -56,7 +56,7 @@ public class MigrateOrEvaluateArgs {
     public Version sourceVersion = null;
 
     @ParametersDelegate
-    public MetadataTransformerParams metadataTransformationParams = new MetadataTransformerParams();
+    public TransformerParams metadataTransformationParams = new MetadataTransformerParams();
 
     @Getter
     public static class MetadataTransformerParams implements TransformerParams {

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Evaluate.java
@@ -25,10 +25,7 @@ public EvaluateResult execute(RootMetadataMigrationContext context) {
             var clusters = createClusters();
             evaluateResult.clusters(clusters);
 
-            var transformer = new CompositeTransformer(
-                    getCustomTransformer(),
-                    selectTransformer(clusters)
-            );
+            var transformer = selectTransformer(clusters);
 
             var items = migrateAllItems(migrationMode, clusters, transformer, context);
             evaluateResult.items(items);

diff --git a/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java b/MetadataMigration/src/main/java/org/opensearch/migrations/commands/Migrate.java
@@ -25,10 +25,7 @@ public MigrateResult execute(RootMetadataMigrationContext context) {
             var clusters = createClusters();
             migrateResult.clusters(clusters);
 
-            var transformer = new CompositeTransformer(
-                    getCustomTransformer(),
-                    selectTransformer(clusters)
-            );
+            var transformer = selectTransformer(clusters);
 
             var items = migrateAllItems(migrationMode, clusters, transformer, context);
             migrateResult.items(items);

diff --git a/...dataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java b/...dataMigration/src/main/java/org/opensearch/migrations/commands/MigratorEvaluatorBase.java
@@ -5,6 +5,7 @@
 
 import org.opensearch.migrations.MigrateOrEvaluateArgs;
 import org.opensearch.migrations.MigrationMode;
+import org.opensearch.migrations.bulkload.transformers.CompositeTransformer;
 import org.opensearch.migrations.bulkload.transformers.TransformFunctions;
 import org.opensearch.migrations.bulkload.transformers.Transformer;
 import org.opensearch.migrations.bulkload.transformers.TransformerToIJsonTransformerAdapter;
@@ -59,7 +60,7 @@ protected Transformer getCustomTransformer() {
             log.atInfo().setMessage("Metadata Transformations config string: {}")
                     .addArgument(transformerConfig).log();
         } else {
-            log.atInfo().setMessage("Using Noop transformation config: {}")
+            log.atInfo().setMessage("Using Noop custom transformation config: {}")
                     .addArgument(NOOP_TRANSFORMATION_CONFIG).log();
             transformerConfig = NOOP_TRANSFORMATION_CONFIG;
         }
@@ -68,13 +69,15 @@ protected Transformer getCustomTransformer() {
     }
 
     protected Transformer selectTransformer(Clusters clusters) {
-        var transformer = TransformFunctions.getTransformer(
+        var versionTransformer = TransformFunctions.getTransformer(
             clusters.getSource().getVersion(),
             clusters.getTarget().getVersion(),
             arguments.minNumberOfReplicas
         );
-        log.atInfo().setMessage("Selected transformer: {}").addArgument(transformer).log();
-        return transformer;
+        var customTransformer = getCustomTransformer();
+        var compositeTransformer = new CompositeTransformer(customTransformer, versionTransformer);
+        log.atInfo().setMessage("Selected transformer: {}").addArgument(compositeTransformer).log();
+        return compositeTransformer;
     }
 
     protected Items migrateAllItems(MigrationMode migrationMode, Clusters clusters, Transformer transformer, RootMetadataMigrationContext context) {

diff --git a/MetadataMigration/src/main/resources/log4j2.properties b/MetadataMigration/src/main/resources/log4j2.properties
@@ -3,69 +3,72 @@ status = WARN
 property.logsDir = ${env:SHARED_LOGS_DIR_PATH:-./logs}
 property.failedLoggerFileNamePrefix = ${logsDir}/${hostName}/failedRequests/failedRequests
 property.metadataTuplesFileNamePrefix = ${logsDir}/${hostName}/metadataTuples/tuples
-
-appenders = console, FailedRequests, MetadataRun, MetadataTuples
-
-appender.FailedRequests.type = RollingRandomAccessFile
-appender.FailedRequests.name = FailedRequests
-appender.FailedRequests.fileName = ${failedLoggerFileNamePrefix}.log
-appender.FailedRequests.filePattern = ${failedLoggerFileNamePrefix}-%d{yyyy-MM-dd-HH:mm}{UTC}-%i.log.gz
-appender.FailedRequests.layout.type = PatternLayout
-appender.FailedRequests.layout.pattern = %m%n
-appender.FailedRequests.policies.type = Policies
-appender.FailedRequests.policies.size.type = SizeBasedTriggeringPolicy
-appender.FailedRequests.policies.size.size = 10 MB
-appender.FailedRequests.strategy.type = DefaultRolloverStrategy
-appender.FailedRequests.immediateFlush = false
-
-logger.FailedRequestsLogger.name = FailedRequestsLogger
-logger.FailedRequestsLogger.level = info
-logger.FailedRequestsLogger.additivity = false
-logger.FailedRequestsLogger.appenderRef.FailedRequests.ref = FailedRequests
-
 property.runTime = ${date:yyyy-MM-dd_HH-mm-ss}
-property.metadataRunLoggerFileNamePrefix = ${logsDir}/${hostName}/metadata/metadata_
-
-appender.MetadataRun.type = File
-appender.MetadataRun.name = MetadataRun
-appender.MetadataRun.fileName = ${metadataRunLoggerFileNamePrefix}${runTime}.log
-appender.MetadataRun.layout.type = PatternLayout
-appender.MetadataRun.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n
-appender.MetadataRun.immediateFlush = true
+property.metadataRunLoggerFileNamePrefix = ${logsDir}/${hostName}/metadata/metadata
 
-logger.MetadataLogger.name = MetadataLogger
-logger.MetadataLogger.level = debug
-logger.MetadataLogger.additivity = false
-logger.MetadataLogger.appenderRef.MetadataRun.ref = MetadataRun
+appenders = console, MetadataTuples, FailedRequests, MetadataRun
 
 appender.console.type = Console
 appender.console.name = Console
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %m%n
 
-rootLogger.level = info
+rootLogger.level = ERROR
 rootLogger.appenderRef.console.ref = MetadataRun
 
+# Metadata Migration
 logger.MetadataMigration.name = org.opensearch.migrations.MetadataMigration
 logger.MetadataMigration.level = info
 logger.MetadataMigration.additivity = false
 logger.MetadataMigration.appenderRef.stdout.ref = Console
 logger.MetadataMigration.appenderRef.MetadataRun.ref = MetadataRun
 
+# Metadata Tuples
 appender.MetadataTuples.type = RollingRandomAccessFile
 appender.MetadataTuples.name = MetadataTuples
 appender.MetadataTuples.fileName = ${metadataTuplesFileNamePrefix}.log
-appender.MetadataTuples.filePattern = ${metadataTuplesFileNamePrefix}-%d{yyyy-MM-dd-HH-mm}{UTC}-%i.log
+appender.MetadataTuples.filePattern = ${metadataTuplesFileNamePrefix}_${runTime}-%i.log
 appender.MetadataTuples.layout.type = PatternLayout
 appender.MetadataTuples.layout.pattern = %m%n
 appender.MetadataTuples.policies.type = Policies
-appender.MetadataTuples.policies.size.type = SizeBasedTriggeringPolicy
-appender.MetadataTuples.policies.size.size = 10 MB
+appender.MetadataTuples.policies.startup.type = OnStartupTriggeringPolicy
+appender.MetadataTuples.policies.startup.minSize = 0
 appender.MetadataTuples.strategy.type = DefaultRolloverStrategy
 appender.MetadataTuples.immediateFlush = false
 
 logger.OutputTransformationJsonLogger.name = OutputTransformationJsonLogger
 logger.OutputTransformationJsonLogger.level = info
 logger.OutputTransformationJsonLogger.additivity = false
 logger.OutputTransformationJsonLogger.appenderRef.MetadataTuples.ref = MetadataTuples
+
+# MetadataRun Logs
+appender.MetadataRun.type = File
+appender.MetadataRun.name = MetadataRun
+appender.MetadataRun.fileName = ${metadataRunLoggerFileNamePrefix}${runTime}-%i.log
+appender.MetadataRun.layout.type = PatternLayout
+appender.MetadataRun.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS}{UTC} %p %c{1.} [%t] %m%n
+appender.MetadataRun.immediateFlush = false
+
+logger.MetadataLogger.name = MetadataLogger
+logger.MetadataLogger.level = debug
+logger.MetadataLogger.additivity = false
+logger.MetadataLogger.appenderRef.MetadataRun.ref = MetadataRun
+
+# Failed Requestss
+appender.FailedRequests.type = RollingRandomAccessFile
+appender.FailedRequests.name = FailedRequests
+appender.FailedRequests.fileName = ${failedLoggerFileNamePrefix}.log
+appender.FailedRequests.filePattern = ${failedLoggerFileNamePrefix}-%d{yyyy-MM-dd-HH:mm}{UTC}-%i.log.gz
+appender.FailedRequests.layout.type = PatternLayout
+appender.FailedRequests.layout.pattern = %m%n
+appender.FailedRequests.policies.type = Policies
+appender.FailedRequests.policies.size.type = SizeBasedTriggeringPolicy
+appender.FailedRequests.policies.size.size = 10 MB
+appender.FailedRequests.strategy.type = DefaultRolloverStrategy
+appender.FailedRequests.immediateFlush = false
+
+logger.FailedRequestsLogger.name = FailedRequestsLogger
+logger.FailedRequestsLogger.level = info
+logger.FailedRequestsLogger.additivity = false
+logger.FailedRequestsLogger.appenderRef.FailedRequests.ref = FailedRequests