Merge branch 'main' into TypeMappingsTransformation

opensearch-project · Dec 10, 2024 · 00968cd · 00968cd
2 parents c13ef4a + 99bd50b
commit 00968cd
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 11 deletions.
diff --git a/...gration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java b/...gration/src/test/java/org/opensearch/migrations/bulkload/PerformanceVerificationTest.java
@@ -23,6 +23,7 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import reactor.core.publisher.Mono;
@@ -36,6 +37,7 @@
 import static org.mockito.Mockito.when;
 
 @Slf4j
+@Disabled("https://opensearch.atlassian.net/browse/MIGRATIONS-2254")
 public class PerformanceVerificationTest {
 
     @Test

diff --git a/RFS/src/main/java/org/opensearch/migrations/bulkload/common/LuceneDocumentsReader.java b/RFS/src/main/java/org/opensearch/migrations/bulkload/common/LuceneDocumentsReader.java
@@ -161,13 +161,17 @@ protected DirectoryReader wrapReader(DirectoryReader reader, boolean softDeletes
         return reader;
     }
 
-    protected RfsLuceneDocument getDocument(IndexReader reader, int docId, boolean isLive) {
+    protected RfsLuceneDocument getDocument(IndexReader reader, int docSegId, boolean isLive) {
         Document document;
         try {
-            document = reader.document(docId);
+            document = reader.document(docSegId);
         } catch (IOException e) {
-            log.atError().setCause(e).setMessage("Failed to read document at Lucene index location {}")
-                .addArgument(docId).log();
+            log.atError()
+                .setCause(e)
+                .setMessage("Failed to read document segment id {} from source {}")
+                .addArgument(docSegId)
+                .addArgument(indexDirectoryPath)
+                .log();
             return null;
         }
 
@@ -207,18 +211,21 @@ protected RfsLuceneDocument getDocument(IndexReader reader, int docId, boolean i
                 }
             }
             if (id == null) {
-                log.atError().setMessage("Document with index {} does not have an id. Skipping")
-                    .addArgument(docId).log();
+                log.atWarn().setMessage("Skipping document segment id {} from source {}, it does not have an referenceable id.")
+                    .addArgument(docSegId)
+                    .addArgument(indexDirectoryPath)
+                    .log();
                 return null;  // Skip documents with missing id
             }
 
             if (sourceBytes == null || sourceBytes.bytes.length == 0) {
-                log.atWarn().setMessage("Document {} doesn't have the _source field enabled")
-                    .addArgument(id).log();
+                log.atWarn().setMessage("Skipping document segment id {} document id {} from source {}, it doesn't have the _source field enabled.")
+                    .addArgument(docSegId)
+                    .addArgument(id)
+                    .addArgument(indexDirectoryPath)
+                    .log();
                 return null;  // Skip these
             }
-
-            log.atDebug().setMessage("Reading document {}").addArgument(id).log();
         } catch (RuntimeException e) {
             StringBuilder errorMessage = new StringBuilder();
             errorMessage.append("Unable to parse Document id from Document.  The Document's Fields: ");
@@ -232,7 +239,10 @@ protected RfsLuceneDocument getDocument(IndexReader reader, int docId, boolean i
             return null; // Skip these
         }
 
-        log.atDebug().setMessage("Document {} read successfully").addArgument(id).log();
+        log.atDebug().setMessage("Document id {} from source {} read successfully.")
+            .addArgument(id)
+            .addArgument(indexDirectoryPath)
+            .log();
         return new RfsLuceneDocument(id, type, sourceBytes.utf8ToString(), routing);
     }
 }