Add logging for running count of source paths not found

apache · Oct 13, 2022 · 72b5662 · 72b5662
1 parent 5cbc405
commit 72b5662
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/...agement/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java b/...agement/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDataset.java
@@ -233,6 +233,7 @@ protected Map<Path, FileStatus> getFilePathsToFileStatus(FileSystem targetFs, Co
     );
 
     Map<Path, FileStatus> results = Maps.newHashMap();
+    long numSourceFilesNotFound = 0L;
     Iterable<String> filePathsIterable = () -> filePathsIterator;
     try {
       // TODO: investigate whether streaming initialization of `Map` preferable--`getFileStatus()` network calls likely
@@ -251,9 +252,10 @@ protected Map<Path, FileStatus> getFilePathsToFileStatus(FileSystem targetFs, Co
             throw fnfe;
           } else {
             // log, but otherwise swallow... to continue on
+            String total = ++numSourceFilesNotFound + " total";
             String speculation = "either premature deletion broke time-travel or metadata read interleaved among delete";
             errorConsolidator.prepLogMsg(path).ifPresent(msg ->
-                log.warn("~{}.{}~ source {} ({}...)", dbName, inputTableName, msg, speculation)
+                log.warn("~{}.{}~ source {} ({}... {})", dbName, inputTableName, msg, speculation, total)
             );
           }
         }