[GOBBLIN-1592] Make hive copy be able to apply filter on directory (#…

…3446)
apache · Dec 20, 2021 · 6e5ba1f · 6e5ba1f
1 parent 59df057
commit 6e5ba1f
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/...management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java b/...management/src/main/java/org/apache/gobblin/data/management/copy/RecursivePathFinder.java
@@ -47,6 +47,7 @@ public class RecursivePathFinder {
   private final FileSystem fs;
   private final PathFilter pathFilter;
   private final boolean includeEmptyDirectories;
+  private final boolean applyFilterToDirectories;
 
   public RecursivePathFinder(final FileSystem fs, Path rootPath, Properties properties) {
     this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath);
@@ -55,6 +56,7 @@ public RecursivePathFinder(final FileSystem fs, Path rootPath, Properties proper
     this.pathFilter = DatasetUtils.instantiatePathFilter(properties);
     this.includeEmptyDirectories =
         Boolean.parseBoolean(properties.getProperty(CopyConfiguration.INCLUDE_EMPTY_DIRECTORIES));
+    this.applyFilterToDirectories = Boolean.parseBoolean(properties.getProperty(CopyConfiguration.APPLY_FILTER_TO_DIRECTORIES));
   }
 
   public Set<FileStatus> getPaths(boolean skipHiddenPaths)
@@ -66,7 +68,7 @@ public Set<FileStatus> getPaths(boolean skipHiddenPaths)
     PathFilter actualFilter =
         skipHiddenPaths ? new AndPathFilter(new HiddenFilter(), this.pathFilter) : this.pathFilter;
     List<FileStatus> files =
-        FileListUtils.listFilesToCopyAtPath(this.fs, this.rootPath, actualFilter, includeEmptyDirectories);
+        FileListUtils.listFilesToCopyAtPath(this.fs, this.rootPath, actualFilter, this.applyFilterToDirectories, includeEmptyDirectories);
 
     return Sets.newHashSet(files);
   }