Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDFS-15529: getChildFilesystems should include fallback fs as well #2234

Merged
merged 3 commits into from
Sep 3, 2020
Merged

HDFS-15529: getChildFilesystems should include fallback fs as well #2234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d0136d3
HDFS-15529: getChildFilesystems should include fallback fs as well
umamaheswararao Aug 20, 2020
a36b4e7
Fixed a test failure.
umamaheswararao Aug 30, 2020
71665bc
Fixed a checkstyle comment.
umamaheswararao Sep 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,15 @@ private boolean hasFallbackLink() {
return rootFallbackLink != null;
}

/**
* @return true if the root represented as internalDir. In LinkMergeSlash,
* there will be root to root mapping. So, root does not represent as
* internalDir.
*/
protected boolean isRootInternalDir() {
return root.isInternalDir();
}

protected INodeLink<T> getRootFallbackLink() {
Preconditions.checkState(root.isInternalDir());
return rootFallbackLink;
Expand Down
6 changes: 6 additions & 0 deletions ...ommon-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -939,6 +939,12 @@ public FileSystem[] getChildFileSystems() {
FileSystem targetFs = mountPoint.target.targetFileSystem;
children.addAll(Arrays.asList(targetFs.getChildFileSystems()));
}

if (fsState.isRootInternalDir() && fsState.getRootFallbackLink() != null) {
children.addAll(Arrays.asList(
fsState.getRootFallbackLink().targetFileSystem
.getChildFileSystems()));
}
return children.toArray(new FileSystem[]{});
}

Expand Down
16 changes: 13 additions & 3 deletions ...test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,10 +476,18 @@ public void testViewFsOverloadSchemeWithInnerCache()
// 2. Two hdfs file systems should be there if no cache.
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(2, vfs.getChildFileSystems().length);
Assert.assertEquals(isFallBackExist(conf) ? 3 : 2,
vfs.getChildFileSystems().length);
}
}

// HDFS-15529: if any extended tests added fallback, then getChildFileSystems
// will include fallback as well.
private boolean isFallBackExist(Configuration config) {
return config.get(ConfigUtil.getConfigViewFsPrefix(defaultFSURI
.getAuthority()) + "." + Constants.CONFIG_VIEWFS_LINK_FALLBACK) != null;
}

/**
* Create mount links as follows
* hdfs://localhost:xxx/HDFSUser0 --> hdfs://localhost:xxx/HDFSUser/
Expand All @@ -501,7 +509,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets()
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
// Two hdfs file systems should be there if no cache.
try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(2, vfs.getChildFileSystems().length);
Assert.assertEquals(isFallBackExist(conf) ? 3 : 2,
vfs.getChildFileSystems().length);
}
}

Expand All @@ -528,7 +537,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets()
// cache should work.
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(1, vfs.getChildFileSystems().length);
Assert.assertEquals(isFallBackExist(conf) ? 2 : 1,
vfs.getChildFileSystems().length);
}
}

Expand Down