Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enable logging other types of ops. #254

Merged
merged 1 commit into from
Mar 21, 2020
Merged

enable logging other types of ops. #254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions bench/hdfs_create.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#! /bin/bash

for ((i=4;i<=64;i=i*2))
do
for ((j=1;j<=1000000;j=j*10))
do
for k in {1..2}
do
./sbin/stop-dfs.sh

# restart hadoop hdfs
rm -rf ~/hadoop/data/*
rm -rf ~/hadoop/name/*
rm -rf ~/hadoop/tmp/*
rm -rf logs/*
./bin/hdfs namenode -format -force
./sbin/start-dfs.sh

sleep 10
./bin/hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark -op create -threads ${i} -files ${j} -filesPerDir 10000000 -keepResults -logLevel INFO &>> hdfs_create_${i}_${j}.txt
done
done
done
2 changes: 1 addition & 1 deletion bench/hopsfs_create.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ do
./bin/hdfs namenode -format
./sbin/stop-nn.sh

sleep 10
sleep 10
./bin/hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark -op create -threads ${i} -files ${j} -filesPerDir 10000000 -keepResults -logLevel INFO &>> hopsfs_create_${i}_${j}.txt
done
done
Expand Down
10 changes: 5 additions & 5 deletions ...-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ static FileStatus modifyAclEntries(
List<AclEntry> newAcl = AclTransformation.mergeAclEntries(
existingAcl, aclSpec);
AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
// fsd.getEditLog().logSetAcl(src, newAcl);
fsd.getEditLog().logSetAcl(src, newAcl);
} catch (AclException e){
throw new AclException(e.getMessage() + " Path: " + src, e);
} finally {
Expand All @@ -78,7 +78,7 @@ static FileStatus removeAclEntries(
List<AclEntry> newAcl = AclTransformation.filterAclEntriesByAclSpec(
existingAcl, aclSpec);
AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
// fsd.getEditLog().logSetAcl(src, newAcl);
fsd.getEditLog().logSetAcl(src, newAcl);
} catch (AclException e){
throw new AclException(e.getMessage() + " Path: " + src, e);
} finally {
Expand All @@ -103,7 +103,7 @@ static FileStatus removeDefaultAcl(FSDirectory fsd, FSPermissionChecker pc,
List<AclEntry> newAcl = AclTransformation.filterDefaultAclEntries(
existingAcl);
AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
// fsd.getEditLog().logSetAcl(src, newAcl);
fsd.getEditLog().logSetAcl(src, newAcl);
} catch (AclException e){
throw new AclException(e.getMessage() + " Path: " + src, e);
} finally {
Expand All @@ -128,7 +128,7 @@ static FileStatus removeAcl(FSDirectory fsd, FSPermissionChecker pc,
} finally {
fsd.writeUnlock();
}
// fsd.getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST);
fsd.getEditLog().logSetAcl(src, AclFeature.EMPTY_ENTRY_LIST);
return fsd.getAuditFileInfo(iip);
}

Expand All @@ -143,7 +143,7 @@ static FileStatus setAcl(
iip = fsd.resolvePath(pc, src, DirOp.WRITE);
fsd.checkOwner(pc, iip);
List<AclEntry> newAcl = unprotectedSetAcl(fsd, iip, aclSpec, false);
// fsd.getEditLog().logSetAcl(iip.getPath(), newAcl);
fsd.getEditLog().logSetAcl(iip.getPath(), newAcl);
} catch (AclException e){
throw new AclException(e.getMessage() + " Path: " + src, e);
} finally {
Expand Down
14 changes: 7 additions & 7 deletions ...project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,13 +234,13 @@ static void setQuota(FSDirectory fsd, FSPermissionChecker pc, String src,
INodeDirectory changed =
unprotectedSetQuota(fsd, iip, nsQuota, ssQuota, type);
if (changed != null) {
// final QuotaCounts q = changed.getQuotaCounts();
// if (type == null) {
// fsd.getEditLog().logSetQuota(src, q.getNameSpace(), q.getStorageSpace());
// } else {
// fsd.getEditLog().logSetQuotaByStorageType(
// src, q.getTypeSpaces().get(type), type);
// }
final QuotaCounts q = changed.getQuotaCounts();
if (type == null) {
fsd.getEditLog().logSetQuota(src, q.getNameSpace(), q.getStorageSpace());
} else {
fsd.getEditLog().logSetQuotaByStorageType(
src, q.getTypeSpaces().get(type), type);
}
}
} finally {
fsd.writeUnlock();
Expand Down
16 changes: 8 additions & 8 deletions ...ect/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ static void allowSnapshot(FSDirectory fsd, SnapshotManager snapshotManager,
} finally {
fsd.writeUnlock();
}
// fsd.getEditLog().logAllowSnapshot(path);
fsd.getEditLog().logAllowSnapshot(path);
}

static void disallowSnapshot(
Expand All @@ -75,7 +75,7 @@ static void disallowSnapshot(
} finally {
fsd.writeUnlock();
}
// fsd.getEditLog().logDisallowSnapshot(path);
fsd.getEditLog().logDisallowSnapshot(path);
}

/**
Expand Down Expand Up @@ -112,8 +112,8 @@ static String createSnapshot(
} finally {
fsd.writeUnlock();
}
// fsd.getEditLog().logCreateSnapshot(snapshotRoot, snapshotName,
// logRetryCache);
fsd.getEditLog().logCreateSnapshot(snapshotRoot, snapshotName,
logRetryCache);

return snapshotPath;
}
Expand All @@ -133,8 +133,8 @@ static void renameSnapshot(FSDirectory fsd, FSPermissionChecker pc,
} finally {
fsd.writeUnlock();
}
// fsd.getEditLog().logRenameSnapshot(path, snapshotOldName,
// snapshotNewName, logRetryCache);
fsd.getEditLog().logRenameSnapshot(path, snapshotOldName,
snapshotNewName, logRetryCache);
}

static SnapshottableDirectoryStatus[] getSnapshottableDirListing(
Expand Down Expand Up @@ -259,8 +259,8 @@ static INode.BlocksMapUpdateInfo deleteSnapshot(
fsd.writeUnlock();
}
removedINodes.clear();
// fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName,
// logRetryCache);
fsd.getEditLog().logDeleteSnapshot(snapshotRoot, snapshotName,
logRetryCache);

return collectedBlocks;
}
Expand Down
4 changes: 2 additions & 2 deletions ...ject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,8 @@ private static INodeSymlink addSymlink(FSDirectory fsd, String path,
NameNode.stateChangeLog.info("addSymlink: failed to add " + path);
return null;
}
// fsd.getEditLog().logSymlink(path, target, mtime, mtime, newNode,
// logRetryCache);
fsd.getEditLog().logSymlink(path, target, mtime, mtime, newNode,
logRetryCache);

if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("addSymlink: " + path + " is added");
Expand Down
Loading