Skip to content

Commit

Permalink
NNThroughputBenchmark support specifying the base directory
Browse files Browse the repository at this point in the history
  • Loading branch information
eddy.cao committed Dec 2, 2023
1 parent d0b460f commit a41f6a9
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 14 deletions.
48 changes: 34 additions & 14 deletions ...doop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ static void setNameNodeLoggingLevel(Level logLevel) {
* specific name-node operation.
*/
abstract class OperationStatsBase {
protected static final String BASE_DIR_NAME = "/nnThroughputBenchmark";
protected String baseDirName = "/nnThroughputBenchmark";
protected static final String OP_ALL_NAME = "all";
protected static final String OP_ALL_USAGE = "-op all <other ops options>";

protected final String baseDir;
protected String baseDir;
protected short replication;
protected int blockSize;
protected int numThreads = 0; // number of threads
Expand Down Expand Up @@ -228,7 +228,7 @@ abstract class OperationStatsBase {
abstract void printResults();

OperationStatsBase() {
baseDir = BASE_DIR_NAME + "/" + getOpName();
baseDir = baseDirName + "/" + getOpName();
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
blockSize = config.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
numOpsRequired = 10;
Expand Down Expand Up @@ -317,6 +317,7 @@ long getAverageTime() {
}

String getBaseDir() {
baseDir = baseDirName + "/" + getOpName();
return baseDir;
}

Expand Down Expand Up @@ -494,15 +495,15 @@ long executeOp(int daemonId, int inputIdx, String ignore)
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
long start = Time.now();
clientProto.delete(BASE_DIR_NAME, true);
clientProto.delete(baseDirName, true);
long end = Time.now();
return end-start;
}

@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("Remove directory " + BASE_DIR_NAME);
LOG.info("Remove directory " + baseDirName);
printStats();
}
}
Expand All @@ -517,9 +518,9 @@ void printResults() {
class CreateFileStats extends OperationStatsBase {
// Operation types
static final String OP_CREATE_NAME = "create";
static final String OP_CREATE_USAGE =
static final String OP_CREATE_USAGE =
"-op create [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-close]";
+ " [-baseDirName D] [-close]";

protected FileNameGenerator nameGenerator;
protected String[][] fileNames;
Expand Down Expand Up @@ -553,6 +554,9 @@ void parseArguments(List<String> args) {
} else if(args.get(i).equals("-filesPerDir")) {
if(i+1 == args.size()) printUsage();
nrFilesPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
} else if(args.get(i).equals("-close")) {
closeUponCreate = true;
} else if(!ignoreUnrelatedOptions)
Expand All @@ -568,6 +572,7 @@ void generateInputs(int[] opsPerThread) throws IOException {
false);
// int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
LOG.info("basedir: " + getBaseDir());
fileNames = new String[numThreads][];
try {
for(int idx=0; idx < numThreads; idx++) {
Expand Down Expand Up @@ -618,6 +623,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("nrFiles = " + numOpsRequired);
LOG.info("nrThreads = " + numThreads);
LOG.info("nrFilesPerDir = " + nameGenerator.getFilesPerDirectory());
Expand All @@ -635,7 +641,7 @@ class MkdirsStats extends OperationStatsBase {
// Operation types
static final String OP_MKDIRS_NAME = "mkdirs";
static final String OP_MKDIRS_USAGE = "-op mkdirs [-threads T] [-dirs N] " +
"[-dirsPerDir P]";
"[-dirsPerDir P] [-baseDirName D]";

protected FileNameGenerator nameGenerator;
protected String[][] dirPaths;
Expand Down Expand Up @@ -664,6 +670,9 @@ void parseArguments(List<String> args) {
} else if(args.get(i).equals("-dirsPerDir")) {
if(i+1 == args.size()) printUsage();
nrDirsPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -718,6 +727,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("nrDirs = " + numOpsRequired);
LOG.info("nrThreads = " + numThreads);
LOG.info("nrDirsPerDir = " + nameGenerator.getFilesPerDirectory());
Expand All @@ -736,7 +746,7 @@ class OpenFileStats extends CreateFileStats {
static final String OP_OPEN_NAME = "open";
static final String OP_USAGE_ARGS =
" [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-useExisting]";
+ " [-baseDirName D] [-useExisting]";
static final String OP_OPEN_USAGE =
"-op " + OP_OPEN_NAME + OP_USAGE_ARGS;

Expand Down Expand Up @@ -771,6 +781,7 @@ void generateInputs(int[] opsPerThread) throws IOException {
"-blockSize", String.valueOf(blockSize),
"-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-baseDirName", this.baseDirName,
"-close"};
CreateFileStats opCreate = new CreateFileStats(Arrays.asList(createArgs));

Expand Down Expand Up @@ -1135,9 +1146,9 @@ private int transferBlocks( Block blocks[],
*/
class BlockReportStats extends OperationStatsBase {
static final String OP_BLOCK_REPORT_NAME = "blockReport";
static final String OP_BLOCK_REPORT_USAGE =
static final String OP_BLOCK_REPORT_USAGE =
"-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S]";
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S] [-baseDirName D]";

private int blocksPerReport;
private int blocksPerFile;
Expand Down Expand Up @@ -1187,6 +1198,9 @@ void parseArguments(List<String> args) {
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -1330,6 +1344,7 @@ void printResults() {
}
blockDistribution += ")";
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("reports = " + numOpsRequired);
LOG.info("datanodes = " + numThreads + " " + blockDistribution);
LOG.info("blocksPerReport = " + blocksPerReport);
Expand All @@ -1348,7 +1363,7 @@ class ReplicationStats extends OperationStatsBase {
static final String OP_REPLICATION_USAGE =
"-op replication [-datanodes T] [-nodesToDecommission D] " +
"[-nodeReplicationLimit C] [-totalBlocks B] [-blockSize S] "
+ "[-replication R]";
+ "[-replication R] [-baseDirName N]";

private final BlockReportStats blockReportObject;
private int numDatanodes;
Expand Down Expand Up @@ -1377,7 +1392,8 @@ class ReplicationStats extends OperationStatsBase {
"-datanodes", String.valueOf(numDatanodes),
"-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes),
"-blocksPerFile", String.valueOf(numDatanodes),
"-blockSize", String.valueOf(blockSize)};
"-blockSize", String.valueOf(blockSize),
"-baseDirName", baseDirName};
blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs));
numDecommissionedBlocks = 0;
numPendingBlocks = 0;
Expand Down Expand Up @@ -1410,6 +1426,9 @@ void parseArguments(List<String> args) {
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -1485,6 +1504,7 @@ void printResults() {
}
blockDistribution += ")";
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("numOpsRequired = " + numOpsRequired);
LOG.info("datanodes = " + numDatanodes + " " + blockDistribution);
LOG.info("decommissioned datanodes = " + nodesToDecommission);
Expand Down Expand Up @@ -1631,7 +1651,7 @@ public int run(String[] aArgs) throws Exception {
}
// run each benchmark
for(OperationStatsBase op : ops) {
LOG.info("Starting benchmark: " + op.getOpName());
LOG.info("Starting benchmark: " + op.getOpName() + ", baseDir: " + op.getBaseDir());
op.benchmark();
op.cleanUp();
}
Expand Down
12 changes: 12 additions & 0 deletions ...-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,18 @@ public void testNNThroughput() throws Exception {
NNThroughputBenchmark.runBenchmark(conf, new String[] {"-op", "all"});
}

@Test
public void testNNThroughputWithBaseDir() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
File nameDir = new File(MiniDFSCluster.getBaseDirectory(), "name");
conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,
nameDir.getAbsolutePath());
DFSTestUtil.formatNameNode(conf);
NNThroughputBenchmark.runBenchmark(conf,
new String[] {"-op", "all", "-baseDirName", "/nnThroughputBenchmark1"});
}

/**
* This test runs all benchmarks defined in {@link NNThroughputBenchmark},
* with explicit local -fs option.
Expand Down

0 comments on commit a41f6a9

Please sign in to comment.