Skip to content

Commit

Permalink
HDFS-17272. NNThroughputBenchmark should support specifying the base …
Browse files Browse the repository at this point in the history 
…directory for multi-client test (apache#6319). Contributed by caozhiqiang.

Reviewed-by: Tao Li <[email protected]>
Signed-off-by: Ayush Saxena <[email protected]>
  • Loading branch information
@lfxy
lfxy authored Dec 10, 2023
1 parent f96361f commit 37d6cad
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 33 deletions.
19 changes: 10 additions & 9 deletions hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,15 @@ Following are all the operations supported along with their respective operation
| OPERATION\_OPTION | Operation-specific parameters |
|:---- |:---- |
|`all` | _options for other operations_ |
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] |
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] |
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] |
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] |
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] |
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] [`-baseDirName /nnThroughputBenchmark`] |
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] [`-baseDirName /nnThroughputBenchmark`] |
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] [`-baseDirName /nnThroughputBenchmark`] |
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] [`-baseDirName /nnThroughputBenchmark`] |
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] [`-baseDirName /nnThroughputBenchmark`] |
|`clean` | N/A |

##### Operation Options
Expand All @@ -86,6 +86,7 @@ When running benchmarks with the above operation(s), please provide operation-sp
|`-nodeReplicationLimit` | The maximum number of outgoing replication streams for a data-node. |
|`-totalBlocks` | Number of total blocks to operate. |
|`-replication` | Replication factor. Will be adjusted to number of data-nodes if it is larger than that. |
|`-baseDirName` | The base dir name for benchmarks, to support multiple clients submitting benchmark tests at the same time. |

### Reports

Expand Down
85 changes: 61 additions & 24 deletions ...doop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ static void setNameNodeLoggingLevel(Level logLevel) {
* specific name-node operation.
*/
abstract class OperationStatsBase {
protected static final String BASE_DIR_NAME = "/nnThroughputBenchmark";
private String baseDirName = "/nnThroughputBenchmark";
protected static final String OP_ALL_NAME = "all";
protected static final String OP_ALL_USAGE = "-op all <other ops options>";

protected final String baseDir;
private String baseDir;
protected short replication;
protected int blockSize;
protected int numThreads = 0; // number of threads
Expand Down Expand Up @@ -228,7 +228,7 @@ abstract class OperationStatsBase {
abstract void printResults();

OperationStatsBase() {
baseDir = BASE_DIR_NAME + "/" + getOpName();
baseDir = baseDirName + "/" + getOpName();
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, 3);
blockSize = config.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
numOpsRequired = 10;
Expand Down Expand Up @@ -295,7 +295,12 @@ void cleanUp() throws IOException {
clientProto.saveNamespace(0, 0);
}
}

public String getBaseDirName() {
return baseDirName;
}
public void setBaseDirName(String baseDirName) {
this.baseDirName = baseDirName;
}
int getNumOpsExecuted() {
return numOpsExecuted;
}
Expand All @@ -316,10 +321,15 @@ long getAverageTime() {
return elapsedTime == 0 ? 0 : 1000*(double)numOpsExecuted / elapsedTime;
}

String getBaseDir() {
public String getBaseDir() {
setBaseDir(baseDirName + "/" + getOpName());
return baseDir;
}

public void setBaseDir(String baseDir) {
this.baseDir = baseDir;
}

String getClientName(int idx) {
return getOpName() + "-client-" + idx;
}
Expand Down Expand Up @@ -494,15 +504,15 @@ long executeOp(int daemonId, int inputIdx, String ignore)
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
long start = Time.now();
clientProto.delete(BASE_DIR_NAME, true);
clientProto.delete(getBaseDirName(), true);
long end = Time.now();
return end-start;
}

@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("Remove directory " + BASE_DIR_NAME);
LOG.info("Remove directory " + getBaseDirName());
printStats();
}
}
Expand All @@ -517,9 +527,9 @@ void printResults() {
class CreateFileStats extends OperationStatsBase {
// Operation types
static final String OP_CREATE_NAME = "create";
static final String OP_CREATE_USAGE =
static final String OP_CREATE_USAGE =
"-op create [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-close]";
+ " [-baseDirName D] [-close]";

protected FileNameGenerator nameGenerator;
protected String[][] fileNames;
Expand Down Expand Up @@ -553,6 +563,11 @@ void parseArguments(List<String> args) {
} else if(args.get(i).equals("-filesPerDir")) {
if(i+1 == args.size()) printUsage();
nrFilesPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(args.get(i).equals("-close")) {
closeUponCreate = true;
} else if(!ignoreUnrelatedOptions)
Expand All @@ -568,6 +583,7 @@ void generateInputs(int[] opsPerThread) throws IOException {
false);
// int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
LOG.info("basedir: " + getBaseDir());
fileNames = new String[numThreads][];
try {
for(int idx=0; idx < numThreads; idx++) {
Expand Down Expand Up @@ -618,6 +634,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("nrFiles = " + numOpsRequired);
LOG.info("nrThreads = " + numThreads);
LOG.info("nrFilesPerDir = " + nameGenerator.getFilesPerDirectory());
Expand All @@ -635,7 +652,7 @@ class MkdirsStats extends OperationStatsBase {
// Operation types
static final String OP_MKDIRS_NAME = "mkdirs";
static final String OP_MKDIRS_USAGE = "-op mkdirs [-threads T] [-dirs N] " +
"[-dirsPerDir P]";
"[-dirsPerDir P] [-baseDirName D]";

protected FileNameGenerator nameGenerator;
protected String[][] dirPaths;
Expand Down Expand Up @@ -664,6 +681,11 @@ void parseArguments(List<String> args) {
} else if(args.get(i).equals("-dirsPerDir")) {
if(i+1 == args.size()) printUsage();
nrDirsPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -718,6 +740,7 @@ long executeOp(int daemonId, int inputIdx, String clientName)
@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("nrDirs = " + numOpsRequired);
LOG.info("nrThreads = " + numThreads);
LOG.info("nrDirsPerDir = " + nameGenerator.getFilesPerDirectory());
Expand All @@ -736,7 +759,7 @@ class OpenFileStats extends CreateFileStats {
static final String OP_OPEN_NAME = "open";
static final String OP_USAGE_ARGS =
" [-threads T] [-files N] [-blockSize S] [-filesPerDir P]"
+ " [-useExisting]";
+ " [-useExisting] [-baseDirName D]";
static final String OP_OPEN_USAGE =
"-op " + OP_OPEN_NAME + OP_USAGE_ARGS;

Expand Down Expand Up @@ -765,13 +788,14 @@ void parseArguments(List<String> args) {
void generateInputs(int[] opsPerThread) throws IOException {
// create files using opsPerThread
String[] createArgs = new String[] {
"-op", "create",
"-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired),
"-blockSize", String.valueOf(blockSize),
"-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-close"};
"-op", "create",
"-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired),
"-blockSize", String.valueOf(blockSize),
"-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-baseDirName", getBaseDirName(),
"-close"};
CreateFileStats opCreate = new CreateFileStats(Arrays.asList(createArgs));

if(!useExisting) { // create files if they were not created before
Expand Down Expand Up @@ -1135,9 +1159,9 @@ private int transferBlocks( Block blocks[],
*/
class BlockReportStats extends OperationStatsBase {
static final String OP_BLOCK_REPORT_NAME = "blockReport";
static final String OP_BLOCK_REPORT_USAGE =
"-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S]";
static final String OP_BLOCK_REPORT_USAGE =
"-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S] [-baseDirName D]";

private int blocksPerReport;
private int blocksPerFile;
Expand Down Expand Up @@ -1187,6 +1211,11 @@ void parseArguments(List<String> args) {
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -1330,6 +1359,7 @@ void printResults() {
}
blockDistribution += ")";
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("reports = " + numOpsRequired);
LOG.info("datanodes = " + numThreads + " " + blockDistribution);
LOG.info("blocksPerReport = " + blocksPerReport);
Expand All @@ -1348,7 +1378,7 @@ class ReplicationStats extends OperationStatsBase {
static final String OP_REPLICATION_USAGE =
"-op replication [-datanodes T] [-nodesToDecommission D] " +
"[-nodeReplicationLimit C] [-totalBlocks B] [-blockSize S] "
+ "[-replication R]";
+ "[-replication R] [-baseDirName D]";

private final BlockReportStats blockReportObject;
private int numDatanodes;
Expand Down Expand Up @@ -1377,7 +1407,8 @@ class ReplicationStats extends OperationStatsBase {
"-datanodes", String.valueOf(numDatanodes),
"-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes),
"-blocksPerFile", String.valueOf(numDatanodes),
"-blockSize", String.valueOf(blockSize)};
"-blockSize", String.valueOf(blockSize),
"-baseDirName", getBaseDirName()};
blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs));
numDecommissionedBlocks = 0;
numPendingBlocks = 0;
Expand Down Expand Up @@ -1410,6 +1441,11 @@ void parseArguments(List<String> args) {
} else if (args.get(i).equals("-blockSize")) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -1485,6 +1521,7 @@ void printResults() {
}
blockDistribution += ")";
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("baseDir = " + getBaseDir());
LOG.info("numOpsRequired = " + numOpsRequired);
LOG.info("datanodes = " + numDatanodes + " " + blockDistribution);
LOG.info("decommissioned datanodes = " + nodesToDecommission);
Expand Down Expand Up @@ -1631,7 +1668,7 @@ public int run(String[] aArgs) throws Exception {
}
// run each benchmark
for(OperationStatsBase op : ops) {
LOG.info("Starting benchmark: " + op.getOpName());
LOG.info("Starting benchmark: " + op.getOpName() + ", baseDir: " + op.getBaseDir());
op.benchmark();
op.cleanUp();
}
Expand Down
36 changes: 36 additions & 0 deletions ...-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
Expand Down Expand Up @@ -187,4 +189,38 @@ public void testNNThroughputForBlockReportOp() throws Exception {
"blockReport", "-datanodes", "3", "-reports", "2"});
}
}

/**
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
* with explicit -baseDirName option.
*/
@Test(timeout = 120000)
public void testNNThroughputWithBaseDir() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
final Configuration benchConf = new HdfsConfiguration();
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
FileSystem.setDefaultUri(benchConf, cluster.getURI());
DistributedFileSystem fs = cluster.getFileSystem();

NNThroughputBenchmark.runBenchmark(benchConf,
new String[] {"-op", "create", "-keepResults", "-files", "3", "-baseDirName",
"/nnThroughputBenchmark1", "-close"});
Assert.assertTrue(fs.exists(new Path("/nnThroughputBenchmark1")));
Assert.assertFalse(fs.exists(new Path("/nnThroughputBenchmark")));

NNThroughputBenchmark.runBenchmark(benchConf,
new String[] {"-op", "all", "-baseDirName", "/nnThroughputBenchmark1"});
Assert.assertTrue(fs.exists(new Path("/nnThroughputBenchmark1")));
Assert.assertFalse(fs.exists(new Path("/nnThroughputBenchmark")));
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
}

0 comments on commit 37d6cad

Please sign in to comment.