Skip to content

Commit

Permalink
fix the checkstyle
Browse files Browse the repository at this point in the history
  • Loading branch information
eddy.cao committed Dec 5, 2023
1 parent fa1ce21 commit 84cd5ee
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 47 deletions.
19 changes: 10 additions & 9 deletions hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,15 @@ Following are all the operations supported along with their respective operation
| OPERATION\_OPTION | Operation-specific parameters |
|:---- |:---- |
|`all` | _options for other operations_ |
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] |
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] |
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] |
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] |
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] |
|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] [`-baseDirName /nnThroughputBenchmark`] |
|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] [`-baseDirName /nnThroughputBenchmark`] |
|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`append` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-appendNewBlk`] [`-baseDirName /nnThroughputBenchmark`] |
|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] [`-baseDirName /nnThroughputBenchmark`] |
|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] [`-baseDirName /nnThroughputBenchmark`] |
|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] [`-baseDirName /nnThroughputBenchmark`] |
|`clean` | N/A |

##### Operation Options
Expand All @@ -86,6 +86,7 @@ When running benchmarks with the above operation(s), please provide operation-sp
|`-nodeReplicationLimit` | The maximum number of outgoing replication streams for a data-node. |
|`-totalBlocks` | Number of total blocks to operate. |
|`-replication` | Replication factor. Will be adjusted to number of data-nodes if it is larger than that. |
|`-baseDirName` | The base dir name for benchmarks, to support multiple clients submitting benchmark tests at the same time. |

### Reports

Expand Down
69 changes: 43 additions & 26 deletions ...doop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,11 @@ static void setNameNodeLoggingLevel(Level logLevel) {
* specific name-node operation.
*/
abstract class OperationStatsBase {
protected String baseDirName = "/nnThroughputBenchmark";
private String baseDirName = "/nnThroughputBenchmark";
protected static final String OP_ALL_NAME = "all";
protected static final String OP_ALL_USAGE = "-op all <other ops options>";

protected String baseDir;
private String baseDir;
protected short replication;
protected int blockSize;
protected int numThreads = 0; // number of threads
Expand Down Expand Up @@ -295,7 +295,12 @@ void cleanUp() throws IOException {
clientProto.saveNamespace(0, 0);
}
}

public String getBaseDirName() {
return baseDirName;
}
public void setBaseDirName(String baseDirName) {
this.baseDirName = baseDirName;
}
int getNumOpsExecuted() {
return numOpsExecuted;
}
Expand All @@ -316,11 +321,15 @@ long getAverageTime() {
return elapsedTime == 0 ? 0 : 1000*(double)numOpsExecuted / elapsedTime;
}

String getBaseDir() {
baseDir = baseDirName + "/" + getOpName();
public String getBaseDir() {
setBaseDir(baseDirName + "/" + getOpName());
return baseDir;
}

public void setBaseDir(String baseDir) {
this.baseDir = baseDir;
}

String getClientName(int idx) {
return getOpName() + "-client-" + idx;
}
Expand Down Expand Up @@ -495,15 +504,15 @@ long executeOp(int daemonId, int inputIdx, String ignore)
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
long start = Time.now();
clientProto.delete(baseDirName, true);
clientProto.delete(getBaseDirName(), true);
long end = Time.now();
return end-start;
}

@Override
void printResults() {
LOG.info("--- " + getOpName() + " inputs ---");
LOG.info("Remove directory " + baseDirName);
LOG.info("Remove directory " + getBaseDirName());
printStats();
}
}
Expand Down Expand Up @@ -555,8 +564,10 @@ void parseArguments(List<String> args) {
if(i+1 == args.size()) printUsage();
nrFilesPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(args.get(i).equals("-close")) {
closeUponCreate = true;
} else if(!ignoreUnrelatedOptions)
Expand Down Expand Up @@ -671,8 +682,10 @@ void parseArguments(List<String> args) {
if(i+1 == args.size()) printUsage();
nrDirsPerDir = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -775,14 +788,14 @@ void parseArguments(List<String> args) {
void generateInputs(int[] opsPerThread) throws IOException {
// create files using opsPerThread
String[] createArgs = new String[] {
"-op", "create",
"-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired),
"-blockSize", String.valueOf(blockSize),
"-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-baseDirName", this.baseDirName,
"-close"};
"-op", "create",
"-threads", String.valueOf(this.numThreads),
"-files", String.valueOf(numOpsRequired),
"-blockSize", String.valueOf(blockSize),
"-filesPerDir",
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-baseDirName", getBaseDirName(),
"-close"};
CreateFileStats opCreate = new CreateFileStats(Arrays.asList(createArgs));

if(!useExisting) { // create files if they were not created before
Expand Down Expand Up @@ -1147,8 +1160,8 @@ private int transferBlocks( Block blocks[],
class BlockReportStats extends OperationStatsBase {
static final String OP_BLOCK_REPORT_NAME = "blockReport";
static final String OP_BLOCK_REPORT_USAGE =
"-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S] [-baseDirName D]";
"-op blockReport [-datanodes T] [-reports N] " +
"[-blocksPerReport B] [-blocksPerFile F] [-blockSize S] [-baseDirName D]";

private int blocksPerReport;
private int blocksPerFile;
Expand Down Expand Up @@ -1199,8 +1212,10 @@ void parseArguments(List<String> args) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down Expand Up @@ -1393,7 +1408,7 @@ class ReplicationStats extends OperationStatsBase {
"-blocksPerReport", String.valueOf(totalBlocks*replication/numDatanodes),
"-blocksPerFile", String.valueOf(numDatanodes),
"-blockSize", String.valueOf(blockSize),
"-baseDirName", baseDirName};
"-baseDirName", getBaseDirName()};
blockReportObject = new BlockReportStats(Arrays.asList(blkReportArgs));
numDecommissionedBlocks = 0;
numPendingBlocks = 0;
Expand Down Expand Up @@ -1427,8 +1442,10 @@ void parseArguments(List<String> args) {
if(i+1 == args.size()) printUsage();
blockSize = Integer.parseInt(args.get(++i));
} else if(args.get(i).equals("-baseDirName")) {
if(i+1 == args.size()) printUsage();
baseDirName = args.get(++i);
if (i + 1 == args.size()) {
printUsage();
}
setBaseDirName(args.get(++i));
} else if(!ignoreUnrelatedOptions)
printUsage();
}
Expand Down
24 changes: 12 additions & 12 deletions ...-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,34 +209,34 @@ public void testNNThroughputWithBaseDir() throws Exception {
"/nnThroughputBenchmark1", "-close"});
FSNamesystem fsNamesystem = cluster.getNamesystem();
DirectoryListing listing = fsNamesystem.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
Boolean b_dir_exist1 = false;
Boolean b_dir_exist2 = false;
Boolean dirExist1 = false;
Boolean dirExist2 = false;
for (HdfsFileStatus f : listing.getPartialListing()) {
if (f.getFullName("/").equals("/nnThroughputBenchmark1")) {
b_dir_exist1 = true;
dirExist1 = true;
}
if (f.getFullName("/").equals("/nnThroughputBenchmark")) {
b_dir_exist2 = true;
dirExist2 = true;
}
}
Assert.assertEquals(b_dir_exist1, true);
Assert.assertEquals(b_dir_exist2, false);
Assert.assertEquals(dirExist1, true);
Assert.assertEquals(dirExist2, false);

NNThroughputBenchmark.runBenchmark(benchConf,
new String[] {"-op", "all", "-baseDirName", "/nnThroughputBenchmark1"});
listing = fsNamesystem.getListing("/", HdfsFileStatus.EMPTY_NAME, false);
b_dir_exist1 = false;
b_dir_exist2 = false;
dirExist1 = false;
dirExist2 = false;
for (HdfsFileStatus f : listing.getPartialListing()) {
if (f.getFullName("/").equals("/nnThroughputBenchmark1")) {
b_dir_exist1 = true;
dirExist1 = true;
}
if (f.getFullName("/").equals("/nnThroughputBenchmark")) {
b_dir_exist2 = true;
dirExist2 = true;
}
}
Assert.assertEquals(b_dir_exist1, true);
Assert.assertEquals(b_dir_exist2, false);
Assert.assertEquals(dirExist1, true);
Assert.assertEquals(dirExist2, false);
} finally {
if (cluster != null) {
cluster.shutdown();
Expand Down

0 comments on commit 84cd5ee

Please sign in to comment.