diff --git a/bench/aws_hdfs_create.py b/bench/aws_hdfs_create.py new file mode 100644 index 00000000000..a9d9d489a85 --- /dev/null +++ b/bench/aws_hdfs_create.py @@ -0,0 +1,15 @@ +import sys +from fabric import Connection +from tee import StdoutTee, StderrTee + +cnn = "3.89.124.4" + +node = {"1":"3.89.124.4"} + +connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']} + + +with Connection(host=cnn, user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/hdfs/hadoop-3.3.0-SNAPSHOT;" + "source etc/hadoop/hadoop-env.sh;" + "bash bench.sh") diff --git a/bench/aws_hdfs_open.py b/bench/aws_hdfs_open.py new file mode 100644 index 00000000000..ee628986f31 --- /dev/null +++ b/bench/aws_hdfs_open.py @@ -0,0 +1,15 @@ +import sys +from fabric import Connection +from tee import StdoutTee, StderrTee + +cnn = "34.203.247.95" + +node = {"1":"34.203.247.95"} + +connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']} + + +with Connection(host=cnn, user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/hdfs/hadoop-3.3.0-SNAPSHOT;" + "source etc/hadoop/hadoop-env.sh;" + "bash bench.sh") diff --git a/bench/aws_voltfs_create32.py b/bench/aws_voltfs_create32.py new file mode 100644 index 00000000000..ef5ed73e2b6 --- /dev/null +++ b/bench/aws_voltfs_create32.py @@ -0,0 +1,56 @@ +import sys +from fabric import Connection +from tee import StdoutTee, StderrTee + +cnn = "3.95.218.253" + +node = {"1":"3.95.218.253", + "2":"3.86.224.36", + "3":"34.230.74.46", + "4":"54.174.45.253", + "5":"35.175.146.114", + "6":"54.236.178.100", + "7":"52.91.200.92", + "8":"35.171.23.86", + "9":"34.238.119.140", + "10":"34.238.152.118", + "11":"34.238.235.121", + "12":"3.95.8.223", + "13":"3.82.145.128", + "14":"18.207.195.249", + "15":"184.72.102.92", + "16":"54.87.184.159", + "17":"34.230.71.202", + "18":"54.152.88.48", + "19":"18.204.213.108", + "20":"54.88.232.184", + "21":"54.243.12.184", + "22":"3.85.160.202", + "23":"3.95.154.153", + "24":"34.207.167.5", + "25":"18.212.38.73", + "26":"100.26.194.178", + "27":"3.83.87.206", + "28":"3.89.251.96", + "29":"3.82.139.86", + "30":"100.27.19.118", + "31":"54.144.62.126", + "32":"3.93.248.58"} + +connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']} + +for key, value in node.items(): + if int(key) >= 3: + continue + print(value) + with Connection(host=value, user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/voltfs/hadoop-3.3.0-SNAPSHOT;" + "source etc/hadoop/hadoop-env.sh;" + "bash test.sh") + +with StderrTee("aws_voltfs_create_2.txt"), Connection(host=cnn, user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/voltfs/hadoop-3.3.0-SNAPSHOT;" + + "source etc/hadoop/hadoop-env.sh;" + + "./bin/hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark " + + "-fs hdfs://localhost:65212 -op create -threads 2 -files 2000000 -filesPerDir 1000005 " + + "-keepResults -logLevel INFO") diff --git a/bench/aws_voltfs_open.py b/bench/aws_voltfs_open.py new file mode 100644 index 00000000000..e0db885d91d --- /dev/null +++ b/bench/aws_voltfs_open.py @@ -0,0 +1,32 @@ +import sys +from fabric import Connection +from tee import StdoutTee, StderrTee + +cnn = "54.210.61.195" + +node = {"1":"54.210.61.195", + "2":"18.207.218.241", + "3":"3.80.206.89", + "4":"3.86.187.51", + "5":"34.227.14.189", + "6":"100.26.112.54", + "7":"35.175.146.118", + "8":"54.91.147.7"} + +connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']} + +for key, value in node.items(): + if int(key) >= 5: + continue + print(value) + with Connection(host=value, user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/voltfs/hadoop-3.3.0-SNAPSHOT;" + "source etc/hadoop/hadoop-env.sh;" + "bash test.sh") + +with StderrTee("aws_voltfs_create_2nn1.txt"), Connection(host=node["2"], user="ec2-user", connect_kwargs=connect_kwargs) as c: + c.run("cd /home/ec2-user/voltfs/hadoop-3.3.0-SNAPSHOT;" + + "source etc/hadoop/hadoop-env.sh;" + + "./bin/hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark " + + "-fs hdfs://localhost:9000 -op create -threads 1 -files 1000000 -filesPerDir 1000005 " + + "-keepResults -logLevel INFO") diff --git a/bench/parse.py b/bench/parse.py new file mode 100644 index 00000000000..7344c97d288 --- /dev/null +++ b/bench/parse.py @@ -0,0 +1,12 @@ +f = open("./aws_stderr.txt", "r") +searchlines = f.readlines() +f.close() + +sum = 0.0 +iter = 0 +for i, line in enumerate(searchlines): + if "Ops per sec: " in line: + iter = iter + 1 + sum += float(line[line.rfind(':')+1:]) + print float(line[line.rfind(':')+1:]) +print sum / iter \ No newline at end of file