diff --git a/wasm-engines/fluence_bencher/.WasmVMBencher.py.swp b/wasm-engines/fluence_bencher/.WasmVMBencher.py.swp deleted file mode 100644 index 274ae1e..0000000 Binary files a/wasm-engines/fluence_bencher/.WasmVMBencher.py.swp and /dev/null differ diff --git a/wasm-engines/fluence_bencher/.settings.py.swp b/wasm-engines/fluence_bencher/.settings.py.swp deleted file mode 100644 index 71d9e99..0000000 Binary files a/wasm-engines/fluence_bencher/.settings.py.swp and /dev/null differ diff --git a/wasm-engines/fluence_bencher/VMDescriptor.py b/wasm-engines/fluence_bencher/VMDescriptor.py deleted file mode 100644 index a7629ec..0000000 --- a/wasm-engines/fluence_bencher/VMDescriptor.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Copyright 2018 Fluence Labs Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - - -class VMDescriptor: - """Wasm VM descriptor that specifies how VM has to be launched. - - Attributes - ---------- - vm_relative_binary_path : str - A relative path to VM binary in its main folder. - vm_launch_cmd : str - An format string with command for launch this vm with provided test. - is_compiler_type : bool - True, if vm is compiler-type (JIT, AOT, ...). - - """ - def __init__(self, vm_binary_name="", vm_launch_cmd=""): - self.vm_binary_name = vm_binary_name - self.vm_launch_cmd = vm_launch_cmd diff --git a/wasm-engines/fluence_bencher/WasmVMBencher.py b/wasm-engines/fluence_bencher/WasmVMBencher.py deleted file mode 100644 index 72d14af..0000000 --- a/wasm-engines/fluence_bencher/WasmVMBencher.py +++ /dev/null @@ -1,534 +0,0 @@ -""" -Copyright 2018 Fluence Labs Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from os import listdir -from os.path import join -from time import time -import subprocess -#from subprocess import Popen -from collections import defaultdict -import logging -# TODO: use nanodurationpy because asmble uses nanoseconds -import durationpy -import re -import shlex - - -class Record: - """Contains measures of one test launch. - - Attributes - ---------- - time : time_type - The execution time of one test. - cpu_load : int - The cpu load (in percents) of one test (currently not supported). - - """ - def __init__(self, time=0.0, compile_time=0.0, exec_time=0.0, cpu_load=0): - self.time = time - self.compile_time = compile_time - self.exec_time = exec_time - self.cpu_load = cpu_load # TODO - - -class WasmVMBencher: - """Launches each VM on given directory on each provided test.""" - - def __init__(self, vm_dir="/", error_log="wasm_vm_errors.log"): - self.vm_dir = vm_dir - self.enabled_vm = [] - self.logger = logging.getLogger("wasm_bencher_logger") - self.error_log = error_log - - def log_error(self, error_msg): - self.logger.info(error_msg) - print(error_msg, file=open(self.error_log, 'a')) - - def run_tests(self, test_descriptors, vm_bin_path, vm_descriptors): - """Launches provided tests and returns their execution time. - - Parameters - ---------- - test_descriptors - Descriptors of test that should be used for benchmark Wasm VM. - vm_descriptors - Descriptors of Wasm VM that should be tested on provided tests. - - Returns - ------- - {vm : {test_name : [Records]}} - Collected test results. - - """ - # {vm : {test_name : [Records]}} - results = defaultdict(lambda: defaultdict(list)) - - test_export_function_name = 'main' - self.enabled_vm = list(vm_descriptors.keys()) - - for test_name, test_path in test_descriptors.items(): - self.logger.info(": launch {} test".format(test_name)) - for vm in self.enabled_vm: - if vm not in vm_descriptors: - continue - - # ------------------- TODO don't hardcode the below - bin_path = "./results/bin" - vm_binary_full_path = join(vm_descriptors[vm].vm_binary_path) - cmd = vm_binary_full_path + " " \ - + vm_descriptors[vm].vm_launch_cmd.format(wasm_file_path=test_path, - function_name=test_export_function_name) - - # determine repititions for a benchmark by checking the time it takes for one run - # if its very short, then do many repetitions - # if > 20 seconds or 30 seconds, then do three repetitions - try: - self.logger.info(": {}".format(cmd)) - result_record = self.run_engine(vm, cmd) - results[vm][test_name].append(result_record) - self.logger.info(": {} result collected: time={} compiletime={} exectime={}".format(vm, result_record.time, result_record.compile_time, result_record.exec_time)) - except (subprocess.TimeoutExpired, Exception) as e: - self.log_error(": {} ({}) got exception:\n{}".format(vm, cmd, e)) - self.log_error(": skipping engine {} for bench input {}".format(vm, test_name)) - continue - - # target 60 seconds total time per benchmark - repetitions = round(60 / result_record.time) - if repetitions < 3: - repetitions = 3 # minimum - if repetitions > 50: - repetitions = 50 # maximum - - for i in range(repetitions - 1): - # run (i+2) of repetitions. 2nd run of 1-based index - self.logger.info(" {} run {} of {}: {}".format(vm, i + 2, repetitions, cmd)) - result_record = self.run_engine(vm, cmd) - results[vm][test_name].append(result_record) - self.logger.info(": {} result collected: time={} compiletime={} exectime={}".format(vm, result_record.time, result_record.compile_time, result_record.exec_time)) - - - return results - - def run_engine(self, vm, cmd): - if vm == "lifePolymerase": - result_record = self.do_life_poly_test(cmd) - elif vm == "life": - result_record = self.do_life_test(cmd) - elif vm == "wavm": - result_record = self.do_wavm_test(cmd) - elif vm == "wasmer": - result_record = self.do_wasmer_test(cmd) - elif vm == "wasmtime": - result_record = self.do_wasmtime_test(cmd) - elif vm == "v8-liftoff" or vm == "v8-turbofan" or vm == "v8-interpreter": - result_record = self.do_v8_test(cmd) - elif vm == "asmble": - result_record = self.do_asmble_test(cmd) - elif vm == "wagon": - result_record = self.do_wagon_test(cmd) - elif vm == "wasmi": - result_record = self.do_wasmi_test(cmd) - elif vm == "wabt": - result_record = self.do_wabt_test(cmd) - elif vm == "vanilla-wabt": - result_record = self.do_vanilla_wabt_test(cmd) - elif vm == "wamr-interp": - result_record = self.do_wamr_interp_test(cmd) - elif vm == "wamr-jit": - result_record = self.do_wamr_jit_test(cmd) - elif vm == "wamr-aot": - result_record = self.do_wamr_aot_test(cmd) - elif vm == "wasm3": - result_record = self.do_wasm3_test(cmd) - elif vm == "fizzy": - result_record = self.do_fizzy_test(cmd) - elif vm == "ssvm": - result_record = self.do_ssvm_test(cmd) - else: - result_record = self.doElapsedTest(cmd) - - return result_record - - def do_wasmtime_test(self, vm_cmd): - """ - module compile time: 78.598076ms - exec time: 73.882µs - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "module compile time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info) - return Record(time=result.time, compile_time=result.compile_time, exec_time=result.exec_time) - - def do_wasmer_test(self, vm_cmd): - """02/26/2019 04:43:14 PM : /engines/wasmer-master/target/release/wasmer run /wasmfiles/sha1-42488-bits.wasm │·············· - compile time: 58.099186ms │·············· - run time: 54.399µs - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "run time: ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info) - return Record(time=result.time, compile_time=result.compile_time, exec_time=result.exec_time) - - def do_wasmer_v014_test(self, vm_cmd): - """02/15/2019 11:23:55 PM : /engines/wasmer/target/release/wasmer run /wasmfiles/ecpairing.wasm - compile time: 88.381ms - total run time (compile + execute): 172.762637ms - 02/15/2019 11:23:55 PM : wasmer result collected: time=0.18068552017211914 compiletime=0.0 - """ - # the other wasmer patch prints "run time: 172.762637ms" - #runtime_match = re.search("run time: ([\w\.]+)", runtime_line) - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "total run time \(compile \+ execute\): ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info, stderr_redir=False) - execution_time = result.exec_time - result.compile_time - return Record(time=result.time, compile_time=result.compile_time, exec_time=execution_time) - - def do_wavm_test(self, vm_cmd): - """02/16/2019 12:03:32 AM : /engines/wavm-build/bin/wavm-run /wasmfiles/example.wasm -f main - Object size: 600392 │·············· - Object size: 800 │·············· - Instantiation/compile time: 4210686us │·············· - Invoke/run time: 113563us - """ - # TODO: read wavm compiled object size - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation/compile time: ([\w\.]+)", - 'exec_regex': "Invoke/run time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_life_test(self, vm_cmd): - """03/10/2019 05:43:33 AM life run 2 of 12: /engines/life/life -entry main /wasmfiles/bn128_pairing-one_point.wasm - disasm time: 150.823183ms - compile time: 194.341499ms - parse/instantiation time: 348.134957ms - return value = 0, exec duration = 4.640145939s - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse/instantiation time: ([\w\.]+)", - 'exec_regex': "duration = ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_life_poly_test(self, vm_cmd): - """03/10/2019 12:22:31 AM : /engines/life/life -polymerase -entry main /wasmfiles/bn128_pairing-one_point.wasm - disasm time: 146.013493ms - compile time: 208.066141ms - parse/instantiation time: 357.112707ms - [Polymerase] Compilation started. - [Polymerase] Compilation finished successfully in 45.485905579s. - return value = 0, exec duration = 15.903258ms - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "Compilation finished successfully in ([\w\.]+s).", - 'exec_regex': "duration = ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_v8_test(self, vm_cmd): - """02/17/2019 07:14:04 PM : /engines/node/node --wasm-interpret-all /engines/node/node-timer.js /wasmfiles/ecpairing.wasm - args: [ '/wasmfiles/ecpairing.wasm' ] - ---- reading wasm file.. - ---- wasm file read. - instantiate: 67.677ms - ---- calling main... - run-main: 13406.809ms - ---- wasm returns: undefined - """ - time_parse_info = { - 'compile_line_num' : 3, - 'exec_line_num' : 5, - 'compile_regex': "instantiate: ([\w\.]+)", - 'exec_regex': "run-main: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wabt_test(self, vm_cmd): - """02/16/2019 09:56:43 PM : /engines/wabt/bin/wasm-interp /wasmfiles/ecpairing.wasm --run-all-exports - ec_pairing() => error: argument type mismatch - main() => - parse time: 45430us - exec time: 62390657us - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w]+)", - 'exec_regex': "exec time: ([\w]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_vanilla_wabt_test(self, vm_cmd): - """02/16/2019 09:56:43 PM : /engines/vanilla-wabt/bin/wasm-interp /wasmfiles/ecpairing.wasm --run-all-exports - ec_pairing() => error: argument type mismatch - main() => - parse time: 45430us - exec time: 62390657us - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w]+)", - 'exec_regex': "exec time: ([\w]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - - def do_wagon_test(self, vm_cmd): - """03/10/2019 12:14:46 AM : /engines/wagon/cmd/wasm-run/wasm-run /wasmfiles/bn128_pairing-one_point.wasm - parse time: 1.378798236s - () - exec time: 3.972499051s - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_asmble_test(self, vm_cmd): - """03/11/2019 01:24:20 PM asmble run 2 of 12: /engines/asmble/bin/asmble invoke -in /wasmfiles/bn128_add-cdetrio11.wasm main -defmaxmempages 20000 - compile time: 4585504732ns - exec time: 7276335ns - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wasmi_test(self, vm_cmd): - """02/24/2019 05:01:58 PM : /engines/wasmi/target/release/examples/invoke /wasmfiles/sha1-42488-bits.wasm main │·············· - module parse time: 3.499567ms │·············· - Result: None │·············· - exec time: 4.630356ms - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "module parse time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_interp_test(self, vm_cmd): - """ - Instantiation time: 0.001776 - - execution time: 0.003867 - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_jit_test(self, vm_cmd): - """ - Instantiation time: 0.001776 - - execution time: 0.003867 - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_aot_test(self, vm_cmd): - """ - engine output: - Runtime load time: 0.000205s - Create AoT compiler with: - target: x86_64 - target cpu: broadwell - cpu features: - opt level: 3 - size level: 3 - output format: AoT file - Compilation time: 0.002127s - Compile success, file wasm.aot was generated. - Instantiation time: 0.000860s - Exception: invalid input argument count. - execution time: 0.000001s - """ - time_parse_info = { - 'compile_line_num' : 8, - 'exec_line_num' : -1, - 'compile_regex': "Compilation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - - - def do_wasm3_test(self, vm_cmd): - """ - Result: - Instantiation time: 0.000046 - execution time: 0.001948 - """ - - time_parse_info = { - 'compile_line_num' : 1, - 'exec_line_num' : -1, - 'compile_regex' : "Instantiation time: ([\w\.]+)", - 'exec_regex' : "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_fizzy_test(self, vm_cmd): - """ - 2020-02-26 03:37:35 - Running /engines/fizzy/build/bin/fizzy-bench - Run on (4 X 3100 MHz CPU s) - CPU Caches: - L1 Data 32K (x2) - L1 Instruction 32K (x2) - L2 Unified 256K (x2) - L3 Unified 3072K (x1) - Load Average: 0.54, 0.65, 0.76 - ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. - -------------------------------------------------------------------------------------- - Benchmark Time CPU Iterations UserCounters... - -------------------------------------------------------------------------------------- - fizzy/parse/testcase 159223ns 158598ns 4375 rate=1.50602G/s size=238.852k - fizzy/instantiate/testcase 30759ns 30557ns 23588 - fizzy/execute/testcase/test 2274841ns 2263767ns 315 - """ - - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex' : "fizzy\/instantiate\/testcase\s+([0-9]+ *[a-z]+)", - 'exec_regex': "fizzy\/execute\/testcase\/test\s+([0-9]+ *[a-z]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_ssvm_test(self, vm_cmd): - """ - Instantiation time: 0.003753s - 2020-04-22 13:49:02,940 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:33] Start running... - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:36] Execution succeeded. - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:44] Done. - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:52] - ================= Statistics ================= - Total execution time: 4448us - Wasm instructions execution time: 4448us - Host functions execution time: 0us - Executed wasm instructions count: 137054 - Gas costs: 137054 - Instructions per second: 30812500 - """ - time_parse_info = { - 'compile_line_num': 0, - 'exec_line_num': -6, - 'compile_regex' : "Instantiation time: ([\w\.]+)", - 'exec_regex' : "Total execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def doElapsedTest(self, vm_cmd): - """Launches provided shell command string via subprocess.Popen and measure its execution time. - - Parameters - ---------- - vm_cmd : str - An exactly command that should be executed. - - Returns - ------- - time_type - An elapsed time of provided cmd execution. - - """ - start_time = time() - #Popen(vm_cmd, shell=True).wait(300) - cmd_args = shlex.split(vm_cmd) - vm_process = subprocess.run(cmd_args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=300) - end_time = time() - #stdoutlines = [str(line, 'utf8') for line in vm_process.stdout] - stdoutlines = [line for line in vm_process.stdout.decode('utf8').rstrip().split('\n')] - print(("\n").join(stdoutlines), end="\n") - return Record(end_time - start_time) - - def doCompilerTest(self, vm_cmd, time_parse_info, stderr_redir=True): - start_time = time() - cmd_args = shlex.split(vm_cmd) - print(cmd_args) - if stderr_redir: - vm_process = subprocess.run(cmd_args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=300) - else: - vm_process = subprocess.run(cmd_args, stdout=subprocess.PIPE, shell=True, timeout=300) - #vm_process.wait(300) - end_time = time() - total_time = end_time - start_time - #stdoutlines = [str(line, 'utf8') for line in vm_process.stdout] - stdoutlines = [line for line in vm_process.stdout.decode('utf8').rstrip().split('\n')] - print('stdout_lines: ', ("\n").join(stdoutlines), end="\n") - try: - compile_line = stdoutlines[time_parse_info['compile_line_num']] - print('compile_lines: ', compile_line) - compile_match = re.search(time_parse_info['compile_regex'], compile_line) - compile_match = compile_match[1].replace(' ', '') - print('compile_match: ', compile_match) - compile_time = durationpy.from_str(compile_match) - print('compile_time: ', compile_time) - - exec_line = stdoutlines[time_parse_info['exec_line_num']] - print('exec_line: ', exec_line) - exec_match = re.search(time_parse_info['exec_regex'], exec_line) - exec_match = exec_match[1].replace(' ', '') - print('exec_match: ', exec_match) - exec_time = durationpy.from_str(exec_match) - print('exec_time: ', exec_time) - except Exception as e: - error_msg = ["Error parsing engine output. exception: {}".format(e)] + \ - ["engine output:"] + stdoutlines - raise Exception("\n".join(error_msg)) - return Record(time=total_time, compile_time=compile_time.total_seconds(), exec_time=exec_time.total_seconds()) - diff --git a/wasm-engines/fluence_bencher/__init__.py b/wasm-engines/fluence_bencher/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/wasm-engines/fluence_bencher/settings.py b/wasm-engines/fluence_bencher/settings.py deleted file mode 100644 index d7a07f7..0000000 --- a/wasm-engines/fluence_bencher/settings.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Copyright 2018 Fluence Labs Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from fluence_bencher.VMDescriptor import VMDescriptor - -# export function name that should be called from each Wasm module -test_export_function_name = "main" - -""" - Attributes - ---------- - vm_relative_binary_path : str - A relative path to VM binary in its main folder. - vm_launch_cmd : str - An format string with command for launch this vm with provided test. - - VMDescriptor(vm_relative_binary_path="", vm_launch_cmd="") -""" - -vm_descriptors = { - "wagon" : VMDescriptor("wasm-run", "{wasm_file_path}"), - "wabt" : VMDescriptor("wabt-wasm-interp", "{wasm_file_path} --run-all-exports"), - "vanilla-wabt" : VMDescriptor("vanilla-wabt-wasm-interp", "{wasm_file_path} --run-all-exports"), - "v8-liftoff" : VMDescriptor("node", "--liftoff --no-wasm-tier-up /engines/node/node-timer.js {wasm_file_path}"), - "v8-turbofan" : VMDescriptor("node", "--no-liftoff /engines/node/node-timer.js {wasm_file_path}"), - "v8-interpreter" : VMDescriptor("node", "--wasm-interpret-all --liftoff --no-wasm-tier-up /engines/node/node-timer.js {wasm_file_path}"), - "wasmtime": VMDescriptor("wasmtime", "{wasm_file_path} --invoke=main"), - "wavm" : VMDescriptor("wavm-run", "{wasm_file_path} -f {function_name}"), - "life-polymerase" : VMDescriptor("life", "-polymerase -entry {function_name} {wasm_file_path}"), - "life" : VMDescriptor("life", "-entry {function_name} {wasm_file_path}"), - "wasmi" : VMDescriptor("invoke", "{wasm_file_path} {function_name}"), - "asmble" : VMDescriptor("asmble", "invoke -in {wasm_file_path} {function_name} -defmaxmempages 20000"), - "wamr-interp" : VMDescriptor("iwasm", "-f {function_name} {wasm_file_path}"), - "wamr-jit" : VMDescriptor("iwasm", "-f {function_name} {wasm_file_path}"), - "wamr-aot" : VMDescriptor("wamr_aot.sh", "{function_name} {wasm_file_path}"), - "wasm3" : VMDescriptor("wasm3", "--func {function_name} {wasm_file_path}"), - "fizzy" : VMDescriptor("fizzy.sh", "{function_name} {wasm_file_path}"), - "ssvm" : VMDescriptor("ssvm", "{wasm_file_path} {function_name}"), -} diff --git a/wasm-engines/run.sh b/wasm-engines/run.sh index cc5f2cc..e9f6e8c 100755 --- a/wasm-engines/run.sh +++ b/wasm-engines/run.sh @@ -36,7 +36,7 @@ docker cp $(docker run -d -t ewasm/vanilla-wabt:1 sleep 3s):/vanilla-wabt/build/ docker cp $(docker run -d -t ewasm/wabt:1 sleep 3s):/wabt/build/wasm-interp $WASM_ENGINE_BIN_DIR/wabt docker cp $(docker run -d -t ewasm/wagon:1 sleep 3s):/wagon/cmd/wasm-run/wasm-run $WASM_ENGINE_BIN_DIR/wagon -docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/product-mini/platforms/linux/build_interp/iwasm $WASM_ENGINE_BIN_DIR/wamr-interp +docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/product-mini/platforms/linux/build_interp/iwasm $WASM_ENGINE_BIN_DIR/iwasm docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/wamr-compiler/build/wamrc $WASM_ENGINE_BIN_DIR/wamrc docker cp $(docker run -d -t ewasm/wasm3:1 sleep 3s):/wasm3/build/wasm3 $WASM_ENGINE_BIN_DIR/wasm3 @@ -49,6 +49,13 @@ docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-compile docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-run $WASM_ENGINE_BIN_DIR/wavm-run docker run -v $RUST_BIN_DIR:/rust-bin -v $INPUT_VECTORS_DIR:/inputvectors -v $RESULTS_DIR:/results -v $(pwd)/rust-code:/rust-code -v $(pwd)/scripts:/scripts -it ewasm/bench-build-base:1 bash /scripts/fill_rust.sh + +ln -s $(which node) $WASM_ENGINE_BIN_DIR/node + +# copy scripts which invoke engines to wasm_bin_dir (fizzy and wamr-aot) +cp scripts/fizzy.sh $WASM_ENGINE_BIN_DIR/ +cp scripts/wamr-aot.sh $WASM_ENGINE_BIN_DIR/ + sudo chown -R 1000:1000 $RESULTS_DIR echo "TODO minify rust wasm" @@ -59,6 +66,8 @@ python3 scripts/bench_native.py --rustbindir=$RUST_BIN_DIR --csvresults=$RESULTS # run wasm benchmarks +python3 scripts/bench_wasm.py --wasmenginedir $WASM_ENGINE_BIN_DIR --csvfile $RESULTS_DIR/wasm_engines.csv --wasmdir $WASM_FILE_DIR + echo "TODO run rust native benchmarks" # do stuff with the output diff --git a/wasm-engines/scripts/bench_native.py b/wasm-engines/scripts/bench_native.py index 7124aaf..6c0b843 100644 --- a/wasm-engines/scripts/bench_native.py +++ b/wasm-engines/scripts/bench_native.py @@ -62,7 +62,6 @@ def main(): result_times = ",".join(list(map(lambda x: str(x), result_times))) results.append({"test_name": os.path.basename(binary).strip("_native"), "elapsed_times": result_times}) - import pdb; pdb.set_trace() write_output_to_csv(results, os.path.join(args['csvresults'], "native_benchmarks.csv")) if __name__ == "__main__": diff --git a/wasm-engines/bench_wasm.py b/wasm-engines/scripts/bench_wasm.py similarity index 92% rename from wasm-engines/bench_wasm.py rename to wasm-engines/scripts/bench_wasm.py index d9f6bc6..e0a412d 100755 --- a/wasm-engines/bench_wasm.py +++ b/wasm-engines/scripts/bench_wasm.py @@ -24,6 +24,7 @@ parser.add_argument('--wasmdir', help='full path of dir containing wasm files') parser.add_argument('--csvfile', help='name of csv result file') parser.add_argument('--engines', help='comma-separated list of engines to benchmark') +parser.add_argument('--wasmenginedir', help='directory containing executables for all wasm engines') args = vars(parser.parse_args()) @@ -77,6 +78,7 @@ def main(): wasm_dir = args['wasmdir'] csv_file_path = args['csvfile'] engines_to_run = args['engines'] + engine_bin_dir = args['wasmenginedir'] # "wagon,wabt,v8-liftoff,v8-turbofan,v8-interpreter,wasmtime,wavm,life-polymerase,life,wasmi,asmble" vms_to_run = {} if engines_to_run is not None: @@ -87,13 +89,15 @@ def main(): vms_to_run = vm_descriptors print("vms_to_run:", vms_to_run) - ## TODO: print version of each engine vm_bencher = WasmVMBencher() test_descriptors = getTestDescriptors(wasm_dir) - test_results = vm_bencher.run_tests(test_descriptors, vms_to_run) + # TODO no hardcode script path here + test_results = vm_bencher.run_tests(test_descriptors, engine_bin_dir, vms_to_run) print("test_results:") print(test_results) + + import pdb; pdb.set_trace() save_test_results(csv_file_path, test_results) diff --git a/wasm-engines/scripts/fizzy.sh b/wasm-engines/scripts/fizzy.sh index b4f0e77..2c3db8c 100755 --- a/wasm-engines/scripts/fizzy.sh +++ b/wasm-engines/scripts/fizzy.sh @@ -5,6 +5,8 @@ set -e # $1 = function name # $2 = wasm file +CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # Create input file echo test > testcase.inputs # 1. Test name echo $1 >> testcase.inputs # 2. Function name @@ -17,6 +19,6 @@ echo $'\n' >> testcase.inputs # 7. Expected memory # Copy wasm file cp $2 ./testcase.wasm -/engines/fizzy/fizzy-bench --benchmark_filter=fizzy/* --benchmark_color=false ./ | sed 's/\([0-9][0-9]*\)\s\([nm]s\)/\1\2/g' +$CWD/fizzy-bench --benchmark_filter=fizzy/* --benchmark_color=false ./ | sed 's/\([0-9][0-9]*\)\s\([nm]s\)/\1\2/g' rm -f testcase.inputs testcase.wasm diff --git a/wasm-engines/scripts/requirements.txt b/wasm-engines/scripts/requirements.txt index 627eeab..6b3ad57 100644 --- a/wasm-engines/scripts/requirements.txt +++ b/wasm-engines/scripts/requirements.txt @@ -1,3 +1,4 @@ +durationpy==0.5 Jinja2==2.11.2 MarkupSafe==1.1.1 numpy==1.19.2 diff --git a/wasm-engines/scripts/wamr_aot.sh b/wasm-engines/scripts/wamr_aot.sh index 922eb59..9440811 100755 --- a/wasm-engines/scripts/wamr_aot.sh +++ b/wasm-engines/scripts/wamr_aot.sh @@ -2,5 +2,7 @@ set -e -/engines/wamr/wamrc -o wasm.aot $2 -/engines/wamr/iwasm -f $1 wasm.aot +CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +$CWD/wamrc -o wasm.aot $2 +$CWD/iwasm -f $1 wasm.aot