From 5790f000dfb38ed4b294bf4776e96820efbb76ba Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 26 Oct 2020 15:02:04 -0700 Subject: [PATCH] bench wasm engines appears to work now --- .../fluence_bencher/.WasmVMBencher.py.swp | Bin 36864 -> 0 bytes wasm-engines/fluence_bencher/.settings.py.swp | Bin 12288 -> 0 bytes wasm-engines/fluence_bencher/VMDescriptor.py | 33 -- wasm-engines/fluence_bencher/WasmVMBencher.py | 534 ------------------ wasm-engines/fluence_bencher/__init__.py | 0 wasm-engines/fluence_bencher/settings.py | 51 -- wasm-engines/run.sh | 11 +- wasm-engines/scripts/bench_native.py | 1 - wasm-engines/{ => scripts}/bench_wasm.py | 8 +- wasm-engines/scripts/fizzy.sh | 4 +- wasm-engines/scripts/requirements.txt | 1 + wasm-engines/scripts/wamr_aot.sh | 6 +- 12 files changed, 24 insertions(+), 625 deletions(-) delete mode 100644 wasm-engines/fluence_bencher/.WasmVMBencher.py.swp delete mode 100644 wasm-engines/fluence_bencher/.settings.py.swp delete mode 100644 wasm-engines/fluence_bencher/VMDescriptor.py delete mode 100644 wasm-engines/fluence_bencher/WasmVMBencher.py delete mode 100644 wasm-engines/fluence_bencher/__init__.py delete mode 100644 wasm-engines/fluence_bencher/settings.py rename wasm-engines/{ => scripts}/bench_wasm.py (92%) diff --git a/wasm-engines/fluence_bencher/.WasmVMBencher.py.swp b/wasm-engines/fluence_bencher/.WasmVMBencher.py.swp deleted file mode 100644 index 274ae1e6698809754d8b35b57c7f3e4117106f32..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36864 zcmeI532E&9FXLebq zu-ns@^YF4uFO>9lJK0(vEx&Q9F$cyR@Ho)xE+>j-#Il)mO73i%c&7T}KRP;Q&zJ*a z4vaZ4=D?T(V-ActFy_FR17i;S0db()dSvV+)c0XtotM48r}X`Ol=pqV_t)-w{z&h8 z$@}|(zUQZU-xs~VKk9pan)luE{yx_$fM1SBc=vaEfB&iP`NO^Y=XigA()ZlEJ^nN1 zz?cJL4vaZ4=D?T(V-ActFy_FR17i-1IWXqHA0!7X7Wj+Q{D)#Mv;Obf|Gzmi7W+E* z8h8V^3?xAU{O)nF*l)o{!5T<{ADj`3eHPpSZU;Am*Mn<80~`d~!P($DkB!Ct3)~Jq z3O)ip4E_;Z4Xy$;um~;&3Y-Bx^_W=fHt;cU92^2u;1S@PkB-Ie0^b0)gO`G9!4B{Y z@G$VH(_^u>fwzLIz~$iS;B4>}92|cGZUk2Y2OI>afgj`Ccn5epcr|z_cmXJYJUADe z0`9~~aR+!aSOzwj1YgCuaWi-sut60l@N=9Z?*TpVG;j+}jQ4{B;0fS%oHFkRSAYc2 zZS;FcsTy*%?M|=O?aZ&%7w5Znr(14Vb$c;swOV$g8h4+T+jgbdu1>{H?KXQJhM%QSGQ^TV;c9+g10N-B8sgf7`ZA9a?tR={n6uXE-UR z+fl1_dtG&oHQU|r^}5wq-{S46-L>0wqFN*!dFWgeRZ^A;0$|kAx+1itB9SmsHq~|N zwsN|vYaO>cI(1Vh^cqOc?|Civ8WkyoZs|GQ=Jtse)hb7x4077d_dK;)FIVkOrR}u3 z&9;`DB)4i7bdQy3B)`3MOD;P#yWFz6$K0Ic)AoTX@Dh>(ktL^Lwbv;~t)^d6=cx() zEeyOdEbW}SqUkgcxM9WfC`HRGY1-JF_8PJ3Nqn!6JN8E;dG!Al)Eu=@bw)R^q5 zV^+IbX;x`MnsLdlH3^PVLoUR!>oj`yz(XW=wXPb?u5ucEyp9UG$Pg(>rMr>cu$F3e zwY*x7CfmA?xlKn`Qz8vPe+$%s(d#77;6D98G=}h0N}<0Pl96<}cD)my!X@n2zptP) zdd=x{`2@rdtS_U-V^J3et&-;CbR|Na%J?=V%euUyO&Q+_u_IqZ_x|?Pj}N@2qTB zqSoe`jFFmh(tW}}wp~qHlZI3cbiROwE8bE{UMuU{UMa&Q13gCwEtkFW zyUFGzrYE+CrJQqf=b0{eCrSPjK8wx9wynxBk!c!p#aWe6+F|;^NV6`t^wWquB&9=E zYF4MCyF(BqDhpA{yL8nmtsa%OWTXi`s%=_zM-z%PmGuA&m;KMtB$Z~pB?Ft6%KREl z=p~Rp`oR!($+NjsGQ~8>pz7WUviA~rRhUb`ImH!GMn@QMs2MF!CF3aMmffz{jc#X3 zbcM_xbZ3rNdhIs1YwLQD>GWDM_}kSfx8DT3DWTK5Iv2@Fh)27bNz{E+SGJ`4V7a^A za%CP^)aflDq!k;X^wJy@W#JPHAG?{7wz1lQS}s@I@TP!nv7K`xGM+M!Lll+L$41L(C@8=n%H?* z?L9o9F5G!|@8RvSqm7ynZQC#Ep&%Id))hue&8f&Nh`OxFl;2umde6Yzl=;b8o5^}0dOG7TYNTW5OT-d%h4AKsJ5B#E1SezT|Q z7Mf^DB$gR`rP<=!a)hx~H@5DjlUbYySoauJk}(5jRP2_X+DUJ(ITm5=Y7Iu2B$6?& z)lBB>)P?igbGy2X@lu973Z~-HkQ3fZ6I18MV!J3<+gUl*#Ry6l)kSVVz1n9jbs&MD z5ZNz6BliCd*sd2~=ZgJ*j%Qzf89V<=;1!?-_5iW{p9tQGz5hCp0Z#a%Q2Gbib$@+REzDJG4+ z#Wfdv>Q5BftU7DUVTQFw&#RaZ#U~aT6G`m}3=Qe%8@j~afAbTrpISA0-Bzz#npni= z$fTODHQ9<7GZO~{#Wf$6+QzQioK?Cj2s=UQFWR zu2~g3J}HHqoE%a*O^^55cCuqz?aDDNEI44=9UC{oa&z9TYTI71PfRX)Rh6cQ5;Kb| zPs0)qrqyMX+vICB?YxNTV9t`T-00OO7l~ChTi?sl6knaIc_Ug#^T`HH;wKrx=f3>H zhS}tu$Ro0heBZK#BB?iTBtL1;PX3KS_V?985_BSr8+^qiwpF8XB5H7`W8A+0}P5#rzS4F9B?G+D)iT%|{_|FSqa zm28Vy+lo&mI~af6_~b%kaw>GQZR?{iomNQGncVj`&nzSqDqo68HC#_s0`a$4=+Qm~ zpGU#dIqkWLhn3xC)Mm}1wAoZDd@ApU38a+*r-!Wn$#PxUYOi##x3xN*GoByZdvFi; zkJ+`_+>v&V`bm+{xpR>$>{_Q!8;+bIs@lx(@R8m4EjBL7(Ck@>q3BRz0TeGwsFz(y z@+%m!#J?lV5Wkey$P}6^* zzDHFWCpsnv&1?$6l(zp_YL30eTmSdt^S_9le+T$0a09TwUD)~`0ylyMAnX4BgT4PA z@FwsQumE;|A7S(V8~7kN0oK4#@M-M+H-J^}d~iMxUqB4V+Wi;62f(XA8_a<-!D-+# z_yg_)?*eh~GkgKx2LB1(4gMCq7|et7!Rg>Gd>dZ^uLrLK4)`8x@;8IO11|*?a5*?1 zoCSW#+WhU{jo?Ob7#sr61KYtrvsV8~unH~%&jm^FSnxxb`4RA!;3Z%exDfmaXtRcY z0A#>9;L+gw@clDD^WUAyN!z*Cq_5jtEn89(-VH4^B^94vSX)Rgo;NjNuBo~MsHhFC z4q3)ZM;`k`%@nCHg$Se}WQgchF~30KG4emlsOpttCNopWX7k4Cd9<9ld@-L-Tb*gwbt`MN-bXxr ztIgxKN}!GPp%UVD2vhK3P)u5u!-C&Rf;E$J3%7e$i;5`^n%r=42%5&z zzxT2{Nbe7_97P(oRu?14LL&2fDU3oUJDV!xXL@0qnvI^z=dvl|ne@^4l$yIBs3kdbh_ZFoY_*oP|9ToRgNJys}>?!vl468 zQdeskQ5xtLR;yd4;Mf(fJg!zrB#^L#RjVc3xCOVKo-*-hmWAeU8B*Rg79n?y)8~hw z3m-fpbWe)T{U$OZ&e9FdNk`DHhN&GEbh{QIPF5G&JRG<}wc!tR<}@T9dF&5tW{PZ> zV$_L>tUxP0*1O#a<5QMLmzf|{gDQmeOJ{SbLb8~et%u4<0#nU5fNG8Kjmq<+u zGs(itO#iDocfJ1Cwk?)FeRFW8qu6Vy_IN$rl>@}3VfbGNN zb(c1~^zu0?wG9X3w2DbvkwtuoM_iXBWS>@ND!Ij!`(7GDMR2@9Vbx+ym--PnqSc~N z<#awdo62VLMMDnd&H01P+B&Ugb?imr)+B~yiwfuR$y_ndq%A))Ys8OjN=%!#vdAx~ z{7Vn-(_vSUm1~itlFH_1lIe72CS~>-dClwoUb0N8m`oK4>0G+6QbY>KJl-Xi&J~g= zdhl%4NHs8B%w$U0^rq9rnzL*Z1F%fDXazh|oKJ^G$u${_4(Zw!TTdE=9`niC`U9v- z$ol`;*qxv7?0?I(L1X>)zm5HW6np<-Z2R-EtQ zw|^GA8~i);dM9pGB90L}q1@CC8!!P~)gpaOP*)4^A;>pu%_ z1g`>r4aELm0rOxEoCSV?jsJh(2jKJIz2KEV;`yHn9uNK-oBzw;ec&D7dhk3T@%>K$ zzoQ-Q0@5BI2b*4g=Td54MjdSs#e>euA&%u`ZJYHr&8+Di?39YuoGrV8XZCuSnA8pl~=A$FZG;S zRZpb#_k<21jrkSHq%x^QDwD_*R4QA_&XltG;2FJGfA$6z{rY|1k?^Kyi#!ZBHD@>d-yeC znl}G!*v`d5Dx3$qj38#vofo-r(0byUvNbIt05ic&wPRe+RwGg7C zC1zIBTWS+HyT+2lKob3!9My?ppg_UI9^Q4NbaFkgI?gbux(BiqgSUV*cry4rJZS>q%?{yD|K*FA8xQhOj3Wx7 zMk0YKQt$gT5@*D~nr(WQm24;0;WA--(G+^2KYmHODg7OMGBr0MpJaL$^IKQ`?qus> z&1%WUS3RF63?bNsNM4~u7Fs=$F5CtuM8wWJUHq`Tg|y3e+gM*ePY$I%os1tYrPrQ+qcJg1w^k# z911QsT-CS7<=;S9)4dTw;m=v!kO=X;9Fb_`zS3>rxf$>PoSg5*&pTMPDyc-loK73` zjWl;-zAQ6%&!xvEX?EJVVbr+PQ^j;9??tePJUj=(ywYSNG_gS0nN&VBVfX0vLab%r z#YbRJ)ECn(M!mIGnod7j!Dj zu==0Q#e@NJY4f|}nmrY}RJzDJ2*bZ~4#C;;fxGhX;ma;%1z?FehphEuSMxhGuBGn% zNmg2|Qa-a7x)faZWiFA)z~)kJwv@_jpUvgf?mZVCeU6%^AILsYa$!$k``OEOezv$Y zo1d*zE8LR|OQk(0*pAuVvD$R17by4E-mm~AiXW7wRASN6uSDEUi`Uhy=3#bB+m~7m z!kOCVNB}-9>(L`kOf%%(;z=&IsCHuvgt~Cd7%eQ>!Xg_Le2=SkqS>6zFy<3W@n)yW zW*4PTv%MP$msJFFxqY}Sr5?bY%vCTF3K9DKwclwbpAEf$eJ-U=hf)1LaVNRDpfBoB zFRv_Md5N^z3)51P3bvU0X`ypod_I+!T|7^nJ8vbf>o@V(SOqU z*-R!IyqzWnceWUORVtUA$t&^H&!v<3R3Vjl&U8n+*yl3YVzQXe9FO_r!l~pv`6(cO zq5nH@etyIL9uYUXdC7Vs>lO@$x?gqFQ3lmrvZ}$xcvE5AlVuKR#r(3Id+xa$hp>O| z{^#(?T?HjlNG8ne`S;FKA@Y|E`r&aotQ*u^L&BRoOVA<*U~sem=l3+4PKVPBnE9^} zd&s1W6FZo?cH5SclNpXhLWuUsX&*be%}CGmE1A!2=l?=p{x=-ftN~|(C1S)(uK=gr z_0G^{uh8pfjjX7R6zyEUjNVG5BNCt|9i3V-;7Pa29ARr zK!M+4vwsu35r{qi0`PQjHux2`dlOs&B+mZl)J@jX{{_4jya+rGNSz-CjLUCSV~!pT zs~#G{au0fEw^CAm0sYa!{qC*$Y!VWL-!_*@=kl}J*>K2Fu8(3k|l_;$LuaY#-oPZ(|~n;i_v;e0S+aF_)Q z3H>)>bnbz;ocm;`?HgqG>a7aXQazVfp&-Tlh8`w~kqO0gv`In|=HR*2uo)~5oy>sg z7tudqM*R~0QQB-tUd6xYe1{2b0))%sdSx7#c)i}pb8WxI$ z-hC2r+Z?3V zCc@Y({Z-VGvC`r#!g%`|mUR?lOqd+3Y>a5In1=)qlYvJhDcY~^lmPFEZ$r2##2nEs zTYWfyUi8*q!?3f!!{mE*JvzagDnX2d&eSBWD)FLOU3mIl3B>FZF;WU)JKPjQxd->$Dna2pk8I^D1%sfCc_+8 zY_57I+ysUAhb}pqRQf{R+bYH`!)z&^AGQSP<>d*0cg&kzbBT(nPGaNuNScksv}DQeIW|N zJJZnUg!`1&Tk%8I|9^@Byc;`N?En3qF}#Cy|6k$*crBO!r-4&}_ycYL%b*0F1ip{m z|G(fvpbmbA?f*$|E%-S0{;l9+;GN(d;3Yuf|Nj)64!(;0e=B$eI1UQn$w2n@zX~jZ zXM@LppWqj`6-b=_ac~fv4StQU;NOAl>6iWdyTKXY$M_FE4BiA*z$M_x;5YaXJ_BwA ze*q+ZzyZg=vq26#2K*R*!gs*!;9WrC2rdRs0cV0Q!nfl<&gd5&?gD>kmk4xpj2Ok3 zey|P?GLt+xbKQ#xi|XG|?eu$Wer666Mjd9d$zm~MjuS(RJ5x;JK2FV;fqy=BQ~U`(~L~d3>e>pWWAjruKZ-E2BFTIQzhBSP&I>eHz(|- z9Fg|mcKZ#9o4fr-q#o|b-jJ#!8JSuT9=HiP(u_=xV=uOlQGJ%hi1ZLqj#u7-pqpq! zimToR;SH<1T1?>S?KYQ}C*(+Hyo=ILxyz1@rRK8L=)3L8hK|=5u?W(sLdCUVLHuMV zK^1dJE5l7`ati-w3P-#$Thbd@ed*n(o0m?Kj94~JE!^80sc@)WB~`cw*&{8(2X>49 zgF)hWk>6s#AB2`SegeT;-8SxT8*KQCLcWuy-^S+$P`oi(oAi8@ zank!7{49!;EHWf`-mB<6kC664^`{SfQB&a{7F2)_UJW0hzsV%O6C-9Oe~5y->YB&_ zm%5}*=?Fw+&M3!f2P2AE2%Q_OCMw~`!UqI?;k9Cuo zY35W`2y&3X2n2Vi>>#};wKfae)p#MLriTdMAZKx+xtrT~!~Y2^*C!?7tb+xE)#%|J zX~W+qlDhi6%=_JvfnV+F9CunRSpxUAwjnFQ@5RtevJuhvVJ{IHQAGL|sW#{nTjfX& z@vE+x;0$o4FwV{^oWLr6o?8CzuC1N`_}n{CTLdrd2i?A_ul)xH#7`s>lB!!Da+8Kab)^!f9w&YDiB>T}=(-WsVvwojqx&=EM|lOj0u=@Jv%{;0R$TLe z;5B;rr3;l2zQHTt74Qmp1-t@Y0k42pz$@St`2Q+k^F8c3!rMCz@$mS*XV-oF=x<&D zuYgy;E8rFI3U~#)0$u^HfLFjP;1%!+Jc0^v{J{V1;oPH_5!~>%h=bz`@kR1F!mAfD)8ykjGX{(J;m53z%k&~KE^%2g~MV3a+|yRIOMhqLp*o zyGB>zzUo++sg~lYxE3remprk&a%B`$c#7tl=>?8!ek$$eo^m;xhpSv=Gl;^_G3|Ec zZhWZ@d1e<;rn-fm`}dH_aEBa}J8}V~bZ)wE!}tL`l*$U7SDkz)MmW(jL#r#e=JF*M6j5oRNd5hmQ6ghFU9HEW-H9rc)uW1k zcPu40FU+%KlnTww0zIb0w+GRK8X59p4ONQYwTHT|7AjCJk2yW9OhvGJRXP=m4k)oZ zWD1ch2?SuIbv2snpY~H@2_aS7A8&ac#J&aTy53Yz$?*X3qc7ojq4!#yG(yV~E9T=ZKn+E4rc-XoSdA0A zG?cbSoh(hcid+|!Gs*U(0a4kH6Tt%+iHNK)wpNhM*+7_}yKRIfu94g0$Y$6Ksq~WY zc9hnTTT`%?L@4kH;bhiqKr__g^eQGaEFx(4;ZmUxbH268^tR?UMyMDROPC#?vY#gX zh$Pk{UoBcpbqK;vBo+}N$^gOwy}^Y&7yuH7%Q50oqw*)riNtdN^EZbFFZ_S zDlIaWDMNjQz+qU>S=kZFh#K1v&XZ|Fkqq~2a?z2o>>x9$+sAG}-E1JVa@os;PNgvg zB@)QCo15>hU)ZGW#`*J&waw<~4O%}>C)d|bH8-2MNl%W(Q=7xT2u3LCuYT-fnC(HXTb zcnZ>h_mfw+4ii>kJjtM;jqSYq(js)-K~Sj z$!6+jn: launch {} test".format(test_name)) - for vm in self.enabled_vm: - if vm not in vm_descriptors: - continue - - # ------------------- TODO don't hardcode the below - bin_path = "./results/bin" - vm_binary_full_path = join(vm_descriptors[vm].vm_binary_path) - cmd = vm_binary_full_path + " " \ - + vm_descriptors[vm].vm_launch_cmd.format(wasm_file_path=test_path, - function_name=test_export_function_name) - - # determine repititions for a benchmark by checking the time it takes for one run - # if its very short, then do many repetitions - # if > 20 seconds or 30 seconds, then do three repetitions - try: - self.logger.info(": {}".format(cmd)) - result_record = self.run_engine(vm, cmd) - results[vm][test_name].append(result_record) - self.logger.info(": {} result collected: time={} compiletime={} exectime={}".format(vm, result_record.time, result_record.compile_time, result_record.exec_time)) - except (subprocess.TimeoutExpired, Exception) as e: - self.log_error(": {} ({}) got exception:\n{}".format(vm, cmd, e)) - self.log_error(": skipping engine {} for bench input {}".format(vm, test_name)) - continue - - # target 60 seconds total time per benchmark - repetitions = round(60 / result_record.time) - if repetitions < 3: - repetitions = 3 # minimum - if repetitions > 50: - repetitions = 50 # maximum - - for i in range(repetitions - 1): - # run (i+2) of repetitions. 2nd run of 1-based index - self.logger.info(" {} run {} of {}: {}".format(vm, i + 2, repetitions, cmd)) - result_record = self.run_engine(vm, cmd) - results[vm][test_name].append(result_record) - self.logger.info(": {} result collected: time={} compiletime={} exectime={}".format(vm, result_record.time, result_record.compile_time, result_record.exec_time)) - - - return results - - def run_engine(self, vm, cmd): - if vm == "lifePolymerase": - result_record = self.do_life_poly_test(cmd) - elif vm == "life": - result_record = self.do_life_test(cmd) - elif vm == "wavm": - result_record = self.do_wavm_test(cmd) - elif vm == "wasmer": - result_record = self.do_wasmer_test(cmd) - elif vm == "wasmtime": - result_record = self.do_wasmtime_test(cmd) - elif vm == "v8-liftoff" or vm == "v8-turbofan" or vm == "v8-interpreter": - result_record = self.do_v8_test(cmd) - elif vm == "asmble": - result_record = self.do_asmble_test(cmd) - elif vm == "wagon": - result_record = self.do_wagon_test(cmd) - elif vm == "wasmi": - result_record = self.do_wasmi_test(cmd) - elif vm == "wabt": - result_record = self.do_wabt_test(cmd) - elif vm == "vanilla-wabt": - result_record = self.do_vanilla_wabt_test(cmd) - elif vm == "wamr-interp": - result_record = self.do_wamr_interp_test(cmd) - elif vm == "wamr-jit": - result_record = self.do_wamr_jit_test(cmd) - elif vm == "wamr-aot": - result_record = self.do_wamr_aot_test(cmd) - elif vm == "wasm3": - result_record = self.do_wasm3_test(cmd) - elif vm == "fizzy": - result_record = self.do_fizzy_test(cmd) - elif vm == "ssvm": - result_record = self.do_ssvm_test(cmd) - else: - result_record = self.doElapsedTest(cmd) - - return result_record - - def do_wasmtime_test(self, vm_cmd): - """ - module compile time: 78.598076ms - exec time: 73.882µs - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "module compile time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info) - return Record(time=result.time, compile_time=result.compile_time, exec_time=result.exec_time) - - def do_wasmer_test(self, vm_cmd): - """02/26/2019 04:43:14 PM : /engines/wasmer-master/target/release/wasmer run /wasmfiles/sha1-42488-bits.wasm │·············· - compile time: 58.099186ms │·············· - run time: 54.399µs - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "run time: ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info) - return Record(time=result.time, compile_time=result.compile_time, exec_time=result.exec_time) - - def do_wasmer_v014_test(self, vm_cmd): - """02/15/2019 11:23:55 PM : /engines/wasmer/target/release/wasmer run /wasmfiles/ecpairing.wasm - compile time: 88.381ms - total run time (compile + execute): 172.762637ms - 02/15/2019 11:23:55 PM : wasmer result collected: time=0.18068552017211914 compiletime=0.0 - """ - # the other wasmer patch prints "run time: 172.762637ms" - #runtime_match = re.search("run time: ([\w\.]+)", runtime_line) - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "total run time \(compile \+ execute\): ([\w\.]+)" - } - result = self.doCompilerTest(vm_cmd, time_parse_info, stderr_redir=False) - execution_time = result.exec_time - result.compile_time - return Record(time=result.time, compile_time=result.compile_time, exec_time=execution_time) - - def do_wavm_test(self, vm_cmd): - """02/16/2019 12:03:32 AM : /engines/wavm-build/bin/wavm-run /wasmfiles/example.wasm -f main - Object size: 600392 │·············· - Object size: 800 │·············· - Instantiation/compile time: 4210686us │·············· - Invoke/run time: 113563us - """ - # TODO: read wavm compiled object size - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation/compile time: ([\w\.]+)", - 'exec_regex': "Invoke/run time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_life_test(self, vm_cmd): - """03/10/2019 05:43:33 AM life run 2 of 12: /engines/life/life -entry main /wasmfiles/bn128_pairing-one_point.wasm - disasm time: 150.823183ms - compile time: 194.341499ms - parse/instantiation time: 348.134957ms - return value = 0, exec duration = 4.640145939s - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse/instantiation time: ([\w\.]+)", - 'exec_regex': "duration = ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_life_poly_test(self, vm_cmd): - """03/10/2019 12:22:31 AM : /engines/life/life -polymerase -entry main /wasmfiles/bn128_pairing-one_point.wasm - disasm time: 146.013493ms - compile time: 208.066141ms - parse/instantiation time: 357.112707ms - [Polymerase] Compilation started. - [Polymerase] Compilation finished successfully in 45.485905579s. - return value = 0, exec duration = 15.903258ms - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "Compilation finished successfully in ([\w\.]+s).", - 'exec_regex': "duration = ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_v8_test(self, vm_cmd): - """02/17/2019 07:14:04 PM : /engines/node/node --wasm-interpret-all /engines/node/node-timer.js /wasmfiles/ecpairing.wasm - args: [ '/wasmfiles/ecpairing.wasm' ] - ---- reading wasm file.. - ---- wasm file read. - instantiate: 67.677ms - ---- calling main... - run-main: 13406.809ms - ---- wasm returns: undefined - """ - time_parse_info = { - 'compile_line_num' : 3, - 'exec_line_num' : 5, - 'compile_regex': "instantiate: ([\w\.]+)", - 'exec_regex': "run-main: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wabt_test(self, vm_cmd): - """02/16/2019 09:56:43 PM : /engines/wabt/bin/wasm-interp /wasmfiles/ecpairing.wasm --run-all-exports - ec_pairing() => error: argument type mismatch - main() => - parse time: 45430us - exec time: 62390657us - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w]+)", - 'exec_regex': "exec time: ([\w]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_vanilla_wabt_test(self, vm_cmd): - """02/16/2019 09:56:43 PM : /engines/vanilla-wabt/bin/wasm-interp /wasmfiles/ecpairing.wasm --run-all-exports - ec_pairing() => error: argument type mismatch - main() => - parse time: 45430us - exec time: 62390657us - """ - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w]+)", - 'exec_regex': "exec time: ([\w]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - - def do_wagon_test(self, vm_cmd): - """03/10/2019 12:14:46 AM : /engines/wagon/cmd/wasm-run/wasm-run /wasmfiles/bn128_pairing-one_point.wasm - parse time: 1.378798236s - () - exec time: 3.972499051s - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "parse time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_asmble_test(self, vm_cmd): - """03/11/2019 01:24:20 PM asmble run 2 of 12: /engines/asmble/bin/asmble invoke -in /wasmfiles/bn128_add-cdetrio11.wasm main -defmaxmempages 20000 - compile time: 4585504732ns - exec time: 7276335ns - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : 1, - 'compile_regex': "compile time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wasmi_test(self, vm_cmd): - """02/24/2019 05:01:58 PM : /engines/wasmi/target/release/examples/invoke /wasmfiles/sha1-42488-bits.wasm main │·············· - module parse time: 3.499567ms │·············· - Result: None │·············· - exec time: 4.630356ms - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "module parse time: ([\w\.]+)", - 'exec_regex': "exec time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_interp_test(self, vm_cmd): - """ - Instantiation time: 0.001776 - - execution time: 0.003867 - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_jit_test(self, vm_cmd): - """ - Instantiation time: 0.001776 - - execution time: 0.003867 - """ - time_parse_info = { - 'compile_line_num' : 0, - 'exec_line_num' : -1, - 'compile_regex': "Instantiation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_wamr_aot_test(self, vm_cmd): - """ - engine output: - Runtime load time: 0.000205s - Create AoT compiler with: - target: x86_64 - target cpu: broadwell - cpu features: - opt level: 3 - size level: 3 - output format: AoT file - Compilation time: 0.002127s - Compile success, file wasm.aot was generated. - Instantiation time: 0.000860s - Exception: invalid input argument count. - execution time: 0.000001s - """ - time_parse_info = { - 'compile_line_num' : 8, - 'exec_line_num' : -1, - 'compile_regex': "Compilation time: ([\w\.]+)", - 'exec_regex': "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - - - def do_wasm3_test(self, vm_cmd): - """ - Result: - Instantiation time: 0.000046 - execution time: 0.001948 - """ - - time_parse_info = { - 'compile_line_num' : 1, - 'exec_line_num' : -1, - 'compile_regex' : "Instantiation time: ([\w\.]+)", - 'exec_regex' : "execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_fizzy_test(self, vm_cmd): - """ - 2020-02-26 03:37:35 - Running /engines/fizzy/build/bin/fizzy-bench - Run on (4 X 3100 MHz CPU s) - CPU Caches: - L1 Data 32K (x2) - L1 Instruction 32K (x2) - L2 Unified 256K (x2) - L3 Unified 3072K (x1) - Load Average: 0.54, 0.65, 0.76 - ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead. - -------------------------------------------------------------------------------------- - Benchmark Time CPU Iterations UserCounters... - -------------------------------------------------------------------------------------- - fizzy/parse/testcase 159223ns 158598ns 4375 rate=1.50602G/s size=238.852k - fizzy/instantiate/testcase 30759ns 30557ns 23588 - fizzy/execute/testcase/test 2274841ns 2263767ns 315 - """ - - time_parse_info = { - 'compile_line_num' : -2, - 'exec_line_num' : -1, - 'compile_regex' : "fizzy\/instantiate\/testcase\s+([0-9]+ *[a-z]+)", - 'exec_regex': "fizzy\/execute\/testcase\/test\s+([0-9]+ *[a-z]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def do_ssvm_test(self, vm_cmd): - """ - Instantiation time: 0.003753s - 2020-04-22 13:49:02,940 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:33] Start running... - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:36] Execution succeeded. - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:44] Done. - 2020-04-22 13:49:02,945 DEBUG [default] [user@fe598b959cdc] [SSVM::Expect SSVM::Interpreter::Interpreter::runFunction(SSVM::Runtime::StoreManager&, const SSVM::Run - time::Instance::FunctionInstance&)] [/engines/SSVM/lib/interpreter/engine/engine.cpp:52] - ================= Statistics ================= - Total execution time: 4448us - Wasm instructions execution time: 4448us - Host functions execution time: 0us - Executed wasm instructions count: 137054 - Gas costs: 137054 - Instructions per second: 30812500 - """ - time_parse_info = { - 'compile_line_num': 0, - 'exec_line_num': -6, - 'compile_regex' : "Instantiation time: ([\w\.]+)", - 'exec_regex' : "Total execution time: ([\w\.]+)" - } - return self.doCompilerTest(vm_cmd, time_parse_info) - - def doElapsedTest(self, vm_cmd): - """Launches provided shell command string via subprocess.Popen and measure its execution time. - - Parameters - ---------- - vm_cmd : str - An exactly command that should be executed. - - Returns - ------- - time_type - An elapsed time of provided cmd execution. - - """ - start_time = time() - #Popen(vm_cmd, shell=True).wait(300) - cmd_args = shlex.split(vm_cmd) - vm_process = subprocess.run(cmd_args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=300) - end_time = time() - #stdoutlines = [str(line, 'utf8') for line in vm_process.stdout] - stdoutlines = [line for line in vm_process.stdout.decode('utf8').rstrip().split('\n')] - print(("\n").join(stdoutlines), end="\n") - return Record(end_time - start_time) - - def doCompilerTest(self, vm_cmd, time_parse_info, stderr_redir=True): - start_time = time() - cmd_args = shlex.split(vm_cmd) - print(cmd_args) - if stderr_redir: - vm_process = subprocess.run(cmd_args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, timeout=300) - else: - vm_process = subprocess.run(cmd_args, stdout=subprocess.PIPE, shell=True, timeout=300) - #vm_process.wait(300) - end_time = time() - total_time = end_time - start_time - #stdoutlines = [str(line, 'utf8') for line in vm_process.stdout] - stdoutlines = [line for line in vm_process.stdout.decode('utf8').rstrip().split('\n')] - print('stdout_lines: ', ("\n").join(stdoutlines), end="\n") - try: - compile_line = stdoutlines[time_parse_info['compile_line_num']] - print('compile_lines: ', compile_line) - compile_match = re.search(time_parse_info['compile_regex'], compile_line) - compile_match = compile_match[1].replace(' ', '') - print('compile_match: ', compile_match) - compile_time = durationpy.from_str(compile_match) - print('compile_time: ', compile_time) - - exec_line = stdoutlines[time_parse_info['exec_line_num']] - print('exec_line: ', exec_line) - exec_match = re.search(time_parse_info['exec_regex'], exec_line) - exec_match = exec_match[1].replace(' ', '') - print('exec_match: ', exec_match) - exec_time = durationpy.from_str(exec_match) - print('exec_time: ', exec_time) - except Exception as e: - error_msg = ["Error parsing engine output. exception: {}".format(e)] + \ - ["engine output:"] + stdoutlines - raise Exception("\n".join(error_msg)) - return Record(time=total_time, compile_time=compile_time.total_seconds(), exec_time=exec_time.total_seconds()) - diff --git a/wasm-engines/fluence_bencher/__init__.py b/wasm-engines/fluence_bencher/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/wasm-engines/fluence_bencher/settings.py b/wasm-engines/fluence_bencher/settings.py deleted file mode 100644 index d7a07f7..0000000 --- a/wasm-engines/fluence_bencher/settings.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Copyright 2018 Fluence Labs Limited - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from fluence_bencher.VMDescriptor import VMDescriptor - -# export function name that should be called from each Wasm module -test_export_function_name = "main" - -""" - Attributes - ---------- - vm_relative_binary_path : str - A relative path to VM binary in its main folder. - vm_launch_cmd : str - An format string with command for launch this vm with provided test. - - VMDescriptor(vm_relative_binary_path="", vm_launch_cmd="") -""" - -vm_descriptors = { - "wagon" : VMDescriptor("wasm-run", "{wasm_file_path}"), - "wabt" : VMDescriptor("wabt-wasm-interp", "{wasm_file_path} --run-all-exports"), - "vanilla-wabt" : VMDescriptor("vanilla-wabt-wasm-interp", "{wasm_file_path} --run-all-exports"), - "v8-liftoff" : VMDescriptor("node", "--liftoff --no-wasm-tier-up /engines/node/node-timer.js {wasm_file_path}"), - "v8-turbofan" : VMDescriptor("node", "--no-liftoff /engines/node/node-timer.js {wasm_file_path}"), - "v8-interpreter" : VMDescriptor("node", "--wasm-interpret-all --liftoff --no-wasm-tier-up /engines/node/node-timer.js {wasm_file_path}"), - "wasmtime": VMDescriptor("wasmtime", "{wasm_file_path} --invoke=main"), - "wavm" : VMDescriptor("wavm-run", "{wasm_file_path} -f {function_name}"), - "life-polymerase" : VMDescriptor("life", "-polymerase -entry {function_name} {wasm_file_path}"), - "life" : VMDescriptor("life", "-entry {function_name} {wasm_file_path}"), - "wasmi" : VMDescriptor("invoke", "{wasm_file_path} {function_name}"), - "asmble" : VMDescriptor("asmble", "invoke -in {wasm_file_path} {function_name} -defmaxmempages 20000"), - "wamr-interp" : VMDescriptor("iwasm", "-f {function_name} {wasm_file_path}"), - "wamr-jit" : VMDescriptor("iwasm", "-f {function_name} {wasm_file_path}"), - "wamr-aot" : VMDescriptor("wamr_aot.sh", "{function_name} {wasm_file_path}"), - "wasm3" : VMDescriptor("wasm3", "--func {function_name} {wasm_file_path}"), - "fizzy" : VMDescriptor("fizzy.sh", "{function_name} {wasm_file_path}"), - "ssvm" : VMDescriptor("ssvm", "{wasm_file_path} {function_name}"), -} diff --git a/wasm-engines/run.sh b/wasm-engines/run.sh index cc5f2cc..e9f6e8c 100755 --- a/wasm-engines/run.sh +++ b/wasm-engines/run.sh @@ -36,7 +36,7 @@ docker cp $(docker run -d -t ewasm/vanilla-wabt:1 sleep 3s):/vanilla-wabt/build/ docker cp $(docker run -d -t ewasm/wabt:1 sleep 3s):/wabt/build/wasm-interp $WASM_ENGINE_BIN_DIR/wabt docker cp $(docker run -d -t ewasm/wagon:1 sleep 3s):/wagon/cmd/wasm-run/wasm-run $WASM_ENGINE_BIN_DIR/wagon -docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/product-mini/platforms/linux/build_interp/iwasm $WASM_ENGINE_BIN_DIR/wamr-interp +docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/product-mini/platforms/linux/build_interp/iwasm $WASM_ENGINE_BIN_DIR/iwasm docker cp $(docker run -d -t ewasm/wamr:1 sleep 3s):/wasm-micro-runtime/wamr-compiler/build/wamrc $WASM_ENGINE_BIN_DIR/wamrc docker cp $(docker run -d -t ewasm/wasm3:1 sleep 3s):/wasm3/build/wasm3 $WASM_ENGINE_BIN_DIR/wasm3 @@ -49,6 +49,13 @@ docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-compile docker cp $(docker run -d -t ewasm/wavm:1 sleep 3s):/wavm-build/bin/wavm-run $WASM_ENGINE_BIN_DIR/wavm-run docker run -v $RUST_BIN_DIR:/rust-bin -v $INPUT_VECTORS_DIR:/inputvectors -v $RESULTS_DIR:/results -v $(pwd)/rust-code:/rust-code -v $(pwd)/scripts:/scripts -it ewasm/bench-build-base:1 bash /scripts/fill_rust.sh + +ln -s $(which node) $WASM_ENGINE_BIN_DIR/node + +# copy scripts which invoke engines to wasm_bin_dir (fizzy and wamr-aot) +cp scripts/fizzy.sh $WASM_ENGINE_BIN_DIR/ +cp scripts/wamr-aot.sh $WASM_ENGINE_BIN_DIR/ + sudo chown -R 1000:1000 $RESULTS_DIR echo "TODO minify rust wasm" @@ -59,6 +66,8 @@ python3 scripts/bench_native.py --rustbindir=$RUST_BIN_DIR --csvresults=$RESULTS # run wasm benchmarks +python3 scripts/bench_wasm.py --wasmenginedir $WASM_ENGINE_BIN_DIR --csvfile $RESULTS_DIR/wasm_engines.csv --wasmdir $WASM_FILE_DIR + echo "TODO run rust native benchmarks" # do stuff with the output diff --git a/wasm-engines/scripts/bench_native.py b/wasm-engines/scripts/bench_native.py index 7124aaf..6c0b843 100644 --- a/wasm-engines/scripts/bench_native.py +++ b/wasm-engines/scripts/bench_native.py @@ -62,7 +62,6 @@ def main(): result_times = ",".join(list(map(lambda x: str(x), result_times))) results.append({"test_name": os.path.basename(binary).strip("_native"), "elapsed_times": result_times}) - import pdb; pdb.set_trace() write_output_to_csv(results, os.path.join(args['csvresults'], "native_benchmarks.csv")) if __name__ == "__main__": diff --git a/wasm-engines/bench_wasm.py b/wasm-engines/scripts/bench_wasm.py similarity index 92% rename from wasm-engines/bench_wasm.py rename to wasm-engines/scripts/bench_wasm.py index d9f6bc6..e0a412d 100755 --- a/wasm-engines/bench_wasm.py +++ b/wasm-engines/scripts/bench_wasm.py @@ -24,6 +24,7 @@ parser.add_argument('--wasmdir', help='full path of dir containing wasm files') parser.add_argument('--csvfile', help='name of csv result file') parser.add_argument('--engines', help='comma-separated list of engines to benchmark') +parser.add_argument('--wasmenginedir', help='directory containing executables for all wasm engines') args = vars(parser.parse_args()) @@ -77,6 +78,7 @@ def main(): wasm_dir = args['wasmdir'] csv_file_path = args['csvfile'] engines_to_run = args['engines'] + engine_bin_dir = args['wasmenginedir'] # "wagon,wabt,v8-liftoff,v8-turbofan,v8-interpreter,wasmtime,wavm,life-polymerase,life,wasmi,asmble" vms_to_run = {} if engines_to_run is not None: @@ -87,13 +89,15 @@ def main(): vms_to_run = vm_descriptors print("vms_to_run:", vms_to_run) - ## TODO: print version of each engine vm_bencher = WasmVMBencher() test_descriptors = getTestDescriptors(wasm_dir) - test_results = vm_bencher.run_tests(test_descriptors, vms_to_run) + # TODO no hardcode script path here + test_results = vm_bencher.run_tests(test_descriptors, engine_bin_dir, vms_to_run) print("test_results:") print(test_results) + + import pdb; pdb.set_trace() save_test_results(csv_file_path, test_results) diff --git a/wasm-engines/scripts/fizzy.sh b/wasm-engines/scripts/fizzy.sh index b4f0e77..2c3db8c 100755 --- a/wasm-engines/scripts/fizzy.sh +++ b/wasm-engines/scripts/fizzy.sh @@ -5,6 +5,8 @@ set -e # $1 = function name # $2 = wasm file +CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # Create input file echo test > testcase.inputs # 1. Test name echo $1 >> testcase.inputs # 2. Function name @@ -17,6 +19,6 @@ echo $'\n' >> testcase.inputs # 7. Expected memory # Copy wasm file cp $2 ./testcase.wasm -/engines/fizzy/fizzy-bench --benchmark_filter=fizzy/* --benchmark_color=false ./ | sed 's/\([0-9][0-9]*\)\s\([nm]s\)/\1\2/g' +$CWD/fizzy-bench --benchmark_filter=fizzy/* --benchmark_color=false ./ | sed 's/\([0-9][0-9]*\)\s\([nm]s\)/\1\2/g' rm -f testcase.inputs testcase.wasm diff --git a/wasm-engines/scripts/requirements.txt b/wasm-engines/scripts/requirements.txt index 627eeab..6b3ad57 100644 --- a/wasm-engines/scripts/requirements.txt +++ b/wasm-engines/scripts/requirements.txt @@ -1,3 +1,4 @@ +durationpy==0.5 Jinja2==2.11.2 MarkupSafe==1.1.1 numpy==1.19.2 diff --git a/wasm-engines/scripts/wamr_aot.sh b/wasm-engines/scripts/wamr_aot.sh index 922eb59..9440811 100755 --- a/wasm-engines/scripts/wamr_aot.sh +++ b/wasm-engines/scripts/wamr_aot.sh @@ -2,5 +2,7 @@ set -e -/engines/wamr/wamrc -o wasm.aot $2 -/engines/wamr/iwasm -f $1 wasm.aot +CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +$CWD/wamrc -o wasm.aot $2 +$CWD/iwasm -f $1 wasm.aot