diff --git a/.gitmodules b/.gitmodules index 0b46282acfc..329ceb24210 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,3 +30,6 @@ [submodule "libraries/wabt"] path = libraries/wabt url = https://github.com/EOSIO/wabt +[submodule "libraries/eos-vm"] + path = libraries/eos-vm + url = https://github.com/eosio/eos-vm diff --git a/CMakeLists.txt b/CMakeLists.txt index 55259210105..c6b16e4958c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 3) set(VERSION_MINOR 0) -set(VERSION_PATCH 6) +set(VERSION_PATCH 7) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") @@ -90,6 +90,26 @@ endif() # * definition EOSIO__RUNTIME_ENABLED defined in public libchain interface # * ctest entries with --runtime list(APPEND EOSIO_WASM_RUNTIMES wabt) #always enable wabt; it works everywhere and parts of eosio still assume it's always available +if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32) + list(APPEND EOSIO_WASM_RUNTIMES wavm) + # WAVM requires LLVM, but move the check up here to a central location so that the EosioTester.cmakes + # can be created with the exact version found + find_package(LLVM REQUIRED CONFIG) + if(LLVM_VERSION_MAJOR VERSION_LESS 7 OR LLVM_VERSION_MAJOR VERSION_GREATER 9) + message(FATAL_ERROR "EOSIO requires an LLVM version 7.0 to 9.0") + endif() + + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux" AND "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64") + list(APPEND EOSIO_WASM_RUNTIMES eos-vm-oc) + endif() +endif() + +if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT WIN32) + list(APPEND EOSIO_WASM_RUNTIMES eos-vm) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) + list(APPEND EOSIO_WASM_RUNTIMES eos-vm-jit) + endif() +endif() if(UNIX) if(APPLE) @@ -266,6 +286,8 @@ configure_file(${CMAKE_SOURCE_DIR}/libraries/fc/secp256k1/upstream/COPYING ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.secp256k1 COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/libraries/fc/src/network/LICENSE.go ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.go COPYONLY) +configure_file(${CMAKE_SOURCE_DIR}/libraries/eos-vm/LICENSE + ${CMAKE_BINARY_DIR}/licenses/eosio/LICENSE.eos-vm COPYONLY) install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/) install(FILES libraries/wabt/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wabt) @@ -273,6 +295,7 @@ install(FILES libraries/softfloat/COPYING.txt DESTINATION ${CMAKE_INSTALL_FULL_D install(FILES libraries/wasm-jit/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wavm) install(FILES libraries/fc/secp256k1/upstream/COPYING DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.secp256k1) install(FILES libraries/fc/src/network/LICENSE.go DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ ) +install(FILES libraries/eos-vm/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.eos-vm COMPONENT base) include(package) include(doxygen) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index e04324406d8..32c238e30f2 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -1,4 +1,4 @@ -FROM boscore/builder:v2.0.4 as builder +FROM boscore/builder:v2.0.6 as builder ARG branch=master ARG symbol=EOS @@ -8,7 +8,7 @@ RUN git clone -b $branch https://github.com/boscore/bos.git --recursive \ && cd bos && echo "$branch:$(git rev-parse HEAD)" > /etc/boscore-version \ && cmake -H. -B"/tmp/build" -GNinja -DCMAKE_BUILD_TYPE=Release -DWASM_ROOT=/opt/wasm -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_C_COMPILER=clang -DCMAKE_INSTALL_PREFIX=/tmp/build -DBUILD_MONGO_DB_PLUGIN=true -DCORE_SYMBOL_NAME=$symbol \ - -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" -DCMAKE_CXX_STANDARD_LIBRARIES="-lpthread" \ + -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" -DCMAKE_CXX_STANDARD_LIBRARIES="-lpthread" -DENABLE_TOOLS=OFF \ && cmake --build /tmp/build --target install FROM ubuntu:18.04 diff --git a/Docker/README.md b/Docker/README.md index a193fc44a19..a775ee257ec 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd bos/Docker docker build . -t boscore/bos -s BOS ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v3.0.6 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v3.0.7 tag, you could do the following: ```bash -docker build -t boscore/bos:v3.0.6 --build-arg branch=v3.0.6 . +docker build -t boscore/bos:v3.0.7 --build-arg branch=v3.0.7 . ``` diff --git a/Docker/builder/Dockerfile b/Docker/builder/Dockerfile index efe9290b851..622f1c158a3 100644 --- a/Docker/builder/Dockerfile +++ b/Docker/builder/Dockerfile @@ -1,3 +1,4 @@ + FROM ubuntu:18.04 LABEL author="xiaobo " maintainer="Xiaobo Huang-Ming Huang Winlin " version="0.1.2" \ @@ -8,16 +9,19 @@ RUN echo 'APT::Install-Recommends 0;' >> /etc/apt/apt.conf.d/01norecommends \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y sudo wget curl net-tools ca-certificates unzip gnupg -RUN echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" >> /etc/apt/sources.list.d/llvm.list \ +RUN echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list.d/llvm.list \ && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y git-core automake autoconf libtool build-essential pkg-config libtool \ - mpi-default-dev libicu-dev python-dev python3-dev libbz2-dev zlib1g-dev libssl-dev libgmp-dev \ - clang-7 lld-4.0 llvm-7-dev libclang-4.0-dev ninja-build libusb-1.0-0-dev curl libcurl4-gnutls-dev \ + mpi-default-dev libicu-dev python-dev python3-dev libbz2-dev zlib1g-dev libssl-dev libgmp-dev doxygen graphviz libgmp3-dev \ + ninja-build libusb-1.0-0-dev curl libcurl4-gnutls-dev autotools-dev ruby \ + clang-9 lldb-9 lld-9 libllvm-9-ocaml-dev libllvm9 llvm-9 llvm-9-dev llvm-9-doc llvm-9-examples llvm-9-runtime clang-9 clang-tools-9 \ + clang-9-doc libclang-common-9-dev libclang-9-dev libclang1-9 clang-format-9 python-clang-9 clangd-9 libfuzzer-9-dev lldb-9 lld-9 \ + libc++-9-dev libc++abi-9-dev libomp-9-dev \ && rm -rf /var/lib/apt/lists/* -RUN update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-7/bin/clang 700 \ - && update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-7/bin/clang++ 700 +RUN update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-9/bin/clang 900 \ + && update-alternatives --install /usr/bin/clang++ clang++ /usr/lib/llvm-9/bin/clang++ 900 RUN wget https://cmake.org/files/v3.9/cmake-3.9.6-Linux-x86_64.sh \ && bash cmake-3.9.6-Linux-x86_64.sh --prefix=/usr/local --exclude-subdir --skip-license \ @@ -29,7 +33,7 @@ ENV CXX clang++ RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.bz2 -O - | tar -xj \ && cd boost_1_71_0 \ && ./bootstrap.sh --prefix=/usr/local \ - && echo 'using clang : 7 : clang++-7 ;' >> project-config.jam \ + && echo 'using clang : 9 : clang++-9 ;' >> project-config.jam \ && ./b2 -d0 -j$(nproc) --with-thread --with-date_time --with-system --with-filesystem --with-program_options \ --with-serialization --with-chrono --with-test --with-context --with-locale --with-coroutine --with-iostreams toolset=clang link=static install \ && cd .. && rm -rf boost_1_71_0 @@ -43,12 +47,12 @@ RUN wget https://github.com/mongodb/mongo-c-driver/releases/download/1.10.2/mong && make install \ && cd ../../ && rm -rf mongo-c-driver-1.10.2 -RUN git clone --depth 1 --single-branch --branch release_70 https://github.com/llvm-mirror/llvm.git \ - && git clone --depth 1 --single-branch --branch release_70 https://github.com/llvm-mirror/clang.git llvm/tools/clang \ - && cd llvm \ - && cmake -H. -Bbuild -GNinja -DCMAKE_INSTALL_PREFIX=/opt/wasm -DLLVM_TARGETS_TO_BUILD= -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly -DCMAKE_BUILD_TYPE=Release \ - && cmake --build build --target install \ - && cd .. && rm -rf llvm +# RUN git clone --depth 1 --single-branch --branch release_90 https://github.com/llvm-mirror/llvm.git \ +# && git clone --depth 1 --single-branch --branch release_90 https://github.com/llvm-mirror/clang.git llvm/tools/clang \ +# && cd llvm \ +# && cmake -H. -Bbuild -GNinja -DCMAKE_INSTALL_PREFIX=/opt/wasm -DLLVM_TARGETS_TO_BUILD= -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly -DCMAKE_BUILD_TYPE=Release \ +# && cmake --build build --target install \ +# && cd .. && rm -rf llvm RUN git clone --depth 1 -b releases/v3.3 https://github.com/mongodb/mongo-cxx-driver \ && cd mongo-cxx-driver/build \ @@ -70,4 +74,4 @@ RUN git clone --depth 1 -b 0.2 https://github.com/boscore/cppkafka.git \ && mkdir build && cd build \ && cmake -DCPPKAFKA_RDKAFKA_STATIC_LIB=1 -DCPPKAFKA_BUILD_SHARED=0 .. \ && make install \ - && cd ../../ && rm -rf cppkafka + && cd ../../ && rm -rf cppkafka \ No newline at end of file diff --git a/README.md b/README.md index c576a181fa0..d669170b8b5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # BOSCore - Blockchain financial center building a trusted business ecosystem. -## BOSCore Version: v3.0.6 -### Basic EOSIO Version: v1.6.6 (support REX, part 2.0.x) +## BOSCore Version: v3.0.7 +### Basic EOSIO Version: v1.6.6 (support REX & EOSVM, part 2.0.x) # Background The emergence of EOS has brought new imagination to the blockchain. In just a few months since the main network was launched, the version has undergone dozens of upgrades, not only the stability has been greatly improved, but also the new functions have been gradually realized. The node team is also actively involved in building the EOSIO ecosystem. What is even more exciting is that EOS has attracted more and more development teams. There are already hundreds of DApp running on the EOS main network. The transaction volume and circulation market value far exceed Ethereum, and the space for development is growing broader. diff --git a/README_CN.md b/README_CN.md index 6dcb8c63cfb..d5541436351 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,7 +1,7 @@ # BOSCore - 区块链自由港，构建可信商业生态。 -## BOSCore Version: v3.0.6 -### Basic EOSIO Version: v1.6.6 (support REX, part 2.0.x) +## BOSCore Version: v3.0.7 +### Basic EOSIO Version: v1.6.6 (support REX & EOSVM, part 2.0.x) # 背景 EOS的出现给区块链带来了新的想象力，主网启动短短几个月以来，版本经历了几十次升级，不仅稳定性得到了很大提高，并且新功能也逐步实现，各个节点团队也积极参与建设EOSIO生态。让人更加兴奋的是，EOS已经吸引了越来越多的开发团队，当前已经有数百个DApp在EOS主网上面运行，其交易量和流通市值远超以太坊，可发展的空间愈来愈广阔。 diff --git a/eosio_build.sh b/eosio_build.sh index a60352e23a7..3db66b5e66b 100755 --- a/eosio_build.sh +++ b/eosio_build.sh @@ -273,7 +273,8 @@ -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" -DBUILD_MONGO_DB_PLUGIN=true \ -DENABLE_COVERAGE_TESTING="${ENABLE_COVERAGE_TESTING}" -DBUILD_DOXYGEN="${DOXYGEN}" \ -DCMAKE_CXX_STANDARD_LIBRARIES="-lpthread" \ - -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" ${LOCAL_CMAKE_FLAGS} "${SOURCE_DIR}" + -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" ${LOCAL_CMAKE_FLAGS} "${SOURCE_DIR}" \ + -DENABLE_TOOLS=OFF then printf "\\n\\t>>>>>>>>>>>>>>>>>>>> CMAKE building BOSCore has exited with the above error.\\n\\n" exit -1 diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index eeaf5afa771..788e6194973 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -13,3 +13,6 @@ set(BUILD_TOOLS OFF CACHE BOOL "Build wabt tools") set(RUN_RE2C OFF CACHE BOOL "Run re2c") set(WITH_EXCEPTIONS ON CACHE BOOL "Build with exceptions enabled" FORCE) add_subdirectory( wabt ) +if(eos-vm IN_LIST EOSIO_WASM_RUNTIMES OR eos-vm-jit IN_LIST EOSIO_WASM_RUNTIMES) + add_subdirectory( eos-vm ) +endif() \ No newline at end of file diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 3a3d8e66816..4ab7dceebee 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -125,6 +125,10 @@ foreach(RUNTIME ${EOSIO_WASM_RUNTIMES}) target_compile_definitions(eosio_chain PUBLIC "EOSIO_${RUNTIMEUC}_RUNTIME_ENABLED") endforeach() +if(EOSVMOC_ENABLE_DEVELOPER_OPTIONS) + message(WARNING "EOS VM OC Developer Options are enabled; these are NOT supported") + target_compile_definitions(eosio_chain PUBLIC EOSIO_EOS_VM_OC_DEVELOPER) +endif() install( TARGETS eosio_chain RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 50b3dd50deb..dfe210a5f90 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -158,6 +158,10 @@ struct controller_impl { map< account_name, map > apply_handlers; platform_timer timer; +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) + vm::wasm_allocator wasm_alloc; +#endif + /** * Transactions that were undone by pop_block or abort_block, transactions * are removed from this list if they are re-applied in other blocks. Producers @@ -199,7 +203,7 @@ struct controller_impl { cfg.reversible_cache_size ), blog( cfg.blocks_dir ), fork_db( cfg.state_dir ), - wasmif( cfg.wasm_runtime, db ), + wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config ), resource_limits( db ), authorization( s, db ), conf( cfg ), @@ -244,10 +248,7 @@ struct controller_impl { template void emit( const Signal& s, Arg&& a ) { try { - s( std::forward( a )); - } catch (std::bad_alloc& e) { - wlog( "std::bad_alloc" ); - throw e; + s(std::forward(a)); } catch (boost::interprocess::bad_alloc& e) { wlog( "bad alloc" ); throw e; @@ -767,6 +768,7 @@ struct controller_impl { return enc.result(); } + /** * Sets fork database head to the genesis state. */ @@ -2857,4 +2859,10 @@ void controller::set_upo(uint32_t target_block_num) { } } +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) +vm::wasm_allocator& controller::get_wasm_allocator() { + return my->wasm_alloc; +} +#endif + } } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index cee256a5c09..bcc416fb242 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace chainbase { class database; @@ -98,6 +99,9 @@ namespace eosio { namespace chain { genesis_state genesis; wasm_interface::vm_type wasm_runtime = chain::config::default_wasm_runtime; + eosvmoc::config eosvmoc_config; + bool eosvmoc_tierup = false; + db_read_mode read_mode = db_read_mode::SPECULATIVE; validation_mode block_validation_mode = validation_mode::FULL; @@ -296,6 +300,9 @@ namespace eosio { namespace chain { void reset_pbft_my_prepare(); void reset_pbft_prepared(); void maybe_switch_forks(); +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) + vm::wasm_allocator& get_wasm_allocator(); +#endif signal pre_accepted_block; signal accepted_block_header; diff --git a/libraries/chain/include/eosio/chain/wasm_interface.hpp b/libraries/chain/include/eosio/chain/wasm_interface.hpp index ea6f1c9b8a9..4acbb9b21c5 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface.hpp @@ -2,6 +2,9 @@ #include #include #include +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) +#include +#endif #include "Runtime/Linker.h" #include "Runtime/Runtime.h" @@ -10,6 +13,8 @@ namespace eosio { namespace chain { class apply_context; class wasm_runtime_interface; class controller; + namespace eosvmoc { struct config; } + struct wasm_exit { int32_t code = 0; }; @@ -71,10 +76,13 @@ namespace eosio { namespace chain { public: enum class vm_type { wavm, - wabt + wabt, + eos_vm, + eos_vm_jit, + eos_vm_oc }; - wasm_interface(vm_type vm, const chainbase::database& d); + wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config); ~wasm_interface(); //call before dtor to skip what can be minutes of dtor overhead with some runtimes; can cause leaks @@ -106,4 +114,4 @@ namespace eosio{ namespace chain { std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime); }} -FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (wavm)(wabt) ) +FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (wavm)(wabt)(eos_vm)(eos_vm_jit)(eos_vm_oc) ) diff --git a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp index 957d82ca38b..dedc6d78de4 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp @@ -3,6 +3,12 @@ #include #include #include +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED +#include +#else +#define _REGISTER_EOSVMOC_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) +#endif +#include #include #include #include @@ -16,6 +22,10 @@ #include "WAST/WAST.h" #include "IR/Validate.h" +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) +#include +#endif + using namespace fc; using namespace eosio::chain::webassembly; using namespace IR; @@ -25,6 +35,8 @@ using boost::multi_index_container; namespace eosio { namespace chain { + namespace eosvmoc { struct config; } + struct wasm_interface_impl { struct wasm_cache_entry { digest_type code_hash; @@ -38,15 +50,44 @@ namespace eosio { namespace chain { struct by_first_block_num; struct by_last_block_num; - wasm_interface_impl(wasm_interface::vm_type vm, const chainbase::database& d) : db(d), wasm_runtime_time(vm) { +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + struct eosvmoc_tier { + eosvmoc_tier(const boost::filesystem::path& d, const eosvmoc::config& c, const chainbase::database& db) : cc(d, c, db), exec(cc) {} + eosvmoc::code_cache_async cc; + eosvmoc::executor exec; + eosvmoc::memory mem; + }; +#endif + + wasm_interface_impl(wasm_interface::vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config) : db(d), wasm_runtime_time(vm) { #ifdef EOSIO_WAVM_RUNTIME_ENABLED if(vm == wasm_interface::vm_type::wavm) runtime_interface = std::make_unique(); #endif if(vm == wasm_interface::vm_type::wabt) runtime_interface = std::make_unique(); +#ifdef EOSIO_EOS_VM_RUNTIME_ENABLED + if(vm == wasm_interface::vm_type::eos_vm) + runtime_interface = std::make_unique>(); +#endif +#ifdef EOSIO_EOS_VM_JIT_RUNTIME_ENABLED + if(vm == wasm_interface::vm_type::eos_vm_jit) + runtime_interface = std::make_unique>(); +#endif +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + if(vm == wasm_interface::vm_type::eos_vm_oc) + runtime_interface = std::make_unique(data_dir, eosvmoc_config, d); +#endif if(!runtime_interface) EOS_THROW(wasm_exception, "${r} wasm runtime not supported on this platform and/or configuration", ("r", vm)); + +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + if(eosvmoc_tierup) { + EOS_ASSERT(vm != wasm_interface::vm_type::wavm, wasm_exception, "WAVM is incompatible with EOS VM OC"); + EOS_ASSERT(vm != wasm_interface::vm_type::eos_vm_oc, wasm_exception, "You can't use EOS VM OC as the base runtime when tier up is activated"); + eosvmoc.emplace(data_dir, eosvmoc_config, d); + } +#endif } ~wasm_interface_impl() { @@ -87,6 +128,10 @@ namespace eosio { namespace chain { //anything last used before or on the LIB can be evicted const auto first_it = wasm_instantiation_cache.get().begin(); const auto last_it = wasm_instantiation_cache.get().upper_bound(lib); +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + if(eosvmoc) for(auto it = first_it; it != last_it; it++) + eosvmoc->cc.free_code(it->code_hash, it->vm_version); +#endif wasm_instantiation_cache.get().erase(first_it, last_it); } @@ -174,6 +219,9 @@ namespace eosio { namespace chain { const chainbase::database& db; const wasm_interface::vm_type wasm_runtime_time; +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + fc::optional eosvmoc; +#endif }; #define _ADD_PAREN_1(...) ((__VA_ARGS__)) _ADD_PAREN_2 @@ -184,7 +232,9 @@ namespace eosio { namespace chain { #define _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ _REGISTER_WAVM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ - _REGISTER_WABT_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) + _REGISTER_WABT_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ + _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ + _REGISTER_EOSVMOC_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) #define _REGISTER_INTRINSIC4(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, SIG ) diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp new file mode 100644 index 00000000000..000b8cef33c --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc.hpp @@ -0,0 +1,799 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "IR/Types.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace eosio { namespace chain { namespace webassembly { namespace eosvmoc { + +using namespace IR; +using namespace Runtime; +using namespace fc; +using namespace eosio::chain::webassembly::common; + +using namespace eosio::chain::eosvmoc; + +class eosvmoc_instantiated_module; + +class eosvmoc_runtime : public eosio::chain::wasm_runtime_interface { + public: + eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db); + ~eosvmoc_runtime(); + bool inject_module(IR::Module&) override { return false; } + std::unique_ptr instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, + const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override; + + void immediately_exit_currently_running_module() override; + + friend eosvmoc_instantiated_module; + eosvmoc::code_cache_sync cc; + eosvmoc::executor exec; + eosvmoc::memory mem; +}; + +/** + * class to represent an in-wasm-memory array + * it is a hint to the transcriber that the next parameter will + * be a size (in Ts) and that the pair are validated together + * This triggers the template specialization of intrinsic_invoker_impl + * @tparam T + */ +template +inline array_ptr array_ptr_impl (size_t ptr, size_t length) +{ + constexpr int cb_full_linear_memory_start_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(full_linear_memory_start); + constexpr int cb_first_invalid_memory_address_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(first_invalid_memory_address); + + size_t end = ptr + length*sizeof(T); + + asm volatile("cmp %%gs:%c[firstInvalidMemory], %[End]\n" + "jle 1f\n" + "mov %%gs:%c[maximumEosioWasmMemory], %[Ptr]\n" //always invalid address + "1:\n" + "add %%gs:%c[linearMemoryStart], %[Ptr]\n" + : [Ptr] "+r" (ptr), + [End] "+r" (end) + : [linearMemoryStart] "i" (cb_full_linear_memory_start_segment_offset), + [firstInvalidMemory] "i" (cb_first_invalid_memory_address_segment_offset), + [maximumEosioWasmMemory] "i" (wasm_constraints::maximum_linear_memory) + : "cc" + ); + + + return array_ptr((T*)ptr); +} + +/** + * class to represent an in-wasm-memory char array that must be null terminated + */ +inline null_terminated_ptr null_terminated_ptr_impl(uint64_t ptr) +{ + constexpr int cb_full_linear_memory_start_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(full_linear_memory_start); + constexpr int cb_first_invalid_memory_address_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(first_invalid_memory_address); + + char dumpster; + uint64_t scratch; + + asm volatile("mov %%gs:(%[Ptr]), %[Dumpster]\n" //probe memory location at ptr to see if valid + "mov %%gs:%c[firstInvalidMemory], %[Scratch]\n" //get first invalid memory address + "cmpb $0, %%gs:-1(%[Scratch])\n" //is last byte in valid linear memory 0? + "je 2f\n" //if so, this will be a null terminated string one way or another + "mov %[Ptr],%[Scratch]\n" + "1:\n" //start loop looking for either 0, or until we SEGV + "inc %[Scratch]\n" + "cmpb $0,%%gs:(%[Scratch])\n" + "jne 1b\n" + "2:\n" + "add %%gs:%c[linearMemoryStart], %[Ptr]\n" //add address of linear memory 0 to ptr + : [Ptr] "+r" (ptr), + [Dumpster] "=r" (dumpster), + [Scratch] "=r" (scratch) + : [linearMemoryStart] "i" (cb_full_linear_memory_start_segment_offset), + [firstInvalidMemory] "i" (cb_first_invalid_memory_address_segment_offset) + : "cc" + ); + + return null_terminated_ptr((char*)ptr); +} + +template +struct void_ret_wrapper { + using type = T; +}; + +template<> +struct void_ret_wrapper { + using type = char; +}; + +template +using void_ret_wrapper_t = typename void_ret_wrapper::type; + +/** + * template that maps native types to WASM VM types + * @tparam T the native type + */ +template +struct native_to_wasm { + using type = void; +}; + +/** + * specialization for mapping pointers to int32's + */ +template +struct native_to_wasm { + using type = I32; +}; + +/** + * Mappings for native types + */ +template<> +struct native_to_wasm { + using type = F32; +}; +template<> +struct native_to_wasm { + using type = F64; +}; +template<> +struct native_to_wasm { + using type = I32; +}; +template<> +struct native_to_wasm { + using type = I32; +}; +template<> +struct native_to_wasm { + using type = I64; +}; +template<> +struct native_to_wasm { + using type = I64; +}; +template<> +struct native_to_wasm { + using type = I32; +}; +template<> +struct native_to_wasm { + using type = I64; +}; +template<> +struct native_to_wasm { + using type = I64; +}; +template<> +struct native_to_wasm { + using type = I32; +}; + +template<> +struct native_to_wasm { + using type = I32; +}; + +// convenience alias +template +using native_to_wasm_t = typename native_to_wasm::type; + +template +inline auto convert_native_to_wasm(T val) { + return native_to_wasm_t(val); +} + +inline auto convert_native_to_wasm(const name &val) { + return native_to_wasm_t(val.to_uint64_t()); +} + +inline auto convert_native_to_wasm(const fc::time_point_sec& val) { + return native_to_wasm_t(val.sec_since_epoch()); +} + +inline auto convert_native_to_wasm(char* ptr) { + constexpr int cb_full_linear_memory_start_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(full_linear_memory_start); + char* full_linear_memory_start; + asm("mov %%gs:%c[fullLinearMemOffset], %[fullLinearMem]\n" + : [fullLinearMem] "=r" (full_linear_memory_start) + : [fullLinearMemOffset] "i" (cb_full_linear_memory_start_offset) + ); + U64 delta = (U64)(ptr - full_linear_memory_start); + array_ptr_impl(delta, 1); + return (U32)delta; +} + +template +inline auto convert_wasm_to_native(native_to_wasm_t val) { + return T(val); +} + +template +struct wasm_to_value_type; + +template<> +struct wasm_to_value_type { + static constexpr auto value = ValueType::f32; +}; + +template<> +struct wasm_to_value_type { + static constexpr auto value = ValueType::f64; +}; +template<> +struct wasm_to_value_type { + static constexpr auto value = ValueType::i32; +}; +template<> +struct wasm_to_value_type { + static constexpr auto value = ValueType::i64; +}; + +template +constexpr auto wasm_to_value_type_v = wasm_to_value_type::value; + +template +struct wasm_to_rvalue_type; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::f32; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::f64; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i32; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i64; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::none; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i64; +}; +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i64; +}; + +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i32; +}; + +template<> +struct wasm_to_rvalue_type { + static constexpr auto value = ResultType::i32; +}; + + +template +constexpr auto wasm_to_rvalue_type_v = wasm_to_rvalue_type::value; + +template +struct is_reference_from_value { + static constexpr bool value = false; +}; + +template<> +struct is_reference_from_value { + static constexpr bool value = true; +}; + +template<> +struct is_reference_from_value { + static constexpr bool value = true; +}; + +template +constexpr bool is_reference_from_value_v = is_reference_from_value::value; + + + +struct void_type { +}; + +/** + * Forward declaration of provider for FunctionType given a desired C ABI signature + */ +template +struct wasm_function_type_provider; + +/** + * specialization to destructure return and arguments + */ +template +struct wasm_function_type_provider { + static const FunctionType *type() { + return FunctionType::get(wasm_to_rvalue_type_v, {wasm_to_value_type_v ...}); + } +}; + +/** + * Forward declaration of the invoker type which transcribes arguments to/from a native method + * and injects the appropriate checks + * + * @tparam Ret - the return type of the native function + * @tparam NativeParameters - a std::tuple of the remaining native parameters to transcribe + * @tparam WasmParameters - a std::tuple of the transribed parameters + */ +template +struct intrinsic_invoker_impl; + +/** + * Specialization for the fully transcribed signature + * @tparam Ret - the return type of the native function + * @tparam Translated - the arguments to the wasm function + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_method_type = Ret (*)(Translated...); + + template + static native_to_wasm_t invoke(Translated... translated) { + try { + if constexpr(!is_injected) { + constexpr int cb_current_call_depth_remaining_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining); + constexpr int depth_assertion_intrinsic_offset = OFFSET_OF_FIRST_INTRINSIC - (int)boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.depth_assert"))).value()*8; + asm volatile("cmpl $1,%%gs:%c[callDepthRemainOffset]\n" + "jne 1f\n" + "callq *%%gs:%c[depthAssertionIntrinsicOffset]\n" + "1:\n" + : + : [callDepthRemainOffset] "i" (cb_current_call_depth_remaining_segment_offset), + [depthAssertionIntrinsicOffset] "i" (depth_assertion_intrinsic_offset) + : "cc"); + } + return convert_native_to_wasm(Method(translated...)); + } + catch(...) { + *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::current_exception(); + } + siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); + __builtin_unreachable(); + } + + template + static const auto fn() { + return invoke; + } +}; + +/** + * specialization of the fully transcribed signature for void return values + * @tparam Translated - the arguments to the wasm function + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_method_type = void_type (*)(Translated...); + + template + static void invoke(Translated... translated) { + try { + if constexpr(!is_injected) { + constexpr int cb_current_call_depth_remaining_segment_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining); + constexpr int depth_assertion_intrinsic_offset = OFFSET_OF_FIRST_INTRINSIC - (int)boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.depth_assert"))).value()*8; + asm volatile("cmpl $1,%%gs:%c[callDepthRemainOffset]\n" + "jne 1f\n" + "callq *%%gs:%c[depthAssertionIntrinsicOffset]\n" + "1:\n" + : + : [callDepthRemainOffset] "i" (cb_current_call_depth_remaining_segment_offset), + [depthAssertionIntrinsicOffset] "i" (depth_assertion_intrinsic_offset) + : "cc"); + } + Method(translated...); + return; + } + catch(...) { + *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::current_exception(); + } + siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); + __builtin_unreachable(); + } + + template + static const auto fn() { + return invoke; + } +}; + +/** + * Sepcialization for transcribing a simple type in the native method signature + * @tparam Ret - the return type of the native method + * @tparam Input - the type of the native parameter to transcribe + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using translated_type = native_to_wasm_t; + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret (*)(Input, Inputs..., Translated...); + + template + static Ret translate_one(Inputs... rest, Translated... translated, translated_type last) { + auto native = convert_wasm_to_native(last); + return Then(native, rest..., translated...); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a array_ptr type in the native method signature + * This type transcribes into 2 wasm parameters: a pointer and byte length and checks the validity of that memory + * range before dispatching to the native method + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, uint32_t, Inputs...>, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret(*)(array_ptr, uint32_t, Inputs..., Translated...); + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr, I32 size) -> std::enable_if_t::value, Ret> { + static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); + const auto length = size_t((U32)size); + T* base = array_ptr_impl((U32)ptr, length); + if ( reinterpret_cast(base) % alignof(T) != 0 ) { + std::vector& copy = reinterpret_cast>*>(eos_vm_oc_get_bounce_buffer_list())->emplace_back(length > 0 ? length*sizeof(T) : 1); + T* copy_ptr = (T*)©[0]; + memcpy( (void*)copy.data(), (void*)base, length * sizeof(T) ); + return Then(static_cast>(copy_ptr), length, rest..., translated...); + } + return Then(static_cast>(base), length, rest..., translated...); + }; + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr, I32 size) -> std::enable_if_t::value, Ret> { + static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); + const auto length = size_t((U32)size); + T* base = array_ptr_impl((U32)ptr, length); + if ( reinterpret_cast(base) % alignof(T) != 0 ) { + std::vector& copy = reinterpret_cast>*>(eos_vm_oc_get_bounce_buffer_list())->emplace_back(length > 0 ? length*sizeof(T) : 1); + T* copy_ptr = (T*)©[0]; + memcpy( (void*)copy.data(), (void*)base, length * sizeof(T) ); + Ret ret = Then(static_cast>(copy_ptr), length, rest..., translated...); + memcpy( (void*)base, (void*)copy.data(), length * sizeof(T) ); + return ret; + } + return Then(static_cast>(base), length, rest..., translated...); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a null_terminated_ptr type in the native method signature + * This type transcribes 1 wasm parameters: a char pointer which is validated to contain + * a null value before the end of the allocated memory. + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret(*)(null_terminated_ptr, Inputs..., Translated...); + + template + static Ret translate_one(Inputs... rest, Translated... translated, I32 ptr) { + return Then(null_terminated_ptr_impl((U32)ptr), rest..., translated...); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a pair of array_ptr types in the native method signature that share size + * This type transcribes into 3 wasm parameters: 2 pointers and byte length and checks the validity of those memory + * ranges before dispatching to the native method + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, array_ptr, uint32_t, Inputs...>, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret(*)(array_ptr, array_ptr, uint32_t, Inputs..., Translated...); + + template + static Ret translate_one(Inputs... rest, Translated... translated, I32 ptr_t, I32 ptr_u, I32 size) { + static_assert(std::is_same, char>::value && std::is_same, char>::value, "Currently only support array of (const)chars"); + const auto length = size_t((U32)size); + return Then(array_ptr_impl((U32)ptr_t, length), array_ptr_impl((U32)ptr_u, length), length, rest..., translated...); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing memset parameters + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, int, uint32_t>, std::tuple<>> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret(*)(array_ptr, int, uint32_t); + + template + static Ret translate_one(I32 ptr, I32 value, I32 size) { + const auto length = size_t((U32)size); + return Then(array_ptr_impl((U32)ptr, length), value, length); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a pointer type in the native method signature + * This type transcribes into an int32 pointer checks the validity of that memory + * range before dispatching to the native method + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret (*)(T *, Inputs..., Translated...); + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { + T* base = array_ptr_impl((U32)ptr, 1); + if ( reinterpret_cast(base) % alignof(T) != 0 ) { + std::remove_const_t copy; + T* copy_ptr = © + memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); + return Then(copy_ptr, rest..., translated...); + } + return Then(base, rest..., translated...); + }; + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { + T* base = array_ptr_impl((U32)ptr, 1); + if ( reinterpret_cast(base) % alignof(T) != 0 ) { + std::remove_const_t copy; + T* copy_ptr = © + memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); + Ret ret = Then(copy_ptr, rest..., translated...); + memcpy( (void*)base, (void*)copy_ptr, sizeof(T) ); + return ret; + } + return Then(base, rest..., translated...); + }; + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a reference to a name which can be passed as a native value + * This type transcribes into a native type which is loaded by value into a + * variable on the stack and then passed by reference to the intrinsic. + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple >>; + using then_type = Ret (*)(const name&, Inputs..., Translated...); + + template + static Ret translate_one(Inputs... rest, Translated... translated, native_to_wasm_t wasm_value) { + auto value = name(wasm_value); + return Then(value, rest..., translated...); + } + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * Specialization for transcribing a reference type in the native method signature + * This type transcribes into an int32 pointer checks the validity of that memory + * range before dispatching to the native method + * + * @tparam Ret - the return type of the native method + * @tparam Inputs - the remaining native parameters to transcribe + * @tparam Translated - the list of transcribed wasm parameters + */ +template +struct intrinsic_invoker_impl, std::tuple> { + using next_step = intrinsic_invoker_impl, std::tuple>; + using then_type = Ret (*)(T &, Inputs..., Translated...); + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { + EOS_ASSERT((U32)ptr != 0, wasm_exception, "references cannot be created for null pointers"); + T &base = *array_ptr_impl((uint32_t)ptr, 1); + + if ( reinterpret_cast(&base) % alignof(T) != 0 ) { + std::remove_const_t copy; + T* copy_ptr = © + memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); + return Then(*copy_ptr, rest..., translated...); + } + return Then(base, rest..., translated...); + } + + template + static auto translate_one(Inputs... rest, Translated... translated, I32 ptr) -> std::enable_if_t::value, Ret> { + EOS_ASSERT((U32)ptr != 0, wasm_exception, "reference cannot be created for null pointers"); + T &base = *array_ptr_impl((uint32_t)ptr, 1); + + if ( reinterpret_cast(&base) % alignof(T) != 0 ) { + std::remove_const_t copy; + T* copy_ptr = © + memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); + Ret ret = Then(*copy_ptr, rest..., translated...); + memcpy( (void*)&base, (void*)copy_ptr, sizeof(T) ); + return ret; + } + return Then(base, rest..., translated...); + } + + template + static const auto fn() { + return next_step::template fn>(); + } +}; + +/** + * forward declaration of a wrapper class to call methods of the class + */ +template +struct intrinsic_function_invoker { + using impl = intrinsic_invoker_impl, std::tuple<>>; + + template + static Ret wrapper(Params... params) { + constexpr int cb_ctx_ptr_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(ctx); + apply_context* ctx; + asm("mov %%gs:%c[applyContextOffset], %[cPtr]\n" + : [cPtr] "=r" (ctx) + : [applyContextOffset] "i" (cb_ctx_ptr_offset) + ); + return (class_from_wasm::value(*ctx).*Method)(params...); + } + + template + static const WasmSig *fn() { + auto fn = impl::template fn>(); + static_assert(std::is_same::value, + "Intrinsic function signature does not match the ABI"); + return fn; + } +}; + +template +struct intrinsic_function_invoker { + using impl = intrinsic_invoker_impl, std::tuple<>>; + + template + static void_type wrapper(Params... params) { + constexpr int cb_ctx_ptr_offset = OFFSET_OF_CONTROL_BLOCK_MEMBER(ctx); + apply_context* ctx; + asm("mov %%gs:%c[applyContextOffset], %[cPtr]\n" + : [cPtr] "=r" (ctx) + : [applyContextOffset] "i" (cb_ctx_ptr_offset) + ); + (class_from_wasm::value(*ctx).*Method)(params...); + return void_type(); + } + + template + static const WasmSig *fn() { + auto fn = impl::template fn>(); + static_assert(std::is_same::value, + "Intrinsic function signature does not match the ABI"); + return fn; + } +}; + +template +struct intrinsic_function_invoker_wrapper; + +template +struct intrinsic_function_invoker_wrapper { + static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && + !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), + "intrinsics should only return a reference or pointer with single byte alignment"); + using type = intrinsic_function_invoker; +}; + +template +struct intrinsic_function_invoker_wrapper { + static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && + !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), + "intrinsics should only return a reference or pointer with single byte alignment"); + using type = intrinsic_function_invoker; +}; + +template +struct intrinsic_function_invoker_wrapper { + static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && + !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), + "intrinsics should only return a reference or pointer with single byte alignment"); + using type = intrinsic_function_invoker; +}; + +template +struct intrinsic_function_invoker_wrapper { + static_assert( !(std::is_pointer_v && alignof(std::remove_pointer_t>) != 1) && + !(std::is_lvalue_reference_v && alignof(std::remove_reference_t>) != 1), + "intrinsics should only return a reference or pointer with single byte alignment"); + using type = intrinsic_function_invoker; +}; + +#define _ADD_PAREN_1(...) ((__VA_ARGS__)) _ADD_PAREN_2 +#define _ADD_PAREN_2(...) ((__VA_ARGS__)) _ADD_PAREN_1 +#define _ADD_PAREN_1_END +#define _ADD_PAREN_2_END +#define _WRAPPED_SEQ(SEQ) BOOST_PP_CAT(_ADD_PAREN_1 SEQ, _END) + +#define __INTRINSIC_NAME(LABEL, SUFFIX) LABEL##SUFFIX +#define _INTRINSIC_NAME(LABEL, SUFFIX) __INTRINSIC_NAME(LABEL,SUFFIX) + +#define _REGISTER_EOSVMOC_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ + static eosio::chain::eosvmoc::intrinsic _INTRINSIC_NAME(__intrinsic_fn, __COUNTER__) EOSVMOC_INTRINSIC_INIT_PRIORITY (\ + MOD "." NAME,\ + eosio::chain::webassembly::eosvmoc::wasm_function_type_provider::type(),\ + (void *)eosio::chain::webassembly::eosvmoc::intrinsic_function_invoker_wrapper::type::fn<&CLS::METHOD>(),\ + ::boost::hana::index_if(eosio::chain::eosvmoc::intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING(MOD "." NAME))).value()\ + );\ + + +} } } }// eosio::chain::webassembly::eosvmoc diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp new file mode 100644 index 00000000000..16fb1b47df2 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp @@ -0,0 +1,118 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +#include + +namespace std { + template<> struct hash { + size_t operator()(const eosio::chain::eosvmoc::code_tuple& ct) const noexcept { + return ct.code_id._hash[0]; + } + }; +} + +namespace eosio { namespace chain { namespace eosvmoc { + +using namespace boost::multi_index; +using namespace boost::asio; + +namespace bip = boost::interprocess; +namespace bfs = boost::filesystem; + +using allocator_t = bip::rbtree_best_fit, alignof(std::max_align_t)>; + +struct config; + +class code_cache_base { + public: + code_cache_base(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db); + ~code_cache_base(); + + const int& fd() const { return _cache_fd; } + + void free_code(const digest_type& code_id, const uint8_t& vm_version); + + protected: + struct by_hash; + + typedef boost::multi_index_container< + code_descriptor, + indexed_by< + sequenced<>, + ordered_unique, + composite_key< code_descriptor, + member, + member + > + > + > + > code_cache_index; + code_cache_index _cache_index; + + const chainbase::database& _db; + + bfs::path _cache_file_path; + int _cache_fd; + + io_context _ctx; + local::datagram_protocol::socket _compile_monitor_write_socket{_ctx}; + local::datagram_protocol::socket _compile_monitor_read_socket{_ctx}; + + //these are really only useful to the async code cache, but keep them here so + //free_code can be shared + std::unordered_set _queued_compiles; + std::unordered_map _outstanding_compiles_and_poison; + + size_t _free_bytes_eviction_threshold; + void check_eviction_threshold(size_t free_bytes); + void run_eviction_round(); + + void set_on_disk_region_dirty(bool); + + template + void serialize_cache_index(fc::datastream& ds); +}; + +class code_cache_async : public code_cache_base { + public: + code_cache_async(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db); + ~code_cache_async(); + + //If code is in cache: returns pointer & bumps to front of MRU list + //If code is not in cache, and not blacklisted, and not currently compiling: return nullptr and kick off compile + //otherwise: return nullptr + const code_descriptor* const get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version); + + private: + std::thread _monitor_reply_thread; + boost::lockfree::spsc_queue _result_queue; + void wait_on_compile_monitor_message(); + std::tuple consume_compile_thread_queue(); + std::unordered_set _blacklist; + size_t _threads; +}; + +class code_cache_sync : public code_cache_base { + public: + using code_cache_base::code_cache_base; + ~code_cache_sync(); + + //Can still fail and return nullptr if, for example, there is an expected instantiation failure + const code_descriptor* const get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version); +}; + +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_monitor.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_monitor.hpp new file mode 100644 index 00000000000..7d4229e4616 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_monitor.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include + +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +wrapped_fd get_connection_to_compile_monitor(int cache_fd); + +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_trampoline.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_trampoline.hpp new file mode 100644 index 00000000000..c6be55ad3ea --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/compile_trampoline.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace eosio { namespace chain { namespace eosvmoc { + +void run_compile_trampoline(int fd); + +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp new file mode 100644 index 00000000000..dbab7b0342b --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +struct config { + uint64_t cache_size = 1024u*1024u*1024u; + uint64_t threads = 1u; +}; + +}}} diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h new file mode 100644 index 00000000000..6c5d4e42676 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +#include +#include +namespace eosio { namespace chain {class apply_context;}} +#endif + +struct eos_vm_oc_control_block { + uint64_t magic; + uintptr_t execution_thread_code_start; + size_t execution_thread_code_length; + uintptr_t execution_thread_memory_start; + size_t execution_thread_memory_length; +#ifdef __cplusplus + eosio::chain::apply_context* ctx; + std::exception_ptr* eptr; +#else + void* ctx; + void* eptr; +#endif + unsigned current_call_depth_remaining; + int64_t current_linear_memory_pages; //-1 if no memory + char* full_linear_memory_start; + sigjmp_buf* jmp; +#ifdef __cplusplus + std::list>* bounce_buffers; +#else + void* bounce_buffers; +#endif + uintptr_t running_code_base; + int64_t first_invalid_memory_address; + unsigned is_running; +}; \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp new file mode 100644 index 00000000000..6dfffa12804 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { + +class apply_context; + +namespace eosvmoc { + +using control_block = eos_vm_oc_control_block; + +struct no_offset{}; +struct code_offset{ size_t offset; }; //relative to code_begin +struct intrinsic_ordinal{ size_t ordinal; }; + +using eosvmoc_optional_offset_or_import_t = fc::static_variant; + +struct code_descriptor { + digest_type code_hash; + uint8_t vm_version; + uint8_t codegen_version; + size_t code_begin; + eosvmoc_optional_offset_or_import_t start; + unsigned apply_offset; + int starting_memory_pages; + size_t initdata_begin; + unsigned initdata_size; + unsigned initdata_prologue_size; +}; + +enum eosvmoc_exitcode : int { + EOSVMOC_EXIT_CLEAN_EXIT = 1, + EOSVMOC_EXIT_CHECKTIME_FAIL, + EOSVMOC_EXIT_SEGV, + EOSVMOC_EXIT_EXCEPTION +}; + +}}} + +FC_REFLECT(eosio::chain::eosvmoc::no_offset, ); +FC_REFLECT(eosio::chain::eosvmoc::code_offset, (offset)); +FC_REFLECT(eosio::chain::eosvmoc::intrinsic_ordinal, (ordinal)); +FC_REFLECT(eosio::chain::eosvmoc::code_descriptor, (code_hash)(vm_version)(codegen_version)(code_begin)(start)(apply_offset)(starting_memory_pages)(initdata_begin)(initdata_size)(initdata_prologue_size)); + +#define EOSVMOC_INTRINSIC_INIT_PRIORITY __attribute__((init_priority(198))) \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp new file mode 100644 index 00000000000..e5a092cd57e --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/executor.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace eosio { namespace chain { + +class apply_context; + +namespace eosvmoc { + +class code_cache_base; +class memory; +struct code_descriptor; + +class executor { + public: + executor(const code_cache_base& cc); + ~executor(); + + void execute(const code_descriptor& code, const memory& mem, apply_context& context); + + private: + uint8_t* code_mapping; + size_t code_mapping_size; + bool mapping_is_executable; + + std::exception_ptr executors_exception_ptr; + sigjmp_buf executors_sigjmp_buf; + std::list> executors_bounce_buffers; +}; + +}}} diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h new file mode 100644 index 00000000000..d2baf938017 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/gs_seg_helpers.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include + +#ifdef __clang__ + #define GS_PTR __attribute__((address_space(256))) +#else + #define GS_PTR __seg_gs +#endif + +//This is really rather unfortunate, but on the upside it does allow a static assert to know if +//the values ever slide which would be a PIC breaking event we'd want to know about at compile +//time. +#define EOS_VM_OC_CONTROL_BLOCK_OFFSET (-18944) +#define EOS_VM_OC_MEMORY_STRIDE (UINT64_C(4329598976)) + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max); +sigjmp_buf* eos_vm_oc_get_jmp_buf(); +void* eos_vm_oc_get_exception_ptr(); +void* eos_vm_oc_get_bounce_buffer_list(); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic.hpp new file mode 100644 index 00000000000..d61af22c919 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include +#include + +namespace IR { + struct FunctionType; +} + +namespace eosio { namespace chain { namespace eosvmoc { + +struct intrinsic { + intrinsic(const char* name, const IR::FunctionType* type, void* function_ptr, size_t ordinal); +}; + +struct intrinsic_entry { + const IR::FunctionType* const type; + const void* const function_ptr; + const size_t ordinal; +}; + +using intrinsic_map_t = std::map; + +const intrinsic_map_t& get_intrinsic_map(); + +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp new file mode 100644 index 00000000000..61cdd2bbc86 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp @@ -0,0 +1,258 @@ +#pragma once + +#define BOOST_HANA_CONFIG_ENABLE_STRING_UDL +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +using namespace boost::hana::literals; + +//NEVER reorder or remove indexes; the PIC uses the indexes in this table as an offset in to a jump +// table. Adding on the bottom is fine and requires no other updates elsewhere +constexpr auto intrinsic_table = boost::hana::make_tuple( + "eosvmoc_internal.unreachable"_s, + "eosvmoc_internal.grow_memory"_s, + "eosvmoc_internal.div0_or_overflow"_s, + "eosvmoc_internal.indirect_call_mismatch"_s, + "eosvmoc_internal.indirect_call_oob"_s, + "eosvmoc_internal.depth_assert"_s, + "eosio_injection.call_depth_assert"_s, //these two are never used by EOS VM OC but all intrinsics + "eosio_injection.checktime"_s, //must be mapped + "env.__ashlti3"_s, + "env.__ashrti3"_s, + "env.__lshlti3"_s, + "env.__lshrti3"_s, + "env.__divti3"_s, + "env.__udivti3"_s, + "env.__modti3"_s, + "env.__umodti3"_s, + "env.__multi3"_s, + "env.__addtf3"_s, + "env.__subtf3"_s, + "env.__multf3"_s, + "env.__divtf3"_s, + "env.__eqtf2"_s, + "env.__netf2"_s, + "env.__getf2"_s, + "env.__gttf2"_s, + "env.__lttf2"_s, + "env.__letf2"_s, + "env.__cmptf2"_s, + "env.__unordtf2"_s, + "env.__negtf2"_s, + "env.__floatsitf"_s, + "env.__floatunsitf"_s, + "env.__floatditf"_s, + "env.__floatunditf"_s, + "env.__floattidf"_s, + "env.__floatuntidf"_s, + "env.__floatsidf"_s, + "env.__extendsftf2"_s, + "env.__extenddftf2"_s, + "env.__fixtfti"_s, + "env.__fixtfdi"_s, + "env.__fixtfsi"_s, + "env.__fixunstfti"_s, + "env.__fixunstfdi"_s, + "env.__fixunstfsi"_s, + "env.__fixsfti"_s, + "env.__fixdfti"_s, + "env.__fixunssfti"_s, + "env.__fixunsdfti"_s, + "env.__trunctfdf2"_s, + "env.__trunctfsf2"_s, + "env.is_feature_active"_s, + "env.activate_feature"_s, + "env.get_resource_limits"_s, + "env.set_resource_limits"_s, + "env.set_proposed_producers"_s, + "env.get_blockchain_parameters_packed"_s, + "env.set_blockchain_parameters_packed"_s, + "env.set_name_list_packed"_s, + "env.set_guaranteed_minimum_resources"_s, + "env.is_privileged"_s, + "env.set_privileged"_s, + "env.set_upgrade_parameters_packed"_s, + "env.get_active_producers"_s, + "env.db_store_i64"_s, + "env.db_update_i64"_s, + "env.db_remove_i64"_s, + "env.db_get_i64"_s, + "env.db_next_i64"_s, + "env.db_previous_i64"_s, + "env.db_find_i64"_s, + "env.db_lowerbound_i64"_s, + "env.db_upperbound_i64"_s, + "env.db_end_i64"_s, + "env.db_idx64_store"_s, + "env.db_idx64_remove"_s, + "env.db_idx64_update"_s, + "env.db_idx64_find_primary"_s, + "env.db_idx64_find_secondary"_s, + "env.db_idx64_lowerbound"_s, + "env.db_idx64_upperbound"_s, + "env.db_idx64_end"_s, + "env.db_idx64_next"_s, + "env.db_idx64_previous"_s, + "env.db_idx128_store"_s, + "env.db_idx128_remove"_s, + "env.db_idx128_update"_s, + "env.db_idx128_find_primary"_s, + "env.db_idx128_find_secondary"_s, + "env.db_idx128_lowerbound"_s, + "env.db_idx128_upperbound"_s, + "env.db_idx128_end"_s, + "env.db_idx128_next"_s, + "env.db_idx128_previous"_s, + "env.db_idx_double_store"_s, + "env.db_idx_double_remove"_s, + "env.db_idx_double_update"_s, + "env.db_idx_double_find_primary"_s, + "env.db_idx_double_find_secondary"_s, + "env.db_idx_double_lowerbound"_s, + "env.db_idx_double_upperbound"_s, + "env.db_idx_double_end"_s, + "env.db_idx_double_next"_s, + "env.db_idx_double_previous"_s, + "env.db_idx_long_double_store"_s, + "env.db_idx_long_double_remove"_s, + "env.db_idx_long_double_update"_s, + "env.db_idx_long_double_find_primary"_s, + "env.db_idx_long_double_find_secondary"_s, + "env.db_idx_long_double_lowerbound"_s, + "env.db_idx_long_double_upperbound"_s, + "env.db_idx_long_double_end"_s, + "env.db_idx_long_double_next"_s, + "env.db_idx_long_double_previous"_s, + "env.db_idx256_store"_s, + "env.db_idx256_remove"_s, + "env.db_idx256_update"_s, + "env.db_idx256_find_primary"_s, + "env.db_idx256_find_secondary"_s, + "env.db_idx256_lowerbound"_s, + "env.db_idx256_upperbound"_s, + "env.db_idx256_end"_s, + "env.db_idx256_next"_s, + "env.db_idx256_previous"_s, + "env.assert_recover_key"_s, + "env.recover_key"_s, + "env.assert_sha256"_s, + "env.assert_sha1"_s, + "env.assert_sha512"_s, + "env.assert_ripemd160"_s, + "env.sha1"_s, + "env.sha256"_s, + "env.sha512"_s, + "env.ripemd160"_s, + "env.check_transaction_authorization"_s, + "env.check_permission_authorization"_s, + "env.get_permission_last_used"_s, + "env.get_account_creation_time"_s, + "env.current_time"_s, + "env.publication_time"_s, + "env.abort"_s, + "env.eosio_assert"_s, + "env.eosio_assert_message"_s, + "env.eosio_assert_code"_s, + "env.eosio_exit"_s, + "env.read_action_data"_s, + "env.action_data_size"_s, + "env.current_receiver"_s, + "env.require_recipient"_s, + "env.require_auth"_s, + "env.require_auth2"_s, + "env.has_auth"_s, + "env.is_account"_s, + "env.prints"_s, + "env.prints_l"_s, + "env.printi"_s, + "env.printui"_s, + "env.printi128"_s, + "env.printui128"_s, + "env.printsf"_s, + "env.printdf"_s, + "env.printqf"_s, + "env.printn"_s, + "env.printhex"_s, + "env.read_transaction"_s, + "env.transaction_size"_s, + "env.get_transaction_id"_s, + "env.get_action_sequence"_s, + "env.has_contract"_s, + "env.get_contract_code"_s, + "env.expiration"_s, + "env.tapos_block_prefix"_s, + "env.tapos_block_num"_s, + "env.get_action"_s, + "env.send_inline"_s, + "env.send_context_free_inline"_s, + "env.send_deferred"_s, + "env.cancel_deferred"_s, + "env.get_context_free_data"_s, + "env.memcpy"_s, + "env.memmove"_s, + "env.memcmp"_s, + "env.memset"_s, + "eosio_injection._eosio_f32_add"_s, + "eosio_injection._eosio_f32_sub"_s, + "eosio_injection._eosio_f32_mul"_s, + "eosio_injection._eosio_f32_div"_s, + "eosio_injection._eosio_f32_min"_s, + "eosio_injection._eosio_f32_max"_s, + "eosio_injection._eosio_f32_copysign"_s, + "eosio_injection._eosio_f32_abs"_s, + "eosio_injection._eosio_f32_neg"_s, + "eosio_injection._eosio_f32_sqrt"_s, + "eosio_injection._eosio_f32_ceil"_s, + "eosio_injection._eosio_f32_floor"_s, + "eosio_injection._eosio_f32_trunc"_s, + "eosio_injection._eosio_f32_nearest"_s, + "eosio_injection._eosio_f32_eq"_s, + "eosio_injection._eosio_f32_ne"_s, + "eosio_injection._eosio_f32_lt"_s, + "eosio_injection._eosio_f32_le"_s, + "eosio_injection._eosio_f32_gt"_s, + "eosio_injection._eosio_f32_ge"_s, + "eosio_injection._eosio_f64_add"_s, + "eosio_injection._eosio_f64_sub"_s, + "eosio_injection._eosio_f64_mul"_s, + "eosio_injection._eosio_f64_div"_s, + "eosio_injection._eosio_f64_min"_s, + "eosio_injection._eosio_f64_max"_s, + "eosio_injection._eosio_f64_copysign"_s, + "eosio_injection._eosio_f64_abs"_s, + "eosio_injection._eosio_f64_neg"_s, + "eosio_injection._eosio_f64_sqrt"_s, + "eosio_injection._eosio_f64_ceil"_s, + "eosio_injection._eosio_f64_floor"_s, + "eosio_injection._eosio_f64_trunc"_s, + "eosio_injection._eosio_f64_nearest"_s, + "eosio_injection._eosio_f64_eq"_s, + "eosio_injection._eosio_f64_ne"_s, + "eosio_injection._eosio_f64_lt"_s, + "eosio_injection._eosio_f64_le"_s, + "eosio_injection._eosio_f64_gt"_s, + "eosio_injection._eosio_f64_ge"_s, + "eosio_injection._eosio_f32_promote"_s, + "eosio_injection._eosio_f64_demote"_s, + "eosio_injection._eosio_f32_trunc_i32s"_s, + "eosio_injection._eosio_f64_trunc_i32s"_s, + "eosio_injection._eosio_f32_trunc_i32u"_s, + "eosio_injection._eosio_f64_trunc_i32u"_s, + "eosio_injection._eosio_f32_trunc_i64s"_s, + "eosio_injection._eosio_f64_trunc_i64s"_s, + "eosio_injection._eosio_f32_trunc_i64u"_s, + "eosio_injection._eosio_f64_trunc_i64u"_s, + "eosio_injection._eosio_i32_to_f32"_s, + "eosio_injection._eosio_i64_to_f32"_s, + "eosio_injection._eosio_ui32_to_f32"_s, + "eosio_injection._eosio_ui64_to_f32"_s, + "eosio_injection._eosio_i32_to_f64"_s, + "eosio_injection._eosio_i64_to_f64"_s, + "eosio_injection._eosio_ui32_to_f64"_s, + "eosio_injection._eosio_ui64_to_f64"_s, + "env.bpsig_action_time_seed"_s +); + +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_helpers.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_helpers.hpp new file mode 100644 index 00000000000..fcb58537311 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_helpers.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include + +#include + +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +class wrapped_fd { + public: + wrapped_fd() : _inuse(false) {} + wrapped_fd(int fd) : _inuse(true), _fd(fd) {} + wrapped_fd(const wrapped_fd&) = delete; + wrapped_fd& operator=(const wrapped_fd&) = delete; + wrapped_fd(wrapped_fd&& other) : _inuse(other._inuse), _fd(other._fd) {other._inuse = false;} + wrapped_fd& operator=(wrapped_fd&& other) { + if(_inuse) + close(_fd); + _inuse = other._inuse; + _fd = other._fd; + other._inuse = false; + return *this; + } + + operator int() const { + FC_ASSERT(_inuse, "trying to get the value of a not-in-use wrappedfd"); + return _fd; + } + + int release() { + _inuse = false; + return _fd; + } + + ~wrapped_fd() { + if(_inuse) + close(_fd); + } + + private: + bool _inuse = false;; + int _fd; +}; + +std::tuple> read_message_with_fds(boost::asio::local::datagram_protocol::socket& s); +std::tuple> read_message_with_fds(int fd); +bool write_message_with_fds(boost::asio::local::datagram_protocol::socket& s, const eosvmoc_message& message, const std::vector& fds = std::vector()); +bool write_message_with_fds(int fd_to_send_to, const eosvmoc_message& message, const std::vector& fds = std::vector()); + +template +wrapped_fd memfd_for_bytearray(const T& bytes) { + int fd = syscall(SYS_memfd_create, "eosvmoc_code", MFD_CLOEXEC); + FC_ASSERT(fd >= 0, "Failed to create memfd"); + FC_ASSERT(ftruncate(fd, bytes.size()) == 0, "failed to grow memfd"); + if(bytes.size()) { + uint8_t* b = (uint8_t*)mmap(nullptr, bytes.size(), PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + FC_ASSERT(b != MAP_FAILED, "failed to mmap memfd"); + memcpy(b, bytes.data(), bytes.size()); + munmap(b, bytes.size()); + } + return wrapped_fd(fd); +} + +std::vector vector_for_memfd(const wrapped_fd& memfd); +}}} \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp new file mode 100644 index 00000000000..b38c3bce64b --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +struct initialize_message { + //Two sent fds: 1) communication socket for this instance 2) the cache file +}; + +struct initalize_response_message { + fc::optional error_message; //no error message? everything groovy +}; + +struct code_tuple { + eosio::chain::digest_type code_id; + uint8_t vm_version; + bool operator==(const code_tuple& o) const {return o.code_id == code_id && o.vm_version == vm_version;} +}; + +struct compile_wasm_message { + code_tuple code; + //Two sent fd: 1) communication socket for result, 2) the wasm to compile +}; + +struct evict_wasms_message { + std::vector codes; +}; + +struct code_compilation_result_message { + eosvmoc_optional_offset_or_import_t start; + unsigned apply_offset; + int starting_memory_pages; + unsigned initdata_prologue_size; + //Two sent fds: 1) wasm code, 2) initial memory snapshot +}; + + +struct compilation_result_unknownfailure {}; +struct compilation_result_toofull {}; + +using wasm_compilation_result = fc::static_variant; + +struct wasm_compilation_result_message { + code_tuple code; + wasm_compilation_result result; + size_t cache_free_bytes; +}; + +using eosvmoc_message = fc::static_variant; +}}} + +FC_REFLECT(eosio::chain::eosvmoc::initialize_message, ) +FC_REFLECT(eosio::chain::eosvmoc::initalize_response_message, (error_message)) +FC_REFLECT(eosio::chain::eosvmoc::code_tuple, (code_id)(vm_version)) +FC_REFLECT(eosio::chain::eosvmoc::compile_wasm_message, (code)) +FC_REFLECT(eosio::chain::eosvmoc::evict_wasms_message, (codes)) +FC_REFLECT(eosio::chain::eosvmoc::code_compilation_result_message, (start)(apply_offset)(starting_memory_pages)(initdata_prologue_size)) +FC_REFLECT(eosio::chain::eosvmoc::compilation_result_unknownfailure, ) +FC_REFLECT(eosio::chain::eosvmoc::compilation_result_toofull, ) +FC_REFLECT(eosio::chain::eosvmoc::wasm_compilation_result_message, (code)(result)(cache_free_bytes)) \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp new file mode 100644 index 00000000000..509e183bd80 --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/memory.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +class memory { + static constexpr uint64_t wasm_memory_size = eosio::chain::wasm_constraints::maximum_linear_memory; + static constexpr uint64_t intrinsic_count = boost::hana::length(intrinsic_table); + //warning: changing the following 3 params will invalidate existing PIC + static constexpr uint64_t mutable_global_size = 8u * eosio::chain::wasm_constraints::maximum_mutable_globals/4u; + static constexpr uint64_t table_size = 16u * eosio::chain::wasm_constraints::maximum_table_elements; + static constexpr size_t wcb_allowance = 512u; + static_assert(sizeof(control_block) <= wcb_allowance, "EOS VM OC memory doesn't set aside enough memory for control block"); + + //round up the prologue to multiple of 4K page + static constexpr uint64_t memory_prologue_size = ((memory::wcb_allowance + mutable_global_size + table_size + intrinsic_count*UINT64_C(8))+UINT64_C(4095))/UINT64_C(4096)*UINT64_C(4096); + //prologue + 33MB + 4GB fault buffer + 4096 addtional buffer for safety + static constexpr uint64_t total_memory_per_slice = memory_prologue_size + wasm_memory_size + UINT64_C(0x100000000) + UINT64_C(4096); + + static constexpr uint64_t number_slices = wasm_memory_size/(64u*1024u)+1u; + + public: + memory(); + ~memory(); + + uint8_t* const zero_page_memory_base() const { return zeropage_base; } + uint8_t* const full_page_memory_base() const { return fullpage_base; } + + control_block* const get_control_block() const { return reinterpret_cast(zeropage_base - cb_offset);} + + //these two are really only inteded for SEGV handling + uint8_t* const start_of_memory_slices() const { return mapbase; } + size_t size_of_memory_slice_mapping() const { return mapsize; } + + //to obtain memory protected for n wasm-pages, use the pointer computed from: + // zero_page_memory_base()+stride*n + static constexpr size_t stride = total_memory_per_slice; + + //offsets to various interesting things in the memory + static constexpr uintptr_t linear_memory = 0; + static constexpr uintptr_t cb_offset = wcb_allowance + mutable_global_size + table_size; + static constexpr uintptr_t first_intrinsic_offset = cb_offset + 8u; + + static_assert(-cb_offset == EOS_VM_OC_CONTROL_BLOCK_OFFSET, "EOS VM OC control block offset has slid out of place somehow"); + static_assert(stride == EOS_VM_OC_MEMORY_STRIDE, "EOS VM OC memory stride has slid out of place somehow"); + + private: + uint8_t* mapbase; + uint64_t mapsize; + + uint8_t* zeropage_base; + uint8_t* fullpage_base; +}; + +}}} + +#define OFFSET_OF_CONTROL_BLOCK_MEMBER(M) (-(int)eosio::chain::eosvmoc::memory::cb_offset + (int)offsetof(eosio::chain::eosvmoc::control_block, M)) +#define OFFSET_OF_FIRST_INTRINSIC ((int)-eosio::chain::eosvmoc::memory::first_intrinsic_offset) \ No newline at end of file diff --git a/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp b/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp new file mode 100644 index 00000000000..5119a5c386f --- /dev/null +++ b/libraries/chain/include/eosio/chain/webassembly/eos-vm.hpp @@ -0,0 +1,150 @@ +#pragma once + +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) + +#include +#include +#include +#include +#include + +//eos-vm includes +#include + +// eosio specific specializations +namespace eosio { namespace vm { + + template<> + struct wasm_type_converter { + static auto from_wasm(uint64_t val) { + return eosio::chain::name{val}; + } + static auto to_wasm(eosio::chain::name val) { + return val.to_uint64_t(); + } + }; + + template + struct wasm_type_converter : linear_memory_access { + auto from_wasm(void* val) { + validate_ptr(val, 1); + return eosio::vm::aligned_ptr_wrapper{val}; + } + }; + + template<> + struct wasm_type_converter : linear_memory_access { + void* to_wasm(char* val) { + validate_ptr(val, 1); + return val; + } + }; + + template + struct wasm_type_converter : linear_memory_access { + auto from_wasm(uint32_t val) { + EOS_VM_ASSERT( val != 0, wasm_memory_exception, "references cannot be created for null pointers" ); + void* ptr = get_ptr(val); + validate_ptr(ptr, 1); + return eosio::vm::aligned_ref_wrapper{ptr}; + } + }; + + template + struct wasm_type_converter> : linear_memory_access { + auto from_wasm(void* ptr, uint32_t size) { + validate_ptr(ptr, size); + return aligned_array_wrapper(ptr, size); + } + }; + + template<> + struct wasm_type_converter> : linear_memory_access { + auto from_wasm(void* ptr, uint32_t size) { + validate_ptr(ptr, size); + return eosio::chain::array_ptr((char*)ptr); + } + // memcpy/memmove + auto from_wasm(void* ptr, eosio::chain::array_ptr /*src*/, uint32_t size) { + validate_ptr(ptr, size); + return eosio::chain::array_ptr((char*)ptr); + } + // memset + auto from_wasm(void* ptr, int /*val*/, uint32_t size) { + validate_ptr(ptr, size); + return eosio::chain::array_ptr((char*)ptr); + } + }; + + template<> + struct wasm_type_converter> : linear_memory_access { + auto from_wasm(void* ptr, uint32_t size) { + validate_ptr(ptr, size); + return eosio::chain::array_ptr((char*)ptr); + } + // memcmp + auto from_wasm(void* ptr, eosio::chain::array_ptr /*src*/, uint32_t size) { + validate_ptr(ptr, size); + return eosio::chain::array_ptr((char*)ptr); + } + }; + + template + struct construct_derived { + static auto &value(Ctx& ctx) { return ctx.trx_context; } + }; + + template <> + struct construct_derived { + static auto &value(eosio::chain::apply_context& ctx) { return ctx; } + }; + + template<> + struct wasm_type_converter : linear_memory_access { + auto from_wasm(void* ptr) { + validate_c_str(ptr); + return eosio::chain::null_terminated_ptr{ static_cast(ptr) }; + } + }; + +}} // ns eosio::vm + +namespace eosio { namespace chain { namespace webassembly { namespace eos_vm_runtime { + +using namespace fc; +using namespace eosio::vm; +using namespace eosio::chain::webassembly::common; + +template +class eos_vm_runtime : public eosio::chain::wasm_runtime_interface { + public: + eos_vm_runtime(); + bool inject_module(IR::Module&) override; + std::unique_ptr instantiate_module(const char* code_bytes, size_t code_size, std::vector, + const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) override; + + void immediately_exit_currently_running_module() override; + + private: + // todo: managing this will get more complicated with sync calls; + // immediately_exit_currently_running_module() should probably + // move from wasm_runtime_interface to wasm_instantiated_module_interface. + backend* _bkend = nullptr; // non owning pointer to allow for immediate exit + + template + friend class eos_vm_instantiated_module; +}; + +} } } }// eosio::chain::webassembly::wabt_runtime + +#define __EOS_VM_INTRINSIC_NAME(LBL, SUF) LBL##SUF +#define _EOS_VM_INTRINSIC_NAME(LBL, SUF) __INTRINSIC_NAME(LBL, SUF) + +#define _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) \ + eosio::vm::registered_function _EOS_VM_INTRINSIC_NAME(__eos_vm_intrinsic_fn, __COUNTER__)(std::string(MOD), std::string(NAME)); + +#else + +#define _REGISTER_EOS_VM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) + +#endif diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index 1b1320286fc..a89251f293c 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -25,10 +25,15 @@ #include #include +#if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) +#include +#endif + namespace eosio { namespace chain { using namespace webassembly::common; - wasm_interface::wasm_interface(vm_type vm, const chainbase::database& d) : my( new wasm_interface_impl(vm, d) ) {} + wasm_interface::wasm_interface(vm_type vm, bool eosvmoc_tierup, const chainbase::database& d, const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config) + : my( new wasm_interface_impl(vm, eosvmoc_tierup, d, data_dir, eosvmoc_config) ) {} wasm_interface::~wasm_interface() {} @@ -69,6 +74,26 @@ namespace eosio { namespace chain { } void wasm_interface::apply( const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version, apply_context& context ) { +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + if(my->eosvmoc) { + const chain::eosvmoc::code_descriptor* cd = nullptr; + try { + cd = my->eosvmoc->cc.get_descriptor_for_code(code_hash, vm_version); + } + catch(...) { + //swallow errors here, if EOS VM OC has gone in to the weeds we shouldn't bail: continue to try and run baseline + //In the future, consider moving bits of EOS VM that can fire exceptions and such out of this call path + static bool once_is_enough; + if(!once_is_enough) + elog("EOS VM OC has encountered an unexpected failure"); + once_is_enough = true; + } + if(cd) { + my->eosvmoc->exec.execute(*cd, my->eosvmoc->mem, context); + return; + } + } +#endif my->get_instantiated_module(code_hash, vm_type, vm_version, context.trx_context)->apply(context); } @@ -2100,6 +2125,12 @@ std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime) { runtime = eosio::chain::wasm_interface::vm_type::wavm; else if (s == "wabt") runtime = eosio::chain::wasm_interface::vm_type::wabt; + else if (s == "eos-vm") + runtime = eosio::chain::wasm_interface::vm_type::eos_vm; + else if (s == "eos-vm-jit") + runtime = eosio::chain::wasm_interface::vm_type::eos_vm_jit; + else if (s == "eos-vm-oc") + runtime = eosio::chain::wasm_interface::vm_type::eos_vm_oc; else in.setstate(std::ios_base::failbit); return in; diff --git a/libraries/chain/webassembly/eos-vm-oc.cpp b/libraries/chain/webassembly/eos-vm-oc.cpp new file mode 100644 index 00000000000..3db4399f133 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc.cpp @@ -0,0 +1,54 @@ +#include +#include +#include +#include +#include + +#include +#include + +namespace eosio { namespace chain { namespace webassembly { namespace eosvmoc { + +class eosvmoc_instantiated_module : public wasm_instantiated_module_interface { + public: + eosvmoc_instantiated_module(const digest_type& code_hash, const uint8_t& vm_version, eosvmoc_runtime& wr) : + _code_hash(code_hash), + _vm_version(vm_version), + _eosvmoc_runtime(wr) + { + + } + + ~eosvmoc_instantiated_module() { + _eosvmoc_runtime.cc.free_code(_code_hash, _vm_version); + } + + void apply(apply_context& context) override { + const code_descriptor* const cd = _eosvmoc_runtime.cc.get_descriptor_for_code_sync(_code_hash, _vm_version); + EOS_ASSERT(cd, wasm_execution_error, "EOS VM OC instantiation failed"); + + _eosvmoc_runtime.exec.execute(*cd, _eosvmoc_runtime.mem, context); + } + + const digest_type _code_hash; + const uint8_t _vm_version; + eosvmoc_runtime& _eosvmoc_runtime; +}; + +eosvmoc_runtime::eosvmoc_runtime(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) + : cc(data_dir, eosvmoc_config, db), exec(cc) { +} + +eosvmoc_runtime::~eosvmoc_runtime() { +} + +std::unique_ptr eosvmoc_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory, + const digest_type& code_hash, const uint8_t& vm_type, const uint8_t& vm_version) { + + return std::make_unique(code_hash, vm_type, *this); +} + +//never called. EOS VM OC overrides eosio_exit to its own implementation +void eosvmoc_runtime::immediately_exit_currently_running_module() {} + +}}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/About WAVM b/libraries/chain/webassembly/eos-vm-oc/About WAVM new file mode 100644 index 00000000000..a948eb0736d --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/About WAVM @@ -0,0 +1,26 @@ +The EOS VM Optimized Compiler was created in part based on WAVM +https://github.com/WebAssembly/wasm-jit-prototype +subject the following: +Copyright (c) 2019, Andrew Scheidecker +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are +permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of +conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. +* Neither the name of WAVM nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp b/libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp new file mode 100644 index 00000000000..d52a2b94ab8 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/LLVMEmitIR.cpp @@ -0,0 +1,1320 @@ +/* +The EOS VM Optimized Compiler was created in part based on WAVM +https://github.com/WebAssembly/wasm-jit-prototype +subject the following: + +Copyright (c) 2016-2019, Andrew Scheidecker +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of WAVM nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "LLVMJIT.h" +#include "llvm/ADT/SmallVector.h" +#include "IR/Operators.h" +#include "IR/OperatorPrinter.h" +#include "llvm/Support/raw_ostream.h" + +#include "llvm/Analysis/Passes.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolSize.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/Transforms/InstCombine/InstCombine.h" +#include "llvm/Transforms/Utils.h" + +#include +#include + +#define ENABLE_LOGGING 0 +#define ENABLE_FUNCTION_ENTER_EXIT_HOOKS 0 + +using namespace IR; + +namespace eosio { namespace chain { namespace eosvmoc { +namespace LLVMJIT +{ + static std::string getExternalFunctionName(Uptr functionDefIndex) + { + return "wasmFunc" + std::to_string(functionDefIndex); + } + + bool getFunctionIndexFromExternalName(const char* externalName,Uptr& outFunctionDefIndex) + { + const char wasmFuncPrefix[] = "wasmFunc"; + const Uptr numPrefixChars = sizeof(wasmFuncPrefix) - 1; + if(!strncmp(externalName,wasmFuncPrefix,numPrefixChars)) + { + char* numberEnd = nullptr; + U64 functionDefIndex64 = std::strtoull(externalName + numPrefixChars,&numberEnd,10); + if(functionDefIndex64 > UINTPTR_MAX) { return false; } + outFunctionDefIndex = Uptr(functionDefIndex64); + return true; + } + else { return false; } + } + + llvm::Value* CreateInBoundsGEPWAR(llvm::IRBuilder<>& irBuilder, llvm::Value* Ptr, llvm::Value* v1, llvm::Value* v2 = nullptr); + + llvm::LLVMContext context; + llvm::Type* llvmResultTypes[(Uptr)ResultType::num]; + + llvm::Type* llvmI8Type; + llvm::Type* llvmI16Type; + llvm::Type* llvmI32Type; + llvm::Type* llvmI64Type; + llvm::Type* llvmF32Type; + llvm::Type* llvmF64Type; + llvm::Type* llvmVoidType; + llvm::Type* llvmBoolType; + llvm::Type* llvmI8PtrType; + + llvm::Constant* typedZeroConstants[(Uptr)ValueType::num]; + + // Converts a WebAssembly type to a LLVM type. + inline llvm::Type* asLLVMType(ValueType type) { return llvmResultTypes[(Uptr)asResultType(type)]; } + inline llvm::Type* asLLVMType(ResultType type) { return llvmResultTypes[(Uptr)type]; } + + // Converts a WebAssembly function type to a LLVM type. + inline llvm::FunctionType* asLLVMType(const FunctionType* functionType) + { + auto llvmArgTypes = (llvm::Type**)alloca(sizeof(llvm::Type*) * functionType->parameters.size()); + for(Uptr argIndex = 0;argIndex < functionType->parameters.size();++argIndex) + { + llvmArgTypes[argIndex] = asLLVMType(functionType->parameters[argIndex]); + } + auto llvmResultType = asLLVMType(functionType->ret); + return llvm::FunctionType::get(llvmResultType,llvm::ArrayRef(llvmArgTypes,functionType->parameters.size()),false); + } + + // Overloaded functions that compile a literal value to a LLVM constant of the right type. + inline llvm::ConstantInt* emitLiteral(U32 value) { return (llvm::ConstantInt*)llvm::ConstantInt::get(llvmI32Type,llvm::APInt(32,(U64)value,false)); } + inline llvm::ConstantInt* emitLiteral(I32 value) { return (llvm::ConstantInt*)llvm::ConstantInt::get(llvmI32Type,llvm::APInt(32,(I64)value,false)); } + inline llvm::ConstantInt* emitLiteral(U64 value) { return (llvm::ConstantInt*)llvm::ConstantInt::get(llvmI64Type,llvm::APInt(64,value,false)); } + inline llvm::ConstantInt* emitLiteral(I64 value) { return (llvm::ConstantInt*)llvm::ConstantInt::get(llvmI64Type,llvm::APInt(64,value,false)); } + inline llvm::Constant* emitLiteral(F32 value) { return llvm::ConstantFP::get(context,llvm::APFloat(value)); } + inline llvm::Constant* emitLiteral(F64 value) { return llvm::ConstantFP::get(context,llvm::APFloat(value)); } + inline llvm::Constant* emitLiteral(bool value) { return llvm::ConstantInt::get(llvmBoolType,llvm::APInt(1,value ? 1 : 0,false)); } + inline llvm::Constant* emitLiteralPointer(const void* pointer,llvm::Type* type) + { + auto pointerInt = llvm::APInt(sizeof(Uptr) == 8 ? 64 : 32,reinterpret_cast(pointer)); + return llvm::Constant::getIntegerValue(type,pointerInt); + } + + // The LLVM IR for a module. + struct EmitModuleContext + { + const Module& module; + + llvm::Module* llvmModule; + std::vector functionDefs; + std::vector importedFunctionOffsets; + std::vector globals; + llvm::Constant* defaultTablePointer; + llvm::Constant* defaultTableMaxElementIndex; + llvm::Constant* defaultMemoryBase; + llvm::Constant* depthCounter; + bool tableOnlyHasDefinedFuncs = true; + + llvm::MDNode* likelyFalseBranchWeights; + llvm::MDNode* likelyTrueBranchWeights; + + EmitModuleContext(const Module& inModule) + : module(inModule) + , llvmModule(new llvm::Module("",context)) + { + auto zeroAsMetadata = llvm::ConstantAsMetadata::get(emitLiteral(I32(0))); + auto i32MaxAsMetadata = llvm::ConstantAsMetadata::get(emitLiteral(I32(INT32_MAX))); + likelyFalseBranchWeights = llvm::MDTuple::getDistinct(context,{llvm::MDString::get(context,"branch_weights"),zeroAsMetadata,i32MaxAsMetadata}); + likelyTrueBranchWeights = llvm::MDTuple::getDistinct(context,{llvm::MDString::get(context,"branch_weights"),i32MaxAsMetadata,zeroAsMetadata}); + + } + llvm::Module* emit(); + }; + + // The context used by functions involved in JITing a single AST function. + struct EmitFunctionContext + { + typedef void Result; + + EmitModuleContext& moduleContext; + const Module& module; + const FunctionDef& functionDef; + const FunctionType* functionType; + llvm::Function* llvmFunction; + llvm::IRBuilder<> irBuilder; + + std::vector localPointers; + + llvm::DISubprogram* diFunction; + + // Information about an in-scope control structure. + struct ControlContext + { + enum class Type : U8 + { + function, + block, + ifThen, + ifElse, + loop + }; + + Type type; + llvm::BasicBlock* endBlock; + llvm::PHINode* endPHI; + llvm::BasicBlock* elseBlock; + ResultType resultType; + Uptr outerStackSize; + Uptr outerBranchTargetStackSize; + bool isReachable; + bool isElseReachable; + }; + + struct BranchTarget + { + ResultType argumentType; + llvm::BasicBlock* block; + llvm::PHINode* phi; + }; + + std::vector controlStack; + std::vector branchTargetStack; + std::vector stack; + + EmitFunctionContext(EmitModuleContext& inEmitModuleContext,const Module& inModule,const FunctionDef& inFunctionDef,llvm::Function* inLLVMFunction) + : moduleContext(inEmitModuleContext) + , module(inModule) + , functionDef(inFunctionDef) + , functionType(inModule.types[inFunctionDef.type.index]) + , llvmFunction(inLLVMFunction) + , irBuilder(context) + {} + + void emit(); + + // Operand stack manipulation + llvm::Value* pop() + { + WAVM_ASSERT_THROW(stack.size() - (controlStack.size() ? controlStack.back().outerStackSize : 0) >= 1); + llvm::Value* result = stack.back(); + stack.pop_back(); + return result; + } + + void popMultiple(llvm::Value** outValues,Uptr num) + { + WAVM_ASSERT_THROW(stack.size() - (controlStack.size() ? controlStack.back().outerStackSize : 0) >= num); + std::copy(stack.end() - num,stack.end(),outValues); + stack.resize(stack.size() - num); + } + + llvm::Value* getTopValue() const + { + return stack.back(); + } + + void push(llvm::Value* value) + { + stack.push_back(value); + } + + // Creates a PHI node for the argument of branches to a basic block. + llvm::PHINode* createPHI(llvm::BasicBlock* basicBlock,ResultType type) + { + if(type == ResultType::none) { return nullptr; } + else + { + auto originalBlock = irBuilder.GetInsertBlock(); + irBuilder.SetInsertPoint(basicBlock); + auto phi = irBuilder.CreatePHI(asLLVMType(type),2); + if(originalBlock) { irBuilder.SetInsertPoint(originalBlock); } + return phi; + } + } + + // Debug logging. + void logOperator(const std::string& operatorDescription) + { + if(ENABLE_LOGGING) + { + std::string controlStackString; + for(Uptr stackIndex = 0;stackIndex < controlStack.size();++stackIndex) + { + if(!controlStack[stackIndex].isReachable) { controlStackString += "("; } + switch(controlStack[stackIndex].type) + { + case ControlContext::Type::function: controlStackString += "F"; break; + case ControlContext::Type::block: controlStackString += "B"; break; + case ControlContext::Type::ifThen: controlStackString += "T"; break; + case ControlContext::Type::ifElse: controlStackString += "E"; break; + case ControlContext::Type::loop: controlStackString += "L"; break; + default: Errors::unreachable(); + }; + if(!controlStack[stackIndex].isReachable) { controlStackString += ")"; } + } + + std::string stackString; + const Uptr stackBase = controlStack.size() == 0 ? 0 : controlStack.back().outerStackSize; + for(Uptr stackIndex = 0;stackIndex < stack.size();++stackIndex) + { + if(stackIndex == stackBase) { stackString += "| "; } + { + llvm::raw_string_ostream stackTypeStream(stackString); + stack[stackIndex]->getType()->print(stackTypeStream,true); + } + stackString += " "; + } + if(stack.size() == stackBase) { stackString += "|"; } + + //Log::printf(Log::Category::debug,"%-50s %-50s %-50s\n",controlStackString.c_str(),operatorDescription.c_str(),stackString.c_str()); + } + } + + // Coerces an I32 value to an I1, and vice-versa. + llvm::Value* coerceI32ToBool(llvm::Value* i32Value) + { + return irBuilder.CreateICmpNE(i32Value,typedZeroConstants[(Uptr)ValueType::i32]); + } + llvm::Value* coerceBoolToI32(llvm::Value* boolValue) + { + return irBuilder.CreateZExt(boolValue,llvmI32Type); + } + + // Bounds checks and converts a memory operation I32 address operand to a LLVM pointer. + llvm::Value* coerceByteIndexToPointer(llvm::Value* byteIndex,U32 offset,llvm::Type* memoryType) + { + + // On a 64 bit runtime, if the address is 32-bits, zext it to 64-bits. + // This is crucial for security, as LLVM will otherwise implicitly sign extend it to 64-bits in the GEP below, + // interpreting it as a signed offset and allowing access to memory outside the sandboxed memory range. + // There are no 'far addresses' in a 32 bit runtime. + byteIndex = irBuilder.CreateZExt(byteIndex,llvmI64Type); + + // Add the offset to the byte index. + if(offset) + { + byteIndex = irBuilder.CreateAdd(byteIndex,irBuilder.CreateZExt(emitLiteral(offset),llvmI64Type)); + } + + // Cast the pointer to the appropriate type. + auto bytePointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultMemoryBase, byteIndex); + + return irBuilder.CreatePointerCast(bytePointer,memoryType->getPointerTo(256)); + } + + // Traps a divide-by-zero + void trapDivideByZero(ValueType type,llvm::Value* divisor) + { + emitConditionalTrapIntrinsic( + irBuilder.CreateICmpEQ(divisor,typedZeroConstants[(Uptr)type]), + "eosvmoc_internal.div0_or_overflow",FunctionType::get(),{}); + } + + // Traps on (x / 0) or (INT_MIN / -1). + void trapDivideByZeroOrIntegerOverflow(ValueType type,llvm::Value* left,llvm::Value* right) + { + emitConditionalTrapIntrinsic( + irBuilder.CreateOr( + irBuilder.CreateAnd( + irBuilder.CreateICmpEQ(left,type == ValueType::i32 ? emitLiteral((U32)INT32_MIN) : emitLiteral((U64)INT64_MIN)), + irBuilder.CreateICmpEQ(right,type == ValueType::i32 ? emitLiteral((U32)-1) : emitLiteral((U64)-1)) + ), + irBuilder.CreateICmpEQ(right,typedZeroConstants[(Uptr)type]) + ), + "eosvmoc_internal.div0_or_overflow",FunctionType::get(),{}); + } + + llvm::Value* getLLVMIntrinsic(const std::initializer_list& argTypes,llvm::Intrinsic::ID id) + { + return llvm::Intrinsic::getDeclaration(moduleContext.llvmModule,id,llvm::ArrayRef(argTypes.begin(),argTypes.end())); + } + + // Emits a call to a WAVM intrinsic function. + llvm::Value* emitRuntimeIntrinsic(const char* intrinsicName,const FunctionType* intrinsicType,const std::initializer_list& args) + { + const eosio::chain::eosvmoc::intrinsic_entry& ie = eosio::chain::eosvmoc::get_intrinsic_map().at(intrinsicName); + llvm::Value* ic = irBuilder.CreateLoad( emitLiteralPointer((void*)(OFFSET_OF_FIRST_INTRINSIC-ie.ordinal*8), llvmI64Type->getPointerTo(256)) ); + llvm::Value* itp = irBuilder.CreateIntToPtr(ic, asLLVMType(ie.type)->getPointerTo()); + return irBuilder.CreateCall(itp,llvm::ArrayRef(args.begin(),args.end())); + } + + // A helper function to emit a conditional call to a non-returning intrinsic function. + void emitConditionalTrapIntrinsic(llvm::Value* booleanCondition,const char* intrinsicName,const FunctionType* intrinsicType,const std::initializer_list& args) + { + auto trueBlock = llvm::BasicBlock::Create(context,llvm::Twine(intrinsicName) + "Trap",llvmFunction); + auto endBlock = llvm::BasicBlock::Create(context,llvm::Twine(intrinsicName) + "Skip",llvmFunction); + + irBuilder.CreateCondBr(booleanCondition,trueBlock,endBlock,moduleContext.likelyFalseBranchWeights); + + irBuilder.SetInsertPoint(trueBlock); + emitRuntimeIntrinsic(intrinsicName,intrinsicType,args); + irBuilder.CreateUnreachable(); + + irBuilder.SetInsertPoint(endBlock); + } + + // + // Misc operators + // + + void nop(NoImm) {} + void unknown(Opcode opcode) { Errors::unreachable(); } + + // + // Control structure operators + // + + void pushControlStack( + ControlContext::Type type, + ResultType resultType, + llvm::BasicBlock* endBlock, + llvm::PHINode* endPHI, + llvm::BasicBlock* elseBlock = nullptr + ) + { + // The unreachable operator filtering should filter out any opcodes that call pushControlStack. + if(controlStack.size()) { errorUnless(controlStack.back().isReachable); } + + controlStack.push_back({type,endBlock,endPHI,elseBlock,resultType,stack.size(),branchTargetStack.size(),true,true}); + } + + void pushBranchTarget(ResultType branchArgumentType,llvm::BasicBlock* branchTargetBlock,llvm::PHINode* branchTargetPHI) + { + branchTargetStack.push_back({branchArgumentType,branchTargetBlock,branchTargetPHI}); + } + + void block(ControlStructureImm imm) + { + // Create an end block+phi for the block result. + auto endBlock = llvm::BasicBlock::Create(context,"blockEnd",llvmFunction); + auto endPHI = createPHI(endBlock,imm.resultType); + + // Push a control context that ends at the end block/phi. + pushControlStack(ControlContext::Type::block,imm.resultType,endBlock,endPHI); + + // Push a branch target for the end block/phi. + pushBranchTarget(imm.resultType,endBlock,endPHI); + } + void loop(ControlStructureImm imm) + { + // Create a loop block, and an end block+phi for the loop result. + auto loopBodyBlock = llvm::BasicBlock::Create(context,"loopBody",llvmFunction); + auto endBlock = llvm::BasicBlock::Create(context,"loopEnd",llvmFunction); + auto endPHI = createPHI(endBlock,imm.resultType); + + // Branch to the loop body and switch the IR builder to emit there. + irBuilder.CreateBr(loopBodyBlock); + irBuilder.SetInsertPoint(loopBodyBlock); + + // Push a control context that ends at the end block/phi. + pushControlStack(ControlContext::Type::loop,imm.resultType,endBlock,endPHI); + + // Push a branch target for the loop body start. + pushBranchTarget(ResultType::none,loopBodyBlock,nullptr); + } + void if_(ControlStructureImm imm) + { + // Create a then block and else block for the if, and an end block+phi for the if result. + auto thenBlock = llvm::BasicBlock::Create(context,"ifThen",llvmFunction); + auto elseBlock = llvm::BasicBlock::Create(context,"ifElse",llvmFunction); + auto endBlock = llvm::BasicBlock::Create(context,"ifElseEnd",llvmFunction); + auto endPHI = createPHI(endBlock,imm.resultType); + + // Pop the if condition from the operand stack. + auto condition = pop(); + irBuilder.CreateCondBr(coerceI32ToBool(condition),thenBlock,elseBlock); + + // Switch the IR builder to emit the then block. + irBuilder.SetInsertPoint(thenBlock); + + // Push an ifThen control context that ultimately ends at the end block/phi, but may + // be terminated by an else operator that changes the control context to the else block. + pushControlStack(ControlContext::Type::ifThen,imm.resultType,endBlock,endPHI,elseBlock); + + // Push a branch target for the if end. + pushBranchTarget(imm.resultType,endBlock,endPHI); + + } + void else_(NoImm imm) + { + WAVM_ASSERT_THROW(controlStack.size()); + ControlContext& currentContext = controlStack.back(); + + if(currentContext.isReachable) + { + // If the control context expects a result, take it from the operand stack and add it to the + // control context's end PHI. + if(currentContext.resultType != ResultType::none) + { + llvm::Value* result = pop(); + currentContext.endPHI->addIncoming(result,irBuilder.GetInsertBlock()); + } + + // Branch to the control context's end. + irBuilder.CreateBr(currentContext.endBlock); + } + WAVM_ASSERT_THROW(stack.size() == currentContext.outerStackSize); + + // Switch the IR emitter to the else block. + WAVM_ASSERT_THROW(currentContext.elseBlock); + WAVM_ASSERT_THROW(currentContext.type == ControlContext::Type::ifThen); + currentContext.elseBlock->moveAfter(irBuilder.GetInsertBlock()); + irBuilder.SetInsertPoint(currentContext.elseBlock); + + // Change the top of the control stack to an else clause. + currentContext.type = ControlContext::Type::ifElse; + currentContext.isReachable = currentContext.isElseReachable; + currentContext.elseBlock = nullptr; + } + void end(NoImm) + { + WAVM_ASSERT_THROW(controlStack.size()); + ControlContext& currentContext = controlStack.back(); + + if(currentContext.isReachable) + { + // If the control context yields a result, take the top of the operand stack and + // add it to the control context's end PHI. + if(currentContext.resultType != ResultType::none) + { + llvm::Value* result = pop(); + currentContext.endPHI->addIncoming(result,irBuilder.GetInsertBlock()); + } + + // Branch to the control context's end. + irBuilder.CreateBr(currentContext.endBlock); + } + WAVM_ASSERT_THROW(stack.size() == currentContext.outerStackSize); + + if(currentContext.elseBlock) + { + // If this is the end of an if without an else clause, create a dummy else clause. + currentContext.elseBlock->moveAfter(irBuilder.GetInsertBlock()); + irBuilder.SetInsertPoint(currentContext.elseBlock); + irBuilder.CreateBr(currentContext.endBlock); + } + + // Switch the IR emitter to the end block. + currentContext.endBlock->moveAfter(irBuilder.GetInsertBlock()); + irBuilder.SetInsertPoint(currentContext.endBlock); + + if(currentContext.endPHI) + { + // If the control context yields a result, take the PHI that merges all the control flow + // to the end and push it onto the operand stack. + if(currentContext.endPHI->getNumIncomingValues()) { push(currentContext.endPHI); } + else + { + // If there weren't any incoming values for the end PHI, remove it and push a dummy value. + currentContext.endPHI->eraseFromParent(); + WAVM_ASSERT_THROW(currentContext.resultType != ResultType::none); + push(typedZeroConstants[(Uptr)asValueType(currentContext.resultType)]); + } + } + + // Pop and branch targets introduced by this control context. + WAVM_ASSERT_THROW(currentContext.outerBranchTargetStackSize <= branchTargetStack.size()); + branchTargetStack.resize(currentContext.outerBranchTargetStackSize); + + // Pop this control context. + controlStack.pop_back(); + } + + // + // Control flow operators + // + + BranchTarget& getBranchTargetByDepth(Uptr depth) + { + WAVM_ASSERT_THROW(depth < branchTargetStack.size()); + return branchTargetStack[branchTargetStack.size() - depth - 1]; + } + + // This is called after unconditional control flow to indicate that operators following it are unreachable until the control stack is popped. + void enterUnreachable() + { + // Unwind the operand stack to the outer control context. + WAVM_ASSERT_THROW(controlStack.back().outerStackSize <= stack.size()); + stack.resize(controlStack.back().outerStackSize); + + // Mark the current control context as unreachable: this will cause the outer loop to stop dispatching operators to us + // until an else/end for the current control context is reached. + controlStack.back().isReachable = false; + } + + void br_if(BranchImm imm) + { + // Pop the condition from operand stack. + auto condition = pop(); + + BranchTarget& target = getBranchTargetByDepth(imm.targetDepth); + if(target.argumentType != ResultType::none) + { + // Use the stack top as the branch argument (don't pop it) and add it to the target phi's incoming values. + llvm::Value* argument = getTopValue(); + target.phi->addIncoming(argument,irBuilder.GetInsertBlock()); + } + + // Create a new basic block for the case where the branch is not taken. + auto falseBlock = llvm::BasicBlock::Create(context,"br_ifElse",llvmFunction); + + // Emit a conditional branch to either the falseBlock or the target block. + irBuilder.CreateCondBr(coerceI32ToBool(condition),target.block,falseBlock); + + // Resume emitting instructions in the falseBlock. + irBuilder.SetInsertPoint(falseBlock); + } + + void br(BranchImm imm) + { + BranchTarget& target = getBranchTargetByDepth(imm.targetDepth); + if(target.argumentType != ResultType::none) + { + // Pop the branch argument from the stack and add it to the target phi's incoming values. + llvm::Value* argument = pop(); + target.phi->addIncoming(argument,irBuilder.GetInsertBlock()); + } + + // Branch to the target block. + irBuilder.CreateBr(target.block); + + enterUnreachable(); + } + void br_table(BranchTableImm imm) + { + // Pop the table index from the operand stack. + auto index = pop(); + + // Look up the default branch target, and assume its argument type applies to all targets. + // (this is guaranteed by the validator) + BranchTarget& defaultTarget = getBranchTargetByDepth(imm.defaultTargetDepth); + const ResultType argumentType = defaultTarget.argumentType; + llvm::Value* argument = nullptr; + if(argumentType != ResultType::none) + { + // Pop the branch argument from the stack and add it to the default target phi's incoming values. + argument = pop(); + defaultTarget.phi->addIncoming(argument,irBuilder.GetInsertBlock()); + } + + // Create a LLVM switch instruction. + WAVM_ASSERT_THROW(imm.branchTableIndex < functionDef.branchTables.size()); + const std::vector& targetDepths = functionDef.branchTables[imm.branchTableIndex]; + auto llvmSwitch = irBuilder.CreateSwitch(index,defaultTarget.block,(unsigned int)targetDepths.size()); + + for(Uptr targetIndex = 0;targetIndex < targetDepths.size();++targetIndex) + { + BranchTarget& target = getBranchTargetByDepth(targetDepths[targetIndex]); + + // Add this target to the switch instruction. + llvmSwitch->addCase(emitLiteral((U32)targetIndex),target.block); + + if(argumentType != ResultType::none) + { + // If this is the first case in the table for this branch target, add the branch argument to + // the target phi's incoming values. + target.phi->addIncoming(argument,irBuilder.GetInsertBlock()); + } + } + + enterUnreachable(); + } + void return_(NoImm) + { + if(functionType->ret != ResultType::none) + { + // Pop the return value from the stack and add it to the return phi's incoming values. + llvm::Value* result = pop(); + controlStack[0].endPHI->addIncoming(result,irBuilder.GetInsertBlock()); + } + + // Branch to the return block. + irBuilder.CreateBr(controlStack[0].endBlock); + + enterUnreachable(); + } + + void unreachable(NoImm) + { + // Call an intrinsic that causes a trap, and insert the LLVM unreachable terminator. + emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(),{}); + irBuilder.CreateUnreachable(); + + enterUnreachable(); + } + + // + // Polymorphic operators + // + + void drop(NoImm) { stack.pop_back(); } + + void select(NoImm) + { + auto condition = pop(); + auto falseValue = pop(); + auto trueValue = pop(); + push(irBuilder.CreateSelect(coerceI32ToBool(condition),trueValue,falseValue)); + } + + // + // Call operators + // + + void call(CallImm imm) + { + // Map the callee function index to either an imported function pointer or a function in this module. + llvm::Value* callee; + const FunctionType* calleeType; + bool isExit = false; + if(imm.functionIndex < moduleContext.importedFunctionOffsets.size()) + { + calleeType = module.types[module.functions.imports[imm.functionIndex].type.index]; + llvm::Value* ic = irBuilder.CreateLoad( emitLiteralPointer((void*)(OFFSET_OF_FIRST_INTRINSIC-moduleContext.importedFunctionOffsets[imm.functionIndex]*8), llvmI64Type->getPointerTo(256)) ); + callee = irBuilder.CreateIntToPtr(ic, asLLVMType(calleeType)->getPointerTo()); + isExit = module.functions.imports[imm.functionIndex].moduleName == "env" && module.functions.imports[imm.functionIndex].exportName == "eosio_exit"; + } + else + { + const Uptr calleeIndex = imm.functionIndex - moduleContext.importedFunctionOffsets.size(); + callee = moduleContext.functionDefs[calleeIndex]; + calleeType = module.types[module.functions.defs[calleeIndex].type.index]; + } + + // Pop the call arguments from the operand stack. + auto llvmArgs = (llvm::Value**)alloca(sizeof(llvm::Value*) * calleeType->parameters.size()); + popMultiple(llvmArgs,calleeType->parameters.size()); + + // Call the function. + auto result = irBuilder.CreateCall(callee,llvm::ArrayRef(llvmArgs,calleeType->parameters.size())); + if(isExit) { + irBuilder.CreateUnreachable(); + enterUnreachable(); + } + + // Push the result on the operand stack. + if(calleeType->ret != ResultType::none) { push(result); } + } + void call_indirect(CallIndirectImm imm) + { + WAVM_ASSERT_THROW(imm.type.index < module.types.size()); + + auto calleeType = module.types[imm.type.index]; + auto functionPointerType = asLLVMType(calleeType)->getPointerTo(); + + // Compile the function index. + auto tableElementIndex = pop(); + + // Compile the call arguments. + auto llvmArgs = (llvm::Value**)alloca(sizeof(llvm::Value*) * calleeType->parameters.size()); + popMultiple(llvmArgs,calleeType->parameters.size()); + + // Zero extend the function index to the pointer size. + auto functionIndexZExt = irBuilder.CreateZExt(tableElementIndex, llvmI64Type); + + // If the function index is larger than the function table size, trap. + emitConditionalTrapIntrinsic( + irBuilder.CreateICmpUGE(functionIndexZExt,moduleContext.defaultTableMaxElementIndex), + "eosvmoc_internal.indirect_call_oob",FunctionType::get(),{}); + + // Load the type for this table entry. + auto functionTypePointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)0)); + auto functionTypePointer = irBuilder.CreateLoad(functionTypePointerPointer); + auto llvmCalleeType = emitLiteralPointer(calleeType,llvmI8PtrType); + + // If the function type doesn't match, trap. + emitConditionalTrapIntrinsic( + irBuilder.CreateICmpNE(llvmCalleeType,functionTypePointer), + "eosvmoc_internal.indirect_call_mismatch", + FunctionType::get(),{} + ); + + //If the WASM only contains table elements to function definitions internal to the wasm, we can take a + // simple and approach + if(moduleContext.tableOnlyHasDefinedFuncs) { + auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)1)); + auto functionInfo = irBuilder.CreateLoad(functionPointerPointer); //offset of code + llvm::Value* running_code_start = irBuilder.CreateLoad(emitLiteralPointer((void*)OFFSET_OF_CONTROL_BLOCK_MEMBER(running_code_base), llvmI64Type->getPointerTo(256))); + llvm::Value* offset_from_start = irBuilder.CreateAdd(running_code_start, functionInfo); + llvm::Value* ptr_cast = irBuilder.CreateIntToPtr(offset_from_start, functionPointerType); + auto result = irBuilder.CreateCall(ptr_cast,llvm::ArrayRef(llvmArgs,calleeType->parameters.size())); + + // Push the result on the operand stack. + if(calleeType->ret != ResultType::none) { push(result); } + } + else { + auto functionPointerPointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultTablePointer, functionIndexZExt, emitLiteral((U32)1)); + auto functionInfo = irBuilder.CreateLoad(functionPointerPointer); //offset of code + + auto is_intrnsic = irBuilder.CreateICmpSLT(functionInfo, typedZeroConstants[(Uptr)ValueType::i64]); + + llvm::BasicBlock* is_intrinsic_block = llvm::BasicBlock::Create(context, "isintrinsic", llvmFunction); + llvm::BasicBlock* is_code_offset_block = llvm::BasicBlock::Create(context, "isoffset"); + llvm::BasicBlock* continuation_block = llvm::BasicBlock::Create(context, "cont"); + + irBuilder.CreateCondBr(is_intrnsic, is_intrinsic_block, is_code_offset_block, moduleContext.likelyFalseBranchWeights); + + irBuilder.SetInsertPoint(is_intrinsic_block); + llvm::Value* intrinsic_start = emitLiteral((I64)OFFSET_OF_FIRST_INTRINSIC); + llvm::Value* intrinsic_offset = irBuilder.CreateAdd(intrinsic_start, functionInfo); + llvm::Value* intrinsic_ptr = irBuilder.CreateLoad(irBuilder.CreateIntToPtr(intrinsic_offset, llvmI64Type->getPointerTo(256))); + irBuilder.CreateBr(continuation_block); + + llvmFunction->getBasicBlockList().push_back(is_code_offset_block); + irBuilder.SetInsertPoint(is_code_offset_block); + llvm::Value* running_code_start = irBuilder.CreateLoad(emitLiteralPointer((void*)OFFSET_OF_CONTROL_BLOCK_MEMBER(running_code_base), llvmI64Type->getPointerTo(256))); + llvm::Value* offset_from_start = irBuilder.CreateAdd(running_code_start, functionInfo); + irBuilder.CreateBr(continuation_block); + + llvmFunction->getBasicBlockList().push_back(continuation_block); + irBuilder.SetInsertPoint(continuation_block); + + llvm::PHINode* PN = irBuilder.CreatePHI(llvmI64Type, 2, "indirecttypephi"); + PN->addIncoming(intrinsic_ptr, is_intrinsic_block); + PN->addIncoming(offset_from_start, is_code_offset_block); + + llvm::Value* ptr_cast = irBuilder.CreateIntToPtr(PN, functionPointerType); + auto result = irBuilder.CreateCall(ptr_cast,llvm::ArrayRef(llvmArgs,calleeType->parameters.size())); + + // Push the result on the operand stack. + if(calleeType->ret != ResultType::none) { push(result); } + } + } + + // + // Local/global operators + // + + void get_local(GetOrSetVariableImm imm) + { + WAVM_ASSERT_THROW(imm.variableIndex < localPointers.size()); + push(irBuilder.CreateLoad(localPointers[imm.variableIndex])); + } + void set_local(GetOrSetVariableImm imm) + { + WAVM_ASSERT_THROW(imm.variableIndex < localPointers.size()); + auto value = irBuilder.CreateBitCast(pop(),localPointers[imm.variableIndex]->getType()->getPointerElementType()); + irBuilder.CreateStore(value,localPointers[imm.variableIndex]); + } + void tee_local(GetOrSetVariableImm imm) + { + WAVM_ASSERT_THROW(imm.variableIndex < localPointers.size()); + auto value = irBuilder.CreateBitCast(getTopValue(),localPointers[imm.variableIndex]->getType()->getPointerElementType()); + irBuilder.CreateStore(value,localPointers[imm.variableIndex]); + } + + void get_global(GetOrSetVariableImm imm) + { + WAVM_ASSERT_THROW(imm.variableIndex < moduleContext.globals.size()); + if(moduleContext.globals[imm.variableIndex]->getType()->isPointerTy()) + push(irBuilder.CreateLoad(moduleContext.globals[imm.variableIndex])); + else + push(moduleContext.globals[imm.variableIndex]); + } + void set_global(GetOrSetVariableImm imm) + { + WAVM_ASSERT_THROW(imm.variableIndex < moduleContext.globals.size()); + auto value = irBuilder.CreateBitCast(pop(),moduleContext.globals[imm.variableIndex]->getType()->getPointerElementType()); + irBuilder.CreateStore(value,moduleContext.globals[imm.variableIndex]); + } + + // + // Memory size operators + // These just call out to wavmIntrinsics.growMemory/currentMemory, passing a pointer to the default memory for the module. + // + + void grow_memory(MemoryImm) + { + auto deltaNumPages = pop(); + auto maxMemoryPages = emitLiteral((U32)moduleContext.module.memories.defs[0].type.size.max); + auto previousNumPages = emitRuntimeIntrinsic( + "eosvmoc_internal.grow_memory", + FunctionType::get(ResultType::i32,{ValueType::i32,ValueType::i32}), + {deltaNumPages,maxMemoryPages}); + push(previousNumPages); + } + void current_memory(MemoryImm) + { + auto offset = emitLiteral((I32)OFFSET_OF_CONTROL_BLOCK_MEMBER(current_linear_memory_pages)); + auto bytePointer = CreateInBoundsGEPWAR(irBuilder, moduleContext.defaultMemoryBase, offset); + auto ptrTo = irBuilder.CreatePointerCast(bytePointer,llvmI32Type->getPointerTo(256)); + auto load = irBuilder.CreateLoad(ptrTo); + push(load); + } + + // + // Constant operators + // + + #define EMIT_CONST(typeId,nativeType) void typeId##_const(LiteralImm imm) { push(emitLiteral(imm.value)); } + EMIT_CONST(i32,I32) EMIT_CONST(i64,I64) + EMIT_CONST(f32,F32) EMIT_CONST(f64,F64) + + // + // Load/store operators + // + + #define EMIT_LOAD_OP(valueTypeId,name,llvmMemoryType,naturalAlignmentLog2,conversionOp) \ + void valueTypeId##_##name(LoadOrStoreImm imm) \ + { \ + auto byteIndex = pop(); \ + auto pointer = coerceByteIndexToPointer(byteIndex,imm.offset,llvmMemoryType); \ + auto load = irBuilder.CreateLoad(pointer); \ + load->setAlignment(1); \ + load->setVolatile(true); \ + push(conversionOp(load,asLLVMType(ValueType::valueTypeId))); \ + } + #define EMIT_STORE_OP(valueTypeId,name,llvmMemoryType,naturalAlignmentLog2,conversionOp) \ + void valueTypeId##_##name(LoadOrStoreImm imm) \ + { \ + auto value = pop(); \ + auto byteIndex = pop(); \ + auto pointer = coerceByteIndexToPointer(byteIndex,imm.offset,llvmMemoryType); \ + auto memoryValue = conversionOp(value,llvmMemoryType); \ + auto store = irBuilder.CreateStore(memoryValue,pointer); \ + store->setVolatile(true); \ + store->setAlignment(1); \ + } + + llvm::Value* identityConversion(llvm::Value* value,llvm::Type* type) { return value; } + + EMIT_LOAD_OP(i32,load8_s,llvmI8Type,0,irBuilder.CreateSExt) EMIT_LOAD_OP(i32,load8_u,llvmI8Type,0,irBuilder.CreateZExt) + EMIT_LOAD_OP(i32,load16_s,llvmI16Type,1,irBuilder.CreateSExt) EMIT_LOAD_OP(i32,load16_u,llvmI16Type,1,irBuilder.CreateZExt) + EMIT_LOAD_OP(i64,load8_s,llvmI8Type,0,irBuilder.CreateSExt) EMIT_LOAD_OP(i64,load8_u,llvmI8Type,0,irBuilder.CreateZExt) + EMIT_LOAD_OP(i64,load16_s,llvmI16Type,1,irBuilder.CreateSExt) EMIT_LOAD_OP(i64,load16_u,llvmI16Type,1,irBuilder.CreateZExt) + EMIT_LOAD_OP(i64,load32_s,llvmI32Type,2,irBuilder.CreateSExt) EMIT_LOAD_OP(i64,load32_u,llvmI32Type,2,irBuilder.CreateZExt) + + EMIT_LOAD_OP(i32,load,llvmI32Type,2,identityConversion) EMIT_LOAD_OP(i64,load,llvmI64Type,3,identityConversion) + EMIT_LOAD_OP(f32,load,llvmF32Type,2,identityConversion) EMIT_LOAD_OP(f64,load,llvmF64Type,3,identityConversion) + + EMIT_STORE_OP(i32,store8,llvmI8Type,0,irBuilder.CreateTrunc) EMIT_STORE_OP(i64,store8,llvmI8Type,0,irBuilder.CreateTrunc) + EMIT_STORE_OP(i32,store16,llvmI16Type,1,irBuilder.CreateTrunc) EMIT_STORE_OP(i64,store16,llvmI16Type,1,irBuilder.CreateTrunc) + EMIT_STORE_OP(i32,store,llvmI32Type,2,irBuilder.CreateTrunc) EMIT_STORE_OP(i64,store32,llvmI32Type,2,irBuilder.CreateTrunc) + EMIT_STORE_OP(i64,store,llvmI64Type,3,identityConversion) + EMIT_STORE_OP(f32,store,llvmF32Type,2,identityConversion) EMIT_STORE_OP(f64,store,llvmF64Type,3,identityConversion) + + // + // Numeric operator macros + // + + #define EMIT_BINARY_OP(typeId,name,emitCode) void typeId##_##name(NoImm) \ + { \ + const ValueType type = ValueType::typeId; SUPPRESS_UNUSED(type); \ + auto right = pop(); \ + auto left = pop(); \ + push(emitCode); \ + } + #define EMIT_INT_BINARY_OP(name,emitCode) EMIT_BINARY_OP(i32,name,emitCode) EMIT_BINARY_OP(i64,name,emitCode) + #define EMIT_FP_BINARY_OP(name,emitCode) EMIT_BINARY_OP(f32,name,emitCode) EMIT_BINARY_OP(f64,name,emitCode) + + #define EMIT_UNARY_OP(typeId,name,emitCode) void typeId##_##name(NoImm) \ + { \ + const ValueType type = ValueType::typeId; SUPPRESS_UNUSED(type); \ + auto operand = pop(); \ + push(emitCode); \ + } + #define EMIT_INT_UNARY_OP(name,emitCode) EMIT_UNARY_OP(i32,name,emitCode) EMIT_UNARY_OP(i64,name,emitCode) + #define EMIT_FP_UNARY_OP(name,emitCode) EMIT_UNARY_OP(f32,name,emitCode) EMIT_UNARY_OP(f64,name,emitCode) + + // + // Int operators + // + + llvm::Value* emitSRem(ValueType type,llvm::Value* left,llvm::Value* right) + { + // Trap if the dividend is zero. + trapDivideByZero(type,right); + + // LLVM's srem has undefined behavior where WebAssembly's rem_s defines that it should not trap if the corresponding + // division would overflow a signed integer. To avoid this case, we just branch around the srem if the INT_MAX%-1 case + // that overflows is detected. + auto preOverflowBlock = irBuilder.GetInsertBlock(); + auto noOverflowBlock = llvm::BasicBlock::Create(context,"sremNoOverflow",llvmFunction); + auto endBlock = llvm::BasicBlock::Create(context,"sremEnd",llvmFunction); + auto noOverflow = irBuilder.CreateOr( + irBuilder.CreateICmpNE(left,type == ValueType::i32 ? emitLiteral((U32)INT32_MIN) : emitLiteral((U64)INT64_MIN)), + irBuilder.CreateICmpNE(right,type == ValueType::i32 ? emitLiteral((U32)-1) : emitLiteral((U64)-1)) + ); + irBuilder.CreateCondBr(noOverflow,noOverflowBlock,endBlock,moduleContext.likelyTrueBranchWeights); + + irBuilder.SetInsertPoint(noOverflowBlock); + auto noOverflowValue = irBuilder.CreateSRem(left,right); + irBuilder.CreateBr(endBlock); + + irBuilder.SetInsertPoint(endBlock); + auto phi = irBuilder.CreatePHI(asLLVMType(type),2); + phi->addIncoming(typedZeroConstants[(Uptr)type],preOverflowBlock); + phi->addIncoming(noOverflowValue,noOverflowBlock); + return phi; + } + + llvm::Value* emitShiftCountMask(ValueType type,llvm::Value* shiftCount) + { + // LLVM's shifts have undefined behavior where WebAssembly specifies that the shift count will wrap numbers + // grather than the bit count of the operands. This matches x86's native shift instructions, but explicitly mask + // the shift count anyway to support other platforms, and ensure the optimizer doesn't take advantage of the UB. + auto bitsMinusOne = irBuilder.CreateZExt(emitLiteral((U8)(getTypeBitWidth(type) - 1)),asLLVMType(type)); + return irBuilder.CreateAnd(shiftCount,bitsMinusOne); + } + + llvm::Value* emitRotl(ValueType type,llvm::Value* left,llvm::Value* right) + { + auto bitWidthMinusRight = irBuilder.CreateSub( + irBuilder.CreateZExt(emitLiteral(getTypeBitWidth(type)),asLLVMType(type)), + right + ); + return irBuilder.CreateOr( + irBuilder.CreateShl(left,emitShiftCountMask(type,right)), + irBuilder.CreateLShr(left,emitShiftCountMask(type,bitWidthMinusRight)) + ); + } + + llvm::Value* emitRotr(ValueType type,llvm::Value* left,llvm::Value* right) + { + auto bitWidthMinusRight = irBuilder.CreateSub( + irBuilder.CreateZExt(emitLiteral(getTypeBitWidth(type)),asLLVMType(type)), + right + ); + return irBuilder.CreateOr( + irBuilder.CreateShl(left,emitShiftCountMask(type,bitWidthMinusRight)), + irBuilder.CreateLShr(left,emitShiftCountMask(type,right)) + ); + } + + EMIT_INT_BINARY_OP(add,irBuilder.CreateAdd(left,right)) + EMIT_INT_BINARY_OP(sub,irBuilder.CreateSub(left,right)) + EMIT_INT_BINARY_OP(mul,irBuilder.CreateMul(left,right)) + EMIT_INT_BINARY_OP(and,irBuilder.CreateAnd(left,right)) + EMIT_INT_BINARY_OP(or,irBuilder.CreateOr(left,right)) + EMIT_INT_BINARY_OP(xor,irBuilder.CreateXor(left,right)) + EMIT_INT_BINARY_OP(rotr,emitRotr(type,left,right)) + EMIT_INT_BINARY_OP(rotl,emitRotl(type,left,right)) + + // Divides use trapDivideByZero to avoid the undefined behavior in LLVM's division instructions. + EMIT_INT_BINARY_OP(div_s, (trapDivideByZeroOrIntegerOverflow(type,left,right), irBuilder.CreateSDiv(left,right)) ) + EMIT_INT_BINARY_OP(rem_s, emitSRem(type,left,right) ) + EMIT_INT_BINARY_OP(div_u, (trapDivideByZero(type,right), irBuilder.CreateUDiv(left,right)) ) + EMIT_INT_BINARY_OP(rem_u, (trapDivideByZero(type,right), irBuilder.CreateURem(left,right)) ) + + // Explicitly mask the shift amount operand to the word size to avoid LLVM's undefined behavior. + EMIT_INT_BINARY_OP(shl,irBuilder.CreateShl(left,emitShiftCountMask(type,right))) + EMIT_INT_BINARY_OP(shr_s,irBuilder.CreateAShr(left,emitShiftCountMask(type,right))) + EMIT_INT_BINARY_OP(shr_u,irBuilder.CreateLShr(left,emitShiftCountMask(type,right))) + + static llvm::Value* getNonConstantZero(llvm::IRBuilder<>& irBuilder, llvm::Constant* zero) { + llvm::Value* zeroAlloca = irBuilder.CreateAlloca(zero->getType(), nullptr, "nonConstantZero"); + irBuilder.CreateStore(zero, zeroAlloca); + return irBuilder.CreateLoad(zeroAlloca); + } + + #define EMIT_INT_COMPARE_OP(name, llvmSourceType, llvmDestType, valueType, emitCode) \ + void name(NoImm) { \ + auto right = irBuilder.CreateOr( \ + irBuilder.CreateBitCast(pop(), llvmSourceType), \ + irBuilder.CreateBitCast( \ + getNonConstantZero(irBuilder, typedZeroConstants[Uptr(valueType)]), \ + llvmSourceType)); \ + auto left = irBuilder.CreateBitCast(pop(), llvmSourceType); \ + push(coerceBoolToI32(emitCode)); \ + } + + #define EMIT_INT_COMPARE(name, emitCode) \ + EMIT_INT_COMPARE_OP(i32_##name, llvmI32Type, llvmI32Type, ValueType::i32, emitCode) \ + EMIT_INT_COMPARE_OP(i64_##name, llvmI64Type, llvmI32Type, ValueType::i64, emitCode) + +#if LLVM_VERSION_MAJOR < 9 + EMIT_INT_COMPARE(eq, irBuilder.CreateICmpEQ(left, right)) + EMIT_INT_COMPARE(ne, irBuilder.CreateICmpNE(left, right)) + EMIT_INT_COMPARE(lt_s, irBuilder.CreateICmpSLT(left, right)) + EMIT_INT_COMPARE(lt_u, irBuilder.CreateICmpULT(left, right)) + EMIT_INT_COMPARE(le_s, irBuilder.CreateICmpSLE(left, right)) + EMIT_INT_COMPARE(le_u, irBuilder.CreateICmpULE(left, right)) + EMIT_INT_COMPARE(gt_s, irBuilder.CreateICmpSGT(left, right)) + EMIT_INT_COMPARE(gt_u, irBuilder.CreateICmpUGT(left, right)) + EMIT_INT_COMPARE(ge_s, irBuilder.CreateICmpSGE(left, right)) + EMIT_INT_COMPARE(ge_u, irBuilder.CreateICmpUGE(left, right)) +#else + EMIT_INT_BINARY_OP(eq, coerceBoolToI32(irBuilder.CreateICmpEQ(left, right))) + EMIT_INT_BINARY_OP(ne, coerceBoolToI32(irBuilder.CreateICmpNE(left, right))) + EMIT_INT_BINARY_OP(lt_s, coerceBoolToI32(irBuilder.CreateICmpSLT(left, right))) + EMIT_INT_BINARY_OP(lt_u, coerceBoolToI32(irBuilder.CreateICmpULT(left, right))) + EMIT_INT_BINARY_OP(le_s, coerceBoolToI32(irBuilder.CreateICmpSLE(left, right))) + EMIT_INT_BINARY_OP(le_u, coerceBoolToI32(irBuilder.CreateICmpULE(left, right))) + EMIT_INT_BINARY_OP(gt_s, coerceBoolToI32(irBuilder.CreateICmpSGT(left, right))) + EMIT_INT_BINARY_OP(gt_u, coerceBoolToI32(irBuilder.CreateICmpUGT(left, right))) + EMIT_INT_BINARY_OP(ge_s, coerceBoolToI32(irBuilder.CreateICmpSGE(left, right))) + EMIT_INT_BINARY_OP(ge_u, coerceBoolToI32(irBuilder.CreateICmpUGE(left, right))) +#endif + + EMIT_INT_UNARY_OP(clz,irBuilder.CreateCall(getLLVMIntrinsic({operand->getType()},llvm::Intrinsic::ctlz),llvm::ArrayRef({operand,emitLiteral(false)}))) + EMIT_INT_UNARY_OP(ctz,irBuilder.CreateCall(getLLVMIntrinsic({operand->getType()},llvm::Intrinsic::cttz),llvm::ArrayRef({operand,emitLiteral(false)}))) + EMIT_INT_UNARY_OP(popcnt,irBuilder.CreateCall(getLLVMIntrinsic({operand->getType()},llvm::Intrinsic::ctpop),llvm::ArrayRef({operand}))) + EMIT_INT_UNARY_OP(eqz,coerceBoolToI32(irBuilder.CreateICmpEQ(operand,typedZeroConstants[(Uptr)type]))) + + // + // FP operators + // + + EMIT_FP_BINARY_OP(add,irBuilder.CreateFAdd(left,right)) + EMIT_FP_BINARY_OP(sub,irBuilder.CreateFSub(left,right)) + EMIT_FP_BINARY_OP(mul,irBuilder.CreateFMul(left,right)) + EMIT_FP_BINARY_OP(div,irBuilder.CreateFDiv(left,right)) + EMIT_FP_BINARY_OP(copysign,irBuilder.CreateCall(getLLVMIntrinsic({left->getType()},llvm::Intrinsic::copysign),llvm::ArrayRef({left,right}))) + + EMIT_FP_UNARY_OP(neg,irBuilder.CreateFNeg(operand)) + EMIT_FP_UNARY_OP(abs,irBuilder.CreateCall(getLLVMIntrinsic({operand->getType()},llvm::Intrinsic::fabs),llvm::ArrayRef({operand}))) + EMIT_FP_UNARY_OP(sqrt,irBuilder.CreateCall(getLLVMIntrinsic({operand->getType()},llvm::Intrinsic::sqrt),llvm::ArrayRef({operand}))) + + EMIT_FP_BINARY_OP(eq,coerceBoolToI32(irBuilder.CreateFCmpOEQ(left,right))) + EMIT_FP_BINARY_OP(ne,coerceBoolToI32(irBuilder.CreateFCmpUNE(left,right))) + EMIT_FP_BINARY_OP(lt,coerceBoolToI32(irBuilder.CreateFCmpOLT(left,right))) + EMIT_FP_BINARY_OP(le,coerceBoolToI32(irBuilder.CreateFCmpOLE(left,right))) + EMIT_FP_BINARY_OP(gt,coerceBoolToI32(irBuilder.CreateFCmpOGT(left,right))) + EMIT_FP_BINARY_OP(ge,coerceBoolToI32(irBuilder.CreateFCmpOGE(left,right))) + + EMIT_UNARY_OP(i32,wrap_i64,irBuilder.CreateTrunc(operand,llvmI32Type)) + EMIT_UNARY_OP(i64,extend_s_i32,irBuilder.CreateSExt(operand,llvmI64Type)) + EMIT_UNARY_OP(i64,extend_u_i32,irBuilder.CreateZExt(operand,llvmI64Type)) + + EMIT_FP_UNARY_OP(convert_s_i32,irBuilder.CreateSIToFP(operand,asLLVMType(type))) + EMIT_FP_UNARY_OP(convert_s_i64,irBuilder.CreateSIToFP(operand,asLLVMType(type))) + EMIT_FP_UNARY_OP(convert_u_i32,irBuilder.CreateUIToFP(operand,asLLVMType(type))) + EMIT_FP_UNARY_OP(convert_u_i64,irBuilder.CreateUIToFP(operand,asLLVMType(type))) + + EMIT_UNARY_OP(f32,demote_f64,irBuilder.CreateFPTrunc(operand,llvmF32Type)) + EMIT_UNARY_OP(f64,promote_f32,irBuilder.CreateFPExt(operand,llvmF64Type)) + EMIT_UNARY_OP(f32,reinterpret_i32,irBuilder.CreateBitCast(operand,llvmF32Type)) + EMIT_UNARY_OP(f64,reinterpret_i64,irBuilder.CreateBitCast(operand,llvmF64Type)) + EMIT_UNARY_OP(i32,reinterpret_f32,irBuilder.CreateBitCast(operand,llvmI32Type)) + EMIT_UNARY_OP(i64,reinterpret_f64,irBuilder.CreateBitCast(operand,llvmI64Type)) + + // These operations don't match LLVM's semantics exactly, so just call out to C++ implementations. + EMIT_FP_BINARY_OP(min,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type,type}),{left,right})) + EMIT_FP_BINARY_OP(max,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type,type}),{left,right})) + EMIT_FP_UNARY_OP(ceil,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type}),{operand})) + EMIT_FP_UNARY_OP(floor,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type}),{operand})) + EMIT_FP_UNARY_OP(trunc,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type}),{operand})) + EMIT_FP_UNARY_OP(nearest,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{type}),{operand})) + EMIT_INT_UNARY_OP(trunc_s_f32,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{ValueType::f32}),{operand})) + EMIT_INT_UNARY_OP(trunc_s_f64,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{ValueType::f64}),{operand})) + EMIT_INT_UNARY_OP(trunc_u_f32,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{ValueType::f32}),{operand})) + EMIT_INT_UNARY_OP(trunc_u_f64,emitRuntimeIntrinsic("eosvmoc_internal.unreachable",FunctionType::get(asResultType(type),{ValueType::f64}),{operand})) + }; + + // A do-nothing visitor used to decode past unreachable operators (but supporting logging, and passing the end operator through). + struct UnreachableOpVisitor + { + typedef void Result; + + UnreachableOpVisitor(EmitFunctionContext& inContext): context(inContext), unreachableControlDepth(0) {} + #define VISIT_OP(opcode,name,nameString,Imm,...) void name(Imm imm) {} + ENUM_NONCONTROL_OPERATORS(VISIT_OP) + VISIT_OP(_,unknown,"unknown",Opcode) + #undef VISIT_OP + + // Keep track of control structure nesting level in unreachable code, so we know when we reach the end of the unreachable code. + void block(ControlStructureImm) { ++unreachableControlDepth; } + void loop(ControlStructureImm) { ++unreachableControlDepth; } + void if_(ControlStructureImm) { ++unreachableControlDepth; } + + // If an else or end opcode would signal an end to the unreachable code, then pass it through to the IR emitter. + void else_(NoImm imm) + { + if(!unreachableControlDepth) { context.else_(imm); } + } + void end(NoImm imm) + { + if(!unreachableControlDepth) { context.end(imm); } + else { --unreachableControlDepth; } + } + + private: + EmitFunctionContext& context; + Uptr unreachableControlDepth; + }; + + void EmitFunctionContext::emit() + { + // Create the return basic block, and push the root control context for the function. + auto returnBlock = llvm::BasicBlock::Create(context,"return",llvmFunction); + auto returnPHI = createPHI(returnBlock,functionType->ret); + pushControlStack(ControlContext::Type::function,functionType->ret,returnBlock,returnPHI); + pushBranchTarget(functionType->ret,returnBlock,returnPHI); + + // Create an initial basic block for the function. + auto entryBasicBlock = llvm::BasicBlock::Create(context,"entry",llvmFunction); + irBuilder.SetInsertPoint(entryBasicBlock); + + // Create and initialize allocas for all the locals and parameters. + auto llvmArgIt = llvmFunction->arg_begin(); + for(Uptr localIndex = 0;localIndex < functionType->parameters.size() + functionDef.nonParameterLocalTypes.size();++localIndex) + { + auto localType = localIndex < functionType->parameters.size() + ? functionType->parameters[localIndex] + : functionDef.nonParameterLocalTypes[localIndex - functionType->parameters.size()]; + auto localPointer = irBuilder.CreateAlloca(asLLVMType(localType),nullptr,""); + localPointers.push_back(localPointer); + + if(localIndex < functionType->parameters.size()) + { + // Copy the parameter value into the local that stores it. + irBuilder.CreateStore((llvm::Argument*)&(*llvmArgIt),localPointer); + ++llvmArgIt; + } + else + { + // Initialize non-parameter locals to zero. + irBuilder.CreateStore(typedZeroConstants[(Uptr)localType],localPointer); + } + } + + llvm::LoadInst* depth_loadinst; + llvm::StoreInst* depth_storeinst; + llvm::Value* depth = depth_loadinst = irBuilder.CreateLoad(moduleContext.depthCounter); + depth = irBuilder.CreateSub(depth, emitLiteral((I32)1)); + depth_storeinst = irBuilder.CreateStore(depth, moduleContext.depthCounter); + emitConditionalTrapIntrinsic(irBuilder.CreateICmpEQ(depth, emitLiteral((I32)0)), "eosvmoc_internal.depth_assert", FunctionType::get(), {}); + depth_loadinst->setVolatile(true); + depth_storeinst->setVolatile(true); + + // Decode the WebAssembly opcodes and emit LLVM IR for them. + OperatorDecoderStream decoder(functionDef.code); + UnreachableOpVisitor unreachableOpVisitor(*this); + OperatorPrinter operatorPrinter(module,functionDef); + while(decoder) + { + if(ENABLE_LOGGING) + { + logOperator(decoder.decodeOpWithoutConsume(operatorPrinter)); + } + WAVM_ASSERT_THROW(!controlStack.empty()); + + if(controlStack.back().isReachable) { decoder.decodeOp(*this); } + else { decoder.decodeOp(unreachableOpVisitor); } + }; + WAVM_ASSERT_THROW(irBuilder.GetInsertBlock() == returnBlock); + + depth = depth_loadinst = irBuilder.CreateLoad(moduleContext.depthCounter); + depth = irBuilder.CreateAdd(depth, emitLiteral((I32)1)); + depth_storeinst = irBuilder.CreateStore(depth, moduleContext.depthCounter); + depth_loadinst->setVolatile(true); + depth_storeinst->setVolatile(true); + + // Emit the function return. + if(functionType->ret == ResultType::none) { irBuilder.CreateRetVoid(); } + else { irBuilder.CreateRet(pop()); } + } + + llvm::Module* EmitModuleContext::emit() + { + defaultMemoryBase = emitLiteralPointer(0,llvmI8Type->getPointerTo(256)); + + depthCounter = emitLiteralPointer((void*)OFFSET_OF_CONTROL_BLOCK_MEMBER(current_call_depth_remaining), llvmI32Type->getPointerTo(256)); + + // Create LLVM pointer constants for the module's imported functions. + for(Uptr functionIndex = 0;functionIndex < module.functions.imports.size();++functionIndex) + { + const intrinsic_entry& ie =get_intrinsic_map().at(module.functions.imports[functionIndex].moduleName + "." + module.functions.imports[functionIndex].exportName); + importedFunctionOffsets.push_back(ie.ordinal); + } + + int current_prologue = -8; + + for(const GlobalDef& global : module.globals.defs) { + if(global.type.isMutable) { + globals.push_back(emitLiteralPointer((void*)current_prologue,asLLVMType(global.type.valueType)->getPointerTo(256))); + current_prologue -= 8; + } + else { + switch(global.type.valueType) { + case ValueType::i32: globals.push_back(emitLiteral(global.initializer.i32)); break; + case ValueType::i64: globals.push_back(emitLiteral(global.initializer.i64)); break; + case ValueType::f32: globals.push_back(emitLiteral(global.initializer.f32)); break; + case ValueType::f64: globals.push_back(emitLiteral(global.initializer.f64)); break; + default: break; //impossible + } + } + } + + if(module.tables.size()) { + current_prologue -= 8; //now pointing to LAST element + current_prologue -= 16*(module.tables.defs[0].type.size.min-1); //now pointing to FIRST element + auto tableElementType = llvm::StructType::get(context,{llvmI8PtrType, llvmI64Type}); + defaultTablePointer = emitLiteralPointer((void*)current_prologue,tableElementType->getPointerTo(256)); + defaultTableMaxElementIndex = emitLiteral((U64)module.tables.defs[0].type.size.min); + + for(const TableSegment& table_segment : module.tableSegments) + for(Uptr i = 0; i < table_segment.indices.size(); ++i) + if(table_segment.indices[i] < module.functions.imports.size()) + tableOnlyHasDefinedFuncs = false; + } + + // Create the LLVM functions. + functionDefs.resize(module.functions.defs.size()); + for(Uptr functionDefIndex = 0;functionDefIndex < module.functions.defs.size();++functionDefIndex) + { + auto llvmFunctionType = asLLVMType(module.types[module.functions.defs[functionDefIndex].type.index]); + auto externalName = getExternalFunctionName(functionDefIndex); + functionDefs[functionDefIndex] = llvm::Function::Create(llvmFunctionType,llvm::Function::ExternalLinkage,externalName,llvmModule); + } + + // Compile each function in the module. + for(Uptr functionDefIndex = 0;functionDefIndex < module.functions.defs.size();++functionDefIndex) + { EmitFunctionContext(*this,module,module.functions.defs[functionDefIndex],functionDefs[functionDefIndex]).emit(); } + + return llvmModule; + } + + llvm::Module* emitModule(const Module& module) + { + static bool inited; + if(!inited) { + llvmI8Type = llvm::Type::getInt8Ty(context); + llvmI16Type = llvm::Type::getInt16Ty(context); + llvmI32Type = llvm::Type::getInt32Ty(context); + llvmI64Type = llvm::Type::getInt64Ty(context); + llvmF32Type = llvm::Type::getFloatTy(context); + llvmF64Type = llvm::Type::getDoubleTy(context); + llvmVoidType = llvm::Type::getVoidTy(context); + llvmBoolType = llvm::Type::getInt1Ty(context); + llvmI8PtrType = llvmI8Type->getPointerTo(); + + llvmResultTypes[(Uptr)ResultType::none] = llvm::Type::getVoidTy(context); + llvmResultTypes[(Uptr)ResultType::i32] = llvmI32Type; + llvmResultTypes[(Uptr)ResultType::i64] = llvmI64Type; + llvmResultTypes[(Uptr)ResultType::f32] = llvmF32Type; + llvmResultTypes[(Uptr)ResultType::f64] = llvmF64Type; + + // Create zero constants of each type. + typedZeroConstants[(Uptr)ValueType::any] = nullptr; + typedZeroConstants[(Uptr)ValueType::i32] = emitLiteral((U32)0); + typedZeroConstants[(Uptr)ValueType::i64] = emitLiteral((U64)0); + typedZeroConstants[(Uptr)ValueType::f32] = emitLiteral((F32)0.0f); + typedZeroConstants[(Uptr)ValueType::f64] = emitLiteral((F64)0.0); + } + + return EmitModuleContext(module).emit(); + } +} +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp b/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp new file mode 100644 index 00000000000..d3fdb28a5d2 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.cpp @@ -0,0 +1,324 @@ +/* +The EOS VM Optimized Compiler was created in part based on WAVM +https://github.com/WebAssembly/wasm-jit-prototype +subject the following: + +Copyright (c) 2016-2019, Andrew Scheidecker +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of WAVM nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "LLVMJIT.h" + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/NullResolver.h" +#include "llvm/ExecutionEngine/Orc/Core.h" + +#include "llvm/Analysis/Passes.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolSize.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/Transforms/InstCombine/InstCombine.h" +#include "llvm/Transforms/Utils.h" +#include + +#include +#include +#include + +#include "llvm/Support/LEB128.h" + +#if LLVM_VERSION_MAJOR == 7 +namespace llvm { namespace orc { + using LegacyRTDyldObjectLinkingLayer = RTDyldObjectLinkingLayer; + template + using LegacyIRCompileLayer = IRCompileLayer; +}} +#endif + +#define DUMP_UNOPTIMIZED_MODULE 0 +#define VERIFY_MODULE 0 +#define DUMP_OPTIMIZED_MODULE 0 +#define PRINT_DISASSEMBLY 0 + +#if PRINT_DISASSEMBLY +#include "llvm-c/Disassembler.h" +static void disassembleFunction(U8* bytes,Uptr numBytes) +{ + LLVMDisasmContextRef disasmRef = LLVMCreateDisasm(llvm::sys::getProcessTriple().c_str(),nullptr,0,nullptr,nullptr); + + U8* nextByte = bytes; + Uptr numBytesRemaining = numBytes; + while(numBytesRemaining) + { + char instructionBuffer[256]; + const Uptr numInstructionBytes = LLVMDisasmInstruction( + disasmRef, + nextByte, + numBytesRemaining, + reinterpret_cast(nextByte), + instructionBuffer, + sizeof(instructionBuffer) + ); + if(numInstructionBytes == 0 || numInstructionBytes > numBytesRemaining) + break; + numBytesRemaining -= numInstructionBytes; + nextByte += numInstructionBytes; + + printf("\t\t%s\n",instructionBuffer); + }; + + LLVMDisasmDispose(disasmRef); +} +#endif + +namespace eosio { namespace chain { namespace eosvmoc { + +namespace LLVMJIT +{ + llvm::TargetMachine* targetMachine = nullptr; + + // Allocates memory for the LLVM object loader. + struct UnitMemoryManager : llvm::RTDyldMemoryManager + { + UnitMemoryManager() {} + virtual ~UnitMemoryManager() override + {} + + void registerEHFrames(U8* addr, U64 loadAddr,uintptr_t numBytes) override {} + void deregisterEHFrames() override {} + + virtual bool needsToReserveAllocationSpace() override { return true; } + virtual void reserveAllocationSpace(uintptr_t numCodeBytes,U32 codeAlignment,uintptr_t numReadOnlyBytes,U32 readOnlyAlignment,uintptr_t numReadWriteBytes,U32 readWriteAlignment) override { + code = std::make_unique>(numCodeBytes + numReadOnlyBytes + numReadWriteBytes); + ptr = code->data(); + } + virtual U8* allocateCodeSection(uintptr_t numBytes,U32 alignment,U32 sectionID,llvm::StringRef sectionName) override + { + return get_next_code_ptr(numBytes, alignment); + } + virtual U8* allocateDataSection(uintptr_t numBytes,U32 alignment,U32 sectionID,llvm::StringRef SectionName,bool isReadOnly) override + { + if(SectionName == ".eh_frame") { + dumpster.resize(numBytes); + return dumpster.data(); + } + if(SectionName == ".stack_sizes") { + return stack_sizes.emplace_back(numBytes).data(); + } + WAVM_ASSERT_THROW(isReadOnly); + + return get_next_code_ptr(numBytes, alignment); + } + + virtual bool finalizeMemory(std::string* ErrMsg = nullptr) override { + code->resize(ptr - code->data()); + return true; + } + + std::unique_ptr> code; + uint8_t* ptr; + + std::vector dumpster; + std::list> stack_sizes; + + U8* get_next_code_ptr(uintptr_t numBytes, U32 alignment) { + FC_ASSERT(alignment <= alignof(std::max_align_t), "alignment of section exceeds max_align_t"); + uintptr_t p = (uintptr_t)ptr; + p += alignment - 1LL; + p &= ~(alignment - 1LL); + uint8_t* this_section = (uint8_t*)p; + ptr = this_section + numBytes; + + return this_section; + } + + UnitMemoryManager(const UnitMemoryManager&) = delete; + void operator=(const UnitMemoryManager&) = delete; + }; + + // The JIT compilation unit for a WebAssembly module instance. + struct JITModule + { + JITModule() { + objectLayer = llvm::make_unique(ES,[this](llvm::orc::VModuleKey K) { + return llvm::orc::LegacyRTDyldObjectLinkingLayer::Resources{ + unitmemorymanager, std::make_shared() + }; + }, + [](llvm::orc::VModuleKey, const llvm::object::ObjectFile &Obj, const llvm::RuntimeDyld::LoadedObjectInfo &o) { + //nothing to do + }, + [this](llvm::orc::VModuleKey, const llvm::object::ObjectFile &Obj, const llvm::RuntimeDyld::LoadedObjectInfo &o) { + for(auto symbolSizePair : llvm::object::computeSymbolSizes(Obj)) { + auto symbol = symbolSizePair.first; + auto name = symbol.getName(); + auto address = symbol.getAddress(); + if(symbol.getType() && symbol.getType().get() == llvm::object::SymbolRef::ST_Function && name && address) { + Uptr loadedAddress = Uptr(*address); + auto symbolSection = symbol.getSection(); + if(symbolSection) + loadedAddress += (Uptr)o.getSectionLoadAddress(*symbolSection.get()); + Uptr functionDefIndex; + if(getFunctionIndexFromExternalName(name->data(),functionDefIndex)) + function_to_offsets[functionDefIndex] = loadedAddress-(uintptr_t)unitmemorymanager->code->data(); +#if PRINT_DISASSEMBLY + disassembleFunction((U8*)loadedAddress, symbolSizePair.second); +#endif + } + } + } + ); + objectLayer->setProcessAllSections(true); + compileLayer = llvm::make_unique(*objectLayer,llvm::orc::SimpleCompiler(*targetMachine)); + } + + void compile(llvm::Module* llvmModule); + + std::shared_ptr unitmemorymanager = std::make_shared(); + + std::map function_to_offsets; + std::vector final_pic_code; + + ~JITModule() + { + } + private: + typedef llvm::orc::LegacyIRCompileLayer CompileLayer; + + llvm::orc::ExecutionSession ES; + std::unique_ptr objectLayer; + std::unique_ptr compileLayer; + }; + + static Uptr printedModuleId = 0; + + void printModule(const llvm::Module* llvmModule,const char* filename) + { + std::error_code errorCode; + std::string augmentedFilename = std::string(filename) + std::to_string(printedModuleId++) + ".ll"; + llvm::raw_fd_ostream dumpFileStream(augmentedFilename,errorCode,llvm::sys::fs::OpenFlags::F_Text); + llvmModule->print(dumpFileStream,nullptr); + ///Log::printf(Log::Category::debug,"Dumped LLVM module to: %s\n",augmentedFilename.c_str()); + } + + void JITModule::compile(llvm::Module* llvmModule) + { + // Get a target machine object for this host, and set the module to use its data layout. + llvmModule->setDataLayout(targetMachine->createDataLayout()); + + // Verify the module. + if(DUMP_UNOPTIMIZED_MODULE) { printModule(llvmModule,"llvmDump"); } + if(VERIFY_MODULE) + { + std::string verifyOutputString; + llvm::raw_string_ostream verifyOutputStream(verifyOutputString); + if(llvm::verifyModule(*llvmModule,&verifyOutputStream)) + { Errors::fatalf("LLVM verification errors:\n%s\n",verifyOutputString.c_str()); } + ///Log::printf(Log::Category::debug,"Verified LLVM module\n"); + } + + auto fpm = new llvm::legacy::FunctionPassManager(llvmModule); + fpm->add(llvm::createPromoteMemoryToRegisterPass()); + fpm->add(llvm::createInstructionCombiningPass()); + fpm->add(llvm::createCFGSimplificationPass()); + fpm->add(llvm::createJumpThreadingPass()); + fpm->add(llvm::createConstantPropagationPass()); + fpm->doInitialization(); + + for(auto functionIt = llvmModule->begin();functionIt != llvmModule->end();++functionIt) + { fpm->run(*functionIt); } + delete fpm; + + if(DUMP_OPTIMIZED_MODULE) { printModule(llvmModule,"llvmOptimizedDump"); } + + llvm::orc::VModuleKey K = ES.allocateVModule(); + std::unique_ptr mod(llvmModule); + WAVM_ASSERT_THROW(!compileLayer->addModule(K, std::move(mod))); + WAVM_ASSERT_THROW(!compileLayer->emitAndFinalize(K)); + + final_pic_code = std::move(*unitmemorymanager->code); + } + + instantiated_code instantiateModule(const IR::Module& module) + { + static bool inited; + if(!inited) { + inited = true; + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + llvm::InitializeNativeTargetAsmParser(); + llvm::InitializeNativeTargetDisassembler(); + llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr); + + auto targetTriple = llvm::sys::getProcessTriple(); + + llvm::TargetOptions to; + to.EmitStackSizeSection = 1; + + targetMachine = llvm::EngineBuilder().setRelocationModel(llvm::Reloc::PIC_).setCodeModel(llvm::CodeModel::Small).setTargetOptions(to).selectTarget( + llvm::Triple(targetTriple),"","",llvm::SmallVector() + ); + } + + // Emit LLVM IR for the module. + auto llvmModule = emitModule(module); + + // Construct the JIT compilation pipeline for this module. + auto jitModule = new JITModule(); + // Compile the module. + jitModule->compile(llvmModule); + + unsigned num_functions_stack_size_found = 0; + for(const auto& stacksizes : jitModule->unitmemorymanager->stack_sizes) { + fc::datastream ds(stacksizes.data(), stacksizes.size()); + while(ds.remaining()) { + uint64_t funcaddr; + fc::unsigned_int stack_size; + fc::raw::unpack(ds, funcaddr); + fc::raw::unpack(ds, stack_size); + + ++num_functions_stack_size_found; + if(stack_size > 16u*1024u) + _exit(1); + } + } + if(num_functions_stack_size_found != module.functions.defs.size()) + _exit(1); + if(jitModule->final_pic_code.size() >= 16u*1024u*1024u) + _exit(1); + + instantiated_code ret; + ret.code = jitModule->final_pic_code; + ret.function_offsets = jitModule->function_to_offsets; + return ret; + } +} +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h b/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h new file mode 100644 index 00000000000..5b1152f56d6 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/LLVMJIT.h @@ -0,0 +1,25 @@ +#pragma once + +#include "Inline/BasicTypes.h" +#include "IR/Module.h" + +#pragma push_macro("N") +#undef N +#include "llvm/IR/Module.h" +#pragma pop_macro("N") +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +struct instantiated_code { + std::vector code; + std::map function_offsets; +}; + +namespace LLVMJIT { + bool getFunctionIndexFromExternalName(const char* externalName,Uptr& outFunctionDefIndex); + llvm::Module* emitModule(const IR::Module& module); + instantiated_code instantiateModule(const IR::Module& module); +} +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/code_cache.cpp b/libraries/chain/webassembly/eos-vm-oc/code_cache.cpp new file mode 100644 index 00000000000..9d93b8a2111 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/code_cache.cpp @@ -0,0 +1,385 @@ +#include //set_thread_name + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "IR/Module.h" +#include "IR/Validate.h" +#include "WASM/WASM.h" +#include "LLVMJIT.h" + +using namespace IR; +using namespace Runtime; + +namespace eosio { namespace chain { namespace eosvmoc { + +static constexpr size_t header_offset = 512u; +static constexpr size_t header_size = 512u; +static constexpr size_t total_header_size = header_offset + header_size; +static constexpr uint64_t header_id = 0x32434f4d56534f45ULL; //"EOSVMOC2" little endian + +struct code_cache_header { + uint64_t id = header_id; + bool dirty = false; + uintptr_t serialized_descriptor_index = 0; +} __attribute__ ((packed)); +static constexpr size_t header_dirty_bit_offset_from_file_start = header_offset + offsetof(code_cache_header, dirty); +static constexpr size_t descriptor_ptr_from_file_start = header_offset + offsetof(code_cache_header, serialized_descriptor_index); + +static_assert(sizeof(code_cache_header) <= header_size, "code_cache_header too big"); + +code_cache_async::code_cache_async(const bfs::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) : + code_cache_base(data_dir, eosvmoc_config, db), + _result_queue(eosvmoc_config.threads * 2), + _threads(eosvmoc_config.threads) +{ + FC_ASSERT(_threads, "EOS VM OC requires at least 1 compile thread"); + + wait_on_compile_monitor_message(); + + _monitor_reply_thread = std::thread([this]() { + fc::set_os_thread_name("oc-monitor"); + _ctx.run(); + }); +} + +code_cache_async::~code_cache_async() { + _compile_monitor_write_socket.shutdown(local::datagram_protocol::socket::shutdown_send); + _monitor_reply_thread.join(); + consume_compile_thread_queue(); +} + +//remember again: wait_on_compile_monitor_message's callback is non-main thread! +void code_cache_async::wait_on_compile_monitor_message() { + _compile_monitor_read_socket.async_wait(local::datagram_protocol::socket::wait_read, [this](auto ec) { + if(ec) { + _ctx.stop(); + return; + } + + auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket); + if(!success || !message.contains()) { + _ctx.stop(); + return; + } + + _result_queue.push(message.get()); + + wait_on_compile_monitor_message(); + }); +} + + +//number processed, bytes available (only if number processed > 0) +std::tuple code_cache_async::consume_compile_thread_queue() { + size_t bytes_remaining = 0; + size_t gotsome = _result_queue.consume_all([&](const wasm_compilation_result_message& result) { + if(_outstanding_compiles_and_poison[result.code] == false) { + result.result.visit(overloaded { + [&](const code_descriptor& cd) { + _cache_index.push_front(cd); + }, + [&](const compilation_result_unknownfailure&) { + wlog("code ${c} failed to tier-up with EOS VM OC", ("c", result.code.code_id)); + _blacklist.emplace(result.code); + }, + [&](const compilation_result_toofull&) { + run_eviction_round(); + } + }); + } + _outstanding_compiles_and_poison.erase(result.code); + bytes_remaining = result.cache_free_bytes; + }); + + return {gotsome, bytes_remaining}; +} + +const code_descriptor* const code_cache_async::get_descriptor_for_code(const digest_type& code_id, const uint8_t& vm_version) { + //if there are any outstanding compiles, process the result queue now + if(_outstanding_compiles_and_poison.size()) { + auto [count_processed, bytes_remaining] = consume_compile_thread_queue(); + + if(count_processed) + check_eviction_threshold(bytes_remaining); + + while(count_processed && _queued_compiles.size()) { + auto nextup = _queued_compiles.begin(); + + //it's not clear this check is required: if apply() was called for code then it existed in the code_index; and then + // if we got notification of it no longer existing we would have removed it from queued_compiles + const code_object* const codeobject = _db.find(boost::make_tuple(nextup->code_id, 0, nextup->vm_version)); + if(codeobject) { + _outstanding_compiles_and_poison.emplace(*nextup, false); + std::vector fds_to_pass; + fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code)); + FC_ASSERT(write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ *nextup }, fds_to_pass), "EOS VM failed to communicate to OOP manager"); + --count_processed; + } + _queued_compiles.erase(nextup); + } + } + + //check for entry in cache + code_cache_index::index::type::iterator it = _cache_index.get().find(boost::make_tuple(code_id, vm_version)); + if(it != _cache_index.get().end()) { + _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it)); + return &*it; + } + + const code_tuple ct = code_tuple{code_id, vm_version}; + + if(_blacklist.find(ct) != _blacklist.end()) + return nullptr; + if(auto it = _outstanding_compiles_and_poison.find(ct); it != _outstanding_compiles_and_poison.end()) { + it->second = false; + return nullptr; + } + if(_queued_compiles.find(ct) != _queued_compiles.end()) + return nullptr; + + if(_outstanding_compiles_and_poison.size() >= _threads) { + _queued_compiles.emplace(ct); + return nullptr; + } + + const code_object* const codeobject = _db.find(boost::make_tuple(code_id, 0, vm_version)); + if(!codeobject) //should be impossible right? + return nullptr; + + _outstanding_compiles_and_poison.emplace(ct, false); + std::vector fds_to_pass; + fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code)); + write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass); + return nullptr; +} + +code_cache_sync::~code_cache_sync() { + //it's exceedingly critical that we wait for the compile monitor to be done with all its work + //This is easy in the sync case + _compile_monitor_write_socket.shutdown(local::datagram_protocol::socket::shutdown_send); + auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket); + if(success) + elog("unexpected response from EOS VM OC compile monitor during shutdown"); +} + +const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const digest_type& code_id, const uint8_t& vm_version) { + //check for entry in cache + code_cache_index::index::type::iterator it = _cache_index.get().find(boost::make_tuple(code_id, vm_version)); + if(it != _cache_index.get().end()) { + _cache_index.relocate(_cache_index.begin(), _cache_index.project<0>(it)); + return &*it; + } + + const code_object* const codeobject = _db.find(boost::make_tuple(code_id, 0, vm_version)); + if(!codeobject) //should be impossible right? + return nullptr; + + std::vector fds_to_pass; + fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code)); + + write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ {code_id, vm_version} }, fds_to_pass); + auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket); + EOS_ASSERT(success, wasm_execution_error, "failed to read response from monitor process"); + EOS_ASSERT(message.contains(), wasm_execution_error, "unexpected response from monitor process"); + + wasm_compilation_result_message result = message.get(); + EOS_ASSERT(result.result.contains(), wasm_execution_error, "failed to compile wasm"); + + check_eviction_threshold(result.cache_free_bytes); + + return &*_cache_index.push_front(std::move(result.result.get())).first; +} + +code_cache_base::code_cache_base(const boost::filesystem::path data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) : + _db(db), + _cache_file_path(data_dir/"code_cache.bin") +{ + static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator"); + + bfs::create_directories(data_dir); + + if(!bfs::exists(_cache_file_path)) { + EOS_ASSERT(eosvmoc_config.cache_size >= allocator_t::get_min_size(total_header_size), database_exception, "configured code cache size is too small"); + std::ofstream ofs(_cache_file_path.generic_string(), std::ofstream::trunc); + EOS_ASSERT(ofs.good(), database_exception, "unable to create EOS VM Optimized Compiler code cache"); + bfs::resize_file(_cache_file_path, eosvmoc_config.cache_size); + bip::file_mapping creation_mapping(_cache_file_path.generic_string().c_str(), bip::read_write); + bip::mapped_region creation_region(creation_mapping, bip::read_write); + new (creation_region.get_address()) allocator_t(eosvmoc_config.cache_size, total_header_size); + new ((char*)creation_region.get_address() + header_offset) code_cache_header; + } + + code_cache_header cache_header; + { + char header_buff[total_header_size]; + std::ifstream hs(_cache_file_path.generic_string(), std::ifstream::binary); + hs.read(header_buff, sizeof(header_buff)); + EOS_ASSERT(!hs.fail(), bad_database_version_exception, "failed to read code cache header"); + memcpy((char*)&cache_header, header_buff + header_offset, sizeof(cache_header)); + } + + EOS_ASSERT(cache_header.id == header_id, bad_database_version_exception, "existing EOS VM OC code cache not compatible with this version"); + EOS_ASSERT(!cache_header.dirty, database_exception, "code cache is dirty"); + + set_on_disk_region_dirty(true); + + auto existing_file_size = bfs::file_size(_cache_file_path); + if(eosvmoc_config.cache_size > existing_file_size) { + bfs::resize_file(_cache_file_path, eosvmoc_config.cache_size); + + bip::file_mapping resize_mapping(_cache_file_path.generic_string().c_str(), bip::read_write); + bip::mapped_region resize_region(resize_mapping, bip::read_write); + + allocator_t* resize_allocator = reinterpret_cast(resize_region.get_address()); + resize_allocator->grow(eosvmoc_config.cache_size - existing_file_size); + } + + _cache_fd = ::open(_cache_file_path.generic_string().c_str(), O_RDWR | O_CLOEXEC); + EOS_ASSERT(_cache_fd >= 0, database_exception, "failure to open code cache"); + + //load up the previous cache index + char* code_mapping = (char*)mmap(nullptr, eosvmoc_config.cache_size, PROT_READ|PROT_WRITE, MAP_SHARED, _cache_fd, 0); + EOS_ASSERT(code_mapping != MAP_FAILED, database_exception, "failure to mmap code cache"); + + allocator_t* allocator = reinterpret_cast(code_mapping); + + if(cache_header.serialized_descriptor_index) { + fc::datastream ds(code_mapping + cache_header.serialized_descriptor_index, eosvmoc_config.cache_size - cache_header.serialized_descriptor_index); + unsigned number_entries; + fc::raw::unpack(ds, number_entries); + for(unsigned i = 0; i < number_entries; ++i) { + code_descriptor cd; + fc::raw::unpack(ds, cd); + if(cd.codegen_version != 0) { + allocator->deallocate(code_mapping + cd.code_begin); + allocator->deallocate(code_mapping + cd.initdata_begin); + continue; + } + _cache_index.push_back(std::move(cd)); + } + allocator->deallocate(code_mapping + cache_header.serialized_descriptor_index); + + ilog("EOS VM Optimized Compiler code cache loaded with ${c} entries; ${f} of ${t} bytes free", ("c", number_entries)("f", allocator->get_free_memory())("t", allocator->get_size())); + } + munmap(code_mapping, eosvmoc_config.cache_size); + + _free_bytes_eviction_threshold = eosvmoc_config.cache_size * .1; + + wrapped_fd compile_monitor_conn = get_connection_to_compile_monitor(_cache_fd); + + //okay, let's do this by the book: we're not allowed to write & read on different threads to the same asio socket. So create two fds + //representing the same unix socket. we'll read on one and write on the other + int duped = dup(compile_monitor_conn); + _compile_monitor_write_socket.assign(local::datagram_protocol(), duped); + _compile_monitor_read_socket.assign(local::datagram_protocol(), compile_monitor_conn.release()); +} + +void code_cache_base::set_on_disk_region_dirty(bool dirty) { + bip::file_mapping dirty_mapping(_cache_file_path.generic_string().c_str(), bip::read_write); + bip::mapped_region dirty_region(dirty_mapping, bip::read_write); + + *((char*)dirty_region.get_address()+header_dirty_bit_offset_from_file_start) = dirty; + if(dirty_region.flush(0, 0, false) == false) + elog("Syncing code cache failed"); +} + +template +void code_cache_base::serialize_cache_index(fc::datastream& ds) { + unsigned entries = _cache_index.size(); + fc::raw::pack(ds, entries); + for(const code_descriptor& cd : _cache_index) + fc::raw::pack(ds, cd); +} + +code_cache_base::~code_cache_base() { + //reopen the code cache in our process + struct stat st; + if(fstat(_cache_fd, &st)) + return; + char* code_mapping = (char*)mmap(nullptr, st.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, _cache_fd, 0); + if(code_mapping == MAP_FAILED) + return; + + allocator_t* allocator = reinterpret_cast(code_mapping); + + //serialize out the cache index + fc::datastream dssz; + serialize_cache_index(dssz); + size_t sz = dssz.tellp(); + + char* p = nullptr; + while(_cache_index.size()) { + p = (char*)allocator->allocate(sz); + if(p != nullptr) + break; + //in theory, there could be too little free space avaiable to store the cache index + //try to free up some space + for(unsigned int i = 0; i < 25 && _cache_index.size(); ++i) { + allocator->deallocate(code_mapping + _cache_index.back().code_begin); + allocator->deallocate(code_mapping + _cache_index.back().initdata_begin); + _cache_index.pop_back(); + } + } + + if(p) { + fc::datastream ds(p, sz); + serialize_cache_index(ds); + + uintptr_t ptr_offset = p-code_mapping; + *((uintptr_t*)(code_mapping+descriptor_ptr_from_file_start)) = ptr_offset; + } + else + *((uintptr_t*)(code_mapping+descriptor_ptr_from_file_start)) = 0; + + msync(code_mapping, allocator->get_size(), MS_SYNC); + munmap(code_mapping, allocator->get_size()); + close(_cache_fd); + set_on_disk_region_dirty(false); + +} + +void code_cache_base::free_code(const digest_type& code_id, const uint8_t& vm_version) { + code_cache_index::index::type::iterator it = _cache_index.get().find(boost::make_tuple(code_id, vm_version)); + if(it != _cache_index.get().end()) { + write_message_with_fds(_compile_monitor_write_socket, evict_wasms_message{ {*it} }); + _cache_index.get().erase(it); + } + + //if it's in the queued list, erase it + _queued_compiles.erase({code_id, vm_version}); + + //however, if it's currently being compiled there is no way to cancel the compile, + //so instead set a poison boolean that indicates not to insert the code in to the cache + //once the compile is complete + const std::unordered_map::iterator compiling_it = _outstanding_compiles_and_poison.find({code_id, vm_version}); + if(compiling_it != _outstanding_compiles_and_poison.end()) + compiling_it->second = true; +} + +void code_cache_base::run_eviction_round() { + evict_wasms_message evict_msg; + for(unsigned int i = 0; i < 25 && _cache_index.size() > 1; ++i) { + evict_msg.codes.emplace_back(_cache_index.back()); + _cache_index.pop_back(); + } + write_message_with_fds(_compile_monitor_write_socket, evict_msg); +} + +void code_cache_base::check_eviction_threshold(size_t free_bytes) { + if(free_bytes < _free_bytes_eviction_threshold) + run_eviction_round(); +} + +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp b/libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp new file mode 100644 index 00000000000..67b85c5ded4 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/compile_monitor.cpp @@ -0,0 +1,326 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +using namespace boost::asio; + +static size_t get_size_of_fd(int fd) { + struct stat st; + FC_ASSERT(fstat(fd, &st) == 0, "failed to get size of fd"); + return st.st_size; +} + +static void copy_memfd_contents_to_pointer(void* dst, int fd) { + struct stat st; + FC_ASSERT(fstat(fd, &st) == 0, "failed to get size of fd"); + if(st.st_size == 0) + return; + void* contents = mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fd, 0); + FC_ASSERT(contents != MAP_FAILED, "failed to map memfd file"); + memcpy(dst, contents, st.st_size); + munmap(contents, st.st_size); +} + +struct compile_monitor_session { + compile_monitor_session(boost::asio::io_context& context, local::datagram_protocol::socket&& n, wrapped_fd&& c, wrapped_fd& t) : + _ctx(context), + _nodeos_instance_socket(std::move(n)), + _cache_fd(std::move(c)), + _trampoline_socket(t) { + + struct stat st; + FC_ASSERT(fstat(_cache_fd, &st) == 0, "failed to stat cache fd"); + _code_size = st.st_size; + _code_mapping = (char*)mmap(nullptr, _code_size, PROT_READ|PROT_WRITE, MAP_SHARED, _cache_fd, 0); + FC_ASSERT(_code_mapping != MAP_FAILED, "failed to mmap cache file"); + _allocator = reinterpret_cast(_code_mapping); + + read_message_from_nodeos(); + } + + ~compile_monitor_session() { + munmap(_code_mapping, _code_size); + } + + void read_message_from_nodeos() { + _nodeos_instance_socket.async_wait(local::datagram_protocol::socket::wait_read, [this](auto ec) { + if(ec) { + connection_dead_signal(); + return; + } + auto [success, message, fds] = read_message_with_fds(_nodeos_instance_socket); + if(!success) { + connection_dead_signal(); + return; + } + + message.visit(overloaded { + [&, &fds=fds](const compile_wasm_message& compile) { + if(fds.size() != 1) { + connection_dead_signal(); + return; + } + kick_compile_off(compile.code, std::move(fds[0])); + }, + [&](const evict_wasms_message& evict) { + for(const code_descriptor& cd : evict.codes) { + _allocator->deallocate(_code_mapping + cd.code_begin); + _allocator->deallocate(_code_mapping + cd.initdata_begin); + } + }, + [&](const auto&) { + //anything else is an error + connection_dead_signal(); + return; + } + }); + + read_message_from_nodeos(); + }); + } + + void kick_compile_off(const code_tuple& code_id, wrapped_fd&& wasm_code) { + //prepare a requst to go out to the trampoline + int socks[2]; + socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks); + local::datagram_protocol::socket response_socket(_ctx); + response_socket.assign(local::datagram_protocol(), socks[0]); + std::vector fds_pass_to_trampoline; + fds_pass_to_trampoline.emplace_back(socks[1]); + fds_pass_to_trampoline.emplace_back(std::move(wasm_code)); + + eosvmoc_message trampoline_compile_request = compile_wasm_message{code_id}; + if(write_message_with_fds(_trampoline_socket, trampoline_compile_request, fds_pass_to_trampoline) == false) { + wasm_compilation_result_message reply{code_id, compilation_result_unknownfailure{}, _allocator->get_free_memory()}; + write_message_with_fds(_nodeos_instance_socket, reply); + return; + } + + current_compiles.emplace_front(code_id, std::move(response_socket)); + read_message_from_compile_task(current_compiles.begin()); + } + + void read_message_from_compile_task(std::list>::iterator current_compile_it) { + auto& [code, socket] = *current_compile_it; + socket.async_wait(local::datagram_protocol::socket::wait_read, [this, current_compile_it](auto ec) { + //at this point we only expect 1 of 2 things to happen: we either get a reply (success), or we get no reply (failure) + auto& [code, socket] = *current_compile_it; + auto [success, message, fds] = read_message_with_fds(socket); + + wasm_compilation_result_message reply{code, compilation_result_unknownfailure{}, _allocator->get_free_memory()}; + + void* code_ptr = nullptr; + void* mem_ptr = nullptr; + try { + if(success && message.contains() && fds.size() == 2) { + code_compilation_result_message& result = message.get(); + code_ptr = _allocator->allocate(get_size_of_fd(fds[0])); + mem_ptr = _allocator->allocate(get_size_of_fd(fds[1])); + + if(code_ptr == nullptr || mem_ptr == nullptr) { + _allocator->deallocate(code_ptr); + _allocator->deallocate(mem_ptr); + reply.result = compilation_result_toofull(); + } + else { + copy_memfd_contents_to_pointer(code_ptr, fds[0]); + copy_memfd_contents_to_pointer(mem_ptr, fds[1]); + + reply.result = code_descriptor { + code.code_id, + code.vm_version, + 0, + (uintptr_t)code_ptr - (uintptr_t)_code_mapping, + result.start, + result.apply_offset, + result.starting_memory_pages, + (uintptr_t)mem_ptr - (uintptr_t)_code_mapping, + (unsigned)get_size_of_fd(fds[1]), + result.initdata_prologue_size + }; + } + } + } + catch(...) { + _allocator->deallocate(code_ptr); + _allocator->deallocate(mem_ptr); + } + + write_message_with_fds(_nodeos_instance_socket, reply); + + //either way, we are done + _ctx.post([this, current_compile_it]() { + current_compiles.erase(current_compile_it); + }); + }); + + } + + boost::signals2::signal connection_dead_signal; + +private: + boost::asio::io_context& _ctx; + local::datagram_protocol::socket _nodeos_instance_socket; + wrapped_fd _cache_fd; + wrapped_fd& _trampoline_socket; + + char* _code_mapping; + size_t _code_size; + allocator_t* _allocator; + + std::list> current_compiles; +}; + +struct compile_monitor { + compile_monitor(boost::asio::io_context& ctx, local::datagram_protocol::socket&& n, wrapped_fd&& t) : _nodeos_socket(std::move(n)), _trampoline_socket(std::move(t)) { + //the only duty of compile_monitor is to create a compile_monitor_session when a code_cache instance + // in nodeos wants one + wait_for_new_incomming_session(ctx); + } + + void wait_for_new_incomming_session(boost::asio::io_context& ctx) { + _nodeos_socket.async_wait(boost::asio::local::datagram_protocol::socket::wait_read, [this, &ctx](auto ec) { + if(ec) { + ctx.stop(); + return; + } + auto [success, message, fds] = read_message_with_fds(_nodeos_socket); + if(!success) { //failure reading indicates that nodeos has shut down + ctx.stop(); + return; + } + if(!message.contains() || fds.size() != 2) { + ctx.stop(); + return; + } + try { + local::datagram_protocol::socket _socket_for_comm(ctx); + _socket_for_comm.assign(local::datagram_protocol(), fds[0].release()); + _compile_sessions.emplace_front(ctx, std::move(_socket_for_comm), std::move(fds[1]), _trampoline_socket); + _compile_sessions.front().connection_dead_signal.connect([&, it = _compile_sessions.begin()]() { + ctx.post([&]() { + _compile_sessions.erase(it); + }); + }); + write_message_with_fds(_nodeos_socket, initalize_response_message()); + } + catch(const fc::exception& e) { + write_message_with_fds(_nodeos_socket, initalize_response_message{e.what()}); + } + catch(...) { + write_message_with_fds(_nodeos_socket, initalize_response_message{"Failed to create compile process"}); + } + + wait_for_new_incomming_session(ctx); + }); + } + + local::datagram_protocol::socket _nodeos_socket; + wrapped_fd _trampoline_socket; + + std::list _compile_sessions; +}; + +void launch_compile_monitor(int nodeos_fd) { + prctl(PR_SET_NAME, "oc-monitor"); + prctl(PR_SET_PDEATHSIG, SIGKILL); + + //first off, let's disable shutdown signals to us; we want all shutdown indicators to come from + // nodeos shutting us down + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGQUIT); + sigprocmask(SIG_BLOCK, &set, nullptr); + + int socks[2]; //0: local trampoline socket, 1: the one we give to trampoline + socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks); + pid_t child = fork(); + if(child == 0) { + close(socks[0]); + run_compile_trampoline(socks[1]); + } + close(socks[1]); + + { + boost::asio::io_context ctx; + boost::asio::local::datagram_protocol::socket nodeos_socket(ctx); + nodeos_socket.assign(boost::asio::local::datagram_protocol(), nodeos_fd); + wrapped_fd trampoline_socket(socks[0]); + compile_monitor monitor(ctx, std::move(nodeos_socket), std::move(trampoline_socket)); + ctx.run(); + if(monitor._compile_sessions.size()) + std::cerr << "ERROR: EOS VM OC compiler monitor exiting with active sessions" << std::endl; + } + + _exit(0); +} + +struct compile_monitor_trampoline { + void start() { + //create communication socket; let's hold off on asio usage until all forks are done + int socks[2]; + socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks); + compile_manager_fd = socks[0]; + + compile_manager_pid = fork(); + if(compile_manager_pid == 0) { + close(socks[0]); + launch_compile_monitor(socks[1]); + } + close(socks[1]); + } + + pid_t compile_manager_pid = -1; + int compile_manager_fd = -1; +}; + +static compile_monitor_trampoline the_compile_monitor_trampoline; +extern "C" int __real_main(int, char*[]); +extern "C" int __wrap_main(int argc, char* argv[]) { + the_compile_monitor_trampoline.start(); + return __real_main(argc, argv); +} + +wrapped_fd get_connection_to_compile_monitor(int cache_fd) { + FC_ASSERT(the_compile_monitor_trampoline.compile_manager_pid >= 0, "EOS VM oop connection doesn't look active"); + + int socks[2]; //0: our socket to compile_manager_session, 1: socket we'll give to compile_maanger_session + socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks); + wrapped_fd socket_to_monitor_session(socks[0]); + wrapped_fd socket_to_hand_to_monitor_session(socks[1]); + + //we don't own cache_fd, so try to be extra careful not to accidentally close it: don't stick it in a wrapped_fd + // to hand off to write_message_with_fds even temporarily. make a copy of it. + int dup_of_cache_fd = dup(cache_fd); + FC_ASSERT(dup_of_cache_fd != -1, "failed to dup cache_fd"); + wrapped_fd dup_cache_fd(dup_of_cache_fd); + + std::vector fds_to_pass; + fds_to_pass.emplace_back(std::move(socket_to_hand_to_monitor_session)); + fds_to_pass.emplace_back(std::move(dup_cache_fd)); + write_message_with_fds(the_compile_monitor_trampoline.compile_manager_fd, initialize_message(), fds_to_pass); + + auto [success, message, fds] = read_message_with_fds(the_compile_monitor_trampoline.compile_manager_fd); + EOS_ASSERT(success, misc_exception, "failed to read response from monitor process"); + EOS_ASSERT(message.contains(), misc_exception, "unexpected response from monitor process"); + EOS_ASSERT(!message.get().error_message, misc_exception, "Error message from monitor process: ${e}", ("e", *message.get().error_message)); + return socket_to_monitor_session; +} + +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp b/libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp new file mode 100644 index 00000000000..07e9ba6b086 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/compile_trampoline.cpp @@ -0,0 +1,188 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "IR/Module.h" +#include "IR/Validate.h" +#include "WASM/WASM.h" +#include "LLVMJIT.h" + +using namespace IR; + +namespace eosio { namespace chain { namespace eosvmoc { + +void run_compile(wrapped_fd&& response_sock, wrapped_fd&& wasm_code) noexcept { //noexcept; we'll just blow up if anything tries to cross this boundry + std::vector wasm = vector_for_memfd(wasm_code); + + //ideally we catch exceptions and sent them upstream as strings for easier reporting + + Module module; + Serialization::MemoryInputStream stream(wasm.data(), wasm.size()); + WASM::serialize(stream, module); + module.userSections.clear(); + wasm_injections::wasm_binary_injection injector(module); + injector.inject(); + + instantiated_code code = LLVMJIT::instantiateModule(module); + + code_compilation_result_message result_message; + + const std::map& function_to_offsets = code.function_offsets; + + if(module.startFunctionIndex == UINTPTR_MAX) + result_message.start = no_offset{}; + else if(module.startFunctionIndex < module.functions.imports.size()) { + const auto& f = module.functions.imports[module.startFunctionIndex]; + const intrinsic_entry& ie = get_intrinsic_map().at(f.moduleName + "." + f.exportName); + result_message.start = intrinsic_ordinal{ie.ordinal}; + } + else + result_message.start = code_offset{function_to_offsets.at(module.startFunctionIndex-module.functions.imports.size())}; + + for(const Export& exprt : module.exports) { + if(exprt.name == "apply") + result_message.apply_offset = function_to_offsets.at(exprt.index-module.functions.imports.size()); + } + + result_message.starting_memory_pages = -1; + if(module.memories.size()) + result_message.starting_memory_pages = module.memories.defs.at(0).type.size.min; + + std::vector prologue(memory::cb_offset); //getting the control block offset gets us as large as table+globals as possible + std::vector::iterator prologue_it = prologue.end(); + + //set up mutable globals + union global_union { + int64_t i64; + int32_t i32; + float f32; + double f64; + }; + + for(const GlobalDef& global : module.globals.defs) { + if(!global.type.isMutable) + continue; + prologue_it -= 8; + global_union* const u = (global_union* const)&*prologue_it; + + switch(global.initializer.type) { + case InitializerExpression::Type::i32_const: u->i32 = global.initializer.i32; break; + case InitializerExpression::Type::i64_const: u->i64 = global.initializer.i64; break; + case InitializerExpression::Type::f32_const: u->f32 = global.initializer.f32; break; + case InitializerExpression::Type::f64_const: u->f64 = global.initializer.f64; break; + default: break; //impossible + } + } + + struct table_entry { + uintptr_t type; + int64_t func; //>= 0 means offset to code in wasm; < 0 means intrinsic call at offset address + }; + + if(module.tables.size()) + prologue_it -= sizeof(table_entry) * module.tables.defs[0].type.size.min; + + for(const TableSegment& table_segment : module.tableSegments) { + struct table_entry* table_index_0 = (struct table_entry*)&*prologue_it; + + if(table_segment.baseOffset.i32 > module.tables.defs[0].type.size.min) + return; + + for(Uptr i = 0; i < table_segment.indices.size(); ++i) { + const Uptr function_index = table_segment.indices[i]; + const long int effective_table_index = table_segment.baseOffset.i32 + i; + + if(effective_table_index >= module.tables.defs[0].type.size.min) + return; + + if(function_index < module.functions.imports.size()) { + const auto& f = module.functions.imports[function_index]; + const intrinsic_entry& ie = get_intrinsic_map().at(f.moduleName + "." + f.exportName); + table_index_0[effective_table_index].func = ie.ordinal*-8; + table_index_0[effective_table_index].type = (uintptr_t)module.types[module.functions.imports[function_index].type.index]; + } + else { + table_index_0[effective_table_index].func = function_to_offsets.at(function_index - module.functions.imports.size()); + table_index_0[effective_table_index].type = (uintptr_t)module.types[module.functions.defs[function_index - module.functions.imports.size()].type.index]; + } + } + } + + //this is somewhat copy pasta from wasm_interface_private, with the asserts removed + std::vector initial_mem; + for(const DataSegment& data_segment : module.dataSegments) { + const U32 base_offset = data_segment.baseOffset.i32; + + if(base_offset + data_segment.data.size() > initial_mem.size()) + initial_mem.resize(base_offset + data_segment.data.size(), 0x00); + memcpy(initial_mem.data() + base_offset, data_segment.data.data(), data_segment.data.size()); + } + + result_message.initdata_prologue_size = prologue.end() - prologue_it; + std::vector initdata_prep; + std::move(prologue_it, prologue.end(), std::back_inserter(initdata_prep)); + std::move(initial_mem.begin(), initial_mem.end(), std::back_inserter(initdata_prep)); + + std::vector fds_to_send; + fds_to_send.emplace_back(memfd_for_bytearray(code.code)); + fds_to_send.emplace_back(memfd_for_bytearray(initdata_prep)); + write_message_with_fds(response_sock, result_message, fds_to_send); +} + +void run_compile_trampoline(int fd) { + prctl(PR_SET_NAME, "oc-trampoline"); + prctl(PR_SET_PDEATHSIG, SIGKILL); + + //squelching this for now, but it means we won't have ability to get compile metrics + struct sigaction act; + sigset_t set; + sigemptyset(&set); + act.sa_handler = SIG_IGN; + act.sa_mask = set; + act.sa_flags = SA_NOCLDWAIT; + act.sa_sigaction = nullptr; + sigaction(SIGCHLD, &act, nullptr); + + while(true) { + auto [success, message, fds] = read_message_with_fds(fd); + if(!success) + break; + + if(!message.contains() || fds.size() != 2) { + std::cerr << "EOS VM OC compile trampoline got unexpected message; ignoring" << std::endl; + continue; + } + + pid_t pid = fork(); + if(pid == 0) { + prctl(PR_SET_NAME, "oc-compile"); + prctl(PR_SET_PDEATHSIG, SIGKILL); + + struct rlimit cpu_limits = {20u, 20u}; + setrlimit(RLIMIT_CPU, &cpu_limits); + + struct rlimit vm_limits = {512u*1024u*1024u, 512u*1024u*1024u}; + setrlimit(RLIMIT_AS, &vm_limits); + + struct rlimit core_limits = {0u, 0u}; + setrlimit(RLIMIT_CORE, &core_limits); + + run_compile(std::move(fds[0]), std::move(fds[1])); + _exit(0); + } + else if(pid == -1) + std::cerr << "EOS VM OC compile trampoline failed to spawn compile task" << std::endl; + } + + _exit(0); +} + +}}} + + diff --git a/libraries/chain/webassembly/eos-vm-oc/executor.cpp b/libraries/chain/webassembly/eos-vm-oc/executor.cpp new file mode 100644 index 00000000000..1dad9744e09 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/executor.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +#if defined(__has_feature) +#if __has_feature(shadow_call_stack) +#error EOS VM OC is not compatible with Clang ShadowCallStack +#endif +#endif + +extern "C" int arch_prctl(int code, unsigned long* addr); + +namespace eosio { namespace chain { namespace eosvmoc { + +static constexpr auto signal_sentinel = 0x4D56534F45534559ul; + +static void(*chained_handler)(int,siginfo_t*,void*); +static void segv_handler(int sig, siginfo_t* info, void* ctx) { + control_block* cb_in_main_segment; + + //a 0 GS value is an indicator an executor hasn't been active on this thread recently + uint64_t current_gs; + syscall(SYS_arch_prctl, ARCH_GET_GS, ¤t_gs); + if(current_gs == 0) + goto notus; + + cb_in_main_segment = reinterpret_cast(current_gs - memory::cb_offset); + + //as a double check that the control block pointer is what we expect, look for the magic + if(cb_in_main_segment->magic != signal_sentinel) + goto notus; + + //was wasm running? If not, this SEGV was not due to us + if(cb_in_main_segment->is_running == false) + goto notus; + + //was the segfault within code? + if((uintptr_t)info->si_addr >= cb_in_main_segment->execution_thread_code_start && + (uintptr_t)info->si_addr < cb_in_main_segment->execution_thread_code_start+cb_in_main_segment->execution_thread_code_length) + siglongjmp(*cb_in_main_segment->jmp, EOSVMOC_EXIT_CHECKTIME_FAIL); + + //was the segfault within data? + if((uintptr_t)info->si_addr >= cb_in_main_segment->execution_thread_memory_start && + (uintptr_t)info->si_addr < cb_in_main_segment->execution_thread_memory_start+cb_in_main_segment->execution_thread_memory_length) + siglongjmp(*cb_in_main_segment->jmp, EOSVMOC_EXIT_SEGV); + +notus: + if(chained_handler) { + chained_handler(sig, info, ctx); + return; + } + ::signal(sig, SIG_DFL); + ::raise(sig); + __builtin_unreachable(); +} + +static intrinsic grow_memory_intrinsic EOSVMOC_INTRINSIC_INIT_PRIORITY("eosvmoc_internal.grow_memory", IR::FunctionType::get(IR::ResultType::i32,{IR::ValueType::i32,IR::ValueType::i32}), + (void*)&eos_vm_oc_grow_memory, + boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("eosvmoc_internal.grow_memory"))).value() +); + +//This is effectively overriding the eosio_exit intrinsic in wasm_interface +static void eosio_exit(int32_t code) { + siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_CLEAN_EXIT); + __builtin_unreachable(); +} +static intrinsic eosio_exit_intrinsic("env.eosio_exit", IR::FunctionType::get(IR::ResultType::none,{IR::ValueType::i32}), (void*)&eosio_exit, + boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING("env.eosio_exit"))).value() +); + +static void throw_internal_exception(const char* const s) { + *reinterpret_cast(eos_vm_oc_get_exception_ptr()) = std::make_exception_ptr(wasm_execution_error(FC_LOG_MESSAGE(error, s))); + siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION); + __builtin_unreachable(); +} + +#define DEFINE_EOSVMOC_TRAP_INTRINSIC(module,name) \ + void name(); \ + static intrinsic name##Function EOSVMOC_INTRINSIC_INIT_PRIORITY(#module "." #name,IR::FunctionType::get(),(void*)&name, \ + boost::hana::index_if(intrinsic_table, ::boost::hana::equal.to(BOOST_HANA_STRING(#module "." #name))).value() \ + ); \ + void name() + +DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,depth_assert) { + throw_internal_exception("Exceeded call depth maximum"); +} + +DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,div0_or_overflow) { + throw_internal_exception("Division by 0 or integer overflow trapped"); +} + +DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,indirect_call_mismatch) { + throw_internal_exception("Indirect call function type mismatch"); +} + +DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,indirect_call_oob) { + throw_internal_exception("Indirect call index out of bounds"); +} + +DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,unreachable) { + throw_internal_exception("Unreachable reached"); +} + +struct executor_signal_init { + executor_signal_init() { + struct sigaction sig_action, old_sig_action; + sig_action.sa_sigaction = segv_handler; + sigemptyset(&sig_action.sa_mask); + sig_action.sa_flags = SA_SIGINFO | SA_NODEFER; + sigaction(SIGSEGV, &sig_action, &old_sig_action); + if(old_sig_action.sa_flags & SA_SIGINFO) + chained_handler = old_sig_action.sa_sigaction; + else if(old_sig_action.sa_handler != SIG_IGN && old_sig_action.sa_handler != SIG_DFL) + chained_handler = (void (*)(int,siginfo_t*,void*))old_sig_action.sa_handler; + } +}; + +executor::executor(const code_cache_base& cc) { + //if we're the first executor created, go setup the signal handling. For now we'll just leave this attached forever + static executor_signal_init the_executor_signal_init; + + uint64_t current_gs; + if(arch_prctl(ARCH_GET_GS, ¤t_gs) || current_gs) + wlog("x86_64 GS register is not set as expected. EOS VM OC may not run correctly on this platform"); + + struct stat s; + FC_ASSERT(fstat(cc.fd(), &s) == 0, "executor failed to get code cache size"); + code_mapping = (uint8_t*)mmap(nullptr, s.st_size, PROT_EXEC|PROT_READ, MAP_SHARED, cc.fd(), 0); + FC_ASSERT(code_mapping != MAP_FAILED, "failed to map code cache in to executor"); + code_mapping_size = s.st_size; + mapping_is_executable = true; +} + +void executor::execute(const code_descriptor& code, const memory& mem, apply_context& context) { + if(mapping_is_executable == false) { + mprotect(code_mapping, code_mapping_size, PROT_EXEC|PROT_READ); + mapping_is_executable = true; + } + + //prepare initial memory, mutable globals, and table data + if(code.starting_memory_pages > 0 ) { + arch_prctl(ARCH_SET_GS, (unsigned long*)(mem.zero_page_memory_base()+code.starting_memory_pages*memory::stride)); + memset(mem.full_page_memory_base(), 0, 64u*1024u*code.starting_memory_pages); + } + else + arch_prctl(ARCH_SET_GS, (unsigned long*)mem.zero_page_memory_base()); + memcpy(mem.full_page_memory_base() - code.initdata_prologue_size, code_mapping + code.initdata_begin, code.initdata_size); + + control_block* const cb = mem.get_control_block(); + cb->magic = signal_sentinel; + cb->execution_thread_code_start = (uintptr_t)code_mapping; + cb->execution_thread_code_length = code_mapping_size; + cb->execution_thread_memory_start = (uintptr_t)mem.start_of_memory_slices(); + cb->execution_thread_memory_length = mem.size_of_memory_slice_mapping(); + cb->ctx = &context; + executors_exception_ptr = nullptr; + cb->eptr = &executors_exception_ptr; + cb->current_call_depth_remaining = eosio::chain::wasm_constraints::maximum_call_depth+2; + cb->current_linear_memory_pages = code.starting_memory_pages; + cb->first_invalid_memory_address = code.starting_memory_pages*64*1024; + cb->full_linear_memory_start = (char*)mem.full_page_memory_base(); + cb->jmp = &executors_sigjmp_buf; + cb->bounce_buffers = &executors_bounce_buffers; + cb->running_code_base = (uintptr_t)(code_mapping + code.code_begin); + cb->is_running = true; + + context.trx_context.transaction_timer.set_expiration_callback([](void* user) { + executor* self = (executor*)user; + syscall(SYS_mprotect, self->code_mapping, self->code_mapping_size, PROT_NONE); + self->mapping_is_executable = false; + }, this); + context.trx_context.checktime(); //catch any expiration that might have occurred before setting up callback + + auto cleanup = fc::make_scoped_exit([cb, &tt=context.trx_context.transaction_timer](){ + cb->is_running = false; + cb->bounce_buffers->clear(); + tt.set_expiration_callback(nullptr, nullptr); + }); + + void(*apply_func)(uint64_t, uint64_t, uint64_t) = (void(*)(uint64_t, uint64_t, uint64_t))(cb->running_code_base + code.apply_offset); + + switch(sigsetjmp(*cb->jmp, 0)) { + case 0: + code.start.visit(overloaded { + [&](const no_offset&) {}, + [&](const intrinsic_ordinal& i) { + void(*start_func)() = (void(*)())(*(uintptr_t*)((uintptr_t)mem.zero_page_memory_base() - memory::first_intrinsic_offset - i.ordinal*8)); + start_func(); + }, + [&](const code_offset& offs) { + void(*start_func)() = (void(*)())(cb->running_code_base + offs.offset); + start_func(); + } + }); + apply_func(context.get_receiver().to_uint64_t(), context.get_action().account.to_uint64_t(), context.get_action().name.to_uint64_t()); + break; + //case 1: clean eosio_exit + case EOSVMOC_EXIT_CHECKTIME_FAIL: + context.trx_context.checktime(); + break; + case EOSVMOC_EXIT_SEGV: + EOS_ASSERT(false, wasm_execution_error, "access violation"); + break; + case EOSVMOC_EXIT_EXCEPTION: //exception + std::rethrow_exception(*cb->eptr); + break; + } +} + +executor::~executor() { + arch_prctl(ARCH_SET_GS, nullptr); +} + +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c b/libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c new file mode 100644 index 00000000000..db4f1014ec6 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/gs_seg_helpers.c @@ -0,0 +1,46 @@ +#include + +#include +#include + +int arch_prctl(int code, unsigned long* addr); + +#define EOSVMOC_MEMORY_PTR_cb_ptr GS_PTR struct eos_vm_oc_control_block* const cb_ptr = ((GS_PTR struct eos_vm_oc_control_block* const)(EOS_VM_OC_CONTROL_BLOCK_OFFSET)); + +int32_t eos_vm_oc_grow_memory(int32_t grow, int32_t max) { + EOSVMOC_MEMORY_PTR_cb_ptr; + uint64_t previous_page_count = cb_ptr->current_linear_memory_pages; + int32_t grow_amount = grow; + uint64_t max_pages = max; + if(grow == 0) + return (int32_t)cb_ptr->current_linear_memory_pages; + if(previous_page_count + grow_amount > max_pages) + return (int32_t)-1; + + uint64_t current_gs; + arch_prctl(ARCH_GET_GS, ¤t_gs); + current_gs += grow_amount * EOS_VM_OC_MEMORY_STRIDE; + arch_prctl(ARCH_SET_GS, (unsigned long*)current_gs); + cb_ptr->current_linear_memory_pages += grow_amount; + cb_ptr->first_invalid_memory_address += grow_amount*64*1024; + + if(grow_amount > 0) + memset(cb_ptr->full_linear_memory_start + previous_page_count*64u*1024u, 0, grow_amount*64u*1024u); + + return (int32_t)previous_page_count; +} + +sigjmp_buf* eos_vm_oc_get_jmp_buf() { + EOSVMOC_MEMORY_PTR_cb_ptr; + return cb_ptr->jmp; +} + +void* eos_vm_oc_get_exception_ptr() { + EOSVMOC_MEMORY_PTR_cb_ptr; + return cb_ptr->eptr; +} + +void* eos_vm_oc_get_bounce_buffer_list() { + EOSVMOC_MEMORY_PTR_cb_ptr; + return cb_ptr->bounce_buffers; +} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/intrinsic.cpp b/libraries/chain/webassembly/eos-vm-oc/intrinsic.cpp new file mode 100644 index 00000000000..f4e15780412 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/intrinsic.cpp @@ -0,0 +1,19 @@ +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +static intrinsic_map_t& the_intrinsic_map() { + static intrinsic_map_t intrinsic_map; + return intrinsic_map; +} + +const intrinsic_map_t& get_intrinsic_map() { + return the_intrinsic_map(); +} + +intrinsic::intrinsic(const char* n, const IR::FunctionType* t, void* f, size_t o) { + the_intrinsic_map().erase(n); + the_intrinsic_map().emplace(n, intrinsic_entry{t, f, o}); +} + +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp b/libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp new file mode 100644 index 00000000000..cc694240957 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/ipc_helpers.cpp @@ -0,0 +1,135 @@ +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +static constexpr size_t max_message_size = 8192; +static constexpr size_t max_num_fds = 4; + +std::tuple> read_message_with_fds(boost::asio::local::datagram_protocol::socket& s) { + return read_message_with_fds(s.native_handle()); +} + +std::tuple> read_message_with_fds(int fd) { + char buff[max_message_size]; + + struct msghdr msg = {}; + struct cmsghdr* cmsg; + + eosvmoc_message message; + std::vector fds; + + struct iovec io = { + .iov_base = buff, + .iov_len = sizeof(buff) + }; + union { + char buf[CMSG_SPACE(max_num_fds * sizeof(int))]; + struct cmsghdr align; + } u; + + msg.msg_iov = &io; + msg.msg_iovlen = 1; + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + + int red; + do { + red = recvmsg(fd, &msg, 0); + } while(red == -1 && errno == EINTR); + if(red < 1 || red >= sizeof(buff)) + return {false, message, std::move(fds)}; + + try { + fc::datastream ds(buff, red); + fc::raw::unpack(ds, message); + } + catch(...) { + return {false, message, std::move(fds)}; + } + + if(msg.msg_controllen) { + cmsg = CMSG_FIRSTHDR(&msg); + unsigned num_of_fds = (cmsg->cmsg_len - CMSG_LEN(0))/sizeof(int); + if(num_of_fds > max_num_fds) + return {false, message, std::move(fds)}; + int* fd_ptr = (int*)CMSG_DATA(cmsg); + for(unsigned i = 0; i < num_of_fds; ++i) + fds.push_back(*fd_ptr++); + } + + return {true, message, std::move(fds)}; +} + +bool write_message_with_fds(boost::asio::local::datagram_protocol::socket& s, const eosvmoc_message& message, const std::vector& fds) { + return write_message_with_fds(s.native_handle(), message, fds); +} + +bool write_message_with_fds(int fd_to_send_to, const eosvmoc_message& message, const std::vector& fds) { + struct msghdr msg = {}; + struct cmsghdr* cmsg; + + size_t sz = fc::raw::pack_size(message); + if(sz > max_message_size) + return false; + char buff[max_message_size]; + try { + fc::datastream ds(buff, max_message_size); + fc::raw::pack(ds, message); + } + catch(...) { + return false; + } + + if(fds.size() > max_num_fds) + return false; + + struct iovec io = { + .iov_base = buff, + .iov_len = sz + }; + union { + char buf[CMSG_SPACE(max_num_fds * sizeof(int))]; + struct cmsghdr align; + } u; + + msg.msg_iov = &io; + msg.msg_iovlen = 1; + if(fds.size()) { + msg.msg_control = u.buf; + msg.msg_controllen = sizeof(u.buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int) * fds.size()); + unsigned char* p = CMSG_DATA(cmsg); + for(const wrapped_fd& fd : fds) { + int thisfd = fd; + memcpy(p, &thisfd, sizeof(thisfd)); + p += sizeof(thisfd); + } + } + + int wrote; + do { + wrote = sendmsg(fd_to_send_to, &msg, 0); + } while(wrote == -1 && errno == EINTR); + + return wrote >= 0; +} + +std::vector vector_for_memfd(const wrapped_fd& memfd) { + struct stat st; + FC_ASSERT(fstat(memfd, &st) == 0, "failed to get memfd size"); + + if(st.st_size == 0) + return std::vector(); + + uint8_t* p = (uint8_t*)mmap(nullptr, st.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, memfd, 0); + FC_ASSERT(p != MAP_FAILED, "failed to map memfd"); + std::vector ret(p, p+st.st_size); + munmap(p, st.st_size); + return ret; +} + +}}} diff --git a/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.cpp b/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.cpp new file mode 100644 index 00000000000..d0b99beb996 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.cpp @@ -0,0 +1,14 @@ +#include +namespace eosio { namespace chain { namespace eosvmoc { +namespace LLVMJIT { + +llvm::Value* CreateInBoundsGEPWAR(llvm::IRBuilder<>& irBuilder, llvm::Value* Ptr, llvm::Value* v1, llvm::Value* v2) { + if(!v2) + return irBuilder.CreateInBoundsGEP(Ptr, v1); + else + return irBuilder.CreateInBoundsGEP(Ptr, {v1, v2}); +} + +} + +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.llvmwar b/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.llvmwar new file mode 120000 index 00000000000..1e2596579da --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/llvmWARshim.llvmwar @@ -0,0 +1 @@ +llvmWARshim.cpp \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm-oc/memory.cpp b/libraries/chain/webassembly/eos-vm-oc/memory.cpp new file mode 100644 index 00000000000..6bf56f01d06 --- /dev/null +++ b/libraries/chain/webassembly/eos-vm-oc/memory.cpp @@ -0,0 +1,47 @@ +#include +#include + +#include + +#include +#include +#include +#include + +namespace eosio { namespace chain { namespace eosvmoc { + +memory::memory() { + int fd = syscall(SYS_memfd_create, "eosvmoc_mem", MFD_CLOEXEC); + FC_ASSERT(fd >= 0, "Failed to create memory memfd"); + auto cleanup_fd = fc::make_scoped_exit([&fd](){close(fd);}); + int ret = ftruncate(fd, wasm_memory_size+memory_prologue_size); + FC_ASSERT(!ret, "Failed to grow memory memfd"); + + mapsize = total_memory_per_slice*number_slices; + mapbase = (uint8_t*)mmap(nullptr, mapsize, PROT_NONE, MAP_PRIVATE|MAP_ANON, 0, 0); + FC_ASSERT(mapbase != MAP_FAILED, "Failed to mmap memory"); + + uint8_t* next_slice = mapbase; + uint8_t* last; + + for(unsigned int p = 0; p < number_slices; ++p) { + last = (uint8_t*)mmap(next_slice, memory_prologue_size+64u*1024u*p, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_FIXED, fd, 0); + FC_ASSERT(last != MAP_FAILED, "Failed to mmap memory"); + next_slice += total_memory_per_slice; + } + + zeropage_base = mapbase + memory_prologue_size; + fullpage_base = last + memory_prologue_size; + + //layout the intrinsic jump table + uintptr_t* const intrinsic_jump_table = reinterpret_cast(zeropage_base - first_intrinsic_offset); + const intrinsic_map_t& intrinsics = get_intrinsic_map(); + for(const auto& intrinsic : intrinsics) + intrinsic_jump_table[-intrinsic.second.ordinal] = (uintptr_t)intrinsic.second.function_ptr; +} + +memory::~memory() { + munmap(mapbase, mapsize); +} + +}}} \ No newline at end of file diff --git a/libraries/chain/webassembly/eos-vm.cpp b/libraries/chain/webassembly/eos-vm.cpp new file mode 100644 index 00000000000..c467b65c44f --- /dev/null +++ b/libraries/chain/webassembly/eos-vm.cpp @@ -0,0 +1,122 @@ +#include +#include +#include +#include +//eos-vm includes +#include + +namespace eosio { namespace chain { namespace webassembly { namespace eos_vm_runtime { + +using namespace eosio::vm; + +namespace wasm_constraints = eosio::chain::wasm_constraints; + +namespace { + + struct checktime_watchdog { + checktime_watchdog(transaction_checktime_timer& timer) : _timer(timer) {} + template + struct guard { + guard(transaction_checktime_timer& timer, F&& func) + : _timer(timer), _func(static_cast(func)) { + _timer.set_expiration_callback(&callback, this); + if(_timer.expired) { + _func(); // it's harmless if _func is invoked twice + } + } + ~guard() { + _timer.set_expiration_callback(nullptr, nullptr); + } + static void callback(void* data) { + guard* self = static_cast(data); + self->_func(); + } + transaction_checktime_timer& _timer; + F _func; + }; + template + guard scoped_run(F&& func) { + return guard{_timer, static_cast(func)}; + } + transaction_checktime_timer& _timer; + }; + +} + +template +class eos_vm_instantiated_module : public wasm_instantiated_module_interface { + using backend_t = backend; + public: + + eos_vm_instantiated_module(eos_vm_runtime* runtime, std::unique_ptr mod) : + _runtime(runtime), + _instantiated_module(std::move(mod)) {} + + void apply(apply_context& context) override { + _instantiated_module->set_wasm_allocator(&context.control.get_wasm_allocator()); + _runtime->_bkend = _instantiated_module.get(); + _runtime->_bkend->initialize(&context); + // clamp WASM memory to maximum_linear_memory/wasm_page_size + auto& module = _runtime->_bkend->get_module(); + if (module.memories.size() && + ((module.memories.at(0).limits.maximum > wasm_constraints::maximum_linear_memory / wasm_constraints::wasm_page_size) + || !module.memories.at(0).limits.flags)) { + module.memories.at(0).limits.flags = true; + module.memories.at(0).limits.maximum = wasm_constraints::maximum_linear_memory / wasm_constraints::wasm_page_size; + } + auto fn = [&]() { + const auto& res = _runtime->_bkend->call( + &context, "env", "apply", context.get_receiver().to_uint64_t(), + context.get_action().account.to_uint64_t(), + context.get_action().name.to_uint64_t()); + }; + try { + checktime_watchdog wd(context.trx_context.transaction_timer); + _runtime->_bkend->timed_run(wd, fn); + } catch(eosio::vm::timeout_exception&) { + context.trx_context.checktime(); + } catch(eosio::vm::wasm_memory_exception& e) { + FC_THROW_EXCEPTION(wasm_execution_error, "access violation"); + } catch(eosio::vm::exception& e) { + // FIXME: Do better translation + FC_THROW_EXCEPTION(wasm_execution_error, "something went wrong..."); + } + _runtime->_bkend = nullptr; + } + + private: + eos_vm_runtime* _runtime; + std::unique_ptr _instantiated_module; +}; + +template +eos_vm_runtime::eos_vm_runtime() {} + +template +void eos_vm_runtime::immediately_exit_currently_running_module() { + throw wasm_exit{}; +} + +template +bool eos_vm_runtime::inject_module(IR::Module& module) { + return false; +} + +template +std::unique_ptr eos_vm_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector, + const digest_type&, const uint8_t&, const uint8_t&) { + using backend_t = backend; + try { + wasm_code_ptr code((uint8_t*)code_bytes, code_size); + std::unique_ptr bkend = std::make_unique(code, code_size); + registered_host_functions::resolve(bkend->get_module()); + return std::make_unique>(this, std::move(bkend)); + } catch(eosio::vm::exception& e) { + FC_THROW_EXCEPTION(wasm_execution_error, "Error building eos-vm interp: ${e}", ("e", e.what())); + } +} + +template class eos_vm_runtime; +template class eos_vm_runtime; + +}}}} diff --git a/libraries/eos-vm b/libraries/eos-vm new file mode 160000 index 00000000000..0b8c291c082 --- /dev/null +++ b/libraries/eos-vm @@ -0,0 +1 @@ +Subproject commit 0b8c291c08280394cab1a21f88a8cb24ebd1050b diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 5eaf21de416..6cf772b6adb 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -101,6 +101,12 @@ namespace eosio { namespace testing { cfg.wasm_runtime = chain::wasm_interface::vm_type::wavm; else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wabt")) cfg.wasm_runtime = chain::wasm_interface::vm_type::wabt; + else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm")) + cfg.wasm_runtime = chain::wasm_interface::vm_type::eos_vm; + else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm-jit")) + cfg.wasm_runtime = chain::wasm_interface::vm_type::eos_vm_jit; + else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--eos-vm-oc")) + cfg.wasm_runtime = chain::wasm_interface::vm_type::eos_vm_oc; } open(nullptr); diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 944d1f2d7af..ff20e506064 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -255,8 +255,16 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to application data dir)") ("checkpoint", bpo::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.") - ("wasm-runtime", bpo::value()->value_name("wavm/wabt"), "Override default WASM runtime") - ("abi-serializer-max-time-ms", bpo::value()->default_value(config::default_abi_serializer_max_time_ms), "Override default maximum ABI serialization time allowed in ms") + ("wasm-runtime", bpo::value()->value_name("runtime")->notifier([](const auto& vm){ +#ifndef EOSIO_EOS_VM_OC_DEVELOPER + //throwing an exception here (like EOS_ASSERT) is just gobbled up with a "Failed to initialize" error :( + if(vm == wasm_interface::vm_type::eos_vm_oc) { + elog("EOS VM OC is a tier-up compiler and works in conjunction with the configured base WASM runtime. Enable EOS VM OC via 'eos-vm-oc-enable' option"); + EOS_ASSERT(false, plugin_exception, ""); + } +#endif + }), "Override default WASM runtime") ("abi-serializer-max-time-ms", bpo::value()->default_value(config::default_abi_serializer_max_time_ms), + "Override default maximum ABI serialization time allowed in ms") ("chain-state-db-size-mb", bpo::value()->default_value(config::default_state_size / (1024 * 1024)), "Maximum size (in MiB) of the chain state database") ("chain-state-db-guard-size-mb", bpo::value()->default_value(config::default_state_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the chain state database drops below this size (in MiB).") ("reversible-blocks-db-size-mb", bpo::value()->default_value(config::default_reversible_cache_size / (1024 * 1024)), "Maximum size (in MiB) of the reversible blocks database") @@ -295,6 +303,16 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("disable-ram-billing-notify-checks", bpo::bool_switch()->default_value(false), "Disable the check which subjectively fails a transaction if a contract bills more RAM to another account within the context of a notification handler (i.e. when the receiver is not the code of the action).") ("trusted-producer", bpo::value>()->composing(), "Indicate a producer whose blocks headers signed by it will be fully validated, but transactions in those validated blocks will be trusted.") +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + ("eos-vm-oc-cache-size-mb", bpo::value()->default_value(eosvmoc::config().cache_size / (1024u*1024u)), "Maximum size (in MiB) of the EOS VM OC code cache") + ("eos-vm-oc-compile-threads", bpo::value()->default_value(1u)->notifier([](const auto t) { + if(t == 0) { + elog("eos-vm-oc-compile-threads must be set to a non-zero value"); + EOS_ASSERT(false, plugin_exception, ""); + } + }), "Number of threads to use for EOS VM OC tier-up") + ("eos-vm-oc-enable", bpo::bool_switch(), "Enable EOS VM OC tier-up runtime") +#endif ; // TODO: rate limiting @@ -736,6 +754,15 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->chain_config->block_validation_mode = options.at("validation-mode").as(); } +#ifdef EOSIO_EOS_VM_OC_RUNTIME_ENABLED + if( options.count("eos-vm-oc-cache-size-mb") ) + my->chain_config->eosvmoc_config.cache_size = options.at( "eos-vm-oc-cache-size-mb" ).as() * 1024u * 1024u; + if( options.count("eos-vm-oc-compile-threads") ) + my->chain_config->eosvmoc_config.threads = options.at("eos-vm-oc-compile-threads").as(); + if( options["eos-vm-oc-enable"].as() ) + my->chain_config->eosvmoc_tierup = true; +#endif + my->chain.emplace( *my->chain_config ); my->chain_id.emplace( my->chain->get_chain_id()); diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index 94d749deb34..f6672720ec8 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -401,16 +401,12 @@ namespace eosio { if (v) ilog("configured http with Access-Control-Allow-Credentials: true"); })->default_value(false), "Specify if Access-Control-Allow-Credentials: true should be returned on each request.") - ("max-body-size", bpo::value()->default_value(1024*1024), - "The maximum body size in bytes allowed for incoming RPC requests") - ("http-max-bytes-in-flight-mb", bpo::value()->default_value(500), - "Maximum size in megabytes http_plugin should use for processing http requests. 503 error response when exceeded." ) - ("verbose-http-errors", bpo::bool_switch()->default_value(false), - "Append the error log to HTTP responses") - ("http-validate-host", boost::program_options::value()->default_value(true), - "If set to false, then any incoming \"Host\" header is considered valid") - ("http-alias", bpo::value>()->composing(), - "Additionaly acceptable values for the \"Host\" header of incoming HTTP requests, can be specified multiple times. Includes http/s_server_address by default.") + ("max-body-size", bpo::value()->default_value(1024*1024), "The maximum body size in bytes allowed for incoming RPC requests") + ("http-max-bytes-in-flight-mb", bpo::value()->default_value(500), + "Maximum size in megabytes http_plugin should use for processing http requests. 503 error response when exceeded." ) + ("verbose-http-errors", bpo::bool_switch()->default_value(false), "Append the error log to HTTP responses") + ("http-validate-host", boost::program_options::value()->default_value(true), "If set to false, then any incoming \"Host\" header is considered valid") + ("http-alias", bpo::value>()->composing(), "Additionaly acceptable values for the \"Host\" header of incoming HTTP requests, can be specified multiple times. Includes http/s_server_address by default.") ; } diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp index afa9fb30365..69a896c05df 100644 --- a/plugins/producer_api_plugin/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/producer_api_plugin.cpp @@ -39,6 +39,9 @@ using namespace eosio; #define INVOKE_R_R(api_handle, call_name, in_param) \ auto result = api_handle.call_name(fc::json::from_string(body).as()); +#define INVOKE_R_R_O(api_handle, call_name, in_param) \ + auto result = (body == "{}" ? api_handle.call_name() : api_handle.call_name(fc::json::from_string(body).as())); + #define INVOKE_R_R_R_R(api_handle, call_name, in_param0, in_param1, in_param2) \ const auto& vs = fc::json::json::from_string(body).as(); \ auto result = api_handle.call_name(vs.at(0).as(), vs.at(1).as(), vs.at(2).as()); @@ -89,7 +92,7 @@ void producer_api_plugin::plugin_startup() { CALL(producer, producer, get_integrity_hash, INVOKE_R_V(producer, get_integrity_hash), 201), CALL(producer, producer, create_snapshot, - INVOKE_R_R(producer, create_snapshot, producer_plugin::export_snapshot_type), 201), + INVOKE_R_R_O(producer, create_snapshot, producer_plugin::export_snapshot_type), 201), }); } diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index 0208ba9f83d..141b92d4a32 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -85,7 +85,7 @@ class producer_plugin : public appbase::plugin { void set_whitelist_blacklist(const whitelist_blacklist& params); integrity_hash_information get_integrity_hash() const; - snapshot_information create_snapshot(export_snapshot_type type) const; + snapshot_information create_snapshot(export_snapshot_type type=export_snapshot_type::snapshot) const; signal confirmed_block; private: diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 288837205be..9a5ad5f64ba 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -975,7 +975,7 @@ producer_plugin::snapshot_information producer_plugin::create_blocks_snapshot(ch producer_plugin::snapshot_information producer_plugin::create_acts_snapshot(chain::controller& chain) const { auto head_id = chain.head_block_id(); - std::string acts_snapshot_path = (my->_snapshots_dir / fc::format_string("acts-snapshot-${id}.txt", fc::mutable_variant_object()("id", head_id))).generic_string(); + std::string acts_snapshot_path = (my->_snapshots_dir / fc::format_string("acts-snapshot-${id}.csv", fc::mutable_variant_object()("id", head_id))).generic_string(); EOS_ASSERT( !fc::is_regular_file(acts_snapshot_path), snapshot_exists_exception, "acts-snapshot named ${name} already exists", ("name", acts_snapshot_path));