diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3c7f63def07..893a19f9ff9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -398,6 +398,7 @@ steps: - "os=high-sierra" artifact_paths: - "build/packages/*.tar.gz" + - "build/packages/*.rb" timeout: 60 - command: | @@ -412,6 +413,7 @@ steps: - "os=mojave" artifact_paths: - "build/packages/*.tar.gz" + - "build/packages/*.rb" timeout: 60 - command: | @@ -507,3 +509,18 @@ steps: OS: "el7" PKGTYPE: "rpm" timeout: 60 + + - wait + + - command: | + echo "--- :arrow_down: Downloading brew files" && \ + buildkite-agent artifact download "build/packages/eosio.rb" . --step ":darwin: High Sierra Package Builder" && \ + mv build/packages/eosio.rb build/packages/eosio_highsierra.rb && \ + buildkite-agent artifact download "build/packages/eosio.rb" . --step ":darwin: Mojave Package Builder" + label: ":darwin: Brew Updater" + agents: + queue: "automation-large-builder-fleet" + artifact_paths: + - "build/packages/eosio_highsierra.rb" + - "build/packages/eosio.rb" + timeout: 60 diff --git a/.gitmodules b/.gitmodules index 7d0f8a37f7b..205a731ffce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,4 +29,4 @@ url = https://github.com/EOSIO/fc [submodule "libraries/wabt"] path = libraries/wabt - url = http://github.com/EOSIO/wabt + url = https://github.com/EOSIO/wabt diff --git a/CMakeLists.txt b/CMakeLists.txt index 2303eee93b3..daacda9a0c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") if (UNIX) if (APPLE) if (LLVM_DIR STREQUAL "" OR NOT LLVM_DIR) - set(LLVM_DIR "/usr/local/Cellar/llvm@4/4.0.1/lib/cmake/llvm") + set(LLVM_DIR "/usr/local/opt/llvm@4/lib/cmake/llvm/") endif() endif() endif() @@ -35,7 +35,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 2) set(VERSION_MINOR 0) -set(VERSION_PATCH 1) +set(VERSION_PATCH 2) if(VERSION_SUFFIX) set(VERSION_FULL "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}-${VERSION_SUFFIX}") diff --git a/Docker/README.md b/Docker/README.md index 582b8936e0d..5be2cbe6a1f 100644 --- a/Docker/README.md +++ b/Docker/README.md @@ -20,10 +20,10 @@ cd bos/Docker docker build . -t boscore/bos -s BOS ``` -The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v2.0.1 tag, you could do the following: +The above will build off the most recent commit to the master branch by default. If you would like to target a specific branch/tag, you may use a build argument. For example, if you wished to generate a docker image based off of the v2.0.2 tag, you could do the following: ```bash -docker build -t boscore/bos:v2.0.1 --build-arg branch=v2.0.1 . +docker build -t boscore/bos:v2.0.2 --build-arg branch=v2.0.2 . ``` @@ -198,4 +198,4 @@ The `blocks` data are stored under `--data-dir` by default, and the wallet files ### About MongoDB Plugin -Currently, the mongodb plugin is disabled in `config.ini` by default, you have to change it manually in `config.ini` or you can mount a `config.ini` file to `/opt/eosio/bin/data-dir/config.ini` in the docker-compose file. +Currently, the mongodb plugin is disabled in `config.ini` by default, you have to change it manually in `config.ini` or you can mount a `config.ini` file to `/opt/eosio/bin/data-dir/config.ini` in the docker-compose file. \ No newline at end of file diff --git a/Docker/config.ini b/Docker/config.ini index 71ae5c6c0ed..9014d16a888 100644 --- a/Docker/config.ini +++ b/Docker/config.ini @@ -113,9 +113,6 @@ network-version-match = 0 # number of blocks to retrieve in a chunk from any individual peer during synchronization (eosio::net_plugin) sync-fetch-span = 100 -# maximum sizes of transaction or block messages that are sent without first sending a notice (eosio::net_plugin) -max-implicit-request = 1500 - # Enable block production, even if the chain is stale. (eosio::producer_plugin) enable-stale-production = false diff --git a/README.md b/README.md index a237aa1cf77..eae2abca3f3 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # BOSCore - Born for DApps. Born for Usability. -## BOSCore Version: v2.0.1 -### Basic EOSIO Version: v1.5.3-patched +## BOSCore Version: v2.0.2 +### Basic EOSIO Version: v1.6.2 # Background The emergence of EOS has brought new imagination to the blockchain. In just a few months since the main network was launched, the version has undergone dozens of upgrades, not only the stability has been greatly improved, but also the new functions have been gradually realized. The node team is also actively involved in building the EOSIO ecosystem. What is even more exciting is that EOS has attracted more and more development teams. There are already hundreds of DApp running on the EOS main network. The transaction volume and circulation market value far exceed Ethereum, and the space for development is growing broader. @@ -29,6 +29,15 @@ As BOS continues to develop, developer rewards will be appropriately adjusted to 1. Build from code : `bash ./eosio_build.sh -s BOS` 2. Docker Style，check [Docker](./Docker/README.md) +## BOSCore Workflow +BOSCore encourage community developer actively participate in contributing the code, members should follow the workflow below. +![BOSCore Workflow](./images/bos-workflow.png) + +Attention: +1. Only allow Feature Branch or bug fix to submit PR to Develop Branch. +2. Rebase is required before submitting PR to Develop Branch. +3. Treat update of eosio/eos code as new feature. +4. Emergent issues must repaired by adopting hotfixes mode. BOSCore bases on EOSIO, so you can also referer: @@ -36,4 +45,3 @@ BOSCore bases on EOSIO, so you can also referer: [EOSIO Developer Portal](https://developers.eos.io). - diff --git a/README_CN.md b/README_CN.md index e8e8d3a9d41..2c1b2434dc8 100644 --- a/README_CN.md +++ b/README_CN.md @@ -1,7 +1,7 @@ # BOSCore - 更可用的链，为DApp而生。 -## BOSCore Version: v2.0.1 -### Basic EOSIO Version: v1.5.3-patched +## BOSCore Version: v2.0.2 +### Basic EOSIO Version: v1.6.2 # 背景 EOS的出现给区块链带来了新的想象力，主网启动短短几个月以来，版本经历了几十次升级，不仅稳定性得到了很大提高，并且新功能也逐步实现，各个节点团队也积极参与建设EOSIO生态。让人更加兴奋的是，EOS已经吸引了越来越多的开发团队，当前已经有数百个DApp在EOS主网上面运行，其交易量和流通市值远超以太坊，可发展的空间愈来愈广阔。 @@ -29,6 +29,16 @@ BOS链的代码完全由社区贡献并维护，每个生态参与者都可以 1. 源码直接编译: `bash ./eosio_build.sh -s BOS` 2. Docker方式部署，参看 [Docker](./Docker/README.md) +## BOSCore 开发流程 +BOSCore 鼓励社区开发者参与代码贡献，社区成员应当遵循以下工作流： +![BOSCore Workflow](./images/bos-workflow.png) + +注意: +1. 只有待发布的 Feature Branch 或者Bug修复才应该向 Develop Branch 提交 +2. 向 Develop Branch 提交 PR 之前需要现在本地执行 rebase 操作 +3. EOSIO 主网版本作为一个 Feature Branch 来对待 +4. 紧急问题修复采用 hotfixes 模式 + BOSCore是基于EOSIO技术的扩展，所以EOSIO的相关资料也可以参考： [EOSIO 开始](https://developers.eos.io/eosio-nodeos/docs/overview-1) diff --git a/contracts/asserter/asserter.cpp b/contracts/asserter/asserter.cpp index f64ea509a1f..c89db1fc709 100644 --- a/contracts/asserter/asserter.cpp +++ b/contracts/asserter/asserter.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include /// defines assert_def struct (abi) diff --git a/contracts/asserter/asserter.hpp b/contracts/asserter/asserter.hpp index 99680870c04..01d4b343e3f 100644 --- a/contracts/asserter/asserter.hpp +++ b/contracts/asserter/asserter.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/eosio.system/delegate_bandwidth.cpp b/contracts/eosio.system/delegate_bandwidth.cpp index fd4e83fcce7..a2920d70295 100644 --- a/contracts/eosio.system/delegate_bandwidth.cpp +++ b/contracts/eosio.system/delegate_bandwidth.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include "eosio.system.hpp" diff --git a/contracts/eosio.system/eosio.system.hpp b/contracts/eosio.system/eosio.system.hpp index 66964e39659..2b6d16d3d59 100644 --- a/contracts/eosio.system/eosio.system.hpp +++ b/contracts/eosio.system/eosio.system.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosio.system/native.hpp b/contracts/eosio.system/native.hpp index e6395807395..e2bcb319575 100644 --- a/contracts/eosio.system/native.hpp +++ b/contracts/eosio.system/native.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosio.system/voting.cpp b/contracts/eosio.system/voting.cpp index feeb53fc3d8..8076f79886d 100644 --- a/contracts/eosio.system/voting.cpp +++ b/contracts/eosio.system/voting.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include "eosio.system.hpp" diff --git a/contracts/eosio.token/eosio.token.cpp b/contracts/eosio.token/eosio.token.cpp index 351d81c47c5..4570b19b5c9 100644 --- a/contracts/eosio.token/eosio.token.cpp +++ b/contracts/eosio.token/eosio.token.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include "eosio.token.hpp" diff --git a/contracts/eosio.token/eosio.token.hpp b/contracts/eosio.token/eosio.token.hpp index 15875134777..48312c9edb4 100644 --- a/contracts/eosio.token/eosio.token.hpp +++ b/contracts/eosio.token/eosio.token.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/action.h b/contracts/eosiolib/action.h index 3e686fe469d..f8deba837b8 100644 --- a/contracts/eosiolib/action.h +++ b/contracts/eosiolib/action.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/action.hpp b/contracts/eosiolib/action.hpp index 11028f09e93..d4538c2972e 100644 --- a/contracts/eosiolib/action.hpp +++ b/contracts/eosiolib/action.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/chain.h b/contracts/eosiolib/chain.h index 8477968be9e..ec4f219b4b9 100644 --- a/contracts/eosiolib/chain.h +++ b/contracts/eosiolib/chain.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/compiler_builtins.h b/contracts/eosiolib/compiler_builtins.h index 3e0d9435357..c2dc09bb356 100644 --- a/contracts/eosiolib/compiler_builtins.h +++ b/contracts/eosiolib/compiler_builtins.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/core_symbol.hpp.in b/contracts/eosiolib/core_symbol.hpp.in index 4a60220b6a9..3262d2aad10 100644 --- a/contracts/eosiolib/core_symbol.hpp.in +++ b/contracts/eosiolib/core_symbol.hpp.in @@ -1,5 +1,5 @@ /** @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See core_symbol.hpp.in for changes. */ diff --git a/contracts/eosiolib/crypto.h b/contracts/eosiolib/crypto.h index 916c25954f7..0133762c43b 100644 --- a/contracts/eosiolib/crypto.h +++ b/contracts/eosiolib/crypto.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/datastream.hpp b/contracts/eosiolib/datastream.hpp index 65666e78cc4..97aed9f4cf7 100644 --- a/contracts/eosiolib/datastream.hpp +++ b/contracts/eosiolib/datastream.hpp @@ -1,6 +1,6 @@ /** * @file datastream.hpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/db.h b/contracts/eosiolib/db.h index d2c01b6b15d..f112ef5bf6d 100644 --- a/contracts/eosiolib/db.h +++ b/contracts/eosiolib/db.h @@ -1,6 +1,6 @@ /** * @file db.h - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * @brief Defines C API for interfacing with blockchain database */ #pragma once diff --git a/contracts/eosiolib/eosio.hpp b/contracts/eosiolib/eosio.hpp index 3e850a69dea..2126abb3203 100644 --- a/contracts/eosiolib/eosio.hpp +++ b/contracts/eosiolib/eosio.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/fixed_key.hpp b/contracts/eosiolib/fixed_key.hpp index eb04a45b46c..4e8cd8241da 100644 --- a/contracts/eosiolib/fixed_key.hpp +++ b/contracts/eosiolib/fixed_key.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/memory.h b/contracts/eosiolib/memory.h index 58e5a482b3d..474e7c7b598 100644 --- a/contracts/eosiolib/memory.h +++ b/contracts/eosiolib/memory.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/memory.hpp b/contracts/eosiolib/memory.hpp index 4a4f56a1c48..f2abbf898d2 100644 --- a/contracts/eosiolib/memory.hpp +++ b/contracts/eosiolib/memory.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/multi_index.hpp b/contracts/eosiolib/multi_index.hpp index d7082559059..cba34f103a0 100644 --- a/contracts/eosiolib/multi_index.hpp +++ b/contracts/eosiolib/multi_index.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/permission.h b/contracts/eosiolib/permission.h index 8acd07fd553..743b499bca8 100644 --- a/contracts/eosiolib/permission.h +++ b/contracts/eosiolib/permission.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/permission.hpp b/contracts/eosiolib/permission.hpp index a6956f5dd28..668fa79bff9 100644 --- a/contracts/eosiolib/permission.hpp +++ b/contracts/eosiolib/permission.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/print.h b/contracts/eosiolib/print.h index 63fd1da9d54..ab7dfcc8ade 100644 --- a/contracts/eosiolib/print.h +++ b/contracts/eosiolib/print.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/print.hpp b/contracts/eosiolib/print.hpp index a90a277cf47..84bdf263aa3 100644 --- a/contracts/eosiolib/print.hpp +++ b/contracts/eosiolib/print.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/system.h b/contracts/eosiolib/system.h index 62a5cfeef51..b7d3e614107 100644 --- a/contracts/eosiolib/system.h +++ b/contracts/eosiolib/system.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/transaction.h b/contracts/eosiolib/transaction.h index 95a41322acc..0cbe0f67a57 100644 --- a/contracts/eosiolib/transaction.h +++ b/contracts/eosiolib/transaction.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/transaction.hpp b/contracts/eosiolib/transaction.hpp index 1586b92b09b..2b5e1f1fb2a 100644 --- a/contracts/eosiolib/transaction.hpp +++ b/contracts/eosiolib/transaction.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/types.h b/contracts/eosiolib/types.h index 120c896cd87..1611df2aad2 100644 --- a/contracts/eosiolib/types.h +++ b/contracts/eosiolib/types.h @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/eosiolib/types.hpp b/contracts/eosiolib/types.hpp index d535f49e922..f1e7c10dcd1 100644 --- a/contracts/eosiolib/types.hpp +++ b/contracts/eosiolib/types.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/contracts/eosiolib/varint.hpp b/contracts/eosiolib/varint.hpp index b8240839aa0..dcc444f20d6 100644 --- a/contracts/eosiolib/varint.hpp +++ b/contracts/eosiolib/varint.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/noop/noop.cpp b/contracts/noop/noop.cpp index 67f3f101f14..6699eca57c9 100644 --- a/contracts/noop/noop.cpp +++ b/contracts/noop/noop.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/proxy/proxy.cpp b/contracts/proxy/proxy.cpp index ff27e5d5af2..3d243fd4e02 100644 --- a/contracts/proxy/proxy.cpp +++ b/contracts/proxy/proxy.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/proxy/proxy.hpp b/contracts/proxy/proxy.hpp index fd439e3c274..924edd24b10 100644 --- a/contracts/proxy/proxy.hpp +++ b/contracts/proxy/proxy.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/skeleton/skeleton.hpp b/contracts/skeleton/skeleton.hpp index ec28c3d168b..2b621b6f2ed 100644 --- a/contracts/skeleton/skeleton.hpp +++ b/contracts/skeleton/skeleton.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/stltest/stltest.cpp b/contracts/stltest/stltest.cpp index 79828877f98..baf975ff703 100644 --- a/contracts/stltest/stltest.cpp +++ b/contracts/stltest/stltest.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ // include entire libc #include diff --git a/contracts/test_api/test_action.cpp b/contracts/test_api/test_action.cpp index 496b30fcc37..20eacaa1bd8 100644 --- a/contracts/test_api/test_action.cpp +++ b/contracts/test_api/test_action.cpp @@ -1,6 +1,6 @@ /** * @file action_test.cpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_api.cpp b/contracts/test_api/test_api.cpp index 9ddcb712752..ce7b44d6c9f 100644 --- a/contracts/test_api/test_api.cpp +++ b/contracts/test_api/test_api.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_api.hpp b/contracts/test_api/test_api.hpp index 4dc5711e3a1..5406dff5520 100644 --- a/contracts/test_api/test_api.hpp +++ b/contracts/test_api/test_api.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include "test_api_common.hpp" diff --git a/contracts/test_api/test_api_common.hpp b/contracts/test_api/test_api_common.hpp index 8d63a535089..d78afa29bb3 100644 --- a/contracts/test_api/test_api_common.hpp +++ b/contracts/test_api/test_api_common.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/contracts/test_api/test_chain.cpp b/contracts/test_api/test_chain.cpp index 6c71eb78eda..22f4ca4f914 100644 --- a/contracts/test_api/test_chain.cpp +++ b/contracts/test_api/test_chain.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_checktime.cpp b/contracts/test_api/test_checktime.cpp index 4f6bef4659a..a227ee91b04 100644 --- a/contracts/test_api/test_checktime.cpp +++ b/contracts/test_api/test_checktime.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/test_api/test_compiler_builtins.cpp b/contracts/test_api/test_compiler_builtins.cpp index 1492110b82e..a7571b0a2ce 100644 --- a/contracts/test_api/test_compiler_builtins.cpp +++ b/contracts/test_api/test_compiler_builtins.cpp @@ -1,6 +1,6 @@ /** * @file test_compiler_builtins.cpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/test_api/test_crypto.cpp b/contracts/test_api/test_crypto.cpp index 526ec4170af..3b96e6e6fad 100644 --- a/contracts/test_api/test_crypto.cpp +++ b/contracts/test_api/test_crypto.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_permission.cpp b/contracts/test_api/test_permission.cpp index 824e8434b7d..f26c120a0a1 100644 --- a/contracts/test_api/test_permission.cpp +++ b/contracts/test_api/test_permission.cpp @@ -1,6 +1,6 @@ /** * @file action_test.cpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_print.cpp b/contracts/test_api/test_print.cpp index 310834037b2..a042ff6f57a 100644 --- a/contracts/test_api/test_print.cpp +++ b/contracts/test_api/test_print.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/test_api/test_transaction.cpp b/contracts/test_api/test_transaction.cpp index 95030419f58..cc1128ae5f6 100644 --- a/contracts/test_api/test_transaction.cpp +++ b/contracts/test_api/test_transaction.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/test_api/test_types.cpp b/contracts/test_api/test_types.cpp index d3cf87b9614..bc9dca52e2a 100644 --- a/contracts/test_api/test_types.cpp +++ b/contracts/test_api/test_types.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/test_api_db/test_api_db.cpp b/contracts/test_api_db/test_api_db.cpp index 706734ba967..d603477a631 100644 --- a/contracts/test_api_db/test_api_db.cpp +++ b/contracts/test_api_db/test_api_db.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include "../test_api/test_api.hpp" diff --git a/contracts/test_api_mem/test_api_mem.cpp b/contracts/test_api_mem/test_api_mem.cpp index e01bba16378..ce59249425d 100644 --- a/contracts/test_api_mem/test_api_mem.cpp +++ b/contracts/test_api_mem/test_api_mem.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include "../test_api/test_api.hpp" diff --git a/contracts/test_api_mem/test_memory.cpp b/contracts/test_api_mem/test_memory.cpp index d3f387fd8b1..3e1eb52243e 100644 --- a/contracts/test_api_mem/test_memory.cpp +++ b/contracts/test_api_mem/test_memory.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/contracts/test_api_multi_index/test_api_multi_index.cpp b/contracts/test_api_multi_index/test_api_multi_index.cpp index 9c5713385ad..828a1d75b88 100644 --- a/contracts/test_api_multi_index/test_api_multi_index.cpp +++ b/contracts/test_api_multi_index/test_api_multi_index.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include "../test_api/test_api.hpp" diff --git a/contracts/test_ram_limit/test_ram_limit.cpp b/contracts/test_ram_limit/test_ram_limit.cpp index 136c27aa72c..c162cea6585 100644 --- a/contracts/test_ram_limit/test_ram_limit.cpp +++ b/contracts/test_ram_limit/test_ram_limit.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/contracts/tic_tac_toe/tic_tac_toe.cpp b/contracts/tic_tac_toe/tic_tac_toe.cpp index ebc5cc46203..a6eb62eccf6 100644 --- a/contracts/tic_tac_toe/tic_tac_toe.cpp +++ b/contracts/tic_tac_toe/tic_tac_toe.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include "tic_tac_toe.hpp" diff --git a/contracts/tic_tac_toe/tic_tac_toe.hpp b/contracts/tic_tac_toe/tic_tac_toe.hpp index 4339ab212e1..f4f32982ff6 100644 --- a/contracts/tic_tac_toe/tic_tac_toe.hpp +++ b/contracts/tic_tac_toe/tic_tac_toe.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/eosio_build.sh b/eosio_build.sh index fafdbbaeb8e..b600e0905ea 100755 --- a/eosio_build.sh +++ b/eosio_build.sh @@ -34,10 +34,14 @@ function usage() { - printf "\\tUsage: %s \\n\\t[Build Option -o ] \\n\\t[CodeCoverage -c] \\n\\t[Doxygen -d] \\n\\t[CoreSymbolName -s <1-7 characters>] \\n\\t[Avoid Compiling -a]\\n\\n" "$0" 1>&2 + printf "\\tUsage: %s \\n\\t[Build Option -o ] \\n\\t[CodeCoverage -c] \\n\\t[Doxygen -d] \\n\\t[CoreSymbolName -s <1-7 characters>] \\n\\t[Avoid Compiling -a]\\n\\t[Noninteractive -y]\\n\\n" "$0" 1>&2 exit 1 } + is_noninteractive() { + [[ -n "${EOSIO_BUILD_NONINTERACTIVE+1}" ]] + } + ARCH=$( uname ) if [ "${SOURCE_DIR}" == "${PWD}" ]; then BUILD_DIR="${PWD}/build" @@ -66,7 +70,7 @@ txtrst=$(tput sgr0) if [ $# -ne 0 ]; then - while getopts ":cdo:s:ah" opt; do + while getopts ":cdo:s:ahy" opt; do case "${opt}" in o ) options=( "Debug" "Release" "RelWithDebInfo" "MinSizeRel" ) @@ -100,6 +104,9 @@ usage exit 1 ;; + y) + EOSIO_BUILD_NONINTERACTIVE=1 + ;; \? ) printf "\\n\\tInvalid Option: %s\\n" "-${OPTARG}" 1>&2 usage @@ -238,7 +245,7 @@ . "$FILE" - printf "\\n\\n>>>>>>>> ALL dependencies sucessfully found or installed . Installing BOSCore\\n\\n" + printf "\\n\\n>>>>>>>> ALL dependencies successfully found or installed . Installing EOSIO\\n\\n" printf ">>>>>>>> CMAKE_BUILD_TYPE=%s\\n" "${CMAKE_BUILD_TYPE}" printf ">>>>>>>> ENABLE_COVERAGE_TESTING=%s\\n" "${ENABLE_COVERAGE_TESTING}" printf ">>>>>>>> DOXYGEN=%s\\n\\n" "${DOXYGEN}" diff --git a/images/bos-workflow.png b/images/bos-workflow.png new file mode 100644 index 00000000000..6c00b9cf715 Binary files /dev/null and b/images/bos-workflow.png differ diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt index 18013317c57..eeaf5afa771 100644 --- a/libraries/CMakeLists.txt +++ b/libraries/CMakeLists.txt @@ -7,8 +7,9 @@ add_subdirectory( appbase ) add_subdirectory( chain ) add_subdirectory( testing ) -#turn these off for now +#turn tools&tests off; not needed for library build set(BUILD_TESTS OFF CACHE BOOL "Build GTest-based tests") set(BUILD_TOOLS OFF CACHE BOOL "Build wabt tools") set(RUN_RE2C OFF CACHE BOOL "Run re2c") +set(WITH_EXCEPTIONS ON CACHE BOOL "Build with exceptions enabled" FORCE) add_subdirectory( wabt ) diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index ee8fe1ebab2..2c430fecea0 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -46,7 +46,7 @@ add_library( eosio_chain # contracts/chain_initializer.cpp -# transaction_metadata.cpp + transaction_metadata.cpp ${HEADERS} ) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index b5f67e51059..2f453e0cf93 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 3a0de3b1466..44a3970f230 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -525,9 +525,8 @@ int apply_context::db_store_i64( uint64_t code, uint64_t scope, uint64_t table, const auto& obj = db.create( [&]( auto& o ) { o.t_id = tableid; o.primary_key = id; - o.value.resize( buffer_size ); + o.value.assign( buffer, buffer_size ); o.payer = payer; - memcpy( o.value.data(), buffer, buffer_size ); }); db.modify( tab, [&]( auto& t ) { @@ -566,8 +565,7 @@ void apply_context::db_update_i64( int iterator, account_name payer, const char* } db.modify( obj, [&]( auto& o ) { - o.value.resize( buffer_size ); - memcpy( o.value.data(), buffer, buffer_size ); + o.value.assign( buffer, buffer_size ); o.payer = payer; }); } diff --git a/libraries/chain/asset.cpp b/libraries/chain/asset.cpp index 142e04cffa3..580156f6d21 100644 --- a/libraries/chain/asset.cpp +++ b/libraries/chain/asset.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/libraries/chain/authorization_manager.cpp b/libraries/chain/authorization_manager.cpp index 6725468cf97..e69f7129121 100644 --- a/libraries/chain/authorization_manager.cpp +++ b/libraries/chain/authorization_manager.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/libraries/chain/block_header.cpp b/libraries/chain/block_header.cpp index b73338b3ca2..bbc9de7b06b 100644 --- a/libraries/chain/block_header.cpp +++ b/libraries/chain/block_header.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index dc769cc9612..41e9756483e 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -473,7 +473,7 @@ namespace eosio { namespace chain { old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); } if( pos != tmp_pos ) { - bad_block = tmp; + bad_block.emplace(std::move(tmp)); break; } diff --git a/libraries/chain/chain_config.cpp b/libraries/chain/chain_config.cpp index efb66bba95f..c39d89cee46 100644 --- a/libraries/chain/chain_config.cpp +++ b/libraries/chain/chain_config.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/libraries/chain/chain_id_type.cpp b/libraries/chain/chain_id_type.cpp index efddb213c6f..634d4623c98 100644 --- a/libraries/chain/chain_id_type.cpp +++ b/libraries/chain/chain_id_type.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include @@ -8,7 +8,7 @@ namespace eosio { namespace chain { - void chain_id_type::reflector_verify()const { + void chain_id_type::reflector_init()const { EOS_ASSERT( *reinterpret_cast(this) != fc::sha256(), chain_id_type_exception, "chain_id_type cannot be zero" ); } diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 85507dea83a..5853f1ff1ec 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -17,15 +17,13 @@ #include #include #include +#include #include #include #include #include -#include -#include - namespace eosio { namespace chain { @@ -138,7 +136,7 @@ struct controller_impl { optional subjective_cpu_leeway; bool trusted_producer_light_validation = false; uint32_t snapshot_head_block = 0; - optional thread_pool; + boost::asio::thread_pool thread_pool; typedef pair handler_key; map< account_name, map > apply_handlers; @@ -148,15 +146,7 @@ struct controller_impl { * are removed from this list if they are re-applied in other blocks. Producers * can query this list when scheduling new transactions into blocks. */ - map unapplied_transactions; - - // async on thread_pool and return future - template - auto async_thread_pool( F&& f ) { - auto task = std::make_shared>( std::forward( f ) ); - boost::asio::post( *thread_pool, [task]() { (*task)(); } ); - return task->get_future(); - } + unapplied_transactions_type unapplied_transactions; void pop_block() { auto prev = fork_db.get_block( head->header.previous ); @@ -197,7 +187,8 @@ struct controller_impl { authorization( s, db ), conf( cfg ), chain_id( cfg.genesis.compute_chain_id() ), - read_mode( cfg.read_mode ) + read_mode( cfg.read_mode ), + thread_pool( cfg.thread_pool_size ) { #define SET_APP_HANDLER( receiver, contract, action) \ @@ -352,8 +343,6 @@ struct controller_impl { void init(std::function shutdown, const snapshot_reader_ptr& snapshot) { - thread_pool.emplace( conf.thread_pool_size ); - bool report_integrity_hash = !!snapshot; if (snapshot) { EOS_ASSERT( !head, fork_database_exception, "" ); @@ -426,11 +415,6 @@ struct controller_impl { ~controller_impl() { pending.reset(); - if( thread_pool ) { - thread_pool->join(); - thread_pool->stop(); - } - db.flush(); reversible_blocks.flush(); } @@ -1106,7 +1090,19 @@ struct controller_impl { transaction_trace_ptr trace; try { - transaction_context trx_context(self, trx->trx, trx->id); + auto start = fc::time_point::now(); + if( !explicit_billed_cpu_time ) { + fc::microseconds already_consumed_time( EOS_PERCENT(trx->sig_cpu_usage.count(), conf.sig_cpu_bill_pct) ); + + if( start.time_since_epoch() < already_consumed_time ) { + start = fc::time_point(); + } else { + start -= already_consumed_time; + } + } + + const signed_transaction& trn = trx->packed_trx->get_signed_transaction(); + transaction_context trx_context(self, trn, trx->id, start); if ((bool)subjective_cpu_leeway && pending->_block_status == controller::block_status::incomplete) { trx_context.leeway = *subjective_cpu_leeway; } @@ -1119,18 +1115,16 @@ struct controller_impl { trx_context.init_for_implicit_trx(); trx_context.enforce_whiteblacklist = false; } else { - bool skip_recording = replay_head_time && (time_point(trx->trx.expiration) <= *replay_head_time); - trx_context.init_for_input_trx( trx->packed_trx.get_unprunable_size(), - trx->packed_trx.get_prunable_size(), - trx->trx.signatures.size(), + bool skip_recording = replay_head_time && (time_point(trn.expiration) <= *replay_head_time); + trx_context.init_for_input_trx( trx->packed_trx->get_unprunable_size(), + trx->packed_trx->get_prunable_size(), skip_recording); } - - trx_context.delay = fc::seconds(trx->trx.delay_sec); + trx_context.delay = fc::seconds(trn.delay_sec); if( !self.skip_auth_check() && !trx->implicit ) { authorization.check_authorization( - trx->trx.actions, + trn.actions, trx->recover_keys( chain_id ), {}, trx_context.delay, @@ -1149,7 +1143,7 @@ struct controller_impl { transaction_receipt::status_enum s = (trx_context.delay == fc::seconds(0)) ? transaction_receipt::executed : transaction_receipt::delayed; - trace->receipt = push_receipt(trx->packed_trx, s, trx_context.billed_cpu_time_us, trace->net_usage); + trace->receipt = push_receipt(*trx->packed_trx, s, trx_context.billed_cpu_time_us, trace->net_usage); pending->_pending_block_state->trxs.emplace_back(trx); } else { transaction_receipt_header r; @@ -1296,15 +1290,9 @@ struct controller_impl { for( const auto& receipt : b->transactions ) { if( receipt.trx.contains()) { auto& pt = receipt.trx.get(); - auto mtrx = std::make_shared( pt ); + auto mtrx = std::make_shared( std::make_shared( pt ) ); if( !self.skip_auth_check() ) { - std::weak_ptr mtrx_wp = mtrx; - mtrx->signing_keys_future = async_thread_pool( [chain_id = this->chain_id, mtrx_wp]() { - auto mtrx = mtrx_wp.lock(); - return mtrx ? - std::make_pair( chain_id, mtrx->trx.get_signature_keys( chain_id ) ) : - std::make_pair( chain_id, decltype( mtrx->trx.get_signature_keys( chain_id ) ){} ); - } ); + transaction_metadata::create_signing_keys_future( mtrx, thread_pool, chain_id, microseconds::maximum() ); } packed_transactions.emplace_back( std::move( mtrx ) ); } @@ -1385,7 +1373,7 @@ struct controller_impl { auto prev = fork_db.get_block( b->previous ); EOS_ASSERT( prev, unlinkable_block_exception, "unlinkable block ${id}", ("id", id)("previous", b->previous) ); - return async_thread_pool( [b, prev]() { + return async_thread_pool( thread_pool, [b, prev]() { const bool skip_validate_signee = false; return std::make_shared( *prev, move( b ), skip_validate_signee ); } ); @@ -1897,6 +1885,10 @@ void controller::abort_block() { my->abort_block(); } +boost::asio::thread_pool& controller::get_thread_pool() { + return my->thread_pool; +} + std::future controller::create_block_state_future( const signed_block_ptr& b ) { return my->create_block_state_future( b ); } @@ -2241,41 +2233,12 @@ const account_object& controller::get_account( account_name name )const return my->db.get(name); } FC_CAPTURE_AND_RETHROW( (name) ) } -vector controller::get_unapplied_transactions() const { - vector result; - if ( my->read_mode == db_read_mode::SPECULATIVE ) { - result.reserve(my->unapplied_transactions.size()); - for ( const auto& entry: my->unapplied_transactions ) { - result.emplace_back(entry.second); - } - } else { - EOS_ASSERT( my->unapplied_transactions.empty(), transaction_exception, "not empty unapplied_transactions in non-speculative mode" ); //should never happen - } - return result; -} - -void controller::drop_unapplied_transaction(const transaction_metadata_ptr& trx) { - my->unapplied_transactions.erase(trx->signed_id); -} - -void controller::drop_all_unapplied_transactions() { - my->unapplied_transactions.clear(); -} - -vector controller::get_scheduled_transactions() const { - const auto& idx = db().get_index(); - - vector result; - - static const size_t max_reserve = 64; - result.reserve(std::min(idx.size(), max_reserve)); - - auto itr = idx.begin(); - while( itr != idx.end() && itr->delay_until <= pending_block_time() ) { - result.emplace_back(itr->trx_id); - ++itr; +unapplied_transactions_type& controller::get_unapplied_transactions() { + if ( my->read_mode != db_read_mode::SPECULATIVE ) { + EOS_ASSERT( my->unapplied_transactions.empty(), transaction_exception, + "not empty unapplied_transactions in non-speculative mode" ); //should never happen } - return result; + return my->unapplied_transactions; } bool controller::sender_avoids_whitelist_blacklist_enforcement( account_name sender )const { diff --git a/libraries/chain/eosio_contract.cpp b/libraries/chain/eosio_contract.cpp index e41fe815834..03e0fed7f7f 100644 --- a/libraries/chain/eosio_contract.cpp +++ b/libraries/chain/eosio_contract.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -155,10 +155,11 @@ void apply_eosio_setcode(apply_context& context) { // TODO: update setcode message to include the hash, then validate it in validate a.last_code_update = context.control.pending_block_time(); a.code_version = code_id; - a.code.resize( code_size ); - if( code_size > 0 ) - memcpy( a.code.data(), act.code.data(), code_size ); - + if ( code_size > 0 ) { + a.code.assign(act.code.data(), code_size); + } else { + a.code.resize(0); + } }); const auto& account_sequence = db.get(act.account); @@ -185,9 +186,11 @@ void apply_eosio_setabi(apply_context& context) { int64_t new_size = abi_size; db.modify( account, [&]( auto& a ) { - a.abi.resize( abi_size ); - if( abi_size > 0 ) - memcpy( a.abi.data(), act.abi.data(), abi_size ); + if (abi_size > 0) { + a.abi.assign(act.abi.data(), abi_size); + } else { + a.abi.resize(0); + } }); const auto& account_sequence = db.get(act.account); diff --git a/libraries/chain/genesis_state.cpp b/libraries/chain/genesis_state.cpp index 40257a6168b..23d8986e5bd 100644 --- a/libraries/chain/genesis_state.cpp +++ b/libraries/chain/genesis_state.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/libraries/chain/genesis_state_root_key.cpp.in b/libraries/chain/genesis_state_root_key.cpp.in index 6530698e33a..11bbde26397 100644 --- a/libraries/chain/genesis_state_root_key.cpp.in +++ b/libraries/chain/genesis_state_root_key.cpp.in @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/libraries/chain/include/eosio/chain/abi_def.hpp b/libraries/chain/include/eosio/chain/abi_def.hpp index f00dd19884d..9b1d211d623 100644 --- a/libraries/chain/include/eosio/chain/abi_def.hpp +++ b/libraries/chain/include/eosio/chain/abi_def.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index 221424e041e..8f8fca4cdeb 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -418,11 +418,11 @@ namespace impl { mutable_variant_object mvo; auto trx = ptrx.get_transaction(); mvo("id", trx.id()); - mvo("signatures", ptrx.signatures); - mvo("compression", ptrx.compression); - mvo("packed_context_free_data", ptrx.packed_context_free_data); + mvo("signatures", ptrx.get_signatures()); + mvo("compression", ptrx.get_compression()); + mvo("packed_context_free_data", ptrx.get_packed_context_free_data()); mvo("context_free_data", ptrx.get_context_free_data()); - mvo("packed_trx", ptrx.packed_trx); + mvo("packed_trx", ptrx.get_packed_transaction()); add(mvo, "transaction", trx, resolver, ctx); out(name, std::move(mvo)); @@ -577,32 +577,42 @@ namespace impl { const variant_object& vo = v.get_object(); EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); - from_variant(vo["signatures"], ptrx.signatures); - from_variant(vo["compression"], ptrx.compression); + std::vector signatures; + packed_transaction::compression_type compression; + from_variant(vo["signatures"], signatures); + from_variant(vo["compression"], compression); + + bytes packed_cfd; + std::vector cfd; + bool use_packed_cfd = false; + if( vo.contains("packed_context_free_data") && vo["packed_context_free_data"].is_string() && !vo["packed_context_free_data"].as_string().empty() ) { + from_variant(vo["packed_context_free_data"], packed_cfd ); + use_packed_cfd = true; + } else if( vo.contains("context_free_data") ) { + from_variant(vo["context_free_data"], cfd); + } - // TODO: Make this nicer eventually. But for now, if it works... good enough. if( vo.contains("packed_trx") && vo["packed_trx"].is_string() && !vo["packed_trx"].as_string().empty() ) { - from_variant(vo["packed_trx"], ptrx.packed_trx); - auto trx = ptrx.get_transaction(); // Validates transaction data provided. - if( vo.contains("packed_context_free_data") && vo["packed_context_free_data"].is_string() && !vo["packed_context_free_data"].as_string().empty() ) { - from_variant(vo["packed_context_free_data"], ptrx.packed_context_free_data ); - } else if( vo.contains("context_free_data") ) { - vector context_free_data; - from_variant(vo["context_free_data"], context_free_data); - ptrx.set_transaction(trx, context_free_data, ptrx.compression); + bytes packed_trx; + from_variant(vo["packed_trx"], packed_trx); + if( use_packed_cfd ) { + ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( packed_cfd ), compression ); + } else { + ptrx = packed_transaction( std::move( packed_trx ), std::move( signatures ), std::move( cfd ), compression ); } } else { EOS_ASSERT(vo.contains("transaction"), packed_transaction_type_exception, "Missing transaction"); - transaction trx; - vector context_free_data; - extract(vo["transaction"], trx, resolver, ctx); - if( vo.contains("packed_context_free_data") && vo["packed_context_free_data"].is_string() && !vo["packed_context_free_data"].as_string().empty() ) { - from_variant(vo["packed_context_free_data"], ptrx.packed_context_free_data ); - context_free_data = ptrx.get_context_free_data(); - } else if( vo.contains("context_free_data") ) { - from_variant(vo["context_free_data"], context_free_data); + if( use_packed_cfd ) { + transaction trx; + extract( vo["transaction"], trx, resolver, ctx ); + ptrx = packed_transaction( std::move(trx), std::move(signatures), std::move(packed_cfd), compression ); + } else { + signed_transaction trx; + extract( vo["transaction"], trx, resolver, ctx ); + trx.signatures = std::move( signatures ); + trx.context_free_data = std::move(cfd); + ptrx = packed_transaction( std::move( trx ), compression ); } - ptrx.set_transaction(trx, context_free_data, ptrx.compression); } } }; @@ -615,11 +625,11 @@ namespace impl { * @tparam Reslover - callable with the signature (const name& code_account) -> optional */ template - class abi_from_variant_visitor : reflector_verifier_visitor + class abi_from_variant_visitor : reflector_init_visitor { public: abi_from_variant_visitor( const variant_object& _vo, T& v, Resolver _resolver, abi_traverse_context& _ctx ) - : reflector_verifier_visitor(v) + : reflector_init_visitor(v) ,_vo(_vo) ,_resolver(_resolver) ,_ctx(_ctx) diff --git a/libraries/chain/include/eosio/chain/account_object.hpp b/libraries/chain/include/eosio/chain/account_object.hpp index 8a945512b3c..b995a8508a1 100644 --- a/libraries/chain/include/eosio/chain/account_object.hpp +++ b/libraries/chain/include/eosio/chain/account_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/action.hpp b/libraries/chain/include/eosio/chain/action.hpp index 9eb989bda91..9912e2f31b7 100644 --- a/libraries/chain/include/eosio/chain/action.hpp +++ b/libraries/chain/include/eosio/chain/action.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/action_receipt.hpp b/libraries/chain/include/eosio/chain/action_receipt.hpp index 9b5d138117f..17fe4f46713 100644 --- a/libraries/chain/include/eosio/chain/action_receipt.hpp +++ b/libraries/chain/include/eosio/chain/action_receipt.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index 3a81b3ffc9d..3909906dbd5 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/asset.hpp b/libraries/chain/include/eosio/chain/asset.hpp index 2c1f2bb4fc3..5c9bb9669bc 100644 --- a/libraries/chain/include/eosio/chain/asset.hpp +++ b/libraries/chain/include/eosio/chain/asset.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -84,7 +84,7 @@ struct asset friend struct fc::reflector; - void reflector_verify()const { + void reflector_init()const { EOS_ASSERT( is_amount_within_range(), asset_type_exception, "magnitude of asset amount must be less than 2^62" ); EOS_ASSERT( sym.valid(), asset_type_exception, "invalid symbol" ); } diff --git a/libraries/chain/include/eosio/chain/authority.hpp b/libraries/chain/include/eosio/chain/authority.hpp index d59382de2c9..373deb56120 100644 --- a/libraries/chain/include/eosio/chain/authority.hpp +++ b/libraries/chain/include/eosio/chain/authority.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/authority_checker.hpp b/libraries/chain/include/eosio/chain/authority_checker.hpp index 068cc5a9137..f17eeac10bf 100644 --- a/libraries/chain/include/eosio/chain/authority_checker.hpp +++ b/libraries/chain/include/eosio/chain/authority_checker.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/authorization_manager.hpp b/libraries/chain/include/eosio/chain/authorization_manager.hpp index a6df7ad2568..3e9ffd0bb42 100644 --- a/libraries/chain/include/eosio/chain/authorization_manager.hpp +++ b/libraries/chain/include/eosio/chain/authorization_manager.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/block.hpp b/libraries/chain/include/eosio/chain/block.hpp index 28c49f9772e..9cd942026cf 100644 --- a/libraries/chain/include/eosio/chain/block.hpp +++ b/libraries/chain/include/eosio/chain/block.hpp @@ -19,22 +19,22 @@ namespace eosio { namespace chain { }; transaction_receipt_header():status(hard_fail){} - transaction_receipt_header( status_enum s ):status(s){} + explicit transaction_receipt_header( status_enum s ):status(s){} friend inline bool operator ==( const transaction_receipt_header& lhs, const transaction_receipt_header& rhs ) { return std::tie(lhs.status, lhs.cpu_usage_us, lhs.net_usage_words) == std::tie(rhs.status, rhs.cpu_usage_us, rhs.net_usage_words); } fc::enum_type status; - uint32_t cpu_usage_us; ///< total billed CPU usage (microseconds) + uint32_t cpu_usage_us = 0; ///< total billed CPU usage (microseconds) fc::unsigned_int net_usage_words; ///< total billed NET usage, so we can reconstruct resource state when skipping context free data... hard failures... }; struct transaction_receipt : public transaction_receipt_header { transaction_receipt():transaction_receipt_header(){} - transaction_receipt( transaction_id_type tid ):transaction_receipt_header(executed),trx(tid){} - transaction_receipt( packed_transaction ptrx ):transaction_receipt_header(executed),trx(ptrx){} + explicit transaction_receipt( const transaction_id_type& tid ):transaction_receipt_header(executed),trx(tid){} + explicit transaction_receipt( const packed_transaction& ptrx ):transaction_receipt_header(executed),trx(ptrx){} fc::static_variant trx; @@ -58,9 +58,14 @@ namespace eosio { namespace chain { /** */ struct signed_block : public signed_block_header { - using signed_block_header::signed_block_header; + private: + signed_block( const signed_block& ) = default; + public: signed_block() = default; - signed_block( const signed_block_header& h ):signed_block_header(h){} + explicit signed_block( const signed_block_header& h ):signed_block_header(h){} + signed_block( signed_block&& ) = default; + signed_block& operator=(const signed_block&) = delete; + signed_block clone() const { return *this; } vector transactions; /// new or generated transactions extensions_type block_extensions; diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 24ff0ad2835..26e1dcb41fa 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 6870d04f5b2..2292392ade4 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/block_summary_object.hpp b/libraries/chain/include/eosio/chain/block_summary_object.hpp index d0224c9a9d5..0bb24ee7976 100644 --- a/libraries/chain/include/eosio/chain/block_summary_object.hpp +++ b/libraries/chain/include/eosio/chain/block_summary_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/chain_config.hpp b/libraries/chain/include/eosio/chain/chain_config.hpp index 649d11a3cdd..da258010fbb 100644 --- a/libraries/chain/include/eosio/chain/chain_config.hpp +++ b/libraries/chain/include/eosio/chain/chain_config.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/chain_id_type.hpp b/libraries/chain/include/eosio/chain/chain_id_type.hpp index 3b583a65997..a16fc143ae6 100644 --- a/libraries/chain/include/eosio/chain/chain_id_type.hpp +++ b/libraries/chain/include/eosio/chain/chain_id_type.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once @@ -34,7 +34,7 @@ namespace chain { return ds; } - void reflector_verify()const; + void reflector_init()const; private: chain_id_type() = default; diff --git a/libraries/chain/include/eosio/chain/chain_snapshot.hpp b/libraries/chain/include/eosio/chain/chain_snapshot.hpp index 7174e69c5b9..3b3e64f264f 100644 --- a/libraries/chain/include/eosio/chain/chain_snapshot.hpp +++ b/libraries/chain/include/eosio/chain/chain_snapshot.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 4fb86f86ae8..f34b7702095 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -80,6 +80,7 @@ const static uint32_t default_max_trx_delay = 45*24*3600; // const static uint32_t default_max_inline_action_size = 4 * 1024; // 4 KB const static uint16_t default_max_inline_action_depth = 4; const static uint16_t default_max_auth_depth = 6; +const static uint32_t default_sig_cpu_bill_pct = 50 * percent_1; // billable percentage of signature recovery const static uint16_t default_controller_thread_pool_size = 2; const static uint32_t min_net_usage_delta_between_base_and_max_for_trx = 10*1024; diff --git a/libraries/chain/include/eosio/chain/contract_table_objects.hpp b/libraries/chain/include/eosio/chain/contract_table_objects.hpp index 8be51a98daf..dc6b25b0501 100644 --- a/libraries/chain/include/eosio/chain/contract_table_objects.hpp +++ b/libraries/chain/include/eosio/chain/contract_table_objects.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 62251d08512..ac3581f580f 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -11,7 +11,9 @@ namespace chainbase { class database; } - +namespace boost { namespace asio { + class thread_pool; +}} namespace eosio { namespace chain { @@ -32,6 +34,7 @@ namespace eosio { namespace chain { class account_object; using resource_limits::resource_limits_manager; using apply_handler = std::function; + using unapplied_transactions_type = map; class fork_database; @@ -79,6 +82,7 @@ namespace eosio { namespace chain { uint64_t state_guard_size = chain::config::default_state_guard_size; uint64_t reversible_cache_size = chain::config::default_reversible_cache_size; uint64_t reversible_guard_size = chain::config::default_reversible_guard_size; + uint32_t sig_cpu_bill_pct = chain::config::default_sig_cpu_bill_pct; uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size; bool read_only = false; bool force_all_checks = false; @@ -103,7 +107,7 @@ namespace eosio { namespace chain { incomplete = 3, ///< this is an incomplete block (either being produced by a producer or speculatively produced by a node) }; - controller( const config& cfg ); + explicit controller( const config& cfg ); ~controller(); void add_indices(); @@ -124,22 +128,9 @@ namespace eosio { namespace chain { * The caller is responsible for calling drop_unapplied_transaction on a failing transaction that * they never intend to retry * - * @return vector of transactions which have been unapplied + * @return map of transactions which have been unapplied */ - vector get_unapplied_transactions() const; - void drop_unapplied_transaction(const transaction_metadata_ptr& trx); - void drop_all_unapplied_transactions(); - - /** - * These transaction IDs represent transactions available in the head chain state as scheduled - * or otherwise generated transactions. - * - * calling push_scheduled_transaction with these IDs will remove the associated transaction from - * the chain state IFF it succeeds or objectively fails - * - * @return - */ - vector get_scheduled_transactions() const; + unapplied_transactions_type& get_unapplied_transactions(); /** * @@ -160,6 +151,8 @@ namespace eosio { namespace chain { std::future create_block_state_future( const signed_block_ptr& b ); void push_block( std::future& block_state_future ); + boost::asio::thread_pool& get_thread_pool(); + const chainbase::database& db()const; const fork_database& fork_db()const; diff --git a/libraries/chain/include/eosio/chain/core_symbol.hpp.in b/libraries/chain/include/eosio/chain/core_symbol.hpp.in index ee8e368296d..c07f1ae267d 100644 --- a/libraries/chain/include/eosio/chain/core_symbol.hpp.in +++ b/libraries/chain/include/eosio/chain/core_symbol.hpp.in @@ -1,5 +1,5 @@ /** @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See core_symbol.hpp.in for changes. */ diff --git a/libraries/chain/include/eosio/chain/database_utils.hpp b/libraries/chain/include/eosio/chain/database_utils.hpp index eee26bce1c0..19de97bd3da 100644 --- a/libraries/chain/include/eosio/chain/database_utils.hpp +++ b/libraries/chain/include/eosio/chain/database_utils.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/eosio_contract.hpp b/libraries/chain/include/eosio/chain/eosio_contract.hpp index 88566b30982..1bf9163827e 100644 --- a/libraries/chain/include/eosio/chain/eosio_contract.hpp +++ b/libraries/chain/include/eosio/chain/eosio_contract.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 01a42793e94..6c3e504d349 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/fixed_key.hpp b/libraries/chain/include/eosio/chain/fixed_key.hpp index 1839225a4f9..5dd90f901ad 100644 --- a/libraries/chain/include/eosio/chain/fixed_key.hpp +++ b/libraries/chain/include/eosio/chain/fixed_key.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/generated_transaction_object.hpp b/libraries/chain/include/eosio/chain/generated_transaction_object.hpp index 6d3e74dd558..24db926b4a8 100644 --- a/libraries/chain/include/eosio/chain/generated_transaction_object.hpp +++ b/libraries/chain/include/eosio/chain/generated_transaction_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/genesis_state.hpp b/libraries/chain/include/eosio/chain/genesis_state.hpp index 7b1da43043f..364f505629e 100644 --- a/libraries/chain/include/eosio/chain/genesis_state.hpp +++ b/libraries/chain/include/eosio/chain/genesis_state.hpp @@ -1,7 +1,7 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 6f4f35ffd80..bdb49d3ce06 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/multi_index_includes.hpp b/libraries/chain/include/eosio/chain/multi_index_includes.hpp index 5d2deaf04d0..2bb5485194c 100644 --- a/libraries/chain/include/eosio/chain/multi_index_includes.hpp +++ b/libraries/chain/include/eosio/chain/multi_index_includes.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/parallel_markers.hpp b/libraries/chain/include/eosio/chain/parallel_markers.hpp index c78ebdabdae..7436f970630 100644 --- a/libraries/chain/include/eosio/chain/parallel_markers.hpp +++ b/libraries/chain/include/eosio/chain/parallel_markers.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/permission_link_object.hpp b/libraries/chain/include/eosio/chain/permission_link_object.hpp index 9930b647ad8..b7b4ea76f57 100644 --- a/libraries/chain/include/eosio/chain/permission_link_object.hpp +++ b/libraries/chain/include/eosio/chain/permission_link_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/permission_object.hpp b/libraries/chain/include/eosio/chain/permission_object.hpp index ee43f0e52a7..7db580b74d8 100644 --- a/libraries/chain/include/eosio/chain/permission_object.hpp +++ b/libraries/chain/include/eosio/chain/permission_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/producer_object.hpp b/libraries/chain/include/eosio/chain/producer_object.hpp index a2bb729dcc2..61d23870280 100644 --- a/libraries/chain/include/eosio/chain/producer_object.hpp +++ b/libraries/chain/include/eosio/chain/producer_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/protocol.hpp b/libraries/chain/include/eosio/chain/protocol.hpp index 134f953fb85..389e9c08723 100644 --- a/libraries/chain/include/eosio/chain/protocol.hpp +++ b/libraries/chain/include/eosio/chain/protocol.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/resource_limits_private.hpp b/libraries/chain/include/eosio/chain/resource_limits_private.hpp index 877c77e5acb..0bbe7e0e3e1 100644 --- a/libraries/chain/include/eosio/chain/resource_limits_private.hpp +++ b/libraries/chain/include/eosio/chain/resource_limits_private.hpp @@ -287,10 +287,10 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::resource_limits::gmr_config_object, eosio FC_REFLECT(eosio::chain::resource_limits::usage_accumulator, (last_ordinal)(value_ex)(consumed)) +// @ignore pending FC_REFLECT(eosio::chain::resource_limits::resource_limits_object, (owner)(net_weight)(cpu_weight)(ram_bytes)) FC_REFLECT(eosio::chain::resource_limits::resource_usage_object, (owner)(net_usage)(cpu_usage)(ram_usage)) FC_REFLECT(eosio::chain::resource_limits::resource_limits_config_object, (cpu_limit_parameters)(net_limit_parameters)(account_cpu_usage_average_window)(account_net_usage_average_window)) FC_REFLECT(eosio::chain::resource_limits::resource_limits_state_object, (average_block_net_usage)(average_block_cpu_usage)(pending_net_usage)(pending_cpu_usage)(total_net_weight)(total_cpu_weight)(total_ram_bytes)(virtual_net_limit)(virtual_cpu_limit)) - - FC_REFLECT(eosio::chain::resource_limits::gmr_config_object, (res_parameters)) + diff --git a/libraries/chain/include/eosio/chain/reversible_block_object.hpp b/libraries/chain/include/eosio/chain/reversible_block_object.hpp index e0502b7e72f..ea9a4c9e122 100644 --- a/libraries/chain/include/eosio/chain/reversible_block_object.hpp +++ b/libraries/chain/include/eosio/chain/reversible_block_object.hpp @@ -1,7 +1,7 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/snapshot.hpp b/libraries/chain/include/eosio/chain/snapshot.hpp index 47f69dd07e2..499fbe29960 100644 --- a/libraries/chain/include/eosio/chain/snapshot.hpp +++ b/libraries/chain/include/eosio/chain/snapshot.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/symbol.hpp b/libraries/chain/include/eosio/chain/symbol.hpp index 3d6a4f72b35..437b1d36ab7 100644 --- a/libraries/chain/include/eosio/chain/symbol.hpp +++ b/libraries/chain/include/eosio/chain/symbol.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -137,7 +137,7 @@ namespace eosio { return ds << s.to_string(); } - void reflector_verify()const { + void reflector_init()const { EOS_ASSERT( decimals() <= max_precision, symbol_type_exception, "precision ${p} should be <= 18", ("p", decimals()) ); EOS_ASSERT( valid_name(name()), symbol_type_exception, "invalid symbol: ${name}", ("name",name())); } diff --git a/libraries/chain/include/eosio/chain/thread_utils.hpp b/libraries/chain/include/eosio/chain/thread_utils.hpp new file mode 100644 index 00000000000..31b32cbd91f --- /dev/null +++ b/libraries/chain/include/eosio/chain/thread_utils.hpp @@ -0,0 +1,24 @@ +/** + * @file + * @copyright defined in eos/LICENSE.txt + */ +#pragma once + +#include +#include +#include +#include + +namespace eosio { namespace chain { + + // async on thread_pool and return future + template + auto async_thread_pool( boost::asio::thread_pool& thread_pool, F&& f ) { + auto task = std::make_shared>( std::forward( f ) ); + boost::asio::post( thread_pool, [task]() { (*task)(); } ); + return task->get_future(); + } + +} } // eosio::chain + + diff --git a/libraries/chain/include/eosio/chain/trace.hpp b/libraries/chain/include/eosio/chain/trace.hpp index 03750bd512e..58de120bdd8 100644 --- a/libraries/chain/include/eosio/chain/trace.hpp +++ b/libraries/chain/include/eosio/chain/trace.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/libraries/chain/include/eosio/chain/transaction.hpp b/libraries/chain/include/eosio/chain/transaction.hpp index 895d3ee4fb8..a8b53b98e0d 100644 --- a/libraries/chain/include/eosio/chain/transaction.hpp +++ b/libraries/chain/include/eosio/chain/transaction.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once @@ -58,11 +58,12 @@ namespace eosio { namespace chain { transaction_id_type id()const; digest_type sig_digest( const chain_id_type& chain_id, const vector& cfd = vector() )const; - flat_set get_signature_keys( const vector& signatures, + fc::microseconds get_signature_keys( const vector& signatures, const chain_id_type& chain_id, - const vector& cfd = vector(), - bool allow_duplicate_keys = false, - bool use_cache = true )const; + fc::time_point deadline, + const vector& cfd, + flat_set& recovered_pub_keys, + bool allow_duplicate_keys = false) const; uint32_t total_actions()const { return context_free_actions.size() + actions.size(); } account_name first_authorizor()const { @@ -84,15 +85,21 @@ namespace eosio { namespace chain { : transaction(std::move(trx)) , signatures(signatures) , context_free_data(context_free_data) - { - } + {} + signed_transaction( transaction&& trx, const vector& signatures, vector&& context_free_data) + : transaction(std::move(trx)) + , signatures(signatures) + , context_free_data(std::move(context_free_data)) + {} vector signatures; vector context_free_data; ///< for each context-free action, there is an entry here const signature_type& sign(const private_key_type& key, const chain_id_type& chain_id); signature_type sign(const private_key_type& key, const chain_id_type& chain_id)const; - flat_set get_signature_keys( const chain_id_type& chain_id, bool allow_duplicate_keys = false, bool use_cache = true )const; + fc::microseconds get_signature_keys( const chain_id_type& chain_id, fc::time_point deadline, + flat_set& recovered_pub_keys, + bool allow_duplicate_keys = false )const; }; struct packed_transaction { @@ -102,47 +109,65 @@ namespace eosio { namespace chain { }; packed_transaction() = default; - - explicit packed_transaction(const transaction& t, compression_type _compression = none) - { - set_transaction(t, _compression); - } + packed_transaction(packed_transaction&&) = default; + explicit packed_transaction(const packed_transaction&) = default; + packed_transaction& operator=(const packed_transaction&) = delete; + packed_transaction& operator=(packed_transaction&&) = default; explicit packed_transaction(const signed_transaction& t, compression_type _compression = none) - :signatures(t.signatures) + :signatures(t.signatures), compression(_compression), unpacked_trx(t) { - set_transaction(t, t.context_free_data, _compression); + local_pack_transaction(); + local_pack_context_free_data(); } explicit packed_transaction(signed_transaction&& t, compression_type _compression = none) - :signatures(std::move(t.signatures)) + :signatures(t.signatures), compression(_compression), unpacked_trx(std::move(t)) { - set_transaction(t, std::move(t.context_free_data), _compression); + local_pack_transaction(); + local_pack_context_free_data(); } + // used by abi_serializer + packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ); + packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ); + uint32_t get_unprunable_size()const; uint32_t get_prunable_size()const; digest_type packed_digest()const; + transaction_id_type id()const { return unpacked_trx.id(); } + bytes get_raw_transaction()const; + + time_point_sec expiration()const { return unpacked_trx.expiration; } + const vector& get_context_free_data()const { return unpacked_trx.context_free_data; } + const transaction& get_transaction()const { return unpacked_trx; } + const signed_transaction& get_signed_transaction()const { return unpacked_trx; } + const vector& get_signatures()const { return signatures; } + const fc::enum_type& get_compression()const { return compression; } + const bytes& get_packed_context_free_data()const { return packed_context_free_data; } + const bytes& get_packed_transaction()const { return packed_trx; } + + private: + void local_unpack_transaction(vector&& context_free_data); + void local_unpack_context_free_data(); + void local_pack_transaction(); + void local_pack_context_free_data(); + + friend struct fc::reflector; + friend struct fc::reflector_init_visitor; + void reflector_init(); + private: vector signatures; fc::enum_type compression; bytes packed_context_free_data; bytes packed_trx; - time_point_sec expiration()const; - transaction_id_type id()const; - transaction_id_type get_uncached_id()const; // thread safe - bytes get_raw_transaction()const; // thread safe - vector get_context_free_data()const; - transaction get_transaction()const; - signed_transaction get_signed_transaction()const; - void set_transaction(const transaction& t, compression_type _compression = none); - void set_transaction(const transaction& t, const vector& cfd, compression_type _compression = none); - private: - mutable optional unpacked_trx; // <-- intermediate buffer used to retrieve values - void local_unpack()const; + // cache unpacked trx, for thread safety do not modify after construction + signed_transaction unpacked_trx; }; using packed_transaction_ptr = std::shared_ptr; diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 2e215f00ec2..b0327dafb18 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -33,7 +33,6 @@ namespace eosio { namespace chain { void init_for_input_trx( uint64_t packed_trx_unprunable_size, uint64_t packed_trx_prunable_size, - uint32_t num_signatures, bool skip_recording); void init_for_deferred_trx( fc::time_point published ); diff --git a/libraries/chain/include/eosio/chain/transaction_metadata.hpp b/libraries/chain/include/eosio/chain/transaction_metadata.hpp index c5eb0c790f4..6136580fa44 100644 --- a/libraries/chain/include/eosio/chain/transaction_metadata.hpp +++ b/libraries/chain/include/eosio/chain/transaction_metadata.hpp @@ -1,14 +1,20 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include #include #include +namespace boost { namespace asio { + class thread_pool; +}} + namespace eosio { namespace chain { +class transaction_metadata; +using transaction_metadata_ptr = std::shared_ptr; /** * This data structure should store context-free cached data about a transaction such as * packed/unpacked/compressed and recovered keys @@ -17,45 +23,38 @@ class transaction_metadata { public: transaction_id_type id; transaction_id_type signed_id; - signed_transaction trx; - packed_transaction packed_trx; + packed_transaction_ptr packed_trx; + fc::microseconds sig_cpu_usage; optional>> signing_keys; - std::future>> signing_keys_future; + std::future>> + signing_keys_future; bool accepted = false; bool implicit = false; bool scheduled = false; + transaction_metadata() = delete; + transaction_metadata(const transaction_metadata&) = delete; + transaction_metadata(transaction_metadata&&) = delete; + transaction_metadata operator=(transaction_metadata&) = delete; + transaction_metadata operator=(transaction_metadata&&) = delete; + explicit transaction_metadata( const signed_transaction& t, packed_transaction::compression_type c = packed_transaction::none ) - :trx(t),packed_trx(t, c) { - id = trx.id(); + :id(t.id()), packed_trx(std::make_shared(t, c)) { //raw_packed = fc::raw::pack( static_cast(trx) ); - signed_id = digest_type::hash(packed_trx); + signed_id = digest_type::hash(*packed_trx); } - explicit transaction_metadata( const packed_transaction& ptrx ) - :trx( ptrx.get_signed_transaction() ), packed_trx(ptrx) { - id = trx.id(); + explicit transaction_metadata( const packed_transaction_ptr& ptrx ) + :id(ptrx->id()), packed_trx(ptrx) { //raw_packed = fc::raw::pack( static_cast(trx) ); - signed_id = digest_type::hash(packed_trx); + signed_id = digest_type::hash(*packed_trx); } - const flat_set& recover_keys( const chain_id_type& chain_id ) { - // Unlikely for more than one chain_id to be used in one nodeos instance - if( !signing_keys || signing_keys->first != chain_id ) { - if( signing_keys_future.valid() ) { - signing_keys = signing_keys_future.get(); - if( signing_keys->first == chain_id ) { - return signing_keys->second; - } - } - signing_keys = std::make_pair( chain_id, trx.get_signature_keys( chain_id )); - } - return signing_keys->second; - } + const flat_set& recover_keys( const chain_id_type& chain_id ); - uint32_t total_actions()const { return trx.context_free_actions.size() + trx.actions.size(); } -}; + static void create_signing_keys_future( const transaction_metadata_ptr& mtrx, boost::asio::thread_pool& thread_pool, + const chain_id_type& chain_id, fc::microseconds time_limit ); -using transaction_metadata_ptr = std::shared_ptr; +}; } } // eosio::chain diff --git a/libraries/chain/include/eosio/chain/transaction_object.hpp b/libraries/chain/include/eosio/chain/transaction_object.hpp index 0d049267e8e..50a7eb62cdd 100644 --- a/libraries/chain/include/eosio/chain/transaction_object.hpp +++ b/libraries/chain/include/eosio/chain/transaction_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/include/eosio/chain/types.hpp b/libraries/chain/include/eosio/chain/types.hpp index 028b050a595..bddeb1dd553 100644 --- a/libraries/chain/include/eosio/chain/types.hpp +++ b/libraries/chain/include/eosio/chain/types.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -103,7 +103,22 @@ namespace eosio { namespace chain { */ class shared_blob : public shared_string { public: - shared_blob() = default; + shared_blob() = delete; + shared_blob(shared_blob&&) = default; + + shared_blob(const shared_blob& s) + :shared_string(s.get_allocator()) + { + assign(s.c_str(), s.size()); + } + + + shared_blob& operator=(const shared_blob& s) { + assign(s.c_str(), s.size()); + return *this; + } + + shared_blob& operator=(shared_blob&& ) = default; template shared_blob(InputIterator f, InputIterator l, const allocator_type& a) diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp index f5ebf01c1f7..a67ca9ef696 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp @@ -272,7 +272,7 @@ namespace eosio { namespace chain { namespace wasm_injections { }; - struct call_depth_check { + struct call_depth_check_and_insert_checktime { static constexpr bool kills = true; static constexpr bool post = false; static int32_t global_idx; @@ -290,6 +290,7 @@ namespace eosio { namespace chain { namespace wasm_injections { injector_utils::add_import(*(arg.module), "call_depth_assert", assert_idx); wasm_ops::op_types<>::call_t call_assert; + wasm_ops::op_types<>::call_t call_checktime; wasm_ops::op_types<>::get_global_t get_global_inst; wasm_ops::op_types<>::set_global_t set_global_inst; @@ -301,6 +302,7 @@ namespace eosio { namespace chain { namespace wasm_injections { wasm_ops::op_types<>::else__t else_inst; call_assert.field = assert_idx; + call_checktime.field = checktime_injection::chktm_idx; get_global_inst.field = global_idx; set_global_inst.field = global_idx; const_inst.field = -1; @@ -334,6 +336,7 @@ namespace eosio { namespace chain { namespace wasm_injections { INSERT_INJECTED(const_inst); INSERT_INJECTED(add_inst); INSERT_INJECTED(set_global_inst); + INSERT_INJECTED(call_checktime); #undef INSERT_INJECTED } @@ -679,8 +682,8 @@ namespace eosio { namespace chain { namespace wasm_injections { }; struct pre_op_injectors : wasm_ops::op_types { - using call_t = wasm_ops::call ; - using call_indirect_t = wasm_ops::call_indirect ; + using call_t = wasm_ops::call ; + using call_indirect_t = wasm_ops::call_indirect ; // float binops using f32_add_t = wasm_ops::f32_add >; @@ -785,7 +788,7 @@ namespace eosio { namespace chain { namespace wasm_injections { // initialize static fields of injectors injector_utils::init( mod ); checktime_injection::init(); - call_depth_check::init(); + call_depth_check_and_insert_checktime::init(); } void inject() { diff --git a/libraries/chain/include/eosio/chain/wast_to_wasm.hpp b/libraries/chain/include/eosio/chain/wast_to_wasm.hpp index e77d23b1360..1493f2f99f7 100644 --- a/libraries/chain/include/eosio/chain/wast_to_wasm.hpp +++ b/libraries/chain/include/eosio/chain/wast_to_wasm.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/libraries/chain/transaction.cpp b/libraries/chain/transaction.cpp index 6e6639bac52..2724a31b28d 100644 --- a/libraries/chain/transaction.cpp +++ b/libraries/chain/transaction.cpp @@ -1,11 +1,12 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include #include #include +#include #include #include @@ -28,6 +29,7 @@ struct cached_pub_key { transaction_id_type trx_id; public_key_type pub_key; signature_type sig; + fc::microseconds cpu_usage; cached_pub_key(const cached_pub_key&) = delete; cached_pub_key() = delete; cached_pub_key& operator=(const cached_pub_key&) = delete; @@ -81,43 +83,55 @@ digest_type transaction::sig_digest( const chain_id_type& chain_id, const vector return enc.result(); } -flat_set transaction::get_signature_keys( const vector& signatures, - const chain_id_type& chain_id, const vector& cfd, bool allow_duplicate_keys, bool use_cache )const +fc::microseconds transaction::get_signature_keys( const vector& signatures, + const chain_id_type& chain_id, fc::time_point deadline, const vector& cfd, + flat_set& recovered_pub_keys, bool allow_duplicate_keys)const { try { using boost::adaptors::transformed; - constexpr size_t recovery_cache_size = 1000; - static thread_local recovery_cache_type recovery_cache; + constexpr size_t recovery_cache_size = 10000; + static recovery_cache_type recovery_cache; + static std::mutex cache_mtx; + + auto start = fc::time_point::now(); + recovered_pub_keys.clear(); const digest_type digest = sig_digest(chain_id, cfd); - flat_set recovered_pub_keys; + std::unique_lock lock(cache_mtx, std::defer_lock); + fc::microseconds sig_cpu_usage; for(const signature_type& sig : signatures) { + auto now = fc::time_point::now(); + EOS_ASSERT( now < deadline, tx_cpu_usage_exceeded, "transaction signature verification executed for too long", + ("now", now)("deadline", deadline)("start", start) ); public_key_type recov; - if( use_cache ) { - recovery_cache_type::index::type::iterator it = recovery_cache.get().find( sig ); - if( it == recovery_cache.get().end() || it->trx_id != id()) { - recov = public_key_type( sig, digest ); - recovery_cache.emplace_back(cached_pub_key{id(), recov, sig} ); //could fail on dup signatures; not a problem - } else { - recov = it->pub_key; - } - } else { + const auto& tid = id(); + lock.lock(); + recovery_cache_type::index::type::iterator it = recovery_cache.get().find( sig ); + if( it == recovery_cache.get().end() || it->trx_id != tid ) { + lock.unlock(); recov = public_key_type( sig, digest ); + fc::microseconds cpu_usage = fc::time_point::now() - start; + lock.lock(); + recovery_cache.emplace_back( cached_pub_key{tid, recov, sig, cpu_usage} ); //could fail on dup signatures; not a problem + sig_cpu_usage += cpu_usage; + } else { + recov = it->pub_key; + sig_cpu_usage += it->cpu_usage; } + lock.unlock(); bool successful_insertion = false; std::tie(std::ignore, successful_insertion) = recovered_pub_keys.insert(recov); EOS_ASSERT( allow_duplicate_keys || successful_insertion, tx_duplicate_sig, "transaction includes more than one signature signed using the same key associated with public key: ${key}", - ("key", recov) - ); + ("key", recov) ); } - if( use_cache ) { - while ( recovery_cache.size() > recovery_cache_size ) - recovery_cache.erase( recovery_cache.begin() ); - } + lock.lock(); + while ( recovery_cache.size() > recovery_cache_size ) + recovery_cache.erase( recovery_cache.begin()); + lock.unlock(); - return recovered_pub_keys; + return sig_cpu_usage; } FC_CAPTURE_AND_RETHROW() } @@ -130,9 +144,12 @@ signature_type signed_transaction::sign(const private_key_type& key, const chain return key.sign(sig_digest(chain_id, context_free_data)); } -flat_set signed_transaction::get_signature_keys( const chain_id_type& chain_id, bool allow_duplicate_keys, bool use_cache )const +fc::microseconds +signed_transaction::get_signature_keys( const chain_id_type& chain_id, fc::time_point deadline, + flat_set& recovered_pub_keys, + bool allow_duplicate_keys)const { - return transaction::get_signature_keys(signatures, chain_id, context_free_data, allow_duplicate_keys, use_cache); + return transaction::get_signature_keys(signatures, chain_id, deadline, context_free_data, recovered_pub_keys, allow_duplicate_keys); } uint32_t packed_transaction::get_unprunable_size()const { @@ -272,112 +289,115 @@ bytes packed_transaction::get_raw_transaction() const } FC_CAPTURE_AND_RETHROW((compression)(packed_trx)) } -vector packed_transaction::get_context_free_data()const +packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +:signatures(std::move(sigs)) +,compression(_compression) +,packed_context_free_data(std::move(packed_cfd)) +,packed_trx(std::move(packed_txn)) { - try { - switch(compression) { - case none: - return unpack_context_free_data(packed_context_free_data); - case zlib: - return zlib_decompress_context_free_data(packed_context_free_data); - default: - EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); - } - } FC_CAPTURE_AND_RETHROW((compression)(packed_context_free_data)) + local_unpack_transaction({}); + if( !packed_context_free_data.empty() ) { + local_unpack_context_free_data(); + } } -time_point_sec packed_transaction::expiration()const +packed_transaction::packed_transaction( bytes&& packed_txn, vector&& sigs, vector&& cfd, compression_type _compression ) +:signatures(std::move(sigs)) +,compression(_compression) +,packed_trx(std::move(packed_txn)) { - local_unpack(); - return unpacked_trx->expiration; + local_unpack_transaction( std::move( cfd ) ); + if( !unpacked_trx.context_free_data.empty() ) { + local_pack_context_free_data(); + } } -transaction_id_type packed_transaction::id()const +packed_transaction::packed_transaction( transaction&& t, vector&& sigs, bytes&& packed_cfd, compression_type _compression ) +:signatures(std::move(sigs)) +,compression(_compression) +,packed_context_free_data(std::move(packed_cfd)) +,unpacked_trx(std::move(t), signatures, {}) { - local_unpack(); - return get_transaction().id(); + local_pack_transaction(); + if( !packed_context_free_data.empty() ) { + local_unpack_context_free_data(); + } } -transaction_id_type packed_transaction::get_uncached_id()const +void packed_transaction::reflector_init() { - const auto raw = get_raw_transaction(); - return fc::raw::unpack( raw ).id(); + // called after construction, but always on the same thread and before packed_transaction passed to any other threads + static_assert(&fc::reflector_init_visitor::reflector_init, "FC with reflector_init required"); + static_assert(fc::raw::has_feature_reflector_init_on_unpacked_reflected_types, + "FC unpack needs to call reflector_init otherwise unpacked_trx will not be initialized"); + EOS_ASSERT( unpacked_trx.expiration == time_point_sec(), tx_decompression_error, "packed_transaction already unpacked" ); + local_unpack_transaction({}); + local_unpack_context_free_data(); } -void packed_transaction::local_unpack()const +void packed_transaction::local_unpack_transaction(vector&& context_free_data) { - if (!unpacked_trx) { - try { - switch(compression) { + try { + switch( compression ) { case none: - unpacked_trx = unpack_transaction(packed_trx); + unpacked_trx = signed_transaction( unpack_transaction( packed_trx ), signatures, std::move(context_free_data) ); break; case zlib: - unpacked_trx = zlib_decompress_transaction(packed_trx); + unpacked_trx = signed_transaction( zlib_decompress_transaction( packed_trx ), signatures, std::move(context_free_data) ); break; default: - EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); - } - } FC_CAPTURE_AND_RETHROW((compression)(packed_trx)) - } -} - -transaction packed_transaction::get_transaction()const -{ - local_unpack(); - return transaction(*unpacked_trx); + EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); + } + } FC_CAPTURE_AND_RETHROW( (compression) ) } -signed_transaction packed_transaction::get_signed_transaction() const +void packed_transaction::local_unpack_context_free_data() { try { - switch(compression) { + EOS_ASSERT(unpacked_trx.context_free_data.empty(), tx_decompression_error, "packed_transaction.context_free_data not empty"); + switch( compression ) { case none: - return signed_transaction(get_transaction(), signatures, unpack_context_free_data(packed_context_free_data)); + unpacked_trx.context_free_data = unpack_context_free_data( packed_context_free_data ); + break; case zlib: - return signed_transaction(get_transaction(), signatures, zlib_decompress_context_free_data(packed_context_free_data)); + unpacked_trx.context_free_data = zlib_decompress_context_free_data( packed_context_free_data ); + break; default: - EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); + EOS_THROW( unknown_transaction_compression, "Unknown transaction compression algorithm" ); } - } FC_CAPTURE_AND_RETHROW((compression)(packed_trx)(packed_context_free_data)) - + } FC_CAPTURE_AND_RETHROW( (compression) ) } -void packed_transaction::set_transaction(const transaction& t, packed_transaction::compression_type _compression) +void packed_transaction::local_pack_transaction() { try { - switch(_compression) { + switch(compression) { case none: - packed_trx = pack_transaction(t); + packed_trx = pack_transaction(unpacked_trx); break; case zlib: - packed_trx = zlib_compress_transaction(t); + packed_trx = zlib_compress_transaction(unpacked_trx); break; default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } - } FC_CAPTURE_AND_RETHROW((_compression)(t)) - packed_context_free_data.clear(); - compression = _compression; + } FC_CAPTURE_AND_RETHROW((compression)) } -void packed_transaction::set_transaction(const transaction& t, const vector& cfd, packed_transaction::compression_type _compression) +void packed_transaction::local_pack_context_free_data() { try { - switch(_compression) { + switch(compression) { case none: - packed_trx = pack_transaction(t); - packed_context_free_data = pack_context_free_data(cfd); + packed_context_free_data = pack_context_free_data(unpacked_trx.context_free_data); break; case zlib: - packed_trx = zlib_compress_transaction(t); - packed_context_free_data = zlib_compress_context_free_data(cfd); + packed_context_free_data = zlib_compress_context_free_data(unpacked_trx.context_free_data); break; default: EOS_THROW(unknown_transaction_compression, "Unknown transaction compression algorithm"); } - } FC_CAPTURE_AND_RETHROW((_compression)(t)) - compression = _compression; + } FC_CAPTURE_AND_RETHROW((compression)) } diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 989905ef9a0..d5da8ca279b 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -284,7 +284,6 @@ namespace bacc = boost::accumulators; void transaction_context::init_for_input_trx( uint64_t packed_trx_unprunable_size, uint64_t packed_trx_prunable_size, - uint32_t num_signatures, bool skip_recording ) { const auto& cfg = control.get_global_properties().configuration; diff --git a/libraries/chain/transaction_metadata.cpp b/libraries/chain/transaction_metadata.cpp new file mode 100644 index 00000000000..2053669c0d7 --- /dev/null +++ b/libraries/chain/transaction_metadata.cpp @@ -0,0 +1,47 @@ +#include +#include +#include + +namespace eosio { namespace chain { + + +const flat_set& transaction_metadata::recover_keys( const chain_id_type& chain_id ) { + // Unlikely for more than one chain_id to be used in one nodeos instance + if( !signing_keys || signing_keys->first != chain_id ) { + if( signing_keys_future.valid() ) { + std::tuple> sig_keys = signing_keys_future.get(); + if( std::get<0>( sig_keys ) == chain_id ) { + sig_cpu_usage = std::get<1>( sig_keys ); + signing_keys.emplace( std::get<0>( sig_keys ), std::move( std::get<2>( sig_keys ))); + return signing_keys->second; + } + } + flat_set recovered_pub_keys; + sig_cpu_usage = packed_trx->get_signed_transaction().get_signature_keys( chain_id, fc::time_point::maximum(), recovered_pub_keys ); + signing_keys.emplace( chain_id, std::move( recovered_pub_keys )); + } + return signing_keys->second; +} + +void transaction_metadata::create_signing_keys_future( const transaction_metadata_ptr& mtrx, + boost::asio::thread_pool& thread_pool, const chain_id_type& chain_id, fc::microseconds time_limit ) { + if( mtrx->signing_keys.valid() ) // already created + return; + + std::weak_ptr mtrx_wp = mtrx; + mtrx->signing_keys_future = async_thread_pool( thread_pool, [time_limit, chain_id, mtrx_wp]() { + fc::time_point deadline = time_limit == fc::microseconds::maximum() ? + fc::time_point::maximum() : fc::time_point::now() + time_limit; + auto mtrx = mtrx_wp.lock(); + fc::microseconds cpu_usage; + flat_set recovered_pub_keys; + if( mtrx ) { + const signed_transaction& trn = mtrx->packed_trx->get_signed_transaction(); + cpu_usage = trn.get_signature_keys( chain_id, deadline, recovered_pub_keys ); + } + return std::make_tuple( chain_id, cpu_usage, std::move( recovered_pub_keys )); + } ); +} + + +} } // eosio::chain diff --git a/libraries/chain/wasm_eosio_injection.cpp b/libraries/chain/wasm_eosio_injection.cpp index a4afa44d46d..2c627e13ea7 100644 --- a/libraries/chain/wasm_eosio_injection.cpp +++ b/libraries/chain/wasm_eosio_injection.cpp @@ -35,7 +35,7 @@ void max_memory_injection_visitor::inject( Module& m ) { } void max_memory_injection_visitor::initializer() {} -int32_t call_depth_check::global_idx = -1; +int32_t call_depth_check_and_insert_checktime::global_idx = -1; uint32_t instruction_counter::icnt = 0; uint32_t instruction_counter::tcnt = 0; uint32_t instruction_counter::bcnt = 0; diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index 9204f3d27bb..7d3553e379b 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace eosio { namespace chain { using namespace webassembly; @@ -937,6 +938,8 @@ class system_api : public context_aware_api { }; +constexpr size_t max_assert_message = 1024; + class context_free_system_api : public context_aware_api { public: explicit context_free_system_api( apply_context& ctx ) @@ -949,14 +952,16 @@ class context_free_system_api : public context_aware_api { // Kept as intrinsic rather than implementing on WASM side (using eosio_assert_message and strlen) because strlen is faster on native side. void eosio_assert( bool condition, null_terminated_ptr msg ) { if( BOOST_UNLIKELY( !condition ) ) { - std::string message( msg ); + const size_t sz = strnlen( msg, max_assert_message ); + std::string message( msg, sz ); EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); } } void eosio_assert_message( bool condition, array_ptr msg, size_t msg_len ) { if( BOOST_UNLIKELY( !condition ) ) { - std::string message( msg, msg_len ); + const size_t sz = msg_len > max_assert_message ? max_assert_message : msg_len; + std::string message( msg, sz ); EOS_THROW( eosio_assert_message_exception, "assertion failure with message: ${s}", ("s",message) ); } } @@ -1099,12 +1104,16 @@ class console_api : public context_aware_api { auto& console = context.get_console_stream(); auto orig_prec = console.precision(); +#ifdef __x86_64__ console.precision( std::numeric_limits::digits10 ); - extFloat80_t val_approx; f128M_to_extF80M(&val, &val_approx); context.console_append( *(long double*)(&val_approx) ); - +#else + console.precision( std::numeric_limits::digits10 ); + double val_approx = from_softfloat64( f128M_to_f64(&val) ); + context.console_append(val_approx); +#endif console.precision( orig_prec ); } } diff --git a/libraries/chain/wast_to_wasm.cpp b/libraries/chain/wast_to_wasm.cpp index 6a2ca2e1bbc..09add7a0b6e 100644 --- a/libraries/chain/wast_to_wasm.cpp +++ b/libraries/chain/wast_to_wasm.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/libraries/fc b/libraries/fc index e5ad1ad65b0..b2fa419ddf6 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit e5ad1ad65b012103222a113239a28fc1e4bddbb4 +Subproject commit b2fa419ddf68c6b5fc902de53cf8e691206cc8f3 diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 906d2cb3c44..5f0a5206fdc 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -99,6 +99,17 @@ namespace eosio { namespace testing { void produce_min_num_of_blocks_to_spend_time_wo_inactive_prod(const fc::microseconds target_elapsed_time = fc::microseconds()); signed_block_ptr push_block(signed_block_ptr b); + /** + * These transaction IDs represent transactions available in the head chain state as scheduled + * or otherwise generated transactions. + * + * calling push_scheduled_transaction with these IDs will remove the associated transaction from + * the chain state IFF it succeeds or objectively fails + * + * @return + */ + vector get_scheduled_transactions() const; + transaction_trace_ptr push_transaction( packed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); transaction_trace_ptr push_transaction( signed_transaction& trx, fc::time_point deadline = fc::time_point::maximum(), uint32_t billed_cpu_time_us = DEFAULT_BILLED_CPU_TIME_US ); action_result push_action(action&& cert_act, uint64_t authorizer); // TODO/QUESTION: Is this needed? diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index a2307904da3..a632cb40643 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -163,16 +164,16 @@ namespace eosio { namespace testing { } if( !skip_pending_trxs ) { - auto unapplied_trxs = control->get_unapplied_transactions(); - for (const auto& trx : unapplied_trxs ) { - auto trace = control->push_transaction(trx, fc::time_point::maximum()); + unapplied_transactions_type unapplied_trxs = control->get_unapplied_transactions(); // make copy of map + for (const auto& entry : unapplied_trxs ) { + auto trace = control->push_transaction(entry.second, fc::time_point::maximum()); if(trace->except) { trace->except->dynamic_rethrow_exception(); } } vector scheduled_trxs; - while( (scheduled_trxs = control->get_scheduled_transactions() ).size() > 0 ) { + while( (scheduled_trxs = get_scheduled_transactions() ).size() > 0 ) { for (const auto& trx : scheduled_trxs ) { auto trace = control->push_scheduled_transaction(trx, fc::time_point::maximum()); if(trace->except) { @@ -237,6 +238,18 @@ namespace eosio { namespace testing { } } + vector base_tester::get_scheduled_transactions() const { + const auto& idx = control->db().get_index(); + + vector result; + + auto itr = idx.begin(); + while( itr != idx.end() && itr->delay_until <= control->pending_block_time() ) { + result.emplace_back(itr->trx_id); + ++itr; + } + return result; + } void base_tester::produce_blocks_until_end_of_round() { uint64_t blocks_per_round; @@ -333,7 +346,7 @@ namespace eosio { namespace testing { { try { if( !control->pending_block_state() ) _start_block(control->head_block_time() + fc::microseconds(config::block_interval_us)); - auto r = control->push_transaction( std::make_shared(trx), deadline, billed_cpu_time_us ); + auto r = control->push_transaction( std::make_shared(std::make_shared(trx)), deadline, billed_cpu_time_us ); if( r->except_ptr ) std::rethrow_exception( r->except_ptr ); if( r->except ) throw *r->except; return r; diff --git a/plugins/COMMUNITY.md b/plugins/COMMUNITY.md index db7d2139d83..55cce74a33b 100644 --- a/plugins/COMMUNITY.md +++ b/plugins/COMMUNITY.md @@ -13,6 +13,10 @@ Third parties are encouraged to make pull requests to this file (`develop` branc | SQL | https://github.com/asiniscalchi/eosio_sql_plugin | | Watch for specific actions and send them to an HTTP URL | https://github.com/eosauthority/eosio-watcher-plugin | | ZMQ / history | https://github.com/cc32d9/eos_zmq_plugin | +| ZMQ Light History API | https://github.com/cc32d9/eos_zmq_light_api | +| Chintai ZMQ Watcher | https://github.com/acoutts/chintai-zeromq-watcher-plugin | +| Mongo History API | https://github.com/CryptoLions/EOS-mongo-history-API | +| State History API | https://github.com/acoutts/EOS-state-history-API | ## DISCLAIMER: diff --git a/plugins/bnet_plugin/bnet_plugin.cpp b/plugins/bnet_plugin/bnet_plugin.cpp index 392df4b31eb..b25631cbabc 100644 --- a/plugins/bnet_plugin/bnet_plugin.cpp +++ b/plugins/bnet_plugin/bnet_plugin.cpp @@ -441,7 +441,7 @@ namespace eosio { if( itr != _transaction_status.end() ) { if( !itr->known_by_peer() ) { _transaction_status.modify( itr, [&]( auto& stat ) { - stat.expired = std::min( fc::time_point::now() + fc::seconds(5), t->trx.expiration ); + stat.expired = std::min( fc::time_point::now() + fc::seconds(5), t->packed_trx->expiration() ); }); } return; @@ -555,8 +555,7 @@ namespace eosio { for( const auto& receipt : s->block->transactions ) { if( receipt.trx.which() == 1 ) { const auto& pt = receipt.trx.get(); - // get id via get_uncached_id() as packed_transaction.id() mutates internal transaction state - const auto& tid = pt.get_uncached_id(); + const auto& tid = pt.id(); auto itr = _transaction_status.find( tid ); if( itr != _transaction_status.end() ) _transaction_status.erase(itr); @@ -765,7 +764,7 @@ namespace eosio { return false; - auto ptrx_ptr = std::make_shared( start->trx->packed_trx ); + auto ptrx_ptr = start->trx->packed_trx; idx.modify( start, [&]( auto& stat ) { stat.mark_known_by_peer(); @@ -1014,8 +1013,7 @@ namespace eosio { for( const auto& receipt : b->transactions ) { if( receipt.trx.which() == 1 ) { const auto& pt = receipt.trx.get(); - // get id via get_uncached_id() as packed_transaction.id() mutates internal transaction state - const auto& id = pt.get_uncached_id(); + const auto& id = pt.id(); mark_transaction_known_by_peer(id); } } @@ -1552,12 +1550,13 @@ namespace eosio { // ilog( "recv trx ${n}", ("n", id) ); if( p->expiration() < fc::time_point::now() ) return; - // get id via get_uncached_id() as packed_transaction.id() mutates internal transaction state - const auto& id = p->get_uncached_id(); + const auto& id = p->id(); if( mark_transaction_known_by_peer( id ) ) return; - app().get_channel().publish(p); + auto ptr = std::make_shared(p); + + app().get_channel().publish(ptr); } } /// namespace eosio diff --git a/plugins/bnet_plugin/include/eosio/bnet_plugin/bnet_plugin.hpp b/plugins/bnet_plugin/include/eosio/bnet_plugin/bnet_plugin.hpp index 5a749ea44ef..9eb3a54c9a4 100644 --- a/plugins/bnet_plugin/include/eosio/bnet_plugin/bnet_plugin.hpp +++ b/plugins/bnet_plugin/include/eosio/bnet_plugin/bnet_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/chain_api_plugin/chain_api_plugin.cpp b/plugins/chain_api_plugin/chain_api_plugin.cpp index f07e48a7f04..8b9fd3f843c 100644 --- a/plugins/chain_api_plugin/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/chain_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/chain_api_plugin/include/eosio/chain_api_plugin/chain_api_plugin.hpp b/plugins/chain_api_plugin/include/eosio/chain_api_plugin/chain_api_plugin.hpp index ad64fcaa1d3..9a854cf0b56 100644 --- a/plugins/chain_api_plugin/include/eosio/chain_api_plugin/chain_api_plugin.hpp +++ b/plugins/chain_api_plugin/include/eosio/chain_api_plugin/chain_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp index 7303768cab4..b62915b5220 100644 --- a/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp +++ b/plugins/chain_interface/include/eosio/chain/plugin_interface.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once @@ -45,19 +45,19 @@ namespace eosio { namespace chain { namespace plugin_interface { namespace incoming { namespace channels { using block = channel_decl; - using transaction = channel_decl; + using transaction = channel_decl; } namespace methods { // synchronously push a block/trx to a single provider using block_sync = method_decl; - using transaction_async = method_decl), first_provider_policy>; + using transaction_async = method_decl), first_provider_policy>; } } namespace compat { namespace channels { - using transaction_ack = channel_decl>; + using transaction_ack = channel_decl>; } } diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index ad67318528b..91d130d3ec5 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -218,6 +218,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("chain-state-db-guard-size-mb", bpo::value()->default_value(config::default_state_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the chain state database drops below this size (in MiB).") ("reversible-blocks-db-size-mb", bpo::value()->default_value(config::default_reversible_cache_size / (1024 * 1024)), "Maximum size (in MiB) of the reversible blocks database") ("reversible-blocks-db-guard-size-mb", bpo::value()->default_value(config::default_reversible_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the reverseible blocks database drops below this size (in MiB).") + ("signature-cpu-billable-pct", bpo::value()->default_value(config::default_sig_cpu_bill_pct / config::percent_1), + "Percentage of actual signature recovery cpu to bill. Whole number percentages, e.g. 50 for 50%") ("chain-threads", bpo::value()->default_value(config::default_controller_thread_pool_size), "Number of worker threads in controller thread pool") ("contracts-console", bpo::bool_switch()->default_value(false), @@ -426,6 +428,11 @@ void chain_plugin::plugin_initialize(const variables_map& options) { "chain-threads ${num} must be greater than 0", ("num", my->chain_config->thread_pool_size) ); } + my->chain_config->sig_cpu_bill_pct = options.at("signature-cpu-billable-pct").as(); + EOS_ASSERT( my->chain_config->sig_cpu_bill_pct >= 0 && my->chain_config->sig_cpu_bill_pct <= 100, plugin_config_exception, + "signature-cpu-billable-pct must be 0 - 100, ${pct}", ("pct", my->chain_config->sig_cpu_bill_pct) ); + my->chain_config->sig_cpu_bill_pct *= config::percent_1; + if( my->wasm_runtime ) my->chain_config->wasm_runtime = *my->wasm_runtime; @@ -738,6 +745,8 @@ void chain_plugin::plugin_shutdown() { my->accepted_transaction_connection.reset(); my->applied_transaction_connection.reset(); my->accepted_confirmation_connection.reset(); + my->chain->get_thread_pool().stop(); + my->chain->get_thread_pool().join(); my->chain.reset(); } @@ -756,7 +765,11 @@ void chain_plugin::accept_block(const signed_block_ptr& block ) { } void chain_plugin::accept_transaction(const chain::packed_transaction& trx, next_function next) { - my->incoming_transaction_async_method(std::make_shared(trx), false, std::forward(next)); + my->incoming_transaction_async_method(std::make_shared(std::make_shared(trx)), false, std::forward(next)); +} + +void chain_plugin::accept_transaction(const chain::transaction_metadata_ptr& trx, next_function next) { + my->incoming_transaction_async_method(trx, false, std::forward(next)); } bool chain_plugin::block_is_on_preferred_chain(const block_id_type& block_id) { @@ -901,7 +914,7 @@ bool chain_plugin::import_reversible_blocks( const fc::path& reversible_dir, new_reversible.create( [&]( auto& ubo ) { ubo.blocknum = num; - ubo.set_block( std::make_shared(tmp) ); + ubo.set_block( std::make_shared(std::move(tmp)) ); }); end = num; } @@ -1526,9 +1539,9 @@ fc::variant read_only::get_block_header_state(const get_block_header_state_param return vo; } -void read_write::push_block(const read_write::push_block_params& params, next_function next) { +void read_write::push_block(read_write::push_block_params&& params, next_function next) { try { - app().get_method()(std::make_shared(params)); + app().get_method()(std::make_shared(std::move(params))); next(read_write::push_block_results{}); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); @@ -1540,18 +1553,19 @@ void read_write::push_transaction(const read_write::push_transaction_params& par try { auto pretty_input = std::make_shared(); auto resolver = make_resolver(this, abi_serializer_max_time); + transaction_metadata_ptr ptrx; try { abi_serializer::from_variant(params, *pretty_input, resolver, abi_serializer_max_time); + ptrx = std::make_shared( pretty_input ); } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction") - app().get_method()(pretty_input, true, [this, next](const fc::static_variant& result) -> void{ + app().get_method()(ptrx, true, [this, next](const fc::static_variant& result) -> void{ if (result.contains()) { next(result.get()); } else { auto trx_trace_ptr = result.get(); try { - chain::transaction_id_type id = trx_trace_ptr->id; fc::variant output; try { output = db.to_variant_with_abi( *trx_trace_ptr, abi_serializer_max_time ); @@ -1559,6 +1573,7 @@ void read_write::push_transaction(const read_write::push_transaction_params& par output = *trx_trace_ptr; } + const chain::transaction_id_type& id = trx_trace_ptr->id; next(read_write::push_transaction_results{id, output}); } CATCH_AND_CALL(next); } diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 832f0b24485..4a41d99455c 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -569,7 +569,7 @@ class read_write { using push_block_params = chain::signed_block; using push_block_results = empty; - void push_block(const push_block_params& params, chain::plugin_interface::next_function next); + void push_block(push_block_params&& params, chain::plugin_interface::next_function next); using push_transaction_params = fc::variant_object; struct push_transaction_results { @@ -664,6 +664,7 @@ class chain_plugin : public plugin { void accept_block( const chain::signed_block_ptr& block ); void accept_transaction(const chain::packed_transaction& trx, chain::plugin_interface::next_function next); + void accept_transaction(const chain::transaction_metadata_ptr& trx, chain::plugin_interface::next_function next); bool block_is_on_preferred_chain(const chain::block_id_type& block_id); diff --git a/plugins/db_size_api_plugin/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/db_size_api_plugin.cpp index 13b717c0789..8eed8b388ed 100644 --- a/plugins/db_size_api_plugin/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/db_size_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp b/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp index 2b0b46bc7ff..54a2d8ef63b 100644 --- a/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp +++ b/plugins/db_size_api_plugin/include/eosio/db_size_api_plugin/db_size_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp b/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp index 39374e692cd..d6f8f53e7b3 100644 --- a/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp +++ b/plugins/faucet_testnet_plugin/faucet_testnet_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp b/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp index 849745044a1..f59b1ef9bb3 100644 --- a/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp +++ b/plugins/faucet_testnet_plugin/include/eosio/faucet_testnet_plugin/faucet_testnet_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/history_api_plugin/history_api_plugin.cpp b/plugins/history_api_plugin/history_api_plugin.cpp index 286e97f6e54..67321fa58d0 100644 --- a/plugins/history_api_plugin/history_api_plugin.cpp +++ b/plugins/history_api_plugin/history_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -32,12 +32,15 @@ void history_api_plugin::plugin_initialize(const variables_map&) {} }} #define CHAIN_RO_CALL(call_name) CALL(history, ro_api, history_apis::read_only, call_name) +//#define CHAIN_RW_CALL(call_name) CALL(history, rw_api, history_apis::read_write, call_name) void history_api_plugin::plugin_startup() { ilog( "starting history_api_plugin" ); auto ro_api = app().get_plugin().get_read_only_api(); + //auto rw_api = app().get_plugin().get_read_write_api(); app().get_plugin().add_api({ +// CHAIN_RO_CALL(get_transaction), CHAIN_RO_CALL(get_actions), CHAIN_RO_CALL(get_transaction), CHAIN_RO_CALL(get_block_detail), diff --git a/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp b/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp index 52ac764a45d..5e6dd936e24 100644 --- a/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp +++ b/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp index f3838726726..5ce12dba17f 100644 --- a/plugins/history_plugin/history_plugin.cpp +++ b/plugins/history_plugin/history_plugin.cpp @@ -505,10 +505,9 @@ namespace eosio { for (const auto &receipt: blk->transactions) { if (receipt.trx.contains()) { auto &pt = receipt.trx.get(); - auto mtrx = transaction_metadata(pt); - if (mtrx.id == result.id) { + if (pt.id() == result.id) { fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(mtrx.trx, abi_serializer_max_time)); + r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer_max_time)); result.trx = move(r); break; } @@ -529,14 +528,14 @@ namespace eosio { for (const auto& receipt: blk->transactions) { if (receipt.trx.contains()) { auto& pt = receipt.trx.get(); - auto mtrx = transaction_metadata(pt); - if( txn_id_matched(mtrx.id) ) { - result.id = mtrx.id; + const auto& id = pt.id(); + if( txn_id_matched(id) ) { + result.id = id; result.last_irreversible_block = chain.last_irreversible_block_num(); result.block_num = *p.block_num_hint; result.block_time = blk->timestamp; fc::mutable_variant_object r("receipt", receipt); - r("trx", chain.to_variant_with_abi(mtrx.trx, abi_serializer_max_time)); + r("trx", chain.to_variant_with_abi(pt.get_signed_transaction(), abi_serializer_max_time)); result.trx = move(r); found = true; break; diff --git a/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp b/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp index 55aec70abe5..5005c279f5a 100644 --- a/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp +++ b/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp b/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp index 838f9b24662..337bf97c239 100644 --- a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp +++ b/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp b/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp index 996aff7c04b..2e8e37f0049 100644 --- a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp +++ b/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/http_client_plugin/http_client_plugin.cpp b/plugins/http_client_plugin/http_client_plugin.cpp index dc27ac34681..cb9d7c8e580 100644 --- a/plugins/http_client_plugin/http_client_plugin.cpp +++ b/plugins/http_client_plugin/http_client_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/http_client_plugin/include/eosio/http_client_plugin/http_client_plugin.hpp b/plugins/http_client_plugin/include/eosio/http_client_plugin/http_client_plugin.hpp index b4bb8a5740e..95803bed1f8 100644 --- a/plugins/http_client_plugin/include/eosio/http_client_plugin/http_client_plugin.hpp +++ b/plugins/http_client_plugin/include/eosio/http_client_plugin/http_client_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index d9be006bb45..c808c5bb99f 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -548,6 +548,8 @@ namespace eosio { my->server.stop_listening(); if(my->https_server.is_listening()) my->https_server.stop_listening(); + if(my->unix_server.is_listening()) + my->unix_server.stop_listening(); } void http_plugin::add_handler(const string& url, const url_handler& handler) { @@ -561,6 +563,9 @@ namespace eosio { try { try { throw; + } catch (chain::unknown_block_exception& e) { + error_results results{400, "Unknown Block", error_results::error_info(e, verbose_http_errors)}; + cb( 400, fc::json::to_string( results )); } catch (chain::unsatisfied_authorization& e) { error_results results{401, "UnAuthorized", error_results::error_info(e, verbose_http_errors)}; cb( 401, fc::json::to_string( results )); diff --git a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp index 7f9aedb01e4..c7bc1ebb2b6 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/login_plugin/include/eosio/login_plugin/login_plugin.hpp b/plugins/login_plugin/include/eosio/login_plugin/login_plugin.hpp index 0bd5d59efed..29c2660c6f3 100644 --- a/plugins/login_plugin/include/eosio/login_plugin/login_plugin.hpp +++ b/plugins/login_plugin/include/eosio/login_plugin/login_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/login_plugin/login_plugin.cpp b/plugins/login_plugin/login_plugin.cpp index 5a7bc5eae3e..0aeac67dce4 100644 --- a/plugins/login_plugin/login_plugin.cpp +++ b/plugins/login_plugin/login_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp b/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp index b9456898b6d..3b636651056 100644 --- a/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp +++ b/plugins/mongo_db_plugin/include/eosio/mongo_db_plugin/mongo_db_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index 3ef0ee252ac..70327c3d96b 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -570,6 +570,15 @@ void handle_mongo_exception( const std::string& desc, int line_num ) { } } +// custom oid to avoid monotonic throttling +// https://docs.mongodb.com/master/core/bulk-write-operations/#avoid-monotonic-throttling +bsoncxx::oid make_custom_oid() { + bsoncxx::oid x = bsoncxx::oid(); + const char* p = x.bytes(); + std::swap((short&)p[0], (short&)p[10]); + return x; +} + } // anonymous namespace void mongo_db_plugin_impl::purge_abi_cache() { @@ -725,7 +734,7 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti using bsoncxx::builder::basic::make_array; namespace bbb = bsoncxx::builder::basic; - const auto& trx = t->trx; + const signed_transaction& trx = t->packed_trx->get_signed_transaction(); if( !filter_include( trx ) ) return; @@ -761,9 +770,10 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti if( t->signing_keys.valid() ) { signing_keys_json = fc::json::to_string( t->signing_keys->second ); } else { - auto signing_keys = trx.get_signature_keys( *chain_id, false, false ); - if( !signing_keys.empty() ) { - signing_keys_json = fc::json::to_string( signing_keys ); + flat_set keys; + trx.get_signature_keys( *chain_id, fc::time_point::maximum(), keys, false ); + if( !keys.empty() ) { + signing_keys_json = fc::json::to_string( keys ); } } @@ -818,6 +828,9 @@ mongo_db_plugin_impl::add_action_trace( mongocxx::bulk_write& bulk_action_traces auto action_traces_doc = bsoncxx::builder::basic::document{}; const chain::base_action_trace& base = atrace; // without inline action traces + // improve data distributivity when using mongodb sharding + action_traces_doc.append( kvp( "_id", make_custom_oid() ) ); + auto v = to_variant_with_abi( base ); string json = fc::json::to_string( v ); try { @@ -1082,11 +1095,8 @@ void mongo_db_plugin_impl::_process_irreversible_block(const chain::block_state_ string trx_id_str; if( receipt.trx.contains() ) { const auto& pt = receipt.trx.get(); - // get id via get_raw_transaction() as packed_transaction.id() mutates internal transaction state - const auto& raw = pt.get_raw_transaction(); - const auto& trx = fc::raw::unpack( raw ); - if( !filter_include( trx ) ) continue; - const auto& id = trx.id(); + if( !filter_include( pt.get_signed_transaction() ) ) continue; + const auto& id = pt.id(); trx_id_str = id.str(); } else { const auto& id = receipt.trx.get(); @@ -1426,7 +1436,7 @@ void mongo_db_plugin_impl::init() { // action traces indexes auto action_traces = mongo_conn[db_name][action_traces_col]; - action_traces.create_index( bsoncxx::from_json( R"xxx({ "trx_id" : 1 })xxx" )); + action_traces.create_index( bsoncxx::from_json( R"xxx({ "block_num" : 1 })xxx" )); // pub_keys indexes auto pub_keys = mongo_conn[db_name][pub_keys_col]; diff --git a/plugins/net_api_plugin/include/eosio/net_api_plugin/net_api_plugin.hpp b/plugins/net_api_plugin/include/eosio/net_api_plugin/net_api_plugin.hpp index 57c5929777e..3c7e1e232ac 100644 --- a/plugins/net_api_plugin/include/eosio/net_api_plugin/net_api_plugin.hpp +++ b/plugins/net_api_plugin/include/eosio/net_api_plugin/net_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/net_api_plugin/net_api_plugin.cpp b/plugins/net_api_plugin/net_api_plugin.cpp index caa31bdad63..3b7327c4313 100644 --- a/plugins/net_api_plugin/net_api_plugin.cpp +++ b/plugins/net_api_plugin/net_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp index 2c7683ccb5e..d732b18cf0c 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/net_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp index bbdd357eec7..76f11da2411 100644 --- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp +++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include @@ -147,11 +147,10 @@ struct request_p2p_message{ notice_message, request_message, sync_request_message, - signed_block, - packed_transaction, + signed_block, // which = 7 + packed_transaction, // which = 8 response_p2p_message, request_p2p_message>; - } // namespace eosio FC_REFLECT( eosio::select_ids, (mode)(pending)(ids) ) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 23b0c0ddb62..889eefb9ed0 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -26,7 +25,6 @@ #include #include #include -#include using namespace eosio::chain::plugin_interface::compat; @@ -42,13 +40,11 @@ namespace eosio { using boost::asio::ip::tcp; using boost::asio::ip::address_v4; using boost::asio::ip::host_name; - using boost::intrusive::rbtree; using boost::multi_index_container; using fc::time_point; using fc::time_point_sec; using eosio::chain::transaction_id_type; - namespace bip = boost::interprocess; class connection; @@ -60,40 +56,24 @@ namespace eosio { using socket_ptr = std::shared_ptr; - using net_message_ptr = shared_ptr; - struct node_transaction_state { transaction_id_type id; time_point_sec expires; /// time after which this may be purged. - /// Expires increased while the txn is - /// "in flight" to anoher peer - packed_transaction packed_txn; - vector serialized_txn; /// the received raw bundle uint32_t block_num = 0; /// block transaction was included in - uint32_t true_block = 0; /// used to reset block_uum when request is 0 - uint16_t requests = 0; /// the number of "in flight" requests for this txn + std::shared_ptr> serialized_txn; /// the received raw bundle }; - struct update_in_flight { - int32_t incr; - update_in_flight (int32_t delta) : incr (delta) {} - void operator() (node_transaction_state& nts) { - int32_t exp = nts.expires.sec_since_epoch(); - nts.expires = fc::time_point_sec (exp + incr * 60); - if( nts.requests == 0 ) { - nts.true_block = nts.block_num; - nts.block_num = 0; - } - nts.requests += incr; - if( nts.requests == 0 ) { - nts.block_num = nts.true_block; - } - } - } incr_in_flight(1), decr_in_flight(-1); - struct by_expiry; struct by_block_num; + struct sha256_less { + bool operator()( const sha256& lhs, const sha256& rhs ) const { + return + std::tie(lhs._hash[0], lhs._hash[1], lhs._hash[2], lhs._hash[3]) < + std::tie(rhs._hash[0], rhs._hash[1], rhs._hash[2], rhs._hash[3]); + } + }; + typedef multi_index_container< node_transaction_state, indexed_by< @@ -101,7 +81,8 @@ namespace eosio { tag< by_id >, member < node_transaction_state, transaction_id_type, - &node_transaction_state::id > >, + &node_transaction_state::id >, + sha256_less >, ordered_non_unique< tag< by_expiry >, member< node_transaction_state, @@ -149,7 +130,7 @@ namespace eosio { }; possible_connections allowed_connections{None}; - connection_ptr find_connection( string host )const; + connection_ptr find_connection(const string& host)const; std::set< connection_ptr > connections; bool done = false; @@ -173,6 +154,7 @@ namespace eosio { string user_agent_name; chain_plugin* chain_plug = nullptr; + producer_plugin* producer_plug = nullptr; int started_sessions = 0; node_transaction_index local_txns; @@ -183,34 +165,30 @@ namespace eosio { channels::transaction_ack::channel_type::handle incoming_transaction_ack_subscription; - void connect( connection_ptr c ); - void connect( connection_ptr c, tcp::resolver::iterator endpoint_itr ); - bool start_session( connection_ptr c ); - void start_listen_loop( ); - void start_read_message( connection_ptr c); + void connect(const connection_ptr& c); + void connect(const connection_ptr& c, tcp::resolver::iterator endpoint_itr); + bool start_session(const connection_ptr& c); + void start_listen_loop(); + void start_read_message(const connection_ptr& c); - void close( connection_ptr c ); + void close(const connection_ptr& c); size_t count_open_sockets() const; template - void send_all( const net_message &msg, VerifierFunc verify ); + void send_all( const std::shared_ptr>& send_buffer, VerifierFunc verify ); - void accepted_block_header(const block_state_ptr&); void accepted_block(const block_state_ptr&); - void irreversible_block(const block_state_ptr&); - void accepted_transaction(const transaction_metadata_ptr&); - void applied_transaction(const transaction_trace_ptr&); - void accepted_confirmation(const header_confirmation&); - - void transaction_ack(const std::pair&); + void transaction_ack(const std::pair&); bool is_valid( const handshake_message &msg); + void send_p2p_request(connection_ptr c); - void handle_message( connection_ptr c, const handshake_message &msg); - void handle_message( connection_ptr c, const chain_size_message &msg); - void handle_message( connection_ptr c, const go_away_message &msg ); + void handle_message(const connection_ptr& c, const handshake_message& msg); + void handle_message(const connection_ptr& c, const chain_size_message& msg); + void handle_message(const connection_ptr& c, const go_away_message& msg ); + /** \name Peer Timestamps * Time message handling * @{ @@ -224,21 +202,24 @@ namespace eosio { * floating-double arithmetic with rounding done by the hardware. * This is necessary in order to avoid overflow and preserve precision. */ - void handle_message( connection_ptr c, const time_message &msg); + void handle_message(const connection_ptr& c, const time_message& msg); /** @} */ - void handle_message( connection_ptr c, const notice_message &msg); - void handle_message( connection_ptr c, const request_message &msg); - void handle_message( connection_ptr c, const sync_request_message &msg); - void handle_message( connection_ptr c, const signed_block &msg); - void handle_message( connection_ptr c, const packed_transaction &msg); + void handle_message(const connection_ptr& c, const notice_message& msg); + void handle_message(const connection_ptr& c, const request_message& msg); + void handle_message(const connection_ptr& c, const sync_request_message& msg); + void handle_message(const connection_ptr& c, const signed_block& msg) = delete; // signed_block_ptr overload used instead + void handle_message(const connection_ptr& c, const signed_block_ptr& msg); + void handle_message(const connection_ptr& c, const packed_transaction& msg) = delete; // packed_transaction_ptr overload used instead + void handle_message(const connection_ptr& c, const packed_transaction_ptr& msg); void handle_message( connection_ptr c, const request_p2p_message &msg); void handle_message( connection_ptr c, const response_p2p_message &msg); void start_conn_timer(boost::asio::steady_timer::duration du, std::weak_ptr from_connection); - void start_txn_timer( ); - void start_monitors( ); + void start_txn_timer(); + void start_monitors(); - void expire_txns( ); + void expire_txns(); + void expire_local_txns(); void connection_monitor(std::weak_ptr from_connection); /** \name Peer Timestamps * Time message handling @@ -318,14 +299,16 @@ namespace eosio { */ constexpr auto def_send_buffer_size_mb = 4; constexpr auto def_send_buffer_size = 1024*1024*def_send_buffer_size_mb; + constexpr auto def_max_write_queue_size = def_send_buffer_size*10; + constexpr boost::asio::chrono::milliseconds def_read_delay_for_full_write_queue{100}; + constexpr auto def_max_reads_in_flight = 1000; + constexpr auto def_max_trx_in_progress_size = 100*1024*1024; // 100 MB constexpr auto def_max_clients = 25; // 0 for unlimited clients constexpr auto def_max_nodes_per_host = 1; constexpr auto def_conn_retry_wait = 30; constexpr auto def_txn_expire_wait = std::chrono::seconds(3); constexpr auto def_resp_expected_wait = std::chrono::seconds(5); constexpr auto def_sync_fetch_span = 100; - constexpr uint32_t def_max_just_send = 1500; // roughly 1 "mtu" - constexpr bool large_msg_notify = false; constexpr auto message_header_size = 4; @@ -352,34 +335,16 @@ namespace eosio { constexpr uint16_t net_version = proto_explicit_sync; - /** - * Index by id - * Index by is_known, block_num, validated_time, this is the order we will broadcast - * to peer. - * Index by is_noticed, validated_time - * - */ struct transaction_state { transaction_id_type id; - bool is_known_by_peer = false; ///< true if we sent or received this trx to this peer or received notice from peer - bool is_noticed_to_peer = false; ///< have we sent peer notice we know it (true if we receive from this peer) uint32_t block_num = 0; ///< the block number the transaction was included in time_point_sec expires; - time_point requested_time; /// in case we fetch large trx - }; - - struct update_txn_expiry { - time_point_sec new_expiry; - update_txn_expiry(time_point_sec e) : new_expiry(e) {} - void operator() (transaction_state& ts) { - ts.expires = new_expiry; - } }; typedef multi_index_container< transaction_state, indexed_by< - ordered_unique< tag, member >, + ordered_unique< tag, member, sha256_less >, ordered_non_unique< tag< by_expiry >, member< transaction_state,fc::time_point_sec,&transaction_state::expires >>, ordered_non_unique< tag, @@ -402,10 +367,7 @@ namespace eosio { }; struct update_request_time { - void operator() (struct transaction_state &ts) { - ts.requested_time = time_point::now(); - } - void operator () (struct eosio::peer_block_state &bs) { + void operator() (struct eosio::peer_block_state &bs) { bs.requested_time = time_point::now(); } } set_request_time; @@ -413,7 +375,7 @@ namespace eosio { typedef multi_index_container< eosio::peer_block_state, indexed_by< - ordered_unique< tag, member >, + ordered_unique< tag, member, sha256_less >, ordered_unique< tag, member > > > peer_block_state_index; @@ -423,9 +385,6 @@ namespace eosio { void operator() (eosio::peer_block_state& bs) { bs.is_known = true; } - void operator() (transaction_state& ts) { - ts.is_known_by_peer = true; - } } set_is_known; @@ -433,12 +392,7 @@ namespace eosio { uint32_t new_bnum; update_block_num(uint32_t bnum) : new_bnum(bnum) {} void operator() (node_transaction_state& nts) { - if (nts.requests ) { - nts.true_block = new_bnum; - } - else { - nts.block_num = new_bnum; - } + nts.block_num = new_bnum; } void operator() (transaction_state& ts) { ts.block_num = new_bnum; @@ -466,6 +420,86 @@ namespace eosio { static void populate(handshake_message &hello); }; + class queued_buffer : boost::noncopyable { + public: + void clear_write_queue() { + _write_queue.clear(); + _sync_write_queue.clear(); + _write_queue_size = 0; + } + + void clear_out_queue() { + while ( _out_queue.size() > 0 ) { + _out_queue.pop_front(); + } + } + + uint32_t write_queue_size() const { return _write_queue_size; } + + bool is_out_queue_empty() const { return _out_queue.empty(); } + + bool ready_to_send() const { + // if out_queue is not empty then async_write is in progress + return ((!_sync_write_queue.empty() || !_write_queue.empty()) && _out_queue.empty()); + } + + bool add_write_queue( const std::shared_ptr>& buff, + std::function callback, + bool to_sync_queue ) { + if( to_sync_queue ) { + _sync_write_queue.push_back( {buff, callback} ); + } else { + _write_queue.push_back( {buff, callback} ); + } + _write_queue_size += buff->size(); + if( _write_queue_size > 2 * def_max_write_queue_size ) { + return false; + } + return true; + } + + void fill_out_buffer( std::vector& bufs ) { + if( _sync_write_queue.size() > 0 ) { // always send msgs from sync_write_queue first + fill_out_buffer( bufs, _sync_write_queue ); + } else { // postpone real_time write_queue if sync queue is not empty + fill_out_buffer( bufs, _write_queue ); + EOS_ASSERT( _write_queue_size == 0, plugin_exception, "write queue size expected to be zero" ); + } + } + + void out_callback( boost::system::error_code ec, std::size_t w ) { + for( auto& m : _out_queue ) { + m.callback( ec, w ); + } + } + + private: + struct queued_write; + void fill_out_buffer( std::vector& bufs, + deque& w_queue ) { + while ( w_queue.size() > 0 ) { + auto& m = w_queue.front(); + bufs.push_back( boost::asio::buffer( *m.buff )); + _write_queue_size -= m.buff->size(); + _out_queue.emplace_back( m ); + w_queue.pop_front(); + } + } + + private: + struct queued_write { + std::shared_ptr> buff; + std::function callback; + }; + + uint32_t _write_queue_size = 0; + deque _write_queue; + deque _sync_write_queue; // sync_write_queue will be sent first + deque _out_queue; + + }; // queued_buffer + + class connection : public std::enable_shared_from_this { public: explicit connection( string endpoint ); @@ -481,14 +515,12 @@ namespace eosio { fc::message_buffer<1024*1024> pending_message_buffer; fc::optional outstanding_read_bytes; - vector blk_buffer; - struct queued_write { - std::shared_ptr> buff; - std::function callback; - }; - deque write_queue; - deque out_queue; + + queued_buffer buffer_queue; + + uint32_t reads_in_flight = 0; + uint32_t trx_in_progress_size = 0; fc::sha256 node_id; handshake_message last_handshake_recv; handshake_message last_handshake_sent; @@ -498,6 +530,7 @@ namespace eosio { uint16_t protocol_version = 0; string peer_addr; unique_ptr response_expected; + unique_ptr read_delay_timer; optional pending_fetch; go_away_reason no_retry = no_reason; block_id_type fork_head; @@ -539,10 +572,6 @@ namespace eosio { /** \name Peer Timestamps * Time message handling */ - /** @{ */ - /** \brief Convert an std::chrono nanosecond rep to a human readable string - */ - char* convert_tstamp(const tstamp& t); /** \brief Populate and queue time_message */ void send_time(); @@ -564,18 +593,22 @@ namespace eosio { const string peer_name(); - void txn_send_pending(const vector &ids); - void txn_send(const vector &txn_lis); + void txn_send_pending(const vector& ids); + void txn_send(const vector& txn_lis); void blk_send_branch(); - void blk_send(const vector &txn_lis); + void blk_send(const block_id_type& blkid); void stop_send(); void enqueue( const net_message &msg, bool trigger_send = true ); + void enqueue_block( const signed_block_ptr& sb, bool trigger_send = true, bool to_sync_queue = false ); + void enqueue_buffer( const std::shared_ptr>& send_buffer, + bool trigger_send, go_away_reason close_after_send, + bool to_sync_queue = false); void cancel_sync(go_away_reason); void flush_queues(); bool enqueue_sync_block(); - void request_sync_blocks (uint32_t start, uint32_t end); + void request_sync_blocks(uint32_t start, uint32_t end); void cancel_wait(); void sync_wait(); @@ -583,9 +616,10 @@ namespace eosio { void sync_timeout(boost::system::error_code ec); void fetch_timeout(boost::system::error_code ec); - void queue_write(std::shared_ptr> buff, + void queue_write(const std::shared_ptr>& buff, bool trigger_send, - std::function callback); + std::function callback, + bool to_sync_queue = false); void do_queue_write(); void send_p2p_request(bool discoverable); @@ -601,7 +635,7 @@ namespace eosio { */ bool process_next_message(net_plugin_impl& impl, uint32_t message_length); - bool add_peer_block(const peer_block_state &pbs); + bool add_peer_block(const peer_block_state& pbs); fc::optional _logger_variant; const fc::variant_object& get_logger_variant() { @@ -629,15 +663,35 @@ namespace eosio { } }; - struct msgHandler : public fc::visitor { + struct msg_handler : public fc::visitor { net_plugin_impl &impl; connection_ptr c; - msgHandler( net_plugin_impl &imp, connection_ptr conn) : impl(imp), c(conn) {} + msg_handler( net_plugin_impl &imp, const connection_ptr& conn) : impl(imp), c(conn) {} + + void operator()( const signed_block& msg ) const { + EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should be called" ); + } + void operator()( signed_block& msg ) const { + EOS_ASSERT( false, plugin_config_exception, "operator()(signed_block&&) should be called" ); + } + void operator()( const packed_transaction& msg ) const { + EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should be called" ); + } + void operator()( packed_transaction& msg ) const { + EOS_ASSERT( false, plugin_config_exception, "operator()(packed_transaction&&) should be called" ); + } + + void operator()( signed_block&& msg ) const { + impl.handle_message( c, std::make_shared( std::move( msg ) ) ); + } + void operator()( packed_transaction&& msg ) const { + impl.handle_message( c, std::make_shared( std::move( msg ) ) ); + } template - void operator()(const T &msg) const + void operator()( T&& msg ) const { - impl.handle_message( c, msg); + impl.handle_message( c, std::forward(msg) ); } }; @@ -665,37 +719,33 @@ namespace eosio { void set_state(stages s); bool sync_required(); void send_handshakes(); - bool is_active(connection_ptr conn); - void reset_lib_num(connection_ptr conn); - void request_next_chunk(connection_ptr conn = connection_ptr() ); - void start_sync(connection_ptr c, uint32_t target); - void reassign_fetch(connection_ptr c, go_away_reason reason); - void verify_catchup(connection_ptr c, uint32_t num, block_id_type id); - void rejected_block(connection_ptr c, uint32_t blk_num); - void recv_block(connection_ptr c, const block_id_type &blk_id, uint32_t blk_num); - void recv_handshake(connection_ptr c, const handshake_message& msg); - void recv_notice(connection_ptr c, const notice_message& msg); + bool is_active(const connection_ptr& conn); + void reset_lib_num(const connection_ptr& conn); + void request_next_chunk(const connection_ptr& conn = connection_ptr()); + void start_sync(const connection_ptr& c, uint32_t target); + void reassign_fetch(const connection_ptr& c, go_away_reason reason); + void verify_catchup(const connection_ptr& c, uint32_t num, const block_id_type& id); + void rejected_block(const connection_ptr& c, uint32_t blk_num); + void recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num); + void recv_handshake(const connection_ptr& c, const handshake_message& msg); + void recv_notice(const connection_ptr& c, const notice_message& msg); }; class dispatch_manager { public: - uint32_t just_send_it_max = 0; + std::multimap received_blocks; + std::multimap received_transactions; - vector req_trx; + void bcast_transaction(const transaction_metadata_ptr& trx); + void rejected_transaction(const transaction_id_type& msg); + void bcast_block(const block_state_ptr& bs); + void rejected_block(const block_id_type& id); - std::multimap received_blocks; - std::multimap received_transactions; + void recv_block(const connection_ptr& conn, const block_id_type& msg, uint32_t bnum); + void recv_transaction(const connection_ptr& conn, const transaction_id_type& id); + void recv_notice(const connection_ptr& conn, const notice_message& msg, bool generated); - void bcast_transaction (const packed_transaction& msg); - void rejected_transaction (const transaction_id_type& msg); - void bcast_block (const signed_block& msg); - void rejected_block (const block_id_type &id); - - void recv_block (connection_ptr conn, const block_id_type& msg, uint32_t bnum); - void recv_transaction(connection_ptr conn, const transaction_id_type& id); - void recv_notice (connection_ptr conn, const notice_message& msg, bool generated); - - void retry_fetch (connection_ptr conn); + void retry_fetch(const connection_ptr& conn); }; //--------------------------------------------------------------------------- @@ -714,6 +764,7 @@ namespace eosio { protocol_version(0), peer_addr(endpoint), response_expected(), + read_delay_timer(), pending_fetch(), no_retry(no_reason), fork_head(), @@ -738,6 +789,7 @@ namespace eosio { protocol_version(0), peer_addr(), response_expected(), + read_delay_timer(), pending_fetch(), no_retry(no_reason), fork_head(), @@ -754,6 +806,7 @@ namespace eosio { auto *rnd = node_id.data(); rnd[0] = 0; response_expected.reset(new boost::asio::steady_timer(app().get_io_service())); + read_delay_timer.reset(new boost::asio::steady_timer(app().get_io_service())); } bool connection::connected() { @@ -771,7 +824,7 @@ namespace eosio { } void connection::flush_queues() { - write_queue.clear(); + buffer_queue.clear_write_queue(); } void connection::close() { @@ -785,7 +838,7 @@ namespace eosio { connecting = false; syncing = false; if( last_req ) { - my_impl->dispatcher->retry_fetch (shared_from_this()); + my_impl->dispatcher->retry_fetch(shared_from_this()); } reset(); sent_handshake_count = 0; @@ -794,60 +847,33 @@ namespace eosio { my_impl->sync_master->reset_lib_num(shared_from_this()); fc_dlog(logger, "canceling wait on ${p}", ("p",peer_name())); cancel_wait(); + if( read_delay_timer ) read_delay_timer->cancel(); pending_message_buffer.reset(); } - void connection::txn_send_pending(const vector &ids) { + void connection::txn_send_pending(const vector& ids) { + const std::set known_ids(ids.cbegin(), ids.cend()); + my_impl->expire_local_txns(); for(auto tx = my_impl->local_txns.begin(); tx != my_impl->local_txns.end(); ++tx ){ - if(tx->serialized_txn.size() && tx->block_num == 0) { - bool found = false; - for(auto known : ids) { - if( known == tx->id) { - found = true; - break; - } - } - if(!found) { - my_impl->local_txns.modify(tx,incr_in_flight); - queue_write(std::make_shared>(tx->serialized_txn), - true, - [tx_id=tx->id](boost::system::error_code ec, std::size_t ) { - auto& local_txns = my_impl->local_txns; - auto tx = local_txns.get().find(tx_id); - if (tx != local_txns.end()) { - local_txns.modify(tx, decr_in_flight); - } else { - fc_wlog(logger, "Local pending TX erased before queued_write called callback"); - } - }); - } + const bool found = known_ids.find( tx->id ) != known_ids.cend(); + if( !found ) { + queue_write( tx->serialized_txn, true, []( boost::system::error_code ec, std::size_t ) {} ); } } } - void connection::txn_send(const vector &ids) { - for(auto t : ids) { + void connection::txn_send(const vector& ids) { + for(const auto& t : ids) { auto tx = my_impl->local_txns.get().find(t); - if( tx != my_impl->local_txns.end() && tx->serialized_txn.size()) { - my_impl->local_txns.modify( tx,incr_in_flight); - queue_write(std::make_shared>(tx->serialized_txn), - true, - [t](boost::system::error_code ec, std::size_t ) { - auto& local_txns = my_impl->local_txns; - auto tx = local_txns.get().find(t); - if (tx != local_txns.end()) { - local_txns.modify(tx, decr_in_flight); - } else { - fc_wlog(logger, "Local TX erased before queued_write called callback"); - } - }); + if( tx != my_impl->local_txns.end() ) { + queue_write( tx->serialized_txn, true, []( boost::system::error_code ec, std::size_t ) {} ); } } } void connection::blk_send_branch() { - controller &cc = my_impl->chain_plug->chain(); - uint32_t head_num = cc.fork_db_head_block_num (); + controller& cc = my_impl->chain_plug->chain(); + uint32_t head_num = cc.fork_db_head_block_num(); notice_message note; note.known_blocks.mode = normal; note.known_blocks.pending = 0; @@ -876,85 +902,53 @@ namespace eosio { } head_id = cc.fork_db_head_block_id(); } - catch (const assert_exception &ex) { + catch (const assert_exception& ex) { elog( "unable to retrieve block info: ${n} for ${p}",("n",ex.to_string())("p",peer_name())); enqueue(note); return; } - catch (const fc::exception &ex) { + catch (const fc::exception& ex) { } catch (...) { } - vector bstack; - block_id_type null_id; - for (auto bid = head_id; bid != null_id && bid != lib_id; ) { - try { - - // if the last handshake received indicates that we are catching up on a fork - // that the peer is already partially aware of, no need to resend blocks - if (remote_head_id == bid) { - break; - } - - signed_block_ptr b = cc.fetch_block_by_id(bid); - if ( b ) { - bid = b->previous; - bstack.push_back(b); - } - else { - break; - } - } catch (...) { - break; - } - } - size_t count = 0; - if (!bstack.empty()) { - if (bstack.back()->previous == lib_id || bstack.back()->previous == remote_head_id) { - count = bstack.size(); - while (bstack.size()) { - enqueue(*bstack.back()); - bstack.pop_back(); - } - } - fc_ilog(logger, "Sent ${n} blocks on my fork",("n",count)); + if( !peer_requested ) { + peer_requested = sync_state( block_header::num_from_id(lib_id)+1, + block_header::num_from_id(head_id), + block_header::num_from_id(lib_id) ); } else { - fc_ilog(logger, "Nothing to send on fork request"); + uint32_t start = std::min( peer_requested->last + 1, block_header::num_from_id(lib_id)+1 ); + uint32_t end = std::max( peer_requested->end_block, block_header::num_from_id(head_id) ); + peer_requested = sync_state( start, end, start - 1 ); } + enqueue_sync_block(); + // still want to send transactions along during blk branch sync syncing = false; } - void connection::blk_send(const vector &ids) { + void connection::blk_send(const block_id_type& blkid) { controller &cc = my_impl->chain_plug->chain(); - int count = 0; - for(auto &blkid : ids) { - ++count; - try { - signed_block_ptr b = cc.fetch_block_by_id(blkid); - if(b) { - fc_dlog(logger,"found block for id at num ${n}",("n",b->block_num())); - enqueue(net_message(*b)); - } - else { - ilog("fetch block by id returned null, id ${id} on block ${c} of ${s} for ${p}", - ("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; - } - } - catch (const assert_exception &ex) { - elog( "caught assert on fetch_block_by_id, ${ex}, id ${id} on block ${c} of ${s} for ${p}", - ("ex",ex.to_string())("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; - } - catch (...) { - elog( "caught othser exception fetching block id ${id} on block ${c} of ${s} for ${p}", - ("id",blkid)("c",count)("s",ids.size())("p",peer_name())); - break; + try { + signed_block_ptr b = cc.fetch_block_by_id(blkid); + if(b) { + fc_dlog(logger,"found block for id at num ${n}",("n",b->block_num())); + peer_block_state pbstate = {blkid, block_header::num_from_id(blkid), true, true, time_point()}; + add_peer_block(pbstate); + enqueue_block( b ); + } else { + ilog("fetch block by id returned null, id ${id} for ${p}", + ("id",blkid)("p",peer_name())); } } - + catch (const assert_exception &ex) { + elog( "caught assert on fetch_block_by_id, ${ex}, id ${id} for ${p}", + ("ex",ex.to_string())("id",blkid)("p",peer_name())); + } + catch (...) { + elog( "caught other exception fetching block id ${id} for ${p}", + ("id",blkid)("p",peer_name())); + } } void connection::send_p2p_request(bool discoverable) @@ -984,7 +978,7 @@ namespace eosio { syncing = false; } - void connection::send_handshake( ) { + void connection::send_handshake() { handshake_initializer::populate(last_handshake_sent); last_handshake_sent.generation = ++sent_handshake_count; fc_dlog(logger, "Sending handshake generation ${g} to ${ep}", @@ -992,15 +986,6 @@ namespace eosio { enqueue(last_handshake_sent); } - char* connection::convert_tstamp(const tstamp& t) - { - const long long NsecPerSec{1000000000}; - time_t seconds = t / NsecPerSec; - strftime(ts, ts_buffer_size, "%F %T", localtime(&seconds)); - snprintf(ts+19, ts_buffer_size-19, ".%lld", t % NsecPerSec); - return ts; - } - void connection::send_time() { time_message xpkt; xpkt.org = rec; @@ -1018,16 +1003,23 @@ namespace eosio { enqueue(xpkt); } - void connection::queue_write(std::shared_ptr> buff, + void connection::queue_write(const std::shared_ptr>& buff, bool trigger_send, - std::function callback) { - write_queue.push_back({buff, callback}); - if(out_queue.empty() && trigger_send) + std::function callback, + bool to_sync_queue) { + if( !buffer_queue.add_write_queue( buff, callback, to_sync_queue )) { + fc_wlog( logger, "write_queue full ${s} bytes, giving up on connection ${p}", + ("s", buffer_queue.write_queue_size())("p", peer_name()) ); + my_impl->close( shared_from_this() ); + return; + } + if( buffer_queue.is_out_queue_empty() && trigger_send) { do_queue_write(); + } } void connection::do_queue_write() { - if(write_queue.empty() || !out_queue.empty()) + if( !buffer_queue.ready_to_send() ) return; connection_wptr c(shared_from_this()); if(!socket->is_open()) { @@ -1036,21 +1028,14 @@ namespace eosio { return; } std::vector bufs; - while (write_queue.size() > 0) { - auto& m = write_queue.front(); - bufs.push_back(boost::asio::buffer(*m.buff)); - out_queue.push_back(m); - write_queue.pop_front(); - } + buffer_queue.fill_out_buffer( bufs ); boost::asio::async_write(*socket, bufs, [c](boost::system::error_code ec, std::size_t w) { try { auto conn = c.lock(); if(!conn) return; - for (auto& m: conn->out_queue) { - m.callback(ec, w); - } + conn->buffer_queue.out_callback( ec, w ); if(ec) { string pname = conn ? conn->peer_name() : "no connection name"; @@ -1063,9 +1048,7 @@ namespace eosio { my_impl->close(conn); return; } - while (conn->out_queue.size() > 0) { - conn->out_queue.pop_front(); - } + conn->buffer_queue.clear_out_queue(); conn->enqueue_sync_block(); conn->do_queue_write(); } @@ -1088,8 +1071,8 @@ namespace eosio { } void connection::cancel_sync(go_away_reason reason) { - fc_dlog(logger,"cancel sync reason = ${m}, write queue size ${o} peer ${p}", - ("m",reason_str(reason)) ("o", write_queue.size())("p", peer_name())); + fc_dlog(logger,"cancel sync reason = ${m}, write queue size ${o} bytes peer ${p}", + ("m",reason_str(reason)) ("o", buffer_queue.write_queue_size())("p", peer_name())); cancel_wait(); flush_queues(); switch (reason) { @@ -1106,7 +1089,6 @@ namespace eosio { } bool connection::enqueue_sync_block() { - controller& cc = app().find_plugin()->chain(); if (!peer_requested) return false; uint32_t num = ++peer_requested->last; @@ -1115,9 +1097,10 @@ namespace eosio { peer_requested.reset(); } try { + controller& cc = my_impl->chain_plug->chain(); signed_block_ptr sb = cc.fetch_block_by_number(num); if(sb) { - enqueue( *sb, trigger_send); + enqueue_block( sb, trigger_send, true); return true; } } catch ( ... ) { @@ -1126,22 +1109,50 @@ namespace eosio { return false; } - void connection::enqueue( const net_message &m, bool trigger_send ) { + void connection::enqueue( const net_message& m, bool trigger_send ) { go_away_reason close_after_send = no_reason; if (m.contains()) { close_after_send = m.get().reason; } uint32_t payload_size = fc::raw::pack_size( m ); - char * header = reinterpret_cast(&payload_size); - size_t header_size = sizeof(payload_size); + char* header = reinterpret_cast(&payload_size); + size_t header_size = sizeof(payload_size); size_t buffer_size = header_size + payload_size; auto send_buffer = std::make_shared>(buffer_size); fc::datastream ds( send_buffer->data(), buffer_size); ds.write( header, header_size ); fc::raw::pack( ds, m ); + + enqueue_buffer( send_buffer, trigger_send, close_after_send ); + } + + void connection::enqueue_block( const signed_block_ptr& sb, bool trigger_send, bool to_sync_queue ) { + // this implementation is to avoid copy of signed_block to net_message + int which = 7; // matches which of net_message for signed_block + + uint32_t which_size = fc::raw::pack_size( unsigned_int( which )); + uint32_t payload_size = which_size + fc::raw::pack_size( *sb ); + + char* header = reinterpret_cast(&payload_size); + size_t header_size = sizeof(payload_size); + size_t buffer_size = header_size + payload_size; + + auto send_buffer = std::make_shared>(buffer_size); + fc::datastream ds( send_buffer->data(), buffer_size); + ds.write( header, header_size ); + fc::raw::pack( ds, unsigned_int( which )); + fc::raw::pack( ds, *sb ); + + enqueue_buffer( send_buffer, trigger_send, no_reason, to_sync_queue ); + } + + void connection::enqueue_buffer( const std::shared_ptr>& send_buffer, bool trigger_send, + go_away_reason close_after_send, + bool to_sync_queue ) + { connection_wptr weak_this = shared_from_this(); queue_write(send_buffer,trigger_send, [weak_this, close_after_send](boost::system::error_code ec, std::size_t ) { @@ -1155,7 +1166,8 @@ namespace eosio { } else { fc_wlog(logger, "connection expired before enqueued net_message called callback!"); } - }); + }, + to_sync_queue); } void connection::cancel_wait() { @@ -1163,7 +1175,7 @@ namespace eosio { response_expected->cancel(); } - void connection::sync_wait( ) { + void connection::sync_wait() { response_expected->expires_from_now( my_impl->resp_expected_period); connection_wptr c(shared_from_this()); response_expected->async_wait( [c]( boost::system::error_code ec){ @@ -1177,7 +1189,7 @@ namespace eosio { } ); } - void connection::fetch_wait( ) { + void connection::fetch_wait() { response_expected->expires_from_now( my_impl->resp_expected_period); connection_wptr c(shared_from_this()); response_expected->async_wait( [c]( boost::system::error_code ec){ @@ -1193,12 +1205,12 @@ namespace eosio { void connection::sync_timeout( boost::system::error_code ec ) { if( !ec ) { - my_impl->sync_master->reassign_fetch (shared_from_this(),benign_other); + my_impl->sync_master->reassign_fetch(shared_from_this(), benign_other); } else if( ec == boost::asio::error::operation_aborted) { } else { - elog ("setting timer for sync request got error ${ec}",("ec", ec.message())); + elog("setting timer for sync request got error ${ec}",("ec", ec.message())); } } @@ -1214,21 +1226,21 @@ namespace eosio { void connection::fetch_timeout( boost::system::error_code ec ) { if( !ec ) { - if( pending_fetch.valid() && !( pending_fetch->req_trx.empty( ) || pending_fetch->req_blocks.empty( ) ) ) { - my_impl->dispatcher->retry_fetch (shared_from_this() ); + if( pending_fetch.valid() && !( pending_fetch->req_trx.empty() || pending_fetch->req_blocks.empty() ) ) { + my_impl->dispatcher->retry_fetch(shared_from_this()); } } else if( ec == boost::asio::error::operation_aborted ) { - if( !connected( ) ) { + if( !connected() ) { fc_dlog(logger, "fetch timeout was cancelled due to dead connection"); } } else { - elog( "setting timer for fetch request got error ${ec}", ("ec", ec.message( ) ) ); + elog( "setting timer for fetch request got error ${ec}", ("ec", ec.message() ) ); } } - void connection::request_sync_blocks (uint32_t start, uint32_t end) { + void connection::request_sync_blocks(uint32_t start, uint32_t end) { sync_request_message srm = {start,end}; enqueue( net_message(srm)); sync_wait(); @@ -1236,27 +1248,17 @@ namespace eosio { bool connection::process_next_message(net_plugin_impl& impl, uint32_t message_length) { try { - // If it is a signed_block, then save the raw message for the cache - // This must be done before we unpack the message. - // This code is copied from fc::io::unpack(..., unsigned_int) - auto index = pending_message_buffer.read_index(); - uint64_t which = 0; char b = 0; uint8_t by = 0; - do { - pending_message_buffer.peek(&b, 1, index); - which |= uint32_t(uint8_t(b) & 0x7f) << by; - by += 7; - } while( uint8_t(b) & 0x80 && by < 32); - - if (which == uint64_t(net_message::tag::value)) { - blk_buffer.resize(message_length); - auto index = pending_message_buffer.read_index(); - pending_message_buffer.peek(blk_buffer.data(), message_length, index); - } auto ds = pending_message_buffer.create_datastream(); net_message msg; fc::raw::unpack(ds, msg); - msgHandler m(impl, shared_from_this() ); - msg.visit(m); + msg_handler m(impl, shared_from_this() ); + if( msg.contains() ) { + m( std::move( msg.get() ) ); + } else if( msg.contains() ) { + m( std::move( msg.get() ) ); + } else { + msg.visit( m ); + } } catch( const fc::exception& e ) { edump((e.to_detail_string() )); impl.close( shared_from_this() ); @@ -1265,7 +1267,7 @@ namespace eosio { return true; } - bool connection::add_peer_block(const peer_block_state &entry) { + bool connection::add_peer_block(const peer_block_state& entry) { auto bptr = blk_state.get().find(entry.id); bool added = (bptr == blk_state.end()); if (added){ @@ -1293,11 +1295,11 @@ namespace eosio { ,source() ,state(in_sync) { - chain_plug = app( ).find_plugin( ); + chain_plug = app().find_plugin(); EOS_ASSERT( chain_plug, chain::missing_chain_plugin_exception, "" ); } - constexpr auto sync_manager::stage_str(stages s ) { + constexpr auto sync_manager::stage_str(stages s) { switch (s) { case in_sync : return "in sync"; case lib_catchup: return "lib catchup"; @@ -1310,11 +1312,11 @@ namespace eosio { if (state == newstate) { return; } - fc_dlog(logger, "old state ${os} becoming ${ns}",("os",stage_str (state))("ns",stage_str (newstate))); + fc_dlog(logger, "old state ${os} becoming ${ns}",("os",stage_str(state))("ns",stage_str(newstate))); state = newstate; } - bool sync_manager::is_active(connection_ptr c) { + bool sync_manager::is_active(const connection_ptr& c) { if (state == head_catchup && c) { bool fhset = c->fork_head != block_id_type(); fc_dlog(logger, "fork_head_num = ${fn} fork_head set = ${s}", @@ -1324,7 +1326,7 @@ namespace eosio { return state != in_sync; } - void sync_manager::reset_lib_num(connection_ptr c) { + void sync_manager::reset_lib_num(const connection_ptr& c) { if(state == in_sync) { source.reset(); } @@ -1338,19 +1340,19 @@ namespace eosio { } } - bool sync_manager::sync_required( ) { + bool sync_manager::sync_required() { fc_dlog(logger, "last req = ${req}, last recv = ${recv} known = ${known} our head = ${head}", - ("req",sync_last_requested_num)("recv",sync_next_expected_num)("known",sync_known_lib_num)("head",chain_plug->chain( ).fork_db_head_block_num( ))); + ("req",sync_last_requested_num)("recv",sync_next_expected_num)("known",sync_known_lib_num)("head",chain_plug->chain().fork_db_head_block_num())); return( sync_last_requested_num < sync_known_lib_num || - chain_plug->chain( ).fork_db_head_block_num( ) < sync_last_requested_num ); + chain_plug->chain().fork_db_head_block_num() < sync_last_requested_num ); } - void sync_manager::request_next_chunk( connection_ptr conn ) { + void sync_manager::request_next_chunk( const connection_ptr& conn ) { uint32_t head_block = chain_plug->chain().fork_db_head_block_num(); if (head_block < sync_last_requested_num && source && source->current()) { - fc_ilog (logger, "ignoring request, head is ${h} last req = ${r} source is ${p}", + fc_ilog(logger, "ignoring request, head is ${h} last req = ${r} source is ${p}", ("h",head_block)("r",sync_last_requested_num)("p",source->peer_name())); return; } @@ -1429,7 +1431,7 @@ namespace eosio { } } - void sync_manager::send_handshakes () + void sync_manager::send_handshakes() { for( auto &ci : my_impl->connections) { if( ci->current()) { @@ -1438,7 +1440,7 @@ namespace eosio { } } - void sync_manager::start_sync( connection_ptr c, uint32_t target) { + void sync_manager::start_sync(const connection_ptr& c, uint32_t target) { if( target > sync_known_lib_num) { sync_known_lib_num = target; } @@ -1462,20 +1464,20 @@ namespace eosio { request_next_chunk(c); } - void sync_manager::reassign_fetch(connection_ptr c, go_away_reason reason) { + void sync_manager::reassign_fetch(const connection_ptr& c, go_away_reason reason) { fc_ilog(logger, "reassign_fetch, our last req is ${cc}, next expected is ${ne} peer ${p}", ( "cc",sync_last_requested_num)("ne",sync_next_expected_num)("p",c->peer_name())); if (c == source) { - c->cancel_sync (reason); + c->cancel_sync(reason); sync_last_requested_num = 0; request_next_chunk(); } } - void sync_manager::recv_handshake (connection_ptr c, const handshake_message &msg) { + void sync_manager::recv_handshake(const connection_ptr& c, const handshake_message& msg) { controller& cc = chain_plug->chain(); - uint32_t lib_num = cc.last_irreversible_block_num( ); + uint32_t lib_num = cc.last_irreversible_block_num(); uint32_t peer_lib = msg.last_irreversible_block_num; reset_lib_num(c); c->syncing = false; @@ -1483,16 +1485,16 @@ namespace eosio { //-------------------------------- // sync need checks; (lib == last irreversible block) // - // 0. my head block id == peer head id means we are all caugnt up block wise + // 0. my head block id == peer head id means we are all caught up block wise // 1. my head block num < peer lib - start sync locally // 2. my lib > peer head num - send an last_irr_catch_up notice if not the first generation // // 3 my head block num <= peer head block num - update sync state and send a catchup request - // 4 my head block num > peer block num ssend a notice catchup if this is not the first generation + // 4 my head block num > peer block num send a notice catchup if this is not the first generation // //----------------------------- - uint32_t head = cc.fork_db_head_block_num( ); + uint32_t head = cc.fork_db_head_block_num(); block_id_type head_id = cc.fork_db_head_block_id(); if (head_id == msg.head_id) { fc_dlog(logger, "sync check state 0"); @@ -1528,7 +1530,7 @@ namespace eosio { if (head <= msg.head_num ) { fc_dlog(logger, "sync check state 3"); - verify_catchup (c, msg.head_num, msg.head_id); + verify_catchup(c, msg.head_num, msg.head_id); return; } else { @@ -1544,13 +1546,13 @@ namespace eosio { c->syncing = true; return; } - elog ("sync check failed to resolve status"); + elog("sync check failed to resolve status"); } - void sync_manager::verify_catchup(connection_ptr c, uint32_t num, block_id_type id) { + void sync_manager::verify_catchup(const connection_ptr& c, uint32_t num, const block_id_type& id) { request_message req; req.req_blocks.mode = catch_up; - for (auto cc : my_impl->connections) { + for (const auto& cc : my_impl->connections) { if (cc->fork_head == id || cc->fork_head_num > num) { req.req_blocks.mode = none; @@ -1560,7 +1562,8 @@ namespace eosio { if( req.req_blocks.mode == catch_up ) { c->fork_head = id; c->fork_head_num = num; - ilog ("got a catch_up notice while in ${s}, fork head num = ${fhn} target LIB = ${lib} next_expected = ${ne}", ("s",stage_str(state))("fhn",num)("lib",sync_known_lib_num)("ne", sync_next_expected_num)); + ilog("got a catch_up notice while in ${s}, fork head num = ${fhn} target LIB = ${lib} next_expected = ${ne}", + ("s",stage_str(state))("fhn",num)("lib",sync_known_lib_num)("ne", sync_next_expected_num)); if (state == lib_catchup) return; set_state(head_catchup); @@ -1573,26 +1576,30 @@ namespace eosio { c->enqueue( req ); } - void sync_manager::recv_notice (connection_ptr c, const notice_message &msg) { - fc_ilog (logger, "sync_manager got ${m} block notice",("m",modes_str(msg.known_blocks.mode))); + void sync_manager::recv_notice(const connection_ptr& c, const notice_message& msg) { + fc_ilog(logger, "sync_manager got ${m} block notice",("m",modes_str(msg.known_blocks.mode))); + if( msg.known_blocks.ids.size() > 1 ) { + elog( "Invalid notice_message, known_blocks.ids.size ${s}", ("s", msg.known_blocks.ids.size()) ); + my_impl->close(c); + return; + } if (msg.known_blocks.mode == catch_up) { if (msg.known_blocks.ids.size() == 0) { - elog ("got a catch up with ids size = 0"); - } - else { - verify_catchup(c, msg.known_blocks.pending, msg.known_blocks.ids.back()); + elog("got a catch up with ids size = 0"); + } else { + verify_catchup(c, msg.known_blocks.pending, msg.known_blocks.ids.back()); } } else { c->last_handshake_recv.last_irreversible_block_num = msg.known_trx.pending; - reset_lib_num (c); + reset_lib_num(c); start_sync(c, msg.known_trx.pending); } } - void sync_manager::rejected_block (connection_ptr c, uint32_t blk_num) { + void sync_manager::rejected_block(const connection_ptr& c, uint32_t blk_num) { if (state != in_sync ) { - fc_ilog (logger, "block ${bn} not accepted from ${p}",("bn",blk_num)("p",c->peer_name())); + fc_ilog(logger, "block ${bn} not accepted from ${p}",("bn",blk_num)("p",c->peer_name())); sync_last_requested_num = 0; source.reset(); my_impl->close(c); @@ -1600,23 +1607,23 @@ namespace eosio { send_handshakes(); } } - void sync_manager::recv_block (connection_ptr c, const block_id_type &blk_id, uint32_t blk_num) { + void sync_manager::recv_block(const connection_ptr& c, const block_id_type& blk_id, uint32_t blk_num) { fc_dlog(logger," got block ${bn} from ${p}",("bn",blk_num)("p",c->peer_name())); if (state == lib_catchup) { if (blk_num != sync_next_expected_num) { - fc_ilog (logger, "expected block ${ne} but got ${bn}",("ne",sync_next_expected_num)("bn",blk_num)); + fc_ilog(logger, "expected block ${ne} but got ${bn}",("ne",sync_next_expected_num)("bn",blk_num)); my_impl->close(c); return; } sync_next_expected_num = blk_num + 1; } if (state == head_catchup) { - fc_dlog (logger, "sync_manager in head_catchup state"); + fc_dlog(logger, "sync_manager in head_catchup state"); set_state(in_sync); source.reset(); block_id_type null_id; - for (auto cp : my_impl->connections) { + for (const auto& cp : my_impl->connections) { if (cp->fork_head == null_id) { continue; } @@ -1647,52 +1654,34 @@ namespace eosio { //------------------------------------------------------------------------ - void dispatch_manager::bcast_block (const signed_block &bsum) { + void dispatch_manager::bcast_block(const block_state_ptr& bs) { std::set skips; - auto range = received_blocks.equal_range(bsum.id()); + auto range = received_blocks.equal_range(bs->id); for (auto org = range.first; org != range.second; ++org) { skips.insert(org->second); } received_blocks.erase(range.first, range.second); - net_message msg(bsum); - uint32_t packsiz = fc::raw::pack_size(msg); - uint32_t msgsiz = packsiz + sizeof(packsiz); - notice_message pending_notify; - block_id_type bid = bsum.id(); - uint32_t bnum = bsum.block_num(); - pending_notify.known_blocks.mode = normal; - pending_notify.known_blocks.ids.push_back( bid ); - pending_notify.known_trx.mode = none; - - peer_block_state pbstate = {bid, bnum, false,true,time_point()}; - // skip will be empty if our producer emitted this block so just send it - if (( large_msg_notify && msgsiz > just_send_it_max) && !skips.empty()) { - fc_ilog(logger, "block size is ${ms}, sending notify",("ms", msgsiz)); - my_impl->send_all(pending_notify, [&skips, pbstate](connection_ptr c) -> bool { - if (skips.find(c) != skips.end() || !c->current()) - return false; - - bool unknown = c->add_peer_block(pbstate); - if (!unknown) { - elog("${p} already has knowledge of block ${b}", ("p",c->peer_name())("b",pbstate.block_num)); - } - return unknown; - }); - } - else { - pbstate.is_known = true; - for (auto cp : my_impl->connections) { - if (skips.find(cp) != skips.end() || !cp->current()) { - continue; - } - cp->add_peer_block(pbstate); - cp->enqueue( bsum ); + block_id_type bid = bs->id; + uint32_t bnum = bs->block_num; + peer_block_state pbstate = {bid, bnum, false, true, time_point()}; + + pbstate.is_known = true; + for( auto& cp : my_impl->connections ) { + if( skips.find( cp ) != skips.end() || !cp->current() ) { + continue; + } + bool has_block = cp->last_handshake_recv.last_irreversible_block_num >= bnum; + if( !has_block ) { + fc_dlog(logger, "bcast block ${b} to ${p}", ("b", bnum)("p", cp->peer_name())); + cp->add_peer_block( pbstate ); + cp->enqueue_block( bs->block ); } } + } - void dispatch_manager::recv_block (connection_ptr c, const block_id_type& id, uint32_t bnum) { + void dispatch_manager::recv_block(const connection_ptr& c, const block_id_type& id, uint32_t bnum) { received_blocks.insert(std::make_pair(id, c)); if (c && c->last_req && @@ -1707,15 +1696,15 @@ namespace eosio { c->cancel_wait(); } - void dispatch_manager::rejected_block (const block_id_type& id) { + void dispatch_manager::rejected_block(const block_id_type& id) { fc_dlog(logger,"not sending rejected transaction ${tid}",("tid",id)); auto range = received_blocks.equal_range(id); received_blocks.erase(range.first, range.second); } - void dispatch_manager::bcast_transaction (const packed_transaction& trx) { + void dispatch_manager::bcast_transaction(const transaction_metadata_ptr& ptrx) { std::set skips; - transaction_id_type id = trx.id(); + const auto& id = ptrx->id; auto range = received_transactions.equal_range(id); for (auto org = range.first; org != range.second; ++org) { @@ -1723,78 +1712,49 @@ namespace eosio { } received_transactions.erase(range.first, range.second); - for (auto ref = req_trx.begin(); ref != req_trx.end(); ++ref) { - if (*ref == id) { - req_trx.erase(ref); - break; - } - } - - if( my_impl->local_txns.get().find( id ) != my_impl->local_txns.end( ) ) { //found + if( my_impl->local_txns.get().find( id ) != my_impl->local_txns.end() ) { //found fc_dlog(logger, "found trxid in local_trxs" ); return; } - uint32_t packsiz = 0; - uint32_t bufsiz = 0; - - time_point_sec trx_expiration = trx.expiration(); - - net_message msg(trx); - packsiz = fc::raw::pack_size(msg); - bufsiz = packsiz + sizeof(packsiz); - vector buff(bufsiz); - fc::datastream ds( buff.data(), bufsiz); - ds.write( reinterpret_cast(&packsiz), sizeof(packsiz) ); - fc::raw::pack( ds, msg ); - node_transaction_state nts = {id, - trx_expiration, - trx, - std::move(buff), - 0, 0, 0}; + + time_point_sec trx_expiration = ptrx->packed_trx->expiration(); + const packed_transaction& trx = *ptrx->packed_trx; + + // this implementation is to avoid copy of packed_transaction to net_message + int which = 8; // matches which of net_message for packed_transaction + + uint32_t which_size = fc::raw::pack_size( unsigned_int( which )); + uint32_t payload_size = which_size + fc::raw::pack_size( trx ); + + char* header = reinterpret_cast(&payload_size); + size_t header_size = sizeof(payload_size); + size_t buffer_size = header_size + payload_size; + + auto buff = std::make_shared>(buffer_size); + fc::datastream ds( buff->data(), buffer_size); + ds.write( header, header_size ); + fc::raw::pack( ds, unsigned_int( which )); + fc::raw::pack( ds, trx ); + + node_transaction_state nts = {id, trx_expiration, 0, buff}; my_impl->local_txns.insert(std::move(nts)); - if( !large_msg_notify || bufsiz <= just_send_it_max) { - my_impl->send_all( trx, [id, &skips, trx_expiration](connection_ptr c) -> bool { - if( skips.find(c) != skips.end() || c->syncing ) { - return false; - } - const auto& bs = c->trx_state.find(id); - bool unknown = bs == c->trx_state.end(); - if( unknown) { - c->trx_state.insert(transaction_state({id,true,true,0,trx_expiration,time_point() })); - fc_dlog(logger, "sending whole trx to ${n}", ("n",c->peer_name() ) ); - } else { - update_txn_expiry ute(trx_expiration); - c->trx_state.modify(bs, ute); - } - return unknown; - }); - } - else { - notice_message pending_notify; - pending_notify.known_trx.mode = normal; - pending_notify.known_trx.ids.push_back( id ); - pending_notify.known_blocks.mode = none; - my_impl->send_all(pending_notify, [id, &skips, trx_expiration](connection_ptr c) -> bool { - if (skips.find(c) != skips.end() || c->syncing) { - return false; - } - const auto& bs = c->trx_state.find(id); - bool unknown = bs == c->trx_state.end(); - if( unknown) { - fc_dlog(logger, "sending notice to ${n}", ("n",c->peer_name() ) ); - c->trx_state.insert(transaction_state({id,false,true,0,trx_expiration,time_point() })); - } else { - update_txn_expiry ute(trx_expiration); - c->trx_state.modify(bs, ute); - } - return unknown; - }); - } + my_impl->send_all( buff, [&id, &skips, trx_expiration](const connection_ptr& c) -> bool { + if( skips.find(c) != skips.end() || c->syncing ) { + return false; + } + const auto& bs = c->trx_state.find(id); + bool unknown = bs == c->trx_state.end(); + if( unknown ) { + c->trx_state.insert(transaction_state({id,0,trx_expiration})); + fc_dlog(logger, "sending trx to ${n}", ("n",c->peer_name() ) ); + } + return unknown; + }); } - void dispatch_manager::recv_transaction (connection_ptr c, const transaction_id_type& id) { + void dispatch_manager::recv_transaction(const connection_ptr& c, const transaction_id_type& id) { received_transactions.insert(std::make_pair(id, c)); if (c && c->last_req && @@ -1808,50 +1768,32 @@ namespace eosio { c->cancel_wait(); } - void dispatch_manager::rejected_transaction (const transaction_id_type& id) { + void dispatch_manager::rejected_transaction(const transaction_id_type& id) { fc_dlog(logger,"not sending rejected transaction ${tid}",("tid",id)); auto range = received_transactions.equal_range(id); received_transactions.erase(range.first, range.second); } - void dispatch_manager::recv_notice (connection_ptr c, const notice_message& msg, bool generated) { + void dispatch_manager::recv_notice(const connection_ptr& c, const notice_message& msg, bool generated) { request_message req; req.req_trx.mode = none; req.req_blocks.mode = none; bool send_req = false; - controller &cc = my_impl->chain_plug->chain(); if (msg.known_trx.mode == normal) { req.req_trx.mode = normal; req.req_trx.pending = 0; - for( const auto& t : msg.known_trx.ids ) { - const auto &tx = my_impl->local_txns.get( ).find( t ); - - if( tx == my_impl->local_txns.end( ) ) { - fc_dlog(logger,"did not find ${id}",("id",t)); - - //At this point the details of the txn are not known, just its id. This - //effectively gives 120 seconds to learn of the details of the txn which - //will update the expiry in bcast_transaction - c->trx_state.insert( (transaction_state){t,true,true,0,time_point_sec(time_point::now()) + 120, - time_point()} ); - - req.req_trx.ids.push_back( t ); - req_trx.push_back( t ); - } - else { - fc_dlog(logger,"big msg manager found txn id in table, ${id}",("id", t)); - } - } - send_req = !req.req_trx.ids.empty(); - fc_dlog(logger,"big msg manager send_req ids list has ${ids} entries", ("ids", req.req_trx.ids.size())); + send_req = false; } else if (msg.known_trx.mode != none) { - elog ("passed a notice_message with something other than a normal on none known_trx"); + elog("passed a notice_message with something other than a normal on none known_trx"); return; } if (msg.known_blocks.mode == normal) { req.req_blocks.mode = normal; - for( const auto& blkid : msg.known_blocks.ids) { + controller& cc = my_impl->chain_plug->chain(); + // known_blocks.ids is never > 1 + if( !msg.known_blocks.ids.empty() ) { + const block_id_type& blkid = msg.known_blocks.ids.back(); signed_block_ptr b; peer_block_state entry = {blkid,0,true,true,fc::time_point()}; try { @@ -1873,7 +1815,7 @@ namespace eosio { } } else if (msg.known_blocks.mode != none) { - elog ("passed a notice_message with something other than a normal on none known_blocks"); + elog("passed a notice_message with something other than a normal on none known_blocks"); return; } fc_dlog( logger, "send req = ${sr}", ("sr",send_req)); @@ -1884,7 +1826,7 @@ namespace eosio { } } - void dispatch_manager::retry_fetch( connection_ptr c ) { + void dispatch_manager::retry_fetch(const connection_ptr& c) { if (!c->last_req) { return; } @@ -1904,14 +1846,14 @@ namespace eosio { ("b",modes_str(c->last_req->req_blocks.mode))("t",modes_str(c->last_req->req_trx.mode))); return; } - for (auto conn : my_impl->connections) { + for (auto& conn : my_impl->connections) { if (conn == c || conn->last_req) { continue; } bool sendit = false; if (is_txn) { auto trx = conn->trx_state.get().find(tid); - sendit = trx != conn->trx_state.end() && trx->is_known_by_peer; + sendit = trx != conn->trx_state.end(); } else { auto blk = conn->blk_state.get().find(bid); @@ -1934,7 +1876,7 @@ namespace eosio { //------------------------------------------------------------------------ - void net_plugin_impl::connect( connection_ptr c ) { + void net_plugin_impl::connect(const connection_ptr& c) { if( c->no_retry != go_away_reason::no_reason) { fc_dlog( logger, "Skipping connect due to go_away reason ${r}",("r", reason_str( c->no_retry ))); return; @@ -1943,7 +1885,7 @@ namespace eosio { auto colon = c->peer_addr.find(':'); if (colon == std::string::npos || colon == 0) { - elog ("Invalid peer address. must be \"host:port\": ${p}", ("p",c->peer_addr)); + elog("Invalid peer address. must be \"host:port\": ${p}", ("p",c->peer_addr)); for ( auto itr : connections ) { if((*itr).peer_addr == c->peer_addr) { (*itr).reset(); @@ -1976,7 +1918,7 @@ namespace eosio { }); } - void net_plugin_impl::connect( connection_ptr c, tcp::resolver::iterator endpoint_itr ) { + void net_plugin_impl::connect(const connection_ptr& c, tcp::resolver::iterator endpoint_itr) { if( c->no_retry != go_away_reason::no_reason) { string rsn = reason_str(c->no_retry); return; @@ -2039,7 +1981,9 @@ namespace eosio { } } } - bool net_plugin_impl::start_session( connection_ptr con ) { + + bool net_plugin_impl::start_session(const connection_ptr& con) { + boost::asio::ip::tcp::no_delay nodelay( true ); boost::system::error_code ec; con->socket->set_option( nodelay, ec ); @@ -2061,7 +2005,7 @@ namespace eosio { } - void net_plugin_impl::start_listen_loop( ) { + void net_plugin_impl::start_listen_loop() { auto socket = std::make_shared( std::ref( app().get_io_service() ) ); acceptor->async_accept( *socket, [socket,this]( boost::system::error_code ec ) { if( !ec ) { @@ -2084,7 +2028,7 @@ namespace eosio { } } if (num_clients != visitors) { - ilog ("checking max client, visitors = ${v} num clients ${n}",("v",visitors)("n",num_clients)); + ilog("checking max client, visitors = ${v} num clients ${n}",("v",visitors)("n",num_clients)); num_clients = visitors; } if( from_addr < max_nodes_per_host && (max_client_count == 0 || num_clients < max_client_count )) { @@ -2103,7 +2047,7 @@ namespace eosio { fc_elog(logger, "Error max_client_count ${m} exceeded", ( "m", max_client_count) ); } - socket->close( ); + socket->close(); } } } else { @@ -2125,7 +2069,7 @@ namespace eosio { }); } - void net_plugin_impl::start_read_message( connection_ptr conn ) { + void net_plugin_impl::start_read_message(const connection_ptr& conn) { try { if(!conn->socket) { @@ -2150,6 +2094,37 @@ namespace eosio { } }; + if( conn->buffer_queue.write_queue_size() > def_max_write_queue_size || + conn->reads_in_flight > def_max_reads_in_flight || + conn->trx_in_progress_size > def_max_trx_in_progress_size ) + { + // too much queued up, reschedule + if( conn->buffer_queue.write_queue_size() > def_max_write_queue_size ) { + peer_wlog( conn, "write_queue full ${s} bytes", ("s", conn->buffer_queue.write_queue_size()) ); + } else if( conn->reads_in_flight > def_max_reads_in_flight ) { + peer_wlog( conn, "max reads in flight ${s}", ("s", conn->reads_in_flight) ); + } else { + peer_wlog( conn, "max trx in progress ${s} bytes", ("s", conn->trx_in_progress_size) ); + } + if( conn->buffer_queue.write_queue_size() > 2*def_max_write_queue_size || + conn->reads_in_flight > 2*def_max_reads_in_flight || + conn->trx_in_progress_size > 2*def_max_trx_in_progress_size ) + { + fc_wlog( logger, "queues over full, giving up on connection ${p}", ("p", conn->peer_name()) ); + my_impl->close( conn ); + return; + } + if( !conn->read_delay_timer ) return; + conn->read_delay_timer->expires_from_now( def_read_delay_for_full_write_queue ); + conn->read_delay_timer->async_wait([this, weak_conn]( boost::system::error_code ) { + auto conn = weak_conn.lock(); + if( !conn ) return; + start_read_message( conn ); + } ); + return; + } + + ++conn->reads_in_flight; boost::asio::async_read(*conn->socket, conn->pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, [this,weak_conn]( boost::system::error_code ec, std::size_t bytes_transferred ) { @@ -2158,6 +2133,7 @@ namespace eosio { return; } + --conn->reads_in_flight; conn->outstanding_read_bytes.reset(); try { @@ -2180,7 +2156,8 @@ namespace eosio { conn->pending_message_buffer.peek(&message_length, sizeof(message_length), index); if(message_length > def_send_buffer_size*2 || message_length == 0) { boost::system::error_code ec; - elog("incoming message length unexpected (${i}), from ${p}", ("i", message_length)("p",boost::lexical_cast(conn->socket->remote_endpoint(ec)))); + elog("incoming message length unexpected (${i}), from ${p}", + ("i", message_length)("p",boost::lexical_cast(conn->socket->remote_endpoint(ec)))); close(conn); return; } @@ -2250,15 +2227,15 @@ namespace eosio { template - void net_plugin_impl::send_all( const net_message &msg, VerifierFunc verify) { + void net_plugin_impl::send_all(const std::shared_ptr>& send_buffer, VerifierFunc verify) { for( auto &c : connections) { - if( c->current() && verify( c)) { - c->enqueue( msg ); + if( c->current() && verify( c )) { + c->enqueue_buffer( send_buffer, true, no_reason ); } } } - bool net_plugin_impl::is_valid( const handshake_message &msg) { + bool net_plugin_impl::is_valid(const handshake_message& msg) { // Do some basic validation of an incoming handshake_message, so things // that really aren't handshake messages can be quickly discarded without // affecting state. @@ -2283,11 +2260,11 @@ namespace eosio { return valid; } - void net_plugin_impl::handle_message( connection_ptr c, const chain_size_message &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const chain_size_message& msg) { peer_ilog(c, "received chain_size_message"); - } + void net_plugin_impl::handle_message( connection_ptr c, const request_p2p_message &msg){ peer_ilog(c, "received request_p2p_message"); string rspm; @@ -2338,7 +2315,9 @@ namespace eosio { }}} } - void net_plugin_impl::handle_message( connection_ptr c, const handshake_message &msg) { + + void net_plugin_impl::handle_message(const connection_ptr& c, const handshake_message& msg) { + peer_ilog(c, "received handshake_message"); if (!is_valid(msg)) { peer_elog( c, "bad handshake message"); @@ -2346,7 +2325,7 @@ namespace eosio { return; } controller& cc = chain_plug->chain(); - uint32_t lib_num = cc.last_irreversible_block_num( ); + uint32_t lib_num = cc.last_irreversible_block_num(); uint32_t peer_lib = msg.last_irreversible_block_num; if( c->connecting ) { c->connecting = false; @@ -2445,7 +2424,7 @@ namespace eosio { sync_master->recv_handshake(c,msg); } - void net_plugin_impl::handle_message( connection_ptr c, const go_away_message &msg ) { + void net_plugin_impl::handle_message(const connection_ptr& c, const go_away_message& msg) { string rsn = reason_str( msg.reason ); peer_ilog(c, "received go_away_message"); ilog( "received a go away message from ${p}, reason = ${r}", @@ -2455,10 +2434,10 @@ namespace eosio { c->node_id = msg.node_id; } c->flush_queues(); - close (c); + close(c); } - void net_plugin_impl::handle_message(connection_ptr c, const time_message &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const time_message& msg) { peer_ilog(c, "received time_message"); /* We've already lost however many microseconds it took to dispatch * the message, but it can't be helped. @@ -2491,7 +2470,7 @@ namespace eosio { c->rec = 0; } - void net_plugin_impl::handle_message( connection_ptr c, const notice_message &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const notice_message& msg) { // peer tells us about one or more blocks or txns. When done syncing, forward on // notices of previously unknown blocks or txns, // @@ -2517,7 +2496,7 @@ namespace eosio { send_req = true; size_t known_sum = local_txns.size(); if( known_sum ) { - for( const auto& t : local_txns.get( ) ) { + for( const auto& t : local_txns.get() ) { req.req_trx.ids.push_back( t.id ); } } @@ -2525,7 +2504,7 @@ namespace eosio { break; } case normal: { - dispatcher->recv_notice (c, msg, false); + dispatcher->recv_notice(c, msg, false); } } @@ -2542,7 +2521,7 @@ namespace eosio { break; } case normal : { - dispatcher->recv_notice (c, msg, false); + dispatcher->recv_notice(c, msg, false); break; } default: { @@ -2555,15 +2534,23 @@ namespace eosio { } } - void net_plugin_impl::handle_message( connection_ptr c, const request_message &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const request_message& msg) { + if( msg.req_blocks.ids.size() > 1 ) { + elog( "Invalid request_message, req_blocks.ids.size ${s}", ("s", msg.req_blocks.ids.size()) ); + close(c); + return; + } + switch (msg.req_blocks.mode) { case catch_up : peer_ilog(c, "received request_message:catch_up"); - c->blk_send_branch( ); + c->blk_send_branch(); break; case normal : peer_ilog(c, "received request_message:normal"); - c->blk_send(msg.req_blocks.ids); + if( !msg.req_blocks.ids.empty() ) { + c->blk_send(msg.req_blocks.ids.back()); + } break; default:; } @@ -2585,7 +2572,7 @@ namespace eosio { } - void net_plugin_impl::handle_message( connection_ptr c, const sync_request_message &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const sync_request_message& msg) { if( msg.end_block == 0) { c->peer_requested.reset(); c->flush_queues(); @@ -2595,7 +2582,14 @@ namespace eosio { } } - void net_plugin_impl::handle_message( connection_ptr c, const packed_transaction &msg) { + size_t calc_trx_size( const packed_transaction_ptr& trx ) { + // transaction is stored packed and unpacked, double packed_size and size of signed as an approximation of use + return (trx->get_packed_transaction().size() * 2 + sizeof(trx->get_signed_transaction())) * 2 + + trx->get_packed_context_free_data().size() * 4 + + trx->get_signatures().size() * sizeof(signature_type); + } + + void net_plugin_impl::handle_message(const connection_ptr& c, const packed_transaction_ptr& trx) { fc_dlog(logger, "got a packed transaction, cancel wait"); peer_ilog(c, "received packed_transaction"); controller& cc = my_impl->chain_plug->chain(); @@ -2607,35 +2601,40 @@ namespace eosio { fc_dlog(logger, "got a txn during sync - dropping"); return; } - transaction_id_type tid = msg.id(); + + auto ptrx = std::make_shared( trx ); + const auto& tid = ptrx->id; + c->cancel_wait(); if(local_txns.get().find(tid) != local_txns.end()) { fc_dlog(logger, "got a duplicate transaction - dropping"); return; } dispatcher->recv_transaction(c, tid); - chain_plug->accept_transaction(msg, [=](const static_variant& result) { + c->trx_in_progress_size += calc_trx_size( ptrx->packed_trx ); + chain_plug->accept_transaction(ptrx, [c, this, ptrx](const static_variant& result) { + c->trx_in_progress_size -= calc_trx_size( ptrx->packed_trx ); if (result.contains()) { peer_dlog(c, "bad packed_transaction : ${m}", ("m",result.get()->what())); } else { auto trace = result.get(); if (!trace->except) { fc_dlog(logger, "chain accepted transaction"); - dispatcher->bcast_transaction(msg); + this->dispatcher->bcast_transaction(ptrx); return; } peer_elog(c, "bad packed_transaction : ${m}", ("m",trace->except->what())); } - dispatcher->rejected_transaction(tid); + dispatcher->rejected_transaction(ptrx->id); }); } - void net_plugin_impl::handle_message( connection_ptr c, const signed_block &msg) { + void net_plugin_impl::handle_message(const connection_ptr& c, const signed_block_ptr& msg) { controller &cc = chain_plug->chain(); - block_id_type blk_id = msg.id(); - uint32_t blk_num = msg.block_num(); + block_id_type blk_id = msg->id(); + uint32_t blk_num = msg->block_num(); fc_dlog(logger, "canceling wait on ${p}", ("p",c->peer_name())); c->cancel_wait(); @@ -2650,14 +2649,13 @@ namespace eosio { } dispatcher->recv_block(c, blk_id, blk_num); - fc::microseconds age( fc::time_point::now() - msg.timestamp); + fc::microseconds age( fc::time_point::now() - msg->timestamp); peer_ilog(c, "received signed_block : #${n} block age in secs = ${age}", ("n",blk_num)("age",age.to_seconds())); go_away_reason reason = fatal_other; try { - signed_block_ptr sbp = std::make_shared(msg); - chain_plug->accept_block(sbp); //, sync_master->is_active(c)); + chain_plug->accept_block(msg); //, sync_master->is_active(c)); reason = no_reason; } catch( const unlinkable_block_exception &ex) { peer_elog(c, "bad signed_block : ${m}", ("m",ex.what())); @@ -2680,7 +2678,7 @@ namespace eosio { update_block_num ubn(blk_num); if( reason == no_reason ) { - for (const auto &recpt : msg.transactions) { + for (const auto &recpt : msg->transactions) { auto id = (recpt.trx.which() == 0) ? recpt.trx.get() : recpt.trx.get().id(); auto ltx = local_txns.get().find(id); if( ltx != local_txns.end()) { @@ -2715,21 +2713,21 @@ namespace eosio { transaction_check->expires_from_now( txn_exp_period); transaction_check->async_wait( [this](boost::system::error_code ec) { if( !ec) { - expire_txns( ); + expire_txns(); } else { elog( "Error from transaction check monitor: ${m}",( "m", ec.message())); - start_txn_timer( ); + start_txn_timer(); } }); } void net_plugin_impl::ticker() { - keepalive_timer->expires_from_now (keepalive_interval); - keepalive_timer->async_wait ([this](boost::system::error_code ec) { - ticker (); + keepalive_timer->expires_from_now(keepalive_interval); + keepalive_timer->async_wait([this](boost::system::error_code ec) { + ticker(); if (ec) { - wlog ("Peer keepalive ticked sooner than expected: ${m}", ("m", ec.message())); + wlog("Peer keepalive ticked sooner than expected: ${m}", ("m", ec.message())); } for (auto &c : connections ) { if (c->socket->is_open()) { @@ -2747,24 +2745,37 @@ namespace eosio { } void net_plugin_impl::expire_txns() { - start_txn_timer( ); - auto &old = local_txns.get(); - auto ex_up = old.upper_bound( time_point::now()); - auto ex_lo = old.lower_bound( fc::time_point_sec( 0)); - old.erase( ex_lo, ex_up); + start_txn_timer(); - auto &stale = local_txns.get(); - controller &cc = chain_plug->chain(); - uint32_t bn = cc.last_irreversible_block_num(); - stale.erase( stale.lower_bound(1), stale.upper_bound(bn) ); + auto now = time_point::now(); + auto start_size = local_txns.size(); + + expire_local_txns(); + + controller& cc = chain_plug->chain(); + uint32_t lib = cc.last_irreversible_block_num(); for ( auto &c : connections ) { auto &stale_txn = c->trx_state.get(); - stale_txn.erase( stale_txn.lower_bound(1), stale_txn.upper_bound(bn) ); + stale_txn.erase( stale_txn.lower_bound(1), stale_txn.upper_bound(lib) ); auto &stale_txn_e = c->trx_state.get(); stale_txn_e.erase(stale_txn_e.lower_bound(time_point_sec()), stale_txn_e.upper_bound(time_point::now())); auto &stale_blk = c->blk_state.get(); - stale_blk.erase( stale_blk.lower_bound(1), stale_blk.upper_bound(bn) ); + stale_blk.erase( stale_blk.lower_bound(1), stale_blk.upper_bound(lib) ); } + fc_dlog(logger, "expire_txns ${n}us size ${s} removed ${r}", + ("n", time_point::now() - now)("s", start_size)("r", start_size - local_txns.size()) ); + } + + void net_plugin_impl::expire_local_txns() { + auto& old = local_txns.get(); + auto ex_lo = old.lower_bound( fc::time_point_sec(0) ); + auto ex_up = old.upper_bound( time_point::now() ); + old.erase( ex_lo, ex_up ); + + auto& stale = local_txns.get(); + controller& cc = chain_plug->chain(); + uint32_t lib = cc.last_irreversible_block_num(); + stale.erase( stale.lower_bound(1), stale.upper_bound(lib) ); } void net_plugin_impl::connection_monitor(std::weak_ptr from_connection) { @@ -2792,8 +2803,8 @@ namespace eosio { start_conn_timer(connector_period, std::weak_ptr()); } - void net_plugin_impl::close( connection_ptr c ) { - if( c->peer_addr.empty( ) && c->socket->is_open() ) { + void net_plugin_impl::close(const connection_ptr& c) { + if( c->peer_addr.empty() && c->socket->is_open() ) { if (num_clients == 0) { fc_wlog( logger, "num_clients already at 0"); } @@ -2804,40 +2815,19 @@ namespace eosio { c->close(); } - void net_plugin_impl::accepted_block_header(const block_state_ptr& block) { - fc_dlog(logger,"signaled, id = ${id}",("id", block->id)); - } - void net_plugin_impl::accepted_block(const block_state_ptr& block) { fc_dlog(logger,"signaled, id = ${id}",("id", block->id)); - dispatcher->bcast_block(*block->block); + dispatcher->bcast_block(block); } - void net_plugin_impl::irreversible_block(const block_state_ptr&block) { - fc_dlog(logger,"signaled, id = ${id}",("id", block->id)); - } - - void net_plugin_impl::accepted_transaction(const transaction_metadata_ptr& md) { - fc_dlog(logger,"signaled, id = ${id}",("id", md->id)); -// dispatcher->bcast_transaction(md->packed_trx); - } - - void net_plugin_impl::applied_transaction(const transaction_trace_ptr& txn) { - fc_dlog(logger,"signaled, id = ${id}",("id", txn->id)); - } - - void net_plugin_impl::accepted_confirmation(const header_confirmation& head) { - fc_dlog(logger,"signaled, id = ${id}",("id", head.block_id)); - } - - void net_plugin_impl::transaction_ack(const std::pair& results) { - transaction_id_type id = results.second->id(); + void net_plugin_impl::transaction_ack(const std::pair& results) { + const auto& id = results.second->id; if (results.first) { fc_ilog(logger,"signaled NACK, trx-id = ${id} : ${why}",("id", id)("why", results.first->to_detail_string())); dispatcher->rejected_transaction(id); } else { fc_ilog(logger,"signaled ACK, trx-id = ${id}",("id", id)); - dispatcher->bcast_transaction(*results.second); + dispatcher->bcast_transaction(results.second); } } @@ -2852,9 +2842,8 @@ namespace eosio { auto allowed_it = std::find(allowed_peers.begin(), allowed_peers.end(), msg.key); auto private_it = private_keys.find(msg.key); bool found_producer_key = false; - producer_plugin* pp = app().find_plugin(); - if(pp != nullptr) - found_producer_key = pp->is_producer_key(msg.key); + if(producer_plug != nullptr) + found_producer_key = producer_plug->is_producer_key(msg.key); if( allowed_it == allowed_peers.end() && private_it == private_keys.end() && !found_producer_key) { elog( "Peer ${peer} sent a handshake with an unauthorized key: ${key}.", ("peer", msg.p2p_address)("key", msg.key)); @@ -2914,9 +2903,8 @@ namespace eosio { auto private_key_itr = private_keys.find(signer); if(private_key_itr != private_keys.end()) return private_key_itr->second.sign(digest); - producer_plugin* pp = app().find_plugin(); - if(pp != nullptr && pp->get_state() == abstract_plugin::started) - return pp->sign_compact(signer, digest); + if(producer_plug != nullptr && producer_plug->get_state() == abstract_plugin::started) + return producer_plug->sign_compact(signer, digest); return chain::signature_type(); } @@ -2995,7 +2983,6 @@ namespace eosio { ( "network-version-match", bpo::value()->default_value(false), "True to require exact match of peer network version.") ( "sync-fetch-span", bpo::value()->default_value(def_sync_fetch_span), "number of blocks to retrieve in a chunk from any individual peer during synchronization") - ( "max-implicit-request", bpo::value()->default_value(def_max_just_send), "maximum sizes of transaction or block messages that are sent without first sending a notice") ( "use-socket-read-watermark", bpo::value()->default_value(false), "Enable expirimental socket read watermark optimization") ( "p2p-discoverable", bpo::value()->default_value(false), "True to p2p discoverable.") @@ -3031,7 +3018,6 @@ namespace eosio { my->max_cleanup_time_ms = options.at("max-cleanup-time-msec").as(); my->txn_exp_period = def_txn_expire_wait; my->resp_expected_period = def_resp_expected_wait; - my->dispatcher->just_send_it_max = options.at( "max-implicit-request" ).as(); my->max_client_count = options.at( "max-clients" ).as(); my->max_nodes_per_host = options.at( "p2p-max-nodes-per-host" ).as(); my->num_clients = 0; @@ -3042,7 +3028,7 @@ namespace eosio { my->p2p_discoverable=options.at( "p2p-discoverable" ).as(); my->resolver = std::make_shared( std::ref( app().get_io_service())); - if( options.count( "p2p-listen-endpoint" )) { + if( options.count( "p2p-listen-endpoint" ) && options.at("p2p-listen-endpoint").as().length()) { my->p2p_address = options.at( "p2p-listen-endpoint" ).as(); auto host = my->p2p_address.substr( 0, my->p2p_address.find( ':' )); auto port = my->p2p_address.substr( host.size() + 1, my->p2p_address.size()); @@ -3053,21 +3039,22 @@ namespace eosio { my->listen_endpoint = *my->resolver->resolve( query ); my->acceptor.reset( new tcp::acceptor( app().get_io_service())); - } - if( options.count( "p2p-server-address" )) { - my->p2p_address = options.at( "p2p-server-address" ).as(); - } else { - if( my->listen_endpoint.address().to_v4() == address_v4::any()) { - boost::system::error_code ec; - auto host = host_name( ec ); - if( ec.value() != boost::system::errc::success ) { - FC_THROW_EXCEPTION( fc::invalid_arg_exception, - "Unable to retrieve host_name. ${msg}", ("msg", ec.message())); + if( options.count( "p2p-server-address" )) { + my->p2p_address = options.at( "p2p-server-address" ).as(); + } else { + if( my->listen_endpoint.address().to_v4() == address_v4::any()) { + boost::system::error_code ec; + auto host = host_name( ec ); + if( ec.value() != boost::system::errc::success ) { + + FC_THROW_EXCEPTION( fc::invalid_arg_exception, + "Unable to retrieve host_name. ${msg}", ("msg", ec.message())); + } + auto port = my->p2p_address.substr( my->p2p_address.find( ':' ), my->p2p_address.size()); + my->p2p_address = host + port; } - auto port = my->p2p_address.substr( my->p2p_address.find( ':' ), my->p2p_address.size()); - my->p2p_address = host + port; } } @@ -3115,7 +3102,7 @@ namespace eosio { my->chain_plug = app().find_plugin(); EOS_ASSERT( my->chain_plug, chain::missing_chain_plugin_exception, "" ); - my->chain_id = app().get_plugin().get_chain_id(); + my->chain_id = my->chain_plug->get_chain_id(); fc::rand_pseudo_bytes( my->node_id.data(), my->node_id.data_size()); ilog( "my node_id is ${id}", ("id", my->node_id)); @@ -3125,6 +3112,7 @@ namespace eosio { } void net_plugin::plugin_startup() { + my->producer_plug = app().find_plugin(); if( my->acceptor ) { my->acceptor->open(my->listen_endpoint.protocol()); my->acceptor->set_option(tcp::acceptor::reuse_address(true)); @@ -3141,12 +3129,7 @@ namespace eosio { } chain::controller&cc = my->chain_plug->chain(); { - cc.accepted_block_header.connect( boost::bind(&net_plugin_impl::accepted_block_header, my.get(), _1)); cc.accepted_block.connect( boost::bind(&net_plugin_impl::accepted_block, my.get(), _1)); - cc.irreversible_block.connect( boost::bind(&net_plugin_impl::irreversible_block, my.get(), _1)); - cc.accepted_transaction.connect( boost::bind(&net_plugin_impl::accepted_transaction, my.get(), _1)); - cc.applied_transaction.connect( boost::bind(&net_plugin_impl::applied_transaction, my.get(), _1)); - cc.accepted_confirmation.connect( boost::bind(&net_plugin_impl::accepted_confirmation, my.get(), _1)); } my->incoming_transaction_ack_subscription = app().get_channel().subscribe(boost::bind(&net_plugin_impl::transaction_ack, my.get(), _1)); @@ -3241,13 +3224,13 @@ namespace eosio { } return result; } - connection_ptr net_plugin_impl::find_connection( string host )const { + connection_ptr net_plugin_impl::find_connection(const string& host )const { for( const auto& c : connections ) if( c->peer_addr == host ) return c; return connection_ptr(); } - uint16_t net_plugin_impl::to_protocol_version (uint16_t v) { + uint16_t net_plugin_impl::to_protocol_version(uint16_t v) { if (v >= net_version_base) { v -= net_version_base; return (v > net_version_range) ? 0 : v; diff --git a/plugins/producer_api_plugin/include/eosio/producer_api_plugin/producer_api_plugin.hpp b/plugins/producer_api_plugin/include/eosio/producer_api_plugin/producer_api_plugin.hpp index 4e7a5925f8a..137ef7b0703 100644 --- a/plugins/producer_api_plugin/include/eosio/producer_api_plugin/producer_api_plugin.hpp +++ b/plugins/producer_api_plugin/include/eosio/producer_api_plugin/producer_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp index 13599d5f834..7fcde1ac98c 100644 --- a/plugins/producer_api_plugin/producer_api_plugin.cpp +++ b/plugins/producer_api_plugin/producer_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp index 7b967421502..c43f0e0f38b 100644 --- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp +++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index bc29cd87ea3..7bcfb7042b7 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -1,11 +1,12 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include #include #include +#include #include #include @@ -131,6 +132,7 @@ class producer_plugin_impl : public std::enable_shared_from_this _producer_watermarks; pending_block_mode _pending_block_mode; transaction_id_with_expiry_index _persistent_transactions; + fc::optional _thread_pool; int32_t _max_transaction_time_ms; fc::microseconds _max_irreversible_block_age_us; @@ -145,6 +147,7 @@ class producer_plugin_impl : public std::enable_shared_from_this().chain(); + chain::controller& chain = chain_plug->chain(); const auto hbn = bsp->block_num; auto new_block_header = bsp->header; new_block_header.timestamp = new_block_header.timestamp.next(); @@ -293,7 +296,7 @@ class producer_plugin_impl : public std::enable_shared_from_thistimestamp < (fc::time_point::now() + fc::seconds( 7 )), block_from_the_future, "received a block from the future, ignoring it: ${id}", ("id", id) ); - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = chain_plug->chain(); /* de-dupe here... no point in aborting block if we already know the block */ auto existing = chain.fetch_block_by_id( id ); @@ -315,7 +318,7 @@ class producer_plugin_impl : public std::enable_shared_from_this().handle_guard_exception(e); + chain_plug->handle_guard_exception(e); return; } catch( const fc::exception& e ) { elog((e.to_detail_string())); @@ -343,10 +346,23 @@ class producer_plugin_impl : public std::enable_shared_from_this>> _pending_incoming_transactions; + std::deque>> _pending_incoming_transactions; + + void on_incoming_transaction_async(const transaction_metadata_ptr& trx, bool persist_until_expired, next_function next) { + chain::controller& chain = chain_plug->chain(); + const auto& cfg = chain.get_global_properties().configuration; + transaction_metadata::create_signing_keys_future( trx, *_thread_pool, chain.get_chain_id(), fc::microseconds( cfg.max_transaction_cpu_usage ) ); + boost::asio::post( *_thread_pool, [self = this, trx, persist_until_expired, next]() { + if( trx->signing_keys_future.valid() ) + trx->signing_keys_future.wait(); + app().get_io_service().post( [self, trx, persist_until_expired, next]() { + self->process_incoming_transaction_async( trx, persist_until_expired, next ); + }); + }); + } - void on_incoming_transaction_async(const packed_transaction_ptr& trx, bool persist_until_expired, next_function next) { - chain::controller& chain = app().get_plugin().chain(); + void process_incoming_transaction_async(const transaction_metadata_ptr& trx, bool persist_until_expired, next_function next) { + chain::controller& chain = chain_plug->chain(); if (!chain.pending_block_state()) { _pending_incoming_transactions.emplace_back(trx, persist_until_expired, next); return; @@ -357,34 +373,34 @@ class producer_plugin_impl : public std::enable_shared_from_this& response) { next(response); if (response.contains()) { - _transaction_ack_channel.publish(std::pair(response.get(), trx)); + _transaction_ack_channel.publish(std::pair(response.get(), trx)); if (_pending_block_mode == pending_block_mode::producing) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is REJECTING tx: ${txid} : ${why} ", ("block_num", chain.head_block_num() + 1) ("prod", chain.pending_block_state()->header.producer) - ("txid", trx->id()) + ("txid", trx->id) ("why",response.get()->what())); } else { fc_dlog(_trx_trace_log, "[TRX_TRACE] Speculative execution is REJECTING tx: ${txid} : ${why} ", - ("txid", trx->id()) + ("txid", trx->id) ("why",response.get()->what())); } } else { - _transaction_ack_channel.publish(std::pair(nullptr, trx)); + _transaction_ack_channel.publish(std::pair(nullptr, trx)); if (_pending_block_mode == pending_block_mode::producing) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is ACCEPTING tx: ${txid}", ("block_num", chain.head_block_num() + 1) ("prod", chain.pending_block_state()->header.producer) - ("txid", trx->id())); + ("txid", trx->id)); } else { fc_dlog(_trx_trace_log, "[TRX_TRACE] Speculative execution is ACCEPTING tx: ${txid}", - ("txid", trx->id())); + ("txid", trx->id)); } } }; - auto id = trx->id(); - if( fc::time_point(trx->expiration()) < block_time ) { + const auto& id = trx->id; + if( fc::time_point(trx->packed_trx->expiration()) < block_time ) { send_response(std::static_pointer_cast(std::make_shared(FC_LOG_MESSAGE(error, "expired transaction ${id}", ("id", id)) ))); return; } @@ -403,7 +419,7 @@ class producer_plugin_impl : public std::enable_shared_from_this(*trx), deadline); + auto trace = chain.push_transaction(trx, deadline); if (trace->except) { if (failure_is_subjective(*trace->except, deadline_is_subjective)) { _pending_incoming_transactions.emplace_back(trx, persist_until_expired, next); @@ -411,10 +427,10 @@ class producer_plugin_impl : public std::enable_shared_from_thisheader.producer) - ("txid", trx->id())); + ("txid", trx->id)); } else { fc_dlog(_trx_trace_log, "[TRX_TRACE] Speculative execution COULD NOT FIT tx: ${txid} RETRYING", - ("txid", trx->id())); + ("txid", trx->id)); } } else { auto e_ptr = trace->except->dynamic_copy_exception(); @@ -424,13 +440,13 @@ class producer_plugin_impl : public std::enable_shared_from_thisid(), trx->expiration()}); + _persistent_transactions.insert(transaction_id_with_expiry{trx->id, trx->packed_trx->expiration()}); } send_response(trace); } } catch ( const guard_exception& e ) { - app().get_plugin().handle_guard_exception(e); + chain_plug->handle_guard_exception(e); } catch ( boost::interprocess::bad_alloc& ) { chain_plugin::handle_db_exhaustion(); } CATCH_AND_CALL(send_response); @@ -533,6 +549,8 @@ void producer_plugin::set_program_options( "Maximum wall-clock time, in milliseconds, spent retiring scheduled transactions in any block before returning to normal transaction processing.") ("incoming-defer-ratio", bpo::value()->default_value(1.0), "ratio between incoming transations and deferred transactions when both are exhausted") + ("producer-threads", bpo::value()->default_value(config::default_controller_thread_pool_size), + "Number of worker threads in producer thread pool") ("snapshots-dir", bpo::value()->default_value("snapshots"), "the location of the snapshots directory (absolute path or relative to application data dir)") ; @@ -604,6 +622,8 @@ make_keosd_signature_provider(const std::shared_ptr& impl, void producer_plugin::plugin_initialize(const boost::program_options::variables_map& options) { try { + my->chain_plug = app().find_plugin(); + EOS_ASSERT( my->chain_plug, plugin_config_exception, "chain_plugin not found" ); my->_options = &options; LOAD_VALUE_SET(options, "producer-name", my->_producers, types::account_name) @@ -665,6 +685,11 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ my->_incoming_defer_ratio = options.at("incoming-defer-ratio").as(); + auto thread_pool_size = options.at( "producer-threads" ).as(); + EOS_ASSERT( thread_pool_size > 0, plugin_config_exception, + "producer-threads ${num} must be greater than 0", ("num", thread_pool_size)); + my->_thread_pool.emplace( thread_pool_size ); + if( options.count( "snapshots-dir" )) { auto sd = options.at( "snapshots-dir" ).as(); if( sd.is_relative()) { @@ -686,7 +711,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ } FC_LOG_AND_DROP(); }); - my->_incoming_transaction_subscription = app().get_channel().subscribe([this](const packed_transaction_ptr& trx){ + my->_incoming_transaction_subscription = app().get_channel().subscribe([this](const transaction_metadata_ptr& trx){ try { my->on_incoming_transaction_async(trx, false, [](const auto&){}); } FC_LOG_AND_DROP(); @@ -696,7 +721,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_ my->on_incoming_block(block); }); - my->_incoming_transaction_async_provider = app().get_method().register_provider([this](const packed_transaction_ptr& trx, bool persist_until_expired, next_function next) -> void { + my->_incoming_transaction_async_provider = app().get_method().register_provider([this](const transaction_metadata_ptr& trx, bool persist_until_expired, next_function next) -> void { return my->on_incoming_transaction_async(trx, persist_until_expired, next ); }); @@ -724,7 +749,7 @@ void producer_plugin::plugin_startup() ilog("producer plugin: plugin_startup() begin"); - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); EOS_ASSERT( my->_producers.empty() || chain.get_read_mode() == chain::db_read_mode::SPECULATIVE, plugin_config_exception, "node cannot have any producer-name configured because block production is impossible when read_mode is not \"speculative\"" ); @@ -766,6 +791,10 @@ void producer_plugin::plugin_shutdown() { edump((e.to_detail_string())); } + if( my->_thread_pool ) { + my->_thread_pool->join(); + my->_thread_pool->stop(); + } my->_accepted_block_connection.reset(); my->_irreversible_block_connection.reset(); } @@ -780,7 +809,7 @@ void producer_plugin::resume() { // re-evaluate that now // if (my->_pending_block_mode == pending_block_mode::speculating) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); chain.abort_block(); my->schedule_production_loop(); } @@ -819,13 +848,13 @@ void producer_plugin::update_runtime_options(const runtime_options& options) { } if (check_speculating && my->_pending_block_mode == pending_block_mode::speculating) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); chain.abort_block(); my->schedule_production_loop(); } if (options.subjective_cpu_leeway_us) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); chain.set_subjective_cpu_leeway(fc::microseconds(*options.subjective_cpu_leeway_us)); } } @@ -841,21 +870,21 @@ producer_plugin::runtime_options producer_plugin::get_runtime_options() const { } void producer_plugin::add_greylist_accounts(const greylist_params& params) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); for (auto &acc : params.accounts) { chain.add_resource_greylist(acc); } } void producer_plugin::remove_greylist_accounts(const greylist_params& params) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); for (auto &acc : params.accounts) { chain.remove_resource_greylist(acc); } } producer_plugin::greylist_params producer_plugin::get_greylist() const { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); greylist_params result; const auto& list = chain.get_resource_greylist(); result.accounts.reserve(list.size()); @@ -866,7 +895,7 @@ producer_plugin::greylist_params producer_plugin::get_greylist() const { } producer_plugin::whitelist_blacklist producer_plugin::get_whitelist_blacklist() const { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); return { chain.get_actor_whitelist(), chain.get_actor_blacklist(), @@ -878,7 +907,7 @@ producer_plugin::whitelist_blacklist producer_plugin::get_whitelist_blacklist() } void producer_plugin::set_whitelist_blacklist(const producer_plugin::whitelist_blacklist& params) { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); if(params.actor_whitelist.valid()) chain.set_actor_whitelist(*params.actor_whitelist); if(params.actor_blacklist.valid()) chain.set_actor_blacklist(*params.actor_blacklist); if(params.contract_whitelist.valid()) chain.set_contract_whitelist(*params.contract_whitelist); @@ -888,7 +917,7 @@ void producer_plugin::set_whitelist_blacklist(const producer_plugin::whitelist_b } producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash() const { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); auto reschedule = fc::make_scoped_exit([this](){ my->schedule_production_loop(); @@ -905,7 +934,7 @@ producer_plugin::integrity_hash_information producer_plugin::get_integrity_hash( } producer_plugin::snapshot_information producer_plugin::create_snapshot() const { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = my->chain_plug->chain(); auto reschedule = fc::make_scoped_exit([this](){ my->schedule_production_loop(); @@ -936,7 +965,7 @@ producer_plugin::snapshot_information producer_plugin::create_snapshot() const { } optional producer_plugin_impl::calculate_next_block_time(const account_name& producer_name, const block_timestamp_type& current_block_time) const { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = chain_plug->chain(); const auto& hbs = chain.head_block_state(); const auto& active_schedule = hbs->active_schedule.producers; @@ -992,7 +1021,7 @@ optional producer_plugin_impl::calculate_next_block_time(const a } fc::time_point producer_plugin_impl::calculate_pending_block_time() const { - const chain::controller& chain = app().get_plugin().chain(); + const chain::controller& chain = chain_plug->chain(); const fc::time_point now = fc::time_point::now(); const fc::time_point base = std::max(now, chain.head_block_time()); const int64_t min_time_to_next_block = (config::block_interval_us) - (base.time_since_epoch().count() % (config::block_interval_us) ); @@ -1018,7 +1047,7 @@ enum class tx_category { producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = chain_plug->chain(); if( chain.get_read_mode() == chain::db_read_mode::READ_ONLY ) return start_block_result::waiting; @@ -1120,6 +1149,10 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { int orig_count = _persistent_transactions.size(); while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pbs->header.timestamp.to_time_point()) { + if (preprocess_deadline <= fc::time_point::now()) { + exhausted = true; + break; + } auto const& txid = persisted_by_expiry.begin()->trx_id; if (_pending_block_mode == pending_block_mode::producing) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", @@ -1135,9 +1168,15 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { num_expired_persistent++; } - fc_dlog(_log, "Processed ${n} persisted transactions, Expired ${expired}", - ("n", orig_count) - ("expired", num_expired_persistent)); + if( exhausted ) { + fc_wlog( _log, "Unable to process all ${n} persisted transactions before deadline, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } else { + fc_dlog( _log, "Processed ${n} persisted transactions, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } } try { @@ -1148,15 +1187,17 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if (_producers.empty() && persisted_by_id.empty()) { // if this node can never produce and has no persisted transactions, // there is no need for unapplied transactions they can be dropped - chain.drop_all_unapplied_transactions(); + chain.get_unapplied_transactions().clear(); } else { - std::vector apply_trxs; - { // derive appliable transactions from unapplied_transactions and drop droppable transactions - auto unapplied_trxs = chain.get_unapplied_transactions(); - apply_trxs.reserve(unapplied_trxs.size()); - + // derive appliable transactions from unapplied_transactions and drop droppable transactions + unapplied_transactions_type& unapplied_trxs = chain.get_unapplied_transactions(); + if( !unapplied_trxs.empty() ) { + auto unapplied_trxs_size = unapplied_trxs.size(); + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; auto calculate_transaction_category = [&](const transaction_metadata_ptr& trx) { - if (trx->packed_trx.expiration() < pbs->header.timestamp.to_time_point()) { + if (trx->packed_trx->expiration() < pbs->header.timestamp.to_time_point()) { return tx_category::EXPIRED; } else if (persisted_by_id.find(trx->id) != persisted_by_id.end()) { return tx_category::PERSISTED; @@ -1165,64 +1206,66 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } }; - for (auto& trx: unapplied_trxs) { + auto itr = unapplied_trxs.begin(); + while( itr != unapplied_trxs.end() ) { + auto itr_next = itr; // save off next since itr may be invalidated by loop + ++itr_next; + + if( preprocess_deadline <= fc::time_point::now() ) exhausted = true; + if( exhausted ) break; + const auto& trx = itr->second; auto category = calculate_transaction_category(trx); - if (category == tx_category::EXPIRED || (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) { + if (category == tx_category::EXPIRED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) + { if (!_producers.empty()) { fc_dlog(_trx_trace_log, "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", ("txid", trx->id)); } - chain.drop_unapplied_transaction(trx); - } else if (category == tx_category::PERSISTED || (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) { - apply_trxs.emplace_back(std::move(trx)); - } - } - } - if (!apply_trxs.empty()) { - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; - - for (const auto& trx: apply_trxs) { - if (preprocess_deadline <= fc::time_point::now()) exhausted = true; - if (exhausted) { - break; - } - - num_processed++; - - try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); - bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && preprocess_deadline < deadline)) { - deadline_is_subjective = true; - deadline = preprocess_deadline; - } + itr = unapplied_trxs.erase( itr ); // unapplied_trxs map has not been modified, so simply erase and continue + continue; + } else if (category == tx_category::PERSISTED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) + { + ++num_processed; + + try { + auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + bool deadline_is_subjective = false; + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && preprocess_deadline < deadline)) { + deadline_is_subjective = true; + deadline = preprocess_deadline; + } - auto trace = chain.push_transaction(trx, deadline); - if (trace->except) { - if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; + auto trace = chain.push_transaction(trx, deadline); + if (trace->except) { + if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; + break; + } else { + // this failed our configured maximum transaction time, we don't want to replay it + // chain.plus_transactions can modify unapplied_trxs, so erase by id + unapplied_trxs.erase( trx->signed_id ); + ++num_failed; + } } else { - // this failed our configured maximum transaction time, we don't want to replay it - chain.drop_unapplied_transaction(trx); - num_failed++; + ++num_applied; } - } else { - num_applied++; - } - } catch ( const guard_exception& e ) { - app().get_plugin().handle_guard_exception(e); - return start_block_result::failed; - } FC_LOG_AND_DROP(); + } catch ( const guard_exception& e ) { + chain_plug->handle_guard_exception(e); + return start_block_result::failed; + } FC_LOG_AND_DROP(); + } + + itr = itr_next; } fc_dlog(_log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", - ("m", num_processed) - ("n", apply_trxs.size()) - ("applied", num_applied) - ("failed", num_failed)); + ("m", num_processed) + ("n", unapplied_trxs_size) + ("applied", num_applied) + ("failed", num_failed)); } } @@ -1235,6 +1278,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { int orig_count = _blacklisted_transactions.size(); while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= now) { + if (preprocess_deadline <= fc::time_point::now()) break; blacklist_by_expiry.erase(blacklist_by_expiry.begin()); num_expired++; } @@ -1244,85 +1288,105 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { ("expired", num_expired)); } - auto scheduled_trxs = chain.get_scheduled_transactions(); - if (!scheduled_trxs.empty()) { - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; + // scheduled transactions + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; + + auto scheduled_trx_deadline = preprocess_deadline; + if (_max_scheduled_transaction_time_per_block_ms >= 0) { + scheduled_trx_deadline = std::min( + scheduled_trx_deadline, + fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) + ); + } + time_point pending_block_time = chain.pending_block_time(); + const auto& sch_idx = chain.db().get_index(); + const auto scheduled_trxs_size = sch_idx.size(); + auto sch_itr = sch_idx.begin(); + while( sch_itr != sch_idx.end() ) { + if( sch_itr->delay_until > pending_block_time) break; // not scheduled yet + if( sch_itr->published >= pending_block_time ) { + ++sch_itr; + continue; // do not allow schedule and execute in same block + } + if( scheduled_trx_deadline <= fc::time_point::now() ) { + exhausted = true; + break; + } - auto scheduled_trx_deadline = preprocess_deadline; - if (_max_scheduled_transaction_time_per_block_ms >= 0) { - scheduled_trx_deadline = std::min( - scheduled_trx_deadline, - fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) - ); + const transaction_id_type trx_id = sch_itr->trx_id; // make copy since reference could be invalidated + if (blacklist_by_id.find(trx_id) != blacklist_by_id.end()) { + ++sch_itr; + continue; } - for (const auto& trx : scheduled_trxs) { - if (scheduled_trx_deadline <= fc::time_point::now()) exhausted = true; - if (exhausted) { - break; - } + auto sch_itr_next = sch_itr; // save off next since sch_itr may be invalidated by loop + ++sch_itr_next; + const auto next_delay_until = sch_itr_next != sch_idx.end() ? sch_itr_next->delay_until : sch_itr->delay_until; + const auto next_id = sch_itr_next != sch_idx.end() ? sch_itr_next->id : sch_itr->id; - num_processed++; - - // configurable ratio of incoming txns vs deferred txns - while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { - if (scheduled_trx_deadline <= fc::time_point::now()) break; - - auto e = _pending_incoming_transactions.front(); - _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; - _incoming_trx_weight -= 1.0; - on_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - } + num_processed++; - if (scheduled_trx_deadline <= fc::time_point::now()) { - exhausted = true; - break; - } + // configurable ratio of incoming txns vs deferred txns + while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { + if (scheduled_trx_deadline <= fc::time_point::now()) break; - if (blacklist_by_id.find(trx) != blacklist_by_id.end()) { - continue; - } + auto e = _pending_incoming_transactions.front(); + _pending_incoming_transactions.pop_front(); + --orig_pending_txn_size; + _incoming_trx_weight -= 1.0; + process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); + } - try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); - bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { - deadline_is_subjective = true; - deadline = scheduled_trx_deadline; - } + if (scheduled_trx_deadline <= fc::time_point::now()) { + exhausted = true; + break; + } - auto trace = chain.push_scheduled_transaction(trx, deadline); - if (trace->except) { - if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; - } else { - auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); - // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist - _blacklisted_transactions.insert(transaction_id_with_expiry{trx, expiration}); - num_failed++; - } + try { + auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + bool deadline_is_subjective = false; + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { + deadline_is_subjective = true; + deadline = scheduled_trx_deadline; + } + + auto trace = chain.push_scheduled_transaction(trx_id, deadline); + if (trace->except) { + if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; + break; } else { - num_applied++; + auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); + // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist + _blacklisted_transactions.insert(transaction_id_with_expiry{trx_id, expiration}); + num_failed++; } - } catch ( const guard_exception& e ) { - app().get_plugin().handle_guard_exception(e); - return start_block_result::failed; - } FC_LOG_AND_DROP(); + } else { + num_applied++; + } + } catch ( const guard_exception& e ) { + chain_plug->handle_guard_exception(e); + return start_block_result::failed; + } FC_LOG_AND_DROP(); - _incoming_trx_weight += _incoming_defer_ratio; - if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; - } + _incoming_trx_weight += _incoming_defer_ratio; + if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; - fc_dlog(_log, "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", - ("m", num_processed) - ("n", scheduled_trxs.size()) - ("applied", num_applied) - ("failed", num_failed)); + if( sch_itr_next == sch_idx.end() ) break; + sch_itr = sch_idx.lower_bound( boost::make_tuple( next_delay_until, next_id ) ); + } + if( scheduled_trxs_size > 0 ) { + fc_dlog( _log, + "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", + ( "m", num_processed ) + ( "n", scheduled_trxs_size ) + ( "applied", num_applied ) + ( "failed", num_failed ) ); } + } if (exhausted || preprocess_deadline <= fc::time_point::now()) { @@ -1334,11 +1398,11 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if (!_pending_incoming_transactions.empty()) { fc_dlog(_log, "Processing ${n} pending transactions"); while (orig_pending_txn_size && _pending_incoming_transactions.size()) { + if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; auto e = _pending_incoming_transactions.front(); _pending_incoming_transactions.pop_front(); --orig_pending_txn_size; - on_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; + process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); } } return start_block_result::succeeded; @@ -1355,7 +1419,7 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { } void producer_plugin_impl::schedule_production_loop() { - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = chain_plug->chain(); _timer.cancel(); std::weak_ptr weak_this = shared_from_this(); @@ -1466,7 +1530,7 @@ bool producer_plugin_impl::maybe_produce_block() { produce_block(); return true; } catch ( const guard_exception& e ) { - app().get_plugin().handle_guard_exception(e); + chain_plug->handle_guard_exception(e); return false; } FC_LOG_AND_DROP(); } catch ( boost::interprocess::bad_alloc&) { @@ -1475,7 +1539,7 @@ bool producer_plugin_impl::maybe_produce_block() { } fc_dlog(_log, "Aborting block due to produce_block error"); - chain::controller& chain = app().get_plugin().chain(); + chain::controller& chain = chain_plug->chain(); chain.abort_block(); return false; } @@ -1498,7 +1562,7 @@ static auto maybe_make_debug_time_logger() -> fc::optional().chain(); + chain::controller& chain = chain_plug->chain(); const auto& pbs = chain.pending_block_state(); const auto& hbs = chain.head_block_state(); EOS_ASSERT(pbs, missing_pending_block_state, "pending_block_state does not exist but it should, another plugin may have corrupted it"); diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp index a580529c5d5..1644d29a404 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_log.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp index ce532d1c89e..f3429e2d190 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp index b000ac78803..37c817dd9cc 100644 --- a/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp +++ b/plugins/state_history_plugin/include/eosio/state_history_plugin/state_history_serialization.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index e625af5052a..67b61587440 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/plugins/template_plugin/include/eosio/template_plugin/template_plugin.hpp b/plugins/template_plugin/include/eosio/template_plugin/template_plugin.hpp index 52b88b12ea0..3217b3b1640 100644 --- a/plugins/template_plugin/include/eosio/template_plugin/template_plugin.hpp +++ b/plugins/template_plugin/include/eosio/template_plugin/template_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/template_plugin/template_plugin.cpp b/plugins/template_plugin/template_plugin.cpp index 74dc6d7a508..d8a738c30bc 100644 --- a/plugins/template_plugin/template_plugin.cpp +++ b/plugins/template_plugin/template_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/plugins/test_control_api_plugin/include/eosio/test_control_api_plugin/test_control_api_plugin.hpp b/plugins/test_control_api_plugin/include/eosio/test_control_api_plugin/test_control_api_plugin.hpp index feac39a95ff..b9c165b8993 100644 --- a/plugins/test_control_api_plugin/include/eosio/test_control_api_plugin/test_control_api_plugin.hpp +++ b/plugins/test_control_api_plugin/include/eosio/test_control_api_plugin/test_control_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/test_control_api_plugin/test_control_api_plugin.cpp b/plugins/test_control_api_plugin/test_control_api_plugin.cpp index 16510b06460..307cccc197e 100644 --- a/plugins/test_control_api_plugin/test_control_api_plugin.cpp +++ b/plugins/test_control_api_plugin/test_control_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/test_control_plugin/include/eosio/test_control_plugin/test_control_plugin.hpp b/plugins/test_control_plugin/include/eosio/test_control_plugin/test_control_plugin.hpp index 0a40d9b6e36..96badb10c83 100644 --- a/plugins/test_control_plugin/include/eosio/test_control_plugin/test_control_plugin.hpp +++ b/plugins/test_control_plugin/include/eosio/test_control_plugin/test_control_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/test_control_plugin/test_control_plugin.cpp b/plugins/test_control_plugin/test_control_plugin.cpp index f2d630c6c59..483e859c30f 100644 --- a/plugins/test_control_plugin/test_control_plugin.cpp +++ b/plugins/test_control_plugin/test_control_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/txn_test_gen_plugin/include/eosio/txn_test_gen_plugin/txn_test_gen_plugin.hpp b/plugins/txn_test_gen_plugin/include/eosio/txn_test_gen_plugin/txn_test_gen_plugin.hpp index b33e4835158..ebfe3bfb77c 100644 --- a/plugins/txn_test_gen_plugin/include/eosio/txn_test_gen_plugin/txn_test_gen_plugin.hpp +++ b/plugins/txn_test_gen_plugin/include/eosio/txn_test_gen_plugin/txn_test_gen_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp index 6b4753e9919..707f75bd9de 100644 --- a/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp +++ b/plugins/txn_test_gen_plugin/txn_test_gen_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -87,19 +87,38 @@ using namespace eosio::chain; api_handle->call_name(vs.at(0).as(), vs.at(1).as(), result_handler); struct txn_test_gen_plugin_impl { - static void push_next_transaction(const std::shared_ptr>& trxs, size_t index, const std::function& next ) { + + uint64_t _total_us = 0; + uint64_t _txcount = 0; + + int _remain = 0; + + void push_next_transaction(const std::shared_ptr>& trxs, size_t index, const std::function& next ) { chain_plugin& cp = app().get_plugin(); - cp.accept_transaction( packed_transaction(trxs->at(index)), [=](const fc::static_variant& result){ - if (result.contains()) { - next(result.get()); - } else { - if (index + 1 < trxs->size()) { - push_next_transaction(trxs, index + 1, next); + + const int overlap = 20; + int end = std::min(index + overlap, trxs->size()); + _remain = end - index; + for (int i = index; i < end; ++i) { + cp.accept_transaction( packed_transaction(trxs->at(i)), [=](const fc::static_variant& result){ + if (result.contains()) { + next(result.get()); } else { - next(nullptr); + if (result.contains() && result.get()->receipt) { + _total_us += result.get()->receipt->cpu_usage_us; + ++_txcount; + } + --_remain; + if (_remain == 0 ) { + if (end < trxs->size()) { + push_next_transaction(trxs, index + overlap, next); + } else { + next(nullptr); + } + } } - } - }); + }); + } } void push_transactions( std::vector&& trxs, const std::function& next ) { @@ -295,13 +314,11 @@ struct txn_test_gen_plugin_impl { try { controller& cc = app().get_plugin().chain(); auto chainid = app().get_plugin().get_chain_id(); - auto abi_serializer_max_time = app().get_plugin().get_abi_serializer_max_time(); - fc::crypto::private_key a_priv_key = fc::crypto::private_key::regenerate(fc::sha256(std::string(64, 'a'))); - fc::crypto::private_key b_priv_key = fc::crypto::private_key::regenerate(fc::sha256(std::string(64, 'b'))); + static fc::crypto::private_key a_priv_key = fc::crypto::private_key::regenerate(fc::sha256(std::string(64, 'a'))); + static fc::crypto::private_key b_priv_key = fc::crypto::private_key::regenerate(fc::sha256(std::string(64, 'b'))); static uint64_t nonce = static_cast(fc::time_point::now().sec_since_epoch()) << 32; - abi_serializer eosio_serializer(cc.db().find(config::system_account_name)->get_abi(), abi_serializer_max_time); uint32_t reference_block_num = cc.last_irreversible_block_num(); if (txn_reference_block_lag >= 0) { @@ -351,6 +368,11 @@ struct txn_test_gen_plugin_impl { timer.cancel(); running = false; ilog("Stopping transaction generation test"); + + if (_txcount) { + ilog("${d} transactions executed, ${t}us / transaction", ("d", _txcount)("t", _total_us / (double)_txcount)); + _txcount = _total_us = 0; + } } boost::asio::high_resolution_timer timer{app().get_io_service()}; diff --git a/plugins/wallet_api_plugin/include/eosio/wallet_api_plugin/wallet_api_plugin.hpp b/plugins/wallet_api_plugin/include/eosio/wallet_api_plugin/wallet_api_plugin.hpp index 74fc2878943..611eed638d9 100644 --- a/plugins/wallet_api_plugin/include/eosio/wallet_api_plugin/wallet_api_plugin.hpp +++ b/plugins/wallet_api_plugin/include/eosio/wallet_api_plugin/wallet_api_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/wallet_api_plugin/wallet_api_plugin.cpp b/plugins/wallet_api_plugin/wallet_api_plugin.cpp index 315ce810b16..0d88ec2966d 100644 --- a/plugins/wallet_api_plugin/wallet_api_plugin.cpp +++ b/plugins/wallet_api_plugin/wallet_api_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp index e8b7c91a7e3..480e7a32a44 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp index 0c80be0d880..0627eceff33 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_api.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_manager.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_manager.hpp index efc10f1a7f9..e73bc32d971 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_manager.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_manager.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_plugin.hpp b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_plugin.hpp index 1b0986fe8f9..c23911fa6fb 100644 --- a/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_plugin.hpp +++ b/plugins/wallet_plugin/include/eosio/wallet_plugin/wallet_plugin.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/plugins/wallet_plugin/se_wallet.cpp b/plugins/wallet_plugin/se_wallet.cpp index b292495c5cc..8b43d569881 100644 --- a/plugins/wallet_plugin/se_wallet.cpp +++ b/plugins/wallet_plugin/se_wallet.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/wallet_plugin/wallet.cpp b/plugins/wallet_plugin/wallet.cpp index accb2f948b9..53d57697ccd 100644 --- a/plugins/wallet_plugin/wallet.cpp +++ b/plugins/wallet_plugin/wallet.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/plugins/wallet_plugin/wallet_manager.cpp b/plugins/wallet_plugin/wallet_manager.cpp index d851d8156f6..15a39c9d9bd 100644 --- a/plugins/wallet_plugin/wallet_manager.cpp +++ b/plugins/wallet_plugin/wallet_manager.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/wallet_plugin/wallet_plugin.cpp b/plugins/wallet_plugin/wallet_plugin.cpp index 0416227b3a4..cfd435390a6 100644 --- a/plugins/wallet_plugin/wallet_plugin.cpp +++ b/plugins/wallet_plugin/wallet_plugin.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/plugins/wallet_plugin/yubihsm_wallet.cpp b/plugins/wallet_plugin/yubihsm_wallet.cpp index d3856ad8ee5..682fc02bbb0 100644 --- a/plugins/wallet_plugin/yubihsm_wallet.cpp +++ b/plugins/wallet_plugin/yubihsm_wallet.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/programs/cleos/config.hpp.in b/programs/cleos/config.hpp.in index 18673e9c8c2..3fe2051d74c 100644 --- a/programs/cleos/config.hpp.in +++ b/programs/cleos/config.hpp.in @@ -1,5 +1,5 @@ /** @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See config.hpp.in for changes. */ diff --git a/programs/cleos/help_text.cpp b/programs/cleos/help_text.cpp index 84173e0dda8..b1f9161a54d 100644 --- a/programs/cleos/help_text.cpp +++ b/programs/cleos/help_text.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include "help_text.hpp" #include "localize.hpp" diff --git a/programs/cleos/help_text.hpp b/programs/cleos/help_text.hpp index cb0f21a828e..dbae493d714 100644 --- a/programs/cleos/help_text.hpp +++ b/programs/cleos/help_text.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once #include diff --git a/programs/cleos/httpc.hpp b/programs/cleos/httpc.hpp index 850b7bab787..a5db5dbe90f 100644 --- a/programs/cleos/httpc.hpp +++ b/programs/cleos/httpc.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/programs/cleos/localize.hpp b/programs/cleos/localize.hpp index 72cd073e813..ea15ea3009b 100644 --- a/programs/cleos/localize.hpp +++ b/programs/cleos/localize.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index ddcb8c11728..bc9373b84a1 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * @defgroup eosclienttool EOSIO Command Line Client Reference * @brief Tool for sending transactions and querying state from @ref nodeos * @ingroup eosclienttool diff --git a/programs/eosio-launcher/config.hpp.in b/programs/eosio-launcher/config.hpp.in index 1bb4d10b59a..f60e6ab19e5 100644 --- a/programs/eosio-launcher/config.hpp.in +++ b/programs/eosio-launcher/config.hpp.in @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See config.hpp.in for changes. */ diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index 597e473b677..55a027b6928 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * @brief launch testnet nodes **/ #include diff --git a/programs/keosd/config.hpp.in b/programs/keosd/config.hpp.in index a7d34f390dc..370f88db1a9 100644 --- a/programs/keosd/config.hpp.in +++ b/programs/keosd/config.hpp.in @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See config.hpp.in for changes. */ diff --git a/programs/keosd/main.cpp b/programs/keosd/main.cpp index c457af3c0e3..efbf2d567f2 100644 --- a/programs/keosd/main.cpp +++ b/programs/keosd/main.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/programs/nodeos/config.hpp.in b/programs/nodeos/config.hpp.in index ac595383297..dbeba72a3d1 100644 --- a/programs/nodeos/config.hpp.in +++ b/programs/nodeos/config.hpp.in @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE * * \warning This file is machine generated. DO NOT EDIT. See config.hpp.in for changes. */ diff --git a/scripts/boost.rb b/scripts/boost.rb new file mode 100644 index 00000000000..a322acbdd11 --- /dev/null +++ b/scripts/boost.rb @@ -0,0 +1,131 @@ +class Boost < Formula + desc "Collection of portable C++ source libraries" + homepage "https://www.boost.org/" + revision 1 + head "https://github.com/boostorg/boost.git" + + stable do + url "https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2" + sha256 "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba" + + # Remove for > 1.67.0 + # Fix "error: no member named 'next' in namespace 'boost'" + # Upstream commit from 1 Dec 2017 "Add #include ; no + # longer in utility.hpp" + patch :p2 do + url "https://github.com/boostorg/lockfree/commit/12726cd.patch?full_index=1" + sha256 "f165823d961a588b622b20520668b08819eb5fdc08be7894c06edce78026ce0a" + end + end + + bottle do + cellar :any + sha256 "265ab8beaa6fa26a7c305ef2e6aec8bd26ca1db105aca0aaca028f32c5245a90" => :high_sierra + sha256 "567f3e9a294413c1701b698d666a521cfdeec846e256c6e66576d5b70eb26f08" => :sierra + sha256 "3f3f687a620f656fe2ac54f01306e00e6bbc0e9797db284a8d272648d427e640" => :el_capitan + end + + option "with-icu4c", "Build regexp engine with icu support" + option "without-single", "Disable building single-threading variant" + option "without-static", "Disable building static library variant" + + deprecated_option "with-icu" => "with-icu4c" + + depends_on "icu4c" => :optional + + def install + # Force boost to compile with the desired compiler + open("user-config.jam", "a") do |file| + file.write "using darwin : : #{ENV.cxx} ;\n" + end + + # libdir should be set by --prefix but isn't + bootstrap_args = ["--prefix=#{prefix}", "--libdir=#{lib}"] + + if build.with? "icu4c" + icu4c_prefix = Formula["icu4c"].opt_prefix + bootstrap_args << "--with-icu=#{icu4c_prefix}" + else + bootstrap_args << "--without-icu" + end + + # Handle libraries that will not be built. + without_libraries = ["python", "mpi"] + + # Boost.Log cannot be built using Apple GCC at the moment. Disabled + # on such systems. + without_libraries << "log" if ENV.compiler == :gcc + + bootstrap_args << "--without-libraries=#{without_libraries.join(",")}" + + # layout should be synchronized with boost-python and boost-mpi + args = ["--prefix=#{prefix}", + "--libdir=#{lib}", + "-d2", + "-j#{ENV.make_jobs}", + "--layout=tagged", + "--user-config=user-config.jam", + "-sNO_LZMA=1", + "install"] + + if build.with? "single" + args << "threading=multi,single" + else + args << "threading=multi" + end + + if build.with? "static" + args << "link=shared,static" + else + args << "link=shared" + end + + # Trunk starts using "clang++ -x c" to select C compiler which breaks C++11 + # handling using ENV.cxx11. Using "cxxflags" and "linkflags" still works. + args << "cxxflags=-std=c++11" + if ENV.compiler == :clang + args << "cxxflags=-stdlib=libc++" << "linkflags=-stdlib=libc++" + end + + system "./bootstrap.sh", *bootstrap_args + system "./b2", "headers" + system "./b2", *args + end + + def caveats + s = "" + # ENV.compiler doesn't exist in caveats. Check library availability + # instead. + if Dir["#{lib}/libboost_log*"].empty? + s += <<~EOS + Building of Boost.Log is disabled because it requires newer GCC or Clang. + EOS + end + + s + end + + test do + (testpath/"test.cpp").write <<~EOS + #include + #include + #include + #include + using namespace boost::algorithm; + using namespace std; + + int main() + { + string str("a,b"); + vector strVec; + split(strVec, str, is_any_of(",")); + assert(strVec.size()==2); + assert(strVec[0]=="a"); + assert(strVec[1]=="b"); + return 0; + } + EOS + system ENV.cxx, "test.cpp", "-std=c++1y", "-L#{lib}", "-lboost_system", "-o", "test" + system "./test" + end +end diff --git a/scripts/eosio_build_amazon.sh b/scripts/eosio_build_amazon.sh index 2dc523870a8..ea690973846 100644 --- a/scripts/eosio_build_amazon.sh +++ b/scripts/eosio_build_amazon.sh @@ -56,11 +56,11 @@ if [[ "${OS_NAME}" == "Amazon Linux AMI" ]]; then DEP_ARRAY=( git gcc72.x86_64 gcc72-c++.x86_64 autoconf automake libtool make bzip2 \ bzip2-devel.x86_64 openssl-devel.x86_64 gmp-devel.x86_64 libstdc++72.x86_64 \ - python27.x86_64 python36-devel.x86_64 libedit-devel.x86_64 doxygen.x86_64 graphviz.x86_64) + python27.x86_64 python27-devel.x86_64 python36-devel.x86_64 libedit-devel.x86_64 doxygen.x86_64 graphviz.x86_64) else - DEP_ARRAY=( git gcc.x86_64 gcc-c++.x86_64 autoconf automake libtool make bzip2 \ - bzip2-devel.x86_64 openssl-devel.x86_64 gmp-devel.x86_64 libstdc++.x86_64 \ - python3.x86_64 python3-devel.x86_64 libedit-devel.x86_64 doxygen.x86_64 graphviz.x86_64) + DEP_ARRAY=( git gcc gcc-c++ autoconf automake libtool make bzip2 \ + bzip2-devel openssl-devel gmp-devel libstdc++ \ + python3 python3-devel python-devel libedit-devel doxygen graphviz) fi COUNT=1 DISPLAY="" @@ -87,6 +87,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index 60810199f73..49debdf68e8 100644 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -63,6 +63,7 @@ SCL=$( rpm -qa | grep -E 'centos-release-scl-[0-9].*' ) if [ -z "${SCL}" ]; then printf "\\t - Do you wish to install and enable this repository?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -87,6 +88,7 @@ DEVTOOLSET=$( rpm -qa | grep -E 'devtoolset-7-[0-9].*' ) if [ -z "${DEVTOOLSET}" ]; then printf "\\tDo you wish to install devtoolset-7?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -118,6 +120,7 @@ PYTHON33=$( rpm -qa | grep -E 'python33-[0-9].*' ) if [ -z "${PYTHON33}" ]; then printf "\\tDo you wish to install python33?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -170,6 +173,7 @@ printf "\\tThe following dependencies are required to install EOSIO.\\n" printf "\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_darwin.sh b/scripts/eosio_build_darwin.sh index f3ee81cf467..bc4a20ebd58 100644 --- a/scripts/eosio_build_darwin.sh +++ b/scripts/eosio_build_darwin.sh @@ -70,6 +70,7 @@ then printf "\\tHomebrew must be installed to compile EOS.IO\\n\\n" printf "\\tDo you wish to install Home Brew?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case "${yn}" in [Yy]* ) @@ -138,6 +139,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" echo "Do you wish to install these packages?" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -159,13 +161,12 @@ printf "\\tExiting now.\\n\\n" exit 1; fi - if [[ "$DEP" == "llvm@4" ]]; then - "${BREW}" unlink ${DEP} - elif ! "${BREW}" unlink ${DEP} && "${BREW}" link --force ${DEP} - then - printf "\\tHomebrew exited with the above errors.\\n" - printf "\\tExiting now.\\n\\n" - exit 1; + if [ $PERMISSION_GETTEXT -eq 1 ]; then + if ! "${BREW}" link --force gettext; then + printf "\\tHomebrew exited with the above errors.\\n" + printf "\\tExiting now.\\n\\n" + exit 1; + fi fi break;; [Nn]* ) echo "User aborting installation of required dependencies, Exiting now."; exit;; @@ -184,6 +185,7 @@ printf "\\tFound Boost Version %s.\\n" "${BVERSION}" printf "\\tEOS.IO requires Boost version 1.67.\\n" printf "\\tWould you like to uninstall version %s and install Boost version 1.67.\\n" "${BVERSION}" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) @@ -218,7 +220,7 @@ done fi printf "\\tInstalling boost libraries.\\n" - if ! "${BREW}" install https://raw.githubusercontent.com/Homebrew/homebrew-core/f946d12e295c8a27519b73cc810d06593270a07f/Formula/boost.rb + if ! "${BREW}" install "${SOURCE_DIR}/scripts/boost.rb" then printf "\\tUnable to install boost 1.67 libraries at this time. 0\\n" printf "\\tExiting now.\\n\\n" diff --git a/scripts/eosio_build_fedora.sh b/scripts/eosio_build_fedora.sh index fbf6fcbf986..b608cf00978 100644 --- a/scripts/eosio_build_fedora.sh +++ b/scripts/eosio_build_fedora.sh @@ -86,6 +86,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these dependencies?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/eosio_build_ubuntu.sh b/scripts/eosio_build_ubuntu.sh index ab3e8823d8e..373093e237d 100644 --- a/scripts/eosio_build_ubuntu.sh +++ b/scripts/eosio_build_ubuntu.sh @@ -91,6 +91,7 @@ printf "\\n\\tThe following dependencies are required to install EOSIO.\\n" printf "\\n\\t${DISPLAY}\\n\\n" printf "\\tDo you wish to install these packages?\\n" + if is_noninteractive; then exec <<< "1"; fi select yn in "Yes" "No"; do case $yn in [Yy]* ) diff --git a/scripts/generate_deb.sh b/scripts/generate_deb.sh index 5009f7627b3..9af16c069f3 100644 --- a/scripts/generate_deb.sh +++ b/scripts/generate_deb.sh @@ -13,19 +13,16 @@ fi NAME="${PROJECT}_${VERSION_NO_SUFFIX}-${RELEASE}_amd64" -DEPS_STR="" -for dep in "${DEPS[@]}"; do - DEPS_STR="${DEPS_STR} Depends: ${dep}" -done mkdir -p ${PROJECT}/DEBIAN -echo "Package: ${PROJECT} +chmod 0755 ${PROJECT}/DEBIAN +echo "Package: ${PROJECT} Version: ${VERSION_NO_SUFFIX}-${RELEASE} Section: devel Priority: optional -Depends: libbz2-dev (>= 1.0), libssl-dev (>= 1.0), libgmp3-dev, build-essential, libicu-dev, zlib1g-dev +Depends: libbz2-dev (>= 1.0), libssl-dev (>= 1.0), libgmp3-dev, build-essential, libicu-dev, zlib1g-dev, libtinfo5 Architecture: amd64 -Homepage: ${URL} -Maintainer: ${EMAIL} +Homepage: ${URL} +Maintainer: ${EMAIL} Description: ${DESC}" &> ${PROJECT}/DEBIAN/control export PREFIX @@ -35,7 +32,10 @@ export SSUBPREFIX bash generate_tarball.sh ${NAME}.tar.gz -tar -xvzf ${NAME}.tar.gz -C ${PROJECT} -dpkg-deb --build ${PROJECT} +tar -xvzf ${NAME}.tar.gz -C ${PROJECT} +dpkg-deb --build ${PROJECT} +BUILDSTATUS=$? mv ${PROJECT}.deb ${NAME}.deb rm -r ${PROJECT} + +exit $BUILDSTATUS diff --git a/scripts/generate_package.sh.in b/scripts/generate_package.sh.in index 5d8aedebb70..9c190ed7d87 100644 --- a/scripts/generate_package.sh.in +++ b/scripts/generate_package.sh.in @@ -35,4 +35,9 @@ else echo "Error, unknown package type. Use either ['brew', 'deb', 'rpm']." exit -1 fi + +BUILDSTATUS=$? + rm -r tmp + +exit $BUILDSTATUS diff --git a/scripts/generate_rpm.sh b/scripts/generate_rpm.sh index 35b60a4c816..8d7ebf03caa 100644 --- a/scripts/generate_rpm.sh +++ b/scripts/generate_rpm.sh @@ -47,5 +47,8 @@ ${DESC} %files -f filenames.txt" &> ${PROJECT}.spec rpmbuild -bb ${PROJECT}.spec +BUILDSTATUS=$? mv ~/rpmbuild/RPMS/x86_64 ./ rm -r ${PROJECT} ~/rpmbuild/BUILD/filenames.txt ${PROJECT}.spec + +exit $BUILDSTATUS diff --git a/tests/Cluster.py b/tests/Cluster.py index f26c5980339..06a5357e9af 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -896,7 +896,7 @@ def bios_bootstrap(totalNodes, biosHost, biosPort, walletMgr, silent=False): contract="eosio.token" action="transfer" for name, keys in producerKeys.items(): - data="{\"from\":\"eosio\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (name, initialFunds, "init transfer") + data="{\"from\":\"eosio\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (name, initialFunds, "init eosio transfer") opts="--permission eosio@active" if name != "eosio": trans=biosNode.pushMessage(contract, action, data, opts) diff --git a/tests/main.cpp b/tests/main.cpp index ad2b5846210..0644ce80545 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/tests/validate-dirty-db.py b/tests/validate-dirty-db.py index 68a62701dd6..ac7520bc353 100755 --- a/tests/validate-dirty-db.py +++ b/tests/validate-dirty-db.py @@ -91,7 +91,7 @@ def runNodeosAndGetOutput(myTimeout=3): assert(ret) assert(isinstance(ret, tuple)) if not ret[0]: - errorExit("Failed to startup nodeos sucessfully on try number %d" % (i)) + errorExit("Failed to startup nodeos successfully on try number %d" % (i)) assert(ret[1]) assert(isinstance(ret[1], dict)) # pylint: disable=unsubscriptable-object diff --git a/tests/wallet_tests.cpp b/tests/wallet_tests.cpp index 0891b030906..28290fef211 100644 --- a/tests/wallet_tests.cpp +++ b/tests/wallet_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -173,7 +173,8 @@ BOOST_AUTO_TEST_CASE(wallet_manager_test) pubkeys.emplace(pkey1.get_public_key()); pubkeys.emplace(pkey2.get_public_key()); trx = wm.sign_transaction(trx, pubkeys, chain_id ); - const auto& pks = trx.get_signature_keys(chain_id); + flat_set pks; + trx.get_signature_keys(chain_id, fc::time_point::maximum(), pks); BOOST_CHECK_EQUAL(2, pks.size()); BOOST_CHECK(find(pks.cbegin(), pks.cend(), pkey1.get_public_key()) != pks.cend()); BOOST_CHECK(find(pks.cbegin(), pks.cend(), pkey2.get_public_key()) != pks.cend()); diff --git a/tools/validate_reflection.py b/tools/validate_reflection.py new file mode 100755 index 00000000000..52cae39bc30 --- /dev/null +++ b/tools/validate_reflection.py @@ -0,0 +1,698 @@ +#!/usr/bin/env python3 + +import argparse +from collections import OrderedDict +import re +import os +import sys +import traceback + +############################################################### +# validate_reflection +# +# +# Looks for files with FC_REFLECT macros. Requires fields to match class definition (provided in same file), +# unless the FC_REFLECT is proceeded by "// @ignore , , ..., " to indicate that field1, +# field2, ... fieldN are not reflected and/or "// @swap , , ..., " to indicate that +# field1, field2, ... fieldN are not in the same order as the class definition. +# +# NOTE: If swapping fields the script expects you to only indicate fields that are not in the expected order, +# so once it runs into the swapped field, it will remove that field from the order and expect the remaining in +# that order, so if the class has field1, field2, field3, and field4, and the reflect macro has the order +# field1, field3, field2, then field4, it should indicate swapping field2. This will remove field2 from the +# expected order and the rest will now match. Alternatively it should indicate swapping field3, since the remaining +# fields will also match the order. But both field2 and field3 should not be indicated. +# +# +# +############################################################### + +import atexit +import tempfile + +@atexit.register +def close_debug_file(): + if debug_file != None: + debug_file.close() + + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument('-?', action='help', default=argparse.SUPPRESS, + help=argparse._('show this help message and exit')) +parser.add_argument('-d', '--debug', help="generate debug output into a temporary directory", action='store_true') +parser.add_argument('-r', '--recurse', help="recurse through an entire directory (if directory provided for \"file\"", action='store_true') +parser.add_argument('-x', '--extension', type=str, help="extensions array to allow for directory and recursive search. Defaults to \".hpp\" and \".cpp\".", action='append') +parser.add_argument('-e', '--exit-on-error', help="Exit immediately when a validation error is discovered. Default is to run validation on all files and directories provided.", action='store_true') +parser.add_argument('files', metavar='file', nargs='+', type=str, help="File containing nodes info in JSON format.") +args = parser.parse_args() + +recurse = args.recurse +if args.debug: + temp_dir = tempfile.mkdtemp() + print("temporary files writen to %s" % (temp_dir)) + debug_file = open(os.path.join(temp_dir, "validate_reflect.debug"), "w") +else: + debug_file = None +extensions = [] +if args.extension is None or len(args.extension) == 0: + extensions = [".hpp",".cpp"] +else: + for extension in args.extension: + assert len(extension) > 0, "empty --extension passed in" + if extension[0] != ".": + extension = "." + extension + extensions.append(extension) +print("extensions=%s" % (",".join(extensions))) +ignore_str = "@ignore" +swap_str = "@swap" +fc_reflect_str = "FC_REFLECT" +fc_reflect_possible_enum_ext = "(?:_ENUM)?" +fc_reflect_derived_ext = "(?:_DERIVED)" + +def debug(debug_str): + if debug_file is not None: + debug_file.write(debug_str + "\n") + +class EmptyScope: + single_comment_pattern = re.compile(r'//.*\n+') + single_comment_ignore_swap_pattern = re.compile(r'//\s*(?:%s|%s)\s' % (ignore_str, swap_str)) + multi_line_comment_pattern = re.compile(r'/\*(.*?)\*/', re.MULTILINE | re.DOTALL) + ignore_swap_pattern = re.compile(r'^\s*(%s|%s)\s+(.*)$' % (ignore_str, swap_str), re.DOTALL) + strip_extra_pattern = re.compile(r'\n\s*\*\s*') + invalid_chars_pattern = re.compile(r'([^\w\s,])') + multi_line_comment_ignore_swap_pattern = re.compile(r'(\w+)(?:\s*,\s*)?') + namespace_str = "namespace" + struct_str = "struct" + class_str = "class" + enum_str = "enum" + any_scope_pattern = re.compile(r'\{', re.DOTALL) + start_char = "{" + end_char = "}" + + def __init__(self, name, start, content, parent_scope): + pname = parent_scope.name if parent_scope is not None else "" + debug("EmptyScope.__init__ %s %d - Parent %s" % (name, start, pname)) + self.name = name + self.content = content + self.start = start + self.current = start + 1 + self.parent_scope = parent_scope + self.end = len(content) - 1 if start == 0 else None + self.children = {} + self.children_ordered = [] + self.fields = [] + self.usings = {} + self.inherit = None + + def read(self): + debug("EmptyScope(%s).read - %s" % (self.__class__.__name__, self.name)) + end = len(self.content) - 1 + while self.current < end: + next_scope = self.next_scope() + if next_scope is None: + break + self.add(next_scope) + + if self.end is None: + self.end = self.content.find(EmptyScope.end_char, self.current, len(self.content)) + pdesc = str(self.parent_scope) if self.parent_scope is not None else "" + assert self.end != -1, "Could not find \"%s\" in \"%s\" - parent scope - %s" % (EmptyScope.end_char, self.content[self.current:], pdesc) + debug("EmptyScope(%s).read - %s - Done at %s" % (self.__class__.__name__, self.name, self.end)) + + def add(self, child): + debug("EmptyScope.add %s (%s) to %s (%s) - DROP" % (child.name, child.__class__.__name__, self.name, self.__class__.__name__)) + pass + + def find_scope_start(self, content, start, end, find_str): + debug("EmptyScope.find_scope_start") + loc = content.find(find_str, start, end) + if loc == -1: + return loc + else: + return loc + len(find_str) - len(EmptyScope.start_char) + + def find_possible_end(self): + possible = self.content.find(EmptyScope.end_char, self.current) + debug("EmptyScope.find_possible_end current=%s possible end=%s" % (self.current, possible)) + return possible + + def next_scope(self, end = None): + if end is None: + end = self.find_possible_end() + debug("EmptyScope.next_scope current=%s end=%s" % (self.current, end)) + match = EmptyScope.any_scope_pattern.search(self.content[self.current:end]) + if match: + start = self.find_scope_start(self.content, self.current, end, EmptyScope.start_char) + new_scope = EmptyScope(None, start, self.content, self) + new_scope.read() + self.current = new_scope.end + 1 + return new_scope + + return None + + def find_class(self, scoped_name): + scope_separator = "::" + loc = scoped_name.find(scope_separator) + if loc != -1: + child_name = scoped_name[0:loc] + loc += len(scope_separator) + child_scoped_name = scoped_name[loc:] + if child_name in self.children: + debug("find_class traverse child_name: %s, child_scoped_name: %s" % (child_name, child_scoped_name)) + return self.children[child_name].find_class(child_scoped_name) + elif self.inherit is not None and scoped_name in self.inherit.children: + debug("find_class found scoped_name: %s in inherited: %s" % (scoped_name, self.inherit.name)) + return self.inherit.children[scoped_name].find_class(child_scoped_name) + else: + if scoped_name not in self.children: + inherit_children = ",".join(self.inherit.children) if self.inherit is not None else "no inheritance" + inherit_using = ",".join(self.inherit.usings) if self.inherit is not None else "no inheritance" + inherit = self.inherit.name if self.inherit is not None else None + debug("find_class %s not in children, using: %s, inherit: %s - children: %s, using: %s" % (scoped_name, ",".join(self.usings), inherit, inherit_children, inherit_using)) + + if scoped_name in self.children: + debug("find_class found scoped_name: %s" % (scoped_name)) + return self.children[scoped_name] + elif scoped_name in self.usings: + using = self.usings[scoped_name] + debug("find_class found scoped_name: %s, using: %s" % (scoped_name, using)) + return self.find_class(using) + elif self.inherit is not None and scoped_name in self.inherit.children: + debug("find_class found scoped_name: %s in inherited: %s" % (scoped_name, self.inherit.name)) + return self.inherit.children[scoped_name] + else: + debug("find_class could not find scoped_name: %s, children: %s" % (scoped_name, ",".join(self.children))) + return None + + def __str__(self): + indent = "" + next = self.parent_scope + while next is not None: + indent += " " + next = next.parent_scope + desc = "%s%s scope type=\"%s\"\n%s children={\n" % (indent, self.name, self.__class__.__name__, indent) + for child in self.children_ordered: + desc += str(self.children[child]) + "\n" + desc += indent + " }\n" + desc += indent + " fields={\n" + for field in self.fields: + desc += indent + " " + field + "\n" + desc += indent + " }\n" + desc += indent + " usings={\n" + for using in self.usings: + desc += indent + " " + using + ": " + self.usings[using] + "\n" + desc += indent + " }\n" + return desc + +def create_scope(type, name, inherit, start, content, parent_scope): + debug("create_scope") + if type == EmptyScope.namespace_str: + return Namespace(name, inherit, start, content, parent_scope) + elif type == EmptyScope.class_str or type == EmptyScope.struct_str: + return ClassStruct(name, inherit, start, content, parent_scope, is_enum = False) + elif type == EmptyScope.enum_str: + return ClassStruct(name, inherit, start, content, parent_scope, is_enum = True) + else: + assert False, "Script does not account for type = \"%s\" found in \"%s\"" % (type, content[start:]) + +class ClassStruct(EmptyScope): + field_pattern = re.compile(r'\n\s*?(?:mutable\s+)?(\w[\w:\d<>]*)\s+(\w+)\s*(?:=\s[^;]+;|;|=\s*{)', re.MULTILINE | re.DOTALL) + enum_field_pattern = re.compile(r'\n\s*?(\w+)\s*(?:=\s*[^,}\s]+)?\s*(?:,|})', re.MULTILINE | re.DOTALL) + class_pattern = re.compile(r'(%s|%s|%s)\s+(\w+)\s*(:\s*public\s+([^<\s]+)[^{]*)?\s*\{' % (EmptyScope.struct_str, EmptyScope.class_str, EmptyScope.enum_str), re.MULTILINE | re.DOTALL) + cb_obj_pattern = re.compile(r'chainbase::object$') + obj_pattern = re.compile(r'^object$') + using_pattern = re.compile(r'\n\s*?using\s+(\w+)\s*=\s*([\w:]+)(?:<.*>)?;') + + def __init__(self, name, inherit, start, content, parent_scope, is_enum): + debug("ClassStruct.__init__ %s %d" % (name, start)) + EmptyScope.__init__(self, name, start, content, parent_scope) + self.classes = {} + self.pattern = ClassStruct.class_pattern + self.is_enum = is_enum + self.inherit = None + if inherit is None: + self.ignore_id = False + else: + match = ClassStruct.cb_obj_pattern.search(inherit) + if match is None: + match = ClassStruct.obj_pattern.search(inherit) + + self.ignore_id = True if match else False + next = self.parent_scope + while self.inherit is None and next is not None: + self.inherit = next.find_class(inherit) + next = next.parent_scope + debug("Checking for object, ignore_id: %s, inherit: %s, name: %s" % (self.ignore_id, inherit, name)) + + def add(self, child): + debug("ClassStruct.add %s (%s) to %s (%s)" % (child.name, child.__class__.__name__, self.name, self.__class__.__name__)) + if isinstance(child, ClassStruct): + self.classes[child.name] = child + self.children[child.name] = child + self.children_ordered.append(child.name) + + def add_fields(self, start, end): + loc = start + while loc < end: + debug("ClassStruct.add_fields -{\n%s\n}" % (self.content[loc:end + 1])) + if self.is_enum: + loc = self.add_enum_field(loc, end) + else: + loc = self.add_field(loc, end) + debug("ClassStruct.add_fields done") + + def add_field(self, loc, end): + match = ClassStruct.field_pattern.search(self.content[loc:end + 1]) + if match is None: + return end + field = match.group(2) + self.fields.append(field) + all = match.group(0) + loc = self.content.find(all, loc) + len(all) + debug("ClassStruct.add_field - %s (%d) - %s" % (field, len(self.fields), ClassStruct.field_pattern.pattern)) + return loc + + def add_enum_field(self, loc, end): + match = ClassStruct.enum_field_pattern.search(self.content[loc:end + 1]) + if match is None: + return end + field = match.group(1) + self.fields.append(field) + all = match.group(0) + loc = self.content.find(all, loc) + len(all) + debug("ClassStruct.add_enum_field - %s (%d) - %s" % (field, len(self.fields), ClassStruct.enum_field_pattern.pattern)) + return loc + + def add_usings(self, start, end): + loc = start + while loc < end: + debug("ClassStruct.add_usings -{\n%s\n}" % (self.content[loc:end + 1])) + match = ClassStruct.using_pattern.search(self.content[loc:end]) + if match is None: + break + using = match.group(1) + class_struct = match.group(2) + self.usings[using] = class_struct + all = match.group(0) + loc = self.content.find(all, loc) + len(all) + debug("ClassStruct.add_usings - %s (%d)" % (using, len(self.usings))) + debug("ClassStruct.add_usings done") + + def next_scope(self, end = None): + new_scope = None + if end is None: + end = self.find_possible_end() + debug("ClassStruct.next_scope end=%s on %s\n\npossible scope={\n\"%s\"\n\n\npattern=%s" % (end, self.name, self.content[self.current:end], self.pattern.pattern)) + match = self.pattern.search(self.content[self.current:end]) + start = -1 + search_str = None + type = None + name = None + inherit = None + if match: + debug("ClassStruct.next_scope match on %s" % (self.name)) + search_str = match.group(0) + type = match.group(1) + name = match.group(2) + if len(match.groups()) >= 3: + inherit = match.group(4) + + start = self.find_scope_start(self.content, self.current, end, search_str) + debug("all: %s, type: %s, name: %s, start: %s, inherit: %s" % (search_str, type, name, start, inherit)) + + generic_scope_start = self.find_scope_start(self.content, self.current, end, EmptyScope.start_char) + if start == -1 and generic_scope_start == -1: + debug("ClassStruct.next_scope end=%s no scopes add_fields and exit" % (end)) + self.add_fields(self.current, end) + return None + + debug("found \"%s\" - \"%s\" - \"%s\" current=%s, start=%s, end=%s, pattern=%s " % (search_str, type, name, self.current, start, end, self.pattern.pattern)) + # determine if there is a non-namespace/non-class/non-struct scope before a namespace/class/struct scope + if start != -1 and (generic_scope_start == -1 or start <= generic_scope_start): + debug("found %s at %d" % (type, start)) + new_scope = create_scope(type, name, inherit, start, self.content, self) + else: + debug("found EmptyScope (%s) at %d, next scope at %s" % (type, generic_scope_start, start)) + new_scope = EmptyScope("", generic_scope_start, self.content, self) + + self.add_fields(self.current, new_scope.start) + self.add_usings(self.current, new_scope.start) + new_scope.read() + self.current = new_scope.end + 1 + + return new_scope + +class Namespace(ClassStruct): + namespace_class_pattern = re.compile(r'(%s|%s|%s|%s)\s+(\w+)\s*(:\s*public\s+([^<\s]+)[^{]*)?\s*\{' % (EmptyScope.namespace_str, EmptyScope.struct_str, EmptyScope.class_str, EmptyScope.enum_str), re.MULTILINE | re.DOTALL) + + def __init__(self, name, inherit, start, content, parent_scope): + debug("Namespace.__init__ %s %d" % (name, start)) + assert inherit is None, "namespace %s should not inherit from %s" % (name, inherit) + ClassStruct.__init__(self, name, None, start, content, parent_scope, is_enum = False) + self.namespaces = {} + self.pattern = Namespace.namespace_class_pattern + + def add(self, child): + debug("Namespace.add %s (%s) to %s (%s)" % (child.name, child.__class__.__name__, self.name, self.__class__.__name__)) + if isinstance(child, ClassStruct): + ClassStruct.add(self, child) + return + if isinstance(child, Namespace): + self.namespaces[child.name] = child + self.children[child.name] = child + self.children_ordered.append(child.name) + +class Reflection: + def __init__(self, name): + self.name = name + self.fields = [] + self.ignored = [] + self.swapped = [] + self.absent = [] + +class Reflections: + def __init__(self, content): + self.content = content + self.current = 0 + self.end = len(content) + self.classes = OrderedDict() + self.with_2_comments = re.compile(r'(//\s*(%s|%s)\s+([^/]*?)\s*\n\s*//\s*(%s|%s)\s+([^/]*?)\s*\n\s*(%s%s\s*\(\s*(\w[^\s<]*))(?:<[^>]*>)?\s*,)' % (ignore_str, swap_str, ignore_str, swap_str, fc_reflect_str, fc_reflect_possible_enum_ext), re.MULTILINE | re.DOTALL) + self.with_comment = re.compile(r'(//\s*(%s|%s)\s+([^/]*?)\s*\n\s*(%s%s\s*\(\s*(\w[^\s<]*))(?:<[^>]*>)?\s*,)' % (ignore_str, swap_str, fc_reflect_str, fc_reflect_possible_enum_ext), re.MULTILINE | re.DOTALL) + self.reflect_pattern = re.compile(r'(\b(%s%s\s*\(\s*(\w[^\s<]*)(?:<[^>]*>)?\s*,)\s*(\(.*?\))\s*\))[^\)]*%s%s\b' % (fc_reflect_str, fc_reflect_possible_enum_ext, fc_reflect_str, fc_reflect_possible_enum_ext), re.MULTILINE | re.DOTALL) + self.field_pattern = re.compile(r'\(([^\)]+)\)', re.MULTILINE | re.DOTALL) + self.ignore_swap_pattern = re.compile(r'\b([\w\d]+)\b', re.MULTILINE | re.DOTALL) + + def read(self): + while self.current < self.end: + match_2_comments = self.with_2_comments.search(self.content[self.current:]) + match_comment = self.with_comment.search(self.content[self.current:]) + match_reflect = self.reflect_pattern.search(self.content[self.current:]) + match_loc = None + if match_2_comments or match_comment: + loc1 = self.content.find(match_2_comments.group(1), self.current) if match_2_comments else self.end + loc2 = self.content.find(match_comment.group(1), self.current) if match_comment else self.end + debug("loc1=%s and loc2=%s" % (loc1, loc2)) + group1 = match_2_comments.group(1) if match_2_comments else "" + group2 = match_comment.group(1) if match_comment else "" + debug("\n ***** group1={\n%s\n}\n\n\n ***** group2={\n%s\n}\n\n\n" % (group1, group2)) + if loc2 < loc1: + debug("loc2 earlier") + match_2_comments = None + match_loc = loc2 + else: + match_loc = loc1 + if match_reflect and match_loc is not None: + debug("match_reflect and one of the other matches") + loc1 = self.content.find(match_reflect.group(1), self.current) + if loc1 < match_loc: + debug("choose the other matches") + match_comment = None + match_2_comments = None + else: + debug("choose comment") + pass + + if match_2_comments: + debug("match_2_comments") + debug("Groups {") + for g in match_2_comments.groups(): + debug(" %s" % g) + debug("}") + assert len(match_2_comments.groups()) == 7, "match_2_comments wrong size due to regex pattern change" + ignore_or_swap1 = match_2_comments[2] + next_reflect_ignore_swap1 = match_2_comments[3] + ignore_or_swap2 = match_2_comments[4] + next_reflect_ignore_swap2 = match_2_comments[5] + search_string_for_next_reflect_class = match_2_comments[6] + next_reflect_class = match_2_comments[7] + self.add_ignore_swaps(next_reflect_class, next_reflect_ignore_swap1, ignore_or_swap1) + self.add_ignore_swaps(next_reflect_class, next_reflect_ignore_swap2, ignore_or_swap2) + elif match_comment: + debug("match_comment") + debug("Groups {") + for g in match_comment.groups(): + debug(" %s" % g) + debug("}") + assert len(match_comment.groups()) == 5, "match_comment too short due to regex pattern change" + # not using array indices here because for some reason the type of match_2_comments and match_comment are different + ignore_or_swap = match_comment.group(2) + next_reflect_ignore_swap = match_comment.group(3) + search_string_for_next_reflect_class = match_comment.group(4) + next_reflect_class = match_comment.group(5) + self.add_ignore_swaps(next_reflect_class, next_reflect_ignore_swap, ignore_or_swap) + + if match_reflect: + debug("match_reflect") + debug("Groups {") + for g in match_reflect.groups(): + debug(" %s" % g) + debug("}") + assert len(match_reflect.groups()) == 4, "match_reflect too short due to regex pattern change" + next_reflect = match_reflect.group(2) + next_reflect_class = match_reflect.group(3) + next_reflect_fields = match_reflect.group(4) + self.add_fields(next_reflect, next_reflect_class, next_reflect_fields) + else: + debug("search for next reflect done") + self.current = self.end + break + + def find_or_add(self, reflect_class): + if reflect_class not in self.classes: + debug("find_or_add added \"%s\"" % (reflect_class)) + self.classes[reflect_class] = Reflection(reflect_class) + return self.classes[reflect_class] + + def add_fields(self, next_reflect, next_reflect_class, next_reflect_fields): + old = self.current + self.current = self.content.find(next_reflect, self.current) + len(next_reflect) + debug("all={\n\n%s\n\nclass=\n\n%s\n\nfields=\n\n%s\n\n" % (next_reflect, next_reflect_class, next_reflect_fields)) + fields = re.findall(self.field_pattern, next_reflect_fields) + for field in fields: + self.add_field(next_reflect_class, field) + reflect_class = self.find_or_add(next_reflect_class) + debug("add_fields %s done, fields count=%s, ignored count=%s, swapped count=%s" % (next_reflect_class, len(reflect_class.fields), len(reflect_class.ignored), len(reflect_class.swapped))) + + def add_ignore_swaps(self, next_reflect_class, next_reflect_ignores_swaps, ignore_or_swap): + debug("class=\n\n%s\n\n%s=\n\n%s\n\n" % (next_reflect_class, ignore_or_swap, next_reflect_ignores_swaps)) + end = len(next_reflect_ignores_swaps) + current = 0 + while current < end: + ignore_swap_match = self.ignore_swap_pattern.search(next_reflect_ignores_swaps[current:]) + if ignore_swap_match: + ignore_swap = ignore_swap_match.group(1) + reflect_class = self.find_or_add(next_reflect_class) + if (ignore_or_swap == ignore_str): + assert ignore_swap not in reflect_class.ignored, "Reflection for %s repeats %s \"%s\"" % (next_reflect_class, ignore_or_swap) + assert ignore_swap not in reflect_class.swapped, "Reflection for %s references field \"%s\" in %s and %s " % (next_reflect_class, ignore_swap, ignore_str, swap_str) + reflect_class.ignored.append(ignore_swap) + else: + assert ignore_swap not in reflect_class.swapped, "Reflection for %s repeats %s \"%s\"" % (next_reflect_class, ignore_or_swap) + assert ignore_swap not in reflect_class.ignored, "Reflection for %s references field \"%s\" in %s and %s " % (next_reflect_class, ignore_swap, swap_str, ignore_str) + reflect_class.swapped.append(ignore_swap) + debug("ignore or swap %s --> %s, ignored count=%s, swapped count=%s" % (next_reflect_class, ignore_swap, len(reflect_class.ignored), len(reflect_class.swapped))) + current = next_reflect_ignores_swaps.find(ignore_swap_match.group(0), current) + len(ignore_swap_match.group(0)) + else: + break + + + def add_field(self, reflect_class_name, field): + reflect_class = self.find_or_add(reflect_class_name) + assert field not in reflect_class.fields, "Reflection for %s repeats field \"%s\"" % (reflect_class_name, field) + reflect_class.fields.append(field) + debug("add_field %s --> %s" % (reflect_class_name, field)) + +def replace_multi_line_comment(match): + all=match.group(1) + all=EmptyScope.strip_extra_pattern.sub("", all) + debug("multiline found=%s" % (all)) + match=EmptyScope.ignore_swap_pattern.search(all) + if match: + ignore_or_swap = match.group(1) + all = match.group(2) + debug("multiline %s now=%s" % (ignore_or_swap, all)) + invalid_chars=EmptyScope.invalid_chars_pattern.search(all) + if invalid_chars: + for ic in invalid_chars.groups(): + debug("invalid_char=%s" % (ic)) + debug("WARNING: looks like \"%s\" is intending to %s, but there are invalid characters - \"%s\"" % (all, ignore_or_swap, ",".join(invalid_chars.groups()))) + return "" + groups=re.findall(EmptyScope.multi_line_comment_ignore_swap_pattern, all) + if groups is None: + return "" + rtn_str="// %s " % (ignore_or_swap) + rtn_str+=', '.join([group for group in groups if group is not None]) + debug("multiline rtn_str=%s" % (rtn_str)) + return rtn_str + + debug("multiline no match") + return "" + +def replace_line_comment(match): + all=match.group(0) + debug("singleline found=%s" % (all)) + if EmptyScope.single_comment_ignore_swap_pattern.match(all): + return all + else: + return "\n" + +def validate_file(file): + f = open(file, "r") + contents = "\n" + f.read() # lazy fix for complex regex + f.close() + contents = EmptyScope.multi_line_comment_pattern.sub(replace_multi_line_comment, contents) + contents = EmptyScope.single_comment_pattern.sub(replace_line_comment, contents) + found = re.search(fc_reflect_str, contents) + if found is None: + return + print("validate %s" % (file)) + debug("validate %s" % (file)) + global_namespace=Namespace("", None, 0, contents, None) + global_namespace.read() + if args.debug: + _, filename = os.path.split(file) + with open(os.path.join(temp_dir, filename + ".struct"), "w") as f: + f.write("global_namespace=%s" % (global_namespace)) + with open(os.path.join(temp_dir, filename + ".stripped"), "w") as f: + f.write(contents) + reflections=Reflections(contents) + reflections.read() + for reflection_name in reflections.classes: + reflection = reflections.classes[reflection_name] + class_struct = global_namespace.find_class(reflection_name) + if class_struct is None: + match=re.search(r'^(.+?)::id_type$', reflection_name) + if match: + parent_class_name = match.group(1) + parent_class = global_namespace.find_class(parent_class_name) + if parent_class.ignore_id: + # this is a chainbase::object, don't need to worry about id_type definition + continue + class_struct_num_fields = len(class_struct.fields) if class_struct is not None else None + debug("reflection_name=%s, class field count=%s, reflection field count=%s, ingore count=%s, swap count=%s" % (reflection_name, class_struct_num_fields, len(reflection.fields), len(reflection.ignored), len(reflection.swapped))) + assert isinstance(class_struct, ClassStruct), "could not find a %s/%s/%s for %s" % (EmptyScope.class_str, EmptyScope.struct_str, EmptyScope.enum_str, reflection_name) + if class_struct.ignore_id: + id_field = "id" + if id_field not in reflection.ignored and id_field not in reflection.fields: + debug("Object ignore_id Adding id to ignored for %s" % (reflection_name)) + reflection.ignored.append(id_field) + else: + debug("Object ignore_id NOT adding id to ignored for %s" % (reflection_name)) + rf_index = 0 + rf_len = len(reflection.fields) + + processed = [] + back_swapped = [] + fwd_swapped = [] + ignored = [] + f_index = 0 + f_len = len(class_struct.fields) + while f_index < f_len: + field = class_struct.fields[f_index] + reflect_field = reflection.fields[rf_index] if rf_index < rf_len else None + processed.append(field) + debug("\nfield=%s reflect_field=%s" % (field, reflect_field)) + if field in reflection.swapped: + debug("field \"%s\" swapped (back)" % (field)) + reflection.swapped.remove(field) + back_swapped.append(field) + assert field in reflection.fields, "Reflection for %s indicates swapping %s but swapped position is not indicated in the reflection fields. Should it be ignored?" % (reflection_name, field) + assert reflect_field != field, "Reflection for %s should not indicate swapping %s since it is in the correct order" % (reflection_name, field) + f_index += 1 + continue + if reflect_field in reflection.swapped: + debug("field \"%s\" swapped (fwd)" % (field)) + reflection.swapped.remove(reflect_field) + fwd_swapped.append(reflect_field) + assert reflect_field in reflection.fields, "Reflection for %s indicates swapping field %s but it doesn't exist in that class/struct so it should be removed" % (reflection_name, reflect_field) + rf_index += 1 + continue + assert reflect_field not in ignored, "Reflection for %s should not indicate %s for %s; it should indicate %s - %s" % (reflection_name, ignore_str, reflect_field, swap_str, ",".join(ignored)) + if field in reflection.ignored: + debug("ignoring: %s" % (field)) + reflection.ignored.remove(field) + ignored.append(field) + assert reflect_field != field, "Reflection for %s should not indicate ignoring %s since it is in the correct order" % (reflection_name, field) + f_index += 1 + continue + debug("ignored=%s, swapped=%s" % (",".join(reflection.ignored),",".join(reflection.swapped))) + if reflect_field is not None and reflect_field in back_swapped: + back_swapped.remove(reflect_field) + rf_index += 1 + elif field in fwd_swapped: + fwd_swapped.remove(field) + f_index += 1 + else: + assert reflect_field == field, "Reflection for %s should have field %s instead of %s or else it should indicate if the field should be ignored (%s) or swapped (%s)" %(reflection_name, field, reflect_field, ignore_str, swap_str) + f_index += 1 + rf_index += 1 + debug("rf_index=%s, rf_len=%s, f_index=%s, f_len=%s" % (rf_index, rf_len, f_index, f_len)) + + assert len(reflection.ignored) == 0, "Reflection for %s has erroneous ignores - \"%s\"" % (reflection_name, ",".join(reflection.ignored)) + unused_reflect_fields = [] + while rf_index < rf_len: + debug("rf_index=%s, rf_len=%s fields=%s" % (rf_index, rf_len, ",".join(reflection.fields))) + reflect_field = reflection.fields[rf_index] + if reflect_field in back_swapped: + back_swapped.remove(reflect_field) + else: + unused_reflect_fields.append(reflect_field) + rf_index += 1 + assert len(unused_reflect_fields) == 0, "Reflection for %s has fields not in definition for class/struct - \"%s\"" % (reflection_name, ",".join(unused_reflect_fields)) + assert len(reflection.swapped) == 0, "Reflection for %s has erroneous swaps - \"%s\"" % (reflection_name, ",".join(reflection.swapped)) + assert len(back_swapped) == 0, "Reflection for %s indicated swapped fields that were never provided - \"%s\"" % (reflection_name, ",".join(back_swapped)) + assert len(fwd_swapped) == 0, "Reflection for %s indicated and provided swapped fields that are not in the class - \"%s\"" % (reflection_name, ",".join(fwd_swapped)) + + print("%s passed" % (file)) + +success = True + +def walk(current_dir): + result = True + print("Searching for files: %s" % (current_dir)) + for root, dirs, filenames in os.walk(current_dir): + for filename in filenames: + _, extension = os.path.splitext(filename) + if extension not in extensions: + continue + try: + validate_file(os.path.join(root, filename)) + except AssertionError: + _, info, tb = sys.exc_info() + traceback.print_tb(tb) # Fixed format + tb_info = traceback.extract_tb(tb) + filename, line, func, text = tb_info[-1] + + print("An error occurred in %s:%s: %s" % (filename, line, info), file=sys.stderr) + if args.exit_on_error: + exit(1) + result = False + + if not recurse: + break + return result + +for file in args.files: + if os.path.isdir(file): + success &= walk(file) + elif os.path.isfile(file): + try: + validate_file(file) + except AssertionError: + _, info, tb = sys.exc_info() + traceback.print_tb(tb) # Fixed format + tb_info = traceback.extract_tb(tb) + filename, line, func, text = tb_info[-1] + + print("An error occurred in %s:%s: %s" % (filename, line, info), file=sys.stderr) + if args.exit_on_error: + exit(1) + success = False + else: + print("ERROR \"%s\" is neither a directory nor a file" % file) + success = False + +if success: + exit(0) +else: + exit(1) diff --git a/tutorials/bios-boot-tutorial/README.md b/tutorials/bios-boot-tutorial/README.md index bebba5f4a04..ea353898086 100644 --- a/tutorials/bios-boot-tutorial/README.md +++ b/tutorials/bios-boot-tutorial/README.md @@ -2,12 +2,43 @@ The `bios-boot-tutorial.py` script simulates the EOSIO bios boot sequence. -The script can be run with no arguments directly from the `tutorials/bios-boot-tutorial` directory. +``Prerequisites``: + +1. Python 3.x +2. CMake +3. git + +``Steps``: + +1. Install eosio binaries by following the steps outlined in below tutorial +[Install eosio binaries](https://github.com/EOSIO/eos#mac-os-x-brew-install) + +2. Install eosio.cdt binaries by following the steps outlined in below tutorial +[Install eosio.cdt binaries](https://github.com/EOSIO/eosio.cdt#binary-releases) + +3. Compile eosio.contracts + +```bash +$ cd ~ +$ git clone https://github.com/EOSIO/eosio.contracts.git +$ cd ./eosio.contracts/ +$ ./build.sh +$ pwd + +``` + +4. Make note of the directory where the contracts were compiled +The last command in the previous step printed on the bash console the contracts' directory, make note of it, we'll reference it from now on as `EOSIO_CONTRACTS_DIRECTORY` + +5. Launch the `bios-boot-tutorial.py` script +Minimal command line to launch the script below, make sure you replace `EOSIO_CONTRACTS_DIRECTORY` with actual directory ```bash -$ cd tutorials/bios-boot-tutorial +$ cd ~ +$ git clone https://github.com/EOSIO/eos.git +$ cd ./eos/tutorials/bios-boot-tutorial/ +$ python3 bios-boot-tutorial.py --cleos="cleos --wallet-url http://127.0.0.1:6666 " --nodeos=nodeos --keosd=keosd --contracts-dir="/EOSIO_CONTRACTS_DIRECTORY/" -a -$ ./bios-boot-tutorial.py ``` -See [EOSIO Documentation Wiki: Tutorial - Bios Boot](https://github.com/EOSIO/eos/wiki/Tutorial-Bios-Boot-Sequence) for additional information. +See [EOSIO Documentation Wiki: Tutorial - Bios Boot](https://github.com/EOSIO/eos/wiki/Tutorial-Bios-Boot-Sequence) for additional information. \ No newline at end of file diff --git a/tutorials/bios-boot-tutorial/bios-boot-tutorial.py b/tutorials/bios-boot-tutorial/bios-boot-tutorial.py index 59a1590d4d6..358d3b963b8 100755 --- a/tutorials/bios-boot-tutorial/bios-boot-tutorial.py +++ b/tutorials/bios-boot-tutorial/bios-boot-tutorial.py @@ -284,17 +284,20 @@ def stepStartBoot(): startNode(0, {'name': 'eosio', 'pvt': args.private_key, 'pub': args.public_key}) sleep(1.5) def stepInstallSystemContracts(): - run(args.cleos + 'set contract eosio.token ' + args.contracts_dir + 'eosio.token/') - run(args.cleos + 'set contract eosio.msig ' + args.contracts_dir + 'eosio.msig/') + run(args.cleos + 'set contract eosio.token ' + args.contracts_dir + '/eosio.token/') + run(args.cleos + 'set contract eosio.msig ' + args.contracts_dir + '/eosio.msig/') def stepCreateTokens(): run(args.cleos + 'push action eosio.token create \'["eosio", "10000000000.0000 %s"]\' -p eosio.token' % (args.symbol)) totalAllocation = allocateFunds(0, len(accounts)) run(args.cleos + 'push action eosio.token issue \'["eosio", "%s", "memo"]\' -p eosio' % intToCurrency(totalAllocation)) sleep(1) def stepSetSystemContract(): - retry(args.cleos + 'set contract eosio ' + args.contracts_dir + 'eosio.system/') + retry(args.cleos + 'set contract eosio ' + args.contracts_dir + '/eosio.system/') sleep(1) run(args.cleos + 'push action eosio setpriv' + jsonArg(['eosio.msig', 1]) + '-p eosio@active') +def stepInitSystemContract(): + run(args.cleos + 'push action eosio init' + jsonArg(['0', '4,SYS']) + '-p eosio@active') + sleep(1) def stepCreateStakedAccounts(): createStakedAccounts(0, len(accounts)) def stepRegProducers(): @@ -326,23 +329,24 @@ def stepLog(): parser = argparse.ArgumentParser() commands = [ - ('k', 'kill', stepKillAll, True, "Kill all nodeos and keosd processes"), - ('w', 'wallet', stepStartWallet, True, "Start keosd, create wallet, fill with keys"), - ('b', 'boot', stepStartBoot, True, "Start boot node"), - ('s', 'sys', createSystemAccounts, True, "Create system accounts (eosio.*)"), - ('c', 'contracts', stepInstallSystemContracts, True, "Install system contracts (token, msig)"), - ('t', 'tokens', stepCreateTokens, True, "Create tokens"), - ('S', 'sys-contract', stepSetSystemContract, True, "Set system contract"), - ('T', 'stake', stepCreateStakedAccounts, True, "Create staked accounts"), - ('p', 'reg-prod', stepRegProducers, True, "Register producers"), - ('P', 'start-prod', stepStartProducers, True, "Start producers"), - ('v', 'vote', stepVote, True, "Vote for producers"), - ('R', 'claim', claimRewards, True, "Claim rewards"), - ('x', 'proxy', stepProxyVotes, True, "Proxy votes"), - ('q', 'resign', stepResign, True, "Resign eosio"), - ('m', 'msg-replace', msigReplaceSystem, False, "Replace system contract using msig"), - ('X', 'xfer', stepTransfer, False, "Random transfer tokens (infinite loop)"), - ('l', 'log', stepLog, True, "Show tail of node's log"), + ('k', 'kill', stepKillAll, True, "Kill all nodeos and keosd processes"), + ('w', 'wallet', stepStartWallet, True, "Start keosd, create wallet, fill with keys"), + ('b', 'boot', stepStartBoot, True, "Start boot node"), + ('s', 'sys', createSystemAccounts, True, "Create system accounts (eosio.*)"), + ('c', 'contracts', stepInstallSystemContracts, True, "Install system contracts (token, msig)"), + ('t', 'tokens', stepCreateTokens, True, "Create tokens"), + ('S', 'sys-contract', stepSetSystemContract, True, "Set system contract"), + ('I', 'init-sys-contract', stepInitSystemContract, True, "Initialiaze system contract"), + ('T', 'stake', stepCreateStakedAccounts, True, "Create staked accounts"), + ('p', 'reg-prod', stepRegProducers, True, "Register producers"), + ('P', 'start-prod', stepStartProducers, True, "Start producers"), + ('v', 'vote', stepVote, True, "Vote for producers"), + ('R', 'claim', claimRewards, True, "Claim rewards"), + ('x', 'proxy', stepProxyVotes, True, "Proxy votes"), + ('q', 'resign', stepResign, True, "Resign eosio"), + ('m', 'msg-replace', msigReplaceSystem, False, "Replace system contract using msig"), + ('X', 'xfer', stepTransfer, False, "Random transfer tokens (infinite loop)"), + ('l', 'log', stepLog, True, "Show tail of node's log"), ] parser.add_argument('--public-key', metavar='', help="EOSIO Public Key", default='EOS8Znrtgwt8TfpmbVpTKvA2oB8Nqey625CLN8bCN3TEbgx86Dsvr', dest="public_key") diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 3ffcc5af263..f611fe768ce 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -1475,7 +1475,7 @@ public_key_type get_public_key( name keyname, string role ) { BOOST_AUTO_TEST_CASE(packed_transaction) { try { - chain::transaction txn; + chain::signed_transaction txn; txn.ref_block_num = 1; txn.ref_block_prefix = 2; txn.expiration.from_iso_string("2021-12-20T15:30"); diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index e889790ba33..955130130fa 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -1,6 +1,6 @@ /** * @file api_tests.cpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -149,7 +149,8 @@ transaction_trace_ptr CallAction(TESTER& test, T ac, const vector& test.set_transaction_headers(trx); auto sigs = trx.sign(test.get_private_key(scope[0], "active"), test.control->get_chain_id()); - trx.get_signature_keys(test.control->get_chain_id()); + flat_set keys; + trx.get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); auto res = test.push_transaction(trx); BOOST_CHECK_EQUAL(res->receipt->status, transaction_receipt::executed); test.produce_block(); @@ -173,7 +174,8 @@ transaction_trace_ptr CallFunction(TESTER& test, T ac, const vector& data, test.set_transaction_headers(trx, test.DEFAULT_EXPIRATION_DELTA); auto sigs = trx.sign(test.get_private_key(scope[0], "active"), test.control->get_chain_id()); - trx.get_signature_keys(test.control->get_chain_id() ); + flat_set keys; + trx.get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); auto res = test.push_transaction(trx); BOOST_CHECK_EQUAL(res->receipt->status, transaction_receipt::executed); test.produce_block(); @@ -266,7 +268,8 @@ BOOST_FIXTURE_TEST_CASE(action_receipt_tests, TESTER) { try { trx.actions.push_back(act); this->set_transaction_headers(trx, this->DEFAULT_EXPIRATION_DELTA); trx.sign(this->get_private_key(config::system_account_name, "active"), control->get_chain_id()); - trx.get_signature_keys(control->get_chain_id() ); + flat_set keys; + trx.get_signature_keys(control->get_chain_id(), fc::time_point::maximum(), keys); auto res = this->push_transaction(trx); BOOST_CHECK_EQUAL(res->receipt->status, transaction_receipt::executed); this->produce_block(); @@ -745,7 +748,8 @@ void call_test(TESTER& test, T ac, uint32_t billed_cpu_time_us , uint32_t max_cp test.set_transaction_headers(trx); //trx.max_cpu_usage_ms = max_cpu_usage_ms; auto sigs = trx.sign(test.get_private_key(N(testapi), "active"), test.control->get_chain_id()); - trx.get_signature_keys(test.control->get_chain_id() ); + flat_set keys; + trx.get_signature_keys(test.control->get_chain_id(), fc::time_point::maximum(), keys); auto res = test.push_transaction( trx, fc::time_point::now() + fc::milliseconds(max_cpu_usage_ms), billed_cpu_time_us ); BOOST_CHECK_EQUAL(res->receipt->status, transaction_receipt::executed); test.produce_block(); @@ -1085,7 +1089,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { produce_blocks( 3 ); //check that only one deferred transaction executed - auto dtrxs = control->get_scheduled_transactions(); + auto dtrxs = get_scheduled_transactions(); BOOST_CHECK_EQUAL(dtrxs.size(), 1); for (const auto& trx: dtrxs) { control->push_scheduled_transaction(trx, fc::time_point::maximum()); @@ -1110,7 +1114,7 @@ BOOST_FIXTURE_TEST_CASE(deferred_transaction_tests, TESTER) { try { produce_blocks( 3 ); //check that only one deferred transaction executed - auto dtrxs = control->get_scheduled_transactions(); + auto dtrxs = get_scheduled_transactions(); BOOST_CHECK_EQUAL(dtrxs.size(), 1); for (const auto& trx: dtrxs) { control->push_scheduled_transaction(trx, fc::time_point::maximum()); @@ -1728,14 +1732,23 @@ BOOST_FIXTURE_TEST_CASE(print_tests, TESTER) { try { BOOST_CHECK_EQUAL( tx9_act_cnsl.substr(start, end-start), "6.666666666666666e-07" ); // test printqf +#ifdef __x86_64__ + std::string expect1 = "5.000000000000000000e-01"; + std::string expect2 = "-3.750000000000000000e+00"; + std::string expect3 = "6.666666666666666667e-07"; +#else + std::string expect1 = "5.000000000000000e-01"; + std::string expect2 = "-3.750000000000000e+00"; + std::string expect3 = "6.666666666666667e-07"; +#endif auto tx10_trace = CALL_TEST_FUNCTION( *this, "test_print", "test_printqf", {} ); auto tx10_act_cnsl = tx10_trace->action_traces.front().console; start = 0; end = tx10_act_cnsl.find('\n', start); - BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), "5.000000000000000000e-01" ); + BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), expect1 ); start = end + 1; end = tx10_act_cnsl.find('\n', start); - BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), "-3.750000000000000000e+00" ); + BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), expect2 ); start = end + 1; end = tx10_act_cnsl.find('\n', start); - BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), "6.666666666666666667e-07" ); + BOOST_CHECK_EQUAL( tx10_act_cnsl.substr(start, end-start), expect3 ); BOOST_REQUIRE_EQUAL( validate(), true ); } FC_LOG_AND_RETHROW() } diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 10a10dc2060..045255da6d9 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) auto b = main.produce_block(); // Make a copy of the valid block and corrupt the transaction - auto copy_b = std::make_shared(*b); + auto copy_b = std::make_shared(std::move(*b)); auto signed_tx = copy_b->transactions.back().trx.get().get_signed_transaction(); auto& act = signed_tx.actions.back(); auto act_data = act.data_as(); @@ -57,14 +57,15 @@ std::pair corrupt_trx_in_block(validating_te signed_block_ptr b = main.produce_block_no_validation(); // Make a copy of the valid block and corrupt the transaction - auto copy_b = std::make_shared(*b); - auto signed_tx = copy_b->transactions.back().trx.get().get_signed_transaction(); + auto copy_b = std::make_shared(b->clone()); + const auto& packed_trx = copy_b->transactions.back().trx.get(); + auto signed_tx = packed_trx.get_signed_transaction(); // Corrupt one signature signed_tx.signatures.clear(); signed_tx.sign(main.get_private_key(act_name, "active"), main.control->get_chain_id()); // Replace the valid transaction with the invalid transaction - auto invalid_packed_tx = packed_transaction(signed_tx); + auto invalid_packed_tx = packed_transaction(signed_tx, packed_trx.get_compression()); copy_b->transactions.back().trx = invalid_packed_tx; // Re-calculate the transaction merkle diff --git a/unittests/block_timestamp_tests.cpp b/unittests/block_timestamp_tests.cpp index 297789c9089..5cf998a85aa 100644 --- a/unittests/block_timestamp_tests.cpp +++ b/unittests/block_timestamp_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/unittests/contracts/deferred_test/deferred_test.cpp b/unittests/contracts/deferred_test/deferred_test.cpp index 95fc4316c8b..a30a7a2550e 100644 --- a/unittests/contracts/deferred_test/deferred_test.cpp +++ b/unittests/contracts/deferred_test/deferred_test.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/unittests/database_tests.cpp b/unittests/database_tests.cpp index 8f9d3553928..7cb35e85be5 100644 --- a/unittests/database_tests.cpp +++ b/unittests/database_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include @@ -28,8 +28,7 @@ BOOST_AUTO_TEST_SUITE(database_tests) TESTER test; // Bypass read-only restriction on state DB access for this unit test which really needs to mutate the DB to properly conduct its test. - // eosio::chain::database& db = const_cast( test.control->db() ); - chainbase::database &db = const_cast(test.control->db()); + eosio::chain::database& db = const_cast( test.control->db() ); auto ses = db.start_undo_session(true); diff --git a/unittests/delay_tests.cpp b/unittests/delay_tests.cpp index 9f14de4107c..ae1a3d114b5 100644 --- a/unittests/delay_tests.cpp +++ b/unittests/delay_tests.cpp @@ -72,7 +72,7 @@ BOOST_FIXTURE_TEST_CASE( delay_error_create_account, validating_tester) { try { produce_blocks(6); - auto scheduled_trxs = control->get_scheduled_transactions(); + auto scheduled_trxs = get_scheduled_transactions(); BOOST_REQUIRE_EQUAL(scheduled_trxs.size(), 1); auto dtrace = control->push_scheduled_transaction(scheduled_trxs.front(), fc::time_point::maximum()); BOOST_REQUIRE_EQUAL(dtrace->except.valid(), true); diff --git a/unittests/eosio_system_tester.hpp b/unittests/eosio_system_tester.hpp index 345ab49c436..194d52330ea 100644 --- a/unittests/eosio_system_tester.hpp +++ b/unittests/eosio_system_tester.hpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma once diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index d4c8e4d3bd4..9543a791174 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -88,7 +88,7 @@ BOOST_AUTO_TEST_CASE( fork_with_bad_block ) try { auto& fork = forks.at(j); if (j <= i) { - auto copy_b = std::make_shared(*b); + auto copy_b = std::make_shared(b->clone()); if (j == i) { // corrupt this block fork.block_merkle = remote.control->head_block_state()->blockroot_merkle; @@ -277,9 +277,9 @@ BOOST_AUTO_TEST_CASE( forking ) try { } wlog( "end push c2 blocks to c1" ); wlog( "now push dan's block to c1 but first corrupt it so it is a bad block" ); - auto bad_block = *b; + signed_block bad_block = std::move(*b); bad_block.transaction_mroot = bad_block.previous; - auto bad_block_bs = c.control->create_block_state_future( std::make_shared(bad_block) ); + auto bad_block_bs = c.control->create_block_state_future( std::make_shared(std::move(bad_block)) ); c.control->abort_block(); BOOST_REQUIRE_EXCEPTION(c.control->push_block( bad_block_bs ), fc::exception, [] (const fc::exception &ex)->bool { diff --git a/unittests/include/config.hpp.in b/unittests/include/config.hpp.in index cd6dd6435e8..8483e4dd73c 100644 --- a/unittests/include/config.hpp.in +++ b/unittests/include/config.hpp.in @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ namespace eosio { namespace unittests { namespace config { diff --git a/unittests/main.cpp b/unittests/main.cpp index ad2b5846210..0644ce80545 100644 --- a/unittests/main.cpp +++ b/unittests/main.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include diff --git a/unittests/message_buffer_tests.cpp b/unittests/message_buffer_tests.cpp index a0b6529f2c6..ccb69033e8e 100644 --- a/unittests/message_buffer_tests.cpp +++ b/unittests/message_buffer_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/unittests/misc_tests.cpp b/unittests/misc_tests.cpp index 1a93b66e97a..563c1922e9e 100644 --- a/unittests/misc_tests.cpp +++ b/unittests/misc_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include @@ -590,11 +590,9 @@ BOOST_AUTO_TEST_CASE(transaction_test) { try { BOOST_CHECK_EQUAL(1, trx.signatures.size()); trx.validate(); - packed_transaction pkt; - pkt.set_transaction(trx, packed_transaction::none); + packed_transaction pkt(trx, packed_transaction::none); - packed_transaction pkt2; - pkt2.set_transaction(trx, packed_transaction::zlib); + packed_transaction pkt2(trx, packed_transaction::zlib); BOOST_CHECK_EQUAL(true, trx.expiration == pkt.expiration()); BOOST_CHECK_EQUAL(true, trx.expiration == pkt2.expiration()); diff --git a/unittests/producer_schedule_tests.cpp b/unittests/producer_schedule_tests.cpp index 9ca0edecf82..a7279499656 100644 --- a/unittests/producer_schedule_tests.cpp +++ b/unittests/producer_schedule_tests.cpp @@ -260,4 +260,136 @@ BOOST_FIXTURE_TEST_CASE( producer_schedule_promotion_test, TESTER ) try { BOOST_REQUIRE_EQUAL( validate(), true ); } FC_LOG_AND_RETHROW() +BOOST_FIXTURE_TEST_CASE( producer_schedule_reduction, tester ) try { + create_accounts( {N(alice),N(bob),N(carol)} ); + produce_block(); + + auto compare_schedules = [&]( const vector& a, const producer_schedule_type& b ) { + return std::equal( a.begin(), a.end(), b.producers.begin(), b.producers.end() ); + }; + + auto res = set_producers( {N(alice),N(bob),N(carol)} ); + vector sch1 = { + {N(alice), get_public_key(N(alice), "active")}, + {N(bob), get_public_key(N(bob), "active")}, + {N(carol), get_public_key(N(carol), "active")} + }; + wlog("set producer schedule to [alice,bob,carol]"); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers().valid() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers() ) ); + BOOST_CHECK_EQUAL( control->pending_producers().version, 0 ); + produce_block(); // Starts new block which promotes the proposed schedule to pending + BOOST_CHECK_EQUAL( control->pending_producers().version, 1 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->pending_producers() ) ); + BOOST_CHECK_EQUAL( control->active_producers().version, 0 ); + produce_block(); + produce_block(); // Starts new block which promotes the pending schedule to active + BOOST_CHECK_EQUAL( control->active_producers().version, 1 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->active_producers() ) ); + produce_blocks(7); + + res = set_producers( {N(alice),N(bob)} ); + vector sch2 = { + {N(alice), get_public_key(N(alice), "active")}, + {N(bob), get_public_key(N(bob), "active")} + }; + wlog("set producer schedule to [alice,bob]"); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers().valid() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers() ) ); + + produce_blocks(48); + BOOST_REQUIRE_EQUAL( control->head_block_producer(), N(bob) ); + BOOST_REQUIRE_EQUAL( control->pending_block_state()->header.producer, N(carol) ); + BOOST_CHECK_EQUAL( control->pending_producers().version, 2 ); + + produce_blocks(47); + BOOST_CHECK_EQUAL( control->active_producers().version, 1 ); + produce_blocks(1); + + BOOST_REQUIRE_EQUAL( control->head_block_producer(), N(carol) ); + BOOST_REQUIRE_EQUAL( control->pending_block_state()->header.producer, N(alice) ); + BOOST_CHECK_EQUAL( control->active_producers().version, 2 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, control->active_producers() ) ); + + produce_blocks(2); + BOOST_CHECK_EQUAL( control->head_block_producer(), N(bob) ); + + BOOST_REQUIRE_EQUAL( validate(), true ); +} FC_LOG_AND_RETHROW() + +BOOST_FIXTURE_TEST_CASE( empty_producer_schedule_has_no_effect, tester ) try { + create_accounts( {N(alice),N(bob),N(carol)} ); + produce_block(); + + auto compare_schedules = [&]( const vector& a, const producer_schedule_type& b ) { + return std::equal( a.begin(), a.end(), b.producers.begin(), b.producers.end() ); + }; + + auto res = set_producers( {N(alice),N(bob)} ); + vector sch1 = { + {N(alice), get_public_key(N(alice), "active")}, + {N(bob), get_public_key(N(bob), "active")} + }; + wlog("set producer schedule to [alice,bob]"); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers().valid() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, *control->proposed_producers() ) ); + BOOST_CHECK_EQUAL( control->pending_producers().producers.size(), 0 ); + + // Start a new block which promotes the proposed schedule to pending + produce_block(); + BOOST_CHECK_EQUAL( control->pending_producers().version, 1 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->pending_producers() ) ); + BOOST_CHECK_EQUAL( control->active_producers().version, 0 ); + + // Start a new block which promotes the pending schedule to active + produce_block(); + BOOST_CHECK_EQUAL( control->active_producers().version, 1 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch1, control->active_producers() ) ); + produce_blocks(7); + + res = set_producers( {} ); + wlog("set producer schedule to []"); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers().valid() ); + BOOST_CHECK_EQUAL( control->proposed_producers()->producers.size(), 0 ); + BOOST_CHECK_EQUAL( control->proposed_producers()->version, 2 ); + + produce_blocks(12); + BOOST_CHECK_EQUAL( control->pending_producers().version, 1 ); + + // Empty producer schedule does get promoted from proposed to pending + produce_block(); + BOOST_CHECK_EQUAL( control->pending_producers().version, 2 ); + BOOST_CHECK_EQUAL( false, control->proposed_producers().valid() ); + + // However it should not get promoted from pending to active + produce_blocks(24); + BOOST_CHECK_EQUAL( control->active_producers().version, 1 ); + BOOST_CHECK_EQUAL( control->pending_producers().version, 2 ); + + // Setting a new producer schedule should still use version 2 + res = set_producers( {N(alice),N(bob),N(carol)} ); + vector sch2 = { + {N(alice), get_public_key(N(alice), "active")}, + {N(bob), get_public_key(N(bob), "active")}, + {N(carol), get_public_key(N(carol), "active")} + }; + wlog("set producer schedule to [alice,bob,carol]"); + BOOST_REQUIRE_EQUAL( true, control->proposed_producers().valid() ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, *control->proposed_producers() ) ); + BOOST_CHECK_EQUAL( control->proposed_producers()->version, 2 ); + + // Produce enough blocks to promote the proposed schedule to pending, which it can do because the existing pending has zero producers + produce_blocks(24); + BOOST_CHECK_EQUAL( control->active_producers().version, 1 ); + BOOST_CHECK_EQUAL( control->pending_producers().version, 2 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, control->pending_producers() ) ); + + // Produce enough blocks to promote the pending schedule to active + produce_blocks(24); + BOOST_CHECK_EQUAL( control->active_producers().version, 2 ); + BOOST_CHECK_EQUAL( true, compare_schedules( sch2, control->active_producers() ) ); + + BOOST_REQUIRE_EQUAL( validate(), true ); +} FC_LOG_AND_RETHROW() + BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/ram_tests.cpp b/unittests/ram_tests.cpp index 976e5b8cefe..6fe0eec6613 100644 --- a/unittests/ram_tests.cpp +++ b/unittests/ram_tests.cpp @@ -1,6 +1,6 @@ /** * @file api_tests.cpp - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" diff --git a/unittests/snapshot_tests.cpp b/unittests/snapshot_tests.cpp index e5207ca7bb8..1534a49b5dc 100644 --- a/unittests/snapshot_tests.cpp +++ b/unittests/snapshot_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include diff --git a/unittests/special_accounts_tests.cpp b/unittests/special_accounts_tests.cpp index 384b8e78ace..e3ad16fa9bc 100644 --- a/unittests/special_accounts_tests.cpp +++ b/unittests/special_accounts_tests.cpp @@ -1,6 +1,6 @@ /** * @file - * @copyright defined in eos/LICENSE.txt + * @copyright defined in eos/LICENSE */ #include #include