Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

performance improvements and network bug fixes #131

Merged
merged 42 commits into from
Oct 17, 2019
Merged
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
0c04ef1
Add logging for hard replay progress. Resolves #6531.
Dec 28, 2018
fdee487
Fix comment on max trx decompressed size
Jan 29, 2019
6cfa400
Changed to report from peer's LIB, since we don't know if any blocks …
Feb 15, 2019
e606e85
Ensure that a caught-up nodeos reports its new handshake status to ot…
Feb 15, 2019
28a0916
Clear connection node_id to prevent invalid self connect error
Jul 23, 2019
0017e95
Don't do needest work on peer request outside valid range
Jul 18, 2019
9f694cf
fix fork resolve special case
Jul 17, 2019
2e76aa9
Re-create socket on re-connect. Use non-deprecated async_connect.
Aug 12, 2019
32e3abf
merge fix
Sep 10, 2019
1969f14
attempt to fix bad prepared cert
Sep 10, 2019
91018c5
Fix for issue gh#6466 sync
Dec 20, 2018
b969b95
Add remote endpoint peer name to dup connection check
Feb 22, 2019
15b3ea9
fix bug in get_block of chain_plugin which could cause unnecessary fa…
Mar 9, 2019
61fd86f
Check for default transaction id which is possible if file/string doe…
Feb 25, 2019
26c572f
Add automatic handling of action.data if action.hex_data is available…
Feb 25, 2019
1c4a8cb
Prevent core dump by catching exception
Apr 8, 2019
845f900
#6980: memory leak when --trace-history not used
Apr 11, 2019
ee1cbab
Do not update connection fork_head info if in lib_catchup since we sh…
Sep 11, 2019
5d0e5ac
Only request_message for our head greater than msg.head when we suspe…
Aug 30, 2019
1417a13
Update comment for new behavior
Sep 3, 2019
a687eef
Merge branch 'fix-cert-gen' into feature/dpos-pbft-bos-optimise
VincentOCL Sep 19, 2019
73003e3
Log replay progress to logging system.
Jan 16, 2019
17c6b3a
allow opening block log with no blocks (fixes undefined behavior bug)…
Sep 19, 2019
16ef1dd
Remove unneeded request_message
Sep 20, 2019
a12b59a
sync_master recv_block only called on valid blocks, so don't disconne…
Sep 21, 2019
7134004
Check for out of date last irreversible block notice
Sep 22, 2019
829585e
get_block_id_for_num will throw unknown_block_exception when block nu…
Sep 18, 2019
60d6324
Fix #7767 - node never identified its peer is done syncying
Sep 17, 2019
187d02b
#7766 - report peer address on close
Sep 17, 2019
033f246
Fix problem with keepalive timer sometimes causing a crash on exit be…
Sep 25, 2019
c439b5f
Make sure plugin_shutdown is called in case of exception in plugin_st…
Sep 16, 2019
c855e40
WebAssembly checktime fixes
Sep 24, 2019
efd5143
Merge branch 'fix-cert-gen' into feature/dpos-pbft-bos-optimise
VincentOCL Sep 30, 2019
3036e5f
Merge remote-tracking branch 'origin/fix-wasm' into feature/bos-develop
VincentOCL Oct 4, 2019
af99251
fix merge from socket branch
Oct 4, 2019
759b6d6
Remove unneeded read_in_flight. Clear trx_in_progress_size on close s…
Aug 19, 2019
f864452
fix merge
Oct 10, 2019
e39d5f6
Capture socket on async_read to prevent close from destroying socket …
Oct 10, 2019
191416f
fix sync stable checkpoints
Oct 11, 2019
3ab79c7
attempt to fix the crash
Oct 15, 2019
746a76d
merge more commits from upstream
Oct 16, 2019
b6100f4
Merge branch 'release/3.0.x' into feature/bos-develop
oldcold Oct 17, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 76 additions & 46 deletions libraries/chain/block_log.cpp
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@

#define LOG_READ (std::ios::in | std::ios::binary)
#define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app)
#define LOG_RW ( std::ios::in | std::ios::out | std::ios::binary )

namespace eosio { namespace chain {

@@ -32,42 +31,47 @@ namespace eosio { namespace chain {
std::fstream index_stream;
fc::path block_file;
fc::path index_file;
bool open_files = false;
bool block_write;
bool index_write;
bool genesis_written_to_block_log = false;
uint32_t version = 0;
uint32_t first_block_num = 0;

inline void check_open_files() {
if( !open_files ) {
reopen();
inline void check_block_read() {
if (block_write) {
block_stream.close();
block_stream.open(block_file.generic_string().c_str(), LOG_READ);
block_write = false;
}
}
void reopen();

void close() {
if( block_stream.is_open() )
inline void check_block_write() {
if (!block_write) {
block_stream.close();
if( index_stream.is_open() )
index_stream.close();
open_files = false;
block_stream.open(block_file.generic_string().c_str(), LOG_WRITE);
block_write = true;
}
}
};

void block_log_impl::reopen() {
close();

// open to create files if they don't exist
//ilog("Opening block log at ${path}", ("path", my->block_file.generic_string()));
block_stream.open(block_file.generic_string().c_str(), LOG_WRITE);
index_stream.open(index_file.generic_string().c_str(), LOG_WRITE);

close();

block_stream.open(block_file.generic_string().c_str(), LOG_RW);
index_stream.open(index_file.generic_string().c_str(), LOG_RW);
inline void check_index_read() {
try {
if (index_write) {
index_stream.close();
index_stream.open(index_file.generic_string().c_str(), LOG_READ);
index_write = false;
}
}
FC_LOG_AND_RETHROW()
}

open_files = true;
}
inline void check_index_write() {
if (!index_write) {
index_stream.close();
index_stream.open(index_file.generic_string().c_str(), LOG_WRITE);
index_write = true;
}
}
};
}

block_log::block_log(const fc::path& data_dir)
@@ -84,21 +88,26 @@ namespace eosio { namespace chain {
block_log::~block_log() {
if (my) {
flush();
my->close();
my.reset();
}
}

void block_log::open(const fc::path& data_dir) {
my->close();
if (my->block_stream.is_open())
my->block_stream.close();
if (my->index_stream.is_open())
my->index_stream.close();

if (!fc::is_directory(data_dir))
fc::create_directories(data_dir);

my->block_file = data_dir / "blocks.log";
my->index_file = data_dir / "blocks.index";

my->reopen();
//ilog("Opening block log at ${path}", ("path", my->block_file.generic_string()));
my->block_stream.open(my->block_file.generic_string().c_str(), LOG_WRITE);
my->index_stream.open(my->index_file.generic_string().c_str(), LOG_WRITE);
my->block_write = true;
my->index_write = true;

/* On startup of the block log, there are several states the log file and the index file can be
* in relation to each other.
@@ -123,6 +132,7 @@ namespace eosio { namespace chain {

if (log_size) {
ilog("Log is nonempty");
my->check_block_read();
my->block_stream.seekg( 0 );
my->version = 0;
my->block_stream.read( (char*)&my->version, sizeof(my->version) );
@@ -149,6 +159,9 @@ namespace eosio { namespace chain {
}

if (index_size) {
my->check_block_read();
my->check_index_read();

ilog("Index is nonempty");
uint64_t block_pos;
my->block_stream.seekg(-sizeof(uint64_t), std::ios::end);
@@ -171,20 +184,20 @@ namespace eosio { namespace chain {
}
} else if (index_size) {
ilog("Index is nonempty, remove and recreate it");
my->close();
my->index_stream.close();
fc::remove_all(my->index_file);
my->reopen();
my->index_stream.open(my->index_file.generic_string().c_str(), LOG_WRITE);
my->index_write = true;
}
}

uint64_t block_log::append(const signed_block_ptr& b) {
try {
EOS_ASSERT( my->genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" );

my->check_open_files();
my->check_block_write();
my->check_index_write();

my->block_stream.seekp(0, std::ios::end);
my->index_stream.seekp(0, std::ios::end);
uint64_t pos = my->block_stream.tellp();
EOS_ASSERT(my->index_stream.tellp() == sizeof(uint64_t) * (b->block_num() - my->first_block_num),
block_log_append_fail,
@@ -211,17 +224,22 @@ namespace eosio { namespace chain {
}

void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block, uint32_t first_block_num ) {
my->close();
if (my->block_stream.is_open())
my->block_stream.close();
if (my->index_stream.is_open())
my->index_stream.close();

fc::remove_all(my->block_file);
fc::remove_all(my->index_file);

my->reopen();
my->block_stream.open(my->block_file.generic_string().c_str(), LOG_WRITE);
my->index_stream.open(my->index_file.generic_string().c_str(), LOG_WRITE);
my->block_write = true;
my->index_write = true;

auto data = fc::raw::pack(gs);
my->version = 0; // version of 0 is invalid; it indicates that the genesis was not properly written to the block log
my->first_block_num = first_block_num;
my->block_stream.seekp(0, std::ios::end);
my->block_stream.write((char*)&my->version, sizeof(my->version));
my->block_stream.write((char*)&my->first_block_num, sizeof(my->first_block_num));
my->block_stream.write(data.data(), data.size());
@@ -233,20 +251,29 @@ namespace eosio { namespace chain {

if (first_block) {
append(first_block);
} else {
my->head.reset();
my->head_id = {};
}

auto pos = my->block_stream.tellp();

my->block_stream.close();
my->block_stream.open(my->block_file.generic_string().c_str(), std::ios::in | std::ios::out | std::ios::binary ); // Bypass append-only writing just once

static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" );
my->version = block_log::max_supported_version;
my->block_stream.seekp( 0 );
my->block_stream.write( (char*)&my->version, sizeof(my->version) );
my->block_stream.seekp( pos );
flush();

my->block_write = false;
my->check_block_write(); // Reset to append-only writing.
}

std::pair<signed_block_ptr, uint64_t> block_log::read_block(uint64_t pos)const {
my->check_open_files();
my->check_block_read();

my->block_stream.seekg(pos);
std::pair<signed_block_ptr,uint64_t> result;
@@ -270,7 +297,7 @@ namespace eosio { namespace chain {
}

uint64_t block_log::get_block_pos(uint32_t block_num) const {
my->check_open_files();
my->check_index_read();
if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num))
return npos;
my->index_stream.seekg(sizeof(uint64_t) * (block_num - my->first_block_num));
@@ -280,7 +307,7 @@ namespace eosio { namespace chain {
}

signed_block_ptr block_log::read_head()const {
my->check_open_files();
my->check_block_read();

uint64_t pos;

@@ -308,13 +335,13 @@ namespace eosio { namespace chain {

void block_log::construct_index() {
ilog("Reconstructing Block Log Index...");
my->close();

my->index_stream.close();
fc::remove_all(my->index_file);

my->reopen();
my->index_stream.open(my->index_file.generic_string().c_str(), LOG_WRITE);
my->index_write = true;

uint64_t end_pos;
my->check_block_read();

my->block_stream.seekg(-sizeof( uint64_t), std::ios::end);
my->block_stream.read((char*)&end_pos, sizeof(end_pos));
@@ -343,10 +370,11 @@ namespace eosio { namespace chain {
my->block_stream.read((char*) &totem, sizeof(totem));
}

my->index_stream.seekp(0, std::ios::end);
while( pos < end_pos ) {
fc::raw::unpack(my->block_stream, tmp);
my->block_stream.read((char*)&pos, sizeof(pos));
if(tmp.block_num() % 1000 == 0)
ilog( "Block log index reconstructed for block ${n}", ("n", tmp.block_num()));
my->index_stream.write((char*)&pos, sizeof(pos));
}
} // construct_index
@@ -468,6 +496,8 @@ namespace eosio { namespace chain {
new_block_stream.write( data.data(), data.size() );
new_block_stream.write( reinterpret_cast<char*>(&pos), sizeof(pos) );
block_num = tmp.block_num();
if(block_num % 1000 == 0)
ilog( "Recovered block ${num}", ("num", block_num) );
pos = new_block_stream.tellp();
if( block_num == truncate_at_block )
break;
13 changes: 4 additions & 9 deletions libraries/chain/controller.cpp
Original file line number Diff line number Diff line change
@@ -322,15 +322,14 @@ struct controller_impl {
auto start = fc::time_point::now();
while( auto next = blog.read_block_by_num( head->block_num + 1 ) ) {
replay_push_block( next, controller::block_status::irreversible );
if( next->block_num() % 100 == 0 ) {
std::cerr << std::setw(10) << next->block_num() << " of " << blog_head->block_num() <<"\r";
if( next->block_num() % 500 == 0 ) {
ilog( "${n} of ${head}", ("n", next->block_num())("head", blog_head->block_num()) );
if( shutdown() ) break;
}
}
std::cerr<< "\n";
ilog( "${n} blocks replayed", ("n", head->block_num - start_block_num) );

// if the irreverible log is played without undo sessions enabled, we need to sync the
// if the irreversible log is played without undo sessions enabled, we need to sync the
// revision ordinal to the appropriate expected value here.
if( self.skip_db_sessions( controller::block_status::irreversible ) )
db.set_revision(head->block_num);
@@ -389,7 +388,7 @@ struct controller_impl {
report_integrity_hash = true;
}
}

if( shutdown() ) return;

const auto& ubi = reversible_blocks.get_index<reversible_block_index,by_num>();
@@ -921,17 +920,14 @@ struct controller_impl {
void commit_block( bool add_to_fork_db ) {
auto reset_pending_on_exit = fc::make_scoped_exit([this]{
pending.reset();

});

try {

if (add_to_fork_db) {
pending->_pending_block_state->validated = true;

auto new_bsp = fork_db.add(pending->_pending_block_state, true, pbft_enabled);
emit(self.accepted_block_header, pending->_pending_block_state);

head = fork_db.head();
EOS_ASSERT(new_bsp == head, fork_database_exception, "committed block did not become the new head in fork database");
}
@@ -1381,7 +1377,6 @@ struct controller_impl {

pending->_pending_block_state->set_confirmed(confirm_block_count, pbft_enabled);


auto was_pending_promoted = pending->_pending_block_state->maybe_promote_pending(pbft_enabled);

//modify state in speculative block only if we are speculative reads mode (other wise we need clean state for head or irreversible reads)
4 changes: 2 additions & 2 deletions libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp
Original file line number Diff line number Diff line change
@@ -759,8 +759,8 @@ namespace eosio { namespace chain { namespace wasm_injections {


struct post_op_injectors : wasm_ops::op_types<pass_injector> {
using loop_t = wasm_ops::loop <checktime_injection>;
using call_t = wasm_ops::call <fix_call_index>;
using loop_t = wasm_ops::loop <checktime_injection>;
using call_t = wasm_ops::call <fix_call_index>;
using grow_memory_t = wasm_ops::grow_memory <checktime_injection>;
};

2 changes: 1 addition & 1 deletion libraries/chain/transaction.cpp
Original file line number Diff line number Diff line change
@@ -213,7 +213,7 @@ static bytes zlib_decompress(const bytes& data) {
bytes out;
bio::filtering_ostream decomp;
decomp.push(bio::zlib_decompressor());
decomp.push(read_limiter<1*1024*1024>()); // limit to 10 megs decompressed for zip bomb protections
decomp.push(read_limiter<1*1024*1024>()); // limit to 1 meg decompressed for zip bomb protections
oldcold marked this conversation as resolved.
Show resolved Hide resolved
decomp.push(bio::back_inserter(out));
bio::write(decomp, data.data(), data.size());
bio::close(decomp);
26 changes: 18 additions & 8 deletions plugins/chain_plugin/chain_plugin.cpp
Original file line number Diff line number Diff line change
@@ -1273,7 +1273,7 @@ uint64_t convert_to_type(const string& str, const string& desc) {
try {
return boost::lexical_cast<uint64_t>(str.c_str(), str.size());
} catch( ... ) { }

try {
auto trimmed_str = str;
boost::trim(trimmed_str);
@@ -1287,7 +1287,7 @@ uint64_t convert_to_type(const string& str, const string& desc) {
return symb.value();
} catch( ... ) { }
}

try {
return ( eosio::chain::string_to_symbol( 0, str.c_str() ) >> 8 );
} catch( ... ) {
@@ -1668,14 +1668,24 @@ read_only::get_scheduled_transactions( const read_only::get_scheduled_transactio

fc::variant read_only::get_block(const read_only::get_block_params& params) const {
signed_block_ptr block;
EOS_ASSERT(!params.block_num_or_id.empty() && params.block_num_or_id.size() <= 64, chain::block_id_type_exception, "Invalid Block number or ID, must be greater than 0 and less than 64 characters" );
optional<uint64_t> block_num;

EOS_ASSERT( !params.block_num_or_id.empty() && params.block_num_or_id.size() <= 64,
chain::block_id_type_exception,
"Invalid Block number or ID, must be greater than 0 and less than 64 characters"
);

try {
block = db.fetch_block_by_id(fc::variant(params.block_num_or_id).as<block_id_type>());
if (!block) {
block = db.fetch_block_by_number(fc::to_uint64(params.block_num_or_id));
}
block_num = fc::to_uint64(params.block_num_or_id);
} catch( ... ) {}

} EOS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: ${block_num_or_id}", ("block_num_or_id", params.block_num_or_id))
if( block_num.valid() ) {
block = db.fetch_block_by_number( *block_num );
} else {
try {
block = db.fetch_block_by_id( fc::variant(params.block_num_or_id).as<block_id_type>() );
} EOS_RETHROW_EXCEPTIONS(chain::block_id_type_exception, "Invalid block ID: ${block_num_or_id}", ("block_num_or_id", params.block_num_or_id))
}

EOS_ASSERT( block, unknown_block_exception, "Could not find block: ${block}", ("block", params.block_num_or_id));

Loading