From 12bd1c83d0dfc3355623a9c1c1792ae347d304db Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Wed, 26 Jun 2019 15:23:47 -0500
Subject: [PATCH 01/10] Improve speed for creating index from existing block
 log.

---
 libraries/chain/block_log.cpp                 | 103 +++++++++++++-----
 .../chain/include/eosio/chain/block_log.hpp   |   2 +
 2 files changed, 79 insertions(+), 26 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index fef0dd3f637..e17c6dde2e2 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -317,44 +317,95 @@ namespace eosio { namespace chain {
 
       my->reopen();
 
-      uint64_t end_pos;
+      my->close();
 
-      my->block_stream.seekg(-sizeof( uint64_t), std::ios::end);
-      my->block_stream.read((char*)&end_pos, sizeof(end_pos));
+      block_log::construct_index(my->block_file, my->index_file);
 
-      if( end_pos == npos ) {
-         ilog( "Block log contains no blocks. No need to construct index." );
+      my->reopen();
+   } // construct_index
+
+   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name) {
+
+      uint64_t pos;
+
+      FILE* fin = fopen(block_file_name.generic_string().c_str(), "r");
+      if (fin == nullptr) {
+         elog("cannot read block file ${file}", ("file", block_file_name.string()) );
          return;
       }
 
-      signed_block tmp;
 
-      uint64_t pos = 0;
-      if (my->version == 1) {
-         pos = 4; // Skip version which should have already been checked.
-      } else {
-         pos = 8; // Skip version and first block offset which should have already been checked
+      const auto block_stream_position = ftell(fin);
+      const auto pos_size = sizeof(pos);
+      auto status = fseek(fin, -pos_size, SEEK_END);
+      uint64_t current_pos = ftell(fin);
+      auto size = fread((char*)&pos, pos_size, 1, fin);
+
+      if( pos == npos ) {
+         ilog( "Block log contains no blocks. No need to construct index." );
+         return;
       }
-      my->block_stream.seekg(pos);
 
-      genesis_state gs;
-      fc::raw::unpack(my->block_stream, gs);
+      status = fseek(fin, 0, SEEK_SET);
+      uint32_t version = 0;
+      size = fread((char*)&version, sizeof(version), 1, fin);
+      uint32_t first_block_num;
+      if (version == 1) {
+         first_block_num = 1;
+      }
+      else if (version == 2) {
+         size = fread ( (char*)&first_block_num, sizeof(first_block_num), 1, fin );
+         if (first_block_num < 1) {
+            elog( "Invalid first block number: ${first}", ("first", first_block_num) );
+            return;
+         }
+         status = fseek(fin, sizeof(npos), SEEK_CUR);
+      }
+      else {
+         elog( "Block log does not support version ${version}", ("version", version) );
+         return;
+      }
 
-      // skip the totem
-      if (my->version > 1) {
-         uint64_t totem;
-         my->block_stream.read((char*) &totem, sizeof(totem));
+      const auto start = ftell(fin);
+
+      const auto reverse_file_name = index_file_name.generic_string() + ".rev";
+      FILE* fout_rev = fopen(reverse_file_name.c_str(), "w+");
+      size = fwrite((void*)&pos, pos_size, 1, fout_rev);
+      status = fseek(fin, -pos_size, SEEK_END);
+
+      uint32_t num_blocks = 0;
+      const auto last = current_pos;
+      auto diff = current_pos - pos + pos_size;
+      current_pos -= diff;
+      // subsequent calculations of diff need to account for read which shifts us forward pos_size
+      // and our current_pos doesn't have that pos_size shift, so need to take it back out
+      for ( ; ; diff = current_pos - pos + pos_size, current_pos -= diff ) {
+         status = fseek(fin, -diff, SEEK_CUR);
+         size = fread((char*)&pos, pos_size, 1, fin);
+         if (pos == npos)
+            break;
+         ++num_blocks;
+         size = fwrite((void*)&pos, pos_size, 1, fout_rev);
+         current_pos += pos_size;
       }
 
-      my->index_stream.seekp(0, std::ios::end);
-      while( pos < end_pos ) {
-         fc::raw::unpack(my->block_stream, tmp);
-         my->block_stream.read((char*)&pos, sizeof(pos));
-         if(tmp.block_num() % 1000 == 0)
-            ilog( "Block log index reconstructed for block ${n}", ("n", tmp.block_num()));
-         my->index_stream.write((char*)&pos, sizeof(pos));
+      const uint32_t last_block_num = first_block_num + num_blocks;
+
+      std::fstream  index_stream;
+      index_stream.open(index_file_name.generic_string().c_str(), LOG_WRITE);
+      EOS_ASSERT( index_stream.is_open(), block_log_exception, "Not able to open ${file}", ("file", index_file_name.generic_string()) );
+      index_stream.seekg(0, std::ios::beg);
+      int64_t reverse_offset = pos_size;
+      for (uint32_t block_num = last_block_num; block_num >= first_block_num; --block_num) {
+         status = fseek(fout_rev, -reverse_offset, SEEK_CUR);
+         size = fread((char*)&pos, pos_size, 1, fout_rev);
+         index_stream.write((char*)&pos, pos_size);
+         reverse_offset = pos_size + pos_size; //need to counteract the stream moving forward
       }
-   } // construct_index
+      fclose(fin);
+      fclose(fout_rev);
+      index_stream.close();
+   }
 
    fc::path block_log::repair_log( const fc::path& data_dir, uint32_t truncate_at_block ) {
       ilog("Recovering Block Log...");
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index 26e1dcb41fa..7a61bc9c165 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -69,6 +69,8 @@ namespace eosio { namespace chain {
 
          static genesis_state extract_genesis_state( const fc::path& data_dir );
 
+         static void construct_index(const fc::path& block_log_path, const fc::path& index_file_name);
+
       private:
          void open(const fc::path& data_dir);
          void construct_index();

From 4bc67ec305f6bc44b442a4e7ab519a179011c6e6 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 27 Jun 2019 14:27:14 -0500
Subject: [PATCH 02/10] Changed block log tools --make-index to use common
 index creation code.

---
 programs/eosio-blocklog/main.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 61d9f7c86ed..610c3584a18 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -707,10 +707,17 @@ int main(int argc, char** argv) {
          return 0;
       }
       if (blog.make_index) {
-         bfs::path out_file = "blocks.index";
+         const bfs::path blocks_dir = vmap.at("blocks-dir").as<bfs::path>();
+         bfs::path out_file = blocks_dir / "blocks.index";
+         const bfs::path block_file = blocks_dir / "blocks.log";
+
          if (vmap.count("output-file") > 0)
              out_file = vmap.at("output-file").as<bfs::path>();
-         return make_index(vmap.at("blocks-dir").as<bfs::path>(), out_file);
+
+         report_time rt("making index");
+         block_log::construct_index(block_file.generic_string(), out_file.generic_string());
+         rt.report();
+         return 0;
       }
       //else print blocks.log as JSON
       blog.initialize(vmap);

From 059876a2ec29ea510743540dfbf3b66914d899ed Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 27 Jun 2019 14:29:45 -0500
Subject: [PATCH 03/10] Fixed issue with non-empty existing blocks.index file.

---
 libraries/chain/block_log.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index e17c6dde2e2..acbd0f1ebf7 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -9,6 +9,7 @@
 
 #define LOG_READ  (std::ios::in | std::ios::binary)
 #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app)
+#define LOG_WRITE_NEW (std::ios::out | std::ios::binary)
 #define LOG_RW ( std::ios::in | std::ios::out | std::ios::binary )
 
 namespace eosio { namespace chain {
@@ -392,7 +393,7 @@ namespace eosio { namespace chain {
       const uint32_t last_block_num = first_block_num + num_blocks;
 
       std::fstream  index_stream;
-      index_stream.open(index_file_name.generic_string().c_str(), LOG_WRITE);
+      index_stream.open(index_file_name.generic_string().c_str(), LOG_WRITE_NEW);
       EOS_ASSERT( index_stream.is_open(), block_log_exception, "Not able to open ${file}", ("file", index_file_name.generic_string()) );
       index_stream.seekg(0, std::ios::beg);
       int64_t reverse_offset = pos_size;

From 2a9497230aeea3384740b60f0e8b3242adb74d91 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 27 Jun 2019 14:30:52 -0500
Subject: [PATCH 04/10] Minor (temporary) performance logging.

---
 libraries/chain/block_log.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index acbd0f1ebf7..5a7c11396dd 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -347,6 +347,7 @@ namespace eosio { namespace chain {
          return;
       }
 
+      ilog("start block index creation");
       status = fseek(fin, 0, SEEK_SET);
       uint32_t version = 0;
       size = fread((char*)&version, sizeof(version), 1, fin);
@@ -391,6 +392,7 @@ namespace eosio { namespace chain {
       }
 
       const uint32_t last_block_num = first_block_num + num_blocks;
+      ilog("reverse order block index creation");
 
       std::fstream  index_stream;
       index_stream.open(index_file_name.generic_string().c_str(), LOG_WRITE_NEW);
@@ -406,6 +408,7 @@ namespace eosio { namespace chain {
       fclose(fin);
       fclose(fout_rev);
       index_stream.close();
+      ilog("stop block index creation");
    }
 
    fc::path block_log::repair_log( const fc::path& data_dir, uint32_t truncate_at_block ) {

From 5c8bd0e8488bbcbf6a05c4447aeb10950077cd81 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 8 Jul 2019 10:01:20 -0500
Subject: [PATCH 05/10] Added classes to process reading in blocks and writting
 them out from memory buffers and periodically read/written to file.

---
 libraries/chain/block_log.cpp                 | 347 ++++++++++++++----
 .../chain/include/eosio/chain/block_log.hpp   |   2 +-
 programs/eosio-blocklog/main.cpp              |   4 +-
 3 files changed, 274 insertions(+), 79 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 5a7c11396dd..a8b2f91a878 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -5,6 +5,7 @@
 #include <eosio/chain/block_log.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <fstream>
+#include <fc/bitutil.hpp>
 #include <fc/io/raw.hpp>
 
 #define LOG_READ  (std::ios::in | std::ios::binary)
@@ -69,6 +70,63 @@ namespace eosio { namespace chain {
 
          open_files = true;
       }
+
+      class reverse_iterator {
+      public:
+         reverse_iterator();
+         ~reverse_iterator();
+         // open a block log file and return the total number of blocks in it
+         uint32_t open(const fc::path& block_file_name);
+         uint64_t previous();
+         uint32_t version() const { return _version; }
+         uint32_t first_block_num() const { return _first_block_num; }
+      private:
+         void update_buffer();
+
+         FILE*                          _file                             = nullptr;
+         uint32_t                       _version                          = 0;
+         uint32_t                       _first_block_num                  = 0;
+         uint32_t                       _last_block_num                   = 0;
+         uint32_t                       _blocks_found                     = 0;
+         uint32_t                       _blocks_expected                  = 0;
+         uint64_t                       _current_position_in_file         = 0;
+         uint64_t                       _eof_position_in_file             = 0;
+         uint64_t                       _end_of_buffer_position           = _unset_position;
+         uint64_t                       _start_of_buffer_position         = 0;
+         std::unique_ptr<char[]>        _buffer_ptr;
+         std::string                    _block_file_name;
+         constexpr static int64_t       _unset_position                   = -1;
+         constexpr static uint32_t      _buf_len                          = 1U << 24;
+         constexpr static uint64_t      _position_size                    = sizeof(_current_position_in_file);
+         constexpr static int           _blknum_offset_from_pos           = 14; //offset from start of block to 4 byte block number, valid for the only allowed versions (1 & 2)
+      };
+
+      constexpr uint64_t buffer_location_to_file_location(uint32_t buffer_location) { return buffer_location << 3; }
+      constexpr uint32_t file_location_to_buffer_location(uint32_t file_location) { return file_location >> 3; }
+
+      class index_writer {
+      public:
+         index_writer(const fc::path& block_index_name, uint32_t blocks_expected, bool report_status);
+         ~index_writer();
+         void write(uint64_t pos);
+         void complete();
+         void calculate_buffer_position();
+      private:
+         void prepare_buffer();
+         bool shift_buffer();
+
+         FILE*                              _file              = nullptr;
+         const std::string                  _block_index_name;
+         const uint32_t                     _blocks_expected;
+         uint32_t                           _block_written;
+         bool                               _report_status;
+         std::unique_ptr<uint64_t[]>        _buffer_ptr;
+         int64_t                            _current_position;
+         int64_t                            _start_of_buffer_position;
+         int64_t                            _end_of_buffer_position;
+         constexpr static uint64_t          _buffer_bytes      = 1U << 22;
+         constexpr static uint64_t          _max_buffer_length = file_location_to_buffer_location(_buffer_bytes);
+      };
    }
 
    block_log::block_log(const fc::path& data_dir)
@@ -318,6 +376,7 @@ namespace eosio { namespace chain {
 
       my->reopen();
 
+
       my->close();
 
       block_log::construct_index(my->block_file, my->index_file);
@@ -325,90 +384,27 @@ namespace eosio { namespace chain {
       my->reopen();
    } // construct_index
 
-   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name) {
-
-      uint64_t pos;
-
-      FILE* fin = fopen(block_file_name.generic_string().c_str(), "r");
-      if (fin == nullptr) {
-         elog("cannot read block file ${file}", ("file", block_file_name.string()) );
-         return;
+   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name, bool provide_index_log_status) {
+      detail::reverse_iterator block_log_iter;
+      if (provide_index_log_status) {
+         std::cout << "\nblock log version= " << block_log_iter.version() << '\n';
       }
 
-
-      const auto block_stream_position = ftell(fin);
-      const auto pos_size = sizeof(pos);
-      auto status = fseek(fin, -pos_size, SEEK_END);
-      uint64_t current_pos = ftell(fin);
-      auto size = fread((char*)&pos, pos_size, 1, fin);
-
-      if( pos == npos ) {
-         ilog( "Block log contains no blocks. No need to construct index." );
+      const uint32_t num_blocks = block_log_iter.open(block_file_name);
+      if (num_blocks == 0) {
          return;
       }
 
-      ilog("start block index creation");
-      status = fseek(fin, 0, SEEK_SET);
-      uint32_t version = 0;
-      size = fread((char*)&version, sizeof(version), 1, fin);
-      uint32_t first_block_num;
-      if (version == 1) {
-         first_block_num = 1;
-      }
-      else if (version == 2) {
-         size = fread ( (char*)&first_block_num, sizeof(first_block_num), 1, fin );
-         if (first_block_num < 1) {
-            elog( "Invalid first block number: ${first}", ("first", first_block_num) );
-            return;
-         }
-         status = fseek(fin, sizeof(npos), SEEK_CUR);
-      }
-      else {
-         elog( "Block log does not support version ${version}", ("version", version) );
-         return;
+      if (provide_index_log_status) {
+         std::cout << "\nfirst block= " << block_log_iter.first_block_num() << ", last block= " << (block_log_iter.first_block_num() + num_blocks) << "\n\n";
       }
 
-      const auto start = ftell(fin);
-
-      const auto reverse_file_name = index_file_name.generic_string() + ".rev";
-      FILE* fout_rev = fopen(reverse_file_name.c_str(), "w+");
-      size = fwrite((void*)&pos, pos_size, 1, fout_rev);
-      status = fseek(fin, -pos_size, SEEK_END);
-
-      uint32_t num_blocks = 0;
-      const auto last = current_pos;
-      auto diff = current_pos - pos + pos_size;
-      current_pos -= diff;
-      // subsequent calculations of diff need to account for read which shifts us forward pos_size
-      // and our current_pos doesn't have that pos_size shift, so need to take it back out
-      for ( ; ; diff = current_pos - pos + pos_size, current_pos -= diff ) {
-         status = fseek(fin, -diff, SEEK_CUR);
-         size = fread((char*)&pos, pos_size, 1, fin);
-         if (pos == npos)
-            break;
-         ++num_blocks;
-         size = fwrite((void*)&pos, pos_size, 1, fout_rev);
-         current_pos += pos_size;
-      }
-
-      const uint32_t last_block_num = first_block_num + num_blocks;
-      ilog("reverse order block index creation");
-
-      std::fstream  index_stream;
-      index_stream.open(index_file_name.generic_string().c_str(), LOG_WRITE_NEW);
-      EOS_ASSERT( index_stream.is_open(), block_log_exception, "Not able to open ${file}", ("file", index_file_name.generic_string()) );
-      index_stream.seekg(0, std::ios::beg);
-      int64_t reverse_offset = pos_size;
-      for (uint32_t block_num = last_block_num; block_num >= first_block_num; --block_num) {
-         status = fseek(fout_rev, -reverse_offset, SEEK_CUR);
-         size = fread((char*)&pos, pos_size, 1, fout_rev);
-         index_stream.write((char*)&pos, pos_size);
-         reverse_offset = pos_size + pos_size; //need to counteract the stream moving forward
-      }
-      fclose(fin);
-      fclose(fout_rev);
-      index_stream.close();
-      ilog("stop block index creation");
+      detail::index_writer index(index_file_name, num_blocks, provide_index_log_status);
+      uint64_t position;
+      while ((position = block_log_iter.previous()) != npos) {
+         index.write(position);
+      }
+      index.complete();
    }
 
    fc::path block_log::repair_log( const fc::path& data_dir, uint32_t truncate_at_block ) {
@@ -598,4 +594,201 @@ namespace eosio { namespace chain {
       return gs;
    }
 
+   detail::reverse_iterator::reverse_iterator()
+   : _buffer_ptr(std::make_unique<char[]>(_buf_len)) {
+   }
+
+   detail::reverse_iterator::~reverse_iterator() {
+      if (_file != nullptr) {
+         fclose(_file);
+      }
+   }
+
+   uint32_t detail::reverse_iterator::open(const fc::path& block_file_name) {
+      _block_file_name = block_file_name.generic_string();
+      _file = fopen(_block_file_name.c_str(), "r");
+      EOS_ASSERT( _file != nullptr, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
+      _end_of_buffer_position = _unset_position;
+
+      //read block log to see if version 1 or 2 and get first blocknum (implicit 1 if version 1)
+      _version = 0;
+      auto size = fread((char*)&_version, sizeof(_version), 1, _file);
+      EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("file", _block_file_name) );
+      EOS_ASSERT( _version == 1 || _version == 2, block_log_unsupported_version, "block log version ${v} is not supported", ("v", _version));
+      if (_version == 1) {
+         _first_block_num = 1;
+      }
+      else {
+         size = fread((char*)&_first_block_num, sizeof(_first_block_num), 1, _file);
+         EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' not formatted consistently with version ${v}.", ("file", _block_file_name)("v", _version) );
+      }
+
+      auto status = fseek(_file, 0, SEEK_END);
+      EOS_ASSERT( status == 0, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
+
+      _eof_position_in_file = ftell(_file);
+      EOS_ASSERT( _eof_position_in_file > 0, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("blocks_log", _block_file_name) );
+      _current_position_in_file = _eof_position_in_file - _position_size;
+
+      update_buffer();
+
+      _blocks_found = 0;
+      char* buf = _buffer_ptr.get();
+      const uint32_t index_of_pos = _current_position_in_file - _start_of_buffer_position;
+      const uint64_t block_pos = *reinterpret_cast<uint64_t*>(buf + index_of_pos);
+      uint32_t bnum;
+
+      if (block_pos == block_log::npos) {
+         return 0;
+      }
+
+      if (block_pos >= _start_of_buffer_position) {
+         const uint32_t index_of_block = block_pos - _start_of_buffer_position;
+         bnum = *reinterpret_cast<uint32_t*>(buf + index_of_block + _blknum_offset_from_pos);  //block number of previous block (is big endian)
+      }
+      else {
+         const auto blknum_offset_pos = block_pos + _blknum_offset_from_pos;
+         auto status = fseek(_file, blknum_offset_pos, SEEK_SET);
+         EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
+         auto size = fread((void*)&bnum, sizeof(bnum), 1, _file);
+         EOS_ASSERT( size == 1, block_log_exception, "Could not read in '${blocks_log}' at position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
+      }
+      _last_block_num = fc::endian_reverse_u32(bnum) + 1;                     //convert from big endian to little endian and add 1
+      _blocks_expected = _last_block_num - _first_block_num + 1;
+      return _blocks_expected;
+   }
+
+   uint64_t detail::reverse_iterator::previous() {
+      EOS_ASSERT( _current_position_in_file != block_log::npos,
+                  block_log_exception,
+                  "Block log file at '${blocks_log}' first block already returned by former call to previous(), it is no longer valid to call this function.", ("blocks_log", _block_file_name) );
+
+      if (_start_of_buffer_position > _current_position_in_file) {
+         update_buffer();
+      }
+
+      char* buf = _buffer_ptr.get();
+      auto offset = _current_position_in_file - _start_of_buffer_position;
+      uint64_t block_location_in_file = *reinterpret_cast<uint64_t*>(buf + offset);
+
+      ++_blocks_found;
+      if (block_location_in_file == block_log::npos) {
+         _current_position_in_file = block_location_in_file;
+         EOS_ASSERT( _blocks_found != _blocks_expected,
+                    block_log_exception,
+                    "Block log file at '${blocks_log}' formatting indicated last block: ${last_block_num}, first block: ${first_block_num}, but found ${num} blocks",
+                    ("blocks_log", _block_file_name)("last_block_num", _last_block_num)("first_block_num", _first_block_num)("num", _blocks_found) );
+      }
+      else {
+         const uint64_t previous_position_in_file = _current_position_in_file;
+         _current_position_in_file = block_location_in_file - _position_size;
+         EOS_ASSERT( _current_position_in_file < previous_position_in_file,
+                     block_log_exception,
+                     "Block log file at '${blocks_log}' formatting is incorrect, indicates position later location in file: ${pos}, which was retrieved at: ${orig_pos}.",
+                     ("blocks_log", _block_file_name)("pos", _current_position_in_file)("orig_pos", previous_position_in_file) );
+         if (_version == 1 && _blocks_found != _blocks_expected) {
+            _current_position_in_file = block_location_in_file = block_log::npos;
+         }
+      }
+
+      return block_location_in_file;
+   }
+
+   void detail::reverse_iterator::update_buffer() {
+      EOS_ASSERT( _current_position_in_file != block_log::npos, block_log_exception, "Block log file not setup properly" );
+
+      // since we need to read in a new section, just need to ensure the next position is at the very end of the buffer
+      _end_of_buffer_position = _current_position_in_file + _position_size;
+      if (_end_of_buffer_position < _buf_len) {
+         _start_of_buffer_position = 0;
+      }
+      else {
+         _start_of_buffer_position = _end_of_buffer_position - _buf_len;
+      }
+
+      auto status = fseek(_file, _start_of_buffer_position, SEEK_SET);
+      EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", _start_of_buffer_position) );
+      char* buf = _buffer_ptr.get();
+      auto size = fread((void*)buf, (_end_of_buffer_position - _start_of_buffer_position), 1, _file);//read tail of blocks.log file into buf
+      EOS_ASSERT( size == 1, block_log_exception, "blocks.log read fails" );
+   }
+
+   detail::index_writer::index_writer(const fc::path& block_index_name, uint32_t blocks_expected, bool report_status)
+   : _block_index_name(block_index_name.generic_string())
+   , _blocks_expected(blocks_expected)
+   , _block_written(blocks_expected)
+   , _report_status(report_status)
+   , _buffer_ptr(std::make_unique<uint64_t[]>(_max_buffer_length)) {
+   }
+
+   detail::index_writer::~index_writer() {
+      if (_file != nullptr) {
+         fclose(_file);
+      }
+   }
+
+   void detail::index_writer::write(uint64_t pos) {
+      prepare_buffer();
+      uint64_t* buffer = _buffer_ptr.get();
+      buffer[_current_position - _start_of_buffer_position] = pos;
+      --_current_position;
+      if (_report_status && (_block_written & 0xfffff) == 0) {                            //periodically print a progress indicator
+         std::cout << "block: " << std::setw(10) << _block_written << "    position in file " << std::setw(14) << pos << '\n';
+      }
+      --_block_written;
+   }
+
+   void detail::index_writer::prepare_buffer() {
+      if (_file == nullptr) {
+         _file = fopen(_block_index_name.c_str(), "w");
+         EOS_ASSERT( _file != nullptr, block_log_exception, "Could not open Block index file at '${blocks_index}'", ("blocks_index", _block_index_name) );
+         // allocate 8 bytes for each block position to store
+         const auto full_file_size = buffer_location_to_file_location(_blocks_expected);
+         auto status = fseek(_file, full_file_size, SEEK_SET);
+         EOS_ASSERT( status == 0, block_log_exception, "Could not allocate in '${blocks_index}' storage for all the blocks, size: ${size}", ("blocks_index", _block_index_name)("size", full_file_size) );
+         const auto block_end = file_location_to_buffer_location(full_file_size);
+         _current_position = block_end - 1;
+         calculate_buffer_position();
+      }
+
+      shift_buffer();
+   }
+
+   bool detail::index_writer::shift_buffer() {
+      if (_current_position >= _start_of_buffer_position) {
+         return false;
+      }
+
+      const auto file_location_start = buffer_location_to_file_location(_start_of_buffer_position);
+
+      auto status = fseek(_file, file_location_start, SEEK_SET);
+      EOS_ASSERT( status == 0, block_log_exception, "Could not navigate in '${blocks_index}' file_location_start: ${loc}, _start_of_buffer_position: ${_start_of_buffer_position}", ("blocks_index", _block_index_name)("loc", file_location_start)("_start_of_buffer_position",_start_of_buffer_position) );
+
+      const auto buffer_size = _end_of_buffer_position - _start_of_buffer_position;
+      const auto file_size = buffer_location_to_file_location(buffer_size);
+      uint64_t* buf = _buffer_ptr.get();
+      auto size = fwrite((void*)buf, file_size, 1, _file);
+      EOS_ASSERT( size == 1, block_log_exception, "Writing Block Index file '${file}' failed at location: ${loc}", ("file", _block_index_name)("loc", file_location_start) );
+      calculate_buffer_position();
+      return true;
+   }
+
+   void detail::index_writer::complete() {
+      const bool shifted = shift_buffer();
+      EOS_ASSERT(shifted, block_log_exception, "Failed to write buffer to '${blocks_index}'", ("blocks_index", _block_index_name) );
+      EOS_ASSERT(_current_position == -1,
+                 block_log_exception,
+                 "Should have written buffer, starting at the 0 index block position, to '${blocks_index}' but instead writing ${pos} position",
+                 ("blocks_index", _block_index_name)("pos", _current_position) );
+   }
+
+   void detail::index_writer::calculate_buffer_position() {
+      _end_of_buffer_position = _current_position + 1;
+      if (_end_of_buffer_position < _max_buffer_length) {
+         _start_of_buffer_position = 0;
+      }
+      else {
+         _start_of_buffer_position = _end_of_buffer_position - _max_buffer_length;
+      }
+   }
 } } /// eosio::chain
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index 7a61bc9c165..0c4ca853828 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -69,7 +69,7 @@ namespace eosio { namespace chain {
 
          static genesis_state extract_genesis_state( const fc::path& data_dir );
 
-         static void construct_index(const fc::path& block_log_path, const fc::path& index_file_name);
+         static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name, bool provide_index_log_status = false);
 
       private:
          void open(const fc::path& data_dir);
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 610c3584a18..19540142f47 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -715,7 +715,9 @@ int main(int argc, char** argv) {
              out_file = vmap.at("output-file").as<bfs::path>();
 
          report_time rt("making index");
-         block_log::construct_index(block_file.generic_string(), out_file.generic_string());
+         std::cout << "\nWill read existing blocks.log file " << block_file << '\n';
+         std::cout << "Will write new blocks.index file " << out_file << '\n';
+         block_log::construct_index(block_file.generic_string(), out_file.generic_string(), true);
          rt.report();
          return 0;
       }

From 1afee433ef1d5b4e50609e6a78641fe8dce7b180 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 8 Jul 2019 10:49:44 -0500
Subject: [PATCH 06/10] Fixed error with reporting version before it was read.

---
 libraries/chain/block_log.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index a8b2f91a878..83f94355a1b 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -386,11 +386,12 @@ namespace eosio { namespace chain {
 
    void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name, bool provide_index_log_status) {
       detail::reverse_iterator block_log_iter;
+
+      const uint32_t num_blocks = block_log_iter.open(block_file_name);
       if (provide_index_log_status) {
          std::cout << "\nblock log version= " << block_log_iter.version() << '\n';
       }
 
-      const uint32_t num_blocks = block_log_iter.open(block_file_name);
       if (num_blocks == 0) {
          return;
       }

From ce2fc68e77ef98975b15e8b5008339b7775c1b4e Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Tue, 9 Jul 2019 14:21:06 -0500
Subject: [PATCH 07/10] Cleaned up logging and added unique_ptr for PR
 comments.

---
 libraries/chain/block_log.cpp                 | 101 ++++++++----------
 .../chain/include/eosio/chain/block_log.hpp   |   2 +-
 programs/eosio-blocklog/main.cpp              |   7 +-
 3 files changed, 50 insertions(+), 60 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 83f94355a1b..db388f24754 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -26,6 +26,8 @@ namespace eosio { namespace chain {
    const uint32_t block_log::max_supported_version = 2;
 
    namespace detail {
+      using unique_file = std::unique_ptr<FILE, decltype(&fclose)>;
+
       class block_log_impl {
          public:
             signed_block_ptr         head;
@@ -74,7 +76,6 @@ namespace eosio { namespace chain {
       class reverse_iterator {
       public:
          reverse_iterator();
-         ~reverse_iterator();
          // open a block log file and return the total number of blocks in it
          uint32_t open(const fc::path& block_file_name);
          uint64_t previous();
@@ -83,7 +84,7 @@ namespace eosio { namespace chain {
       private:
          void update_buffer();
 
-         FILE*                          _file                             = nullptr;
+         unique_file                    _file;
          uint32_t                       _version                          = 0;
          uint32_t                       _first_block_num                  = 0;
          uint32_t                       _last_block_num                   = 0;
@@ -106,26 +107,24 @@ namespace eosio { namespace chain {
 
       class index_writer {
       public:
-         index_writer(const fc::path& block_index_name, uint32_t blocks_expected, bool report_status);
-         ~index_writer();
+         index_writer(const fc::path& block_index_name, uint32_t blocks_expected);
          void write(uint64_t pos);
          void complete();
-         void calculate_buffer_position();
+         void update_buffer_position();
       private:
          void prepare_buffer();
          bool shift_buffer();
 
-         FILE*                              _file              = nullptr;
+         unique_file                        _file;
          const std::string                  _block_index_name;
          const uint32_t                     _blocks_expected;
          uint32_t                           _block_written;
-         bool                               _report_status;
          std::unique_ptr<uint64_t[]>        _buffer_ptr;
-         int64_t                            _current_position;
-         int64_t                            _start_of_buffer_position;
-         int64_t                            _end_of_buffer_position;
-         constexpr static uint64_t          _buffer_bytes      = 1U << 22;
-         constexpr static uint64_t          _max_buffer_length = file_location_to_buffer_location(_buffer_bytes);
+         int64_t                            _current_position         = 0;
+         int64_t                            _start_of_buffer_position = 0;
+         int64_t                            _end_of_buffer_position   = 0;
+         constexpr static uint64_t          _buffer_bytes             = 1U << 22;
+         constexpr static uint64_t          _max_buffer_length        = file_location_to_buffer_location(_buffer_bytes);
       };
    }
 
@@ -384,23 +383,24 @@ namespace eosio { namespace chain {
       my->reopen();
    } // construct_index
 
-   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name, bool provide_index_log_status) {
+   void block_log::construct_index(const fc::path& block_file_name, const fc::path& index_file_name) {
       detail::reverse_iterator block_log_iter;
 
+      ilog("Will read existing blocks.log file ${file}", ("file", block_file_name.generic_string()));
+      ilog("Will write new blocks.index file ${file}", ("file", index_file_name.generic_string()));
+
       const uint32_t num_blocks = block_log_iter.open(block_file_name);
-      if (provide_index_log_status) {
-         std::cout << "\nblock log version= " << block_log_iter.version() << '\n';
-      }
+
+      ilog("block log version= ${version}", ("version", block_log_iter.version()));
 
       if (num_blocks == 0) {
          return;
       }
 
-      if (provide_index_log_status) {
-         std::cout << "\nfirst block= " << block_log_iter.first_block_num() << ", last block= " << (block_log_iter.first_block_num() + num_blocks) << "\n\n";
-      }
+      ilog("first block= ${first}         last block= ${last}",
+           ("first", block_log_iter.first_block_num())("last", (block_log_iter.first_block_num() + num_blocks)));
 
-      detail::index_writer index(index_file_name, num_blocks, provide_index_log_status);
+      detail::index_writer index(index_file_name, num_blocks);
       uint64_t position;
       while ((position = block_log_iter.previous()) != npos) {
          index.write(position);
@@ -596,38 +596,33 @@ namespace eosio { namespace chain {
    }
 
    detail::reverse_iterator::reverse_iterator()
-   : _buffer_ptr(std::make_unique<char[]>(_buf_len)) {
-   }
-
-   detail::reverse_iterator::~reverse_iterator() {
-      if (_file != nullptr) {
-         fclose(_file);
-      }
+   : _file(nullptr, &fclose)
+   , _buffer_ptr(std::make_unique<char[]>(_buf_len)) {
    }
 
    uint32_t detail::reverse_iterator::open(const fc::path& block_file_name) {
       _block_file_name = block_file_name.generic_string();
-      _file = fopen(_block_file_name.c_str(), "r");
-      EOS_ASSERT( _file != nullptr, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
+      _file.reset(fopen(_block_file_name.c_str(), "r"));
+      EOS_ASSERT( _file, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
       _end_of_buffer_position = _unset_position;
 
       //read block log to see if version 1 or 2 and get first blocknum (implicit 1 if version 1)
       _version = 0;
-      auto size = fread((char*)&_version, sizeof(_version), 1, _file);
+      auto size = fread((char*)&_version, sizeof(_version), 1, _file.get());
       EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("file", _block_file_name) );
       EOS_ASSERT( _version == 1 || _version == 2, block_log_unsupported_version, "block log version ${v} is not supported", ("v", _version));
       if (_version == 1) {
          _first_block_num = 1;
       }
       else {
-         size = fread((char*)&_first_block_num, sizeof(_first_block_num), 1, _file);
+         size = fread((char*)&_first_block_num, sizeof(_first_block_num), 1, _file.get());
          EOS_ASSERT( size == 1, block_log_exception, "Block log file at '${blocks_log}' not formatted consistently with version ${v}.", ("file", _block_file_name)("v", _version) );
       }
 
-      auto status = fseek(_file, 0, SEEK_END);
+      auto status = fseek(_file.get(), 0, SEEK_END);
       EOS_ASSERT( status == 0, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
 
-      _eof_position_in_file = ftell(_file);
+      _eof_position_in_file = ftell(_file.get());
       EOS_ASSERT( _eof_position_in_file > 0, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("blocks_log", _block_file_name) );
       _current_position_in_file = _eof_position_in_file - _position_size;
 
@@ -649,9 +644,9 @@ namespace eosio { namespace chain {
       }
       else {
          const auto blknum_offset_pos = block_pos + _blknum_offset_from_pos;
-         auto status = fseek(_file, blknum_offset_pos, SEEK_SET);
+         auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET);
          EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
-         auto size = fread((void*)&bnum, sizeof(bnum), 1, _file);
+         auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get());
          EOS_ASSERT( size == 1, block_log_exception, "Could not read in '${blocks_log}' at position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
       }
       _last_block_num = fc::endian_reverse_u32(bnum) + 1;                     //convert from big endian to little endian and add 1
@@ -707,49 +702,43 @@ namespace eosio { namespace chain {
          _start_of_buffer_position = _end_of_buffer_position - _buf_len;
       }
 
-      auto status = fseek(_file, _start_of_buffer_position, SEEK_SET);
+      auto status = fseek(_file.get(), _start_of_buffer_position, SEEK_SET);
       EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", _start_of_buffer_position) );
       char* buf = _buffer_ptr.get();
-      auto size = fread((void*)buf, (_end_of_buffer_position - _start_of_buffer_position), 1, _file);//read tail of blocks.log file into buf
+      auto size = fread((void*)buf, (_end_of_buffer_position - _start_of_buffer_position), 1, _file.get());//read tail of blocks.log file into buf
       EOS_ASSERT( size == 1, block_log_exception, "blocks.log read fails" );
    }
 
-   detail::index_writer::index_writer(const fc::path& block_index_name, uint32_t blocks_expected, bool report_status)
-   : _block_index_name(block_index_name.generic_string())
+   detail::index_writer::index_writer(const fc::path& block_index_name, uint32_t blocks_expected)
+   : _file(nullptr, &fclose)
+   , _block_index_name(block_index_name.generic_string())
    , _blocks_expected(blocks_expected)
    , _block_written(blocks_expected)
-   , _report_status(report_status)
    , _buffer_ptr(std::make_unique<uint64_t[]>(_max_buffer_length)) {
    }
 
-   detail::index_writer::~index_writer() {
-      if (_file != nullptr) {
-         fclose(_file);
-      }
-   }
-
    void detail::index_writer::write(uint64_t pos) {
       prepare_buffer();
       uint64_t* buffer = _buffer_ptr.get();
       buffer[_current_position - _start_of_buffer_position] = pos;
       --_current_position;
-      if (_report_status && (_block_written & 0xfffff) == 0) {                            //periodically print a progress indicator
-         std::cout << "block: " << std::setw(10) << _block_written << "    position in file " << std::setw(14) << pos << '\n';
+      if ((_block_written & 0xfffff) == 0) {                            //periodically print a progress indicator
+         dlog("block: ${block_written}      position in file: ${pos}", ("block_written", _block_written)("pos",pos));
       }
       --_block_written;
    }
 
    void detail::index_writer::prepare_buffer() {
       if (_file == nullptr) {
-         _file = fopen(_block_index_name.c_str(), "w");
-         EOS_ASSERT( _file != nullptr, block_log_exception, "Could not open Block index file at '${blocks_index}'", ("blocks_index", _block_index_name) );
+         _file.reset(fopen(_block_index_name.c_str(), "w"));
+         EOS_ASSERT( _file, block_log_exception, "Could not open Block index file at '${blocks_index}'", ("blocks_index", _block_index_name) );
          // allocate 8 bytes for each block position to store
          const auto full_file_size = buffer_location_to_file_location(_blocks_expected);
-         auto status = fseek(_file, full_file_size, SEEK_SET);
+         auto status = fseek(_file.get(), full_file_size, SEEK_SET);
          EOS_ASSERT( status == 0, block_log_exception, "Could not allocate in '${blocks_index}' storage for all the blocks, size: ${size}", ("blocks_index", _block_index_name)("size", full_file_size) );
          const auto block_end = file_location_to_buffer_location(full_file_size);
          _current_position = block_end - 1;
-         calculate_buffer_position();
+         update_buffer_position();
       }
 
       shift_buffer();
@@ -762,15 +751,15 @@ namespace eosio { namespace chain {
 
       const auto file_location_start = buffer_location_to_file_location(_start_of_buffer_position);
 
-      auto status = fseek(_file, file_location_start, SEEK_SET);
+      auto status = fseek(_file.get(), file_location_start, SEEK_SET);
       EOS_ASSERT( status == 0, block_log_exception, "Could not navigate in '${blocks_index}' file_location_start: ${loc}, _start_of_buffer_position: ${_start_of_buffer_position}", ("blocks_index", _block_index_name)("loc", file_location_start)("_start_of_buffer_position",_start_of_buffer_position) );
 
       const auto buffer_size = _end_of_buffer_position - _start_of_buffer_position;
       const auto file_size = buffer_location_to_file_location(buffer_size);
       uint64_t* buf = _buffer_ptr.get();
-      auto size = fwrite((void*)buf, file_size, 1, _file);
+      auto size = fwrite((void*)buf, file_size, 1, _file.get());
       EOS_ASSERT( size == 1, block_log_exception, "Writing Block Index file '${file}' failed at location: ${loc}", ("file", _block_index_name)("loc", file_location_start) );
-      calculate_buffer_position();
+      update_buffer_position();
       return true;
    }
 
@@ -783,7 +772,7 @@ namespace eosio { namespace chain {
                  ("blocks_index", _block_index_name)("pos", _current_position) );
    }
 
-   void detail::index_writer::calculate_buffer_position() {
+   void detail::index_writer::update_buffer_position() {
       _end_of_buffer_position = _current_position + 1;
       if (_end_of_buffer_position < _max_buffer_length) {
          _start_of_buffer_position = 0;
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index 0c4ca853828..c0684d566a6 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -69,7 +69,7 @@ namespace eosio { namespace chain {
 
          static genesis_state extract_genesis_state( const fc::path& data_dir );
 
-         static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name, bool provide_index_log_status = false);
+         static void construct_index(const fc::path& block_file_name, const fc::path& index_file_name);
 
       private:
          void open(const fc::path& data_dir);
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 19540142f47..d244e7974e9 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -715,9 +715,10 @@ int main(int argc, char** argv) {
              out_file = vmap.at("output-file").as<bfs::path>();
 
          report_time rt("making index");
-         std::cout << "\nWill read existing blocks.log file " << block_file << '\n';
-         std::cout << "Will write new blocks.index file " << out_file << '\n';
-         block_log::construct_index(block_file.generic_string(), out_file.generic_string(), true);
+         const auto log_level = fc::logger::get(DEFAULT_LOGGER).get_log_level();
+         fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
+         block_log::construct_index(block_file.generic_string(), out_file.generic_string());
+         fc::logger::get(DEFAULT_LOGGER).set_log_level(log_level);
          rt.report();
          return 0;
       }

From 60123231babab22a6610c9512aad0c9c74520373 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Tue, 9 Jul 2019 14:31:10 -0500
Subject: [PATCH 08/10] Added status to log messages and minor edits for PR
 comments.

---
 libraries/chain/block_log.cpp | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index db388f24754..d979b087fbf 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -10,7 +10,6 @@
 
 #define LOG_READ  (std::ios::in | std::ios::binary)
 #define LOG_WRITE (std::ios::out | std::ios::binary | std::ios::app)
-#define LOG_WRITE_NEW (std::ios::out | std::ios::binary)
 #define LOG_RW ( std::ios::in | std::ios::out | std::ios::binary )
 
 namespace eosio { namespace chain {
@@ -620,7 +619,7 @@ namespace eosio { namespace chain {
       }
 
       auto status = fseek(_file.get(), 0, SEEK_END);
-      EOS_ASSERT( status == 0, block_log_exception, "Could not open Block log file at '${blocks_log}'", ("blocks_log", _block_file_name) );
+      EOS_ASSERT( status == 0, block_log_exception, "Could not open Block log file at '${blocks_log}'. Returned status: ${status}", ("blocks_log", _block_file_name)("status", status) );
 
       _eof_position_in_file = ftell(_file.get());
       EOS_ASSERT( _eof_position_in_file > 0, block_log_exception, "Block log file at '${blocks_log}' could not be read.", ("blocks_log", _block_file_name) );
@@ -632,12 +631,12 @@ namespace eosio { namespace chain {
       char* buf = _buffer_ptr.get();
       const uint32_t index_of_pos = _current_position_in_file - _start_of_buffer_position;
       const uint64_t block_pos = *reinterpret_cast<uint64_t*>(buf + index_of_pos);
-      uint32_t bnum;
 
       if (block_pos == block_log::npos) {
          return 0;
       }
 
+      uint32_t bnum = 0;
       if (block_pos >= _start_of_buffer_position) {
          const uint32_t index_of_block = block_pos - _start_of_buffer_position;
          bnum = *reinterpret_cast<uint32_t*>(buf + index_of_block + _blknum_offset_from_pos);  //block number of previous block (is big endian)
@@ -645,7 +644,7 @@ namespace eosio { namespace chain {
       else {
          const auto blknum_offset_pos = block_pos + _blknum_offset_from_pos;
          auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
+         EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos)("status", status) );
          auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get());
          EOS_ASSERT( size == 1, block_log_exception, "Could not read in '${blocks_log}' at position: ${pos}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos) );
       }
@@ -703,7 +702,7 @@ namespace eosio { namespace chain {
       }
 
       auto status = fseek(_file.get(), _start_of_buffer_position, SEEK_SET);
-      EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}", ("blocks_log", _block_file_name)("pos", _start_of_buffer_position) );
+      EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", _start_of_buffer_position)("status", status) );
       char* buf = _buffer_ptr.get();
       auto size = fread((void*)buf, (_end_of_buffer_position - _start_of_buffer_position), 1, _file.get());//read tail of blocks.log file into buf
       EOS_ASSERT( size == 1, block_log_exception, "blocks.log read fails" );
@@ -735,7 +734,7 @@ namespace eosio { namespace chain {
          // allocate 8 bytes for each block position to store
          const auto full_file_size = buffer_location_to_file_location(_blocks_expected);
          auto status = fseek(_file.get(), full_file_size, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "Could not allocate in '${blocks_index}' storage for all the blocks, size: ${size}", ("blocks_index", _block_index_name)("size", full_file_size) );
+         EOS_ASSERT( status == 0, block_log_exception, "Could not allocate in '${blocks_index}' storage for all the blocks, size: ${size}. Returned status: ${status}", ("blocks_index", _block_index_name)("size", full_file_size)("status", status) );
          const auto block_end = file_location_to_buffer_location(full_file_size);
          _current_position = block_end - 1;
          update_buffer_position();
@@ -752,7 +751,7 @@ namespace eosio { namespace chain {
       const auto file_location_start = buffer_location_to_file_location(_start_of_buffer_position);
 
       auto status = fseek(_file.get(), file_location_start, SEEK_SET);
-      EOS_ASSERT( status == 0, block_log_exception, "Could not navigate in '${blocks_index}' file_location_start: ${loc}, _start_of_buffer_position: ${_start_of_buffer_position}", ("blocks_index", _block_index_name)("loc", file_location_start)("_start_of_buffer_position",_start_of_buffer_position) );
+      EOS_ASSERT( status == 0, block_log_exception, "Could not navigate in '${blocks_index}' file_location_start: ${loc}, _start_of_buffer_position: ${_start_of_buffer_position}. Returned status: ${status}", ("blocks_index", _block_index_name)("loc", file_location_start)("_start_of_buffer_position",_start_of_buffer_position)("status", status) );
 
       const auto buffer_size = _end_of_buffer_position - _start_of_buffer_position;
       const auto file_size = buffer_location_to_file_location(buffer_size);

From 393170935acd7dea85107fae3abfc8ecfef46512 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Tue, 9 Jul 2019 15:03:00 -0500
Subject: [PATCH 09/10] Removed ununsed make_index function.

---
 programs/eosio-blocklog/main.cpp | 139 -------------------------------
 1 file changed, 139 deletions(-)

diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index d244e7974e9..37beec44ca0 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -505,145 +505,6 @@ int trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first b
 }
 
 
-int make_index(const bfs::path& block_dir, const bfs::path& out_file) {
-   report_time rt("making index");
-   //this code makes blocks.index much faster than nodeos (in recent test 80 seconds vs. 90 minutes)
-   using namespace std;
-   bfs::path block_file_name = block_dir / "blocks.log";
-   bfs::path out_file_name = block_dir / out_file;
-   cout << '\n';
-   cout << "Will read existing blocks.log file " << block_file_name << '\n';
-   cout << "Will write new blocks.index file " << out_file_name << '\n';
-   FILE* fin = fopen(block_file_name.c_str(), "r");
-   EOS_ASSERT( fin != nullptr, block_log_not_found, "cannot read block file ${file}", ("file", block_file_name.string()) );
-
-   //will read big chunks of blocks.log into buf, will fill fpos_list with file positions before write to blocks.index
-   constexpr uint32_t buf_len{1U << 24};                      //buf_len must be power of 2 >= largest possible block == one MByte
-   auto buffer = make_unique<char[]>(buf_len + 8);            //can write up to 8 bytes past end of buf
-   char* buf = buffer.get();
-   constexpr uint64_t fpos_list_len{1U << 22};                //length of fpos_list[] in bytes
-   auto fpos_buffer = make_unique<uint64_t[]>(fpos_list_len >> 3);
-   uint64_t* fpos_list = fpos_buffer.get();
-
-   //read blocks.log to see if version 1 or 2 and get first_blocknum (implicit 1 if version 1)
-   uint32_t version = 0, first_block = 0;
-   auto size = fread((char*)&version, sizeof(version), 1, fin);
-   EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", block_file_name.string()) );
-   cout << "block log version= " << version << '\n';
-   EOS_ASSERT( version == 1 || version == 2, block_log_unsupported_version, "block log version ${v} is not supported", ("v",version));
-   if (version == 1)
-      first_block = 1;
-   else {
-      size = fread((void*)&first_block, sizeof(first_block), 1, fin);
-      EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", block_file_name.string()) );
-   }
-   cout << "first block= " << first_block << '\n';
-
-   auto status = fseek(fin, 0, SEEK_END);                    //get blocks.log file length
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", block_file_name.string())("pos", 0) );
-   uint64_t pos = ftell(fin);
-   uint64_t last_buf_len = pos & ((uint64_t)buf_len-1);     //buf_len is a power of 2 so -1 creates low bits all 1
-   if (!last_buf_len)                                       //will read integral number of buf_len and one time read last_buf_len
-      last_buf_len = buf_len;
-   status = fseek(fin, -(uint64_t)last_buf_len, SEEK_END);     //one time read last_buf_len
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", block_file_name.string())("pos", last_buf_len) );
-   pos = ftell(fin);
-   uint64_t did_read = fread((void*)buf, last_buf_len, 1, fin);//read tail of blocks.log file into buf
-   EOS_ASSERT( did_read == 1, block_log_exception, "blocks.log read fails" );
-
-   //we traverse linked list of blocks in buf (from end to start), for each block we know this:
-   uint32_t index_start;                                    //buf index for block start
-   uint32_t index_end;                                      //buf index for block end == buf index for block start file position
-   uint64_t file_pos;                                       //file pos of block start
-   uint32_t bnum;                                           //block number
-
-   index_end = last_buf_len - 8;                            //index in buf where last block ends and block file position starts
-   cout << "last_buf_len=" << last_buf_len << " index_end=" << index_end <<  '\n';
-   file_pos = *(uint64_t*)(buf + index_end);                //file pos of block start
-   cout << "file_pos=" << file_pos << " buf=" << (uint64_t)buf <<  '\n';
-   index_start = file_pos - pos;                            //buf index for block start
-   bnum = *(uint32_t*)(buf + index_start + blknum_offset);  //block number of previous block (is big endian)
-   bnum = endian_reverse_u32(bnum) + 1;                     //convert from big endian to little endian and add 1
-   cout << "last block=  " << bnum << '\n';
-   cout << '\n';
-   cout << "block " << setw(10) << bnum << "    file_pos " << setw(14) << file_pos << '\n';  //first progress indicator
-   uint64_t last_file_pos = file_pos;                       //used to check that file_pos is strictly decreasing
-   uint32_t end_block{bnum};                                //save for message at end
-
-   //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-   FILE* fout = fopen(out_file_name.c_str(), "w");
-   EOS_ASSERT( fout != nullptr, block_index_not_found, "cannot write blocks.index" );
-
-   uint64_t ind_file_len = (bnum + 1 - first_block) << 3;            //index file holds 8 bytes for each block in blocks.log
-   uint64_t last_ind_buf_len = ind_file_len & (fpos_list_len - 1); //fpos_list_len is a power of 2 so -1 creates low bits all 1
-   if (!last_ind_buf_len)                                          //will write integral number of buf_len and last_ind_buf_len one time to index file
-      last_ind_buf_len = buf_len;
-   status = fseek(fout, ind_file_len - last_ind_buf_len, SEEK_SET);
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", out_file_name.string())("pos", ind_file_len - last_ind_buf_len) );
-   uint64_t ind_pos = ftell(fout);
-   EOS_ASSERT( ind_pos == ind_file_len - last_ind_buf_len, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", out_file_name.string())("pos", ind_file_len - last_ind_buf_len) );
-   uint64_t blk_base = (ind_pos >> 3) + first_block;              //first entry in fpos_list is for block blk_base
-   cout << "ind_pos= " << ind_pos << "  blk_base= " << blk_base << '\n';
-   fpos_list[bnum - blk_base] = file_pos;                         //write filepos for block bnum
-
-   for (;;) {
-      if (bnum == blk_base) {                                        //if fpos_list is full
-         size = fwrite((void*)fpos_list, last_ind_buf_len, 1, fout); //write fpos_list to index file
-         EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", out_file_name.string()) );
-         if (ind_pos == 0) {                                         //if done writing index file
-            cout << "block " << setw(10) << bnum << "    file_pos " << setw(14) << file_pos << '\n';  //last progress indicator
-            EOS_ASSERT( bnum == first_block, block_log_exception, "blocks.log does not contain consecutive block numbers" );
-            break;
-         }
-         ind_pos -= fpos_list_len;
-         blk_base -= (fpos_list_len >> 3);
-         status = fseek(fout, ind_pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", out_file_name.string())("pos", ind_pos) );
-         did_read = ftell(fout);
-         EOS_ASSERT( did_read == ind_pos, block_log_exception, "${file} seek fails", ("file", out_file_name.string()) );
-         last_ind_buf_len = fpos_list_len;                           //from now on all writes to index file write a full fpos_list[]
-      }
-      if (index_start < 8) {                                //if block start is split across buf boundary
-         memcpy(buf + buf_len, buf, 8);                     //copy portion at start of buf to past end of buf
-         pos -= buf_len;                                    //file position of buf
-         status = fseek(fin, pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", pos) );
-         did_read = fread((void*)buf, buf_len, 1, fin);     //read next buf
-         EOS_ASSERT( did_read == 1, block_log_exception, "blocks.log read fails" );
-         index_start += buf_len;
-      }
-      --bnum;                                               //now move index_start and index_end to prior block
-      index_end = index_start - 8;                          //index in buf where block ends and block file position starts
-      file_pos = *(uint64_t*)(buf + index_end);             //file pos of block start
-      if (file_pos >= last_file_pos) {                      //file_pos will decrease if linked list is not corrupt
-         cout << '\n';
-         cout << "file pos for block " << bnum+1 << " is " << last_file_pos << '\n';
-         cout << "file pos for block " << bnum   << " is " <<      file_pos << '\n';
-         cout << "The linked list of blocks in blocks.log should run from last block to first block in reverse order\n";
-         EOS_ASSERT( file_pos < last_file_pos, block_log_exception, "blocks.log linked list of blocks is corrupt" );
-      }
-      last_file_pos = file_pos;
-      if (file_pos < pos) {                                 //if block start is in prior buf
-         pos -= buf_len;                                    //file position of buf
-         status = fseek(fin, pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", pos) );
-         did_read = fread(buf, buf_len, 1, fin);            //read next buf
-         EOS_ASSERT( did_read == 1, block_log_exception, "blocks.log read fails" );
-         index_end += buf_len;
-      }
-      index_start = file_pos - pos;                          //buf index for block start
-      fpos_list[bnum - blk_base]= file_pos;                  //write filepos for block bnum
-      if ((bnum & 0xfffff) == 0)                             //periodically print a progress indicator
-         cout << "block " << setw(10) << bnum << "    file_pos " << setw(14) << file_pos << '\n';
-   }
-
-   fclose(fout);
-   fclose(fin);
-   cout << "\nwrote " << (end_block+1-first_block) << " file positions to " << out_file_name << '\n';
-   rt.report();
-   return 0;
-}
-
 void smoke_test(bfs::path block_dir) {
    using namespace std;
    cout << "\nSmoke test of blocks.log and blocks.index in directory " << block_dir << '\n';

From cf1143694e3a69fd1045c9e3225b6443addb8be1 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Tue, 9 Jul 2019 16:48:01 -0500
Subject: [PATCH 10/10] Fixed integration test based on changes and fixed
 documentation.

---
 programs/eosio-blocklog/main.cpp | 2 +-
 tests/block_log_util_test.py     | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 37beec44ca0..4e8c42037d7 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -172,7 +172,7 @@ void blocklog::set_program_options(options_description& cli)
          ("as-json-array", bpo::bool_switch(&as_json_array)->default_value(false),
           "Print out json blocks wrapped in json array (otherwise the output is free-standing json objects).")
          ("make-index", bpo::bool_switch(&make_index)->default_value(false),
-          "Create blocks.index from blocks.log. Must give 'blocks-dir'. Give 'output-file' relative to blocks-dir (default is blocks.index).")
+          "Create blocks.index from blocks.log. Must give 'blocks-dir'. Give 'output-file' relative to current directory or absolute path (default is <blocks-dir>/blocks.index).")
          ("trim-blocklog", bpo::bool_switch(&trim_log)->default_value(false),
           "Trim blocks.log and blocks.index. Must give 'blocks-dir' and 'first and/or 'last'.")
          ("smoke-test", bpo::bool_switch(&smoke_test)->default_value(false),
diff --git a/tests/block_log_util_test.py b/tests/block_log_util_test.py
index 073347c4b79..2d0a292b7f2 100755
--- a/tests/block_log_util_test.py
+++ b/tests/block_log_util_test.py
@@ -106,14 +106,13 @@ def checkBlockLog(blockLog, blockNumsToFind, firstBlockNum=1):
     expectedStr="no problems found"
     assert output.find(expectedStr) != -1, "Couldn't find \"%s\" in:\n\"%s\"\n" % (expectedStr, output)
 
-    duplicateIndexFileName="duplicate.index"
+    blockLogDir=Utils.getNodeDataDir(0, "blocks")
+    duplicateIndexFileName=os.path.join(blockLogDir, "duplicate.index")
     output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.make_index, outputFile=duplicateIndexFileName)
     assert output is not None, "Couldn't make new index file \"%s\"\n" % (duplicateIndexFileName)
 
-    blockLogDir=Utils.getNodeDataDir(0, "blocks")
     blockIndexFileName=os.path.join(blockLogDir, "blocks.index")
     blockIndexFile=open(blockIndexFileName,"rb")
-    duplicateIndexFileName=os.path.join(blockLogDir, duplicateIndexFileName)
     duplicateIndexFile=open(duplicateIndexFileName,"rb")
     blockIndexStr=blockIndexFile.read()
     duplicateIndexStr=duplicateIndexFile.read()