New Transaction Queue implementation (openethereum#8074)

* Implementation of Verifier, Scoring and Ready.

* Queue in progress.

* TransactionPool.

* Prepare for txpool release.

* Miner refactor [WiP]

* WiP reworking miner.

* Make it compile.

* Add some docs.

* Split blockchain access to a separate file.

* Work on miner API.

* Fix ethcore tests.

* Refactor miner interface for sealing/work packages.

* Implement next nonce.

* RPC compiles.

* Implement couple of missing methdods for RPC.

* Add transaction queue listeners.

* Compiles!

* Clean-up and parallelize.

* Get rid of RefCell in header.

* Revert "Get rid of RefCell in header."

This reverts commit 0f2424c.

* Override Sync requirement.

* Fix status display.

* Unify logging.

* Extract some cheap checks.

* Measurements and optimizations.

* Fix scoring bug, heap size of bug and add cache

* Disable tx queueing and parallel verification.

* Make ethcore and ethcore-miner compile again.

* Make RPC compile again.

* Bunch of txpool tests.

* Migrate transaction queue tests.

* Nonce Cap

* Nonce cap cache and tests.

* Remove stale future transactions from the queue.

* Optimize scoring and write some tests.

* Simple penalization.

* Clean up and support for different scoring algorithms.

* Add CLI parameters for the new queue.

* Remove banning queue.

* Disable debug build.

* Change per_sender limit to be 1% instead of 5%

* Avoid cloning when propagating transactions.

* Remove old todo.

* Post-review fixes.

* Fix miner options default.

* Implement back ready transactions for light client.

* Get rid of from_pending_block

* Pass rejection reason.

* Add more details to drop.

* Rollback heap size of.

* Avoid cloning hashes when propagating and include more details on rejection.

* Fix tests.

* Introduce nonces cache.

* Remove uneccessary hashes allocation.

* Lower the mem limit.

* Re-enable parallel verification.

* Add miner log. Don't check the type if not below min_gas_price.

* Add more traces, fix disabling miner.

* Fix creating pending blocks twice on AuRa authorities.

* Fix tests.

* re-use pending blocks in AuRa

* Use reseal_min_period to prevent too frequent update_sealing.

* Fix log to contain hash not sender.

* Optimize local transactions.

* Fix aura tests.

* Update locks comments.

* Get rid of unsafe Sync impl.

* Review fixes.

* Remove excessive matches.

* Fix compilation errors.

* Use new pool in private transactions.

* Fix private-tx test.

* Fix secret store tests.

* Actually use gas_floor_target

* Fix config tests.

* Fix pool tests.

* Address grumbles.

ethcore/Cargo.toml

@@ -34,7 +34,6 @@ ethjson = { path = "../json" }
 ethkey = { path = "../ethkey" }
 ethstore = { path = "../ethstore" }
 evm = { path = "evm" }
-futures-cpupool = "0.1"
 hardware-wallet = { path = "../hw" }
 heapsize = "0.4"
 itertools = "0.5"
@@ -45,7 +44,6 @@ num = { version = "0.1", default-features = false, features = ["bigint"] }
 num_cpus = "1.2"
 parity-machine = { path = "../machine" }
 parking_lot = "0.5"
-price-info = { path = "../price-info" }
 rayon = "1.0"
 rand = "0.4"
 rlp = { path = "../util/rlp" }
@@ -63,7 +61,6 @@ rustc-hex = "1.0"
 stats = { path = "../util/stats" }
 trace-time = { path = "../util/trace-time" }
 using_queue = { path = "../util/using_queue" }
-table = { path = "../util/table" }
 vm = { path = "vm" }
 wasm = { path = "wasm" }
 keccak-hash = { path = "../util/hash" }
@@ -76,10 +73,18 @@ tempdir = "0.3"
 trie-standardmap = { path = "../util/trie-standardmap" }
 
 [features]
+# Display EVM debug traces.
 evm-debug = ["slow-blocks"]
+# Display EVM debug traces when running tests.
 evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"]
-slow-blocks = [] # Use SLOW_TX_DURATION="50" (compile time!) to track transactions over 50ms
+# Measure time of transaction execution.
+# Whenever the transaction execution time (in millis) exceeds the value of
+# SLOW_TX_DURATION env variable (provided compile time!)
+# EVM debug traces are printed.
+slow-blocks = []
+# Run JSON consensus tests.
 json-tests = ["ethcore-transaction/json-tests"]
+# Run memory/cpu heavy tests.
 test-heavy = []
-default = []
+# Compile benches
 benches = []

ethcore/light/src/provider.rs

@@ -282,6 +282,9 @@ impl<T: ProvingBlockChainClient + ?Sized> Provider for T {
 
 	fn ready_transactions(&self) -> Vec<PendingTransaction> {
 		BlockChainClient::ready_transactions(self)
+			.into_iter()
+			.map(|tx| tx.pending().clone())
+			.collect()
 	}
 
 	fn epoch_signal(&self, req: request::CompleteSignalRequest) -> Option<request::SignalResponse> {

ethcore/light/src/transaction_queue.rs

@@ -120,6 +120,18 @@ impl AccountTransactions {
 	}
 }
 
+/// Transaction import result.
+pub enum ImportDestination {
+	/// Transaction has been imported to the current queue.
+	///
+	/// It's going to be propagated to peers.
+	Current,
+	/// Transaction has been imported to future queue.
+	///
+	/// It means it won't be propagated until the gap is filled.
+	Future,
+}
+
 type Listener = Box<Fn(&[H256]) + Send + Sync>;
 
 /// Light transaction queue. See module docs for more details.
@@ -142,7 +154,7 @@ impl fmt::Debug for TransactionQueue {
 
 impl TransactionQueue {
 	/// Import a pending transaction to be queued.
-	pub fn import(&mut self, tx: PendingTransaction) -> Result<transaction::ImportResult, transaction::Error>  {
+	pub fn import(&mut self, tx: PendingTransaction) -> Result<ImportDestination, transaction::Error>  {
 		let sender = tx.sender();
 		let hash = tx.hash();
 		let nonce = tx.nonce;
@@ -158,7 +170,7 @@ impl TransactionQueue {
 					future: BTreeMap::new(),
 				});
 
-				(transaction::ImportResult::Current, vec![hash])
+				(ImportDestination::Current, vec![hash])
 			}
 			Entry::Occupied(mut entry) => {
 				let acct_txs = entry.get_mut();
@@ -180,7 +192,7 @@ impl TransactionQueue {
 						let old = ::std::mem::replace(&mut acct_txs.current[idx], tx_info);
 						self.by_hash.remove(&old.hash);
 
-						(transaction::ImportResult::Current, vec![hash])
+						(ImportDestination::Current, vec![hash])
 					}
 					Err(idx) => {
 						let cur_len = acct_txs.current.len();
@@ -202,13 +214,13 @@ impl TransactionQueue {
 								acct_txs.future.insert(future_nonce, future);
 							}
 
-							(transaction::ImportResult::Current, vec![hash])
+							(ImportDestination::Current, vec![hash])
 						} else if idx == cur_len && acct_txs.current.last().map_or(false, |f| f.nonce + 1.into() != nonce) {
 							trace!(target: "txqueue", "Queued future transaction for {}, nonce={}", sender, nonce);
 							let future_nonce = nonce;
 							acct_txs.future.insert(future_nonce, tx_info);
 
-							(transaction::ImportResult::Future, vec![])
+							(ImportDestination::Future, vec![])
 						} else {
 							trace!(target: "txqueue", "Queued current transaction for {}, nonce={}", sender, nonce);
 
@@ -217,7 +229,7 @@ impl TransactionQueue {
 							let mut promoted = acct_txs.adjust_future();
 							promoted.insert(0, hash);
 
-							(transaction::ImportResult::Current, promoted)
+							(ImportDestination::Current, promoted)
 						}
 					}
 				}

ethcore/node_filter/src/lib.rs

@@ -132,7 +132,7 @@ mod test {
 			ClientConfig::default(),
 			&spec,
 			client_db,
-			Arc::new(Miner::with_spec(&spec)),
+			Arc::new(Miner::new_for_tests(&spec, None)),
 			IoChannel::disconnected(),
 		).unwrap();
 		let filter = NodeFilter::new(Arc::downgrade(&client) as Weak<BlockChainClient>, contract_addr);