perf(api): More efficient gas estimation

core/node/api_server/Cargo.toml

@@ -56,6 +56,7 @@ tower-http = { workspace = true, features = ["cors", "metrics"] }
 lru.workspace = true
 
 [dev-dependencies]
+zk_evm_1_5_0.workspace = true
 zksync_node_genesis.workspace = true
 zksync_node_test_utils.workspace = true
 

core/node/api_server/src/execution_sandbox/storage.rs

@@ -8,7 +8,7 @@ use std::{
 use zksync_multivm::interface::storage::ReadStorage;
 use zksync_types::{
     api::state_override::{OverrideState, StateOverride},
-    get_code_key, get_nonce_key,
+    get_code_key, get_known_code_key, get_nonce_key,
     utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance},
     AccountTreeId, StorageKey, StorageValue, H256,
 };
@@ -56,6 +56,10 @@ impl<S: ReadStorage> StorageWithOverrides<S> {
                 let code_key = get_code_key(account);
                 let code_hash = code.hash();
                 self.overridden_slots.insert(code_key, code_hash);
+                // FIXME: is this addition correct?
+                let known_code_key = get_known_code_key(&code_hash);
+                self.overridden_slots
+                    .insert(known_code_key, H256::from_low_u64_be(1));
                 self.store_factory_dep(code_hash, code.clone().into_bytes());
             }
 

core/node/api_server/src/execution_sandbox/tests.rs

@@ -11,15 +11,15 @@ use zksync_node_test_utils::{create_l2_block, prepare_recovery_snapshot};
 use zksync_state::PostgresStorageCaches;
 use zksync_types::{
     api::state_override::{OverrideAccount, StateOverride},
-    fee::Fee,
     fee_model::BatchFeeInput,
-    l2::L2Tx,
-    transaction_request::PaymasterParams,
-    Address, K256PrivateKey, L2ChainId, Nonce, ProtocolVersionId, Transaction, U256,
+    K256PrivateKey, ProtocolVersionId, Transaction, U256,
 };
 
 use super::*;
-use crate::{execution_sandbox::execute::SandboxExecutor, tx_sender::SandboxExecutorOptions};
+use crate::{
+    execution_sandbox::execute::SandboxExecutor, testonly::TestAccount,
+    tx_sender::SandboxExecutorOptions,
+};
 
 #[tokio::test]
 async fn creating_block_args() {
@@ -210,7 +210,11 @@ async fn test_instantiating_vm(connection: Connection<'static, Core>, block_args
     let fee_input = BatchFeeInput::l1_pegged(55, 555);
     let (base_fee, gas_per_pubdata) =
         derive_base_fee_and_gas_per_pubdata(fee_input, ProtocolVersionId::latest().into());
-    let tx = Transaction::from(create_transfer(base_fee, gas_per_pubdata));
+    let tx = Transaction::from(K256PrivateKey::random().create_transfer(
+        0.into(),
+        base_fee,
+        gas_per_pubdata,
+    ));
 
     let (limiter, _) = VmConcurrencyLimiter::new(1);
     let vm_permit = limiter.acquire().await.unwrap();
@@ -229,27 +233,6 @@ async fn test_instantiating_vm(connection: Connection<'static, Core>, block_args
     assert!(!tx_result.result.is_failed(), "{tx_result:#?}");
 }
 
-fn create_transfer(fee_per_gas: u64, gas_per_pubdata: u64) -> L2Tx {
-    let fee = Fee {
-        gas_limit: 200_000.into(),
-        max_fee_per_gas: fee_per_gas.into(),
-        max_priority_fee_per_gas: 0_u64.into(),
-        gas_per_pubdata_limit: gas_per_pubdata.into(),
-    };
-    L2Tx::new_signed(
-        Some(Address::random()),
-        vec![],
-        Nonce(0),
-        fee,
-        U256::zero(),
-        L2ChainId::default(),
-        &K256PrivateKey::random(),
-        vec![],
-        PaymasterParams::default(),
-    )
-    .unwrap()
-}
-
 #[test_casing(2, [false, true])]
 #[tokio::test]
 async fn validating_transaction(set_balance: bool) {
@@ -270,7 +253,7 @@ async fn validating_transaction(set_balance: bool) {
     let fee_input = BatchFeeInput::l1_pegged(55, 555);
     let (base_fee, gas_per_pubdata) =
         derive_base_fee_and_gas_per_pubdata(fee_input, ProtocolVersionId::latest().into());
-    let tx = create_transfer(base_fee, gas_per_pubdata);
+    let tx = K256PrivateKey::random().create_transfer(0.into(), base_fee, gas_per_pubdata);
 
     let (limiter, _) = VmConcurrencyLimiter::new(1);
     let vm_permit = limiter.acquire().await.unwrap();

core/node/api_server/src/execution_sandbox/vm_metrics.rs

@@ -89,8 +89,18 @@ pub(crate) struct SandboxMetrics {
     pub(super) sandbox_execution_permits: Histogram<usize>,
     #[metrics(buckets = Buckets::LATENCIES)]
     submit_tx: Family<SubmitTxStage, Histogram<Duration>>,
+
+    /// Number of iterations necessary to estimate gas for a transaction.
     #[metrics(buckets = Buckets::linear(0.0..=30.0, 3.0))]
     pub estimate_gas_binary_search_iterations: Histogram<usize>,
+    /// Relative difference between the unscaled final gas estimate and the optimized lower bound. Positive if the lower bound
+    /// is (as expected) lower than the final gas estimate.
+    #[metrics(buckets = Buckets::linear(-0.05..=0.15, 0.01))]
+    pub estimate_gas_lower_bound_relative_diff: Histogram<f64>,
+    /// Relative difference between the optimistic gas limit and the unscaled final gas estimate. Positive if the optimistic gas limit
+    /// is (as expected) greater than the final gas estimate.
+    #[metrics(buckets = Buckets::linear(-0.05..=0.15, 0.01))]
+    pub estimate_gas_optimistic_gas_limit_relative_diff: Histogram<f64>,
 }
 
 impl SandboxMetrics {

core/node/api_server/src/lib.rs

@@ -4,5 +4,7 @@
 mod utils;
 pub mod execution_sandbox;
 pub mod healthcheck;
+#[cfg(test)]
+mod testonly;
 pub mod tx_sender;
 pub mod web3;

core/node/api_server/src/testonly.rs

@@ -0,0 +1,230 @@
+//! Test utils shared among multiple modules.
+
+use std::iter;
+
+use zk_evm_1_5_0::zkevm_opcode_defs::decoding::{EncodingModeProduction, VmEncodingMode};
+use zksync_contracts::{
+    get_loadnext_contract, load_contract, read_bytecode,
+    test_contracts::LoadnextContractExecutionParams,
+};
+use zksync_types::{
+    ethabi::Token, fee::Fee, l2::L2Tx, transaction_request::PaymasterParams, Address,
+    K256PrivateKey, L2ChainId, Nonce, H256, U256,
+};
+
+pub(crate) const LOAD_TEST_ADDRESS: Address = Address::repeat_byte(1);
+
+const EXPENSIVE_CONTRACT_PATH: &str =
+    "etc/contracts-test-data/artifacts-zk/contracts/expensive/expensive.sol/Expensive.json";
+pub(crate) const EXPENSIVE_CONTRACT_ADDRESS: Address = Address::repeat_byte(2);
+
+const PRECOMPILES_CONTRACT_PATH: &str =
+    "etc/contracts-test-data/artifacts-zk/contracts/precompiles/precompiles.sol/Precompiles.json";
+pub(crate) const PRECOMPILES_CONTRACT_ADDRESS: Address = Address::repeat_byte(3);
+
+const COUNTER_CONTRACT_PATH: &str =
+    "etc/contracts-test-data/artifacts-zk/contracts/counter/counter.sol/Counter.json";
+pub(crate) const COUNTER_CONTRACT_ADDRESS: Address = Address::repeat_byte(4);
+
+const INFINITE_LOOP_CONTRACT_PATH: &str =
+    "etc/contracts-test-data/artifacts-zk/contracts/infinite/infinite.sol/InfiniteLoop.json";
+pub(crate) const INFINITE_LOOP_CONTRACT_ADDRESS: Address = Address::repeat_byte(5);
+
+pub(crate) fn read_expensive_contract_bytecode() -> Vec<u8> {
+    read_bytecode(EXPENSIVE_CONTRACT_PATH)
+}
+
+pub(crate) fn read_precompiles_contract_bytecode() -> Vec<u8> {
+    read_bytecode(PRECOMPILES_CONTRACT_PATH)
+}
+
+pub(crate) fn read_counter_contract_bytecode() -> Vec<u8> {
+    read_bytecode(COUNTER_CONTRACT_PATH)
+}
+
+pub(crate) fn read_infinite_loop_contract_bytecode() -> Vec<u8> {
+    read_bytecode(INFINITE_LOOP_CONTRACT_PATH)
+}
+
+/// Inflates the provided bytecode by appending the specified amount of NOP instructions at the end.
+pub(crate) fn inflate_bytecode(bytecode: &mut Vec<u8>, nop_count: usize) {
+    bytecode.extend(
+        iter::repeat(EncodingModeProduction::nop_encoding().to_be_bytes())
+            .take(nop_count)
+            .flatten(),
+    );
+}
+
+fn default_fee() -> Fee {
+    Fee {
+        gas_limit: 200_000.into(),
+        max_fee_per_gas: 55.into(),
+        max_priority_fee_per_gas: 0_u64.into(),
+        gas_per_pubdata_limit: 555.into(),
+    }
+}
+
+pub(crate) trait TestAccount {
+    fn create_transfer(&self, value: U256, fee_per_gas: u64, gas_per_pubdata: u64) -> L2Tx {
+        let fee = Fee {
+            gas_limit: 200_000.into(),
+            max_fee_per_gas: fee_per_gas.into(),
+            max_priority_fee_per_gas: 0_u64.into(),
+            gas_per_pubdata_limit: gas_per_pubdata.into(),
+        };
+        self.create_transfer_with_fee(value, fee)
+    }
+
+    fn create_transfer_with_fee(&self, value: U256, fee: Fee) -> L2Tx;
+
+    fn create_load_test_tx(&self, params: LoadnextContractExecutionParams) -> L2Tx;
+
+    fn create_expensive_tx(&self, write_count: usize) -> L2Tx;
+
+    fn create_expensive_cleanup_tx(&self) -> L2Tx;
+
+    fn create_code_oracle_tx(&self, bytecode_hash: H256, expected_keccak_hash: H256) -> L2Tx;
+
+    fn create_reverting_counter_tx(&self) -> L2Tx;
+
+    fn create_infinite_loop_tx(&self) -> L2Tx;
+}
+
+impl TestAccount for K256PrivateKey {
+    fn create_transfer_with_fee(&self, value: U256, fee: Fee) -> L2Tx {
+        L2Tx::new_signed(
+            Some(Address::random()),
+            vec![],
+            Nonce(0),
+            fee,
+            value,
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_load_test_tx(&self, params: LoadnextContractExecutionParams) -> L2Tx {
+        L2Tx::new_signed(
+            Some(LOAD_TEST_ADDRESS),
+            params.to_bytes(),
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            if params.deploys > 0 {
+                get_loadnext_contract().factory_deps
+            } else {
+                vec![]
+            },
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_expensive_tx(&self, write_count: usize) -> L2Tx {
+        let calldata = load_contract(EXPENSIVE_CONTRACT_PATH)
+            .function("expensive")
+            .expect("no `expensive` function in contract")
+            .encode_input(&[Token::Uint(write_count.into())])
+            .expect("failed encoding `expensive` function");
+        L2Tx::new_signed(
+            Some(EXPENSIVE_CONTRACT_ADDRESS),
+            calldata,
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_expensive_cleanup_tx(&self) -> L2Tx {
+        let calldata = load_contract(EXPENSIVE_CONTRACT_PATH)
+            .function("cleanUp")
+            .expect("no `cleanUp` function in contract")
+            .encode_input(&[])
+            .expect("failed encoding `cleanUp` input");
+        L2Tx::new_signed(
+            Some(EXPENSIVE_CONTRACT_ADDRESS),
+            calldata,
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_code_oracle_tx(&self, bytecode_hash: H256, expected_keccak_hash: H256) -> L2Tx {
+        let calldata = load_contract(PRECOMPILES_CONTRACT_PATH)
+            .function("callCodeOracle")
+            .expect("no `callCodeOracle` function")
+            .encode_input(&[
+                Token::FixedBytes(bytecode_hash.0.to_vec()),
+                Token::FixedBytes(expected_keccak_hash.0.to_vec()),
+            ])
+            .expect("failed encoding `callCodeOracle` input");
+        L2Tx::new_signed(
+            Some(PRECOMPILES_CONTRACT_ADDRESS),
+            calldata,
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_reverting_counter_tx(&self) -> L2Tx {
+        let calldata = load_contract(COUNTER_CONTRACT_PATH)
+            .function("incrementWithRevert")
+            .expect("no `incrementWithRevert` function")
+            .encode_input(&[Token::Uint(1.into()), Token::Bool(true)])
+            .expect("failed encoding `incrementWithRevert` input");
+        L2Tx::new_signed(
+            Some(COUNTER_CONTRACT_ADDRESS),
+            calldata,
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+
+    fn create_infinite_loop_tx(&self) -> L2Tx {
+        let calldata = load_contract(INFINITE_LOOP_CONTRACT_PATH)
+            .function("infiniteLoop")
+            .expect("no `infiniteLoop` function")
+            .encode_input(&[])
+            .expect("failed encoding `infiniteLoop` input");
+        L2Tx::new_signed(
+            Some(INFINITE_LOOP_CONTRACT_ADDRESS),
+            calldata,
+            Nonce(0),
+            default_fee(),
+            0.into(),
+            L2ChainId::default(),
+            self,
+            vec![],
+            PaymasterParams::default(),
+        )
+        .unwrap()
+    }
+}