fuzz console log & test cases (foundry-rs#8387)

* fuze console log & test cases test fuzz console.log * rename to show_fuzz_logs rename to show_fuzz_logs * add logs field in FuzzTestData add logs field in FuzzTestData add logs field in FuzzTestData * rename to show_logs * removed `decoded_logs` in FuzzTestResult & refactored some code fmt * fmt --------- Co-authored-by: Matthias Seitz <[email protected]>
vlayer-xyz · Jul 9, 2024 · f557626 · f557626
1 parent 687625d
commit f557626
Show file tree

Hide file tree

Showing 7 changed files with 148 additions and 13 deletions.
diff --git a/crates/config/src/fuzz.rs b/crates/config/src/fuzz.rs
@@ -1,7 +1,8 @@
 //! Configuration for fuzz testing.
 
 use crate::inline::{
-    parse_config_u32, InlineConfigParser, InlineConfigParserError, INLINE_CONFIG_FUZZ_KEY,
+    parse_config_bool, parse_config_u32, InlineConfigParser, InlineConfigParserError,
+    INLINE_CONFIG_FUZZ_KEY,
 };
 use alloy_primitives::U256;
 use serde::{Deserialize, Serialize};
@@ -29,6 +30,8 @@ pub struct FuzzConfig {
     pub failure_persist_dir: Option<PathBuf>,
     /// Name of the file to record fuzz failures, defaults to `failures`.
     pub failure_persist_file: Option<String>,
+    /// show `console.log` in fuzz test, defaults to `false`
+    pub show_logs: bool,
 }
 
 impl Default for FuzzConfig {
@@ -41,6 +44,7 @@ impl Default for FuzzConfig {
             gas_report_samples: 256,
             failure_persist_dir: None,
             failure_persist_file: None,
+            show_logs: false,
         }
     }
 }
@@ -56,6 +60,7 @@ impl FuzzConfig {
             gas_report_samples: 256,
             failure_persist_dir: Some(cache_dir),
             failure_persist_file: Some("failures".to_string()),
+            show_logs: false,
         }
     }
 }
@@ -84,6 +89,7 @@ impl InlineConfigParser for FuzzConfig {
                     conf_clone.dictionary.dictionary_weight = parse_config_u32(key, value)?
                 }
                 "failure-persist-file" => conf_clone.failure_persist_file = Some(value),
+                "show-logs" => conf_clone.show_logs = parse_config_bool(key, value)?,
                 _ => Err(InlineConfigParserError::InvalidConfigProperty(key))?,
             }
         }

diff --git a/crates/evm/evm/src/executors/fuzz/mod.rs b/crates/evm/evm/src/executors/fuzz/mod.rs
@@ -1,14 +1,11 @@
 use crate::executors::{Executor, RawCallResult};
 use alloy_dyn_abi::JsonAbiExt;
 use alloy_json_abi::Function;
-use alloy_primitives::{Address, Bytes, U256};
+use alloy_primitives::{Address, Bytes, Log, U256};
 use eyre::Result;
 use foundry_common::evm::Breakpoints;
 use foundry_config::FuzzConfig;
-use foundry_evm_core::{
-    constants::MAGIC_ASSUME,
-    decode::{decode_console_logs, RevertDecoder},
-};
+use foundry_evm_core::{constants::MAGIC_ASSUME, decode::RevertDecoder};
 use foundry_evm_coverage::HitMaps;
 use foundry_evm_fuzz::{
     strategies::{fuzz_calldata, fuzz_calldata_from_state, EvmFuzzState},
@@ -37,6 +34,8 @@ pub struct FuzzTestData {
     pub breakpoints: Option<Breakpoints>,
     // Stores coverage information for all fuzz cases.
     pub coverage: Option<HitMaps>,
+    // Stores logs for all fuzz cases
+    pub logs: Vec<Log>,
 }
 
 /// Wrapper around an [`Executor`] which provides fuzzing support using [`proptest`].
@@ -90,6 +89,7 @@ impl FuzzedExecutor {
         ];
         // We want to collect at least one trace which will be displayed to user.
         let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize;
+        let show_logs = self.config.show_logs;
 
         let run_result = self.runner.clone().run(&strat, |calldata| {
             let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;
@@ -113,7 +113,9 @@ impl FuzzedExecutor {
                         data.traces.push(call_traces);
                         data.breakpoints.replace(case.breakpoints);
                     }
-
+                    if show_logs {
+                        data.logs.extend(case.logs);
+                    }
                     // Collect and merge coverage if `forge snapshot` context.
                     match &mut data.coverage {
                         Some(prev) => prev.merge(case.coverage.unwrap()),
@@ -133,6 +135,7 @@ impl FuzzedExecutor {
                     // to run at least one more case to find a minimal failure
                     // case.
                     let reason = rd.maybe_decode(&outcome.1.result, Some(status));
+                    execution_data.borrow_mut().logs.extend(outcome.1.logs.clone());
                     execution_data.borrow_mut().counterexample = outcome;
                     // HACK: we have to use an empty string here to denote `None`.
                     Err(TestCaseError::fail(reason.unwrap_or_default()))
@@ -156,8 +159,7 @@ impl FuzzedExecutor {
             success: run_result.is_ok(),
             reason: None,
             counterexample: None,
-            decoded_logs: decode_console_logs(&call.logs),
-            logs: call.logs,
+            logs: fuzz_result.logs,
             labeled_addresses: call.labels,
             traces: last_run_traces,
             breakpoints: last_run_breakpoints,
@@ -229,6 +231,7 @@ impl FuzzedExecutor {
                 traces: call.traces,
                 coverage: call.coverage,
                 breakpoints,
+                logs: call.logs,
             }))
         } else {
             Ok(FuzzOutcome::CounterExample(CounterExampleOutcome {

diff --git a/crates/evm/evm/src/executors/fuzz/types.rs b/crates/evm/evm/src/executors/fuzz/types.rs
@@ -1,5 +1,5 @@
 use crate::executors::RawCallResult;
-use alloy_primitives::Bytes;
+use alloy_primitives::{Bytes, Log};
 use foundry_common::evm::Breakpoints;
 use foundry_evm_coverage::HitMaps;
 use foundry_evm_fuzz::FuzzCase;
@@ -17,6 +17,8 @@ pub struct CaseOutcome {
     pub coverage: Option<HitMaps>,
     /// Breakpoints char pc map
     pub breakpoints: Breakpoints,
+    /// logs of a single fuzz test case
+    pub logs: Vec<Log>,
 }
 
 /// Returned by a single fuzz when a counterexample has been discovered

diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs
@@ -163,9 +163,6 @@ pub struct FuzzTestResult {
     /// be printed to the user.
     pub logs: Vec<Log>,
 
-    /// The decoded DSTest logging events and Hardhat's `console.log` from [logs](Self::logs).
-    pub decoded_logs: Vec<String>,
-
     /// Labeled addresses
     pub labeled_addresses: HashMap<Address, String>,
 

diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs
@@ -74,6 +74,7 @@ forgetest!(can_extract_config_values, |prj, cmd| {
             seed: Some(U256::from(1000)),
             failure_persist_dir: Some("test-cache/fuzz".into()),
             failure_persist_file: Some("failures".to_string()),
+            show_logs: false,
             ..Default::default()
         },
         invariant: InvariantConfig {

diff --git a/crates/forge/tests/cli/test_cmd.rs b/crates/forge/tests/cli/test_cmd.rs
@@ -728,3 +728,128 @@ contract PrecompileLabelsTest is Test {
     assert!(output.contains("Blake2F: [0x0000000000000000000000000000000000000009]"));
     assert!(output.contains("PointEvaluation: [0x000000000000000000000000000000000000000A]"));
 });
+
+// tests that `forge test` with config `show_logs: true` for fuzz tests will
+// display `console.log` info
+forgetest_init!(should_show_logs_when_fuzz_test, |prj, cmd| {
+    prj.wipe_contracts();
+
+    // run fuzz test 3 times
+    let config = Config {
+        fuzz: { FuzzConfig { runs: 3, show_logs: true, ..Default::default() } },
+        ..Default::default()
+    };
+    prj.write_config(config);
+    let config = cmd.config();
+    assert_eq!(config.fuzz.runs, 3);
+
+    prj.add_test(
+        "ContractFuzz.t.sol",
+        r#"pragma solidity 0.8.24;
+        import {Test, console2} from "forge-std/Test.sol";
+    contract ContractFuzz is Test {
+      function testFuzzConsoleLog(uint256 x) public pure {
+        console2.log("inside fuzz test, x is:", x);
+      }
+    }
+     "#,
+    )
+    .unwrap();
+    cmd.args(["test", "-vv"]);
+    let stdout = cmd.stdout_lossy();
+    assert!(stdout.contains("inside fuzz test, x is:"), "\n{stdout}");
+});
+
+// tests that `forge test` with inline config `show_logs = true` for fuzz tests will
+// display `console.log` info
+forgetest_init!(should_show_logs_when_fuzz_test_inline_config, |prj, cmd| {
+    prj.wipe_contracts();
+
+    // run fuzz test 3 times
+    let config =
+        Config { fuzz: { FuzzConfig { runs: 3, ..Default::default() } }, ..Default::default() };
+    prj.write_config(config);
+    let config = cmd.config();
+    assert_eq!(config.fuzz.runs, 3);
+
+    prj.add_test(
+        "ContractFuzz.t.sol",
+        r#"pragma solidity 0.8.24;
+        import {Test, console2} from "forge-std/Test.sol";
+    contract ContractFuzz is Test {
+
+      /// forge-config: default.fuzz.show-logs = true
+      function testFuzzConsoleLog(uint256 x) public pure {
+        console2.log("inside fuzz test, x is:", x);
+      }
+    }
+     "#,
+    )
+    .unwrap();
+    cmd.args(["test", "-vv"]);
+    let stdout = cmd.stdout_lossy();
+    assert!(stdout.contains("inside fuzz test, x is:"), "\n{stdout}");
+});
+
+// tests that `forge test` with config `show_logs: false` for fuzz tests will not display
+// `console.log` info
+forgetest_init!(should_not_show_logs_when_fuzz_test, |prj, cmd| {
+    prj.wipe_contracts();
+
+    // run fuzz test 3 times
+    let config = Config {
+        fuzz: { FuzzConfig { runs: 3, show_logs: false, ..Default::default() } },
+        ..Default::default()
+    };
+    prj.write_config(config);
+    let config = cmd.config();
+    assert_eq!(config.fuzz.runs, 3);
+
+    prj.add_test(
+        "ContractFuzz.t.sol",
+        r#"pragma solidity 0.8.24;
+        import {Test, console2} from "forge-std/Test.sol";
+    contract ContractFuzz is Test {
+
+      function testFuzzConsoleLog(uint256 x) public pure {
+        console2.log("inside fuzz test, x is:", x);
+      }
+    }
+     "#,
+    )
+    .unwrap();
+    cmd.args(["test", "-vv"]);
+    let stdout = cmd.stdout_lossy();
+    assert!(!stdout.contains("inside fuzz test, x is:"), "\n{stdout}");
+});
+
+// tests that `forge test` with inline config `show_logs = false` for fuzz tests will not
+// display `console.log` info
+forgetest_init!(should_not_show_logs_when_fuzz_test_inline_config, |prj, cmd| {
+    prj.wipe_contracts();
+
+    // run fuzz test 3 times
+    let config =
+        Config { fuzz: { FuzzConfig { runs: 3, ..Default::default() } }, ..Default::default() };
+    prj.write_config(config);
+    let config = cmd.config();
+    assert_eq!(config.fuzz.runs, 3);
+
+    prj.add_test(
+        "ContractFuzz.t.sol",
+        r#"pragma solidity 0.8.24;
+        import {Test, console2} from "forge-std/Test.sol";
+    contract ContractFuzz is Test {
+
+      /// forge-config: default.fuzz.show-logs = false
+      function testFuzzConsoleLog(uint256 x) public pure {
+        console2.log("inside fuzz test, x is:", x);
+      }
+    }
+     "#,
+    )
+    .unwrap();
+    cmd.args(["test", "-vv"]);
+    let stdout = cmd.stdout_lossy();
+    assert!(!stdout.contains("inside fuzz test, x is:"), "\n{stdout}");
+});
diff --git a/crates/forge/tests/it/test_helpers.rs b/crates/forge/tests/it/test_helpers.rs
@@ -93,6 +93,7 @@ impl ForgeTestProfile {
                 gas_report_samples: 256,
                 failure_persist_dir: Some(tempfile::tempdir().unwrap().into_path()),
                 failure_persist_file: Some("testfailure".to_string()),
+                show_logs: false,
             })
             .invariant(InvariantConfig {
                 runs: 256,