Merge branch 'main' into lunkai/p256

.github/workflows/benchmark-call.yml

@@ -131,7 +131,8 @@ jobs:
 
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref || github.ref }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
 
       - uses: dtolnay/rust-toolchain@nightly
       - uses: Swatinem/rust-cache@v2
@@ -186,7 +187,7 @@ jobs:
       # When uploading to S3, use ${METRIC_NAME}-${current_sha}.[md/json]
       - name: Set metric name
         run: |
-          METRIC_NAME=${{ inputs.benchmark_id }}
+          METRIC_NAME=${{ inputs.benchmark_id || inputs.benchmark_name }}
           echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV
           METRIC_PATH=".bench_metrics/${METRIC_NAME}.json"
           echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV

.github/workflows/benchmarks.yml

@@ -23,6 +23,7 @@ env:
   CARGO_TERM_COLOR: always
   OPENVM_FAST_TEST: "1"
   CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+  REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
 
 permissions:
@@ -39,7 +40,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref || github.ref }}
+          ref: ${{ CURRENT_SHA }}
+          repository: ${{ REPO }}
 
       - name: Create benchmark matrix from JSON
         id: create-matrix
@@ -133,7 +135,8 @@ jobs:
       ##########################################################################
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref || github.ref }}
+          ref: ${{ CURRENT_SHA }}
+          repository: ${{ REPO }}
 
       - name: Set github pages path for PR
         if: github.event_name == 'pull_request'

.gitignore

@@ -28,3 +28,7 @@ __pycache__/
 
 # Javascript
 **/node_modules/
+
+# Profiling
+**/flamegraph.svg
+**/profile.json

Cargo.toml

@@ -112,8 +112,8 @@ openvm-platform = { path = "crates/toolchain/platform", default-features = false
 openvm-transpiler = { path = "crates/toolchain/transpiler", default-features = false }
 openvm-circuit = { path = "crates/vm", default-features = false }
 openvm-circuit-derive = { path = "crates/vm/derive", default-features = false }
-openvm-stark-backend = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v0.1.1-alpha", default-features = false }
-openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v0.1.1-alpha", default-features = false }
+openvm-stark-backend = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v0.1.2-alpha", default-features = false }
+openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", tag = "v0.1.2-alpha", default-features = false }
 
 # Extensions
 openvm-algebra-circuit = { path = "extensions/algebra/circuit", default-features = false }
@@ -164,6 +164,7 @@ p3-merkle-tree = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c
 p3-monty-31 = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
 p3-poseidon = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
 p3-poseidon2 = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
+p3-poseidon2-air = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
 p3-symmetric = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
 p3-uni-stark = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" }
 p3-maybe-rayon = { git = "https://github.com/Plonky3/Plonky3.git", rev = "9b267c4" } # the "parallel" feature is NOT on by default to allow single-threaded benchmarking

benchmarks/README.md

@@ -90,18 +90,41 @@ Different labels can be added to provide more granularity on the metrics, but th
 
 Most benchmarks are binaries that run once since proving benchmarks take longer. For smaller benchmarks, such as to benchmark VM runtime, we use Criterion. These are in the `benches` directory.
 
-### Usage
-
 ```bash
 cargo bench --bench fibonacci_execute
+cargo bench --bench regex_execute
 ```
 
 will run the normal criterion benchmark.
 
+## Profiling
+
+We profile using executables without criterion in [`examples`](./examples). To prevent the ELF build time from being included in the benchmark, we pre-build the ELF using the CLI. Check that the included ELF file in `examples` is up to date before proceeding.
+
+### Flamegraph
+
+To generate flamegraphs, install `cargo-flamegraph` and run:
+
+```bash
+cargo flamegraph --example regex_execute --profile=profiling
+```
+
+will generate a flamegraph at `flamegraph.svg` without running any criterion analysis.
+On MacOS, you will need to run the above command with `sudo`.
+
+### Samply
+
+To use [samply](https://github.com/mstange/samply), install it and then we must first build the executable.
+
 ```bash
-cargo bench --bench fibonacci_execute -- --profile-time=30
+cargo build --example regex_execute --profile=profiling
 ```
 
-will generate a flamegraph report without running any criterion analysis.
+Then, run:
+
+```bash
+samply record ../target/profiling/examples/regex_execute
+```
 
-Flamegraph reports can be found in `target/criterion/fibonacci/execute/profile/flamegraph.svg` of the repo root directory.
+It will open an interactive UI in your browser (currently only Firefox and Chrome are supported).
+See the samply github page for more information.

benchmarks/benches/fibonacci_execute.rs

@@ -8,7 +8,6 @@ use openvm_rv32im_transpiler::{
 use openvm_sdk::StdIn;
 use openvm_stark_sdk::p3_baby_bear::BabyBear;
 use openvm_transpiler::{transpiler::Transpiler, FromElf};
-use pprof::criterion::{Output, PProfProfiler};
 
 fn benchmark_function(c: &mut Criterion) {
     let elf = build_bench_program("fibonacci").unwrap();
@@ -37,9 +36,5 @@ fn benchmark_function(c: &mut Criterion) {
     group.finish();
 }
 
-criterion_group! {
-    name = benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
-    targets = benchmark_function
-}
+criterion_group!(benches, benchmark_function);
 criterion_main!(benches);

benchmarks/benches/regex_execute.rs

@@ -9,7 +9,6 @@ use openvm_rv32im_transpiler::{
 use openvm_sdk::StdIn;
 use openvm_stark_sdk::p3_baby_bear::BabyBear;
 use openvm_transpiler::{transpiler::Transpiler, FromElf};
-use pprof::criterion::{Output, PProfProfiler};
 
 fn benchmark_function(c: &mut Criterion) {
     let elf = build_bench_program("regex").unwrap();
@@ -42,9 +41,5 @@ fn benchmark_function(c: &mut Criterion) {
     group.finish();
 }
 
-criterion_group! {
-    name = benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(10, Output::Flamegraph(None)));
-    targets = benchmark_function
-}
+criterion_group!(benches, benchmark_function);
 criterion_main!(benches);

benchmarks/examples/regex_execute.rs

@@ -0,0 +1,33 @@
+use openvm_circuit::arch::{instructions::exe::VmExe, VmExecutor};
+use openvm_keccak256_circuit::Keccak256Rv32Config;
+use openvm_keccak256_transpiler::Keccak256TranspilerExtension;
+use openvm_rv32im_transpiler::{
+    Rv32ITranspilerExtension, Rv32IoTranspilerExtension, Rv32MTranspilerExtension,
+};
+use openvm_sdk::StdIn;
+use openvm_stark_sdk::p3_baby_bear::BabyBear;
+use openvm_transpiler::{
+    elf::Elf, openvm_platform::memory::MEM_SIZE, transpiler::Transpiler, FromElf,
+};
+
+fn main() {
+    let elf = Elf::decode(include_bytes!("regex-elf"), MEM_SIZE as u32).unwrap();
+    let exe = VmExe::from_elf(
+        elf,
+        Transpiler::<BabyBear>::default()
+            .with_extension(Rv32ITranspilerExtension)
+            .with_extension(Rv32MTranspilerExtension)
+            .with_extension(Rv32IoTranspilerExtension)
+            .with_extension(Keccak256TranspilerExtension),
+    )
+    .unwrap();
+
+    let config = Keccak256Rv32Config::default();
+    let executor = VmExecutor::<BabyBear, Keccak256Rv32Config>::new(config);
+
+    let data = include_str!("../programs/regex/regex_email.txt");
+
+    executor
+        .execute(exe.clone(), StdIn::from_bytes(data.as_bytes()))
+        .unwrap();
+}

benchmarks/programs/regex/src/main.rs

@@ -7,14 +7,14 @@ use regex::Regex;
 
 openvm::entry!(main);
 
-const pattern: &str = r"(?m)(\r\n|^)From:([^\r\n]+<)?(?P<email>[^<>]+)>?";
+const PATTERN: &str = r"(?m)(\r\n|^)From:([^\r\n]+<)?(?P<email>[^<>]+)>?";
 
 pub fn main() {
     let data = openvm::io::read_vec();
     let data = core::str::from_utf8(&data).expect("Invalid UTF-8");
 
     // Compile the regex
-    let re = Regex::new(pattern).expect("Invalid regex");
+    let re = Regex::new(PATTERN).expect("Invalid regex");
 
     let caps = re.captures(data).expect("No match found.");
     let email = caps.name("email").expect("No email found.");