Feat/benchmark multitier

Cargo.toml

@@ -1,6 +1,8 @@
 [workspace]
 members = [
+    "bin/common",
     "bin/afs",
+    "bin/afs-1b",
     "bin/olap",
     "benchmark",
     "db/page",
@@ -104,4 +106,4 @@ zkhash = { git = "https://github.com/HorizenLabs/poseidon2.git", rev = "bb476b9"
 # p3-symmetric = { path = "../Plonky3/symmetric" }
 # p3-uni-stark = { path = "../Plonky3/uni-stark" }
 # p3-maybe-rayon = { path = "../Plonky3/maybe-rayon" }
-# p3-bn254-fr = { path = "../Plonky3/bn254-fr" }
+# p3-bn254-fr = { path = "../Plonky3/bn254-fr" }

benchmark/Cargo.toml

@@ -7,6 +7,7 @@ edition.workspace = true
 [dependencies]
 csv = "1.3.0"
 afs = { path = "../bin/afs", default-features = false }
+afs-1b = { path = "../bin/afs-1b", default-features = false }
 olap = { path = "../bin/olap", default-features = false }
 afs-stark-backend = { path = "../stark-backend", default-features = false }
 logical-interface = { path = "../logical-interface" }

benchmark/README.md

@@ -56,6 +56,16 @@ Pass in an .afo file that contains the predicate instruction. Example .afo file:
 FILTER 0xfade INDEX <= 0xdac0
 ```
 
+## Predicate
+
+Run this command for multitier read/write
+
+```bash
+RUSTFLAGS="-Ctarget-cpu=native" cargo run --release --bin benchmark -- mtrw -n -i 0
+```
+
+This command goes through many configs, including large and small page heights. We run benchmarks on the case that n leaves are revealed via n insert operations, where n is 1 or 8. To account for the worst case scenario, we double the leaves in the final tree for certain configs (so 2n). The -n flag tells it to use a new, empty Btree (this should always be on, I haven't implemented large tree yet). The -i flag decides where to start the benchmarking, in terms of config #. To directly jump to worst case testing, setting -i to 8 suffices.
+
 ## Additional test commands
 
 To run only small configs for testing

benchmark/src/cli/mod.rs

@@ -3,12 +3,17 @@ use olap::commands::parse_afo_file;
 
 use crate::{
     commands::{
-        benchmark::benchmark_execute,
+        benchmark::{benchmark_execute, benchmark_multitier_execute},
+        multitier_rw::{run_mtrw_bench, MultitierRwCommand},
         predicate::{run_bench_predicate, PredicateCommand},
         rw::{run_bench_rw, RwCommand},
     },
-    config::benchmark_data::{benchmark_data_predicate, benchmark_data_rw},
-    utils::table_gen::{generate_incremental_afi_rw, generate_random_afi_rw},
+    config::benchmark_data::{
+        benchmark_data_multitier_rw, benchmark_data_predicate, benchmark_data_rw,
+    },
+    utils::table_gen::{
+        generate_incremental_afi_rw, generate_random_afi_rw, generate_random_multitier_afi_rw,
+    },
 };
 
 #[derive(Debug, Parser)]
@@ -21,6 +26,10 @@ pub struct Cli {
 
 #[derive(Debug, Subcommand)]
 pub enum Commands {
+    #[command(name = "mtrw", about = "Benchmark Multitier Read/Write")]
+    /// Read/Write functions
+    MtRw(MultitierRwCommand),
+
     #[command(name = "rw", about = "Benchmark Read/Write")]
     /// Read/Write functions
     Rw(RwCommand),
@@ -34,6 +43,23 @@ impl Cli {
     pub fn run() {
         let cli = Self::parse();
         match cli.command {
+            Commands::MtRw(mtrw) => {
+                let benchmark_name = "MultitierReadWrite".to_string();
+                let scenario = format!("New Tree: {}", mtrw.new_tree);
+                let common = mtrw.common;
+                let extra_data = format!("{}", mtrw.new_tree);
+                benchmark_multitier_execute(
+                    benchmark_name,
+                    scenario,
+                    common,
+                    extra_data,
+                    mtrw.start_idx,
+                    run_mtrw_bench,
+                    benchmark_data_multitier_rw,
+                    generate_random_multitier_afi_rw,
+                )
+                .unwrap();
+            }
             Commands::Rw(rw) => {
                 let benchmark_name = "ReadWrite".to_string();
                 let scenario = format!("r{}%, w{}%", rw.percent_reads, rw.percent_writes);

benchmark/src/commands/benchmark.rs

@@ -1,18 +1,22 @@
 use std::{collections::HashMap, fs, path::Path, time::Instant};
 
-use afs_test_utils::page_config::PageConfig;
+use afs_test_utils::page_config::{MultitierPageConfig, PageConfig};
 use chrono::Local;
 use color_eyre::eyre::Result;
 
 use crate::{
-    config::{benchmark_data::BenchmarkData, config_gen::get_configs},
+    config::{
+        benchmark_data::BenchmarkData,
+        config_gen::{get_configs, get_multitier_configs},
+    },
     utils::{
         output_writer::{
-            default_output_filename, save_afi_to_new_db, write_csv_header, write_csv_line,
+            default_output_filename, multitier_page_config_to_row, page_config_to_row,
+            save_afi_to_new_db, write_csv_header, write_csv_line,
         },
         tracing::{clear_tracing_log, extract_event_data_from_log, extract_timing_data_from_log},
     },
-    AFI_FILE_PATH, DB_FILE_PATH, TABLE_ID, TMP_FOLDER, TMP_TRACING_LOG,
+    AFI_FILE_PATH, DB_FILE_PATH, MULTITIER_TABLE_ID, TABLE_ID, TMP_FOLDER, TMP_TRACING_LOG,
 };
 
 use super::CommonCommands;
@@ -132,14 +136,114 @@ pub fn benchmark_execute(
         let mut log_data: HashMap<String, String> = event_data;
         log_data.extend(timing_data);
 
-        write_csv_line(
-            output_file.clone(),
-            benchmark_name.clone(),
-            scenario.clone(),
+        let init_row = page_config_to_row(benchmark_name.clone(), scenario.clone(), config);
+        write_csv_line(output_file.clone(), init_row, &benchmark_data, &log_data)?;
+    }
+
+    println!("Benchmark [{}: {}] completed.", benchmark_name, scenario);
+
+    Ok(())
+}
+
+/// Function for setting up the benchmark
+pub fn benchmark_multitier_setup(
+    benchmark_name: String,
+    config_folder: Option<String>,
+    output_file: Option<String>,
+    benchmark_data: &BenchmarkData,
+) -> (Vec<MultitierPageConfig>, String) {
+    // Generate/Parse config(s)
+    let configs = get_multitier_configs(config_folder);
+
+    // Create tmp folder
+    let _ = fs::create_dir_all(TMP_FOLDER);
+
+    // Write .csv file
+    let output_file = output_file
+        .clone()
+        .unwrap_or(default_output_filename(benchmark_name.clone()));
+
+    println!("Output file: {}", output_file.clone());
+    write_csv_header(
+        output_file.clone(),
+        benchmark_data.sections.clone(),
+        benchmark_data.headers.clone(),
+    )
+    .unwrap();
+
+    (configs, output_file)
+}
+
+#[allow(clippy::too_many_arguments)]
+pub fn benchmark_multitier_execute(
+    benchmark_name: String,
+    scenario: String,
+    common: CommonCommands,
+    extra_data: String,
+    start_idx: usize,
+    benchmark_fn: fn(&MultitierPageConfig, String) -> Result<()>,
+    benchmark_data_fn: fn() -> BenchmarkData,
+    afi_gen_fn: fn(&MultitierPageConfig, String, String) -> Result<()>,
+) -> Result<()> {
+    println!("Executing [{}: {}] benchmark...", benchmark_name, scenario);
+    let benchmark_data = benchmark_data_fn();
+    let (configs, output_file) = benchmark_multitier_setup(
+        benchmark_name.clone(),
+        common.config_folder.clone(),
+        common.output_file.clone(),
+        &benchmark_data,
+    );
+    let configs_len = configs.len();
+    println!("Output file: {}", output_file.clone());
+
+    // Run benchmark for each config
+    for (idx, config) in configs.iter().rev().enumerate() {
+        if idx < start_idx {
+            continue;
+        }
+        let timestamp = Local::now().format("%H:%M:%S");
+        println!(
+            "[{}] Running config {:?}: {} of {}",
+            timestamp,
+            config.generate_filename(),
+            idx + 1,
+            configs_len
+        );
+
+        clear_tracing_log(TMP_TRACING_LOG.as_str())?;
+
+        // Generate AFI file
+        let generate_afi_instant = Instant::now();
+        afi_gen_fn(
             config,
-            &benchmark_data,
-            &log_data,
+            MULTITIER_TABLE_ID.to_string(),
+            AFI_FILE_PATH.to_string(),
         )?;
+        let generate_afi_duration = generate_afi_instant.elapsed();
+        println!("Setup: generate AFI duration: {:?}", generate_afi_duration);
+
+        // Run the benchmark function
+        benchmark_fn(config, extra_data.clone()).unwrap();
+
+        let event_data = extract_event_data_from_log(
+            TMP_TRACING_LOG.as_str(),
+            benchmark_data.event_filters.clone(),
+        )?;
+        let timing_data = extract_timing_data_from_log(
+            TMP_TRACING_LOG.as_str(),
+            benchmark_data.timing_filters.clone(),
+        )?;
+
+        println!("Config: {:?}", config);
+        println!("Event data: {:?}", event_data);
+        println!("Timing data: {:?}", timing_data);
+        println!("Output file: {}", output_file.clone());
+
+        let mut log_data: HashMap<String, String> = event_data;
+        log_data.extend(timing_data);
+        let init_row =
+            multitier_page_config_to_row(benchmark_name.clone(), scenario.clone(), config);
+        write_csv_line(output_file.clone(), init_row, &benchmark_data, &log_data)?;
     }
 
     println!("Benchmark [{}: {}] completed.", benchmark_name, scenario);

benchmark/src/commands/mod.rs

@@ -1,9 +1,11 @@
 use std::fs;
 
-use afs_test_utils::page_config::PageConfig;
+use afs_test_utils::page_config::{MultitierPageConfig, PageConfig};
 use clap::Parser;
 use color_eyre::eyre::Result;
 
+pub mod multitier_rw;
+
 pub mod benchmark;
 pub mod predicate;
 pub mod rw;
@@ -48,3 +50,17 @@ pub fn parse_config_folder(config_folder: String) -> Vec<PageConfig> {
     }
     configs
 }
+
+pub fn parse_multitier_config_folder(config_folder: String) -> Vec<MultitierPageConfig> {
+    let mut configs = Vec::new();
+    if let Ok(entries) = fs::read_dir(config_folder) {
+        for entry in entries.filter_map(Result::ok) {
+            let path = entry.path();
+            if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("toml") {
+                let config = MultitierPageConfig::read_config_file(path.to_str().unwrap());
+                configs.push(config);
+            }
+        }
+    }
+    configs
+}