From c0961b28398f11c8be1e1d54ed74c4fb4d8da20d Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 15:23:54 +0800 Subject: [PATCH 01/13] chore(core): Separate out initialize_zkey --- mopro-core/src/middleware/circom/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mopro-core/src/middleware/circom/mod.rs b/mopro-core/src/middleware/circom/mod.rs index 25b26c1f..94b5c819 100644 --- a/mopro-core/src/middleware/circom/mod.rs +++ b/mopro-core/src/middleware/circom/mod.rs @@ -59,6 +59,7 @@ impl Default for CircomState { const ZKEY_BYTES: &[u8] = include_bytes!(env!("BUILD_RS_ZKEY_FILE")); +// TODO: This can probably be pre-processed static ZKEY: Lazy<(ProvingKey, ConstraintMatrices)> = Lazy::new(|| { let mut reader = Cursor::new(ZKEY_BYTES); read_zkey(&mut reader).expect("Failed to read zkey") @@ -88,6 +89,14 @@ pub fn initialize(dylib_path: &Path) { println!("Initializing zkey took: {:.2?}", now.elapsed()); } +#[cfg(not(feature = "dylib"))] +pub fn initialize_zkey() { + println!("Initializing zkey"); + let now = std::time::Instant::now(); + Lazy::force(&ZKEY); + println!("Initializing zkey took: {:.2?}", now.elapsed()); +} + /// Creates a `WitnessCalculator` instance from a dylib file. #[cfg(feature = "dylib")] fn from_dylib(path: &Path) -> Mutex { @@ -512,6 +521,12 @@ mod tests { initialize(Path::new(&dylib_path)); } + #[cfg(not(feature = "dylib"))] + { + // Do this separately to remove zkey loading from proof generation time + initialize_zkey(); + } + let input_vec = vec![ 116, 101, 115, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, From 8cfb3af95183d1923aea91fcfded032a68de6aca Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 17:08:22 +0800 Subject: [PATCH 02/13] chore(core): Setup zarkkey bin --- mopro-core/Cargo.toml | 4 ++++ mopro-core/src/middleware/circom/mod.rs | 1 + mopro-core/src/middleware/circom/zarkkey.rs | 5 +++++ mopro-core/zarkkey-util.rs | 7 +++++++ 4 files changed, 17 insertions(+) create mode 100644 mopro-core/src/middleware/circom/zarkkey.rs create mode 100644 mopro-core/zarkkey-util.rs diff --git a/mopro-core/Cargo.toml b/mopro-core/Cargo.toml index b995a986..52c8b96a 100644 --- a/mopro-core/Cargo.toml +++ b/mopro-core/Cargo.toml @@ -5,6 +5,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "zarkkey-util" +path = "zarkkey-util.rs" + [patch.crates-io] # NOTE: Forked wasmer to work around memory limits # See https://github.com/wasmerio/wasmer/commit/09c7070 diff --git a/mopro-core/src/middleware/circom/mod.rs b/mopro-core/src/middleware/circom/mod.rs index 94b5c819..5c4516f6 100644 --- a/mopro-core/src/middleware/circom/mod.rs +++ b/mopro-core/src/middleware/circom/mod.rs @@ -34,6 +34,7 @@ use { pub mod serialization; pub mod utils; +pub mod zarkkey; type GrothBn = Groth16; diff --git a/mopro-core/src/middleware/circom/zarkkey.rs b/mopro-core/src/middleware/circom/zarkkey.rs new file mode 100644 index 00000000..1907f4d2 --- /dev/null +++ b/mopro-core/src/middleware/circom/zarkkey.rs @@ -0,0 +1,5 @@ +// Util to convert zkey to zarkkey + +pub fn hello() { + println!("Hello, world!"); +} diff --git a/mopro-core/zarkkey-util.rs b/mopro-core/zarkkey-util.rs new file mode 100644 index 00000000..be191ff5 --- /dev/null +++ b/mopro-core/zarkkey-util.rs @@ -0,0 +1,7 @@ +// Util to convert zkey to zarkkey + +use mopro_core::middleware; + +fn main() { + middleware::circom::zarkkey::hello(); +} From 67927b3a463a20baa8b31c4460f14ff65999d8f6 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 17:40:55 +0800 Subject: [PATCH 03/13] feat(ark-zkey): Init --- ark-zkey/Cargo.toml | 8 ++++++++ ark-zkey/src/lib.rs | 14 ++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 ark-zkey/Cargo.toml create mode 100644 ark-zkey/src/lib.rs diff --git a/ark-zkey/Cargo.toml b/ark-zkey/Cargo.toml new file mode 100644 index 00000000..e001bc8f --- /dev/null +++ b/ark-zkey/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "ark-zkey" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs new file mode 100644 index 00000000..7d12d9af --- /dev/null +++ b/ark-zkey/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From 6e439ad5a8cd7abb21c1e0f6d0e7167ca37c543e Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 18:47:52 +0800 Subject: [PATCH 04/13] chore(ark-zkey): Add gitignore --- ark-zkey/.gitignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ark-zkey/.gitignore diff --git a/ark-zkey/.gitignore b/ark-zkey/.gitignore new file mode 100644 index 00000000..6985cf1b --- /dev/null +++ b/ark-zkey/.gitignore @@ -0,0 +1,14 @@ +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb From 8ae92fbbbfc3a4d2841859bdb0df6c139a8b7816 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 18:49:01 +0800 Subject: [PATCH 05/13] feat(ark-zkey): Read zkey proving key and serialize to file --- ark-zkey/Cargo.toml | 14 +++++++++ ark-zkey/src/lib.rs | 70 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/ark-zkey/Cargo.toml b/ark-zkey/Cargo.toml index e001bc8f..cbaa0f72 100644 --- a/ark-zkey/Cargo.toml +++ b/ark-zkey/Cargo.toml @@ -5,4 +5,18 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +# XXX: Shouldn't be necessary, but this way we stay consistent with wasmer version and fix +# error[E0432]: unresolved import `wasmer` error +# (likely due to other packages) +[patch.crates-io] +# NOTE: Forked wasmer to work around memory limits +# See https://github.com/wasmerio/wasmer/commit/09c7070 +wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" } + [dependencies] +color-eyre = "0.6" + +ark-serialize = { version = "=0.4.1", features = ["derive"] } +ark-bn254 = { version = "=0.4.0" } +ark-groth16 = { version = "=0.4.0" } +ark-circom = { git = "https://github.com/arkworks-rs/circom-compat.git" } diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 7d12d9af..54a645ff 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -1,5 +1,52 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right +use ark_bn254::Bn254; +use ark_circom::read_zkey; +use ark_groth16::ProvingKey; +use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; +use color_eyre::eyre::{self, Result, WrapErr}; +use std::env; +use std::fs::File; +use std::io::{self, Cursor, Read, Write}; +use std::path::PathBuf; + +// NOTE: Starting with ProvingKey +// TODO: Add ConstraintMatrices + +#[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] +pub struct SerializableProvingKey(pub ProvingKey); + +pub fn serialize_proving_key(pk: &SerializableProvingKey) -> Vec { + let mut serialized_data = Vec::new(); + pk.serialize_uncompressed(&mut serialized_data) + .expect("Serialization failed"); + serialized_data +} + +pub fn deserialize_proving_key(data: Vec) -> SerializableProvingKey { + SerializableProvingKey::deserialize_uncompressed(&mut &data[..]) + .expect("Deserialization failed") +} + +pub fn convert_zkey(zkey_path: &str, arkzkey_path: &str) -> Result<()> { + let zkey_file_path = PathBuf::from(zkey_path); + let arkzkey_file_path = PathBuf::from(arkzkey_path); + println!("zkey_file_path: {:?}", zkey_file_path); + println!("arkzkey_file_path: {:?}", arkzkey_file_path); + + // Read the zkey file and get a SerializableProvingKey + let mut zkey_file = File::open(zkey_file_path).wrap_err("Failed to open zkey file")?; + + // TODO: Add ConstraintMatrices + let (proving_key, _) = read_zkey(&mut zkey_file).wrap_err("Failed to read zkey file")?; + + let serialized_path = PathBuf::from(arkzkey_file_path); + + let mut file = + File::create(&serialized_path).wrap_err("Failed to create serialized proving key file")?; + SerializableProvingKey(proving_key) + .serialize_uncompressed(&mut file) + .wrap_err("Failed to serialize proving key")?; + + Ok(()) } #[cfg(test)] @@ -7,8 +54,21 @@ mod tests { use super::*; #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + fn test_serialization_deserialization() -> Result<()> { + // multiplier + let dir = "../mopro-core/examples/circom/multiplier2"; + let circuit = "multiplier2"; + + // keccak256 + // let dir = "../mopro-core/examples/circom/keccak256"; + // let circuit = "keccak256_256_test"; + + let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); + let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); + + // TODO: Also read it back and compare + convert_zkey(&zkey_path, &arkzkey_path)?; + + Ok(()) } } From 0d0a1055cb4b0452d991ab8541cd226d786d9e1f Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 19:18:35 +0800 Subject: [PATCH 06/13] feat(ark-zkey): Compare with original proving key --- ark-zkey/src/lib.rs | 48 +++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 54a645ff..248112fb 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -2,10 +2,9 @@ use ark_bn254::Bn254; use ark_circom::read_zkey; use ark_groth16::ProvingKey; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; -use color_eyre::eyre::{self, Result, WrapErr}; -use std::env; +use color_eyre::eyre::{Result, WrapErr}; use std::fs::File; -use std::io::{self, Cursor, Read, Write}; +use std::io::Read; use std::path::PathBuf; // NOTE: Starting with ProvingKey @@ -26,23 +25,36 @@ pub fn deserialize_proving_key(data: Vec) -> SerializableProvingKey { .expect("Deserialization failed") } -pub fn convert_zkey(zkey_path: &str, arkzkey_path: &str) -> Result<()> { - let zkey_file_path = PathBuf::from(zkey_path); +pub fn read_arkzkey(arkzkey_path: &str) -> Result { let arkzkey_file_path = PathBuf::from(arkzkey_path); - println!("zkey_file_path: {:?}", zkey_file_path); - println!("arkzkey_file_path: {:?}", arkzkey_file_path); + let mut arkzkey_file = File::open(arkzkey_file_path).wrap_err("Failed to open arkzkey file")?; + let mut serialized_data = Vec::new(); + arkzkey_file + .read_to_end(&mut serialized_data) + .wrap_err("Failed to read arkzkey file")?; + Ok( + SerializableProvingKey::deserialize_uncompressed(&mut &serialized_data[..]) + .wrap_err("Failed to deserialize proving key")?, + ) +} - // Read the zkey file and get a SerializableProvingKey +pub fn read_proving_key_from_zkey(zkey_path: &str) -> Result { + let zkey_file_path = PathBuf::from(zkey_path); let mut zkey_file = File::open(zkey_file_path).wrap_err("Failed to open zkey file")?; - - // TODO: Add ConstraintMatrices let (proving_key, _) = read_zkey(&mut zkey_file).wrap_err("Failed to read zkey file")?; + Ok(SerializableProvingKey(proving_key)) +} + +// TODO: Add ConstraintMatrices +pub fn convert_zkey(proving_key: SerializableProvingKey, arkzkey_path: &str) -> Result<()> { + let arkzkey_file_path = PathBuf::from(arkzkey_path); + println!("arkzkey_file_path: {:?}", arkzkey_file_path); let serialized_path = PathBuf::from(arkzkey_file_path); let mut file = File::create(&serialized_path).wrap_err("Failed to create serialized proving key file")?; - SerializableProvingKey(proving_key) + proving_key .serialize_uncompressed(&mut file) .wrap_err("Failed to serialize proving key")?; @@ -66,8 +78,18 @@ mod tests { let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); - // TODO: Also read it back and compare - convert_zkey(&zkey_path, &arkzkey_path)?; + // Read the original proving key + let original_proving_key = read_proving_key_from_zkey(&zkey_path)?; + convert_zkey(original_proving_key.clone(), &arkzkey_path)?; + + // Read the serialized and then deserialized proving key + let deserialized_proving_key = read_arkzkey(&arkzkey_path)?; + + // Compare the original and deserialized proving keys + assert_eq!( + original_proving_key, deserialized_proving_key, + "Original and deserialized proving keys do not match" + ); Ok(()) } From 83b41aca71bdfc0e08d72cc71ba641db4936f8ea Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 19:57:42 +0800 Subject: [PATCH 07/13] chore(ark-zkey): Add initial support for serializing ConstraintMatrices --- ark-zkey/Cargo.toml | 2 ++ ark-zkey/src/lib.rs | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/ark-zkey/Cargo.toml b/ark-zkey/Cargo.toml index cbaa0f72..b62c7b2e 100644 --- a/ark-zkey/Cargo.toml +++ b/ark-zkey/Cargo.toml @@ -20,3 +20,5 @@ ark-serialize = { version = "=0.4.1", features = ["derive"] } ark-bn254 = { version = "=0.4.0" } ark-groth16 = { version = "=0.4.0" } ark-circom = { git = "https://github.com/arkworks-rs/circom-compat.git" } +ark-relations = { version = "=0.4.0" } +ark-ff = { version = "=0.4.1" } diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 248112fb..3b74776b 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -1,6 +1,8 @@ -use ark_bn254::Bn254; +use ark_bn254::{Bn254, Fr}; use ark_circom::read_zkey; +use ark_ff::Field; use ark_groth16::ProvingKey; +use ark_relations::r1cs::ConstraintMatrices; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use color_eyre::eyre::{Result, WrapErr}; use std::fs::File; @@ -13,6 +15,39 @@ use std::path::PathBuf; #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] pub struct SerializableProvingKey(pub ProvingKey); +#[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] +pub struct SerializableMatrix { + pub data: Vec>, +} + +// #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] +// pub struct SerializableConstraintMatrices(pub ConstraintMatrices); + +#[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] +pub struct SerializableConstraintMatrices { + pub num_instance_variables: usize, + pub num_witness_variables: usize, + pub num_constraints: usize, + pub a_num_non_zero: usize, + pub b_num_non_zero: usize, + pub c_num_non_zero: usize, + pub a: SerializableMatrix, + pub b: SerializableMatrix, + pub c: SerializableMatrix, +} + +impl From>> for SerializableMatrix { + fn from(matrix: Vec>) -> Self { + SerializableMatrix { data: matrix } + } +} + +impl From> for Vec> { + fn from(serializable_matrix: SerializableMatrix) -> Self { + serializable_matrix.data + } +} + pub fn serialize_proving_key(pk: &SerializableProvingKey) -> Vec { let mut serialized_data = Vec::new(); pk.serialize_uncompressed(&mut serialized_data) From c6cefa04ec5dd1a724cdcda000d8c09e9cc53dd4 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 20:14:13 +0800 Subject: [PATCH 08/13] feat(ark-zkey): Rest of ConstraintMatrices logic --- ark-zkey/src/lib.rs | 70 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 3b74776b..0801eee8 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -60,28 +60,53 @@ pub fn deserialize_proving_key(data: Vec) -> SerializableProvingKey { .expect("Deserialization failed") } -pub fn read_arkzkey(arkzkey_path: &str) -> Result { +pub fn read_arkzkey( + arkzkey_path: &str, +) -> Result<(SerializableProvingKey, SerializableConstraintMatrices)> { let arkzkey_file_path = PathBuf::from(arkzkey_path); let mut arkzkey_file = File::open(arkzkey_file_path).wrap_err("Failed to open arkzkey file")?; let mut serialized_data = Vec::new(); arkzkey_file .read_to_end(&mut serialized_data) .wrap_err("Failed to read arkzkey file")?; - Ok( - SerializableProvingKey::deserialize_uncompressed(&mut &serialized_data[..]) - .wrap_err("Failed to deserialize proving key")?, - ) + + let proving_key = SerializableProvingKey::deserialize_uncompressed(&mut &serialized_data[..]) + .wrap_err("Failed to deserialize proving key")?; + let constraint_matrices = + SerializableConstraintMatrices::deserialize_uncompressed(&mut &serialized_data[..]) + .wrap_err("Failed to deserialize constraint matrices")?; + + Ok((proving_key, constraint_matrices)) } -pub fn read_proving_key_from_zkey(zkey_path: &str) -> Result { +pub fn read_proving_key_and_matrices_from_zkey( + zkey_path: &str, +) -> Result<(SerializableProvingKey, SerializableConstraintMatrices)> { let zkey_file_path = PathBuf::from(zkey_path); let mut zkey_file = File::open(zkey_file_path).wrap_err("Failed to open zkey file")?; - let (proving_key, _) = read_zkey(&mut zkey_file).wrap_err("Failed to read zkey file")?; - Ok(SerializableProvingKey(proving_key)) + let (proving_key, matrices) = read_zkey(&mut zkey_file).wrap_err("Failed to read zkey file")?; + + let serializable_proving_key = SerializableProvingKey(proving_key); + let serializable_constrain_matrices = SerializableConstraintMatrices { + num_instance_variables: matrices.num_instance_variables, + num_witness_variables: matrices.num_witness_variables, + num_constraints: matrices.num_constraints, + a_num_non_zero: matrices.a_num_non_zero, + b_num_non_zero: matrices.b_num_non_zero, + c_num_non_zero: matrices.c_num_non_zero, + a: SerializableMatrix { data: matrices.a }, + b: SerializableMatrix { data: matrices.b }, + c: SerializableMatrix { data: matrices.c }, + }; + + Ok((serializable_proving_key, serializable_constrain_matrices)) } -// TODO: Add ConstraintMatrices -pub fn convert_zkey(proving_key: SerializableProvingKey, arkzkey_path: &str) -> Result<()> { +pub fn convert_zkey( + proving_key: SerializableProvingKey, + constraint_matrices: SerializableConstraintMatrices, + arkzkey_path: &str, +) -> Result<()> { let arkzkey_file_path = PathBuf::from(arkzkey_path); println!("arkzkey_file_path: {:?}", arkzkey_file_path); @@ -89,10 +114,15 @@ pub fn convert_zkey(proving_key: SerializableProvingKey, arkzkey_path: &str) -> let mut file = File::create(&serialized_path).wrap_err("Failed to create serialized proving key file")?; + proving_key .serialize_uncompressed(&mut file) .wrap_err("Failed to serialize proving key")?; + constraint_matrices + .serialize_uncompressed(&mut file) + .wrap_err("Failed to serialize constraint matrices")?; + Ok(()) } @@ -114,18 +144,28 @@ mod tests { let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); // Read the original proving key - let original_proving_key = read_proving_key_from_zkey(&zkey_path)?; - convert_zkey(original_proving_key.clone(), &arkzkey_path)?; + let (original_proving_key, original_constraint_matrices) = + read_proving_key_and_matrices_from_zkey(&zkey_path)?; + + convert_zkey( + original_proving_key.clone(), + original_constraint_matrices.clone(), + &arkzkey_path, + )?; - // Read the serialized and then deserialized proving key - let deserialized_proving_key = read_arkzkey(&arkzkey_path)?; + let (deserialized_proving_key, deserialized_constraint_matrices) = + read_arkzkey(&arkzkey_path)?; - // Compare the original and deserialized proving keys assert_eq!( original_proving_key, deserialized_proving_key, "Original and deserialized proving keys do not match" ); + assert_eq!( + original_constraint_matrices, deserialized_constraint_matrices, + "Original and deserialized constraint matrices do not match" + ); + Ok(()) } } From 4c5193877dc3535373029cac79cfcb4818b801e3 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 20:20:06 +0800 Subject: [PATCH 09/13] fix(ark-zkey): Use cursor to read at arkzkey from right place --- ark-zkey/src/lib.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 0801eee8..7d3e725d 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -2,16 +2,13 @@ use ark_bn254::{Bn254, Fr}; use ark_circom::read_zkey; use ark_ff::Field; use ark_groth16::ProvingKey; -use ark_relations::r1cs::ConstraintMatrices; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use color_eyre::eyre::{Result, WrapErr}; use std::fs::File; +use std::io::Cursor; use std::io::Read; use std::path::PathBuf; -// NOTE: Starting with ProvingKey -// TODO: Add ConstraintMatrices - #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] pub struct SerializableProvingKey(pub ProvingKey); @@ -20,9 +17,6 @@ pub struct SerializableMatrix { pub data: Vec>, } -// #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] -// pub struct SerializableConstraintMatrices(pub ConstraintMatrices); - #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] pub struct SerializableConstraintMatrices { pub num_instance_variables: usize, @@ -70,11 +64,12 @@ pub fn read_arkzkey( .read_to_end(&mut serialized_data) .wrap_err("Failed to read arkzkey file")?; - let proving_key = SerializableProvingKey::deserialize_uncompressed(&mut &serialized_data[..]) + let mut cursor = Cursor::new(serialized_data); + + let proving_key = SerializableProvingKey::deserialize_uncompressed(&mut cursor) .wrap_err("Failed to deserialize proving key")?; - let constraint_matrices = - SerializableConstraintMatrices::deserialize_uncompressed(&mut &serialized_data[..]) - .wrap_err("Failed to deserialize constraint matrices")?; + let constraint_matrices = SerializableConstraintMatrices::deserialize_uncompressed(&mut cursor) + .wrap_err("Failed to deserialize constraint matrices")?; Ok((proving_key, constraint_matrices)) } @@ -108,7 +103,6 @@ pub fn convert_zkey( arkzkey_path: &str, ) -> Result<()> { let arkzkey_file_path = PathBuf::from(arkzkey_path); - println!("arkzkey_file_path: {:?}", arkzkey_file_path); let serialized_path = PathBuf::from(arkzkey_file_path); @@ -143,16 +137,18 @@ mod tests { let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); - // Read the original proving key + println!("Reading zkey from: {}", zkey_path); let (original_proving_key, original_constraint_matrices) = read_proving_key_and_matrices_from_zkey(&zkey_path)?; + println!("Writing arkzkey to: {}", arkzkey_path); convert_zkey( original_proving_key.clone(), original_constraint_matrices.clone(), &arkzkey_path, )?; + println!("Reading arkzkey from: {}", arkzkey_path); let (deserialized_proving_key, deserialized_constraint_matrices) = read_arkzkey(&arkzkey_path)?; From 9a02797e0721652670c99780cd47b94a831db07a Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 20:22:22 +0800 Subject: [PATCH 10/13] chore(ark-zkey): Add timings --- ark-zkey/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 7d3e725d..55e83806 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -8,6 +8,7 @@ use std::fs::File; use std::io::Cursor; use std::io::Read; use std::path::PathBuf; +use std::time::Instant; #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] pub struct SerializableProvingKey(pub ProvingKey); @@ -138,19 +139,25 @@ mod tests { let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); println!("Reading zkey from: {}", zkey_path); + let now = Instant::now(); let (original_proving_key, original_constraint_matrices) = read_proving_key_and_matrices_from_zkey(&zkey_path)?; + println!("Time to read zkey: {:?}", now.elapsed()); println!("Writing arkzkey to: {}", arkzkey_path); + let now = Instant::now(); convert_zkey( original_proving_key.clone(), original_constraint_matrices.clone(), &arkzkey_path, )?; + println!("Time to write zkey: {:?}", now.elapsed()); println!("Reading arkzkey from: {}", arkzkey_path); + let now = Instant::now(); let (deserialized_proving_key, deserialized_constraint_matrices) = read_arkzkey(&arkzkey_path)?; + println!("Time to read arkzkey: {:?}", now.elapsed()); assert_eq!( original_proving_key, deserialized_proving_key, From 6844c541a11dea18fc698623c8fb5edc3bd9d230 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 20:39:24 +0800 Subject: [PATCH 11/13] feat(ark-zkey): Use buffered reads and keccak --- ark-zkey/src/lib.rs | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 55e83806..49139f9e 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -7,8 +7,8 @@ use color_eyre::eyre::{Result, WrapErr}; use std::fs::File; use std::io::Cursor; use std::io::Read; +use std::io::{self, BufReader}; use std::path::PathBuf; -use std::time::Instant; #[derive(CanonicalSerialize, CanonicalDeserialize, Clone, Debug, PartialEq)] pub struct SerializableProvingKey(pub ProvingKey); @@ -45,32 +45,27 @@ impl From> for Vec> { pub fn serialize_proving_key(pk: &SerializableProvingKey) -> Vec { let mut serialized_data = Vec::new(); - pk.serialize_uncompressed(&mut serialized_data) + pk.serialize_compressed(&mut serialized_data) .expect("Serialization failed"); serialized_data } pub fn deserialize_proving_key(data: Vec) -> SerializableProvingKey { - SerializableProvingKey::deserialize_uncompressed(&mut &data[..]) - .expect("Deserialization failed") + SerializableProvingKey::deserialize_compressed(&mut &data[..]).expect("Deserialization failed") } pub fn read_arkzkey( arkzkey_path: &str, ) -> Result<(SerializableProvingKey, SerializableConstraintMatrices)> { let arkzkey_file_path = PathBuf::from(arkzkey_path); - let mut arkzkey_file = File::open(arkzkey_file_path).wrap_err("Failed to open arkzkey file")?; - let mut serialized_data = Vec::new(); - arkzkey_file - .read_to_end(&mut serialized_data) - .wrap_err("Failed to read arkzkey file")?; - - let mut cursor = Cursor::new(serialized_data); + let arkzkey_file = File::open(arkzkey_file_path).wrap_err("Failed to open arkzkey file")?; + let mut buf_reader = BufReader::new(arkzkey_file); - let proving_key = SerializableProvingKey::deserialize_uncompressed(&mut cursor) + let proving_key = SerializableProvingKey::deserialize_compressed(&mut buf_reader) .wrap_err("Failed to deserialize proving key")?; - let constraint_matrices = SerializableConstraintMatrices::deserialize_uncompressed(&mut cursor) - .wrap_err("Failed to deserialize constraint matrices")?; + let constraint_matrices = + SerializableConstraintMatrices::deserialize_compressed(&mut buf_reader) + .wrap_err("Failed to deserialize constraint matrices")?; Ok((proving_key, constraint_matrices)) } @@ -80,7 +75,11 @@ pub fn read_proving_key_and_matrices_from_zkey( ) -> Result<(SerializableProvingKey, SerializableConstraintMatrices)> { let zkey_file_path = PathBuf::from(zkey_path); let mut zkey_file = File::open(zkey_file_path).wrap_err("Failed to open zkey file")?; - let (proving_key, matrices) = read_zkey(&mut zkey_file).wrap_err("Failed to read zkey file")?; + + let mut buf_reader = BufReader::new(zkey_file); + + let (proving_key, matrices) = + read_zkey(&mut buf_reader).wrap_err("Failed to read zkey file")?; let serializable_proving_key = SerializableProvingKey(proving_key); let serializable_constrain_matrices = SerializableConstraintMatrices { @@ -111,11 +110,11 @@ pub fn convert_zkey( File::create(&serialized_path).wrap_err("Failed to create serialized proving key file")?; proving_key - .serialize_uncompressed(&mut file) + .serialize_compressed(&mut file) .wrap_err("Failed to serialize proving key")?; constraint_matrices - .serialize_uncompressed(&mut file) + .serialize_compressed(&mut file) .wrap_err("Failed to serialize constraint matrices")?; Ok(()) @@ -124,16 +123,17 @@ pub fn convert_zkey( #[cfg(test)] mod tests { use super::*; + use std::time::Instant; #[test] fn test_serialization_deserialization() -> Result<()> { // multiplier - let dir = "../mopro-core/examples/circom/multiplier2"; - let circuit = "multiplier2"; + // let dir = "../mopro-core/examples/circom/multiplier2"; + // let circuit = "multiplier2"; - // keccak256 - // let dir = "../mopro-core/examples/circom/keccak256"; - // let circuit = "keccak256_256_test"; + //keccak256 + let dir = "../mopro-core/examples/circom/keccak256"; + let circuit = "keccak256_256_test"; let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); From dbd154487b1aa69d8be6a6b0e01ed71cb87dedb6 Mon Sep 17 00:00:00 2001 From: oskarth Date: Fri, 3 Nov 2023 23:43:42 +0800 Subject: [PATCH 12/13] chore(ark-zkey): Try mmap2 --- ark-zkey/Cargo.toml | 1 + ark-zkey/src/lib.rs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/ark-zkey/Cargo.toml b/ark-zkey/Cargo.toml index b62c7b2e..9753babf 100644 --- a/ark-zkey/Cargo.toml +++ b/ark-zkey/Cargo.toml @@ -15,6 +15,7 @@ wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" } [dependencies] color-eyre = "0.6" +memmap2 = "0.9" ark-serialize = { version = "=0.4.1", features = ["derive"] } ark-bn254 = { version = "=0.4.0" } diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index 49139f9e..cb62f5b7 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -2,8 +2,10 @@ use ark_bn254::{Bn254, Fr}; use ark_circom::read_zkey; use ark_ff::Field; use ark_groth16::ProvingKey; +use ark_relations::r1cs::ConstraintMatrices; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use color_eyre::eyre::{Result, WrapErr}; +use memmap2::Mmap; use std::fs::File; use std::io::Cursor; use std::io::Read; @@ -120,6 +122,17 @@ pub fn convert_zkey( Ok(()) } +fn read_zkey_with_mmap(zkey_path: &str) -> Result<(ProvingKey, ConstraintMatrices)> { + let file = File::open(zkey_path)?; + + let mmap = unsafe { Mmap::map(&file)? }; + + let cursor = Cursor::new(&mmap); + let (proving_key, matrices) = read_zkey(&mut cursor.clone())?; + + Ok((proving_key, matrices)) +} + #[cfg(test)] mod tests { use super::*; @@ -138,6 +151,11 @@ mod tests { let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); + println!("Reading mmaped zkey from: {}", zkey_path); + let now = Instant::now(); + let (original_proving_key, original_constraint_matrices) = read_zkey_with_mmap(&zkey_path)?; + println!("Time to read mmaped zkey: {:?}", now.elapsed()); + println!("Reading zkey from: {}", zkey_path); let now = Instant::now(); let (original_proving_key, original_constraint_matrices) = From 1354086bc9f1bbdeee7f9591559c8c648ab3eca2 Mon Sep 17 00:00:00 2001 From: oskarth Date: Sat, 4 Nov 2023 00:49:12 +0800 Subject: [PATCH 13/13] chore(ark-zkey): Try more mmap, flame --- ark-zkey/Cargo.toml | 2 ++ ark-zkey/src/lib.rs | 34 +++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/ark-zkey/Cargo.toml b/ark-zkey/Cargo.toml index 9753babf..88e80c60 100644 --- a/ark-zkey/Cargo.toml +++ b/ark-zkey/Cargo.toml @@ -16,6 +16,8 @@ wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" } [dependencies] color-eyre = "0.6" memmap2 = "0.9" +flame = "0.2" +flamer = "0.5" ark-serialize = { version = "=0.4.1", features = ["derive"] } ark-bn254 = { version = "=0.4.0" } diff --git a/ark-zkey/src/lib.rs b/ark-zkey/src/lib.rs index cb62f5b7..5987324b 100644 --- a/ark-zkey/src/lib.rs +++ b/ark-zkey/src/lib.rs @@ -59,15 +59,29 @@ pub fn deserialize_proving_key(data: Vec) -> SerializableProvingKey { pub fn read_arkzkey( arkzkey_path: &str, ) -> Result<(SerializableProvingKey, SerializableConstraintMatrices)> { + let now = std::time::Instant::now(); let arkzkey_file_path = PathBuf::from(arkzkey_path); let arkzkey_file = File::open(arkzkey_file_path).wrap_err("Failed to open arkzkey file")?; - let mut buf_reader = BufReader::new(arkzkey_file); + println!("Time to open arkzkey file: {:?}", now.elapsed()); - let proving_key = SerializableProvingKey::deserialize_compressed(&mut buf_reader) + //let mut buf_reader = BufReader::new(arkzkey_file); + + // Using mmap + let now = std::time::Instant::now(); + let mmap = unsafe { Mmap::map(&arkzkey_file)? }; + let mut cursor = std::io::Cursor::new(mmap); + println!("Time to mmap: {:?}", now.elapsed()); + + // Was &mut buf_reader + let now = std::time::Instant::now(); + let proving_key = SerializableProvingKey::deserialize_compressed(&mut cursor) .wrap_err("Failed to deserialize proving key")?; - let constraint_matrices = - SerializableConstraintMatrices::deserialize_compressed(&mut buf_reader) - .wrap_err("Failed to deserialize constraint matrices")?; + println!("Time to deserialize proving key: {:?}", now.elapsed()); + + let now = std::time::Instant::now(); + let constraint_matrices = SerializableConstraintMatrices::deserialize_compressed(&mut cursor) + .wrap_err("Failed to deserialize constraint matrices")?; + println!("Time to deserialize matrices: {:?}", now.elapsed()); Ok((proving_key, constraint_matrices)) } @@ -151,10 +165,10 @@ mod tests { let zkey_path = format!("{}/target/{}_final.zkey", dir, circuit); let arkzkey_path = format!("{}/target/{}_final.arkzkey", dir, circuit); - println!("Reading mmaped zkey from: {}", zkey_path); - let now = Instant::now(); - let (original_proving_key, original_constraint_matrices) = read_zkey_with_mmap(&zkey_path)?; - println!("Time to read mmaped zkey: {:?}", now.elapsed()); + // println!("Reading mmaped zkey from: {}", zkey_path); + // let now = Instant::now(); + // let (original_proving_key, original_constraint_matrices) = read_zkey_with_mmap(&zkey_path)?; + // println!("Time to read mmaped zkey: {:?}", now.elapsed()); println!("Reading zkey from: {}", zkey_path); let now = Instant::now(); @@ -187,6 +201,8 @@ mod tests { "Original and deserialized constraint matrices do not match" ); + flame::dump_html(&mut std::fs::File::create("flame-graph.html").unwrap()).unwrap(); + Ok(()) } }