Feat/Msm with gpu (metal shader language) on laptop

Cargo.toml

@@ -7,10 +7,3 @@ exclude = ["mopro-example-app"]
 # NOTE: Forked wasmer to work around memory limits
 # See https://github.com/wasmerio/wasmer/commit/09c7070
 wasmer = { git = "https://github.com/oskarth/wasmer.git", rev = "09c7070" }
-
-# NOTE: For gpu exploration on zprize works, will only compile when `gpu-benchmarks` feature is enabled
-ark-bls12-377-3 = { git = 'https://github.com/arkworks-rs/curves.git', package = 'ark-bls12-377', tag = 'v0.3.0', optional = true}
-ark-ec-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-ec', tag = 'v0.3.0', features = ["parallel"], optional = true }
-ark-ff-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-ff', tag = 'v0.3.0', features = ["parallel"], optional = true }
-ark-serialize-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-serialize', tag = 'v0.3.0', optional = true }
-ark-std-3 = { git = 'https://github.com/arkworks-rs/std.git', package = 'ark-std', tag = 'v0.3.0', optional = true }

mopro-core/.gitignore

@@ -14,4 +14,7 @@ Cargo.lock
 *.pdb
 
 # GPU exploration - preprocessed vectors
-src/middleware/gpu_explorations/utils/vectors
+src/middleware/gpu_explorations/utils/vectors
+
+# GPU exploration - proptest generated files
+proptest-regressions

mopro-core/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 [features]
 default = ["wasmer/dylib"]
 dylib = []                  # NOTE: can probably remove this if we use env config instead
-gpu-benchmarks = ["ark-bls12-377", "ark-bls12-381", "ark-ed-on-bls12-377", "ark-ed-on-bls12-381", "ark-poly", "ark-poly-commit", "ark-sponge", "duration-string", "rand", "rand_chacha", "lazy_static", "ark-ec-3", "ark-ff-3", "ark-serialize-3", "ark-std-3", "ark-bls12-377-3", "parallel"]
+gpu-benchmarks = ["ark-ff", "ark-bls12-381", "metal", "objc", "proptest", "parallel"]
 calc-native-witness = ["witness"] # experimental feature to calculate witness with witness graph
 build-native-witness = ["witness/build-witness"] # only enable build-native-witness feature when building the witness graph
 parallel = ["rayon", "ark-std/parallel"]
@@ -48,27 +48,13 @@ color-eyre = "=0.6.2"
 criterion = "=0.3.6"
 
 # GPU explorations
-ark-bls12-377 = { version = "0.4", optional = true }
-ark-bls12-381 = { version = "0.3", optional = true }
-ark-ed-on-bls12-377 = { version = "0.3", optional = true }
-ark-ed-on-bls12-381 = { version = "0.3", optional = true }
-ark-poly = { version = "0.3", optional = true }
-ark-poly-commit = { version = "0.3", optional = true }
-ark-sponge = { version = "0.3", optional = true }
-duration-string = { version = "0.0.6", optional = true }
-rand = { version = "0.8.0", optional = true }
-rand_chacha = { version = "0.3.1", optional = true }
-lazy_static = { version = "1.4.0", optional = true }
-ark-ff = { version = "=0.4.1", default-features = false, features = [
+ark-ff = { version = "=0.4.1", default-features = false, optional = true, features = [
     "parallel",
 ] }
-
-# GPU explorations from mopro/Cargo.toml patch
-ark-bls12-377-3 = { git = 'https://github.com/arkworks-rs/curves.git', package = 'ark-bls12-377', tag = 'v0.3.0', optional = true}
-ark-ec-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-ec', tag = 'v0.3.0', features = ["parallel"], optional = true}
-ark-ff-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-ff', tag = 'v0.3.0', features = ["parallel"], optional = true }
-ark-serialize-3 = { git = 'https://github.com/arkworks-rs/algebra.git', package = 'ark-serialize', tag = 'v0.3.0', optional = true }
-ark-std-3 = { git = 'https://github.com/arkworks-rs/std.git', package = 'ark-std', tag = 'v0.3.0', optional = true }
+ark-bls12-381 ={ version = "=0.4.0", optional = true }
+metal = { version = "=0.28.0", optional = true }
+objc ={ version = "=0.2.4", optional = true }
+proptest ={ version = "1.4.0", optional = true }
 
 [build-dependencies]
 color-eyre = "0.6"
@@ -81,4 +67,4 @@ witness = { git = "https://github.com/philsippl/circom-witness-rs.git", optional
 
 [dependencies.rayon]
 version = "1"
-optional = true
+optional = true

mopro-core/gpu_explorations/benchmarks/arkworks_pippenger_benchmark.txt

@@ -1,9 +1,7 @@
 msm_size,num_msm,avg_processing_time(ms)
-8,5,3.302555666666667
-8,10,1.7059585000000002
-12,5,11.544680666666668
-12,10,11.7898874
-16,5,128.50465099999997
-16,10,139.1740167
-18,5,472.9359916
-18,10,477.0808459000001
+8,10,1.3042915
+12,10,8.4526458
+16,10,82.19201659999999
+18,10,307.24009179999996
+20,10,1140.8793625
+22,10,4160.4375503

mopro-core/gpu_explorations/benchmarks/halo2curve_multicore_msm_benchmark.txt

@@ -0,0 +1,7 @@
+msm_size,num_msm,avg_processing_time(ms)
+8,10,2.72165
+12,10,12.6313959
+16,10,116.79077500000001
+18,10,410.9840459
+20,10,1544.3454166
+22,10,5254.7052669

mopro-core/gpu_explorations/benchmarks/halo2curve_singlecore_msm_benchmark.txt

@@ -0,0 +1,7 @@
+msm_size,num_msm,avg_processing_time(ms)
+8,10,5.959329
+12,10,39.155546
+16,10,464.353125
+18,10,1649.2290374
+20,10,6154.858971
+22,10,20962.7212291

mopro-core/src/middleware/gpu_explorations/arkworks_pippenger.rs

@@ -1,7 +1,5 @@
-use ark_bls12_377::{Fr as ScalarField, G1Affine, G1Projective};
-// use ark_bn254::{Fr as ScalarField, FrConfig, G1Affine as GAffine, G1Projective as G};
+use ark_bn254::{Fr as ScalarField, G1Projective as G};
 use ark_ec::VariableBaseMSM;
-use ark_ff::BigInt;
 use std::time::{Duration, Instant};
 
 use crate::middleware::gpu_explorations::utils::{benchmark::BenchmarkResult, preprocess};
@@ -17,28 +15,16 @@ where
 
     for instance in instances {
         let points = &instance.0;
-        let scalars = &instance.1;
-        let mut parsed_points = Vec::<G1Affine>::new();
-        let mut parsed_scalars = Vec::<ScalarField>::new();
-
-        // parse points and scalars from arkworks 0.3 compatible format to 0.4 compatible
-        for p in points {
-            let new_p =
-                G1Affine::new_unchecked(BigInt::new(p.x.0 .0).into(), BigInt::new(p.y.0 .0).into());
-            parsed_points.push(new_p);
-        }
-
-        for s in scalars {
-            let new_s = ScalarField::new(BigInt::new(s.0));
-            parsed_scalars.push(new_s);
-        }
-
+        // map each scalar to a ScalarField
+        let scalars = &instance
+            .1
+            .iter()
+            .map(|s| ScalarField::new(*s))
+            .collect::<Vec<ScalarField>>();
         let mut instance_total_duration = Duration::ZERO;
         for _i in 0..iterations {
             let start = Instant::now();
-            let _result =
-                <G1Projective as VariableBaseMSM>::msm(&parsed_points[..], &parsed_scalars[..])
-                    .unwrap();
+            let _result = <G as VariableBaseMSM>::msm(&points[..], &scalars[..]).unwrap();
 
             instance_total_duration += start.elapsed();
         }
@@ -97,12 +83,18 @@ mod tests {
 
     const INSTANCE_SIZE: u32 = 16;
     const NUM_INSTANCE: u32 = 10;
-    const UTILSPATH: &str = "../mopro-core/src/middleware/gpu_explorations/utils/vectors";
-    const BENCHMARKSPATH: &str = "../mopro-core/gpu_explorations/benchmarks";
+    const UTILSPATH: &str = "mopro-core/src/middleware/gpu_explorations/utils/vectors";
+    const BENCHMARKSPATH: &str = "mopro-core/gpu_explorations/benchmarks";
 
     #[test]
     fn test_benchmark_msm() {
-        let dir = format!("{}/{}x{}", UTILSPATH, INSTANCE_SIZE, NUM_INSTANCE);
+        let dir = format!(
+            "{}/{}/{}x{}",
+            preprocess::get_root_path(),
+            UTILSPATH,
+            INSTANCE_SIZE,
+            NUM_INSTANCE
+        );
 
         // Check if the vectors have been generated
         match preprocess::FileInputIterator::open(&dir) {
@@ -121,29 +113,47 @@ mod tests {
 
     #[test]
     fn test_run_benchmark() {
-        let utils_path = format!("{}/{}x{}", &UTILSPATH, INSTANCE_SIZE, NUM_INSTANCE);
+        let utils_path = format!(
+            "{}/{}/{}x{}",
+            preprocess::get_root_path(),
+            &UTILSPATH,
+            INSTANCE_SIZE,
+            NUM_INSTANCE
+        );
         let result = run_benchmark(INSTANCE_SIZE, NUM_INSTANCE, &utils_path).unwrap();
         println!("Benchmark result: {:#?}", result);
     }
 
     #[test]
     fn test_run_multi_benchmarks() {
-        let output_path = format!("{}/{}_benchmark.txt", &BENCHMARKSPATH, "arkworks_pippenger");
+        let output_path = format!(
+            "{}/{}/{}_benchmark.txt",
+            preprocess::get_root_path(),
+            &BENCHMARKSPATH,
+            "arkworks_pippenger"
+        );
         let mut output_file = File::create(output_path).expect("output file creation failed");
-        writeln!(output_file, "msm_size,num_msm,avg_processing_time(ms)");
+        writeln!(output_file, "msm_size,num_msm,avg_processing_time(ms)").unwrap();
 
-        let instance_size = vec![8, 12, 16, 18, 20];
-        let num_instance = vec![5, 10];
+        let instance_size = vec![8, 12, 16, 18, 20, 22];
+        let num_instance = vec![10];
         for size in &instance_size {
             for num in &num_instance {
-                let utils_path = format!("{}/{}x{}", &UTILSPATH, *size, *num);
+                let utils_path = format!(
+                    "{}/{}/{}x{}",
+                    preprocess::get_root_path(),
+                    &UTILSPATH,
+                    *size,
+                    *num
+                );
                 let result = run_benchmark(*size, *num, &utils_path).unwrap();
                 println!("{}x{} result: {:#?}", *size, *num, result);
                 writeln!(
                     output_file,
                     "{},{},{}",
                     result.instance_size, result.num_instance, result.avg_processing_time
-                );
+                )
+                .unwrap();
             }
         }
     }