feat: programs batched mts picked

.github/workflows/forester-build-and-push-to-docr.yml

@@ -3,7 +3,7 @@ name: Docker Build and Push to DOCR
 on:
   push:
     branches:
-      - main
+      - "*"
   pull_request:
     branches:
       - "**"

.github/workflows/forester-tests.yml

@@ -1,7 +1,8 @@
 on:
   push:
     branches:
-      - main
+      - "*"
+      - jorrit/feat-bloomfilter-queues
     paths:
       - "forester/**"
       - "photon-api/**"

.github/workflows/js.yml

@@ -1,7 +1,8 @@
 on:
   push:
     branches:
-      - main
+      - "*"
+      - jorrit/feat-bloomfilter-queues
   pull_request:
     branches:
       - "*"

.github/workflows/light-examples-tests.yml

@@ -1,7 +1,7 @@
 on:
   push:
     branches:
-      - main
+      - "*"
     paths:
       - "examples/**"
       - "macros/light-sdk-macros/**"

.github/workflows/light-system-programs-tests.yml

@@ -1,7 +1,7 @@
 on:
   push:
     branches:
-      - main
+      - "*"
     paths:
       - "programs/**"
       - "test-programs/**"

.github/workflows/lint.yml

@@ -1,7 +1,8 @@
 on:
   push:
     branches:
-      - main
+      - "*"
+      - jorrit/feat-bloomfilter-queues
   pull_request:
     branches:
       - "*"

.github/workflows/prover-test.yml

@@ -2,7 +2,7 @@ name: Test gnark prover
 on:
   push:
     branches:
-      - main
+      - "*"
       - 'release/**'
     paths:
       - "light-prover/**"

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     branches:
-      - main
+      - "*"
 
 jobs:
   release:

.github/workflows/rust.yml

@@ -1,7 +1,7 @@
 on:
   push:
     branches:
-      - main
+      - "*"
     paths:
       - ".cargo/**"
       - "**/*.rs"

Cargo.toml

@@ -102,6 +102,9 @@ num-traits = "0.2.19"
 # HTTP client
 reqwest = "0.11.26"
 
+# Testing
+serial_test = "3.1.1"
+
 [patch.crates-io]
 "solana-account-decoder" = { git = "https://github.com/lightprotocol/agave", branch = "v1.18.22-enforce-cpi-tracking" }
 "solana-accounts-db" = { git = "https://github.com/lightprotocol/agave", branch = "v1.18.22-enforce-cpi-tracking" }

circuit-lib/light-prover-client/Cargo.toml

@@ -53,4 +53,4 @@ sysinfo = "0.31"
 borsh = ">=0.9, <0.11"
 [dev-dependencies]
 duct = "0.13.7"
-serial_test = "3.1.1"
+serial_test = "3.1.1"

circuit-lib/light-prover-client/src/gnark/helpers.rs

@@ -45,7 +45,8 @@ pub enum ProofType {
     Inclusion,
     NonInclusion,
     Combined,
-    BatchAppend,
+    BatchAppendWithSubtrees,
+    BatchAppendWithProofs,
     BatchUpdate,
     BatchAppendWithSubtreesTest,
     BatchUpdateTest,
@@ -61,7 +62,8 @@ impl Display for ProofType {
                 ProofType::Inclusion => "inclusion",
                 ProofType::NonInclusion => "non-inclusion",
                 ProofType::Combined => "combined",
-                ProofType::BatchAppend => "append",
+                ProofType::BatchAppendWithSubtrees => "append-with-subtrees",
+                ProofType::BatchAppendWithProofs => "append-with-proofs",
                 ProofType::BatchUpdate => "update",
                 ProofType::BatchAppendWithSubtreesTest => "append-with-subtrees-test",
                 ProofType::BatchUpdateTest => "update-test",

circuit-lib/light-prover-client/src/mock_batched_forester.rs

@@ -17,7 +17,7 @@ use crate::{
     },
 };
 
-// TODO: rename to MockBatchedForester
+#[derive(Clone, Debug)]
 pub struct MockBatchedForester<const HEIGHT: usize> {
     pub merkle_tree: MerkleTree<Poseidon>,
     pub input_queue_leaves: Vec<([u8; 32], usize)>,

circuit-lib/verifier/src/lib.rs

@@ -248,7 +248,7 @@ fn verify<const N: usize>(
 }
 
 #[inline(never)]
-pub fn verify_batch_append(
+pub fn verify_batch_append_with_subtrees(
     batch_size: usize,
     public_input_hash: [u8; 32],
     compressed_proof: &CompressedProof,

client/src/indexer/test_indexer.rs

@@ -288,6 +288,7 @@ where
                 let proof_json = deserialize_gnark_proof_json(&body).unwrap();
                 let (proof_a, proof_b, proof_c) = proof_from_json_struct(proof_json);
                 let (proof_a, proof_b, proof_c) = compress_proof(&proof_a, &proof_b, &proof_c);
+                let root_indices = root_indices.iter().map(|x| Some(*x)).collect();
                 return ProofRpcResult {
                     root_indices,
                     address_root_indices,

client/src/transaction_params.rs

@@ -34,3 +34,16 @@ impl Default for FeeConfig {
         }
     }
 }
+
+impl FeeConfig {
+    pub fn test_batched() -> Self {
+        Self {
+            // rollover fee plus additonal lamports for the cpi account
+            state_merkle_tree_rollover: 8,
+            address_queue_rollover: 392, // not batched
+            network_fee: 5000,
+            address_network_fee: 5000,
+            solana_network_fee: 5000,
+        }
+    }
+}

examples/name-service/programs/name-service/tests/test.rs

@@ -394,7 +394,7 @@ where
         inputs,
         proof: rpc_result.proof,
         merkle_context,
-        merkle_tree_root_index: rpc_result.root_indices[0],
+        merkle_tree_root_index: rpc_result.root_indices[0].unwrap(),
         address_merkle_context: *address_merkle_context,
         address_merkle_tree_root_index: 0,
         new_rdata: new_rdata.clone(),
@@ -471,7 +471,7 @@ where
         inputs,
         proof: rpc_result.proof,
         merkle_context,
-        merkle_tree_root_index: rpc_result.root_indices[0],
+        merkle_tree_root_index: rpc_result.root_indices[0].unwrap(),
         address_merkle_context: *address_merkle_context,
         address_merkle_tree_root_index: 0,
     };

examples/token-escrow/programs/token-escrow/src/escrow_with_compressed_pda/sdk.rs

@@ -25,7 +25,7 @@ pub struct CreateCompressedPdaEscrowInstructionInputs<'a> {
     pub input_merkle_context: &'a [MerkleContext],
     pub output_compressed_account_merkle_tree_pubkeys: &'a [Pubkey],
     pub output_compressed_accounts: &'a [TokenTransferOutputData],
-    pub root_indices: &'a [u16],
+    pub root_indices: &'a [Option<u16>],
     pub proof: &'a Option<CompressedProof>,
     pub input_token_data: &'a [light_compressed_token::token_data::TokenData],
     pub input_compressed_accounts: &'a [CompressedAccount],
@@ -137,7 +137,7 @@ pub struct CreateCompressedPdaWithdrawalInstructionInputs<'a> {
     pub input_cpda_merkle_context: MerkleContext,
     pub output_compressed_account_merkle_tree_pubkeys: &'a [Pubkey],
     pub output_compressed_accounts: &'a [TokenTransferOutputData],
-    pub root_indices: &'a [u16],
+    pub root_indices: &'a [Option<u16>],
     pub proof: &'a Option<CompressedProof>,
     pub input_token_data: &'a [light_compressed_token::token_data::TokenData],
     pub input_compressed_accounts: &'a [CompressedAccount],
@@ -201,7 +201,7 @@ pub fn create_withdrawal_instruction(
         new_lock_up_time: input_params.new_lock_up_time,
         address: input_params.address,
         merkle_context: merkle_context_packed[0],
-        root_index: input_params.root_indices[0],
+        root_index: input_params.root_indices[0].unwrap_or_default(),
     };
     let instruction_data = crate::instruction::WithdrawCompressedTokensWithCompressedPda {
         proof: input_params.proof.clone().unwrap(),

examples/token-escrow/programs/token-escrow/src/escrow_with_pda/sdk.rs

@@ -29,7 +29,7 @@ pub struct CreateEscrowInstructionInputs<'a> {
     pub input_merkle_context: &'a [MerkleContext],
     pub output_compressed_account_merkle_tree_pubkeys: &'a [Pubkey],
     pub output_compressed_accounts: &'a [TokenTransferOutputData],
-    pub root_indices: &'a [u16],
+    pub root_indices: &'a [Option<u16>],
     pub proof: &'a Option<CompressedProof>,
     pub input_token_data: &'a [light_compressed_token::token_data::TokenData],
     pub input_compressed_accounts: &'a [CompressedAccount],

examples/token-escrow/programs/token-escrow/tests/test.rs

@@ -237,10 +237,12 @@ pub async fn perform_escrow<R: RpcConnection>(
 
     let rpc_result = test_indexer
         .create_proof_for_compressed_accounts(
-            Some(&[input_compressed_account_hash]),
-            Some(&[compressed_input_account_with_context
-                .merkle_context
-                .merkle_tree_pubkey]),
+            Some(vec![input_compressed_account_hash]),
+            Some(vec![
+                compressed_input_account_with_context
+                    .merkle_context
+                    .merkle_tree_pubkey,
+            ]),
             None,
             None,
             rpc,
@@ -301,7 +303,8 @@ pub async fn perform_escrow_with_event<R: RpcConnection>(
         )
         .await?
         .unwrap();
-    test_indexer.add_compressed_accounts_with_token_data(&event.0);
+    let slot = rpc.get_slot().await.unwrap();
+    test_indexer.add_compressed_accounts_with_token_data(slot, &event.0);
     Ok(())
 }
 
@@ -395,10 +398,12 @@ pub async fn perform_withdrawal<R: RpcConnection>(
 
     let rpc_result = test_indexer
         .create_proof_for_compressed_accounts(
-            Some(&[input_compressed_account_hash]),
-            Some(&[compressed_input_account_with_context
-                .merkle_context
-                .merkle_tree_pubkey]),
+            Some(vec![input_compressed_account_hash]),
+            Some(vec![
+                compressed_input_account_with_context
+                    .merkle_context
+                    .merkle_tree_pubkey,
+            ]),
             None,
             None,
             context,
@@ -457,7 +462,8 @@ pub async fn perform_withdrawal_with_event<R: RpcConnection>(
         )
         .await?
         .unwrap();
-    test_indexer.add_compressed_accounts_with_token_data(&event.0);
+    let slot = rpc.get_slot().await.unwrap();
+    test_indexer.add_compressed_accounts_with_token_data(slot, &event.0);
     Ok(())
 }
 

examples/token-escrow/programs/token-escrow/tests/test_compressed_pda.rs

@@ -52,7 +52,7 @@ async fn test_escrow_with_compressed_pda() {
     );
     let mint = create_mint_helper(&mut rpc, &payer).await;
     let mut test_indexer = test_indexer.await;
-
+    test_indexer.state_merkle_trees.remove(1);
     let amount = 10000u64;
     mint_tokens_helper(
         &mut rpc,
@@ -203,7 +203,8 @@ pub async fn perform_escrow_with_event<R: RpcConnection>(
             }),
         )
         .await?;
-    test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0);
+    let slot = rpc.get_slot().await.unwrap();
+    test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0);
     Ok(())
 }
 
@@ -236,10 +237,12 @@ async fn create_escrow_ix<R: RpcConnection>(
 
     let rpc_result = test_indexer
         .create_proof_for_compressed_accounts(
-            Some(&[input_compressed_account_hash]),
-            Some(&[compressed_input_account_with_context
-                .merkle_context
-                .merkle_tree_pubkey]),
+            Some(vec![input_compressed_account_hash]),
+            Some(vec![
+                compressed_input_account_with_context
+                    .merkle_context
+                    .merkle_tree_pubkey,
+            ]),
             Some(&[address]),
             Some(vec![env.address_merkle_tree_pubkey]),
             context,
@@ -371,7 +374,8 @@ pub async fn perform_withdrawal_with_event<R: RpcConnection>(
             None,
         )
         .await?;
-    test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0);
+    let slot = rpc.get_slot().await.unwrap();
+    test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0);
     Ok(())
 }
 
@@ -450,8 +454,8 @@ pub async fn perform_withdrawal<R: RpcConnection>(
     // the compressed pda program executes the transaction
     let rpc_result = test_indexer
         .create_proof_for_compressed_accounts(
-            Some(&[compressed_pda_hash, token_escrow_account_hash]),
-            Some(&[
+            Some(vec![compressed_pda_hash, token_escrow_account_hash]),
+            Some(vec![
                 compressed_escrow_pda.merkle_context.merkle_tree_pubkey,
                 token_escrow_account.merkle_context.merkle_tree_pubkey,
             ]),