diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ccc8322787..8e318c11df 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -80,7 +80,7 @@ jobs: { echo "Testing group: ${{ matrix.group.name }}" echo "Packages: ${{ matrix.group.packages }}" - echo "Rust version: $(rustc --version)" + echo "Rust version: $(rustc --version)" } >> "$GITHUB_STEP_SUMMARY" # Function to format time duration diff --git a/Cargo.lock b/Cargo.lock index f0391181bd..d3bf2e06a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,7 +53,7 @@ dependencies = [ "light-batched-merkle-tree", "light-bloom-filter", "light-bounded-vec", - "light-compressed-token", + "light-client", "light-concurrent-merkle-tree", "light-hash-set", "light-hasher", @@ -62,22 +62,14 @@ dependencies = [ "light-merkle-tree-reference", "light-program-test", "light-prover-client", - "light-system-program", "light-test-utils", "light-utils 1.1.0", "light-verifier", - "memoffset 0.9.1", "num-bigint 0.4.6", - "num-traits", "rand 0.8.5", - "reqwest 0.11.27", - "serde_json", "serial_test", - "solana-cli-output", "solana-program-test", "solana-sdk", - "spl-token", - "thiserror 1.0.64", "tokio", ] @@ -682,12 +674,6 @@ dependencies = [ "syn 2.0.96", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "atty" version = "0.2.14" @@ -1243,21 +1229,16 @@ dependencies = [ "account-compression", "anchor-lang", "anchor-spl", + "light-client", "light-compressed-token", - "light-concurrent-merkle-tree", - "light-hasher", "light-program-test", "light-prover-client", + "light-sdk", "light-system-program", "light-test-utils", - "light-utils 1.1.0", "light-verifier", - "num-bigint 0.4.6", - "num-traits", "rand 0.8.5", - "reqwest 0.11.27", "serial_test", - "solana-program-test", "solana-sdk", "spl-token", "tokio", @@ -1716,11 +1697,11 @@ dependencies = [ "num-bigint 0.4.6", "num-traits", "rand 0.8.5", - "reqwest 0.11.27", + "reqwest", "solana-client", "solana-program-test", "solana-sdk", - "spl-concurrent-merkle-tree 0.2.0", + "spl-concurrent-merkle-tree", "spl-token", "tokio", ] @@ -1985,11 +1966,12 @@ dependencies = [ "light-program-test", "light-prover-client", "light-registry", + "light-sdk", "light-system-program", "light-test-utils", "photon-api", "prometheus", - "reqwest 0.11.27", + "reqwest", "scopeguard", "serde", "serde_json", @@ -2031,7 +2013,7 @@ dependencies = [ "num-bigint 0.4.6", "num-traits", "photon-api", - "reqwest 0.11.27", + "reqwest", "solana-client", "solana-program-test", "solana-sdk", @@ -2248,25 +2230,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "h2" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http 1.1.0", - "indexmap 2.5.0", - "slab", - "tokio", - "tokio-util 0.7.13", - "tracing", -] - [[package]] name = "hash32" version = "0.2.1" @@ -2444,29 +2407,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http 1.1.0", -] - -[[package]] -name = "http-body-util" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" -dependencies = [ - "bytes", - "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "pin-project-lite", -] - [[package]] name = "httparse" version = "1.9.4" @@ -2495,9 +2435,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", - "http-body 0.4.6", + "http-body", "httparse", "httpdate", "itoa", @@ -2509,26 +2449,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2 0.4.6", - "http 1.1.0", - "http-body 1.0.1", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2537,7 +2457,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.30", + "hyper", "rustls", "tokio", "tokio-rustls", @@ -2550,48 +2470,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.30", + "hyper", "native-tls", "tokio", "tokio-native-tls", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper 1.4.1", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "hyper 1.4.1", - "pin-project-lite", - "socket2", - "tokio", - "tower", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2931,7 +2815,7 @@ dependencies = [ "num-traits", "photon-api", "rand 0.8.5", - "reqwest 0.11.27", + "reqwest", "solana-banks-client", "solana-client", "solana-program", @@ -2968,7 +2852,6 @@ dependencies = [ "ark-bn254", "ark-ff", "borsh 0.10.3", - "bytemuck", "light-bounded-vec", "light-hash-set", "light-hasher", @@ -2979,8 +2862,7 @@ dependencies = [ "num-traits", "rand 0.8.5", "solana-program", - "spl-account-compression", - "spl-concurrent-merkle-tree 0.2.0", + "spl-concurrent-merkle-tree", "thiserror 1.0.64", "tokio", ] @@ -3104,6 +2986,7 @@ dependencies = [ "light-compressed-token", "light-hasher", "light-indexed-merkle-tree", + "light-merkle-tree-metadata", "light-merkle-tree-reference", "light-prover-client", "light-registry", @@ -3114,7 +2997,7 @@ dependencies = [ "log", "num-bigint 0.4.6", "num-traits", - "reqwest 0.11.27", + "reqwest", "solana-banks-client", "solana-program-test", "solana-sdk", @@ -3141,7 +3024,7 @@ dependencies = [ "num-bigint 0.4.6", "num-traits", "once_cell", - "reqwest 0.11.27", + "reqwest", "serde", "serde_json", "serial_test", @@ -3171,35 +3054,18 @@ name = "light-sdk" version = "0.11.0" dependencies = [ "account-compression", - "aligned-sized", "anchor-lang", "borsh 0.10.3", - "bytemuck", - "groth16-solana", - "lazy_static", - "light-concurrent-merkle-tree", - "light-hash-set", "light-hasher", "light-heap", "light-indexed-merkle-tree", "light-macros", - "light-merkle-tree-reference", - "light-prover-client", "light-sdk-macros", "light-system-program", "light-utils 1.1.0", - "light-verifier", "num-bigint 0.4.6", - "num-traits", - "rand 0.8.5", - "reqwest 0.12.4", - "serde_json", - "solana-banks-interface", - "solana-cli-output", "solana-program", - "solana-program-test", "solana-sdk", - "tokio", ] [[package]] @@ -3261,6 +3127,7 @@ dependencies = [ "light-program-test", "light-prover-client", "light-registry", + "light-sdk", "light-system-program", "light-utils 1.1.0", "light-verifier", @@ -3270,7 +3137,7 @@ dependencies = [ "num-traits", "photon-api", "rand 0.8.5", - "reqwest 0.11.27", + "reqwest", "serde", "solana-client", "solana-program-test", @@ -3292,10 +3159,8 @@ dependencies = [ "ark-ff", "borsh 0.10.3", "bytemuck", - "light-bounded-vec", "light-hasher", "light-poseidon", - "memoffset 0.9.1", "num-bigint 0.4.6", "rand 0.8.5", "solana-program", @@ -3327,7 +3192,7 @@ dependencies = [ "groth16-solana", "light-prover-client", "light-utils 1.1.0", - "reqwest 0.11.27", + "reqwest", "serial_test", "solana-program", "thiserror 1.0.64", @@ -4027,7 +3892,7 @@ dependencies = [ name = "photon-api" version = "0.45.0" dependencies = [ - "reqwest 0.11.27", + "reqwest", "serde", "serde_derive", "serde_json", @@ -4550,29 +4415,17 @@ version = "1.1.0" dependencies = [ "account-compression", "anchor-lang", - "anchor-spl", "forester-utils", "light-batched-merkle-tree", - "light-compressed-token", - "light-concurrent-merkle-tree", + "light-client", "light-hasher", - "light-indexed-merkle-tree", "light-program-test", "light-prover-client", "light-registry", - "light-system-program", "light-test-utils", "light-utils 1.1.0", - "light-verifier", - "num-bigint 0.4.6", - "num-traits", - "reqwest 0.11.27", - "serde_json", "serial_test", - "solana-cli-output", - "solana-program-test", "solana-sdk", - "spl-token", "tokio", ] @@ -4588,12 +4441,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.3.26", + "h2", "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", + "http-body", + "hyper", "hyper-rustls", - "hyper-tls 0.5.0", + "hyper-tls", "ipnet", "js-sys", "log", @@ -4604,7 +4457,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -4620,49 +4473,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots 0.25.4", - "winreg 0.50.0", -] - -[[package]] -name = "reqwest" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" -dependencies = [ - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.4.6", - "http 1.1.0", - "http-body 1.0.1", - "http-body-util", - "hyper 1.4.1", - "hyper-tls 0.6.0", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile 2.1.3", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg 0.52.0", + "winreg", ] [[package]] @@ -4778,7 +4589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "schannel", "security-framework", ] @@ -4792,22 +4603,6 @@ dependencies = [ "base64 0.21.7", ] -[[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" -dependencies = [ - "base64 0.22.1", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4910,17 +4705,12 @@ name = "sdk-test" version = "0.7.0" dependencies = [ "anchor-lang", - "borsh 0.10.3", "light-client", "light-hasher", - "light-macros", "light-program-test", + "light-prover-client", "light-sdk", - "light-sdk-macros", "light-test-utils", - "light-utils 1.1.0", - "light-verifier", - "solana-program-test", "solana-sdk", "tokio", ] @@ -5654,7 +5444,7 @@ dependencies = [ "gethostname", "lazy_static", "log", - "reqwest 0.11.27", + "reqwest", "solana-sdk", "thiserror 1.0.64", ] @@ -5832,7 +5622,7 @@ dependencies = [ "crossbeam-channel", "futures-util", "log", - "reqwest 0.11.27", + "reqwest", "semver", "serde", "serde_derive", @@ -5912,7 +5702,7 @@ dependencies = [ "bs58 0.4.0", "indicatif", "log", - "reqwest 0.11.27", + "reqwest", "semver", "serde", "serde_derive", @@ -5934,7 +5724,7 @@ dependencies = [ "base64 0.21.7", "bs58 0.4.0", "jsonrpc-core", - "reqwest 0.11.27", + "reqwest", "semver", "serde", "serde_derive", @@ -6392,20 +6182,6 @@ dependencies = [ "der", ] -[[package]] -name = "spl-account-compression" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcf740e5242f2ad63325e600c368702f32db84608fc8b70d70633c68dd1486d" -dependencies = [ - "anchor-lang", - "bytemuck", - "solana-program", - "solana-security-txt", - "spl-concurrent-merkle-tree 0.3.0", - "spl-noop", -] - [[package]] name = "spl-associated-token-account" version = "2.3.0" @@ -6433,17 +6209,6 @@ dependencies = [ "thiserror 1.0.64", ] -[[package]] -name = "spl-concurrent-merkle-tree" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5f45b971d82cbb0416fdffad3c9098f259545d54072e83a0a482f60f8f689" -dependencies = [ - "bytemuck", - "solana-program", - "thiserror 1.0.64", -] - [[package]] name = "spl-discriminator" version = "0.1.0" @@ -6523,15 +6288,6 @@ dependencies = [ "solana-program", ] -[[package]] -name = "spl-noop" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd67ea3d0070a12ff141f5da46f9695f49384a03bce1203a5608f5739437950" -dependencies = [ - "solana-program", -] - [[package]] name = "spl-pod" version = "0.1.0" @@ -7016,19 +6772,21 @@ dependencies = [ "anchor-lang", "anchor-spl", "light-batched-merkle-tree", + "light-client", "light-compressed-token", "light-hasher", "light-merkle-tree-metadata", "light-program-test", "light-prover-client", "light-registry", + "light-sdk", "light-system-program", "light-test-utils", "light-utils 1.1.0", "light-verifier", "num-bigint 0.4.6", "num-traits", - "reqwest 0.11.27", + "reqwest", "serial_test", "solana-program-test", "solana-sdk", @@ -7042,30 +6800,23 @@ version = "1.1.0" dependencies = [ "account-compression", "anchor-lang", - "anchor-spl", "light-batched-merkle-tree", - "light-compressed-token", - "light-concurrent-merkle-tree", + "light-client", "light-hasher", - "light-indexed-merkle-tree", "light-merkle-tree-metadata", "light-program-test", "light-prover-client", "light-registry", + "light-sdk", "light-system-program", "light-test-utils", "light-utils 1.1.0", "light-verifier", - "num-bigint 0.4.6", - "num-traits", "quote", - "reqwest 0.11.27", "serde_json", "serial_test", "solana-cli-output", - "solana-program-test", "solana-sdk", - "spl-token", "tokio", ] @@ -7342,6 +7093,7 @@ dependencies = [ "account-compression", "anchor-lang", "anchor-spl", + "light-client", "light-compressed-token", "light-hasher", "light-program-test", @@ -7522,27 +7274,6 @@ dependencies = [ "winnow 0.6.18", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -7895,7 +7626,7 @@ dependencies = [ "futures-util", "headers", "http 0.2.12", - "hyper 0.14.30", + "hyper", "log", "mime", "mime_guess", @@ -8286,16 +8017,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wyz" version = "0.5.1" diff --git a/examples/name-service/programs/name-service-without-macros/tests/test.rs b/examples/name-service/programs/name-service-without-macros/tests/test.rs index 3d01c29027..f98d5f20dd 100644 --- a/examples/name-service/programs/name-service-without-macros/tests/test.rs +++ b/examples/name-service/programs/name-service-without-macros/tests/test.rs @@ -348,7 +348,8 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } @@ -421,7 +422,8 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } diff --git a/examples/name-service/programs/name-service/tests/test.rs b/examples/name-service/programs/name-service/tests/test.rs index 17a75eefae..35dd036a59 100644 --- a/examples/name-service/programs/name-service/tests/test.rs +++ b/examples/name-service/programs/name-service/tests/test.rs @@ -356,7 +356,8 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } @@ -435,7 +436,8 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } diff --git a/examples/token-escrow/programs/token-escrow/Cargo.toml b/examples/token-escrow/programs/token-escrow/Cargo.toml index 9d0ca72135..812b0e538e 100644 --- a/examples/token-escrow/programs/token-escrow/Cargo.toml +++ b/examples/token-escrow/programs/token-escrow/Cargo.toml @@ -25,13 +25,14 @@ light-compressed-token = { workspace = true } light-system-program = { workspace = true } account-compression = { workspace = true } light-hasher = { workspace = true } -light-verifier = { workspace = true } light-sdk = { workspace = true, features = ["legacy"] } [target.'cfg(not(target_os = "solana"))'.dependencies] solana-sdk = { workspace = true } [dev-dependencies] +light-verifier = { workspace = true } +light-client = { workspace = true } light-test-utils = { workspace = true, features = ["devenv"] } light-program-test = { workspace = true, features = ["devenv"] } tokio = { workspace = true } diff --git a/examples/token-escrow/programs/token-escrow/tests/test.rs b/examples/token-escrow/programs/token-escrow/tests/test.rs index bd0b925041..7ff502fa29 100644 --- a/examples/token-escrow/programs/token-escrow/tests/test.rs +++ b/examples/token-escrow/programs/token-escrow/tests/test.rs @@ -10,15 +10,22 @@ // - create escrow pda and just prove that utxo exists -> read utxo from compressed token account // release compressed tokens +use light_client::indexer::Indexer; use light_hasher::Poseidon; -use light_program_test::test_env::{setup_test_programs_with_accounts, EnvAccounts}; +use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, + test_env::{setup_test_programs_with_accounts, EnvAccounts}, +}; use light_prover_client::gnark::helpers::{ProofType, ProverConfig}; use light_system_program::sdk::{compressed_account::MerkleContext, event::PublicTransactionEvent}; use light_test_utils::{ airdrop_lamports, assert_rpc_error, - indexer::TestIndexer, + conversions::{ + program_to_sdk_public_transaction_event, sdk_to_program_compressed_account, + sdk_to_program_compressed_proof, sdk_to_program_token_data, + }, spl::{create_mint_helper, mint_tokens_helper}, - FeeConfig, Indexer, RpcConnection, RpcError, TransactionParams, + FeeConfig, RpcConnection, RpcError, TransactionParams, }; use light_verifier::VerifierError; use solana_sdk::{ @@ -205,16 +212,16 @@ async fn test_escrow_pda() { ); } -pub async fn perform_escrow( +pub async fn perform_escrow + TestIndexerExtensions>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, escrow_amount: &u64, lock_up_time: &u64, ) -> Instruction { let input_compressed_token_account_data = test_indexer - .token_compressed_accounts + .get_token_compressed_accounts() .iter() .find(|x| { println!("searching token account: {:?}", x.token_data); @@ -253,7 +260,9 @@ pub async fn perform_escrow( .await; let create_ix_inputs = CreateEscrowInstructionInputs { - input_token_data: &[input_compressed_token_account_data.token_data.clone()], + input_token_data: &[sdk_to_program_token_data( + input_compressed_token_account_data.token_data.clone(), + )], lock_up_time: *lock_up_time, signer: &payer_pubkey, input_merkle_context: &[MerkleContext { @@ -270,16 +279,21 @@ pub async fn perform_escrow( ], output_compressed_accounts: &Vec::new(), root_indices: &rpc_result.root_indices, - proof: &Some(rpc_result.proof), + proof: &Some(sdk_to_program_compressed_proof(rpc_result.proof)), mint: &input_compressed_token_account_data.token_data.mint, - input_compressed_accounts: &[compressed_input_account_with_context.compressed_account], + input_compressed_accounts: &[sdk_to_program_compressed_account( + compressed_input_account_with_context.compressed_account, + )], }; create_escrow_instruction(create_ix_inputs, *escrow_amount) } -pub async fn perform_escrow_with_event( +pub async fn perform_escrow_with_event< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, escrow_amount: &u64, @@ -307,13 +321,16 @@ pub async fn perform_escrow_with_event( .await? .unwrap(); let slot = rpc.get_slot().await.unwrap(); - test_indexer.add_compressed_accounts_with_token_data(slot, &event.0); + test_indexer.add_compressed_accounts_with_token_data( + slot, + &program_to_sdk_public_transaction_event(event.0), + ); Ok(()) } -pub async fn perform_escrow_failing( +pub async fn perform_escrow_failing + TestIndexerExtensions>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, escrow_amount: &u64, @@ -330,9 +347,9 @@ pub async fn perform_escrow_failing( rpc.process_transaction(transaction).await } -pub async fn assert_escrow( +pub async fn assert_escrow + TestIndexerExtensions>( rpc: &mut R, - test_indexer: &TestIndexer, + test_indexer: &I, payer_pubkey: &Pubkey, amount: u64, escrow_amount: u64, @@ -340,7 +357,7 @@ pub async fn assert_escrow( ) { let token_owner_pda = get_token_owner_pda(payer_pubkey).0; let token_data_escrow = test_indexer - .token_compressed_accounts + .get_token_compressed_accounts() .iter() .find(|x| x.token_data.owner == token_owner_pda) .unwrap() @@ -349,8 +366,10 @@ pub async fn assert_escrow( assert_eq!(token_data_escrow.amount, escrow_amount); assert_eq!(token_data_escrow.owner, token_owner_pda); - let token_data_change_compressed_token_account = - test_indexer.token_compressed_accounts[0].token_data.clone(); + let token_data_change_compressed_token_account = test_indexer.get_token_compressed_accounts() + [0] + .token_data + .clone(); assert_eq!( token_data_change_compressed_token_account.amount, amount - escrow_amount @@ -369,9 +388,9 @@ pub async fn assert_escrow( assert_eq!(timelock_account.slot, *lock_up_time + current_slot); } -pub async fn perform_withdrawal( +pub async fn perform_withdrawal + TestIndexerExtensions>( context: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, withdrawal_amount: &u64, @@ -380,7 +399,7 @@ pub async fn perform_withdrawal( let payer_pubkey = payer.pubkey(); let token_owner_pda = get_token_owner_pda(&invalid_signer.unwrap_or(payer_pubkey)).0; let escrow_token_data_with_context = test_indexer - .token_compressed_accounts + .get_token_compressed_accounts() .iter() .find(|x| { x.token_data.owner == token_owner_pda && x.token_data.amount >= *withdrawal_amount @@ -414,7 +433,9 @@ pub async fn perform_withdrawal( .await; let create_ix_inputs = CreateEscrowInstructionInputs { - input_token_data: &[escrow_token_data_with_context.token_data.clone()], + input_token_data: &[sdk_to_program_token_data( + escrow_token_data_with_context.token_data.clone(), + )], lock_up_time: 0, signer: &payer_pubkey, input_merkle_context: &[MerkleContext { @@ -431,17 +452,22 @@ pub async fn perform_withdrawal( ], output_compressed_accounts: &Vec::new(), root_indices: &rpc_result.root_indices, - proof: &Some(rpc_result.proof), + proof: &Some(sdk_to_program_compressed_proof(rpc_result.proof)), mint: &escrow_token_data_with_context.token_data.mint, - input_compressed_accounts: &[compressed_input_account_with_context.compressed_account], + input_compressed_accounts: &[sdk_to_program_compressed_account( + compressed_input_account_with_context.compressed_account, + )], }; create_withdrawal_escrow_instruction(create_ix_inputs, *withdrawal_amount) } -pub async fn perform_withdrawal_with_event( +pub async fn perform_withdrawal_with_event< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, withdrawal_amount: &u64, @@ -466,13 +492,19 @@ pub async fn perform_withdrawal_with_event( .await? .unwrap(); let slot = rpc.get_slot().await.unwrap(); - test_indexer.add_compressed_accounts_with_token_data(slot, &event.0); + test_indexer.add_compressed_accounts_with_token_data( + slot, + &program_to_sdk_public_transaction_event(event.0), + ); Ok(()) } -pub async fn perform_withdrawal_failing( +pub async fn perform_withdrawal_failing< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, withdrawal_amount: &u64, @@ -495,15 +527,15 @@ pub async fn perform_withdrawal_failing( ); rpc.process_transaction(transaction).await } -pub fn assert_withdrawal( - test_indexer: &TestIndexer, +pub fn assert_withdrawal + TestIndexerExtensions>( + test_indexer: &I, payer_pubkey: &Pubkey, withdrawal_amount: u64, escrow_amount: u64, ) { let token_owner_pda = get_token_owner_pda(payer_pubkey).0; let token_data_withdrawal = test_indexer - .token_compressed_accounts + .get_token_compressed_accounts() .iter() .any(|x| x.token_data.owner == *payer_pubkey && x.token_data.amount == withdrawal_amount); @@ -512,10 +544,13 @@ pub fn assert_withdrawal( "Withdrawal compressed account doesn't exist or has incorrect amount {} expected amount", withdrawal_amount ); - let token_data_escrow_change = test_indexer.token_compressed_accounts.iter().any(|x| { - x.token_data.owner == token_owner_pda - && x.token_data.amount == escrow_amount - withdrawal_amount - }); + let token_data_escrow_change = test_indexer + .get_token_compressed_accounts() + .iter() + .any(|x| { + x.token_data.owner == token_owner_pda + && x.token_data.amount == escrow_amount - withdrawal_amount + }); assert!( token_data_escrow_change, "Escrow change compressed account doesn't exist or has incorrect amount {} expected amount", diff --git a/examples/token-escrow/programs/token-escrow/tests/test_compressed_pda.rs b/examples/token-escrow/programs/token-escrow/tests/test_compressed_pda.rs index f1f654fbdb..d77add741b 100644 --- a/examples/token-escrow/programs/token-escrow/tests/test_compressed_pda.rs +++ b/examples/token-escrow/programs/token-escrow/tests/test_compressed_pda.rs @@ -14,8 +14,12 @@ // release compressed tokens use anchor_lang::AnchorDeserialize; +use light_client::{indexer::Indexer, rpc::merkle_tree::MerkleTreeExt}; use light_hasher::{Hasher, Poseidon}; -use light_program_test::test_env::{setup_test_programs_with_accounts, EnvAccounts}; +use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, + test_env::{setup_test_programs_with_accounts, EnvAccounts}, +}; use light_prover_client::gnark::helpers::{ProverConfig, ProverMode}; use light_system_program::{ sdk::{ @@ -25,9 +29,12 @@ use light_system_program::{ NewAddressParams, }; use light_test_utils::{ - indexer::TestIndexer, + conversions::{ + program_to_sdk_public_transaction_event, sdk_to_program_compressed_account, + sdk_to_program_compressed_proof, sdk_to_program_token_data, + }, spl::{create_mint_helper, mint_tokens_helper}, - FeeConfig, Indexer, RpcConnection, RpcError, TransactionParams, + FeeConfig, RpcConnection, RpcError, TransactionParams, }; use solana_sdk::{ instruction::{Instruction, InstructionError}, @@ -62,7 +69,7 @@ async fn test_escrow_with_compressed_pda() { ); let mint = create_mint_helper(&mut rpc, &payer).await; let mut test_indexer = test_indexer.await; - test_indexer.state_merkle_trees.remove(1); + let amount = 10000u64; mint_tokens_helper( &mut rpc, @@ -150,7 +157,7 @@ async fn test_escrow_with_compressed_pda() { .await; } -pub async fn perform_escrow_failing( +pub async fn perform_escrow_failing( test_indexer: &mut TestIndexer, rpc: &mut R, env: &EnvAccounts, @@ -180,7 +187,7 @@ pub async fn perform_escrow_failing( rpc.process_transaction(transaction).await } -pub async fn perform_escrow_with_event( +pub async fn perform_escrow_with_event( test_indexer: &mut TestIndexer, rpc: &mut R, env: &EnvAccounts, @@ -214,11 +221,14 @@ pub async fn perform_escrow_with_event( ) .await?; let slot = rpc.get_slot().await.unwrap(); - test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); + test_indexer.add_compressed_accounts_with_token_data( + slot, + &program_to_sdk_public_transaction_event(event.unwrap().0), + ); Ok(()) } -async fn create_escrow_ix( +async fn create_escrow_ix( payer: &Keypair, test_indexer: &mut TestIndexer, env: &EnvAccounts, @@ -266,7 +276,9 @@ async fn create_escrow_ix( address_merkle_tree_root_index: rpc_result.address_root_indices[0], }; let create_ix_inputs = CreateCompressedPdaEscrowInstructionInputs { - input_token_data: &[input_compressed_token_account_data.token_data.clone()], + input_token_data: &[sdk_to_program_token_data( + input_compressed_token_account_data.token_data.clone(), + )], lock_up_time, signer: &payer_pubkey, input_merkle_context: &[MerkleContext { @@ -283,17 +295,19 @@ async fn create_escrow_ix( ], output_compressed_accounts: &Vec::new(), root_indices: &rpc_result.root_indices, - proof: &Some(rpc_result.proof), + proof: &Some(sdk_to_program_compressed_proof(rpc_result.proof)), mint: &input_compressed_token_account_data.token_data.mint, new_address_params, cpi_context_account: &env.cpi_context_account_pubkey, - input_compressed_accounts: &[compressed_input_account_with_context.compressed_account], + input_compressed_accounts: &[sdk_to_program_compressed_account( + compressed_input_account_with_context.compressed_account, + )], }; let instruction = create_escrow_instruction(create_ix_inputs.clone(), escrow_amount); (payer_pubkey, instruction) } -pub async fn assert_escrow( +pub async fn assert_escrow( test_indexer: &mut TestIndexer, env: &EnvAccounts, payer: &Keypair, @@ -327,6 +341,7 @@ pub async fn assert_escrow( .unwrap() .clone(); let address = derive_address_legacy(&env.address_merkle_tree_pubkey, seed).unwrap(); + assert_eq!( compressed_escrow_pda.compressed_account.address.unwrap(), address @@ -357,7 +372,7 @@ pub async fn assert_escrow( Poseidon::hash(&compressed_escrow_pda_data.slot.to_le_bytes()).unwrap(), ); } -pub async fn perform_withdrawal_with_event( +pub async fn perform_withdrawal_with_event( rpc: &mut R, test_indexer: &mut TestIndexer, env: &EnvAccounts, @@ -385,11 +400,14 @@ pub async fn perform_withdrawal_with_event( ) .await?; let slot = rpc.get_slot().await.unwrap(); - test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); + test_indexer.add_compressed_accounts_with_token_data( + slot, + &program_to_sdk_public_transaction_event(event.unwrap().0), + ); Ok(()) } -pub async fn perform_withdrawal_failing( +pub async fn perform_withdrawal_failing( rpc: &mut R, test_indexer: &mut TestIndexer, env: &EnvAccounts, @@ -417,7 +435,7 @@ pub async fn perform_withdrawal_failing( ); rpc.process_transaction(transaction).await } -pub async fn perform_withdrawal( +pub async fn perform_withdrawal( rpc: &mut R, test_indexer: &mut TestIndexer, env: &EnvAccounts, @@ -476,7 +494,7 @@ pub async fn perform_withdrawal( .await; let create_withdrawal_ix_inputs = CreateCompressedPdaWithdrawalInstructionInputs { - input_token_data: &[token_escrow.token_data.clone()], + input_token_data: &[sdk_to_program_token_data(token_escrow.token_data.clone())], signer: &payer_pubkey, input_token_escrow_merkle_context: MerkleContext { leaf_index: token_escrow_account.merkle_context.leaf_index, @@ -496,13 +514,15 @@ pub async fn perform_withdrawal( ], output_compressed_accounts: &Vec::new(), root_indices: &rpc_result.root_indices, - proof: &Some(rpc_result.proof), + proof: &Some(sdk_to_program_compressed_proof(rpc_result.proof)), mint: &token_escrow.token_data.mint, cpi_context_account: &env.cpi_context_account_pubkey, old_lock_up_time, new_lock_up_time, address: compressed_escrow_pda.compressed_account.address.unwrap(), - input_compressed_accounts: &[compressed_escrow_pda.compressed_account], + input_compressed_accounts: &[sdk_to_program_compressed_account( + compressed_escrow_pda.compressed_account, + )], }; create_withdrawal_instruction(create_withdrawal_ix_inputs.clone(), escrow_amount) } @@ -511,7 +531,7 @@ pub async fn perform_withdrawal( /// 2. Withdrawal token account exists /// 3. Compressed pda with update lock-up time exists #[allow(clippy::too_many_arguments)] -pub async fn assert_withdrawal( +pub async fn assert_withdrawal( rpc: &mut R, test_indexer: &mut TestIndexer, env: &EnvAccounts, diff --git a/forester-utils/src/address_merkle_tree_config.rs b/forester-utils/src/address_merkle_tree_config.rs index 53acd13094..ef23fde66c 100644 --- a/forester-utils/src/address_merkle_tree_config.rs +++ b/forester-utils/src/address_merkle_tree_config.rs @@ -4,16 +4,15 @@ use account_compression::{ }; use anchor_lang::Discriminator; use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; -use light_client::rpc::RpcConnection; +use light_client::{ + indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, + rpc::RpcConnection, +}; use light_hasher::{Discriminator as LightDiscriminator, Poseidon}; use num_traits::Zero; use solana_sdk::pubkey::Pubkey; -use crate::{ - get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree, - indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, - AccountZeroCopy, -}; +use crate::{get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree, AccountZeroCopy}; pub async fn get_address_bundle_config( rpc: &mut R, diff --git a/forester-utils/src/indexer/mod.rs b/forester-utils/src/indexer/mod.rs deleted file mode 100644 index 854633fbff..0000000000 --- a/forester-utils/src/indexer/mod.rs +++ /dev/null @@ -1,328 +0,0 @@ -use std::fmt::Debug; - -use account_compression::initialize_address_merkle_tree::{ - Error as AccountCompressionError, Pubkey, -}; -use async_trait::async_trait; -use light_client::rpc::RpcConnection; -use light_compressed_token::TokenData; -use light_hash_set::HashSetError; -use light_hasher::Poseidon; -use light_indexed_merkle_tree::{ - array::{IndexedArray, IndexedElement}, - reference::IndexedMerkleTree, -}; -use light_merkle_tree_reference::MerkleTree; -use light_system_program::{ - invoke::processor::CompressedProof, - sdk::{compressed_account::CompressedAccountWithMerkleContext, event::PublicTransactionEvent}, -}; -use num_bigint::BigUint; -use photon_api::apis::{default_api::GetCompressedAccountProofPostError, Error as PhotonApiError}; -use solana_sdk::signature::Keypair; -use thiserror::Error; - -#[derive(Debug, Clone)] -pub struct TokenDataWithContext { - pub token_data: TokenData, - pub compressed_account: CompressedAccountWithMerkleContext, -} - -#[derive(Debug, Default)] -pub struct BatchedTreeProofRpcResult { - pub proof: Option, - // If none -> proof by index, else included in zkp - pub root_indices: Vec>, - pub address_root_indices: Vec, -} - -#[derive(Debug, Default)] -pub struct ProofRpcResult { - pub proof: CompressedProof, - pub root_indices: Vec>, - pub address_root_indices: Vec, -} - -#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)] -pub struct StateMerkleTreeAccounts { - pub merkle_tree: Pubkey, - pub nullifier_queue: Pubkey, - pub cpi_context: Pubkey, -} - -#[derive(Debug, Clone, Copy)] -pub struct AddressMerkleTreeAccounts { - pub merkle_tree: Pubkey, - pub queue: Pubkey, -} - -#[derive(Debug, Clone)] -pub struct StateMerkleTreeBundle { - pub rollover_fee: i64, - pub merkle_tree: Box>, - pub accounts: StateMerkleTreeAccounts, - pub version: u64, - pub output_queue_elements: Vec<[u8; 32]>, - /// leaf index, leaf, tx hash - pub input_leaf_indices: Vec<(u32, [u8; 32], [u8; 32])>, -} - -#[derive(Debug, Clone)] -pub struct AddressMerkleTreeBundle { - pub rollover_fee: i64, - pub merkle_tree: Box>, - pub indexed_array: Box>, - pub accounts: AddressMerkleTreeAccounts, - pub queue_elements: Vec<[u8; 32]>, -} - -pub struct ProofOfLeaf { - pub leaf: [u8; 32], - pub proof: Vec<[u8; 32]>, -} - -#[async_trait] -pub trait Indexer: Sync + Send + Debug + 'static { - /// Returns queue elements from the queue with the given pubkey. For input - /// queues account compression program does not store queue elements in the - /// account data but only emits these in the public transaction event. The - /// indexer needs the queue elements to create batch update proofs. - async fn get_queue_elements( - &self, - pubkey: [u8; 32], - batch: u64, - start_offset: u64, - end_offset: u64, - ) -> Result, IndexerError>; - - fn get_proof_by_index(&mut self, _merkle_tree_pubkey: Pubkey, _index: u64) -> ProofOfLeaf { - unimplemented!("get_proof_by_index not implemented") - } - - fn get_proofs_by_indices( - &mut self, - _merkle_tree_pubkey: Pubkey, - _indices: &[u64], - ) -> Vec { - unimplemented!("get_proof_by_index not implemented") - } - - fn get_leaf_indices_tx_hashes( - &mut self, - _merkle_tree_pubkey: Pubkey, - _zkp_batch_size: usize, - ) -> Vec<(u32, [u8; 32], [u8; 32])> { - unimplemented!(); - } - - async fn get_subtrees( - &self, - merkle_tree_pubkey: [u8; 32], - ) -> Result, IndexerError>; - - async fn get_multiple_compressed_account_proofs( - &self, - hashes: Vec, - ) -> Result, IndexerError>; - - async fn get_rpc_compressed_accounts_by_owner( - &self, - owner: &Pubkey, - ) -> Result, IndexerError>; - - async fn get_multiple_new_address_proofs( - &self, - merkle_tree_pubkey: [u8; 32], - addresses: Vec<[u8; 32]>, - ) -> Result>, IndexerError>; - - async fn get_multiple_new_address_proofs_full( - &self, - merkle_tree_pubkey: [u8; 32], - addresses: Vec<[u8; 32]>, - ) -> Result>, IndexerError>; - - fn account_nullified(&mut self, _merkle_tree_pubkey: Pubkey, _account_hash: &str) {} - - fn address_tree_updated( - &mut self, - _merkle_tree_pubkey: Pubkey, - _context: &NewAddressProofWithContext<16>, - ) { - } - - fn get_state_merkle_tree_accounts(&self, _pubkeys: &[Pubkey]) -> Vec { - unimplemented!() - } - - fn add_event_and_compressed_accounts( - &mut self, - _slot: u64, - _event: &PublicTransactionEvent, - ) -> ( - Vec, - Vec, - ) { - unimplemented!() - } - - fn get_state_merkle_trees(&self) -> &Vec { - unimplemented!() - } - - fn get_state_merkle_trees_mut(&mut self) -> &mut Vec { - unimplemented!() - } - - fn get_address_merkle_trees(&self) -> &Vec { - unimplemented!() - } - - fn get_address_merkle_trees_mut(&mut self) -> &mut Vec { - unimplemented!() - } - - fn get_token_compressed_accounts(&self) -> &Vec { - unimplemented!() - } - - fn get_payer(&self) -> &Keypair { - unimplemented!() - } - - fn get_group_pda(&self) -> &Pubkey { - unimplemented!() - } - - async fn create_proof_for_compressed_accounts( - &mut self, - _compressed_accounts: Option>, - _state_merkle_tree_pubkeys: Option>, - _new_addresses: Option<&[[u8; 32]]>, - _address_merkle_tree_pubkeys: Option>, - _rpc: &mut R, - ) -> ProofRpcResult { - unimplemented!() - } - - async fn create_proof_for_compressed_accounts2( - &mut self, - _compressed_accounts: Option>, - _state_merkle_tree_pubkeys: Option>, - _new_addresses: Option<&[[u8; 32]]>, - _address_merkle_tree_pubkeys: Option>, - _rpc: &mut R, - ) -> BatchedTreeProofRpcResult { - unimplemented!() - } - - fn add_address_merkle_tree_accounts( - &mut self, - _merkle_tree_keypair: &Keypair, - _queue_keypair: &Keypair, - _owning_program_id: Option, - ) -> AddressMerkleTreeAccounts { - unimplemented!() - } - - fn get_compressed_accounts_by_owner( - &self, - _owner: &Pubkey, - ) -> Vec { - unimplemented!() - } - - fn get_compressed_token_accounts_by_owner(&self, _owner: &Pubkey) -> Vec { - unimplemented!() - } - - fn add_state_bundle(&mut self, _state_bundle: StateMerkleTreeBundle) { - unimplemented!() - } - - async fn update_test_indexer_after_append( - &mut self, - _rpc: &mut R, - _merkle_tree_pubkey: Pubkey, - _output_queue_pubkey: Pubkey, - _num_inserted_zkps: u64, - ) { - unimplemented!() - } - - async fn update_test_indexer_after_nullification( - &mut self, - _rpc: &mut R, - _merkle_tree_pubkey: Pubkey, - _batch_index: usize, - ) { - unimplemented!() - } - - async fn finalize_batched_address_tree_update( - &mut self, - _rpc: &mut R, - _merkle_tree_pubkey: Pubkey, - ) { - unimplemented!() - } -} - -#[derive(Debug, Clone)] -pub struct MerkleProof { - pub hash: String, - pub leaf_index: u64, - pub merkle_tree: String, - pub proof: Vec<[u8; 32]>, - pub root_seq: u64, -} - -// For consistency with the Photon API. -#[derive(Clone, Debug, PartialEq)] -pub struct NewAddressProofWithContext { - pub merkle_tree: [u8; 32], - pub root: [u8; 32], - pub root_seq: u64, - pub low_address_index: u64, - pub low_address_value: [u8; 32], - pub low_address_next_index: u64, - pub low_address_next_value: [u8; 32], - pub low_address_proof: [[u8; 32]; NET_HEIGHT], - pub new_low_element: Option>, - pub new_element: Option>, - pub new_element_next_value: Option, -} - -#[derive(Error, Debug)] -pub enum IndexerError { - #[error("RPC Error: {0}")] - RpcError(#[from] solana_client::client_error::ClientError), - #[error("failed to deserialize account data")] - DeserializeError(#[from] solana_sdk::program_error::ProgramError), - #[error("failed to copy merkle tree")] - CopyMerkleTreeError(#[from] std::io::Error), - #[error(transparent)] - AccountCompressionError(#[from] AccountCompressionError), - #[error(transparent)] - HashSetError(#[from] HashSetError), - #[error(transparent)] - PhotonApiError(PhotonApiErrorWrapper), - #[error("error: {0:?}")] - Custom(String), - #[error("unknown error")] - Unknown, -} - -#[derive(Error, Debug)] -pub enum PhotonApiErrorWrapper { - #[error(transparent)] - GetCompressedAccountProofPostError(#[from] PhotonApiError), -} - -impl From> for IndexerError { - fn from(err: PhotonApiError) -> Self { - IndexerError::PhotonApiError(PhotonApiErrorWrapper::GetCompressedAccountProofPostError( - err, - )) - } -} diff --git a/forester-utils/src/instructions.rs b/forester-utils/src/instructions.rs index c96d49f499..f303de3b05 100644 --- a/forester-utils/src/instructions.rs +++ b/forester-utils/src/instructions.rs @@ -6,7 +6,7 @@ use light_batched_merkle_tree::{ }, queue::BatchedQueueAccount, }; -use light_client::rpc::RpcConnection; +use light_client::{indexer::Indexer, rpc::RpcConnection}; use light_hasher::{Hasher, Poseidon}; use light_prover_client::{ batch_address_append::get_batch_address_append_circuit_inputs, @@ -27,8 +27,6 @@ use reqwest::Client; use solana_sdk::pubkey::Pubkey; use thiserror::Error; -use crate::indexer::Indexer; - #[derive(Error, Debug)] pub enum ForesterUtilsError { #[error("parse error: {0:?}")] @@ -41,11 +39,15 @@ pub enum ForesterUtilsError { IndexerError(String), } -pub async fn create_batch_update_address_tree_instruction_data>( +pub async fn create_batch_update_address_tree_instruction_data( rpc: &mut R, indexer: &mut I, merkle_tree_pubkey: Pubkey, -) -> Result<(InstructionDataBatchNullifyInputs, usize), ForesterUtilsError> { +) -> Result<(InstructionDataBatchNullifyInputs, usize), ForesterUtilsError> +where + R: RpcConnection, + I: Indexer, +{ let mut merkle_tree_account = rpc.get_account(merkle_tree_pubkey).await .map_err(|e| { error!( @@ -114,7 +116,7 @@ pub async fn create_batch_update_address_tree_instruction_data>( merkle_tree_pubkey, &leaf_indices_tx_hashes .iter() - .map(|(index, _, _)| *index as u64) + .map(|leaf_info| leaf_info.leaf_index as u64) .collect::>(), ); - for ((index, leaf, tx_hash), proof) in leaf_indices_tx_hashes.iter().zip(proofs.iter()) { - path_indices.push(*index); - leaves.push(*leaf); + for (leaf_info, proof) in leaf_indices_tx_hashes.iter().zip(proofs.iter()) { + path_indices.push(leaf_info.leaf_index); + leaves.push(leaf_info.leaf); old_leaves.push(proof.leaf); merkle_proofs.push(proof.proof.clone()); - tx_hashes.push(*tx_hash); - let index_bytes = index.to_be_bytes(); - let nullifier = Poseidon::hashv(&[leaf, &index_bytes, tx_hash]).unwrap(); + tx_hashes.push(leaf_info.tx_hash); + let index_bytes = leaf_info.leaf_index.to_be_bytes(); + let nullifier = + Poseidon::hashv(&[&leaf_info.leaf, &index_bytes, &leaf_info.tx_hash]).unwrap(); nullifiers.push(nullifier); } diff --git a/forester-utils/src/lib.rs b/forester-utils/src/lib.rs index d8beb61597..c83a36c21b 100644 --- a/forester-utils/src/lib.rs +++ b/forester-utils/src/lib.rs @@ -16,7 +16,6 @@ use solana_sdk::{ pub mod address_merkle_tree_config; pub mod forester_epoch; -pub mod indexer; pub mod instructions; pub mod registry; diff --git a/forester-utils/src/registry.rs b/forester-utils/src/registry.rs index c798a5ca20..38cd4c90e0 100644 --- a/forester-utils/src/registry.rs +++ b/forester-utils/src/registry.rs @@ -2,7 +2,10 @@ use account_compression::{ AddressMerkleTreeConfig, AddressQueueConfig, NullifierQueueConfig, QueueAccount, StateMerkleTreeConfig, }; -use light_client::rpc::{RpcConnection, RpcError}; +use light_client::{ + indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, + rpc::{RpcConnection, RpcError}, +}; use light_registry::{ account_compression_cpi::sdk::{ create_rollover_state_merkle_tree_instruction, CreateRolloverMerkleTreeInstructionInputs, @@ -21,7 +24,6 @@ use solana_sdk::{ use crate::{ address_merkle_tree_config::{get_address_bundle_config, get_state_bundle_config}, create_account_instruction, - indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, }; /// Creates and asserts forester account creation. diff --git a/forester/Cargo.toml b/forester/Cargo.toml index b807fd1764..5cf562daac 100644 --- a/forester/Cargo.toml +++ b/forester/Cargo.toml @@ -21,7 +21,8 @@ photon-api = { workspace = true } forester-utils = { workspace = true } light-client = { workspace = true } light-merkle-tree-metadata = { workspace = true } - +light-sdk = { workspace = true } +light-program-test = { workspace = true} serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1", features = ["full"] } diff --git a/forester/src/batch_processor/address.rs b/forester/src/batch_processor/address.rs index 2a02f1a0e8..339a6abe72 100644 --- a/forester/src/batch_processor/address.rs +++ b/forester/src/batch_processor/address.rs @@ -1,18 +1,19 @@ use borsh::BorshSerialize; -use forester_utils::{ - indexer::Indexer, instructions::create_batch_update_address_tree_instruction_data, -}; +use forester_utils::instructions::create_batch_update_address_tree_instruction_data; use light_batched_merkle_tree::event::BatchNullifyEvent; -use light_client::rpc::RpcConnection; +use light_client::{indexer::Indexer, rpc::RpcConnection}; use light_registry::account_compression_cpi::sdk::create_batch_update_address_tree_instruction; use solana_sdk::signer::Signer; use tracing::{info, instrument}; use super::common::BatchContext; -use crate::batch_processor::error::{BatchProcessError, Result}; +use crate::{ + batch_processor::error::{BatchProcessError, Result}, + indexer_type::{finalize_batch_address_tree_update, IndexerType}, +}; #[instrument(level = "debug", skip(context), fields(tree = %context.merkle_tree))] -pub(crate) async fn process_batch>( +pub(crate) async fn process_batch + IndexerType>( context: &BatchContext, ) -> Result { info!("Processing address batch operation"); @@ -49,10 +50,9 @@ pub(crate) async fn process_batch>( ) .await?; - let mut indexer = context.indexer.lock().await; - indexer - .finalize_batched_address_tree_update(&mut *rpc, context.merkle_tree) - .await; + finalize_batch_address_tree_update(&mut *rpc, context.indexer.clone(), context.merkle_tree) + .await + .expect("Failed to finalize batch address tree update"); info!( "Address batch processing completed successfully. Batch size: {}", diff --git a/forester/src/batch_processor/common.rs b/forester/src/batch_processor/common.rs index dda238d9b1..3fe4e4b814 100644 --- a/forester/src/batch_processor/common.rs +++ b/forester/src/batch_processor/common.rs @@ -1,18 +1,19 @@ use std::sync::Arc; -use forester_utils::{forester_epoch::TreeType, indexer::Indexer}; +use forester_utils::forester_epoch::TreeType; use light_batched_merkle_tree::{ batch::{Batch, BatchState}, merkle_tree::BatchedMerkleTreeAccount, queue::BatchedQueueAccount, }; -use light_client::{rpc::RpcConnection, rpc_pool::SolanaRpcPool}; +use light_client::{indexer::Indexer, rpc::RpcConnection, rpc_pool::SolanaRpcPool}; use solana_program::pubkey::Pubkey; use solana_sdk::signature::Keypair; use tokio::sync::Mutex; use tracing::info; use super::{address, error::Result, state, BatchProcessError}; +use crate::indexer_type::IndexerType; #[derive(Debug)] pub struct BatchContext> { @@ -33,12 +34,12 @@ pub enum BatchReadyState { } #[derive(Debug)] -pub struct BatchProcessor> { +pub struct BatchProcessor + IndexerType> { context: BatchContext, tree_type: TreeType, } -impl> BatchProcessor { +impl + IndexerType> BatchProcessor { pub fn new(context: BatchContext, tree_type: TreeType) -> Self { Self { context, tree_type } } diff --git a/forester/src/batch_processor/mod.rs b/forester/src/batch_processor/mod.rs index f4013bef7a..9e392aee33 100644 --- a/forester/src/batch_processor/mod.rs +++ b/forester/src/batch_processor/mod.rs @@ -5,7 +5,7 @@ mod state; use common::BatchProcessor; use error::Result; -use forester_utils::{forester_epoch::TreeType, indexer::Indexer}; +use forester_utils::forester_epoch::TreeType; use light_client::rpc::RpcConnection; use tracing::{info, instrument}; @@ -17,7 +17,7 @@ use tracing::{info, instrument}; tree_type = ?tree_type ) )] -pub async fn process_batched_operations>( +pub async fn process_batched_operations + IndexerType>( context: BatchContext, tree_type: TreeType, ) -> Result { @@ -28,3 +28,6 @@ pub async fn process_batched_operations>( pub use common::BatchContext; pub use error::BatchProcessError; +use light_client::indexer::Indexer; + +use crate::indexer_type::IndexerType; diff --git a/forester/src/batch_processor/state.rs b/forester/src/batch_processor/state.rs index a3c38d381e..85e1fb3b05 100644 --- a/forester/src/batch_processor/state.rs +++ b/forester/src/batch_processor/state.rs @@ -1,19 +1,21 @@ use borsh::BorshSerialize; -use forester_utils::{ - indexer::Indexer, - instructions::{create_append_batch_ix_data, create_nullify_batch_ix_data}, -}; +use forester_utils::instructions::{create_append_batch_ix_data, create_nullify_batch_ix_data}; use light_batched_merkle_tree::event::{BatchAppendEvent, BatchNullifyEvent}; -use light_client::rpc::RpcConnection; +use light_client::{indexer::Indexer, rpc::RpcConnection}; use light_registry::account_compression_cpi::sdk::{ create_batch_append_instruction, create_batch_nullify_instruction, }; use solana_sdk::signer::Signer; use super::common::BatchContext; -use crate::batch_processor::error::{BatchProcessError, Result}; +use crate::{ + batch_processor::error::{BatchProcessError, Result}, + indexer_type::{ + update_test_indexer_after_append, update_test_indexer_after_nullification, IndexerType, + }, +}; -pub(crate) async fn perform_append>( +pub(crate) async fn perform_append + IndexerType>( context: &BatchContext, rpc: &mut R, num_inserted_zkps: u64, @@ -46,20 +48,20 @@ pub(crate) async fn perform_append>( ) .await?; - let mut indexer = context.indexer.lock().await; - indexer - .update_test_indexer_after_append( - rpc, - context.merkle_tree, - context.output_queue, - num_inserted_zkps, - ) - .await; + update_test_indexer_after_append( + rpc, + context.indexer.clone(), + context.merkle_tree, + context.output_queue, + num_inserted_zkps, + ) + .await + .expect("Failed to update test indexer after append"); Ok(()) } -pub(crate) async fn perform_nullify>( +pub(crate) async fn perform_nullify + IndexerType>( context: &BatchContext, rpc: &mut R, ) -> Result<()> { @@ -88,12 +90,14 @@ pub(crate) async fn perform_nullify>( ) .await?; - context - .indexer - .lock() - .await - .update_test_indexer_after_nullification(rpc, context.merkle_tree, batch_index) - .await; + update_test_indexer_after_nullification( + rpc, + context.indexer.clone(), + context.merkle_tree, + batch_index, + ) + .await + .expect("Failed to update test indexer after nullification"); Ok(()) } diff --git a/forester/src/epoch_manager.rs b/forester/src/epoch_manager.rs index 566622c09b..067a01f041 100644 --- a/forester/src/epoch_manager.rs +++ b/forester/src/epoch_manager.rs @@ -9,12 +9,12 @@ use std::{ use anyhow::Context; use dashmap::DashMap; -use forester_utils::{ - forester_epoch::{get_epoch_phases, Epoch, TreeAccounts, TreeForesterSchedule, TreeType}, - indexer::{Indexer, MerkleProof, NewAddressProofWithContext}, +use forester_utils::forester_epoch::{ + get_epoch_phases, Epoch, TreeAccounts, TreeForesterSchedule, TreeType, }; use futures::future::join_all; use light_client::{ + indexer::{Indexer, MerkleProof, NewAddressProofWithContext}, rpc::{RetryConfig, RpcConnection, RpcError, SolanaRpcConnection}, rpc_pool::SolanaRpcPool, }; @@ -39,12 +39,11 @@ use crate::{ ChannelError, ConfigurationError, ForesterError, InitializationError, RegistrationError, WorkReportError, }, + indexer_type::{rollover_address_merkle_tree, rollover_state_merkle_tree, IndexerType}, metrics::{push_metrics, queue_metric_update, update_forester_sol_balance}, pagerduty::send_pagerduty_alert, queue_helpers::QueueItemData, - rollover::{ - is_tree_ready_for_rollover, rollover_address_merkle_tree, rollover_state_merkle_tree, - }, + rollover::is_tree_ready_for_rollover, send_transaction::{ send_batched_transactions, BuildTransactionBatchConfig, EpochManagerTransactions, SendBatchedTransactionsConfig, @@ -114,7 +113,7 @@ impl> Clone for EpochManager { } } -impl> EpochManager { +impl + IndexerType> EpochManager { #[allow(clippy::too_many_arguments)] pub async fn new( config: Arc, @@ -1152,7 +1151,7 @@ impl> EpochManager { skip(config, protocol_config, rpc_pool, indexer, shutdown, work_report_sender, slot_tracker), fields(forester = %config.payer_keypair.pubkey()) )] -pub async fn run_service>( +pub async fn run_service + IndexerType>( config: Arc, protocol_config: Arc, rpc_pool: Arc>, diff --git a/forester/src/indexer_type.rs b/forester/src/indexer_type.rs new file mode 100644 index 0000000000..68d891d76a --- /dev/null +++ b/forester/src/indexer_type.rs @@ -0,0 +1,338 @@ +use std::{any::Any, sync::Arc}; + +use async_trait::async_trait; +use forester_utils::forester_epoch::TreeAccounts; +use light_client::{ + indexer::{Indexer, StateMerkleTreeAccounts, StateMerkleTreeBundle}, + rpc::RpcConnection, +}; +use light_hasher::Poseidon; +use light_merkle_tree_reference::MerkleTree; +use light_program_test::indexer::{TestIndexer, TestIndexerExtensions}; +use light_sdk::{STATE_MERKLE_TREE_CANOPY_DEPTH, STATE_MERKLE_TREE_HEIGHT}; +use solana_program::pubkey::Pubkey; +use solana_sdk::{signature::Keypair, signer::Signer}; +use tokio::sync::Mutex; +use tracing::info; + +use crate::{ + errors::ForesterError, + photon_indexer::PhotonIndexer, + rollover::{perform_address_merkle_tree_rollover, perform_state_merkle_tree_rollover_forester}, + ForesterConfig, +}; + +mod sealed { + use light_client::rpc::merkle_tree::MerkleTreeExt; + + use super::*; + pub trait Sealed {} + impl Sealed for TestIndexer {} + impl Sealed for PhotonIndexer {} +} + +#[async_trait] +pub trait IndexerType: sealed::Sealed { + fn handle_state_bundle( + indexer: &mut impl Indexer, + new_merkle_tree: Pubkey, + new_queue: Pubkey, + new_cpi_context: Pubkey, + ) where + Self: Sized; + + fn handle_address_bundle( + indexer: &mut impl Indexer, + new_merkle_tree: &Keypair, + new_queue: &Keypair, + ) where + Self: Sized; + + async fn finalize_batch_address_tree_update( + rpc: &mut R, + indexer: &mut impl Indexer, + new_merkle_tree_pubkey: Pubkey, + ) where + Self: Sized; + + async fn update_test_indexer_after_nullification( + rpc: &mut R, + indexer: &mut impl Indexer, + merkle_tree_pubkey: Pubkey, + batch_index: usize, + ) where + Self: Sized; + + async fn update_test_indexer_after_append( + rpc: &mut R, + indexer: &mut impl Indexer, + merkle_tree_pubkey: Pubkey, + output_queue: Pubkey, + num_inserted_zkps: u64, + ) where + Self: Sized; +} + +#[async_trait] +impl IndexerType + for TestIndexer +{ + fn handle_state_bundle( + indexer: &mut impl Indexer, + new_merkle_tree: Pubkey, + new_queue: Pubkey, + new_cpi_context: Pubkey, + ) { + if let Some(test_indexer) = (indexer as &mut dyn Any).downcast_mut::>() { + let state_bundle = StateMerkleTreeBundle { + rollover_fee: 0, + accounts: StateMerkleTreeAccounts { + merkle_tree: new_merkle_tree, + nullifier_queue: new_queue, + cpi_context: new_cpi_context, + }, + version: 1, + output_queue_elements: vec![], + merkle_tree: Box::new(MerkleTree::::new( + STATE_MERKLE_TREE_HEIGHT, + STATE_MERKLE_TREE_CANOPY_DEPTH, + )), + input_leaf_indices: vec![], + }; + test_indexer.add_state_bundle(state_bundle); + } + } + + fn handle_address_bundle( + indexer: &mut impl Indexer, + new_merkle_tree: &Keypair, + new_queue: &Keypair, + ) { + if let Some(test_indexer) = (indexer as &mut dyn Any).downcast_mut::>() { + test_indexer.add_address_merkle_tree_accounts(new_merkle_tree, new_queue, None); + } + } + + async fn finalize_batch_address_tree_update( + rpc: &mut R, + indexer: &mut impl Indexer, + new_merkle_tree_pubkey: Pubkey, + ) { + if let Some(test_indexer) = (indexer as &mut dyn Any).downcast_mut::>() { + test_indexer + .finalize_batched_address_tree_update(rpc, new_merkle_tree_pubkey) + .await; + } + } + + async fn update_test_indexer_after_nullification( + rpc: &mut R, + indexer: &mut impl Indexer, + merkle_tree_pubkey: Pubkey, + batch_index: usize, + ) where + Self: Sized, + { + if let Some(test_indexer) = (indexer as &mut dyn Any).downcast_mut::>() { + test_indexer + .update_test_indexer_after_nullification(rpc, merkle_tree_pubkey, batch_index) + .await; + } + } + + async fn update_test_indexer_after_append( + rpc: &mut R, + indexer: &mut impl Indexer, + merkle_tree_pubkey: Pubkey, + output_queue: Pubkey, + num_inserted_zkps: u64, + ) where + Self: Sized, + { + if let Some(test_indexer) = (indexer as &mut dyn Any).downcast_mut::>() { + test_indexer + .update_test_indexer_after_append( + rpc, + merkle_tree_pubkey, + output_queue, + num_inserted_zkps, + ) + .await; + } + } +} + +// Implementation for PhotonIndexer - no-op +#[async_trait] +impl IndexerType for PhotonIndexer { + fn handle_state_bundle( + _indexer: &mut impl Indexer, + _new_merkle_tree: Pubkey, + _new_queue: Pubkey, + _new_cpi_context: Pubkey, + ) { + // No-op for production indexer + } + + fn handle_address_bundle( + _indexer: &mut impl Indexer, + _new_merkle_tree: &Keypair, + _new_queue: &Keypair, + ) { + // No-op for production indexer + } + + async fn finalize_batch_address_tree_update( + _rpc: &mut R, + _indexer: &mut impl Indexer, + _new_merkle_tree_pubkey: Pubkey, + ) { + // No-op for production indexer + } + + async fn update_test_indexer_after_nullification( + _rpc: &mut R, + _indexer: &mut impl Indexer, + _merkle_tree_pubkey: Pubkey, + _batch_index: usize, + ) { + // No-op for production indexer + } + + async fn update_test_indexer_after_append( + _rpc: &mut R, + _indexer: &mut impl Indexer, + _merkle_tree_pubkey: Pubkey, + _output_queue: Pubkey, + _num_inserted_zkps: u64, + ) { + // No-op for production indexer + } +} + +pub async fn rollover_state_merkle_tree + IndexerType>( + config: Arc, + rpc: &mut R, + indexer: Arc>, + tree_accounts: &TreeAccounts, + epoch: u64, +) -> Result<(), ForesterError> { + let new_nullifier_queue_keypair = Keypair::new(); + let new_merkle_tree_keypair = Keypair::new(); + let new_cpi_signature_keypair = Keypair::new(); + + let rollover_signature = perform_state_merkle_tree_rollover_forester( + &config.payer_keypair, + &config.derivation_pubkey, + rpc, + &new_nullifier_queue_keypair, + &new_merkle_tree_keypair, + &new_cpi_signature_keypair, + &tree_accounts.merkle_tree, + &tree_accounts.queue, + &Pubkey::default(), + epoch, + ) + .await?; + + info!("State rollover signature: {:?}", rollover_signature); + + I::handle_state_bundle( + &mut *indexer.lock().await, + new_merkle_tree_keypair.pubkey(), + new_nullifier_queue_keypair.pubkey(), + new_cpi_signature_keypair.pubkey(), + ); + + Ok(()) +} + +pub async fn rollover_address_merkle_tree + IndexerType>( + config: Arc, + rpc: &mut R, + indexer: Arc>, + tree_accounts: &TreeAccounts, + epoch: u64, +) -> Result<(), ForesterError> { + let new_nullifier_queue_keypair = Keypair::new(); + let new_merkle_tree_keypair = Keypair::new(); + + let rollover_signature = perform_address_merkle_tree_rollover( + &config.payer_keypair, + &config.derivation_pubkey, + rpc, + &new_nullifier_queue_keypair, + &new_merkle_tree_keypair, + &tree_accounts.merkle_tree, + &tree_accounts.queue, + epoch, + ) + .await?; + + info!("Address rollover signature: {:?}", rollover_signature); + + I::handle_address_bundle( + &mut *indexer.lock().await, + &new_merkle_tree_keypair, + &new_nullifier_queue_keypair, + ); + + Ok(()) +} + +pub async fn finalize_batch_address_tree_update< + R: RpcConnection, + I: Indexer + IndexerType, +>( + rpc: &mut R, + indexer: Arc>, + new_merkle_tree_pubkey: Pubkey, +) -> Result<(), ForesterError> { + I::finalize_batch_address_tree_update( + &mut *rpc, + &mut *indexer.lock().await, + new_merkle_tree_pubkey, + ) + .await; + + Ok(()) +} + +pub async fn update_test_indexer_after_nullification< + R: RpcConnection, + I: Indexer + IndexerType, +>( + rpc: &mut R, + indexer: Arc>, + merkle_tree_pubkey: Pubkey, + batch_index: usize, +) -> Result<(), ForesterError> { + I::update_test_indexer_after_nullification( + &mut *rpc, + &mut *indexer.lock().await, + merkle_tree_pubkey, + batch_index, + ) + .await; + + Ok(()) +} + +pub async fn update_test_indexer_after_append + IndexerType>( + rpc: &mut R, + indexer: Arc>, + merkle_tree_pubkey: Pubkey, + output_queue: Pubkey, + num_inserted_zkps: u64, +) -> Result<(), ForesterError> { + I::update_test_indexer_after_append( + &mut *rpc, + &mut *indexer.lock().await, + merkle_tree_pubkey, + output_queue, + num_inserted_zkps, + ) + .await; + + Ok(()) +} diff --git a/forester/src/lib.rs b/forester/src/lib.rs index d99a5790fa..64e9f5fdfe 100644 --- a/forester/src/lib.rs +++ b/forester/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub mod epoch_manager; pub mod errors; pub mod forester_status; +mod indexer_type; pub mod metrics; pub mod pagerduty; pub mod photon_indexer; @@ -23,11 +24,9 @@ use std::{sync::Arc, time::Duration}; use account_compression::utils::constants::{ADDRESS_QUEUE_VALUES, STATE_NULLIFIER_QUEUE_VALUES}; pub use config::{ForesterConfig, ForesterEpochInfo}; -use forester_utils::{ - forester_epoch::{TreeAccounts, TreeType}, - indexer::Indexer, -}; +use forester_utils::forester_epoch::{TreeAccounts, TreeType}; use light_client::{ + indexer::Indexer, rpc::{RpcConnection, SolanaRpcConnection}, rpc_pool::SolanaRpcPool, }; @@ -37,6 +36,7 @@ use tracing::debug; use crate::{ epoch_manager::{run_service, WorkReport}, + indexer_type::IndexerType, metrics::QUEUE_LENGTH, queue_helpers::fetch_queue_item_data, slot_tracker::SlotTracker, @@ -80,7 +80,7 @@ pub async fn run_queue_info( } } -pub async fn run_pipeline>( +pub async fn run_pipeline + IndexerType>( config: Arc, indexer: Arc>, shutdown: oneshot::Receiver<()>, diff --git a/forester/src/photon_indexer.rs b/forester/src/photon_indexer.rs index cff0687969..abbbf61b80 100644 --- a/forester/src/photon_indexer.rs +++ b/forester/src/photon_indexer.rs @@ -2,8 +2,14 @@ use std::fmt::Debug; use account_compression::initialize_address_merkle_tree::Pubkey; use async_trait::async_trait; -use forester_utils::indexer::{Indexer, IndexerError, MerkleProof, NewAddressProofWithContext}; -use light_client::rpc::RpcConnection; +use light_client::{ + indexer::{ + AddressMerkleTreeBundle, Indexer, IndexerError, LeafIndexInfo, MerkleProof, + NewAddressProofWithContext, ProofOfLeaf, + }, + rpc::RpcConnection, +}; +use light_sdk::proof::ProofRpcResult; use photon_api::{ apis::configuration::{ApiKey, Configuration}, models::{AddressWithTree, GetCompressedAccountsByOwnerPostRequestParams}, @@ -53,19 +59,20 @@ impl Indexer for PhotonIndexer { ) -> Result, IndexerError> { unimplemented!() } - async fn get_subtrees( - &self, - _merkle_tree_pubkey: [u8; 32], - ) -> Result, IndexerError> { + + fn get_subtrees(&self, _merkle_tree_pubkey: [u8; 32]) -> Result, IndexerError> { unimplemented!() } - async fn get_multiple_new_address_proofs_full( - &self, - _merkle_tree_pubkey: [u8; 32], - _addresses: Vec<[u8; 32]>, - ) -> Result>, IndexerError> { - unimplemented!() + async fn create_proof_for_compressed_accounts( + &mut self, + _compressed_accounts: Option>, + _state_merkle_tree_pubkeys: Option>, + _new_addresses: Option<&[[u8; 32]]>, + _address_merkle_tree_pubkeys: Option>, + _rpc: &mut R, + ) -> ProofRpcResult { + todo!() } async fn get_multiple_compressed_account_proofs( &self, @@ -118,7 +125,7 @@ impl Indexer for PhotonIndexer { } } - async fn get_rpc_compressed_accounts_by_owner( + async fn get_compressed_accounts_by_owner( &self, owner: &Pubkey, ) -> Result, IndexerError> { @@ -216,4 +223,32 @@ impl Indexer for PhotonIndexer { Ok(proofs) } + + async fn get_multiple_new_address_proofs_h40( + &self, + _merkle_tree_pubkey: [u8; 32], + _addresses: Vec<[u8; 32]>, + ) -> Result>, IndexerError> { + unimplemented!() + } + + fn get_proofs_by_indices( + &mut self, + _merkle_tree_pubkey: Pubkey, + _indices: &[u64], + ) -> Vec { + todo!() + } + + fn get_leaf_indices_tx_hashes( + &mut self, + _merkle_tree_pubkey: Pubkey, + _zkp_batch_size: usize, + ) -> Vec { + todo!() + } + + fn get_address_merkle_trees(&self) -> &Vec { + todo!() + } } diff --git a/forester/src/rollover/mod.rs b/forester/src/rollover/mod.rs index f963a4df8b..0c4985de26 100644 --- a/forester/src/rollover/mod.rs +++ b/forester/src/rollover/mod.rs @@ -2,7 +2,7 @@ mod operations; mod state; pub use operations::{ - get_tree_fullness, is_tree_ready_for_rollover, rollover_address_merkle_tree, - rollover_state_merkle_tree, + get_tree_fullness, is_tree_ready_for_rollover, perform_address_merkle_tree_rollover, + perform_state_merkle_tree_rollover_forester, }; pub use state::RolloverState; diff --git a/forester/src/rollover/operations.rs b/forester/src/rollover/operations.rs index cc0c2a5e0c..1bfaa608c0 100644 --- a/forester/src/rollover/operations.rs +++ b/forester/src/rollover/operations.rs @@ -1,22 +1,20 @@ -use std::sync::Arc; - use account_compression::{ - utils::constants::{STATE_MERKLE_TREE_CANOPY_DEPTH, STATE_MERKLE_TREE_HEIGHT}, AddressMerkleTreeAccount, AddressMerkleTreeConfig, AddressQueueConfig, NullifierQueueConfig, QueueAccount, StateMerkleTreeAccount, StateMerkleTreeConfig, }; use forester_utils::{ address_merkle_tree_config::{get_address_bundle_config, get_state_bundle_config}, create_account_instruction, - forester_epoch::{TreeAccounts, TreeType}, + forester_epoch::TreeType, get_concurrent_merkle_tree, get_indexed_merkle_tree, - indexer::{AddressMerkleTreeAccounts, Indexer, StateMerkleTreeAccounts, StateMerkleTreeBundle}, registry::RentExemption, }; use light_batched_merkle_tree::merkle_tree::BatchedMerkleTreeAccount; -use light_client::rpc::{RpcConnection, RpcError}; +use light_client::{ + indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, + rpc::{RpcConnection, RpcError}, +}; use light_hasher::Poseidon; -use light_merkle_tree_reference::MerkleTree; use light_registry::{ account_compression_cpi::sdk::{ create_rollover_address_merkle_tree_instruction, @@ -28,10 +26,9 @@ use solana_sdk::{ instruction::Instruction, pubkey::Pubkey, signature::Keypair, signer::Signer, transaction::Transaction, }; -use tokio::sync::Mutex; use tracing::info; -use crate::{errors::ForesterError, ForesterConfig}; +use crate::errors::ForesterError; enum TreeAccount { State(StateMerkleTreeAccount), @@ -285,56 +282,6 @@ pub async fn is_tree_ready_for_rollover( } } -pub async fn rollover_state_merkle_tree>( - config: Arc, - rpc: &mut R, - indexer: Arc>, - tree_accounts: &TreeAccounts, - epoch: u64, -) -> Result<(), ForesterError> { - info!( - "Rolling over state merkle tree {:?}", - tree_accounts.merkle_tree - ); - let new_nullifier_queue_keypair = Keypair::new(); - let new_merkle_tree_keypair = Keypair::new(); - let new_cpi_signature_keypair = Keypair::new(); - - let rollover_signature = perform_state_merkle_tree_rollover_forester( - &config.payer_keypair, - &config.derivation_pubkey, - rpc, - &new_nullifier_queue_keypair, - &new_merkle_tree_keypair, - &new_cpi_signature_keypair, - &tree_accounts.merkle_tree, - &tree_accounts.queue, - &Pubkey::default(), - epoch, - ) - .await?; - info!("State rollover signature: {:?}", rollover_signature); - - let state_bundle = StateMerkleTreeBundle { - // TODO: fetch correct fee when this property is used - rollover_fee: 0, - accounts: StateMerkleTreeAccounts { - merkle_tree: new_merkle_tree_keypair.pubkey(), - nullifier_queue: new_nullifier_queue_keypair.pubkey(), - cpi_context: new_cpi_signature_keypair.pubkey(), - }, - merkle_tree: Box::new(MerkleTree::::new( - STATE_MERKLE_TREE_HEIGHT as usize, - STATE_MERKLE_TREE_CANOPY_DEPTH as usize, - )), - version: 1, - input_leaf_indices: vec![], - output_queue_elements: vec![], - }; - indexer.lock().await.add_state_bundle(state_bundle); - Ok(()) -} - #[allow(clippy::too_many_arguments)] pub async fn perform_state_merkle_tree_rollover_forester( payer: &Keypair, @@ -376,36 +323,6 @@ pub async fn perform_state_merkle_tree_rollover_forester( context.process_transaction(transaction).await } -pub async fn rollover_address_merkle_tree>( - config: Arc, - rpc: &mut R, - indexer: Arc>, - tree_data: &TreeAccounts, - epoch: u64, -) -> Result<(), ForesterError> { - let new_nullifier_queue_keypair = Keypair::new(); - let new_merkle_tree_keypair = Keypair::new(); - let rollover_signature = perform_address_merkle_tree_rollover( - &config.payer_keypair, - &config.derivation_pubkey, - rpc, - &new_nullifier_queue_keypair, - &new_merkle_tree_keypair, - &tree_data.merkle_tree, - &tree_data.queue, - epoch, - ) - .await?; - info!("Address rollover signature: {:?}", rollover_signature); - - indexer.lock().await.add_address_merkle_tree_accounts( - &new_merkle_tree_keypair, - &new_nullifier_queue_keypair, - None, - ); - Ok(()) -} - #[allow(clippy::too_many_arguments)] pub async fn perform_address_merkle_tree_rollover( payer: &Keypair, diff --git a/forester/src/send_transaction.rs b/forester/src/send_transaction.rs index 054ea2d55a..63ac5e4370 100644 --- a/forester/src/send_transaction.rs +++ b/forester/src/send_transaction.rs @@ -5,12 +5,10 @@ use account_compression::utils::constants::{ STATE_MERKLE_TREE_CHANGELOG, STATE_NULLIFIER_QUEUE_VALUES, }; use async_trait::async_trait; -use forester_utils::{ - forester_epoch::{TreeAccounts, TreeType}, - indexer::Indexer, -}; +use forester_utils::forester_epoch::{TreeAccounts, TreeType}; use futures::future::join_all; use light_client::{ + indexer::Indexer, rpc::{RetryConfig, RpcConnection}, rpc_pool::SolanaRpcPool, }; diff --git a/forester/tests/batched_address_test.rs b/forester/tests/batched_address_test.rs index ba4ff36fec..279db9c645 100644 --- a/forester/tests/batched_address_test.rs +++ b/forester/tests/batched_address_test.rs @@ -1,23 +1,20 @@ use std::{sync::Arc, time::Duration}; use forester::run_pipeline; -use forester_utils::{ - indexer::AddressMerkleTreeAccounts, - registry::{register_test_forester, update_test_forester}, -}; +use forester_utils::registry::{register_test_forester, update_test_forester}; use light_batched_merkle_tree::{ batch::BatchState, initialize_address_tree::InitAddressTreeAccountsInstructionData, merkle_tree::BatchedMerkleTreeAccount, }; use light_client::{ + indexer::AddressMerkleTreeAccounts, rpc::{solana_rpc::SolanaRpcUrl, RpcConnection, SolanaRpcConnection}, rpc_pool::SolanaRpcPool, }; -use light_program_test::test_env::EnvAccounts; +use light_program_test::{indexer::TestIndexer, test_env::EnvAccounts}; use light_prover_client::gnark::helpers::{LightValidatorConfig, ProverConfig, ProverMode}; use light_test_utils::{ create_address_test_program_sdk::perform_create_pda_with_event_rnd, e2e_test_env::E2ETestEnv, - indexer::TestIndexer, }; use serial_test::serial; use solana_program::native_token::LAMPORTS_PER_SOL; diff --git a/forester/tests/batched_state_test.rs b/forester/tests/batched_state_test.rs index d19aa0898f..f0da3c225e 100644 --- a/forester/tests/batched_state_test.rs +++ b/forester/tests/batched_state_test.rs @@ -10,12 +10,9 @@ use light_client::{ rpc::{solana_rpc::SolanaRpcUrl, RpcConnection, SolanaRpcConnection}, rpc_pool::SolanaRpcPool, }; -use light_program_test::test_env::EnvAccounts; +use light_program_test::{indexer::TestIndexer, test_env::EnvAccounts}; use light_prover_client::gnark::helpers::LightValidatorConfig; -use light_test_utils::{ - e2e_test_env::{init_program_test_env, E2ETestEnv}, - indexer::TestIndexer, -}; +use light_test_utils::e2e_test_env::{init_program_test_env, E2ETestEnv}; use serial_test::serial; use solana_program::native_token::LAMPORTS_PER_SOL; use solana_sdk::{ diff --git a/forester/tests/e2e_test.rs b/forester/tests/e2e_test.rs index f30045dd13..3c7e81c41f 100644 --- a/forester/tests/e2e_test.rs +++ b/forester/tests/e2e_test.rs @@ -5,15 +5,13 @@ use account_compression::{ AddressMerkleTreeAccount, }; use forester::{queue_helpers::fetch_queue_item_data, run_pipeline, utils::get_protocol_config}; -use forester_utils::{ - indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, - registry::register_test_forester, -}; +use forester_utils::registry::register_test_forester; use light_client::{ + indexer::{AddressMerkleTreeAccounts, StateMerkleTreeAccounts}, rpc::{solana_rpc::SolanaRpcUrl, RpcConnection, RpcError, SolanaRpcConnection}, rpc_pool::SolanaRpcPool, }; -use light_program_test::test_env::EnvAccounts; +use light_program_test::{indexer::TestIndexer, test_env::EnvAccounts}; use light_prover_client::gnark::helpers::{ spawn_prover, LightValidatorConfig, ProverConfig, ProverMode, }; @@ -21,7 +19,7 @@ use light_registry::{ utils::{get_epoch_pda_address, get_forester_epoch_pda_from_authority}, EpochPda, ForesterEpochPda, }; -use light_test_utils::{e2e_test_env::E2ETestEnv, indexer::TestIndexer, update_test_forester}; +use light_test_utils::{e2e_test_env::E2ETestEnv, update_test_forester}; use serial_test::serial; use solana_sdk::{ commitment_config::CommitmentConfig, native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, diff --git a/forester/tests/test_utils.rs b/forester/tests/test_utils.rs index f1a96cca07..4cfbbc6ca5 100644 --- a/forester/tests/test_utils.rs +++ b/forester/tests/test_utils.rs @@ -6,14 +6,13 @@ use forester::{ telemetry::setup_telemetry, ForesterConfig, }; -use forester_utils::indexer::{Indexer, IndexerError, NewAddressProofWithContext}; -use light_client::rpc::{RpcConnection, SolanaRpcConnection}; -use light_program_test::test_env::get_test_env_accounts; -use light_prover_client::gnark::helpers::{spawn_validator, LightValidatorConfig}; -use light_test_utils::{ - e2e_test_env::{GeneralActionConfig, KeypairActionConfig, User}, - indexer::TestIndexer, +use light_client::{ + indexer::{Indexer, IndexerError, NewAddressProofWithContext}, + rpc::RpcConnection, }; +use light_program_test::{indexer::TestIndexerExtensions, test_env::get_test_env_accounts}; +use light_prover_client::gnark::helpers::{spawn_validator, LightValidatorConfig}; +use light_test_utils::e2e_test_env::{GeneralActionConfig, KeypairActionConfig, User}; use solana_sdk::signature::{Keypair, Signer}; use tracing::debug; @@ -108,8 +107,11 @@ pub fn generate_pubkey_254() -> Pubkey { } #[allow(dead_code)] -pub async fn assert_new_address_proofs_for_photon_and_test_indexer( - indexer: &mut TestIndexer, +pub async fn assert_new_address_proofs_for_photon_and_test_indexer< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + indexer: &mut I, trees: &[Pubkey], addresses: &[Pubkey], photon_indexer: &PhotonIndexer, @@ -176,19 +178,22 @@ pub async fn assert_new_address_proofs_for_photon_and_test_indexer( - indexer: &mut TestIndexer, +pub async fn assert_accounts_by_owner< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + indexer: &mut I, user: &User, photon_indexer: &PhotonIndexer, ) { let mut photon_accs = photon_indexer - .get_rpc_compressed_accounts_by_owner(&user.keypair.pubkey()) + .get_compressed_accounts_by_owner(&user.keypair.pubkey()) .await .unwrap(); photon_accs.sort(); let mut test_accs = indexer - .get_rpc_compressed_accounts_by_owner(&user.keypair.pubkey()) + .get_compressed_accounts_by_owner(&user.keypair.pubkey()) .await .unwrap(); test_accs.sort(); @@ -210,14 +215,16 @@ pub async fn assert_accounts_by_owner( } #[allow(dead_code)] -pub async fn assert_account_proofs_for_photon_and_test_indexer( - indexer: &mut TestIndexer, +pub async fn assert_account_proofs_for_photon_and_test_indexer< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + indexer: &mut I, user_pubkey: &Pubkey, photon_indexer: &PhotonIndexer, ) { - let accs: Result, IndexerError> = indexer - .get_rpc_compressed_accounts_by_owner(user_pubkey) - .await; + let accs: Result, IndexerError> = + indexer.get_compressed_accounts_by_owner(user_pubkey).await; for account_hash in accs.unwrap() { let photon_result = photon_indexer .get_multiple_compressed_account_proofs(vec![account_hash.clone()]) diff --git a/js/compressed-token/src/idl.ts b/js/compressed-token/src/idl.ts index 909301341f..5ce45b666e 100644 --- a/js/compressed-token/src/idl.ts +++ b/js/compressed-token/src/idl.ts @@ -1581,23 +1581,158 @@ export type LightCompressedToken = { errors: [ { code: 6000; - name: 'SignerCheckFailed'; - msg: 'Signer check failed'; + name: 'PublicKeyAmountMissmatch'; + msg: 'public keys and amounts must be of same length'; }, { code: 6001; - name: 'CreateTransferInstructionFailed'; - msg: 'Create transfer instruction failed'; + name: 'ComputeInputSumFailed'; + msg: 'ComputeInputSumFailed'; }, { code: 6002; - name: 'AccountNotFound'; - msg: 'Account not found'; + name: 'ComputeOutputSumFailed'; + msg: 'ComputeOutputSumFailed'; }, { code: 6003; - name: 'SerializationError'; - msg: 'Serialization error'; + name: 'ComputeCompressSumFailed'; + msg: 'ComputeCompressSumFailed'; + }, + { + code: 6004; + name: 'ComputeDecompressSumFailed'; + msg: 'ComputeDecompressSumFailed'; + }, + { + code: 6005; + name: 'SumCheckFailed'; + msg: 'SumCheckFailed'; + }, + { + code: 6006; + name: 'DecompressRecipientUndefinedForDecompress'; + msg: 'DecompressRecipientUndefinedForDecompress'; + }, + { + code: 6007; + name: 'CompressedPdaUndefinedForDecompress'; + msg: 'CompressedPdaUndefinedForDecompress'; + }, + { + code: 6008; + name: 'DeCompressAmountUndefinedForDecompress'; + msg: 'DeCompressAmountUndefinedForDecompress'; + }, + { + code: 6009; + name: 'CompressedPdaUndefinedForCompress'; + msg: 'CompressedPdaUndefinedForCompress'; + }, + { + code: 6010; + name: 'DeCompressAmountUndefinedForCompress'; + msg: 'DeCompressAmountUndefinedForCompress'; + }, + { + code: 6011; + name: 'DelegateSignerCheckFailed'; + msg: 'DelegateSignerCheckFailed'; + }, + { + code: 6012; + name: 'MintTooLarge'; + msg: 'Minted amount greater than u64::MAX'; + }, + { + code: 6013; + name: 'SplTokenSupplyMismatch'; + msg: 'SplTokenSupplyMismatch'; + }, + { + code: 6014; + name: 'HeapMemoryCheckFailed'; + msg: 'HeapMemoryCheckFailed'; + }, + { + code: 6015; + name: 'InstructionNotCallable'; + msg: 'The instruction is not callable'; + }, + { + code: 6016; + name: 'ArithmeticUnderflow'; + msg: 'ArithmeticUnderflow'; + }, + { + code: 6017; + name: 'HashToFieldError'; + msg: 'HashToFieldError'; + }, + { + code: 6018; + name: 'InvalidAuthorityMint'; + msg: 'Expected the authority to be also a mint authority'; + }, + { + code: 6019; + name: 'InvalidFreezeAuthority'; + msg: 'Provided authority is not the freeze authority'; + }, + { + code: 6020; + name: 'InvalidDelegateIndex'; + }, + { + code: 6021; + name: 'TokenPoolPdaUndefined'; + }, + { + code: 6022; + name: 'IsTokenPoolPda'; + msg: 'Compress or decompress recipient is the same account as the token pool pda.'; + }, + { + code: 6023; + name: 'InvalidTokenPoolPda'; + }, + { + code: 6024; + name: 'NoInputTokenAccountsProvided'; + }, + { + code: 6025; + name: 'NoInputsProvided'; + }, + { + code: 6026; + name: 'MintHasNoFreezeAuthority'; + }, + { + code: 6027; + name: 'MintWithInvalidExtension'; + }, + { + code: 6028; + name: 'InsufficientTokenAccountBalance'; + msg: 'The token account balance is less than the remaining amount.'; + }, + { + code: 6029; + name: 'InvalidTokenPoolBump'; + msg: 'Max number of token pools reached.'; + }, + { + code: 6030; + name: 'FailedToDecompress'; + }, + { + code: 6031; + name: 'FailedToBurnSplTokensFromTokenPool'; + }, + { + code: 6032; + name: 'NoMatchingBumpFound'; }, ]; }; @@ -3189,23 +3324,158 @@ export const IDL: LightCompressedToken = { errors: [ { code: 6000, - name: 'SignerCheckFailed', - msg: 'Signer check failed', + name: 'PublicKeyAmountMissmatch', + msg: 'public keys and amounts must be of same length', }, { code: 6001, - name: 'CreateTransferInstructionFailed', - msg: 'Create transfer instruction failed', + name: 'ComputeInputSumFailed', + msg: 'ComputeInputSumFailed', }, { code: 6002, - name: 'AccountNotFound', - msg: 'Account not found', + name: 'ComputeOutputSumFailed', + msg: 'ComputeOutputSumFailed', }, { code: 6003, - name: 'SerializationError', - msg: 'Serialization error', + name: 'ComputeCompressSumFailed', + msg: 'ComputeCompressSumFailed', + }, + { + code: 6004, + name: 'ComputeDecompressSumFailed', + msg: 'ComputeDecompressSumFailed', + }, + { + code: 6005, + name: 'SumCheckFailed', + msg: 'SumCheckFailed', + }, + { + code: 6006, + name: 'DecompressRecipientUndefinedForDecompress', + msg: 'DecompressRecipientUndefinedForDecompress', + }, + { + code: 6007, + name: 'CompressedPdaUndefinedForDecompress', + msg: 'CompressedPdaUndefinedForDecompress', + }, + { + code: 6008, + name: 'DeCompressAmountUndefinedForDecompress', + msg: 'DeCompressAmountUndefinedForDecompress', + }, + { + code: 6009, + name: 'CompressedPdaUndefinedForCompress', + msg: 'CompressedPdaUndefinedForCompress', + }, + { + code: 6010, + name: 'DeCompressAmountUndefinedForCompress', + msg: 'DeCompressAmountUndefinedForCompress', + }, + { + code: 6011, + name: 'DelegateSignerCheckFailed', + msg: 'DelegateSignerCheckFailed', + }, + { + code: 6012, + name: 'MintTooLarge', + msg: 'Minted amount greater than u64::MAX', + }, + { + code: 6013, + name: 'SplTokenSupplyMismatch', + msg: 'SplTokenSupplyMismatch', + }, + { + code: 6014, + name: 'HeapMemoryCheckFailed', + msg: 'HeapMemoryCheckFailed', + }, + { + code: 6015, + name: 'InstructionNotCallable', + msg: 'The instruction is not callable', + }, + { + code: 6016, + name: 'ArithmeticUnderflow', + msg: 'ArithmeticUnderflow', + }, + { + code: 6017, + name: 'HashToFieldError', + msg: 'HashToFieldError', + }, + { + code: 6018, + name: 'InvalidAuthorityMint', + msg: 'Expected the authority to be also a mint authority', + }, + { + code: 6019, + name: 'InvalidFreezeAuthority', + msg: 'Provided authority is not the freeze authority', + }, + { + code: 6020, + name: 'InvalidDelegateIndex', + }, + { + code: 6021, + name: 'TokenPoolPdaUndefined', + }, + { + code: 6022, + name: 'IsTokenPoolPda', + msg: 'Compress or decompress recipient is the same account as the token pool pda.', + }, + { + code: 6023, + name: 'InvalidTokenPoolPda', + }, + { + code: 6024, + name: 'NoInputTokenAccountsProvided', + }, + { + code: 6025, + name: 'NoInputsProvided', + }, + { + code: 6026, + name: 'MintHasNoFreezeAuthority', + }, + { + code: 6027, + name: 'MintWithInvalidExtension', + }, + { + code: 6028, + name: 'InsufficientTokenAccountBalance', + msg: 'The token account balance is less than the remaining amount.', + }, + { + code: 6029, + name: 'InvalidTokenPoolBump', + msg: 'Max number of token pools reached.', + }, + { + code: 6030, + name: 'FailedToDecompress', + }, + { + code: 6031, + name: 'FailedToBurnSplTokensFromTokenPool', + }, + { + code: 6032, + name: 'NoMatchingBumpFound', }, ], }; diff --git a/program-libs/concurrent-merkle-tree/Cargo.toml b/program-libs/concurrent-merkle-tree/Cargo.toml index e88ebfe1b8..6e0e756cf9 100644 --- a/program-libs/concurrent-merkle-tree/Cargo.toml +++ b/program-libs/concurrent-merkle-tree/Cargo.toml @@ -16,7 +16,6 @@ solana = [ [dependencies] borsh = "0.10" -bytemuck = { version = "1.17", features = ["derive"] } light-bounded-vec = { workspace = true } light-hasher = { workspace = true } light-utils = { version = "1.1.0"} @@ -31,7 +30,6 @@ light-merkle-tree-reference = { workspace = true} light-hash-set = { workspace = true, features = ["solana"] } rand = "0.8" solana-program = { workspace = true } -spl-account-compression = { version = "0.3.0", default-features = false} spl-concurrent-merkle-tree = { version = "0.2.0", default-features = false} tokio = { workspace = true } num-bigint = "0.4" diff --git a/program-libs/utils/Cargo.toml b/program-libs/utils/Cargo.toml index 2bdf1c0610..3caf66def5 100644 --- a/program-libs/utils/Cargo.toml +++ b/program-libs/utils/Cargo.toml @@ -13,19 +13,17 @@ anchor = ["anchor-lang"] [dependencies] anyhow = "1.0" ark-ff = "0.4" -light-bounded-vec = { workspace = true } -light-hasher = {workspace = true} +light-hasher = { workspace = true } num-bigint = { version = "0.4", features = ["rand"] } thiserror = "1.0" solana-program = { workspace = true } ark-bn254 = "0.4.0" rand = "0.8" -zerocopy = {workspace = true, features = ["derive"]} -borsh = {workspace = true} -bytemuck = {workspace = true} -anchor-lang = {workspace = true , optional = true} +zerocopy = { workspace = true, features = ["derive"] } +borsh = { workspace = true } +bytemuck = { workspace = true } +anchor-lang = { workspace = true, optional = true } [dev-dependencies] -memoffset = "0.9" -light-poseidon = {workspace = true} -borsh = {workspace = true} \ No newline at end of file +light-poseidon = { workspace = true } +borsh = { workspace = true } diff --git a/program-tests/account-compression-test/Cargo.toml b/program-tests/account-compression-test/Cargo.toml index f902ce92b8..7092284608 100644 --- a/program-tests/account-compression-test/Cargo.toml +++ b/program-tests/account-compression-test/Cargo.toml @@ -25,16 +25,12 @@ ark-ff = "0.4.0" solana-program-test = { workspace = true} light-test-utils = { workspace = true, features=["devenv"] } light-program-test = { workspace = true, features = ["devenv"] } -reqwest = "0.11.26" +light-client = { workspace = true } tokio = { workspace = true } light-prover-client = {workspace = true } num-bigint = "0.4.6" -num-traits = "0.2.19" -spl-token = { workspace = true } anchor-spl = { workspace = true } anchor-lang = { workspace = true } -light-compressed-token = { workspace = true } -light-system-program = { workspace = true } account-compression = { workspace = true } light-hasher = {workspace = true} light-hash-set = { workspace = true} @@ -45,11 +41,7 @@ light-bounded-vec = {workspace = true} light-utils = {workspace = true} light-verifier = {workspace = true} rand = "0.8" -solana-cli-output = { workspace = true } -serde_json = "1.0.114" solana-sdk = { workspace = true } -thiserror = "1.0" -memoffset = "0.9.1" serial_test = "3.1.1" light-bloom-filter = { workspace = true } light-batched-merkle-tree = { workspace = true } diff --git a/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs b/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs index 1d67a42ba3..45cda27e39 100644 --- a/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs +++ b/program-tests/account-compression-test/tests/address_merkle_tree_tests.rs @@ -12,6 +12,7 @@ use anchor_lang::error::ErrorCode; use ark_bn254::Fr; use ark_ff::{BigInteger, PrimeField, UniformRand}; use light_bounded_vec::BoundedVecError; +use light_client::indexer::{AddressMerkleTreeAccounts, AddressMerkleTreeBundle}; use light_concurrent_merkle_tree::errors::ConcurrentMerkleTreeError; use light_hash_set::{HashSet, HashSetError}; use light_hasher::Poseidon; @@ -28,7 +29,7 @@ use light_test_utils::{ create_address_merkle_tree_and_queue_account_with_assert, get_hash_set, get_indexed_merkle_tree, test_forester::{empty_address_queue_test, update_merkle_tree}, - AddressMerkleTreeAccounts, AddressMerkleTreeBundle, FeeConfig, RpcConnection, RpcError, + FeeConfig, RpcConnection, RpcError, }; use light_utils::{bigint::bigint_to_be_bytes_array, UtilsError}; use num_bigint::ToBigUint; diff --git a/program-tests/compressed-token-test/Cargo.toml b/program-tests/compressed-token-test/Cargo.toml index 99be0fe0b3..cd05bd8122 100644 --- a/program-tests/compressed-token-test/Cargo.toml +++ b/program-tests/compressed-token-test/Cargo.toml @@ -22,23 +22,18 @@ anchor-lang = { workspace = true } light-compressed-token = { workspace = true } light-system-program = { workspace = true } account-compression = { workspace = true } -light-hasher = {workspace = true} -light-concurrent-merkle-tree = {workspace = true} -light-utils = {workspace = true} -light-verifier = {workspace = true} [target.'cfg(not(target_os = "solana"))'.dependencies] solana-sdk = { workspace = true } [dev-dependencies] -solana-program-test = { workspace = true } +light-client = { workspace = true } +light-sdk = { workspace = true } +light-verifier = {workspace = true} light-test-utils = { workspace = true, features=["devenv"] } light-program-test = { workspace = true, features = ["devenv"] } -reqwest = "0.11.26" tokio = { workspace = true } light-prover-client = {workspace = true } -num-bigint = "0.4.6" -num-traits = "0.2.19" spl-token = { workspace = true } anchor-spl = { workspace = true } rand = "0.8" diff --git a/program-tests/compressed-token-test/tests/test.rs b/program-tests/compressed-token-test/tests/test.rs index ef9820066d..5a361c1696 100644 --- a/program-tests/compressed-token-test/tests/test.rs +++ b/program-tests/compressed-token-test/tests/test.rs @@ -9,6 +9,7 @@ use anchor_spl::{ token::{Mint, TokenAccount}, token_2022::{spl_token_2022, spl_token_2022::extension::ExtensionType}, }; +use light_client::indexer::Indexer; use light_compressed_token::{ constants::NUM_MAX_POOL_ACCOUNTS, delegation::sdk::{ @@ -22,21 +23,27 @@ use light_compressed_token::{ get_cpi_authority_pda, transfer_sdk::create_transfer_instruction, TokenTransferOutputData, }, spl_compression::check_spl_token_pool_derivation_with_index, - token_data::{AccountState, TokenData}, + token_data::TokenData, ErrorCode, }; use light_program_test::{ - test_env::setup_test_programs_with_accounts, test_rpc::ProgramTestRpcConnection, + indexer::{TestIndexer, TestIndexerExtensions}, + test_env::setup_test_programs_with_accounts, + test_rpc::ProgramTestRpcConnection, }; use light_prover_client::gnark::helpers::{kill_prover, spawn_prover, ProofType, ProverConfig}; +use light_sdk::token::{AccountState, TokenDataWithMerkleContext}; use light_system_program::{ invoke::processor::CompressedProof, sdk::compressed_account::{CompressedAccountWithMerkleContext, MerkleContext}, }; use light_test_utils::{ airdrop_lamports, assert_custom_error_or_program_error, assert_rpc_error, + conversions::{ + sdk_to_program_compressed_account, sdk_to_program_compressed_account_with_merkle_context, + sdk_to_program_compressed_proof, sdk_to_program_merkle_context, sdk_to_program_token_data, + }, create_account_instruction, - indexer::TestIndexer, spl::{ approve_test, burn_test, compress_test, compressed_transfer_22_test, compressed_transfer_test, create_additional_token_pools, create_burn_test_instruction, @@ -46,7 +53,7 @@ use light_test_utils::{ mint_tokens_helper_with_lamports, mint_wrapped_sol, perform_compress_spl_token_account, revoke_test, thaw_test, BurnInstructionMode, }, - Indexer, RpcConnection, RpcError, TokenDataWithContext, + RpcConnection, RpcError, }; use light_verifier::VerifierError; use rand::{seq::SliceRandom, thread_rng, Rng}; @@ -1550,9 +1557,12 @@ async fn test_decompression() { kill_prover(); } -pub async fn mint_tokens_to_all_token_pools( +pub async fn mint_tokens_to_all_token_pools< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, merkle_tree_pubkey: &Pubkey, mint_authority: &Keypair, mint: &Pubkey, @@ -1925,7 +1935,7 @@ async fn test_delegation( .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); compressed_transfer_test( &delegate, &mut rpc, @@ -1951,7 +1961,7 @@ async fn test_delegation( .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); compressed_transfer_test( &delegate, &mut rpc, @@ -2059,7 +2069,7 @@ async fn test_delegation_mixed() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let delegate_input_compressed_accounts = test_indexer.get_compressed_token_accounts_by_owner(&delegate.pubkey()); input_compressed_accounts @@ -2098,7 +2108,7 @@ async fn test_delegation_mixed() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let delegate_input_compressed_accounts = test_indexer.get_compressed_token_accounts_by_owner(&delegate.pubkey()); input_compressed_accounts @@ -2139,7 +2149,7 @@ async fn test_delegation_mixed() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let delegate_input_compressed_accounts = test_indexer.get_compressed_token_accounts_by_owner(&delegate.pubkey()); @@ -2281,15 +2291,17 @@ async fn test_approve_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, delegated_amount, @@ -2298,7 +2310,7 @@ async fn test_approve_failing() { change_compressed_account_merkle_tree: delegated_compressed_account_merkle_tree, delegate: delegate.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_approve_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2328,15 +2340,17 @@ async fn test_approve_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, delegated_amount, @@ -2345,7 +2359,7 @@ async fn test_approve_failing() { change_compressed_account_merkle_tree: invalid_change_merkle_tree.pubkey(), delegate: delegate.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_approve_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2379,15 +2393,17 @@ async fn test_approve_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, delegated_amount, @@ -2419,15 +2435,17 @@ async fn test_approve_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint: invalid_mint.pubkey(), delegated_amount, @@ -2436,7 +2454,7 @@ async fn test_approve_failing() { change_compressed_account_merkle_tree: delegated_compressed_account_merkle_tree, delegate: delegate.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_approve_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2462,15 +2480,17 @@ async fn test_approve_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, delegated_amount, @@ -2479,7 +2499,7 @@ async fn test_approve_failing() { change_compressed_account_merkle_tree: delegated_compressed_account_merkle_tree, delegate: delegate.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_approve_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2562,12 +2582,12 @@ async fn test_revoke(num_inputs: usize, mint_amount: u64, delegated_amount: u64) .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let input_compressed_accounts = input_compressed_accounts .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let delegated_compressed_account_merkle_tree = input_compressed_accounts[0] .compressed_account .merkle_context @@ -2680,7 +2700,7 @@ async fn test_revoke_failing() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let input_compressed_account_hashes = input_compressed_accounts .iter() @@ -2710,20 +2730,22 @@ async fn test_revoke_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, output_account_merkle_tree: merkle_tree_pubkey, root_indices: invalid_root_indices, - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_revoke_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2746,20 +2768,22 @@ async fn test_revoke_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, output_account_merkle_tree: invalid_merkle_tree.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_revoke_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2791,20 +2815,22 @@ async fn test_revoke_failing() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint: invalid_mint.pubkey(), output_account_merkle_tree: merkle_tree_pubkey, root_indices: proof_rpc_result.root_indices, - proof: proof_rpc_result.proof, + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof), }; let instruction = create_revoke_instruction(inputs).unwrap(); let context_payer = rpc.get_payer().insecure_clone(); @@ -2924,7 +2950,7 @@ async fn test_burn() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let burn_amount = 100; let change_account_merkle_tree = input_compressed_accounts[0] .compressed_account @@ -2952,7 +2978,7 @@ async fn test_burn() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let burn_amount = input_compressed_accounts .iter() .map(|x| x.token_data.amount) @@ -3210,7 +3236,7 @@ async fn failing_tests_burn() { .iter() .filter(|x| x.token_data.delegate.is_some()) .cloned() - .collect::>(); + .collect::>(); let burn_amount = 1; let change_account_merkle_tree = input_compressed_accounts[0] .compressed_account @@ -3468,7 +3494,7 @@ async fn test_freeze_and_thaw(mint_amount: u64, delegated_amount: u64) { .iter() .filter(|x| x.token_data.state == AccountState::Frozen) .cloned() - .collect::>(); + .collect::>(); let output_merkle_tree = input_compressed_accounts[0] .compressed_account .merkle_context @@ -3532,7 +3558,7 @@ async fn test_freeze_and_thaw(mint_amount: u64, delegated_amount: u64) { .iter() .filter(|x| x.token_data.state == AccountState::Frozen) .cloned() - .collect::>(); + .collect::>(); let output_merkle_tree = input_compressed_accounts[0] .compressed_account .merkle_context @@ -3648,19 +3674,21 @@ async fn test_failing_freeze() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -3682,19 +3710,21 @@ async fn test_failing_freeze() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree: invalid_merkle_tree.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -3730,15 +3760,17 @@ async fn test_failing_freeze() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), @@ -3770,7 +3802,7 @@ async fn test_failing_freeze() { .iter() .filter(|x| x.token_data.state == AccountState::Frozen) .cloned() - .collect::>()[0] + .collect::>()[0] .clone()]; let outputs_merkle_tree = input_compressed_accounts[0] .compressed_account @@ -3800,19 +3832,21 @@ async fn test_failing_freeze() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -3903,7 +3937,7 @@ async fn test_failing_thaw() { .iter() .filter(|x| x.token_data.state == AccountState::Frozen) .cloned() - .collect::>(); + .collect::>(); let outputs_merkle_tree = input_compressed_accounts[0] .compressed_account .merkle_context @@ -3938,19 +3972,21 @@ async fn test_failing_thaw() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -3972,19 +4008,21 @@ async fn test_failing_thaw() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree: invalid_merkle_tree.pubkey(), root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -4020,15 +4058,17 @@ async fn test_failing_thaw() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), @@ -4052,7 +4092,7 @@ async fn test_failing_thaw() { .iter() .filter(|x| x.token_data.state == AccountState::Initialized) .cloned() - .collect::>(); + .collect::>(); let outputs_merkle_tree = input_compressed_accounts[0] .compressed_account .merkle_context @@ -4081,19 +4121,21 @@ async fn test_failing_thaw() { input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), outputs_merkle_tree, root_indices: proof_rpc_result.root_indices.clone(), - proof: proof_rpc_result.proof.clone(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.clone()), }; let instruction = create_instruction::(inputs).unwrap(); let result = rpc @@ -4651,11 +4693,14 @@ async fn test_failing_decompression() { } #[allow(clippy::too_many_arguments)] -pub async fn failing_compress_decompress( +pub async fn failing_compress_decompress< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( payer: &Keypair, rpc: &mut R, - test_indexer: &mut TestIndexer, - input_compressed_accounts: Vec, + test_indexer: &mut I, + input_compressed_accounts: Vec, amount: u64, output_merkle_tree_pubkey: &Pubkey, compression_amount: u64, @@ -4709,25 +4754,33 @@ pub async fn failing_compress_decompress( } else { (Vec::new(), None) }; + + let mut _proof = None; + if let Some(proof) = proof { + _proof = Some(sdk_to_program_compressed_proof(proof)); + } + let instruction = create_transfer_instruction( &rpc.get_payer().pubkey(), &payer.pubkey(), &input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect::>(), &[change_out_compressed_account], &root_indices, - &proof, + &_proof, input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect::>() .as_slice(), &input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), *mint, None, @@ -4852,6 +4905,15 @@ async fn test_invalid_inputs() { &mut rpc, ) .await; + let proof = Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.clone(), + )); + + let input_compressed_accounts = input_compressed_accounts + .iter() + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .collect::>(); + let change_out_compressed_account_0 = TokenTransferOutputData { amount: input_compressed_account_token_data.amount - 1000, owner: recipient_keypair.pubkey(), @@ -4868,6 +4930,7 @@ async fn test_invalid_inputs() { let mut transfer_recipient_out_compressed_account_0 = transfer_recipient_out_compressed_account_0; transfer_recipient_out_compressed_account_0.amount += 1; + // Test 1: invalid token data amount (+ 1) let res = perform_transfer_failing_test( &mut rpc, @@ -4876,7 +4939,9 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.clone(), + )), &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -4901,7 +4966,9 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.clone(), + )), &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -4926,7 +4993,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -4950,7 +5017,9 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.clone(), + )), &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -4974,7 +5043,9 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.clone(), + )), &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -4988,6 +5059,8 @@ async fn test_invalid_inputs() { let mut input_compressed_account_token_data_invalid_amount = test_indexer.token_compressed_accounts[0].token_data.clone(); input_compressed_account_token_data_invalid_amount.amount = 0; + let input_compressed_account_token_data_invalid_amount = + sdk_to_program_token_data(input_compressed_account_token_data_invalid_amount); let mut input_compressed_accounts = vec![test_indexer.token_compressed_accounts[0] .compressed_account .clone()]; @@ -5002,6 +5075,10 @@ async fn test_invalid_inputs() { .as_mut_slice(), ) .unwrap(); + let input_compressed_accounts = input_compressed_accounts + .iter() + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .collect::>(); let change_out_compressed_account_0 = TokenTransferOutputData { amount: input_compressed_account_token_data.amount - 1000, owner: recipient_keypair.pubkey(), @@ -5022,7 +5099,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -5036,9 +5113,16 @@ async fn test_invalid_inputs() { let mut input_compressed_account_token_data = test_indexer.token_compressed_accounts[0].token_data.clone(); input_compressed_account_token_data.delegate = Some(Pubkey::new_unique()); - let mut input_compressed_accounts = vec![test_indexer.token_compressed_accounts[0] + let input_compressed_accounts = vec![test_indexer.token_compressed_accounts[0] .compressed_account .clone()]; + let mut input_compressed_accounts = input_compressed_accounts + .iter() + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .collect::>(); + let input_compressed_account_token_data = + sdk_to_program_token_data(input_compressed_account_token_data); + let mut vec = Vec::new(); crate::TokenData::serialize(&input_compressed_account_token_data, &mut vec).unwrap(); input_compressed_accounts[0] @@ -5054,7 +5138,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &recipient_keypair, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -5073,7 +5157,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &invalid_payer, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, false, @@ -5094,7 +5178,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &payer, - &Some(proof_rpc_result.proof.clone()), + &proof, &root_indices, &input_compressed_accounts, false, @@ -5112,7 +5196,7 @@ async fn test_invalid_inputs() { &merkle_tree_pubkey, &nullifier_queue_pubkey, &payer, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, true, @@ -5130,7 +5214,7 @@ async fn test_invalid_inputs() { &nullifier_queue_pubkey, &nullifier_queue_pubkey, &payer, - &Some(proof_rpc_result.proof.clone()), + &proof, &proof_rpc_result.root_indices, &input_compressed_accounts, false, diff --git a/program-tests/e2e-test/tests/test.rs b/program-tests/e2e-test/tests/test.rs index 14bf08268b..f58cd91069 100644 --- a/program-tests/e2e-test/tests/test.rs +++ b/program-tests/e2e-test/tests/test.rs @@ -5,15 +5,13 @@ use light_batched_merkle_tree::{ initialize_state_tree::InitStateTreeAccountsInstructionData, }; use light_program_test::{ + indexer::TestIndexer, test_env::setup_test_programs_with_accounts_with_protocol_config_and_batched_tree_params, test_rpc::ProgramTestRpcConnection, }; use light_prover_client::gnark::helpers::{ProofType, ProverConfig}; use light_registry::protocol_config::state::ProtocolConfig; -use light_test_utils::{ - e2e_test_env::{E2ETestEnv, GeneralActionConfig, KeypairActionConfig}, - indexer::TestIndexer, -}; +use light_test_utils::e2e_test_env::{E2ETestEnv, GeneralActionConfig, KeypairActionConfig}; #[tokio::test] async fn test_10_all() { diff --git a/program-tests/registry-test/Cargo.toml b/program-tests/registry-test/Cargo.toml index f1419087f5..37f5cb1da6 100644 --- a/program-tests/registry-test/Cargo.toml +++ b/program-tests/registry-test/Cargo.toml @@ -19,31 +19,18 @@ default = ["custom-heap"] [dependencies] - [dev-dependencies] -solana-program-test = { workspace = true } light-test-utils = { workspace = true, features=["devenv"] } light-program-test = { workspace = true, features = ["devenv"] } -reqwest = "0.11.26" tokio = { workspace = true } light-prover-client = {workspace = true, features = ["devenv"] } -num-bigint = "0.4.6" -num-traits = "0.2.19" -spl-token = { workspace = true } -anchor-spl = { workspace = true } +light-client = { workspace = true } anchor-lang = { workspace = true } forester-utils = { workspace = true } light-registry = { workspace = true } -light-compressed-token = { workspace = true } -light-system-program = { workspace = true } account-compression = { workspace = true } light-hasher = {workspace = true} -light-concurrent-merkle-tree = {workspace = true} -light-indexed-merkle-tree = {workspace = true} light-utils = {workspace = true} -light-verifier = {workspace = true} -solana-cli-output = { workspace = true } -serde_json = "1.0.133" solana-sdk = { workspace = true } serial_test = { workspace = true } light-batched-merkle-tree = { workspace = true } diff --git a/program-tests/registry-test/tests/tests.rs b/program-tests/registry-test/tests/tests.rs index 24c0f17ad5..c215372857 100644 --- a/program-tests/registry-test/tests/tests.rs +++ b/program-tests/registry-test/tests/tests.rs @@ -9,7 +9,6 @@ use account_compression::{ use anchor_lang::{AnchorSerialize, InstructionData, ToAccountMetas}; use forester_utils::{ airdrop_lamports, forester_epoch::get_epoch_phases, get_concurrent_merkle_tree, - indexer::Indexer, }; use light_batched_merkle_tree::{ initialize_address_tree::InitAddressTreeAccountsInstructionData, @@ -19,8 +18,10 @@ use light_batched_merkle_tree::{ merkle_tree::{BatchedMerkleTreeAccount, BatchedMerkleTreeMetadata, CreateTreeParams}, queue::BatchedQueueAccount, }; +use light_client::indexer::Indexer; use light_hasher::Poseidon; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_batch_forester::{ assert_perform_state_mt_roll_over, create_append_batch_ix_data, create_batch_address_merkle_tree, @@ -70,7 +71,6 @@ use light_test_utils::{ create_rollover_address_merkle_tree_instructions, create_rollover_state_merkle_tree_instructions, e2e_test_env::{init_program_test_env, init_program_test_env_forester}, - indexer::TestIndexer, register_test_forester, test_forester::{empty_address_queue_test, nullify_compressed_accounts}, update_test_forester, Epoch, RpcConnection, RpcError, SolanaRpcConnection, SolanaRpcUrl, @@ -1427,7 +1427,8 @@ async fn test_migrate_state() { 26, >(&mut rpc, env_accounts.merkle_tree_pubkey) .await; - let compressed_account = &test_indexer.get_compressed_accounts_by_owner(&payer.pubkey())[0]; + let compressed_account = + &test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&payer.pubkey())[0]; let hash = compressed_account.hash().unwrap(); let bundle = &test_indexer .get_state_merkle_trees() @@ -1511,7 +1512,8 @@ async fn test_migrate_state() { 26, >(&mut rpc, env_accounts.merkle_tree_pubkey) .await; - let compressed_account = &test_indexer.get_compressed_accounts_by_owner(&payer.pubkey())[1]; + let compressed_account = + &test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&payer.pubkey())[1]; let hash = compressed_account.hash().unwrap(); let bundle = &test_indexer .get_state_merkle_trees() @@ -1957,9 +1959,12 @@ async fn test_batch_address_tree() { .await; } -pub async fn perform_batch_address_merkle_tree_update( +pub async fn perform_batch_address_merkle_tree_update< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, forester: &Keypair, derivation_pubkey: &Pubkey, merkle_tree_pubkey: &Pubkey, diff --git a/program-tests/sdk-test-program/package.json b/program-tests/sdk-test-program/package.json index ef81bf80c1..2116e5ccad 100644 --- a/program-tests/sdk-test-program/package.json +++ b/program-tests/sdk-test-program/package.json @@ -1,6 +1,6 @@ { "scripts": { - "test": "cargo test-sbf -p sdk-test -- --test-threads 1" + "test": "cargo test-sbf -p sdk-test" }, "dependencies": { "@coral-xyz/anchor": "^0.29.0" diff --git a/program-tests/sdk-test-program/programs/sdk-test/Cargo.toml b/program-tests/sdk-test-program/programs/sdk-test/Cargo.toml index 7366f03911..7e55ca805f 100644 --- a/program-tests/sdk-test-program/programs/sdk-test/Cargo.toml +++ b/program-tests/sdk-test-program/programs/sdk-test/Cargo.toml @@ -22,13 +22,8 @@ idl-build = ["anchor-lang/idl-build", "light-sdk/idl-build"] [dependencies] anchor-lang = { workspace=true} -borsh = { workspace = true } light-hasher = { workspace = true, features = ["solana"] } -light-macros = { workspace = true } light-sdk = { workspace = true } -light-sdk-macros = { workspace = true } -light-utils = { workspace = true } -light-verifier = { workspace = true } [target.'cfg(not(target_os = "solana"))'.dependencies] solana-sdk = { workspace = true } @@ -37,5 +32,5 @@ solana-sdk = { workspace = true } light-client = { workspace = true , features = ["devenv"]} light-program-test = { workspace = true, features = ["devenv"] } light-test-utils = { workspace = true, features = ["devenv"] } -solana-program-test = { workspace = true } +light-prover-client = { workspace = true } tokio = { workspace = true } diff --git a/program-tests/sdk-test-program/programs/sdk-test/tests/test.rs b/program-tests/sdk-test-program/programs/sdk-test/tests/test.rs index 92d4110a87..542cc70739 100644 --- a/program-tests/sdk-test-program/programs/sdk-test/tests/test.rs +++ b/program-tests/sdk-test-program/programs/sdk-test/tests/test.rs @@ -6,10 +6,11 @@ use light_client::{ rpc::merkle_tree::MerkleTreeExt, }; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_env::{setup_test_programs_with_accounts_v2, EnvAccounts}, - test_indexer::TestIndexer, test_rpc::ProgramTestRpcConnection, }; +use light_prover_client::gnark::helpers::{ProofType, ProverConfig}; use light_sdk::{ account_meta::LightAccountMeta, address::derive_address, @@ -36,17 +37,21 @@ async fn test_sdk_test() { let payer = rpc.get_payer().insecure_clone(); let mut test_indexer: TestIndexer = TestIndexer::new( - &[StateMerkleTreeAccounts { + vec![StateMerkleTreeAccounts { merkle_tree: env.merkle_tree_pubkey, nullifier_queue: env.nullifier_queue_pubkey, cpi_context: env.cpi_context_account_pubkey, }], - &[AddressMerkleTreeAccounts { + vec![AddressMerkleTreeAccounts { merkle_tree: env.address_merkle_tree_pubkey, queue: env.address_merkle_tree_queue_pubkey, }], - true, - true, + payer.insecure_clone(), + env.group_pda, + Some(ProverConfig { + circuits: vec![ProofType::Inclusion, ProofType::NonInclusion], + run_mode: None, + }), ) .await; @@ -86,7 +91,8 @@ async fn test_sdk_test() { .unwrap(); // Check that it was created correctly. - let compressed_accounts = test_indexer.get_compressed_accounts_by_owner(&sdk_test::ID); + let compressed_accounts = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&sdk_test::ID); assert_eq!(compressed_accounts.len(), 1); let compressed_account = &compressed_accounts[0]; let record = &compressed_account @@ -126,7 +132,8 @@ async fn test_sdk_test() { .unwrap(); // Check that it was updated correctly. - let compressed_accounts = test_indexer.get_compressed_accounts_by_owner(&sdk_test::ID); + let compressed_accounts = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&sdk_test::ID); assert_eq!(compressed_accounts.len(), 1); let compressed_account = &compressed_accounts[0]; let record = &compressed_account @@ -139,10 +146,10 @@ async fn test_sdk_test() { assert_eq!(record.nested.one, 2); } -async fn with_nested_data( +async fn with_nested_data( name: String, rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, env: &EnvAccounts, remaining_accounts: &mut RemainingAccounts, payer: &Keypair, @@ -153,6 +160,7 @@ async fn with_nested_data( ) -> Result<(), RpcError> where R: RpcConnection + MerkleTreeExt, + I: Indexer + TestIndexerExtensions, { let rpc_result = test_indexer .create_proof_for_compressed_accounts( @@ -209,13 +217,14 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } -async fn update_nested_data( +async fn update_nested_data( rpc: &mut R, - test_indexer: &mut TestIndexer, + test_indexer: &mut I, remaining_accounts: &mut RemainingAccounts, nested_data: NestedData, payer: &Keypair, @@ -226,14 +235,15 @@ async fn update_nested_data( ) -> Result<(), RpcError> where R: RpcConnection + MerkleTreeExt, + I: Indexer + TestIndexerExtensions, { let hash = compressed_account.hash().unwrap(); let merkle_tree_pubkey = compressed_account.merkle_context.merkle_tree_pubkey; let rpc_result = test_indexer .create_proof_for_compressed_accounts( - Some(&[hash]), - Some(&[merkle_tree_pubkey]), + Some(vec![hash]), + Some(vec![merkle_tree_pubkey]), None, None, rpc, @@ -282,6 +292,7 @@ where let event = rpc .create_and_send_transaction_with_event(&[instruction], &payer.pubkey(), &[payer], None) .await?; - test_indexer.add_compressed_accounts_with_token_data(&event.unwrap().0); + let slot = rpc.get_slot().await.unwrap(); + test_indexer.add_compressed_accounts_with_token_data(slot, &event.unwrap().0); Ok(()) } diff --git a/program-tests/system-cpi-test/Cargo.toml b/program-tests/system-cpi-test/Cargo.toml index af81de6cc5..526d00244d 100644 --- a/program-tests/system-cpi-test/Cargo.toml +++ b/program-tests/system-cpi-test/Cargo.toml @@ -36,6 +36,8 @@ solana-sdk = { workspace = true } [dev-dependencies] solana-program-test = { workspace = true } +light-client = { workspace = true } +light-sdk = { workspace = true } light-program-test = { workspace = true, features=["devenv"]} light-test-utils = { workspace = true, features=["devenv"] } reqwest = "0.11.26" diff --git a/program-tests/system-cpi-test/tests/test.rs b/program-tests/system-cpi-test/tests/test.rs index e048fff3fb..e8c9f3c0fa 100644 --- a/program-tests/system-cpi-test/tests/test.rs +++ b/program-tests/system-cpi-test/tests/test.rs @@ -3,12 +3,12 @@ use account_compression::errors::AccountCompressionErrorCode; use anchor_lang::{AnchorDeserialize, AnchorSerialize}; use light_batched_merkle_tree::initialize_state_tree::InitStateTreeAccountsInstructionData; -use light_compressed_token::{ - process_transfer::InputTokenDataWithContext, token_data::AccountState, -}; +use light_client::indexer::Indexer; +use light_compressed_token::process_transfer::InputTokenDataWithContext; use light_hasher::{Hasher, Poseidon}; use light_merkle_tree_metadata::errors::MerkleTreeMetadataError; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_batch_forester::{ create_batch_update_address_tree_instruction_data_with_proof, perform_batch_append, }, @@ -16,6 +16,7 @@ use light_program_test::{ }; use light_prover_client::gnark::helpers::{ProverConfig, ProverMode}; use light_registry::account_compression_cpi::sdk::create_batch_update_address_tree_instruction; +use light_sdk::token::{AccountState, TokenDataWithMerkleContext}; use light_system_program::{ errors::SystemProgramError, sdk::{ @@ -31,11 +32,14 @@ use light_system_program::{ }; use light_test_utils::{ assert_rpc_error, + conversions::{ + program_to_sdk_public_transaction_event, + sdk_to_program_compressed_account_with_merkle_context, sdk_to_program_compressed_proof, + }, e2e_test_env::init_program_test_env, - indexer::TestIndexer, spl::{create_mint_helper, mint_tokens_helper}, system_program::transfer_compressed_sol_test, - Indexer, RpcConnection, RpcError, TokenDataWithContext, + RpcConnection, RpcError, }; use light_utils::{hash_to_bn254_field_size_be, UtilsError}; use light_verifier::VerifierError; @@ -199,25 +203,25 @@ async fn test_read_only_accounts() { // account in batched state mt and value vec let account_in_value_array = e2e_env .indexer - .get_compressed_accounts_by_owner(&ID) + .get_compressed_accounts_with_merkle_context_by_owner(&ID) .iter() .find(|x| { x.merkle_context.leaf_index == 101 && x.merkle_context.merkle_tree_pubkey == env.batched_state_merkle_tree }) - .unwrap() - .clone(); + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .unwrap(); let account_not_in_value_array_and_in_mt = e2e_env .indexer - .get_compressed_accounts_by_owner(&ID) + .get_compressed_accounts_with_merkle_context_by_owner(&ID) .iter() .find(|x| { x.merkle_context.leaf_index == 1 && x.merkle_context.merkle_tree_pubkey == env.batched_state_merkle_tree }) - .unwrap() - .clone(); + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .unwrap(); // 1. functional - 1 read only account proof by index, an create 1 new account { @@ -308,11 +312,11 @@ async fn test_read_only_accounts() { let data = [4u8; 31]; let account_in_v1_tree = e2e_env .indexer - .get_compressed_accounts_by_owner(&ID) + .get_compressed_accounts_with_merkle_context_by_owner(&ID) .iter() .find(|x| x.merkle_context.merkle_tree_pubkey == env.merkle_tree_pubkey) - .unwrap() - .clone(); + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .unwrap(); let result = perform_create_pda_with_event( &mut e2e_env.indexer, &mut e2e_env.rpc, @@ -592,7 +596,7 @@ async fn test_read_only_accounts() { let data = [5u8; 31]; let input_account_in_mt = e2e_env .indexer - .get_compressed_accounts_by_owner(&ID) + .get_compressed_accounts_with_merkle_context_by_owner(&ID) .iter() .find(|x| { x.merkle_context.leaf_index == 2 @@ -602,8 +606,8 @@ async fn test_read_only_accounts() { .merkle_context .leaf_index }) - .unwrap() - .clone(); + .map(|x| sdk_to_program_compressed_account_with_merkle_context(x.clone())) + .unwrap(); perform_create_pda_with_event( &mut e2e_env.indexer, &mut e2e_env.rpc, @@ -992,6 +996,8 @@ async fn only_test_create_pda() { [0] .compressed_account .clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); // Failing 4 input account that is not owned by signer ---------------------------------------------- perform_with_input_accounts( &mut test_indexer, @@ -1006,7 +1012,10 @@ async fn only_test_create_pda() { .await .unwrap(); { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); // Failing 5 provide cpi context but no cpi context account ---------------------------------------------- perform_with_input_accounts( &mut test_indexer, @@ -1096,7 +1105,10 @@ async fn only_test_create_pda() { 26, 211, 193, 195, 11, 219, 9, 155, 58, 172, 58, 200, 254, 75, 231, 106, 31, 168, 183, 76, 179, 113, 234, 101, 191, 99, 156, 98, ]; - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); let keypair = Keypair::from_bytes(&CPI_SYSTEM_TEST_PROGRAM_ID_KEYPAIR).unwrap(); let result = transfer_compressed_sol_test( &mut rpc, @@ -1188,7 +1200,10 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { test_indexer.get_compressed_token_accounts_by_owner(&payer.pubkey())[0].clone(); // 1. Approve functional with cpi context { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); let compressed_token_data = test_indexer.get_compressed_token_accounts_by_owner(&payer.pubkey())[0].clone(); perform_with_input_accounts( @@ -1215,7 +1230,10 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { } // 2. Revoke functional with cpi context { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); let compressed_token_data = test_indexer .get_compressed_token_accounts_by_owner(&payer.pubkey()) .iter() @@ -1241,7 +1259,10 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { } // 3. Freeze functional with cpi context { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); let compressed_token_data = test_indexer.get_compressed_token_accounts_by_owner(&payer.pubkey())[0].clone(); perform_with_input_accounts( @@ -1264,7 +1285,10 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { } // 4. Thaw functional with cpi context { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); + let compressed_account = + sdk_to_program_compressed_account_with_merkle_context(compressed_account); let compressed_token_data = test_indexer.get_compressed_token_accounts_by_owner(&payer.pubkey())[0].clone(); perform_with_input_accounts( @@ -1286,7 +1310,8 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { } // 5. Burn functional with cpi context { - let compressed_account = test_indexer.get_compressed_accounts_by_owner(&ID)[0].clone(); + let compressed_account = + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&ID)[0].clone(); let compressed_token_data = test_indexer.get_compressed_token_accounts_by_owner(&payer.pubkey())[0].clone(); perform_with_input_accounts( @@ -1294,7 +1319,7 @@ async fn test_approve_revoke_burn_freeze_thaw_with_cpi_context() { &mut rpc, &payer, None, - &compressed_account, + &sdk_to_program_compressed_account_with_merkle_context(compressed_account), Some(compressed_token_data), u32::MAX, WithInputAccountsMode::Burn, @@ -1499,8 +1524,11 @@ async fn test_create_pda_in_program_owned_merkle_trees() { } #[allow(clippy::too_many_arguments)] -pub async fn perform_create_pda_failing( - test_indexer: &mut TestIndexer, +pub async fn perform_create_pda_failing< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + test_indexer: &mut I, rpc: &mut R, env: &EnvAccounts, payer: &Keypair, @@ -1535,8 +1563,11 @@ pub async fn perform_create_pda_failing( } #[allow(clippy::too_many_arguments)] -pub async fn perform_create_pda_with_event( - test_indexer: &mut TestIndexer, +pub async fn perform_create_pda_with_event< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + test_indexer: &mut I, rpc: &mut R, env: &EnvAccounts, payer: &Keypair, @@ -1580,10 +1611,10 @@ pub async fn perform_create_pda_with_event( } #[allow(clippy::too_many_arguments)] -async fn perform_create_pda( +async fn perform_create_pda + TestIndexerExtensions>( env: &EnvAccounts, seed: [u8; 32], - test_indexer: &mut TestIndexer, + test_indexer: &mut I, rpc: &mut R, data: &[u8; 31], payer_pubkey: Pubkey, @@ -1761,7 +1792,7 @@ async fn perform_create_pda( data: *data, signer: &payer_pubkey, output_compressed_account_merkle_tree_pubkey, - proof: &rpc_result.proof.unwrap(), + proof: &sdk_to_program_compressed_proof(rpc_result.proof.unwrap()), new_address_params, cpi_context_account: &env.cpi_context_account_pubkey, owner_program, @@ -1775,15 +1806,15 @@ async fn perform_create_pda( create_pda_instruction(create_ix_inputs) } -pub async fn assert_created_pda( - test_indexer: &mut TestIndexer, +pub async fn assert_created_pda + TestIndexerExtensions>( + test_indexer: &mut I, env: &EnvAccounts, payer: &Keypair, seed: &[u8; 32], data: &[u8; 31], ) { let compressed_escrow_pda = test_indexer - .compressed_accounts + .get_compressed_accounts_with_merkle_context_by_owner(&ID) .iter() .find(|x| x.compressed_account.owner == ID) .unwrap() @@ -1820,13 +1851,16 @@ pub async fn assert_created_pda( } #[allow(clippy::too_many_arguments)] -pub async fn perform_with_input_accounts( - test_indexer: &mut TestIndexer, +pub async fn perform_with_input_accounts< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + test_indexer: &mut I, rpc: &mut R, payer: &Keypair, fee_payer: Option<&Keypair>, compressed_account: &CompressedAccountWithMerkleContext, - token_account: Option, + token_account: Option, expected_error_code: u32, mode: WithInputAccountsMode, ) -> Result<(), RpcError> { @@ -1868,7 +1902,7 @@ pub async fn perform_with_input_accounts( _ => None, }; let cpi_context_account_pubkey = test_indexer - .state_merkle_trees + .get_state_merkle_trees() .iter() .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) .unwrap() @@ -1922,7 +1956,7 @@ pub async fn perform_with_input_accounts( input_nullifier_pubkey: &nullifier_pubkey, cpi_context_account: &cpi_context_account_pubkey, cpi_context, - proof: &rpc_result.proof, + proof: &sdk_to_program_compressed_proof(rpc_result.proof), compressed_account: &PackedCompressedAccountWithMerkleContext { compressed_account: compressed_account.compressed_account.clone(), merkle_context: PackedMerkleContext { @@ -1950,7 +1984,10 @@ pub async fn perform_with_input_accounts( if expected_error_code == u32::MAX { let result = result?.unwrap(); let slot: u64 = rpc.get_slot().await.unwrap(); - test_indexer.add_compressed_accounts_with_token_data(slot, &result.0); + test_indexer.add_compressed_accounts_with_token_data( + slot, + &program_to_sdk_public_transaction_event(result.0), + ); Ok(()) } else { assert_rpc_error(result, 0, expected_error_code) diff --git a/program-tests/system-cpi-test/tests/test_program_owned_trees.rs b/program-tests/system-cpi-test/tests/test_program_owned_trees.rs index e270f45abb..0687edb002 100644 --- a/program-tests/system-cpi-test/tests/test_program_owned_trees.rs +++ b/program-tests/system-cpi-test/tests/test_program_owned_trees.rs @@ -10,6 +10,7 @@ use anchor_lang::{system_program, InstructionData, ToAccountMetas}; use light_compressed_token::mint_sdk::create_mint_to_instruction; use light_hasher::Poseidon; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_env::{ initialize_new_group, register_program_with_registry_program, setup_test_programs_with_accounts, NOOP_PROGRAM_ID, @@ -29,8 +30,8 @@ use light_registry::{ }; use light_test_utils::{ airdrop_lamports, assert_custom_error_or_program_error, assert_rpc_error, - create_account_instruction, get_concurrent_merkle_tree, indexer::TestIndexer, - spl::create_mint_helper, FeeConfig, RpcConnection, RpcError, TransactionParams, + create_account_instruction, get_concurrent_merkle_tree, spl::create_mint_helper, FeeConfig, + RpcConnection, RpcError, TransactionParams, }; use serial_test::serial; use solana_sdk::{ diff --git a/program-tests/system-test/Cargo.toml b/program-tests/system-test/Cargo.toml index 9cbbe55c35..8107f1a8b4 100644 --- a/program-tests/system-test/Cargo.toml +++ b/program-tests/system-test/Cargo.toml @@ -21,24 +21,17 @@ default = ["custom-heap"] [dev-dependencies] -solana-program-test = { workspace = true } light-program-test = { workspace = true, features=["devenv"] } light-test-utils = { workspace = true, features=["devenv"]} -reqwest = "0.11.26" tokio = { workspace = true } light-prover-client = {workspace = true } -num-bigint = "0.4.6" -num-traits = "0.2.19" -spl-token = { workspace = true } -anchor-spl = { workspace = true } anchor-lang = { workspace = true } -light-compressed-token = { workspace = true } light-system-program = { workspace = true } account-compression = { workspace = true } light-hasher = {workspace = true} -light-concurrent-merkle-tree = {workspace = true} -light-indexed-merkle-tree = {workspace = true} light-utils = {workspace = true} +light-client = { workspace = true } +light-sdk = { workspace = true } light-verifier = {workspace = true} light-registry = { workspace = true} solana-cli-output = { workspace = true } diff --git a/program-tests/system-test/tests/test.rs b/program-tests/system-test/tests/test.rs index b0c0c525c6..a91cee91c6 100644 --- a/program-tests/system-test/tests/test.rs +++ b/program-tests/system-test/tests/test.rs @@ -1,4 +1,5 @@ #![cfg(feature = "test-sbf")] + use account_compression::errors::AccountCompressionErrorCode; use anchor_lang::{error::ErrorCode, AnchorSerialize, InstructionData, ToAccountMetas}; use light_batched_merkle_tree::{ @@ -6,9 +7,11 @@ use light_batched_merkle_tree::{ initialize_address_tree::InitAddressTreeAccountsInstructionData, initialize_state_tree::InitStateTreeAccountsInstructionData, queue::BatchedQueueAccount, }; +use light_client::indexer::Indexer; use light_hasher::Poseidon; use light_merkle_tree_metadata::errors::MerkleTreeMetadataError; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_batch_forester::perform_batch_append, test_env::{ initialize_accounts, setup_test_programs, setup_test_programs_with_accounts, @@ -18,6 +21,7 @@ use light_program_test::{ }; use light_prover_client::gnark::helpers::{spawn_prover, ProofType, ProverConfig, ProverMode}; use light_registry::protocol_config::state::ProtocolConfig; +use light_sdk::merkle_context::QueueIndex as SdkQueueIndex; use light_system_program::{ errors::SystemProgramError, invoke::processor::CompressedProof, @@ -38,11 +42,14 @@ use light_test_utils::{ airdrop_lamports, assert_compressed_tx::assert_created_compressed_accounts, assert_custom_error_or_program_error, assert_rpc_error, - indexer::TestIndexer, + conversions::{ + sdk_to_program_compressed_account, sdk_to_program_compressed_account_with_merkle_context, + sdk_to_program_compressed_proof, sdk_to_program_merkle_context, + }, system_program::{ compress_sol_test, create_addresses_test, decompress_sol_test, transfer_compressed_sol_test, }, - FeeConfig, Indexer, RpcConnection, RpcError, TransactionParams, + FeeConfig, RpcConnection, RpcError, TransactionParams, }; use light_utils::{hash_to_bn254_field_size_be, UtilsError}; use light_verifier::VerifierError; @@ -174,9 +181,12 @@ async fn invoke_failing_test() { } #[allow(clippy::too_many_arguments)] -pub async fn failing_transaction_inputs( - context: &mut ProgramTestRpcConnection, - test_indexer: &mut TestIndexer, +pub async fn failing_transaction_inputs< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + context: &mut R, + test_indexer: &mut I, payer: &Keypair, env: &EnvAccounts, num_inputs: usize, @@ -202,8 +212,9 @@ pub async fn failing_transaction_inputs( } let (mut new_address_params, derived_addresses) = create_address_test_inputs(env, num_addresses); - let input_compressed_accounts = - test_indexer.get_compressed_accounts_by_owner(&payer.pubkey())[0..num_inputs].to_vec(); + let input_compressed_accounts = test_indexer + .get_compressed_accounts_with_merkle_context_by_owner(&payer.pubkey())[0..num_inputs] + .to_vec(); let hashes = input_compressed_accounts .iter() .map(|x| x.hash().unwrap()) @@ -239,7 +250,10 @@ pub async fn failing_transaction_inputs( for (i, root_index) in proof_rpc_res.address_root_indices.iter().enumerate() { new_address_params[i].address_merkle_tree_root_index = *root_index; } - (proof_rpc_res.root_indices, Some(proof_rpc_res.proof)) + ( + proof_rpc_res.root_indices, + Some(sdk_to_program_compressed_proof(proof_rpc_res.proof)), + ) } else { (Vec::new(), None) }; @@ -276,10 +290,12 @@ pub async fn failing_transaction_inputs( &input_compressed_accounts .iter() .map(|x| x.merkle_context) + .map(|x| sdk_to_program_merkle_context(x)) .collect::>(), &input_compressed_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(|x| sdk_to_program_compressed_account(x)) .collect::>(), &root_indices, &output_merkle_tree_pubkeys, @@ -321,8 +337,8 @@ pub async fn failing_transaction_inputs( Ok(()) } -pub async fn failing_transaction_inputs_inner( - context: &mut ProgramTestRpcConnection, +pub async fn failing_transaction_inputs_inner( + context: &mut R, payer: &Keypair, env: &EnvAccounts, inputs_struct: &InstructionDataInvoke, @@ -581,8 +597,8 @@ fn create_address_test_inputs( (new_address_params, derived_addresses) } -pub async fn failing_transaction_address( - context: &mut ProgramTestRpcConnection, +pub async fn failing_transaction_address( + context: &mut R, payer: &Keypair, env: &EnvAccounts, inputs_struct: &InstructionDataInvoke, @@ -701,8 +717,8 @@ pub async fn failing_transaction_address( /// 2. data but signer is not a program /// 3. invalid output Merkle tree /// 4. address that doesn't exist -pub async fn failing_transaction_output( - context: &mut ProgramTestRpcConnection, +pub async fn failing_transaction_output( + context: &mut R, payer: &Keypair, env: &EnvAccounts, inputs_struct: InstructionDataInvoke, @@ -838,8 +854,8 @@ pub async fn perform_tx_with_output_compressed_accounts( assert_rpc_error(result, 0, expected_error_code) } -pub async fn create_instruction_and_failing_transaction( - context: &mut ProgramTestRpcConnection, +pub async fn create_instruction_and_failing_transaction( + context: &mut R, payer: &Keypair, inputs_struct: InstructionDataInvoke, remaining_accounts: Vec, @@ -943,6 +959,10 @@ async fn invoke_test() { let slot: u64 = context.get_slot().await.unwrap(); let (created_compressed_accounts, _) = test_indexer.add_event_and_compressed_accounts(slot, &event.0); + let created_compressed_accounts = created_compressed_accounts + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>(); assert_created_compressed_accounts( output_compressed_accounts.as_slice(), output_merkle_tree_pubkeys.as_slice(), @@ -1039,7 +1059,11 @@ async fn invoke_test() { &mut context, ) .await; - let input_compressed_accounts = vec![compressed_account_with_context.compressed_account]; + let proof = sdk_to_program_compressed_proof(proof_rpc_res.proof.clone()); + let input_compressed_accounts = vec![sdk_to_program_compressed_account( + compressed_account_with_context.compressed_account, + )]; + let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, @@ -1054,7 +1078,7 @@ async fn invoke_test() { &[merkle_tree_pubkey], &proof_rpc_res.root_indices, &Vec::new(), - Some(proof_rpc_res.proof.clone()), + Some(proof.clone()), None, false, None, @@ -1103,7 +1127,7 @@ async fn invoke_test() { &[merkle_tree_pubkey], &proof_rpc_res.root_indices, &Vec::new(), - Some(proof_rpc_res.proof.clone()), + Some(proof.clone()), None, false, None, @@ -1134,7 +1158,7 @@ async fn invoke_test() { &[merkle_tree_pubkey], &proof_rpc_res.root_indices, &Vec::new(), - Some(proof_rpc_res.proof.clone()), + Some(proof.clone()), None, false, None, @@ -1319,7 +1343,9 @@ async fn test_with_address() { // transfer with address println!("transfer with address-------------------------"); - let compressed_account_with_context = test_indexer.compressed_accounts[0].clone(); + let compressed_account_with_context = sdk_to_program_compressed_account_with_merkle_context( + test_indexer.compressed_accounts[0].clone(), + ); let recipient_pubkey = Keypair::new().pubkey(); transfer_compressed_sol_test( &mut context, @@ -1413,8 +1439,13 @@ async fn test_with_address() { ]; for (n_input_compressed_accounts, n_new_addresses) in test_inputs { let compressed_input_accounts = test_indexer - .get_compressed_accounts_by_owner(&payer_pubkey)[0..n_input_compressed_accounts] + .get_compressed_accounts_with_merkle_context_by_owner(&payer_pubkey) + [0..n_input_compressed_accounts] .to_vec(); + let compressed_input_accounts = compressed_input_accounts + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>(); let mut address_vec = Vec::new(); // creates multiple seeds by taking the number of input accounts and zeroing out the jth byte for j in 0..n_new_addresses { @@ -1559,8 +1590,10 @@ async fn test_with_compression() { &mut context, ) .await; - let input_compressed_accounts = - vec![compressed_account_with_context.clone().compressed_account]; + let proof = sdk_to_program_compressed_proof(proof_rpc_res.proof.clone()); + let input_compressed_accounts = vec![sdk_to_program_compressed_account( + compressed_account_with_context.clone().compressed_account, + )]; let recipient_pubkey = Keypair::new().pubkey(); let output_compressed_accounts = vec![CompressedAccount { lamports: 0, @@ -1583,7 +1616,7 @@ async fn test_with_compression() { &[merkle_tree_pubkey], &proof_rpc_res.root_indices, &Vec::new(), - Some(proof_rpc_res.proof.clone()), + Some(proof.clone()), Some(compress_amount), true, Some(recipient), @@ -1603,7 +1636,10 @@ async fn test_with_compression() { .unwrap(); let compressed_account_with_context = - test_indexer.get_compressed_accounts_by_owner(&payer_pubkey)[0].clone(); + test_indexer.get_compressed_accounts_with_merkle_context_by_owner(&payer_pubkey)[0].clone(); + let compressed_account_with_context = + sdk_to_program_compressed_account_with_merkle_context(compressed_account_with_context); + decompress_sol_test( &mut context, &mut test_indexer, @@ -1892,7 +1928,11 @@ async fn batch_invoke_test() { assert!(proof_rpc_result.proof.is_none()); // No root index since value is in output queue assert!(proof_rpc_result.root_indices[0].is_none()); - let input_compressed_accounts = vec![compressed_account_with_context.compressed_account]; + + let input_compressed_accounts = vec![sdk_to_program_compressed_account( + compressed_account_with_context.compressed_account, + )]; + let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, @@ -1983,7 +2023,7 @@ async fn batch_invoke_test() { // 6. Should fail: invalid leaf index { let input_compressed_account = test_indexer - .get_compressed_accounts_by_owner(&payer_pubkey) + .get_compressed_accounts_with_merkle_context_by_owner(&payer_pubkey) .iter() .filter(|x| x.merkle_context.nullifier_queue_pubkey == output_queue_pubkey) .last() @@ -1998,7 +2038,9 @@ async fn batch_invoke_test() { let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, - &[input_compressed_account.compressed_account], + &[sdk_to_program_compressed_account( + input_compressed_account.compressed_account, + )], &output_compressed_accounts, &[MerkleContext { merkle_tree_pubkey, @@ -2085,10 +2127,27 @@ async fn batch_invoke_test() { &mut context, ) .await; + + let mut proof = None; + if let Some(proof_rpc) = proof_rpc_result.proof { + proof = Some(sdk_to_program_compressed_proof(proof_rpc)); + } + let input_compressed_accounts = vec![ compressed_account_with_context_1.compressed_account, compressed_account_with_context_2.compressed_account, - ]; + ] + .iter() + .map(|x| sdk_to_program_compressed_account(x.clone())) + .collect::>(); + + let merkle_context = vec![ + compressed_account_with_context_1.merkle_context, + compressed_account_with_context_2.merkle_context, + ] + .iter() + .map(|x| sdk_to_program_merkle_context(x.clone())) + .collect::>(); let output_compressed_accounts = vec![ CompressedAccount { lamports: 0, @@ -2108,16 +2167,16 @@ async fn batch_invoke_test() { let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, - &input_compressed_accounts, + input_compressed_accounts.as_slice(), &output_compressed_accounts, - &[merkle_context_1, merkle_context_2], + merkle_context.as_slice(), &[ merkle_context_1.nullifier_queue_pubkey, // output queue merkle_context_2.merkle_tree_pubkey, ], &proof_rpc_result.root_indices, &Vec::new(), - proof_rpc_result.proof, + proof, None, false, None, @@ -2163,7 +2222,9 @@ async fn batch_invoke_test() { &mut test_indexer, &payer, TestMode::ByZkpThenIndex, - compressed_account_with_context_1.clone(), + sdk_to_program_compressed_account_with_merkle_context( + compressed_account_with_context_1.clone(), + ), ) .await; assert_rpc_error( @@ -2194,7 +2255,9 @@ async fn batch_invoke_test() { &mut test_indexer, &payer, TestMode::ByIndexThenZkp, - compressed_account_with_context_1.clone(), + sdk_to_program_compressed_account_with_merkle_context( + compressed_account_with_context_1.clone(), + ), ) .await; assert_rpc_error( @@ -2224,7 +2287,9 @@ async fn batch_invoke_test() { &mut test_indexer, &payer, TestMode::ByIndexThenIndex, - compressed_account_with_context_1.clone(), + sdk_to_program_compressed_account_with_merkle_context( + compressed_account_with_context_1.clone(), + ), ) .await; assert_rpc_error( @@ -2254,7 +2319,9 @@ async fn batch_invoke_test() { &mut test_indexer, &payer, TestMode::ByZkpThenZkp, - compressed_account_with_context_1.clone(), + sdk_to_program_compressed_account_with_merkle_context( + compressed_account_with_context_1.clone(), + ), ) .await; assert_rpc_error( @@ -2311,18 +2378,26 @@ async fn batch_invoke_test() { &mut context, ) .await; - let mut merkle_context = compressed_account_with_context_1.merkle_context; + let mut merkle_context = + sdk_to_program_merkle_context(compressed_account_with_context_1.merkle_context); merkle_context.queue_index = Some(QueueIndex::default()); + let mut proof = None; + if let Some(proof_rpc) = proof_rpc_result.proof { + proof = Some(sdk_to_program_compressed_proof(proof_rpc)); + } + let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, - &[compressed_account_with_context_1.compressed_account], + &[sdk_to_program_compressed_account( + compressed_account_with_context_1.compressed_account, + )], &output_compressed_accounts, &[merkle_context], &[merkle_context.nullifier_queue_pubkey], &[None], &Vec::new(), - proof_rpc_result.proof, + proof, None, false, None, @@ -2355,13 +2430,13 @@ async fn batch_invoke_test() { .clone(); let mut merkle_context = compressed_account_with_context_1.merkle_context; - merkle_context.queue_index = Some(QueueIndex::default()); + merkle_context.queue_index = Some(SdkQueueIndex::default()); let instruction = create_invoke_instruction( &payer_pubkey, &payer_pubkey, &input_compressed_accounts, &output_compressed_accounts, - &[merkle_context], + &[sdk_to_program_merkle_context(merkle_context)], &[merkle_context.merkle_tree_pubkey], &[None], &Vec::new(), @@ -2388,9 +2463,12 @@ pub enum TestMode { ByZkpThenZkp, } -pub async fn double_spend_compressed_account( - context: &mut ProgramTestRpcConnection, - test_indexer: &mut TestIndexer, +pub async fn double_spend_compressed_account< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + context: &mut R, + test_indexer: &mut I, payer: &Keypair, mode: TestMode, compressed_account_with_context_1: CompressedAccountWithMerkleContext, @@ -2408,6 +2486,7 @@ pub async fn double_spend_compressed_account( context, ) .await; + let proof = Some(sdk_to_program_compressed_proof(proof_rpc_result.proof)); let input_compressed_accounts = vec![compressed_account_with_context_1.compressed_account]; let output_compressed_accounts = vec![CompressedAccount { lamports: 0, @@ -2425,7 +2504,7 @@ pub async fn double_spend_compressed_account( &[merkle_context_1.nullifier_queue_pubkey], &proof_rpc_result.root_indices, &Vec::new(), - Some(proof_rpc_result.proof), + proof, None, false, None, @@ -2569,6 +2648,10 @@ pub async fn create_output_accounts( let slot: u64 = context.get_slot().await.unwrap(); let (created_compressed_accounts, _) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let created_compressed_accounts = created_compressed_accounts + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>(); assert_created_compressed_accounts( output_compressed_accounts.as_slice(), output_merkle_tree_pubkeys.as_slice(), diff --git a/program-tests/utils/Cargo.toml b/program-tests/utils/Cargo.toml index 6fdf366af1..27c871923f 100644 --- a/program-tests/utils/Cargo.toml +++ b/program-tests/utils/Cargo.toml @@ -40,6 +40,7 @@ light-verifier = { workspace = true } light-utils = { workspace = true } light-program-test = { workspace = true } forester-utils = { workspace = true } +light-sdk = { workspace = true } memoffset = "0.9.1" rand = "0.8" photon-api = { workspace = true } diff --git a/program-tests/utils/src/assert_address_merkle_tree.rs b/program-tests/utils/src/assert_address_merkle_tree.rs deleted file mode 100644 index cb50eea57f..0000000000 --- a/program-tests/utils/src/assert_address_merkle_tree.rs +++ /dev/null @@ -1,132 +0,0 @@ -use forester_utils::{get_indexed_merkle_tree, AccountZeroCopy}; -use light_client::rpc::RpcConnection; -use light_hasher::Poseidon; -use light_merkle_tree_metadata::access::AccessMetadata; -use solana_sdk::pubkey::Pubkey; - -#[allow(clippy::too_many_arguments)] -pub async fn assert_address_merkle_tree_initialized( - rpc: &mut R, - merkle_tree_pubkey: &Pubkey, - queue_pubkey: &Pubkey, - merkle_tree_config: &account_compression::AddressMerkleTreeConfig, - index: u64, - program_owner: Option, - forester: Option, - expected_changelog_length: usize, - expected_roots_length: usize, - expected_next_index: usize, - expected_rightmost_leaf: &[u8; 32], - owner_pubkey: &Pubkey, - expected_indexed_changelog_length: usize, -) { - let merkle_tree = AccountZeroCopy::::new( - rpc, - *merkle_tree_pubkey, - ) - .await; - let merkle_tree_account = merkle_tree.deserialized(); - - assert_eq!( - merkle_tree_account - .metadata - .rollover_metadata - .rollover_threshold, - merkle_tree_config.rollover_threshold.unwrap_or(u64::MAX) - ); - assert_eq!( - merkle_tree_account.metadata.rollover_metadata.network_fee, - merkle_tree_config.network_fee.unwrap_or_default() - ); - - // The address Merkle tree is never directly called by the user. - // The whole rollover fees are collected by the address queue. - let expected_rollover_fee = 0; - assert_eq!( - merkle_tree_account.metadata.rollover_metadata.rollover_fee, - expected_rollover_fee - ); - - assert_eq!(merkle_tree_account.metadata.rollover_metadata.index, index); - assert_eq!( - merkle_tree_account - .metadata - .rollover_metadata - .rolledover_slot, - u64::MAX - ); - - assert_eq!( - merkle_tree_account - .metadata - .rollover_metadata - .close_threshold, - merkle_tree_config.close_threshold.unwrap_or(u64::MAX) - ); - - assert_eq!( - merkle_tree_account.metadata.next_merkle_tree, - Pubkey::default().into() - ); - let expected_access_meta_data = AccessMetadata { - owner: (*owner_pubkey).into(), - program_owner: program_owner.unwrap_or_default().into(), - forester: forester.unwrap_or_default().into(), - }; - assert_eq!( - merkle_tree_account.metadata.access_metadata, - expected_access_meta_data - ); - assert_eq!( - merkle_tree_account.metadata.associated_queue, - (*queue_pubkey).into() - ); - - let merkle_tree = get_indexed_merkle_tree::< - account_compression::AddressMerkleTreeAccount, - R, - Poseidon, - usize, - 26, - 16, - >(rpc, *merkle_tree_pubkey) - .await; - - assert_eq!(merkle_tree.height, merkle_tree_config.height as usize); - assert_eq!( - merkle_tree.merkle_tree.changelog.capacity(), - merkle_tree_config.changelog_size as usize - ); - assert_eq!( - merkle_tree.merkle_tree.changelog.len(), - expected_changelog_length - ); - assert_eq!( - merkle_tree.merkle_tree.changelog_index(), - expected_changelog_length.saturating_sub(1) - ); - assert_eq!( - merkle_tree.roots.capacity(), - merkle_tree_config.roots_size as usize - ); - assert_eq!(merkle_tree.roots.len(), expected_roots_length); - assert_eq!( - merkle_tree.root_index(), - expected_roots_length.saturating_sub(1) - ); - assert_eq!( - merkle_tree.canopy_depth, - merkle_tree_config.canopy_depth as usize - ); - assert_eq!(merkle_tree.next_index(), expected_next_index); - assert_eq!( - merkle_tree.sequence_number() % merkle_tree_config.roots_size as usize, - expected_roots_length.saturating_sub(1) - ); - assert_eq!(&merkle_tree.rightmost_leaf(), expected_rightmost_leaf); - // TODO: complete asserts - assert_eq!( - merkle_tree.indexed_changelog_index(), - expected_indexed_changelog_length.saturating_sub(1) - ); -} diff --git a/program-tests/utils/src/assert_compressed_tx.rs b/program-tests/utils/src/assert_compressed_tx.rs index 394e7ed249..f13c5c49b6 100644 --- a/program-tests/utils/src/assert_compressed_tx.rs +++ b/program-tests/utils/src/assert_compressed_tx.rs @@ -1,15 +1,15 @@ use account_compression::{state::QueueAccount, StateMerkleTreeAccount}; use anchor_lang::Discriminator; -use forester_utils::{ - get_concurrent_merkle_tree, get_hash_set, - indexer::{Indexer, StateMerkleTreeAccounts}, - AccountZeroCopy, -}; +use forester_utils::{get_concurrent_merkle_tree, get_hash_set, AccountZeroCopy}; use light_batched_merkle_tree::{ merkle_tree::BatchedMerkleTreeAccount, queue::BatchedQueueMetadata, }; -use light_client::rpc::RpcConnection; +use light_client::{ + indexer::{Indexer, StateMerkleTreeAccounts}, + rpc::RpcConnection, +}; use light_hasher::{Discriminator as LightDiscriminator, Poseidon}; +use light_program_test::indexer::TestIndexerExtensions; use light_system_program::sdk::{ compressed_account::{CompressedAccount, CompressedAccountWithMerkleContext}, event::{MerkleTreeSequenceNumber, PublicTransactionEvent}, @@ -19,7 +19,11 @@ use num_bigint::BigUint; use num_traits::FromBytes; use solana_sdk::{account::ReadableAccount, pubkey::Pubkey}; -pub struct AssertCompressedTransactionInputs<'a, R: RpcConnection, I: Indexer> { +pub struct AssertCompressedTransactionInputs< + 'a, + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +> { pub rpc: &'a mut R, pub test_indexer: &'a mut I, pub output_compressed_accounts: &'a [CompressedAccount], @@ -47,7 +51,10 @@ pub struct AssertCompressedTransactionInputs<'a, R: RpcConnection, I: Indexer /// 5. Merkle tree was updated correctly /// 6. TODO: Fees have been paid (after fee refactor) /// 7. Check compression amount was transferred -pub async fn assert_compressed_transaction>( +pub async fn assert_compressed_transaction< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( input: AssertCompressedTransactionInputs<'_, R, I>, ) { // CHECK 1 @@ -315,7 +322,10 @@ pub struct MerkleTreeTestSnapShot { /// Asserts: /// 1. The root has been updated /// 2. The next index has been updated -pub async fn assert_merkle_tree_after_tx>( +pub async fn assert_merkle_tree_after_tx< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, snapshots: &[MerkleTreeTestSnapShot], test_indexer: &mut I, diff --git a/program-tests/utils/src/assert_token_tx.rs b/program-tests/utils/src/assert_token_tx.rs index 82846ca421..69c6596ff4 100644 --- a/program-tests/utils/src/assert_token_tx.rs +++ b/program-tests/utils/src/assert_token_tx.rs @@ -1,7 +1,8 @@ use anchor_lang::AnchorSerialize; -use forester_utils::indexer::{Indexer, TokenDataWithContext}; -use light_client::rpc::RpcConnection; +use light_client::{indexer::Indexer, rpc::RpcConnection}; use light_compressed_token::process_transfer::{get_cpi_authority_pda, TokenTransferOutputData}; +use light_program_test::indexer::TestIndexerExtensions; +use light_sdk::token::TokenDataWithMerkleContext; use light_system_program::sdk::{ compressed_account::CompressedAccountWithMerkleContext, event::PublicTransactionEvent, }; @@ -21,7 +22,7 @@ use crate::assert_compressed_tx::{ /// 6. Check compression amount was transferred (outside of this function) /// No addresses in token transactions #[allow(clippy::too_many_arguments)] -pub async fn assert_transfer>( +pub async fn assert_transfer + TestIndexerExtensions>( context: &mut R, test_indexer: &mut I, out_compressed_accounts: &[TokenTransferOutputData], @@ -78,7 +79,10 @@ pub async fn assert_transfer>( ); } -pub fn assert_compressed_token_accounts>( +pub fn assert_compressed_token_accounts< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( test_indexer: &mut I, out_compressed_accounts: &[TokenTransferOutputData], lamports: Option>>, @@ -188,14 +192,14 @@ pub fn assert_compressed_token_accounts>( } #[allow(clippy::too_many_arguments)] -pub async fn assert_mint_to<'a, R: RpcConnection, I: Indexer>( +pub async fn assert_mint_to<'a, R: RpcConnection, I: Indexer + TestIndexerExtensions>( rpc: &mut R, test_indexer: &'a mut I, recipients: &[Pubkey], mint: Pubkey, amounts: &[u64], snapshots: &[MerkleTreeTestSnapShot], - created_token_accounts: &[TokenDataWithContext], + created_token_accounts: &[TokenDataWithMerkleContext], previous_mint_supply: u64, previous_sol_pool_amount: u64, token_pool_pda: Pubkey, diff --git a/program-tests/utils/src/conversions.rs b/program-tests/utils/src/conversions.rs new file mode 100644 index 0000000000..97ee92a77a --- /dev/null +++ b/program-tests/utils/src/conversions.rs @@ -0,0 +1,247 @@ +use light_compressed_token::{ + token_data::AccountState as ProgramAccountState, TokenData as ProgramTokenData, +}; +use light_sdk::{self as sdk, proof::CompressedProof}; +use light_system_program::{ + invoke::{ + processor::CompressedProof as ProgramCompressedProof, + OutputCompressedAccountWithPackedContext as ProgramOutputCompressedAccountWithPackedContext, + }, + sdk::{ + compressed_account::{ + CompressedAccount as ProgramCompressedAccount, + CompressedAccountData as ProgramCompressedAccountData, + CompressedAccountWithMerkleContext as ProgramCompressedAccountWithMerkleContext, + MerkleContext as ProgramMerkleContext, QueueIndex as ProgramQueueIndex, + }, + event::{ + MerkleTreeSequenceNumber as ProgramMerkleTreeSequenceNumber, + PublicTransactionEvent as ProgramPublicTransactionEvent, + }, + }, +}; + +pub fn sdk_to_program_queue_index( + sdk_queue_index: sdk::merkle_context::QueueIndex, +) -> ProgramQueueIndex { + ProgramQueueIndex { + queue_id: sdk_queue_index.queue_id, + index: sdk_queue_index.index, + } +} + +pub fn program_to_sdk_queue_index( + program_queue_index: ProgramQueueIndex, +) -> sdk::merkle_context::QueueIndex { + sdk::merkle_context::QueueIndex { + queue_id: program_queue_index.queue_id, + index: program_queue_index.index, + } +} + +pub fn sdk_to_program_merkle_context( + sdk_merkle_context: sdk::merkle_context::MerkleContext, +) -> ProgramMerkleContext { + ProgramMerkleContext { + merkle_tree_pubkey: sdk_merkle_context.merkle_tree_pubkey, + nullifier_queue_pubkey: sdk_merkle_context.nullifier_queue_pubkey, + leaf_index: sdk_merkle_context.leaf_index, + queue_index: sdk_merkle_context + .queue_index + .map(sdk_to_program_queue_index), + } +} + +pub fn program_to_sdk_merkle_context( + program_merkle_context: ProgramMerkleContext, +) -> sdk::merkle_context::MerkleContext { + sdk::merkle_context::MerkleContext { + merkle_tree_pubkey: program_merkle_context.merkle_tree_pubkey, + nullifier_queue_pubkey: program_merkle_context.nullifier_queue_pubkey, + leaf_index: program_merkle_context.leaf_index, + queue_index: program_merkle_context + .queue_index + .map(program_to_sdk_queue_index), + } +} +pub fn sdk_to_program_compressed_account_data( + sdk_data: sdk::compressed_account::CompressedAccountData, +) -> ProgramCompressedAccountData { + ProgramCompressedAccountData { + discriminator: sdk_data.discriminator, + data: sdk_data.data, + data_hash: sdk_data.data_hash, + } +} + +pub fn program_to_sdk_compressed_account_data( + program_data: ProgramCompressedAccountData, +) -> sdk::compressed_account::CompressedAccountData { + sdk::compressed_account::CompressedAccountData { + discriminator: program_data.discriminator, + data: program_data.data, + data_hash: program_data.data_hash, + } +} + +pub fn sdk_to_program_compressed_account( + sdk_account: sdk::compressed_account::CompressedAccount, +) -> ProgramCompressedAccount { + ProgramCompressedAccount { + owner: sdk_account.owner, + lamports: sdk_account.lamports, + address: sdk_account.address, + data: sdk_account.data.map(sdk_to_program_compressed_account_data), + } +} + +pub fn program_to_sdk_compressed_account( + program_account: ProgramCompressedAccount, +) -> sdk::compressed_account::CompressedAccount { + sdk::compressed_account::CompressedAccount { + owner: program_account.owner, + lamports: program_account.lamports, + address: program_account.address, + data: program_account + .data + .map(program_to_sdk_compressed_account_data), + } +} + +pub fn sdk_to_program_compressed_account_with_merkle_context( + sdk_account: sdk::compressed_account::CompressedAccountWithMerkleContext, +) -> ProgramCompressedAccountWithMerkleContext { + ProgramCompressedAccountWithMerkleContext { + compressed_account: sdk_to_program_compressed_account(sdk_account.compressed_account), + merkle_context: sdk_to_program_merkle_context(sdk_account.merkle_context), + } +} + +pub fn program_to_sdk_compressed_account_with_merkle_context( + program_account: ProgramCompressedAccountWithMerkleContext, +) -> sdk::compressed_account::CompressedAccountWithMerkleContext { + sdk::compressed_account::CompressedAccountWithMerkleContext { + compressed_account: program_to_sdk_compressed_account(program_account.compressed_account), + merkle_context: program_to_sdk_merkle_context(program_account.merkle_context), + } +} + +pub fn sdk_to_program_account_state(sdk_state: sdk::token::AccountState) -> ProgramAccountState { + match sdk_state { + sdk::token::AccountState::Initialized => ProgramAccountState::Initialized, + sdk::token::AccountState::Frozen => ProgramAccountState::Frozen, + } +} + +pub fn program_to_sdk_account_state( + program_state: ProgramAccountState, +) -> sdk::token::AccountState { + match program_state { + ProgramAccountState::Initialized => sdk::token::AccountState::Initialized, + ProgramAccountState::Frozen => sdk::token::AccountState::Frozen, + } +} + +pub fn sdk_to_program_token_data(sdk_token: sdk::token::TokenData) -> ProgramTokenData { + ProgramTokenData { + mint: sdk_token.mint, + owner: sdk_token.owner, + amount: sdk_token.amount, + delegate: sdk_token.delegate, + state: sdk_to_program_account_state(sdk_token.state), + tlv: sdk_token.tlv, + } +} + +pub fn program_to_sdk_token_data(program_token: ProgramTokenData) -> sdk::token::TokenData { + sdk::token::TokenData { + mint: program_token.mint, + owner: program_token.owner, + amount: program_token.amount, + delegate: program_token.delegate, + state: program_to_sdk_account_state(program_token.state), + tlv: program_token.tlv, + } +} + +pub fn program_to_sdk_compressed_proof(program_proof: ProgramCompressedProof) -> CompressedProof { + CompressedProof { + a: program_proof.a, + b: program_proof.b, + c: program_proof.c, + } +} + +pub fn sdk_to_program_compressed_proof(sdk_proof: CompressedProof) -> ProgramCompressedProof { + ProgramCompressedProof { + a: sdk_proof.a, + b: sdk_proof.b, + c: sdk_proof.c, + } +} + +pub fn sdk_to_program_public_transaction_event( + event: sdk::event::PublicTransactionEvent, +) -> ProgramPublicTransactionEvent { + ProgramPublicTransactionEvent { + input_compressed_account_hashes: event.input_compressed_account_hashes, + output_compressed_account_hashes: event.output_compressed_account_hashes, + output_compressed_accounts: event + .output_compressed_accounts + .into_iter() + .map(|account| ProgramOutputCompressedAccountWithPackedContext { + compressed_account: sdk_to_program_compressed_account(account.compressed_account), + merkle_tree_index: account.merkle_tree_index, + }) + .collect(), + output_leaf_indices: event.output_leaf_indices, + sequence_numbers: event + .sequence_numbers + .into_iter() + .map(|sequence_number| ProgramMerkleTreeSequenceNumber { + pubkey: sequence_number.pubkey, + seq: sequence_number.seq, + }) + .collect(), + relay_fee: event.relay_fee, + is_compress: event.is_compress, + compress_or_decompress_lamports: event.compress_or_decompress_lamports, + pubkey_array: event.pubkey_array, + message: event.message, + } +} + +pub fn program_to_sdk_public_transaction_event( + event: ProgramPublicTransactionEvent, +) -> sdk::event::PublicTransactionEvent { + sdk::event::PublicTransactionEvent { + input_compressed_account_hashes: event.input_compressed_account_hashes, + output_compressed_account_hashes: event.output_compressed_account_hashes, + output_compressed_accounts: event + .output_compressed_accounts + .into_iter() + .map( + |account| sdk::compressed_account::OutputCompressedAccountWithPackedContext { + compressed_account: program_to_sdk_compressed_account( + account.compressed_account, + ), + merkle_tree_index: account.merkle_tree_index, + }, + ) + .collect(), + output_leaf_indices: event.output_leaf_indices, + sequence_numbers: event + .sequence_numbers + .into_iter() + .map(|sequence_number| sdk::event::MerkleTreeSequenceNumber { + pubkey: sequence_number.pubkey, + seq: sequence_number.seq, + }) + .collect(), + relay_fee: event.relay_fee, + is_compress: event.is_compress, + compress_or_decompress_lamports: event.compress_or_decompress_lamports, + pubkey_array: event.pubkey_array, + message: event.message, + } +} diff --git a/program-tests/utils/src/create_address_test_program_sdk.rs b/program-tests/utils/src/create_address_test_program_sdk.rs index b9b962a511..e69159e650 100644 --- a/program-tests/utils/src/create_address_test_program_sdk.rs +++ b/program-tests/utils/src/create_address_test_program_sdk.rs @@ -2,9 +2,12 @@ use std::collections::HashMap; use account_compression::utils::constants::CPI_AUTHORITY_PDA_SEED; use anchor_lang::{InstructionData, ToAccountMetas}; -use light_client::rpc::{RpcConnection, RpcError}; +use light_client::{ + indexer::Indexer, + rpc::{RpcConnection, RpcError}, +}; use light_compressed_token::process_transfer::transfer_sdk::to_account_metas; -use light_program_test::test_env::EnvAccounts; +use light_program_test::{indexer::TestIndexerExtensions, test_env::EnvAccounts}; use light_system_program::{ invoke::processor::CompressedProof, sdk::address::{derive_address, pack_new_address_params}, @@ -12,7 +15,7 @@ use light_system_program::{ }; use solana_sdk::{instruction::Instruction, pubkey::Pubkey, signature::Keypair, signer::Signer}; -use crate::{indexer::TestIndexer, Indexer}; +use crate::conversions::sdk_to_program_compressed_proof; #[derive(Debug, Clone)] pub struct CreateCompressedPdaInstructionInputs<'a> { @@ -68,8 +71,11 @@ pub fn create_pda_instruction(input_params: CreateCompressedPdaInstructionInputs } } -pub async fn perform_create_pda_with_event_rnd( - test_indexer: &mut TestIndexer, +pub async fn perform_create_pda_with_event_rnd< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + test_indexer: &mut I, rpc: &mut R, env: &EnvAccounts, payer: &Keypair, @@ -78,8 +84,11 @@ pub async fn perform_create_pda_with_event_rnd( let data = rand::random(); perform_create_pda_with_event(test_indexer, rpc, env, payer, seed, &data).await } -pub async fn perform_create_pda_with_event( - test_indexer: &mut TestIndexer, +pub async fn perform_create_pda_with_event< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( + test_indexer: &mut I, rpc: &mut R, env: &EnvAccounts, payer: &Keypair, @@ -126,7 +135,7 @@ pub async fn perform_create_pda_with_event( data: *data, signer: &payer.pubkey(), output_compressed_account_merkle_tree_pubkey: &env.merkle_tree_pubkey, - proof: &rpc_result.proof, + proof: &sdk_to_program_compressed_proof(rpc_result.proof), new_address_params, registered_program_pda: &env.registered_program_pda, diff --git a/program-tests/utils/src/e2e_test_env.rs b/program-tests/utils/src/e2e_test_env.rs index c361fcc399..49ec706f3a 100644 --- a/program-tests/utils/src/e2e_test_env.rs +++ b/program-tests/utils/src/e2e_test_env.rs @@ -72,10 +72,6 @@ use forester_utils::{ address_merkle_tree_config::{address_tree_ready_for_rollover, state_tree_ready_for_rollover}, airdrop_lamports, forester_epoch::{Epoch, Forester, TreeAccounts, TreeType}, - indexer::{ - AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, StateMerkleTreeAccounts, - StateMerkleTreeBundle, TokenDataWithContext, - }, registry::register_test_forester, AccountZeroCopy, }; @@ -84,19 +80,23 @@ use light_batched_merkle_tree::{ queue::BatchedQueueAccount, }; use light_client::{ - rpc::{errors::RpcError, RpcConnection}, + indexer::{ + AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, StateMerkleTreeAccounts, + StateMerkleTreeBundle, + }, + rpc::{errors::RpcError, merkle_tree::MerkleTreeExt, RpcConnection}, transaction_params::{FeeConfig, TransactionParams}, }; // TODO: implement traits for context object and indexer that we can implement with an rpc as well // context trait: send_transaction -> return transaction result, get_account_info -> return account info // indexer trait: get_compressed_accounts_by_owner -> return compressed accounts, // refactor all tests to work with that so that we can run all tests with a test validator and concurrency -use light_compressed_token::token_data::AccountState; use light_hasher::Poseidon; use light_indexed_merkle_tree::{ array::IndexedArray, reference::IndexedMerkleTree, HIGHEST_ADDRESS_PLUS_ONE, }; use light_program_test::{ + indexer::{TestIndexer, TestIndexerExtensions}, test_batch_forester::{perform_batch_append, perform_batch_nullify}, test_env::{create_state_merkle_tree_and_queue_account, EnvAccounts}, test_rpc::ProgramTestRpcConnection, @@ -108,6 +108,7 @@ use light_registry::{ utils::get_protocol_config_pda_address, ForesterConfig, }; +use light_sdk::token::{AccountState, TokenDataWithMerkleContext}; use light_system_program::sdk::compressed_account::CompressedAccountWithMerkleContext; use light_utils::{bigint::bigint_to_be_bytes_array, rand::gen_prime}; use log::info; @@ -135,8 +136,8 @@ use crate::{ assert_epoch::{ assert_finalized_epoch_registration, assert_report_work, fetch_epoch_and_forester_pdas, }, + conversions::sdk_to_program_compressed_account_with_merkle_context, create_address_merkle_tree_and_queue_account_with_assert, - indexer::TestIndexer, spl::{ approve_test, burn_test, compress_test, compressed_transfer_test, create_mint_helper, create_token_account, decompress_test, freeze_test, mint_tokens_helper, revoke_test, @@ -212,7 +213,7 @@ impl Stats { println!("Finalized registrations {}", self.finalized_registrations); } } -pub async fn init_program_test_env( +pub async fn init_program_test_env( rpc: R, env_accounts: &EnvAccounts, skip_prover: bool, @@ -288,7 +289,7 @@ pub struct TestForester { is_registered: Option, } -pub struct E2ETestEnv> { +pub struct E2ETestEnv + TestIndexerExtensions> { pub payer: Keypair, pub governance_keypair: Keypair, pub indexer: I, @@ -311,7 +312,7 @@ pub struct E2ETestEnv> { pub registration_epoch: u64, } -impl> E2ETestEnv +impl + TestIndexerExtensions> E2ETestEnv where R: RpcConnection, I: Indexer, @@ -2208,7 +2209,12 @@ where ) -> Vec { let input_compressed_accounts = self .indexer - .get_compressed_accounts_by_owner(&self.users[user_index].keypair.pubkey()); + .get_compressed_accounts_with_merkle_context_by_owner( + &self.users[user_index].keypair.pubkey(), + ) + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>(); if input_compressed_accounts.is_empty() { return vec![]; } @@ -2245,7 +2251,11 @@ where &self, pubkey: &Pubkey, ) -> Vec { - self.indexer.get_compressed_accounts_by_owner(pubkey) + self.indexer + .get_compressed_accounts_with_merkle_context_by_owner(pubkey) + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect() } pub fn get_merkle_tree_pubkeys(&mut self, num: u64) -> Vec { @@ -2320,7 +2330,7 @@ where pub async fn select_random_compressed_token_accounts( &mut self, user: &Pubkey, - ) -> (Pubkey, Vec) { + ) -> (Pubkey, Vec) { self.select_random_compressed_token_accounts_delegated(user, false, None, false) .await } @@ -2328,7 +2338,7 @@ where pub async fn select_random_compressed_token_accounts_frozen( &mut self, user: &Pubkey, - ) -> (Pubkey, Vec) { + ) -> (Pubkey, Vec) { self.select_random_compressed_token_accounts_delegated(user, false, None, true) .await } @@ -2339,7 +2349,7 @@ where delegated: bool, delegate: Option, frozen: bool, - ) -> (Pubkey, Vec) { + ) -> (Pubkey, Vec) { let user_token_accounts = &mut self.indexer.get_compressed_token_accounts_by_owner(user); // clean up dust so that we don't run into issues that account balances are too low user_token_accounts.retain(|t| t.token_data.amount > 1000); @@ -2429,14 +2439,14 @@ where token_account.token_data.mint == mint && tree_version == version }) .map(|token_account| (*token_account).clone()) - .collect::>(); + .collect::>(); } if delegated { token_accounts_with_mint = token_accounts_with_mint .iter() .filter(|token_account| token_account.token_data.delegate.is_some()) .map(|token_account| (*token_account).clone()) - .collect::>(); + .collect::>(); if token_accounts_with_mint.is_empty() { return (mint, Vec::new()); } @@ -2446,14 +2456,14 @@ where .iter() .filter(|token_account| token_account.token_data.delegate.unwrap() == delegate) .map(|token_account| (*token_account).clone()) - .collect::>(); + .collect::>(); } if frozen { token_accounts_with_mint = token_accounts_with_mint .iter() .filter(|token_account| token_account.token_data.state == AccountState::Frozen) .map(|token_account| (*token_account).clone()) - .collect::>(); + .collect::>(); if token_accounts_with_mint.is_empty() { return (mint, Vec::new()); } @@ -2462,7 +2472,7 @@ where .iter() .filter(|token_account| token_account.token_data.state == AccountState::Initialized) .map(|token_account| (*token_account).clone()) - .collect::>(); + .collect::>(); } let range_end = if token_accounts_with_mint.len() == 1 { 1 diff --git a/program-tests/utils/src/indexer/mod.rs b/program-tests/utils/src/indexer/mod.rs deleted file mode 100644 index 4cb99d00ea..0000000000 --- a/program-tests/utils/src/indexer/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod test_indexer; -pub use test_indexer::TestIndexer; diff --git a/program-tests/utils/src/lib.rs b/program-tests/utils/src/lib.rs index e02de430ea..77620f619a 100644 --- a/program-tests/utils/src/lib.rs +++ b/program-tests/utils/src/lib.rs @@ -13,28 +13,27 @@ use solana_sdk::{ pub mod address; pub mod address_tree_rollover; -pub mod assert_address_merkle_tree; pub mod assert_compressed_tx; pub mod assert_epoch; pub mod assert_merkle_tree; pub mod assert_queue; pub mod assert_rollover; pub mod assert_token_tx; +pub mod conversions; pub mod create_address_test_program_sdk; pub mod e2e_test_env; #[allow(unused)] -pub mod indexer; pub mod spl; pub mod state_tree_rollover; pub mod system_program; #[allow(unused)] pub mod test_forester; + pub use create_address_test_program::ID as CREATE_ADDRESS_TEST_PROGRAM_ID; pub use forester_utils::{ airdrop_lamports, create_account_instruction, forester_epoch::{Epoch, TreeAccounts, TreeType}, get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree, - indexer::{AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, TokenDataWithContext}, registry::{ create_rollover_address_merkle_tree_instructions, create_rollover_state_merkle_tree_instructions, register_test_forester, @@ -49,13 +48,13 @@ pub use light_client::{ transaction_params::{FeeConfig, TransactionParams}, }; use light_hasher::Poseidon; -use light_program_test::test_env::create_address_merkle_tree_and_queue_account; +use light_program_test::{ + indexer::utils::assert_address_merkle_tree_initialized, + test_env::create_address_merkle_tree_and_queue_account, +}; use light_registry::account_compression_cpi::sdk::get_registered_program_pda; -use crate::{ - assert_address_merkle_tree::assert_address_merkle_tree_initialized, - assert_queue::assert_address_queue_initialized, -}; +use crate::assert_queue::assert_address_queue_initialized; #[allow(clippy::too_many_arguments)] #[inline(never)] diff --git a/program-tests/utils/src/spl.rs b/program-tests/utils/src/spl.rs index 9911f47b32..9109d0be2e 100644 --- a/program-tests/utils/src/spl.rs +++ b/program-tests/utils/src/spl.rs @@ -1,9 +1,7 @@ use anchor_spl::token::{Mint, TokenAccount}; -use forester_utils::{ - create_account_instruction, - indexer::{Indexer, TokenDataWithContext}, -}; +use forester_utils::create_account_instruction; use light_client::{ + indexer::Indexer, rpc::{errors::RpcError, RpcConnection}, transaction_params::TransactionParams, }; @@ -26,6 +24,8 @@ use light_compressed_token::{ TokenData, }; use light_hasher::Poseidon; +use light_program_test::indexer::TestIndexerExtensions; +use light_sdk::token::TokenDataWithMerkleContext; use light_system_program::{ invoke::processor::CompressedProof, sdk::{compressed_account::MerkleContext, event::PublicTransactionEvent}, @@ -42,9 +42,14 @@ use spl_token::instruction::initialize_mint; use crate::{ assert_compressed_tx::get_merkle_tree_snapshots, assert_token_tx::{assert_create_mint, assert_mint_to, assert_transfer}, + conversions::{ + program_to_sdk_public_transaction_event, program_to_sdk_token_data, + sdk_to_program_compressed_account, sdk_to_program_compressed_account_with_merkle_context, + sdk_to_program_compressed_proof, sdk_to_program_merkle_context, sdk_to_program_token_data, + }, }; -pub async fn mint_tokens_helper>( +pub async fn mint_tokens_helper + TestIndexerExtensions>( rpc: &mut R, test_indexer: &mut I, merkle_tree_pubkey: &Pubkey, @@ -105,7 +110,10 @@ pub async fn mint_spl_tokens( } #[allow(clippy::too_many_arguments)] -pub async fn mint_tokens_helper_with_lamports>( +pub async fn mint_tokens_helper_with_lamports< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, test_indexer: &mut I, merkle_tree_pubkey: &Pubkey, @@ -129,7 +137,10 @@ pub async fn mint_tokens_helper_with_lamports>( .await; } #[allow(clippy::too_many_arguments)] -pub async fn mint_tokens_22_helper_with_lamports>( +pub async fn mint_tokens_22_helper_with_lamports< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, test_indexer: &mut I, merkle_tree_pubkey: &Pubkey, @@ -156,7 +167,10 @@ pub async fn mint_tokens_22_helper_with_lamports } #[allow(clippy::too_many_arguments)] -pub async fn mint_tokens_22_helper_with_lamports_and_bump>( +pub async fn mint_tokens_22_helper_with_lamports_and_bump< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, test_indexer: &mut I, merkle_tree_pubkey: &Pubkey, @@ -206,7 +220,10 @@ pub async fn mint_tokens_22_helper_with_lamports_and_bump( } #[allow(clippy::too_many_arguments)] -pub async fn compressed_transfer_test>( +pub async fn compressed_transfer_test< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( payer: &Keypair, rpc: &mut R, test_indexer: &mut I, @@ -484,7 +504,7 @@ pub async fn compressed_transfer_test>( recipients: &[Pubkey], amounts: &[u64], lamports: Option>>, - input_compressed_accounts: &[TokenDataWithContext], + input_compressed_accounts: &[TokenDataWithMerkleContext], output_merkle_tree_pubkeys: &[Pubkey], delegate_change_account_index: Option, delegate_is_signer: bool, @@ -510,7 +530,10 @@ pub async fn compressed_transfer_test>( } #[allow(clippy::too_many_arguments)] -pub async fn compressed_transfer_22_test>( +pub async fn compressed_transfer_22_test< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( payer: &Keypair, rpc: &mut R, test_indexer: &mut I, @@ -519,7 +542,7 @@ pub async fn compressed_transfer_22_test>( recipients: &[Pubkey], amounts: &[u64], mut lamports: Option>>, - input_compressed_accounts: &[TokenDataWithContext], + input_compressed_accounts: &[TokenDataWithMerkleContext], output_merkle_tree_pubkeys: &[Pubkey], delegate_change_account_index: Option, delegate_is_signer: bool, @@ -616,18 +639,22 @@ pub async fn compressed_transfer_22_test>( }; let authority_signer = if delegate_is_signer { payer } else { from }; + let instruction = create_transfer_instruction( &payer.pubkey(), &authority_signer.pubkey(), // authority &input_merkle_tree_context, &output_compressed_accounts, &rpc_result.root_indices, - &rpc_result.proof, - &input_compressed_account_token_data, // input_token_data + &rpc_result.proof.map(sdk_to_program_compressed_proof), + &input_compressed_account_token_data + .into_iter() + .map(sdk_to_program_token_data) + .collect::>(), // input_token_data &input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), *mint, delegate_pubkey, // owner_if_delegate_change_account_index @@ -691,8 +718,11 @@ pub async fn compressed_transfer_22_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (created_change_output_account, created_token_output_accounts) = - test_indexer.add_event_and_compressed_accounts(slot, &event); + let (created_change_output_account, created_token_output_accounts) = test_indexer + .add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); let delegates = if let Some(index) = delegate_change_account_index { let mut delegates = vec![None; created_token_output_accounts.len()]; delegates[index as usize] = Some(payer.pubkey()); @@ -711,7 +741,11 @@ pub async fn compressed_transfer_22_test>( rpc, test_indexer, &output_compressed_accounts, - created_output_accounts.as_slice(), + created_output_accounts + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>() + .as_slice(), lamports, &input_compressed_account_hashes, &snapshots, @@ -723,11 +757,11 @@ pub async fn compressed_transfer_22_test>( } #[allow(clippy::too_many_arguments)] -pub async fn decompress_test>( +pub async fn decompress_test + TestIndexerExtensions>( payer: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, amount: u64, output_merkle_tree_pubkey: &Pubkey, recipient_token_account: &Pubkey, @@ -775,19 +809,22 @@ pub async fn decompress_test>( &input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect::>(), // input_compressed_account_merkle_tree_pubkeys &[change_out_compressed_account], // output_compressed_accounts &proof_rpc_result.root_indices, // root_indices - &proof_rpc_result.proof, + &Some(sdk_to_program_compressed_proof( + proof_rpc_result.proof.unwrap_or_default(), + )), input_compressed_accounts .iter() - .map(|x| x.token_data.clone()) + .map(|x| sdk_to_program_token_data(x.token_data.clone())) .collect::>() .as_slice(), // input_token_data &input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, // mint None, // owner_if_delegate_change_account_index @@ -858,7 +895,10 @@ pub async fn decompress_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); assert_transfer( rpc, test_indexer, @@ -866,6 +906,7 @@ pub async fn decompress_test>( created_output_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(sdk_to_program_compressed_account_with_merkle_context) .collect::>() .as_slice(), None, @@ -925,7 +966,10 @@ pub async fn decompress_test>( } #[allow(clippy::too_many_arguments)] -pub async fn perform_compress_spl_token_account>( +pub async fn perform_compress_spl_token_account< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, test_indexer: &mut I, payer: &Keypair, @@ -965,7 +1009,10 @@ pub async fn perform_compress_spl_token_account> .unwrap(); // TODO: replace with get_transaction_slot() this only works with Program test let slot = rpc.get_slot().await.unwrap(); - test_indexer.add_event_and_compressed_accounts(slot, &event); + test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); let created_compressed_token_account = test_indexer.get_compressed_token_accounts_by_owner(&token_owner.pubkey())[0].clone(); let expected_token_data = TokenData { @@ -978,7 +1025,7 @@ pub async fn perform_compress_spl_token_account> }; assert_eq!( created_compressed_token_account.token_data, - expected_token_data + program_to_sdk_token_data(expected_token_data) ); assert_eq!( created_compressed_token_account @@ -999,7 +1046,7 @@ pub async fn perform_compress_spl_token_account> } #[allow(clippy::too_many_arguments)] -pub async fn compress_test>( +pub async fn compress_test + TestIndexerExtensions>( payer: &Keypair, rpc: &mut R, test_indexer: &mut I, @@ -1067,7 +1114,10 @@ pub async fn compress_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); assert_transfer( rpc, @@ -1076,6 +1126,7 @@ pub async fn compress_test>( created_output_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(sdk_to_program_compressed_account_with_merkle_context) .collect::>() .as_slice(), None, @@ -1102,11 +1153,11 @@ pub async fn compress_test>( } #[allow(clippy::too_many_arguments)] -pub async fn approve_test>( +pub async fn approve_test + TestIndexerExtensions>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, delegated_amount: u64, delegate_lamports: Option, delegate: &Pubkey, @@ -1143,15 +1194,17 @@ pub async fn approve_test>( input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, delegated_amount, @@ -1160,7 +1213,7 @@ pub async fn approve_test>( change_compressed_account_merkle_tree: *change_compressed_account_merkle_tree, delegate: *delegate, root_indices: proof_rpc_result.root_indices, - proof: proof_rpc_result.proof.unwrap_or_default(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.unwrap_or_default()), }; let instruction = create_approve_instruction(inputs).unwrap(); @@ -1216,7 +1269,10 @@ pub async fn approve_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); let expected_delegated_token_data = TokenData { mint, @@ -1229,7 +1285,7 @@ pub async fn approve_test>( assert_eq!( expected_delegated_token_data, - created_output_accounts[0].token_data + sdk_to_program_token_data(created_output_accounts[0].token_data.clone()) ); let mut expected_token_data = vec![expected_delegated_token_data]; let mut delegates = vec![Some(*delegate)]; @@ -1244,7 +1300,7 @@ pub async fn approve_test>( }; assert_eq!( expected_change_token_data, - created_output_accounts[1].token_data + sdk_to_program_token_data(created_output_accounts[1].token_data.clone()) ); expected_token_data.push(expected_change_token_data); delegates.push(None); @@ -1260,6 +1316,7 @@ pub async fn approve_test>( created_output_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(sdk_to_program_compressed_account_with_merkle_context) .collect::>() .as_slice(), change_lamports, @@ -1273,11 +1330,11 @@ pub async fn approve_test>( } #[allow(clippy::too_many_arguments)] -pub async fn revoke_test>( +pub async fn revoke_test + TestIndexerExtensions>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, output_account_merkle_tree: &Pubkey, transaction_params: Option, ) { @@ -1305,20 +1362,22 @@ pub async fn revoke_test>( input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), mint, output_account_merkle_tree: *output_account_merkle_tree, root_indices: proof_rpc_result.root_indices, - proof: proof_rpc_result.proof.unwrap_or_default(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.unwrap_or_default()), }; let instruction = create_revoke_instruction(inputs).unwrap(); @@ -1343,7 +1402,10 @@ pub async fn revoke_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); let input_amount = input_compressed_accounts .iter() .map(|x| x.token_data.amount) @@ -1356,7 +1418,10 @@ pub async fn revoke_test>( state: AccountState::Initialized, tlv: None, }; - assert_eq!(expected_token_data, created_output_accounts[0].token_data); + assert_eq!( + expected_token_data, + sdk_to_program_token_data(created_output_accounts[0].token_data.clone()) + ); let expected_compressed_output_accounts = create_expected_token_output_data(vec![expected_token_data], &output_merkle_tree_pubkeys); let sum_inputs = input_compressed_accounts @@ -1375,6 +1440,7 @@ pub async fn revoke_test>( created_output_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(sdk_to_program_compressed_account_with_merkle_context) .collect::>() .as_slice(), change_lamports, @@ -1387,11 +1453,11 @@ pub async fn revoke_test>( .await; } -pub async fn freeze_test>( +pub async fn freeze_test + TestIndexerExtensions>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, outputs_merkle_tree: &Pubkey, transaction_params: Option, ) { @@ -1406,11 +1472,11 @@ pub async fn freeze_test>( .await; } -pub async fn thaw_test>( +pub async fn thaw_test + TestIndexerExtensions>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, outputs_merkle_tree: &Pubkey, transaction_params: Option, ) { @@ -1425,11 +1491,15 @@ pub async fn thaw_test>( .await; } -pub async fn freeze_or_thaw_test>( +pub async fn freeze_or_thaw_test< + R: RpcConnection, + const FREEZE: bool, + I: Indexer + TestIndexerExtensions, +>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, outputs_merkle_tree: &Pubkey, transaction_params: Option, ) { @@ -1457,19 +1527,21 @@ pub async fn freeze_or_thaw_test>(), outputs_merkle_tree: *outputs_merkle_tree, root_indices: proof_rpc_result.root_indices, - proof: proof_rpc_result.proof.unwrap_or_default(), + proof: sdk_to_program_compressed_proof(proof_rpc_result.proof.unwrap_or_default()), }; let instruction = create_instruction::(inputs).unwrap(); @@ -1495,7 +1567,10 @@ pub async fn freeze_or_thaw_test>() .as_slice(), change_lamports, @@ -1559,11 +1635,11 @@ pub async fn freeze_or_thaw_test>( +pub async fn burn_test + TestIndexerExtensions>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: Vec, + input_compressed_accounts: Vec, change_account_merkle_tree: &Pubkey, burn_amount: u64, signer_is_delegate: bool, @@ -1627,7 +1703,10 @@ pub async fn burn_test>( .unwrap() .unwrap(); let slot = rpc.get_slot().await.unwrap(); - let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts(slot, &event); + let (_, created_output_accounts) = test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.clone()), + ); let mut delegates = Vec::new(); let mut expected_output_accounts = Vec::new(); @@ -1670,6 +1749,7 @@ pub async fn burn_test>( created_output_accounts .iter() .map(|x| x.compressed_account.clone()) + .map(sdk_to_program_compressed_account_with_merkle_context) .collect::>() .as_slice(), change_lamports, @@ -1702,11 +1782,14 @@ pub enum BurnInstructionMode { } #[allow(clippy::too_many_arguments)] -pub async fn create_burn_test_instruction>( +pub async fn create_burn_test_instruction< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( authority: &Keypair, rpc: &mut R, test_indexer: &mut I, - input_compressed_accounts: &[TokenDataWithContext], + input_compressed_accounts: &[TokenDataWithMerkleContext], change_account_merkle_tree: &Pubkey, burn_amount: u64, signer_is_delegate: bool, @@ -1744,7 +1827,7 @@ pub async fn create_burn_test_instruction>( c: proof_rpc_result.proof.as_ref().unwrap().a, // flip c to make proof invalid but not run into decompress errors } } else { - proof_rpc_result.proof.unwrap_or_default() + sdk_to_program_compressed_proof(proof_rpc_result.proof.unwrap_or_default()) }; let inputs = CreateBurnInstructionInputs { fee_payer: rpc.get_payer().pubkey(), @@ -1752,15 +1835,17 @@ pub async fn create_burn_test_instruction>( input_merkle_contexts: input_compressed_accounts .iter() .map(|x| x.compressed_account.merkle_context) + .map(sdk_to_program_merkle_context) .collect(), input_token_data: input_compressed_accounts .iter() .map(|x| x.token_data.clone()) + .map(sdk_to_program_token_data) .collect(), input_compressed_accounts: input_compressed_accounts .iter() .map(|x| &x.compressed_account.compressed_account) - .cloned() + .map(|x| sdk_to_program_compressed_account(x.clone())) .collect::>(), change_account_merkle_tree: *change_account_merkle_tree, root_indices: proof_rpc_result.root_indices, diff --git a/program-tests/utils/src/system_program.rs b/program-tests/utils/src/system_program.rs index a0b8b0d3c4..c54fc7fca5 100644 --- a/program-tests/utils/src/system_program.rs +++ b/program-tests/utils/src/system_program.rs @@ -1,9 +1,10 @@ -use forester_utils::indexer::Indexer; use light_client::{ + indexer::Indexer, rpc::{errors::RpcError, RpcConnection}, transaction_params::TransactionParams, }; use light_hasher::Poseidon; +use light_program_test::indexer::TestIndexerExtensions; use light_system_program::{ sdk::{ address::derive_address_legacy, @@ -20,12 +21,18 @@ use solana_sdk::{ signature::{Keypair, Signature, Signer}, }; -use crate::assert_compressed_tx::{ - assert_compressed_transaction, get_merkle_tree_snapshots, AssertCompressedTransactionInputs, +use crate::{ + assert_compressed_tx::{ + assert_compressed_transaction, get_merkle_tree_snapshots, AssertCompressedTransactionInputs, + }, + conversions::{ + program_to_sdk_public_transaction_event, + sdk_to_program_compressed_account_with_merkle_context, sdk_to_program_compressed_proof, + }, }; #[allow(clippy::too_many_arguments)] -pub async fn create_addresses_test>( +pub async fn create_addresses_test + TestIndexerExtensions>( rpc: &mut R, test_indexer: &mut I, address_merkle_tree_pubkeys: &[Pubkey], @@ -104,7 +111,7 @@ pub async fn create_addresses_test>( } #[allow(clippy::too_many_arguments)] -pub async fn compress_sol_test>( +pub async fn compress_sol_test + TestIndexerExtensions>( rpc: &mut R, test_indexer: &mut I, authority: &Keypair, @@ -163,7 +170,7 @@ pub async fn compress_sol_test>( } #[allow(clippy::too_many_arguments)] -pub async fn decompress_sol_test>( +pub async fn decompress_sol_test + TestIndexerExtensions>( rpc: &mut R, test_indexer: &mut I, authority: &Keypair, @@ -207,7 +214,10 @@ pub async fn decompress_sol_test>( } #[allow(clippy::too_many_arguments)] -pub async fn transfer_compressed_sol_test>( +pub async fn transfer_compressed_sol_test< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( rpc: &mut R, test_indexer: &mut I, authority: &Keypair, @@ -292,7 +302,10 @@ pub struct CompressedTransactionTestInputs<'a, R: RpcConnection, I: Indexer> } #[allow(clippy::too_many_arguments)] -pub async fn compressed_transaction_test>( +pub async fn compressed_transaction_test< + R: RpcConnection, + I: Indexer + TestIndexerExtensions, +>( inputs: CompressedTransactionTestInputs<'_, R, I>, ) -> Result { let mut compressed_account_hashes = Vec::new(); @@ -324,9 +337,9 @@ pub async fn compressed_transaction_test>( Some(state_input_merkle_trees) }; let mut root_indices = Vec::new(); - let mut proof = None; let mut input_merkle_tree_snapshots = Vec::new(); let mut address_params = Vec::new(); + let mut proof = None; if !inputs.input_compressed_accounts.is_empty() || !inputs.new_address_params.is_empty() { let address_merkle_tree_pubkeys = if inputs.new_address_params.is_empty() { None @@ -351,7 +364,11 @@ pub async fn compressed_transaction_test>( .await; root_indices = proof_rpc_res.root_indices; - proof = proof_rpc_res.proof; + + if let Some(proof_rpc_res) = proof_rpc_res.proof { + proof = Some(sdk_to_program_compressed_proof(proof_rpc_res)); + } + let input_merkle_tree_accounts = inputs .test_indexer .get_state_merkle_tree_accounts(state_input_merkle_trees.unwrap_or(vec![]).as_slice()); @@ -430,9 +447,17 @@ pub async fn compressed_transaction_test>( .unwrap(); let slot = inputs.rpc.get_transaction_slot(&event.1).await.unwrap(); - let (created_output_compressed_accounts, _) = inputs - .test_indexer - .add_event_and_compressed_accounts(slot, &event.0); + let (created_output_compressed_accounts, _) = + inputs.test_indexer.add_event_and_compressed_accounts( + slot, + &program_to_sdk_public_transaction_event(event.0.clone()), + ); + + let created_output_compressed_accounts = created_output_compressed_accounts + .into_iter() + .map(sdk_to_program_compressed_account_with_merkle_context) + .collect::>(); + let input = AssertCompressedTransactionInputs { rpc: inputs.rpc, test_indexer: inputs.test_indexer, diff --git a/program-tests/utils/src/test_forester.rs b/program-tests/utils/src/test_forester.rs index 4ff9e92469..900dee4824 100644 --- a/program-tests/utils/src/test_forester.rs +++ b/program-tests/utils/src/test_forester.rs @@ -5,11 +5,11 @@ use account_compression::{ AddressMerkleTreeAccount, StateMerkleTreeAccount, ID, SAFETY_MARGIN, }; use anchor_lang::{system_program, InstructionData, ToAccountMetas}; -use forester_utils::{ - get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree, +use forester_utils::{get_concurrent_merkle_tree, get_hash_set, get_indexed_merkle_tree}; +use light_client::{ indexer::{AddressMerkleTreeBundle, StateMerkleTreeBundle}, + rpc::{errors::RpcError, RpcConnection}, }; -use light_client::rpc::{errors::RpcError, RpcConnection}; use light_concurrent_merkle_tree::event::MerkleTreeEvent; use light_hasher::Poseidon; use light_indexed_merkle_tree::copy::IndexedMerkleTreeCopy; diff --git a/programs/package.json b/programs/package.json index bd54cfd7fb..f74fd9f97a 100644 --- a/programs/package.json +++ b/programs/package.json @@ -6,14 +6,17 @@ "build": "anchor build", "build-system": "anchor build --program-name light_system_program -- --features idl-build custom-heap", "build-compressed-token": "anchor build --program-name light_compressed_token -- --features idl-build custom-heap", - "test": "pnpm test-account-compression && pnpm test-system && pnpm test-compressed-token && pnpm test-registry", - "test-account-compression": "cargo-test-sbf -p account-compression-test -- --test-threads=1", - "test-system": "cargo test-sbf -p system-test -- --test-threads=1", - "test-compressed-token": "cargo test-sbf -p compressed-token-test -- --test-threads=1", - "test-registry": "cargo-test-sbf -p registry-test -- --test-threads=1", - "token-escrow": "cargo-test-sbf -p token-escrow -- --test-threads=1 --features idl-build", - "program-owned-account-test": "cargo-test-sbf -p program-owned-account-test -- --test-threads=1", - "random-e2e-test": "RUST_MIN_STACK=8388608 cargo-test-sbf -p e2e-test -- --nocapture --test-threads=1" + "test": "RUSTFLAGS=\"-D warnings\" && pnpm test-account-compression && pnpm test-compressed-token && pnpm e2e-test && pnpm test-registry && pnpm sdk-test-program && pnpm test-system && pnpm test-system-cpi", + "test-account-compression": "cargo-test-sbf -p account-compression-test", + "test-compressed-token": "cargo test-sbf -p compressed-token-test", + "e2e-test": "cargo-test-sbf -p e2e-test", + "test-registry": "cargo-test-sbf -p registry-test", + "sdk-test-program": "cargo test-sbf -p sdk-test", + "test-system": "cargo test-sbf -p system-test", + "test-system-cpi": "cargo test-sbf -p system-cpi-test", + "ignored-token-escrow": "cargo-test-sbf -p token-escrow --features idl-build", + "ignored-program-owned-account-test": "cargo-test-sbf -p program-owned-account-test" + }, "nx": { "targets": { diff --git a/programs/system/src/invoke/verify_state_proof.rs b/programs/system/src/invoke/verify_state_proof.rs index da1dee7483..d20364fec2 100644 --- a/programs/system/src/invoke/verify_state_proof.rs +++ b/programs/system/src/invoke/verify_state_proof.rs @@ -248,8 +248,8 @@ fn fetch_root( /// 1. prove inclusion by index in the output queue if leaf index should exist in the output queue. /// 1.1. if inclusion was proven by index, return Ok. /// 2. prove non-inclusion in the bloom filters -/// 2.1. skip wiped batches. -/// 2.2. prove non-inclusion in the bloom filters for each batch. +/// 2.1. skip wiped batches. +/// 2.2. prove non-inclusion in the bloom filters for each batch. #[inline(always)] pub fn verify_read_only_account_inclusion_by_index<'a>( remaining_accounts: &'a [AccountInfo<'_>], diff --git a/prover/client/Cargo.toml b/prover/client/Cargo.toml index 4bb6f38e47..4050fabd40 100644 --- a/prover/client/Cargo.toml +++ b/prover/client/Cargo.toml @@ -41,6 +41,7 @@ num-traits = "0.2.19" tokio = { workspace = true, optional = true } reqwest = { version = "0.11.24", features = ["json", "rustls-tls"], optional = true } sysinfo = "0.33" + [dev-dependencies] serial_test = "3.2.0" diff --git a/sdk-libs/client/src/indexer/mod.rs b/sdk-libs/client/src/indexer/mod.rs index 709c9770c0..cb7b61b88d 100644 --- a/sdk-libs/client/src/indexer/mod.rs +++ b/sdk-libs/client/src/indexer/mod.rs @@ -1,15 +1,13 @@ -use std::{fmt::Debug, future::Future}; +use std::fmt::Debug; +use async_trait::async_trait; use light_concurrent_merkle_tree::light_hasher::Poseidon; use light_indexed_merkle_tree::{ array::{IndexedArray, IndexedElement}, reference::IndexedMerkleTree, }; use light_merkle_tree_reference::MerkleTree; -use light_sdk::{ - compressed_account::CompressedAccountWithMerkleContext, event::PublicTransactionEvent, - proof::ProofRpcResult, token::TokenDataWithMerkleContext, -}; +use light_sdk::proof::ProofRpcResult; use num_bigint::BigUint; use solana_sdk::pubkey::Pubkey; use thiserror::Error; @@ -19,7 +17,7 @@ use crate::rpc::RpcConnection; #[derive(Error, Debug)] pub enum IndexerError { #[error("RPC Error: {0}")] - RpcError(#[from] solana_client::client_error::ClientError), + RpcError(#[from] Box), #[error("failed to deserialize account data")] DeserializeError(#[from] solana_sdk::program_error::ProgramError), #[error("failed to copy merkle tree")] @@ -30,28 +28,71 @@ pub enum IndexerError { Unknown, } +pub struct ProofOfLeaf { + pub leaf: [u8; 32], + pub proof: Vec<[u8; 32]>, +} + +#[async_trait] pub trait Indexer: Sync + Send + Debug + 'static { - fn add_event_and_compressed_accounts( - &mut self, - event: &PublicTransactionEvent, - ) -> ( - Vec, - Vec, - ); + /// Returns queue elements from the queue with the given pubkey. For input + /// queues account compression program does not store queue elements in the + /// account data but only emits these in the public transaction event. The + /// indexer needs the queue elements to create batch update proofs. + async fn get_queue_elements( + &self, + pubkey: [u8; 32], + batch: u64, + start_offset: u64, + end_offset: u64, + ) -> Result, IndexerError>; - fn create_proof_for_compressed_accounts( + fn get_subtrees(&self, merkle_tree_pubkey: [u8; 32]) -> Result, IndexerError>; + + async fn create_proof_for_compressed_accounts( &mut self, - compressed_accounts: Option<&[[u8; 32]]>, - state_merkle_tree_pubkeys: Option<&[Pubkey]>, + compressed_accounts: Option>, + state_merkle_tree_pubkeys: Option>, new_addresses: Option<&[[u8; 32]]>, address_merkle_tree_pubkeys: Option>, rpc: &mut R, - ) -> impl Future; + ) -> ProofRpcResult; + + async fn get_multiple_compressed_account_proofs( + &self, + hashes: Vec, + ) -> Result, IndexerError>; - fn get_compressed_accounts_by_owner( + async fn get_compressed_accounts_by_owner( &self, owner: &Pubkey, - ) -> Vec; + ) -> Result, IndexerError>; + + async fn get_multiple_new_address_proofs( + &self, + merkle_tree_pubkey: [u8; 32], + addresses: Vec<[u8; 32]>, + ) -> Result>, IndexerError>; + + async fn get_multiple_new_address_proofs_h40( + &self, + merkle_tree_pubkey: [u8; 32], + addresses: Vec<[u8; 32]>, + ) -> Result>, IndexerError>; + + fn get_proofs_by_indices( + &mut self, + merkle_tree_pubkey: Pubkey, + indices: &[u64], + ) -> Vec; + + fn get_leaf_indices_tx_hashes( + &mut self, + merkle_tree_pubkey: Pubkey, + zkp_batch_size: usize, + ) -> Vec; + + fn get_address_merkle_trees(&self) -> &Vec; } #[derive(Debug, Clone)] @@ -64,8 +105,8 @@ pub struct MerkleProof { } // For consistency with the Photon API. -#[derive(Clone, Default, Debug, PartialEq)] -pub struct NewAddressProofWithContext { +#[derive(Clone, Debug, PartialEq)] +pub struct NewAddressProofWithContext { pub merkle_tree: [u8; 32], pub root: [u8; 32], pub root_seq: u64, @@ -73,7 +114,7 @@ pub struct NewAddressProofWithContext { pub low_address_value: [u8; 32], pub low_address_next_index: u64, pub low_address_next_value: [u8; 32], - pub low_address_proof: [[u8; 32]; 16], + pub low_address_proof: [[u8; 32]; NET_HEIGHT], pub new_low_element: Option>, pub new_element: Option>, pub new_element_next_value: Option, @@ -92,17 +133,28 @@ pub struct AddressMerkleTreeAccounts { pub queue: Pubkey, } +#[derive(Debug, Clone)] +pub struct LeafIndexInfo { + pub leaf_index: u32, + pub leaf: [u8; 32], + pub tx_hash: [u8; 32], +} + #[derive(Debug, Clone)] pub struct StateMerkleTreeBundle { - pub rollover_fee: u64, + pub rollover_fee: i64, pub merkle_tree: Box>, pub accounts: StateMerkleTreeAccounts, + pub version: u64, + pub output_queue_elements: Vec<[u8; 32]>, + pub input_leaf_indices: Vec, } #[derive(Debug, Clone)] pub struct AddressMerkleTreeBundle { - pub rollover_fee: u64, + pub rollover_fee: i64, pub merkle_tree: Box>, pub indexed_array: Box>, pub accounts: AddressMerkleTreeAccounts, + pub queue_elements: Vec<[u8; 32]>, } diff --git a/sdk-libs/client/src/photon_rpc/photon_client.rs b/sdk-libs/client/src/photon_rpc/photon_client.rs index 023b453c42..65620a3e01 100644 --- a/sdk-libs/client/src/photon_rpc/photon_client.rs +++ b/sdk-libs/client/src/photon_rpc/photon_client.rs @@ -114,7 +114,7 @@ impl PhotonClient { &self, merkle_tree_pubkey: Pubkey, addresses: Vec
, - ) -> Result, PhotonClientError> { + ) -> Result>, PhotonClientError> { let params: Vec = addresses .iter() .map(|x| photon_api::models::AddressWithTree { @@ -141,7 +141,7 @@ impl PhotonClient { } let photon_proofs = result.unwrap().result.unwrap().value; - let mut proofs: Vec = Vec::new(); + let mut proofs: Vec> = Vec::new(); for photon_proof in photon_proofs { let tree_pubkey = Hash::from_base58(&photon_proof.merkle_tree).unwrap(); let low_address_value = Hash::from_base58(&photon_proof.lower_range_address).unwrap(); diff --git a/sdk-libs/client/src/rpc/solana_rpc.rs b/sdk-libs/client/src/rpc/solana_rpc.rs index ce8e0ed381..a98091a0b4 100644 --- a/sdk-libs/client/src/rpc/solana_rpc.rs +++ b/sdk-libs/client/src/rpc/solana_rpc.rs @@ -27,7 +27,7 @@ use solana_transaction_status::{ use tokio::time::{sleep, Instant}; use crate::{ - rpc::{errors::RpcError, rpc_connection::RpcConnection}, + rpc::{errors::RpcError, merkle_tree::MerkleTreeExt, rpc_connection::RpcConnection}, transaction_params::TransactionParams, }; @@ -461,3 +461,5 @@ impl RpcConnection for SolanaRpcConnection { .await } } + +impl MerkleTreeExt for SolanaRpcConnection {} diff --git a/sdk-libs/program-test/Cargo.toml b/sdk-libs/program-test/Cargo.toml index 597e77027a..a9ec2ab938 100644 --- a/sdk-libs/program-test/Cargo.toml +++ b/sdk-libs/program-test/Cargo.toml @@ -13,6 +13,7 @@ light-prover-client = { workspace = true } light-sdk = { workspace = true } light-indexed-merkle-tree = { workspace = true } light-merkle-tree-reference = { workspace = true } +light-merkle-tree-metadata = { workspace = true } light-hasher = { workspace = true } light-registry = { workspace = true } light-system-program = { workspace = true } diff --git a/sdk-libs/program-test/src/indexer/extensions.rs b/sdk-libs/program-test/src/indexer/extensions.rs new file mode 100644 index 0000000000..6766f6fa3f --- /dev/null +++ b/sdk-libs/program-test/src/indexer/extensions.rs @@ -0,0 +1,110 @@ +use account_compression::initialize_address_merkle_tree::Pubkey; +use async_trait::async_trait; +use light_client::{ + indexer::{ + AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, NewAddressProofWithContext, + ProofOfLeaf, StateMerkleTreeAccounts, StateMerkleTreeBundle, + }, + rpc::RpcConnection, +}; +use light_sdk::{ + compressed_account::CompressedAccountWithMerkleContext, event::PublicTransactionEvent, + proof::BatchedTreeProofRpcResult, token::TokenDataWithMerkleContext, +}; +use solana_sdk::signature::Keypair; + +#[async_trait] +pub trait TestIndexerExtensions: Indexer { + fn get_address_merkle_tree( + &self, + merkle_tree_pubkey: Pubkey, + ) -> Option<&AddressMerkleTreeBundle>; + + fn add_compressed_accounts_with_token_data( + &mut self, + slot: u64, + event: &PublicTransactionEvent, + ); + + fn account_nullified(&mut self, merkle_tree_pubkey: Pubkey, account_hash: &str); + + fn address_tree_updated( + &mut self, + merkle_tree_pubkey: Pubkey, + context: &NewAddressProofWithContext<16>, + ); + + fn get_state_merkle_tree_accounts(&self, pubkeys: &[Pubkey]) -> Vec; + + fn get_state_merkle_trees(&self) -> &Vec; + + fn get_state_merkle_trees_mut(&mut self) -> &mut Vec; + + fn get_address_merkle_trees_mut(&mut self) -> &mut Vec; + + fn get_token_compressed_accounts(&self) -> &Vec; + + fn get_payer(&self) -> &Keypair; + + fn get_group_pda(&self) -> &Pubkey; + + async fn create_proof_for_compressed_accounts2( + &mut self, + compressed_accounts: Option>, + state_merkle_tree_pubkeys: Option>, + new_addresses: Option<&[[u8; 32]]>, + address_merkle_tree_pubkeys: Option>, + rpc: &mut R, + ) -> BatchedTreeProofRpcResult; + + fn add_address_merkle_tree_accounts( + &mut self, + merkle_tree_keypair: &Keypair, + queue_keypair: &Keypair, + owning_program_id: Option, + ) -> AddressMerkleTreeAccounts; + + fn get_compressed_accounts_with_merkle_context_by_owner( + &self, + owner: &Pubkey, + ) -> Vec; + + fn get_compressed_token_accounts_by_owner( + &self, + owner: &Pubkey, + ) -> Vec; + + fn add_state_bundle(&mut self, state_bundle: StateMerkleTreeBundle); + + fn add_event_and_compressed_accounts( + &mut self, + slot: u64, + event: &PublicTransactionEvent, + ) -> ( + Vec, + Vec, + ); + + fn get_proof_by_index(&mut self, merkle_tree_pubkey: Pubkey, index: u64) -> ProofOfLeaf; + + async fn update_test_indexer_after_append( + &mut self, + rpc: &mut R, + merkle_tree_pubkey: Pubkey, + output_queue_pubkey: Pubkey, + num_inserted_zkps: u64, + ); + + async fn update_test_indexer_after_nullification( + &mut self, + rpc: &mut R, + merkle_tree_pubkey: Pubkey, + batch_index: usize, + ); + + async fn finalize_batched_address_tree_update( + &mut self, + rpc: &mut R, + merkle_tree_pubkey: Pubkey, + ); +} diff --git a/sdk-libs/program-test/src/indexer/mod.rs b/sdk-libs/program-test/src/indexer/mod.rs new file mode 100644 index 0000000000..838133dc4f --- /dev/null +++ b/sdk-libs/program-test/src/indexer/mod.rs @@ -0,0 +1,6 @@ +mod extensions; +mod test_indexer; +pub mod utils; + +pub use extensions::TestIndexerExtensions; +pub use test_indexer::TestIndexer; diff --git a/program-tests/utils/src/indexer/test_indexer.rs b/sdk-libs/program-test/src/indexer/test_indexer.rs similarity index 91% rename from program-tests/utils/src/indexer/test_indexer.rs rename to sdk-libs/program-test/src/indexer/test_indexer.rs index f057264ae4..aadbc8e2a2 100644 --- a/program-tests/utils/src/indexer/test_indexer.rs +++ b/sdk-libs/program-test/src/indexer/test_indexer.rs @@ -1,26 +1,12 @@ -use std::{ - future::Future, - marker::PhantomData, - sync::{Arc, Mutex}, - time::Duration, -}; +use std::{marker::PhantomData, time::Duration}; use account_compression::{ - utils::constants::{STATE_MERKLE_TREE_CANOPY_DEPTH, STATE_MERKLE_TREE_HEIGHT}, AddressMerkleTreeAccount, AddressMerkleTreeConfig, AddressQueueConfig, NullifierQueueConfig, StateMerkleTreeAccount, StateMerkleTreeConfig, }; -use anchor_lang::AnchorDeserialize; use async_trait::async_trait; -use forester_utils::{ - get_concurrent_merkle_tree, get_indexed_merkle_tree, - indexer::{ - AddressMerkleTreeAccounts, AddressMerkleTreeBundle, BatchedTreeProofRpcResult, Indexer, - IndexerError, MerkleProof, NewAddressProofWithContext, ProofOfLeaf, ProofRpcResult, - StateMerkleTreeAccounts, StateMerkleTreeBundle, TokenDataWithContext, - }, - AccountZeroCopy, -}; +use borsh::BorshDeserialize; +use forester_utils::{get_concurrent_merkle_tree, get_indexed_merkle_tree, AccountZeroCopy}; use light_batched_merkle_tree::{ batch::BatchState, constants::{DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, DEFAULT_BATCH_STATE_TREE_HEIGHT}, @@ -30,33 +16,23 @@ use light_batched_merkle_tree::{ queue::{BatchedQueueAccount, BatchedQueueMetadata}, }; use light_client::{ - rpc::{RpcConnection, RpcError}, + indexer::{ + AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, IndexerError, LeafIndexInfo, + MerkleProof, NewAddressProofWithContext, ProofOfLeaf, StateMerkleTreeAccounts, + StateMerkleTreeBundle, + }, + rpc::{merkle_tree::MerkleTreeExt, RpcConnection}, transaction_params::FeeConfig, }; -use light_compressed_token::{ - constants::TOKEN_COMPRESSED_ACCOUNT_DISCRIMINATOR, get_token_pool_pda, - mint_sdk::create_create_token_pool_instruction, TokenData, -}; use light_hasher::{Hasher, Poseidon}; use light_indexed_merkle_tree::{array::IndexedArray, reference::IndexedMerkleTree}; -use light_macros::pubkey; use light_merkle_tree_reference::MerkleTree; -use light_program_test::{ - test_batch_forester::{create_batch_address_merkle_tree, create_batched_state_merkle_tree}, - test_env::{ - create_address_merkle_tree_and_queue_account, create_state_merkle_tree_and_queue_account, - EnvAccounts, BATCHED_OUTPUT_QUEUE_TEST_KEYPAIR, - }, -}; use light_prover_client::{ gnark::{ - batch_append_with_proofs_json_formatter::BatchAppendWithProofsInputsJson, combined_json_formatter::CombinedJsonStruct, combined_json_formatter_legacy::CombinedJsonStruct as CombinedJsonStructLegacy, constants::{PROVE_PATH, SERVER_ADDRESS}, - helpers::{ - big_int_to_string, spawn_prover, string_to_big_int, ProofType, ProverConfig, ProverMode, - }, + helpers::{big_int_to_string, spawn_prover, string_to_big_int, ProofType, ProverConfig}, inclusion_json_formatter::BatchInclusionJsonStruct, inclusion_json_formatter_legacy::BatchInclusionJsonStruct as BatchInclusionJsonStructLegacy, non_inclusion_json_formatter::BatchNonInclusionJsonStruct, @@ -71,49 +47,61 @@ use light_prover_client::{ }, non_inclusion_legacy::merkle_non_inclusion_proof_inputs::NonInclusionProofInputs as NonInclusionProofInputsLegacy, }; -use light_system_program::{ - invoke::processor::CompressedProof, - sdk::{ - compressed_account::{CompressedAccountWithMerkleContext, MerkleContext, QueueIndex}, - event::PublicTransactionEvent, - }, +use light_sdk::{ + compressed_account::CompressedAccountWithMerkleContext, + event::PublicTransactionEvent, + merkle_context::MerkleContext, + proof::{BatchedTreeProofRpcResult, CompressedProof, ProofRpcResult}, + token::{TokenData, TokenDataWithMerkleContext}, + STATE_MERKLE_TREE_CANOPY_DEPTH, }; use light_utils::{ bigint::bigint_to_be_bytes_array, hashchain::{create_hash_chain_from_slice, create_tx_hash}, }; -use log::{debug, info, warn}; +use log::{info, warn}; use num_bigint::{BigInt, BigUint}; -use num_traits::ops::bytes::FromBytes; +use num_traits::FromBytes; use reqwest::Client; use solana_sdk::{ - bs58, instruction::Instruction, program_pack::Pack, pubkey::Pubkey, signature::Keypair, - signer::Signer, + bs58, + pubkey::Pubkey, + signature::{Keypair, Signer}, }; -use spl_token::instruction::initialize_mint; use crate::{ - create_address_merkle_tree_and_queue_account_with_assert, e2e_test_env::KeypairActionConfig, - spl::create_initialize_mint_instructions, + indexer::{ + utils::create_address_merkle_tree_and_queue_account_with_assert, TestIndexerExtensions, + }, + test_batch_forester::{create_batch_address_merkle_tree, create_batched_state_merkle_tree}, + test_env::{ + create_state_merkle_tree_and_queue_account, EnvAccounts, BATCHED_OUTPUT_QUEUE_TEST_KEYPAIR, + }, }; #[derive(Debug)] -pub struct TestIndexer { +pub struct TestIndexer +where + R: RpcConnection + MerkleTreeExt, +{ pub state_merkle_trees: Vec, pub address_merkle_trees: Vec, pub payer: Keypair, pub group_pda: Pubkey, pub compressed_accounts: Vec, pub nullified_compressed_accounts: Vec, - pub token_compressed_accounts: Vec, - pub token_nullified_compressed_accounts: Vec, + pub token_compressed_accounts: Vec, + pub token_nullified_compressed_accounts: Vec, pub events: Vec, pub prover_config: Option, phantom: PhantomData, } #[async_trait] -impl Indexer for TestIndexer { +impl Indexer for TestIndexer +where + R: RpcConnection + MerkleTreeExt, +{ async fn get_queue_elements( &self, pubkey: [u8; 32], @@ -143,64 +131,7 @@ impl Indexer for TestIndexer { Err(IndexerError::Custom("Merkle tree not found".to_string())) } - fn get_proof_by_index(&mut self, merkle_tree_pubkey: Pubkey, index: u64) -> ProofOfLeaf { - let mut bundle = self - .state_merkle_trees - .iter_mut() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) - .unwrap(); - - while bundle.merkle_tree.leaves().len() <= index as usize { - bundle.merkle_tree.append(&[0u8; 32]).unwrap(); - } - - let leaf = match bundle.merkle_tree.get_leaf(index as usize) { - Ok(leaf) => leaf, - Err(_) => { - bundle.merkle_tree.append(&[0u8; 32]).unwrap(); - bundle.merkle_tree.get_leaf(index as usize).unwrap() - } - }; - - let proof = bundle - .merkle_tree - .get_proof_of_leaf(index as usize, true) - .unwrap() - .to_vec(); - - ProofOfLeaf { leaf, proof } - } - - fn get_proofs_by_indices( - &mut self, - merkle_tree_pubkey: Pubkey, - indices: &[u64], - ) -> Vec { - indices - .iter() - .map(|&index| self.get_proof_by_index(merkle_tree_pubkey, index)) - .collect() - } - - /// leaf index, leaf, tx hash - fn get_leaf_indices_tx_hashes( - &mut self, - merkle_tree_pubkey: Pubkey, - zkp_batch_size: usize, - ) -> Vec<(u32, [u8; 32], [u8; 32])> { - let mut state_merkle_tree_bundle = self - .state_merkle_trees - .iter_mut() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) - .unwrap(); - - state_merkle_tree_bundle.input_leaf_indices[..zkp_batch_size].to_vec() - } - - async fn get_subtrees( - &self, - merkle_tree_pubkey: [u8; 32], - ) -> Result, IndexerError> { + fn get_subtrees(&self, merkle_tree_pubkey: [u8; 32]) -> Result, IndexerError> { let merkle_tree_pubkey = Pubkey::new_from_array(merkle_tree_pubkey); let address_tree_bundle = self .address_merkle_trees @@ -221,186 +152,6 @@ impl Indexer for TestIndexer { } } - async fn get_multiple_compressed_account_proofs( - &self, - hashes: Vec, - ) -> Result, IndexerError> { - info!("Getting proofs for {:?}", hashes); - let mut proofs: Vec = Vec::new(); - hashes.iter().for_each(|hash| { - let hash_array: [u8; 32] = bs58::decode(hash) - .into_vec() - .unwrap() - .as_slice() - .try_into() - .unwrap(); - - self.state_merkle_trees.iter().for_each(|tree| { - if let Some(leaf_index) = tree.merkle_tree.get_leaf_index(&hash_array) { - let proof = tree - .merkle_tree - .get_proof_of_leaf(leaf_index, false) - .unwrap(); - proofs.push(MerkleProof { - hash: hash.clone(), - leaf_index: leaf_index as u64, - merkle_tree: tree.accounts.merkle_tree.to_string(), - proof: proof.to_vec(), - root_seq: tree.merkle_tree.sequence_number as u64, - }); - } - }) - }); - Ok(proofs) - } - - async fn get_rpc_compressed_accounts_by_owner( - &self, - owner: &Pubkey, - ) -> Result, IndexerError> { - let result = self.get_compressed_accounts_by_owner(owner); - let mut hashes: Vec = Vec::new(); - for account in result.iter() { - let hash = account.hash()?; - let bs58_hash = bs58::encode(hash).into_string(); - hashes.push(bs58_hash); - } - Ok(hashes) - } - - async fn get_multiple_new_address_proofs( - &self, - merkle_tree_pubkey: [u8; 32], - addresses: Vec<[u8; 32]>, - ) -> Result>, IndexerError> { - self._get_multiple_new_address_proofs(merkle_tree_pubkey, addresses, false) - .await - } - - async fn get_multiple_new_address_proofs_full( - &self, - merkle_tree_pubkey: [u8; 32], - addresses: Vec<[u8; 32]>, - ) -> Result>, IndexerError> { - self._get_multiple_new_address_proofs(merkle_tree_pubkey, addresses, true) - .await - } - - fn account_nullified(&mut self, merkle_tree_pubkey: Pubkey, account_hash: &str) { - let decoded_hash: [u8; 32] = bs58::decode(account_hash) - .into_vec() - .unwrap() - .as_slice() - .try_into() - .unwrap(); - - if let Some(state_tree_bundle) = self - .state_merkle_trees - .iter_mut() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) - { - if let Some(leaf_index) = state_tree_bundle.merkle_tree.get_leaf_index(&decoded_hash) { - state_tree_bundle - .merkle_tree - .update(&[0u8; 32], leaf_index) - .unwrap(); - } - } - } - - fn address_tree_updated( - &mut self, - merkle_tree_pubkey: Pubkey, - context: &NewAddressProofWithContext<16>, - ) { - info!("Updating address tree..."); - let mut address_tree_bundle: &mut AddressMerkleTreeBundle = self - .address_merkle_trees - .iter_mut() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) - .unwrap(); - - let new_low_element = context.new_low_element.clone().unwrap(); - let new_element = context.new_element.clone().unwrap(); - let new_element_next_value = context.new_element_next_value.clone().unwrap(); - address_tree_bundle - .merkle_tree - .update(&new_low_element, &new_element, &new_element_next_value) - .unwrap(); - address_tree_bundle - .indexed_array - .append_with_low_element_index(new_low_element.index, &new_element.value) - .unwrap(); - info!("Address tree updated"); - } - - fn get_state_merkle_tree_accounts(&self, pubkeys: &[Pubkey]) -> Vec { - pubkeys - .iter() - .map(|x| { - self.state_merkle_trees - .iter() - .find(|y| y.accounts.merkle_tree == *x || y.accounts.nullifier_queue == *x) - .unwrap() - .accounts - }) - .collect::>() - } - - fn add_event_and_compressed_accounts( - &mut self, - slot: u64, - event: &PublicTransactionEvent, - ) -> ( - Vec, - Vec, - ) { - let mut compressed_accounts = Vec::new(); - let mut token_compressed_accounts = Vec::new(); - let event_inputs_len = event.input_compressed_account_hashes.len(); - let event_outputs_len = event.output_compressed_account_hashes.len(); - for i in 0..std::cmp::max(event_inputs_len, event_outputs_len) { - self.process_v1_compressed_account( - slot, - event, - i, - &mut token_compressed_accounts, - &mut compressed_accounts, - ); - } - - self.events.push(event.clone()); - (compressed_accounts, token_compressed_accounts) - } - - fn get_state_merkle_trees(&self) -> &Vec { - &self.state_merkle_trees - } - - fn get_state_merkle_trees_mut(&mut self) -> &mut Vec { - &mut self.state_merkle_trees - } - - fn get_address_merkle_trees(&self) -> &Vec { - &self.address_merkle_trees - } - - fn get_address_merkle_trees_mut(&mut self) -> &mut Vec { - &mut self.address_merkle_trees - } - - fn get_token_compressed_accounts(&self) -> &Vec { - &self.token_compressed_accounts - } - - fn get_payer(&self) -> &Keypair { - &self.payer - } - - fn get_group_pda(&self) -> &Pubkey { - &self.group_pda - } - async fn create_proof_for_compressed_accounts( &mut self, compressed_accounts: Option>, @@ -582,6 +333,214 @@ impl Indexer for TestIndexer { panic!("Failed to get proof from server"); } + async fn get_multiple_compressed_account_proofs( + &self, + hashes: Vec, + ) -> Result, IndexerError> { + info!("Getting proofs for {:?}", hashes); + let mut proofs: Vec = Vec::new(); + hashes.iter().for_each(|hash| { + let hash_array: [u8; 32] = bs58::decode(hash) + .into_vec() + .unwrap() + .as_slice() + .try_into() + .unwrap(); + + self.state_merkle_trees.iter().for_each(|tree| { + if let Some(leaf_index) = tree.merkle_tree.get_leaf_index(&hash_array) { + let proof = tree + .merkle_tree + .get_proof_of_leaf(leaf_index, false) + .unwrap(); + proofs.push(MerkleProof { + hash: hash.clone(), + leaf_index: leaf_index as u64, + merkle_tree: tree.accounts.merkle_tree.to_string(), + proof: proof.to_vec(), + root_seq: tree.merkle_tree.sequence_number as u64, + }); + } + }) + }); + Ok(proofs) + } + + async fn get_compressed_accounts_by_owner( + &self, + owner: &Pubkey, + ) -> Result, IndexerError> { + let result = self.get_compressed_accounts_with_merkle_context_by_owner(owner); + let mut hashes: Vec = Vec::new(); + for account in result.iter() { + let hash = account.hash().unwrap(); + let bs58_hash = bs58::encode(hash).into_string(); + hashes.push(bs58_hash); + } + Ok(hashes) + } + + async fn get_multiple_new_address_proofs( + &self, + merkle_tree_pubkey: [u8; 32], + addresses: Vec<[u8; 32]>, + ) -> Result>, IndexerError> { + self._get_multiple_new_address_proofs(merkle_tree_pubkey, addresses, false) + .await + } + + async fn get_multiple_new_address_proofs_h40( + &self, + merkle_tree_pubkey: [u8; 32], + addresses: Vec<[u8; 32]>, + ) -> Result>, IndexerError> { + self._get_multiple_new_address_proofs(merkle_tree_pubkey, addresses, true) + .await + } + + fn get_proofs_by_indices( + &mut self, + merkle_tree_pubkey: Pubkey, + indices: &[u64], + ) -> Vec { + indices + .iter() + .map(|&index| self.get_proof_by_index(merkle_tree_pubkey, index)) + .collect() + } + + fn get_leaf_indices_tx_hashes( + &mut self, + merkle_tree_pubkey: Pubkey, + zkp_batch_size: usize, + ) -> Vec { + let state_merkle_tree_bundle = self + .state_merkle_trees + .iter_mut() + .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + .unwrap(); + + state_merkle_tree_bundle.input_leaf_indices[..zkp_batch_size].to_vec() + } + + fn get_address_merkle_trees(&self) -> &Vec { + &self.address_merkle_trees + } +} + +#[async_trait] +impl TestIndexerExtensions for TestIndexer +where + R: RpcConnection + MerkleTreeExt, +{ + fn get_address_merkle_tree( + &self, + merkle_tree_pubkey: Pubkey, + ) -> Option<&AddressMerkleTreeBundle> { + self.address_merkle_trees + .iter() + .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + } + + /// deserializes an event + /// adds the output_compressed_accounts to the compressed_accounts + /// removes the input_compressed_accounts from the compressed_accounts + /// adds the input_compressed_accounts to the nullified_compressed_accounts + /// deserialiazes token data from the output_compressed_accounts + /// adds the token_compressed_accounts to the token_compressed_accounts + fn add_compressed_accounts_with_token_data( + &mut self, + slot: u64, + event: &PublicTransactionEvent, + ) { + self.add_event_and_compressed_accounts(slot, event); + } + + fn account_nullified(&mut self, merkle_tree_pubkey: Pubkey, account_hash: &str) { + let decoded_hash: [u8; 32] = bs58::decode(account_hash) + .into_vec() + .unwrap() + .as_slice() + .try_into() + .unwrap(); + + if let Some(state_tree_bundle) = self + .state_merkle_trees + .iter_mut() + .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + { + if let Some(leaf_index) = state_tree_bundle.merkle_tree.get_leaf_index(&decoded_hash) { + state_tree_bundle + .merkle_tree + .update(&[0u8; 32], leaf_index) + .unwrap(); + } + } + } + + fn address_tree_updated( + &mut self, + merkle_tree_pubkey: Pubkey, + context: &NewAddressProofWithContext<16>, + ) { + info!("Updating address tree..."); + let address_tree_bundle: &mut AddressMerkleTreeBundle = self + .address_merkle_trees + .iter_mut() + .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + .unwrap(); + + let new_low_element = context.new_low_element.clone().unwrap(); + let new_element = context.new_element.clone().unwrap(); + let new_element_next_value = context.new_element_next_value.clone().unwrap(); + address_tree_bundle + .merkle_tree + .update(&new_low_element, &new_element, &new_element_next_value) + .unwrap(); + address_tree_bundle + .indexed_array + .append_with_low_element_index(new_low_element.index, &new_element.value) + .unwrap(); + info!("Address tree updated"); + } + + fn get_state_merkle_tree_accounts(&self, pubkeys: &[Pubkey]) -> Vec { + pubkeys + .iter() + .map(|x| { + self.state_merkle_trees + .iter() + .find(|y| y.accounts.merkle_tree == *x || y.accounts.nullifier_queue == *x) + .unwrap() + .accounts + }) + .collect::>() + } + + fn get_state_merkle_trees(&self) -> &Vec { + &self.state_merkle_trees + } + + fn get_state_merkle_trees_mut(&mut self) -> &mut Vec { + &mut self.state_merkle_trees + } + + fn get_address_merkle_trees_mut(&mut self) -> &mut Vec { + &mut self.address_merkle_trees + } + + fn get_token_compressed_accounts(&self) -> &Vec { + &self.token_compressed_accounts + } + + fn get_payer(&self) -> &Keypair { + &self.payer + } + + fn get_group_pda(&self) -> &Pubkey { + &self.group_pda + } + async fn create_proof_for_compressed_accounts2( &mut self, compressed_accounts: Option>, @@ -704,9 +663,7 @@ impl Indexer for TestIndexer { address_merkle_tree_accounts } - /// returns compressed_accounts with the owner pubkey - /// does not return token accounts. - fn get_compressed_accounts_by_owner( + fn get_compressed_accounts_with_merkle_context_by_owner( &self, owner: &Pubkey, ) -> Vec { @@ -717,7 +674,10 @@ impl Indexer for TestIndexer { .collect() } - fn get_compressed_token_accounts_by_owner(&self, owner: &Pubkey) -> Vec { + fn get_compressed_token_accounts_by_owner( + &self, + owner: &Pubkey, + ) -> Vec { self.token_compressed_accounts .iter() .filter(|x| x.token_data.owner == *owner) @@ -729,6 +689,60 @@ impl Indexer for TestIndexer { self.get_state_merkle_trees_mut().push(state_bundle); } + fn add_event_and_compressed_accounts( + &mut self, + slot: u64, + event: &PublicTransactionEvent, + ) -> ( + Vec, + Vec, + ) { + let mut compressed_accounts = Vec::new(); + let mut token_compressed_accounts = Vec::new(); + let event_inputs_len = event.input_compressed_account_hashes.len(); + let event_outputs_len = event.output_compressed_account_hashes.len(); + for i in 0..std::cmp::max(event_inputs_len, event_outputs_len) { + self.process_v1_compressed_account( + slot, + event, + i, + &mut token_compressed_accounts, + &mut compressed_accounts, + ); + } + + self.events.push(event.clone()); + (compressed_accounts, token_compressed_accounts) + } + + fn get_proof_by_index(&mut self, merkle_tree_pubkey: Pubkey, index: u64) -> ProofOfLeaf { + let bundle = self + .state_merkle_trees + .iter_mut() + .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + .unwrap(); + + while bundle.merkle_tree.leaves().len() <= index as usize { + bundle.merkle_tree.append(&[0u8; 32]).unwrap(); + } + + let leaf = match bundle.merkle_tree.get_leaf(index as usize) { + Ok(leaf) => leaf, + Err(_) => { + bundle.merkle_tree.append(&[0u8; 32]).unwrap(); + bundle.merkle_tree.get_leaf(index as usize).unwrap() + } + }; + + let proof = bundle + .merkle_tree + .get_proof_of_leaf(index as usize, true) + .unwrap() + .to_vec(); + + ProofOfLeaf { leaf, proof } + } + async fn update_test_indexer_after_append( &mut self, rpc: &mut R, @@ -736,7 +750,7 @@ impl Indexer for TestIndexer { output_queue_pubkey: Pubkey, num_inserted_zkps: u64, ) { - let mut state_merkle_tree_bundle = self + let state_merkle_tree_bundle = self .state_merkle_trees .iter_mut() .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) @@ -830,12 +844,13 @@ impl Indexer for TestIndexer { let batch_size = batch.zkp_batch_size; let leaf_indices_tx_hashes = state_merkle_tree_bundle.input_leaf_indices[..batch_size as usize].to_vec(); - for (index, leaf, tx_hash) in leaf_indices_tx_hashes.iter() { - let index = *index as usize; - let leaf = *leaf; + for leaf_info in leaf_indices_tx_hashes.iter() { + let index = leaf_info.leaf_index as usize; + let leaf = leaf_info.leaf; let index_bytes = index.to_be_bytes(); - let nullifier = Poseidon::hashv(&[&leaf, &index_bytes, tx_hash]).unwrap(); + let nullifier = + Poseidon::hashv(&[&leaf, &index_bytes, &leaf_info.tx_hash]).unwrap(); state_merkle_tree_bundle.input_leaf_indices.remove(0); state_merkle_tree_bundle @@ -885,90 +900,10 @@ impl Indexer for TestIndexer { } } -impl TestIndexer { - async fn _get_multiple_new_address_proofs( - &self, - merkle_tree_pubkey: [u8; 32], - addresses: Vec<[u8; 32]>, - full: bool, - ) -> Result>, IndexerError> { - let mut proofs: Vec> = Vec::new(); - - for address in addresses.iter() { - info!("Getting new address proof for {:?}", address); - let pubkey = Pubkey::from(merkle_tree_pubkey); - let address_tree_bundle = self - .address_merkle_trees - .iter() - .find(|x| x.accounts.merkle_tree == pubkey) - .unwrap(); - - let address_biguint = BigUint::from_bytes_be(address.as_slice()); - let (old_low_address, _old_low_address_next_value) = address_tree_bundle - .indexed_array - .find_low_element_for_nonexistent(&address_biguint) - .unwrap(); - let address_bundle = address_tree_bundle - .indexed_array - .new_element_with_low_element_index(old_low_address.index, &address_biguint) - .unwrap(); - - let (old_low_address, old_low_address_next_value) = address_tree_bundle - .indexed_array - .find_low_element_for_nonexistent(&address_biguint) - .unwrap(); - - // Get the Merkle proof for updating low element. - let low_address_proof = address_tree_bundle - .merkle_tree - .get_proof_of_leaf(old_low_address.index, full) - .unwrap(); - - let low_address_index: u64 = old_low_address.index as u64; - let low_address_value: [u8; 32] = - bigint_to_be_bytes_array(&old_low_address.value).unwrap(); - let low_address_next_index: u64 = old_low_address.next_index as u64; - let low_address_next_value: [u8; 32] = - bigint_to_be_bytes_array(&old_low_address_next_value).unwrap(); - let low_address_proof: [[u8; 32]; NET_HEIGHT] = low_address_proof.to_array().unwrap(); - let proof = NewAddressProofWithContext:: { - merkle_tree: merkle_tree_pubkey, - low_address_index, - low_address_value, - low_address_next_index, - low_address_next_value, - low_address_proof, - root: address_tree_bundle.merkle_tree.root(), - root_seq: address_tree_bundle.merkle_tree.merkle_tree.sequence_number as u64, - new_low_element: Some(address_bundle.new_low_element), - new_element: Some(address_bundle.new_element), - new_element_next_value: Some(address_bundle.new_element_next_value), - }; - proofs.push(proof); - } - Ok(proofs) - } - - fn count_matching_hashes(&self, query_hashes: &[String]) -> usize { - self.nullified_compressed_accounts - .iter() - .map(|account| self.compute_hash(account)) - .filter(|bs58_hash| query_hashes.contains(bs58_hash)) - .count() - } - - fn compute_hash(&self, account: &CompressedAccountWithMerkleContext) -> String { - // replace AccountType with actual type - let hash = account - .compressed_account - .hash::( - &account.merkle_context.merkle_tree_pubkey, - &account.merkle_context.leaf_index, - ) - .unwrap(); - bs58::encode(hash).into_string() - } - +impl TestIndexer +where + R: RpcConnection + MerkleTreeExt, +{ pub async fn init_from_env( payer: &Keypair, env: &EnvAccounts, @@ -1030,8 +965,8 @@ impl TestIndexer { (2, merkle_tree) } else { let merkle_tree = Box::new(MerkleTree::::new( - STATE_MERKLE_TREE_HEIGHT as usize, - STATE_MERKLE_TREE_CANOPY_DEPTH as usize, + account_compression::utils::constants::STATE_MERKLE_TREE_HEIGHT as usize, + account_compression::utils::constants::STATE_MERKLE_TREE_CANOPY_DEPTH as usize, )); (1, merkle_tree) }; @@ -1076,7 +1011,7 @@ impl TestIndexer { if address_merkle_tree_accounts.merkle_tree == address_merkle_tree_accounts.queue { (40, 0) } else { - (26, STATE_MERKLE_TREE_CANOPY_DEPTH as usize) + (26, STATE_MERKLE_TREE_CANOPY_DEPTH) }; let mut merkle_tree = Box::new(IndexedMerkleTree::::new(height, canopy).unwrap()); @@ -1190,26 +1125,26 @@ impl TestIndexer { ) { let (rollover_fee, merkle_tree) = match version { 1 => { - create_state_merkle_tree_and_queue_account( - &self.payer, - true, - rpc, - merkle_tree_keypair, + create_state_merkle_tree_and_queue_account( + &self.payer, + true, + rpc, + merkle_tree_keypair, queue_keypair, - Some(cpi_context_keypair), - owning_program_id, - forester, - self.state_merkle_trees.len() as u64, - &StateMerkleTreeConfig::default(), - &NullifierQueueConfig::default(), - ) - .await - .unwrap(); - let merkle_tree = Box::new(MerkleTree::::new( - STATE_MERKLE_TREE_HEIGHT as usize, - STATE_MERKLE_TREE_CANOPY_DEPTH as usize, - )); - (FeeConfig::default().state_merkle_tree_rollover as i64,merkle_tree) + Some(cpi_context_keypair), + owning_program_id, + forester, + self.state_merkle_trees.len() as u64, + &StateMerkleTreeConfig::default(), + &NullifierQueueConfig::default(), + ) + .await + .unwrap(); + let merkle_tree = Box::new(MerkleTree::::new( + account_compression::utils::constants::STATE_MERKLE_TREE_HEIGHT as usize, + account_compression::utils::constants::STATE_MERKLE_TREE_CANOPY_DEPTH as usize, + )); + (FeeConfig::default().state_merkle_tree_rollover as i64,merkle_tree) } 2 => { let params = InitStateTreeAccountsInstructionData::test_default(); @@ -1222,7 +1157,7 @@ impl TestIndexer { queue_keypair, cpi_context_keypair, params, - ).await; + ).await.unwrap(); let merkle_tree = Box::new(MerkleTree::::new( DEFAULT_BATCH_STATE_TREE_HEIGHT as usize, 0 @@ -1309,12 +1244,11 @@ impl TestIndexer { }); let (root_index, root) = if version == 1 { - let fetched_merkle_tree = unsafe { + let fetched_merkle_tree = get_concurrent_merkle_tree::( rpc, pubkey, ) - .await - }; + .await; ( fetched_merkle_tree.root_index() as u32, fetched_merkle_tree.root(), @@ -1346,7 +1280,8 @@ impl TestIndexer { )), None, ) - } else if height == STATE_MERKLE_TREE_HEIGHT as usize { + } else if height == account_compression::utils::constants::STATE_MERKLE_TREE_HEIGHT as usize + { let inclusion_proof_inputs = InclusionProofInputsLegacy(inclusion_proofs.as_slice()); ( None, @@ -1361,7 +1296,7 @@ impl TestIndexer { (batch_inclusion_proof_inputs, legacy, root_indices) } - pub async fn process_non_inclusion_proofs( + async fn process_non_inclusion_proofs( &self, address_merkle_tree_pubkeys: &[Pubkey], addresses: &[[u8; 32]], @@ -1420,13 +1355,17 @@ impl TestIndexer { ); } } else { - let fetched_address_merkle_tree = unsafe { - get_indexed_merkle_tree::( - rpc, - address_merkle_tree_pubkeys[i], - ) - .await - }; + let fetched_address_merkle_tree = get_indexed_merkle_tree::< + AddressMerkleTreeAccount, + R, + Poseidon, + usize, + 26, + 16, + >( + rpc, address_merkle_tree_pubkeys[i] + ) + .await; address_root_indices.push(fetched_address_merkle_tree.root_index() as u16); } } @@ -1477,23 +1416,10 @@ impl TestIndexer { pub fn add_lamport_compressed_accounts(&mut self, slot: u64, event_bytes: Vec) { let event_bytes = event_bytes.clone(); let event = PublicTransactionEvent::deserialize(&mut event_bytes.as_slice()).unwrap(); + // TODO: map event type self.add_event_and_compressed_accounts(slot, &event); } - /// deserializes an event - /// adds the output_compressed_accounts to the compressed_accounts - /// removes the input_compressed_accounts from the compressed_accounts - /// adds the input_compressed_accounts to the nullified_compressed_accounts - /// deserialiazes token data from the output_compressed_accounts - /// adds the token_compressed_accounts to the token_compressed_accounts - pub fn add_compressed_accounts_with_token_data( - &mut self, - slot: u64, - event: &PublicTransactionEvent, - ) { - self.add_event_and_compressed_accounts(slot, event); - } - /// returns the compressed sol balance of the owner pubkey pub fn get_compressed_balance(&self, owner: &Pubkey) -> u64 { self.compressed_accounts @@ -1520,7 +1446,7 @@ impl TestIndexer { slot: u64, event: &PublicTransactionEvent, i: usize, - token_compressed_accounts: &mut Vec, + token_compressed_accounts: &mut Vec, compressed_accounts: &mut Vec, ) { let mut input_addresses = vec![]; @@ -1591,9 +1517,11 @@ impl TestIndexer { // Store leaf indices of input accounts for batched trees if bundle.version == 2 { let leaf_hash = event.input_compressed_account_hashes[i]; - bundle - .input_leaf_indices - .push((leaf_index, leaf_hash, tx_hash)); + bundle.input_leaf_indices.push(LeafIndexInfo { + leaf_index, + leaf: leaf_hash, + tx_hash, + }); } } let mut new_addresses = vec![]; @@ -1633,10 +1561,10 @@ impl TestIndexer { match compressed_account.compressed_account.data.as_ref() { Some(data) => { if compressed_account.compressed_account.owner == light_compressed_token::ID - && data.discriminator == TOKEN_COMPRESSED_ACCOUNT_DISCRIMINATOR + && data.discriminator == light_compressed_token::constants::TOKEN_COMPRESSED_ACCOUNT_DISCRIMINATOR { if let Ok(token_data) = TokenData::deserialize(&mut data.data.as_slice()) { - let token_account = TokenDataWithContext { + let token_account = TokenDataWithMerkleContext { token_data, compressed_account: CompressedAccountWithMerkleContext { compressed_account: compressed_account @@ -1758,7 +1686,7 @@ impl TestIndexer { .address_merkle_trees .iter_mut() .enumerate() - .find(|(i, x)| x.accounts.merkle_tree == *pubkey) + .find(|(_, x)| x.accounts.merkle_tree == *pubkey) { address_merkle_tree .queue_elements @@ -1768,12 +1696,66 @@ impl TestIndexer { } } - pub(crate) fn get_address_merkle_tree( + async fn _get_multiple_new_address_proofs( &self, - merkle_tree_pubkey: Pubkey, - ) -> Option<&AddressMerkleTreeBundle> { - self.address_merkle_trees - .iter() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey) + merkle_tree_pubkey: [u8; 32], + addresses: Vec<[u8; 32]>, + full: bool, + ) -> Result>, IndexerError> { + let mut proofs: Vec> = Vec::new(); + + for address in addresses.iter() { + info!("Getting new address proof for {:?}", address); + let pubkey = Pubkey::from(merkle_tree_pubkey); + let address_tree_bundle = self + .address_merkle_trees + .iter() + .find(|x| x.accounts.merkle_tree == pubkey) + .unwrap(); + + let address_biguint = BigUint::from_bytes_be(address.as_slice()); + let (old_low_address, _old_low_address_next_value) = address_tree_bundle + .indexed_array + .find_low_element_for_nonexistent(&address_biguint) + .unwrap(); + let address_bundle = address_tree_bundle + .indexed_array + .new_element_with_low_element_index(old_low_address.index, &address_biguint) + .unwrap(); + + let (old_low_address, old_low_address_next_value) = address_tree_bundle + .indexed_array + .find_low_element_for_nonexistent(&address_biguint) + .unwrap(); + + // Get the Merkle proof for updating low element. + let low_address_proof = address_tree_bundle + .merkle_tree + .get_proof_of_leaf(old_low_address.index, full) + .unwrap(); + + let low_address_index: u64 = old_low_address.index as u64; + let low_address_value: [u8; 32] = + bigint_to_be_bytes_array(&old_low_address.value).unwrap(); + let low_address_next_index: u64 = old_low_address.next_index as u64; + let low_address_next_value: [u8; 32] = + bigint_to_be_bytes_array(&old_low_address_next_value).unwrap(); + let low_address_proof: [[u8; 32]; NET_HEIGHT] = low_address_proof.to_array().unwrap(); + let proof = NewAddressProofWithContext:: { + merkle_tree: merkle_tree_pubkey, + low_address_index, + low_address_value, + low_address_next_index, + low_address_next_value, + low_address_proof, + root: address_tree_bundle.merkle_tree.root(), + root_seq: address_tree_bundle.merkle_tree.merkle_tree.sequence_number as u64, + new_low_element: Some(address_bundle.new_low_element), + new_element: Some(address_bundle.new_element), + new_element_next_value: Some(address_bundle.new_element_next_value), + }; + proofs.push(proof); + } + Ok(proofs) } } diff --git a/sdk-libs/program-test/src/indexer/utils.rs b/sdk-libs/program-test/src/indexer/utils.rs new file mode 100644 index 0000000000..17c180f764 --- /dev/null +++ b/sdk-libs/program-test/src/indexer/utils.rs @@ -0,0 +1,390 @@ +use std::cmp; + +use account_compression::{AddressMerkleTreeConfig, AddressQueueConfig, RegisteredProgram}; +use forester_utils::{get_hash_set, get_indexed_merkle_tree, AccountZeroCopy}; +use light_client::rpc::{RpcConnection, RpcError}; +use light_hasher::Poseidon; +use light_merkle_tree_metadata::{ + access::AccessMetadata, + queue::{QueueMetadata, QueueType}, + rollover::RolloverMetadata, +}; +use light_registry::account_compression_cpi::sdk::get_registered_program_pda; +use light_utils::fee::compute_rollover_fee; +use solana_sdk::{ + pubkey::Pubkey, + signature::{Keypair, Signature, Signer}, +}; + +use crate::test_env::create_address_merkle_tree_and_queue_account; + +#[allow(clippy::too_many_arguments)] +#[inline(never)] +pub async fn create_address_merkle_tree_and_queue_account_with_assert( + payer: &Keypair, + registry: bool, + context: &mut R, + address_merkle_tree_keypair: &Keypair, + address_queue_keypair: &Keypair, + program_owner: Option, + forester: Option, + merkle_tree_config: &AddressMerkleTreeConfig, + queue_config: &AddressQueueConfig, + index: u64, +) -> Result { + let result = create_address_merkle_tree_and_queue_account( + payer, + registry, + context, + address_merkle_tree_keypair, + address_queue_keypair, + program_owner, + forester, + merkle_tree_config, + queue_config, + index, + ) + .await; + + // To initialize the indexed tree we do 4 operations: + // 1. insert 0 append 0 and update 0 + // 2. insert 1 append BN254_FIELD_SIZE -1 and update 0 + // we appended two values this the expected next index is 2; + // The right most leaf is the hash of the indexed array element with value FIELD_SIZE - 1 + // index 1, next_index: 0 + let expected_change_log_length = cmp::min(4, merkle_tree_config.changelog_size as usize); + let expected_roots_length = cmp::min(4, merkle_tree_config.roots_size as usize); + let expected_next_index = 2; + let expected_indexed_change_log_length = + cmp::min(4, merkle_tree_config.address_changelog_size as usize); + + let mut reference_tree = + light_indexed_merkle_tree::reference::IndexedMerkleTree::::new( + account_compression::utils::constants::ADDRESS_MERKLE_TREE_HEIGHT as usize, + account_compression::utils::constants::ADDRESS_MERKLE_TREE_CANOPY_DEPTH as usize, + ) + .unwrap(); + reference_tree.init().unwrap(); + + let expected_right_most_leaf = reference_tree + .merkle_tree + .get_leaf(reference_tree.merkle_tree.rightmost_index - 1) + .unwrap(); + + let _expected_right_most_leaf = [ + 30, 164, 22, 238, 180, 2, 24, 181, 64, 193, 207, 184, 219, 233, 31, 109, 84, 232, 162, 158, + 220, 48, 163, 158, 50, 107, 64, 87, 167, 217, 99, 245, + ]; + assert_eq!(expected_right_most_leaf, _expected_right_most_leaf); + let owner = if registry { + let registered_program = get_registered_program_pda(&light_registry::ID); + let registered_program_account = context + .get_anchor_account::(®istered_program) + .await + .unwrap() + .unwrap(); + registered_program_account.group_authority_pda + } else { + payer.pubkey() + }; + + assert_address_merkle_tree_initialized( + context, + &address_merkle_tree_keypair.pubkey(), + &address_queue_keypair.pubkey(), + merkle_tree_config, + index, + program_owner, + forester, + expected_change_log_length, + expected_roots_length, + expected_next_index, + &expected_right_most_leaf, + &owner, + expected_indexed_change_log_length, + ) + .await; + + assert_address_queue_initialized( + context, + &address_queue_keypair.pubkey(), + queue_config, + &address_merkle_tree_keypair.pubkey(), + merkle_tree_config, + QueueType::AddressQueue, + index, + program_owner, + forester, + &owner, + ) + .await; + + result +} + +#[allow(clippy::too_many_arguments)] +pub async fn assert_address_merkle_tree_initialized( + rpc: &mut R, + merkle_tree_pubkey: &Pubkey, + queue_pubkey: &Pubkey, + merkle_tree_config: &account_compression::AddressMerkleTreeConfig, + index: u64, + program_owner: Option, + forester: Option, + expected_changelog_length: usize, + expected_roots_length: usize, + expected_next_index: usize, + expected_rightmost_leaf: &[u8; 32], + owner_pubkey: &Pubkey, + expected_indexed_changelog_length: usize, +) { + let merkle_tree = AccountZeroCopy::::new( + rpc, + *merkle_tree_pubkey, + ) + .await; + let merkle_tree_account = merkle_tree.deserialized(); + + assert_eq!( + merkle_tree_account + .metadata + .rollover_metadata + .rollover_threshold, + merkle_tree_config.rollover_threshold.unwrap_or(u64::MAX) + ); + assert_eq!( + merkle_tree_account.metadata.rollover_metadata.network_fee, + merkle_tree_config.network_fee.unwrap_or_default() + ); + + // The address Merkle tree is never directly called by the user. + // The whole rollover fees are collected by the address queue. + let expected_rollover_fee = 0; + assert_eq!( + merkle_tree_account.metadata.rollover_metadata.rollover_fee, + expected_rollover_fee + ); + + assert_eq!(merkle_tree_account.metadata.rollover_metadata.index, index); + assert_eq!( + merkle_tree_account + .metadata + .rollover_metadata + .rolledover_slot, + u64::MAX + ); + + assert_eq!( + merkle_tree_account + .metadata + .rollover_metadata + .close_threshold, + merkle_tree_config.close_threshold.unwrap_or(u64::MAX) + ); + + assert_eq!( + merkle_tree_account.metadata.next_merkle_tree, + Pubkey::default().into() + ); + let expected_access_meta_data = AccessMetadata { + owner: (*owner_pubkey).into(), + program_owner: program_owner.unwrap_or_default().into(), + forester: forester.unwrap_or_default().into(), + }; + assert_eq!( + merkle_tree_account.metadata.access_metadata, + expected_access_meta_data + ); + assert_eq!( + merkle_tree_account.metadata.associated_queue, + (*queue_pubkey).into() + ); + + let merkle_tree = get_indexed_merkle_tree::< + account_compression::AddressMerkleTreeAccount, + R, + Poseidon, + usize, + 26, + 16, + >(rpc, *merkle_tree_pubkey) + .await; + + assert_eq!(merkle_tree.height, merkle_tree_config.height as usize); + assert_eq!( + merkle_tree.merkle_tree.changelog.capacity(), + merkle_tree_config.changelog_size as usize + ); + assert_eq!( + merkle_tree.merkle_tree.changelog.len(), + expected_changelog_length + ); + assert_eq!( + merkle_tree.merkle_tree.changelog_index(), + expected_changelog_length.saturating_sub(1) + ); + assert_eq!( + merkle_tree.roots.capacity(), + merkle_tree_config.roots_size as usize + ); + assert_eq!(merkle_tree.roots.len(), expected_roots_length); + assert_eq!( + merkle_tree.root_index(), + expected_roots_length.saturating_sub(1) + ); + assert_eq!( + merkle_tree.canopy_depth, + merkle_tree_config.canopy_depth as usize + ); + assert_eq!(merkle_tree.next_index(), expected_next_index); + assert_eq!( + merkle_tree.sequence_number() % merkle_tree_config.roots_size as usize, + expected_roots_length.saturating_sub(1) + ); + assert_eq!(&merkle_tree.rightmost_leaf(), expected_rightmost_leaf); + // TODO: complete asserts + assert_eq!( + merkle_tree.indexed_changelog_index(), + expected_indexed_changelog_length.saturating_sub(1) + ); +} + +#[allow(clippy::too_many_arguments)] +pub async fn assert_address_queue_initialized( + rpc: &mut R, + queue_pubkey: &Pubkey, + queue_config: &account_compression::AddressQueueConfig, + associated_merkle_tree_pubkey: &Pubkey, + associated_tree_config: &account_compression::AddressMerkleTreeConfig, + expected_queue_type: QueueType, + expected_index: u64, + expected_program_owner: Option, + expected_forester: Option, + payer_pubkey: &Pubkey, +) { + assert_address_queue( + rpc, + queue_pubkey, + queue_config, + associated_merkle_tree_pubkey, + associated_tree_config, + expected_queue_type, + expected_index, + expected_program_owner, + expected_forester, + None, + None, + payer_pubkey, + ) + .await; +} + +#[allow(clippy::too_many_arguments)] +pub async fn assert_address_queue( + rpc: &mut R, + queue_pubkey: &Pubkey, + queue_config: &account_compression::AddressQueueConfig, + associated_merkle_tree_pubkey: &Pubkey, + associated_tree_config: &account_compression::AddressMerkleTreeConfig, + expected_queue_type: QueueType, + expected_index: u64, + expected_program_owner: Option, + expected_forester: Option, + expected_rolledover_slot: Option, + expected_next_queue: Option, + payer_pubkey: &Pubkey, +) { + let balance_merkle_tree = rpc + .get_account(*associated_merkle_tree_pubkey) + .await + .unwrap() + .unwrap() + .lamports; + let balance_queue = rpc + .get_account(*queue_pubkey) + .await + .unwrap() + .unwrap() + .lamports; + // The address queue is the only account that collects the rollover fees. + let expected_rollover_fee = match associated_tree_config.rollover_threshold { + Some(threshold) => { + compute_rollover_fee(threshold, associated_tree_config.height, balance_queue).unwrap() + + compute_rollover_fee( + threshold, + associated_tree_config.height, + balance_merkle_tree, + ) + .unwrap() + } + None => 0, + }; + assert_queue( + rpc, + queue_pubkey, + queue_config, + associated_merkle_tree_pubkey, + associated_tree_config, + expected_rollover_fee, + expected_queue_type, + expected_index, + expected_program_owner, + expected_forester, + expected_rolledover_slot, + expected_next_queue, + payer_pubkey, + ) + .await; +} +#[allow(clippy::too_many_arguments)] +pub async fn assert_queue( + rpc: &mut R, + queue_pubkey: &Pubkey, + queue_config: &account_compression::AddressQueueConfig, + associated_merkle_tree_pubkey: &Pubkey, + associated_tree_config: &account_compression::AddressMerkleTreeConfig, + expected_rollover_fee: u64, + expected_queue_type: QueueType, + expected_index: u64, + expected_program_owner: Option, + expected_forester: Option, + expected_rolledover_slot: Option, + expected_next_queue: Option, + payer_pubkey: &Pubkey, +) { + let queue = AccountZeroCopy::::new(rpc, *queue_pubkey).await; + let queue_account = queue.deserialized(); + + let expected_rollover_meta_data = RolloverMetadata { + index: expected_index, + rolledover_slot: expected_rolledover_slot.unwrap_or(u64::MAX), + rollover_threshold: associated_tree_config + .rollover_threshold + .unwrap_or(u64::MAX), + network_fee: queue_config.network_fee.unwrap_or_default(), + rollover_fee: expected_rollover_fee, + close_threshold: associated_tree_config.close_threshold.unwrap_or(u64::MAX), + additional_bytes: 0, + }; + let expected_access_meta_data = AccessMetadata { + owner: (*payer_pubkey).into(), + program_owner: expected_program_owner.unwrap_or_default().into(), + forester: expected_forester.unwrap_or_default().into(), + }; + let expected_queue_meta_data = QueueMetadata { + access_metadata: expected_access_meta_data, + rollover_metadata: expected_rollover_meta_data, + associated_merkle_tree: (*associated_merkle_tree_pubkey).into(), + next_queue: expected_next_queue.unwrap_or_default().into(), + queue_type: expected_queue_type as u64, + }; + assert_eq!(queue_account.metadata, expected_queue_meta_data); + + let queue = + unsafe { get_hash_set::(rpc, *queue_pubkey).await }; + assert_eq!(queue.get_capacity(), queue_config.capacity as usize); + assert_eq!( + queue.sequence_threshold, + queue_config.sequence_threshold as usize + ); +} diff --git a/sdk-libs/program-test/src/lib.rs b/sdk-libs/program-test/src/lib.rs index d02fe4b890..9d0ba91f5b 100644 --- a/sdk-libs/program-test/src/lib.rs +++ b/sdk-libs/program-test/src/lib.rs @@ -1,5 +1,5 @@ pub mod env_accounts; +pub mod indexer; pub mod test_batch_forester; pub mod test_env; -pub mod test_indexer; pub mod test_rpc; diff --git a/sdk-libs/program-test/src/test_batch_forester.rs b/sdk-libs/program-test/src/test_batch_forester.rs index 07985993be..ee28d087ac 100644 --- a/sdk-libs/program-test/src/test_batch_forester.rs +++ b/sdk-libs/program-test/src/test_batch_forester.rs @@ -1,10 +1,6 @@ use anchor_lang::AnchorDeserialize; use borsh::BorshSerialize; -use forester_utils::{ - create_account_instruction, - indexer::{Indexer, StateMerkleTreeBundle}, - AccountZeroCopy, -}; +use forester_utils::{create_account_instruction, AccountZeroCopy}; use light_batched_merkle_tree::{ constants::{DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, DEFAULT_BATCH_STATE_TREE_HEIGHT}, event::{BatchAppendEvent, BatchNullifyEvent}, @@ -303,10 +299,10 @@ pub async fn get_batched_nullify_ix_data( let mut tx_hashes = Vec::new(); let mut old_leaves = Vec::new(); let mut path_indices = Vec::new(); - for (index, leaf, tx_hash) in leaf_indices_tx_hashes.iter() { - path_indices.push(*index); - let index = *index as usize; - let leaf = *leaf; + for leaf_info in leaf_indices_tx_hashes.iter() { + path_indices.push(leaf_info.leaf_index); + let index = leaf_info.leaf_index as usize; + let leaf = leaf_info.leaf; leaves.push(leaf); // + 2 because next index is + 1 and we need to init the leaf in @@ -328,8 +324,8 @@ pub async fn get_batched_nullify_ix_data( bundle.input_leaf_indices.remove(0); let index_bytes = index.to_be_bytes(); use light_hasher::Hasher; - let nullifier = Poseidon::hashv(&[&leaf, &index_bytes, tx_hash]).unwrap(); - tx_hashes.push(*tx_hash); + let nullifier = Poseidon::hashv(&[&leaf, &index_bytes, &leaf_info.tx_hash]).unwrap(); + tx_hashes.push(leaf_info.tx_hash); nullifiers.push(nullifier); bundle.merkle_tree.update(&nullifier, index).unwrap(); } @@ -393,6 +389,7 @@ pub async fn get_batched_nullify_ix_data( } use anchor_lang::{InstructionData, ToAccountMetas}; +use light_client::indexer::{Indexer, StateMerkleTreeBundle}; pub async fn create_batched_state_merkle_tree( payer: &Keypair, @@ -858,7 +855,7 @@ pub async fn create_batch_update_address_tree_instruction_data_with_proof< let mut low_element_next_values = Vec::new(); let mut low_element_proofs: Vec> = Vec::new(); let non_inclusion_proofs = indexer - .get_multiple_new_address_proofs_full(merkle_tree_pubkey.to_bytes(), addresses.clone()) + .get_multiple_new_address_proofs_h40(merkle_tree_pubkey.to_bytes(), addresses.clone()) .await .unwrap(); for non_inclusion_proof in &non_inclusion_proofs { @@ -881,7 +878,6 @@ pub async fn create_batch_update_address_tree_instruction_data_with_proof< addresses, indexer .get_subtrees(merkle_tree_pubkey.to_bytes()) - .await .unwrap() .try_into() .unwrap(), diff --git a/sdk-libs/program-test/src/test_indexer.rs b/sdk-libs/program-test/src/test_indexer.rs deleted file mode 100644 index 3766dae797..0000000000 --- a/sdk-libs/program-test/src/test_indexer.rs +++ /dev/null @@ -1,664 +0,0 @@ -use std::{marker::PhantomData, time::Duration}; - -use account_compression::StateMerkleTreeAccount; -use anchor_lang::Discriminator; -use borsh::BorshDeserialize; -use forester_utils::get_concurrent_merkle_tree; -use light_batched_merkle_tree::{ - constants::{DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, DEFAULT_BATCH_STATE_TREE_HEIGHT}, - merkle_tree::BatchedMerkleTreeAccount, -}; -use light_client::{ - indexer::{ - AddressMerkleTreeAccounts, AddressMerkleTreeBundle, Indexer, StateMerkleTreeAccounts, - StateMerkleTreeBundle, - }, - rpc::{merkle_tree::MerkleTreeExt, RpcConnection}, - transaction_params::FeeConfig, -}; -use light_hasher::{Discriminator as LightDiscriminator, Poseidon}; -use light_indexed_merkle_tree::{array::IndexedArray, reference::IndexedMerkleTree}; -use light_merkle_tree_reference::MerkleTree; -use light_prover_client::{ - gnark::{ - combined_json_formatter::CombinedJsonStruct, - combined_json_formatter_legacy::CombinedJsonStruct as CombinedJsonStructLegacy, - constants::{PROVE_PATH, SERVER_ADDRESS}, - helpers::{ - big_int_to_string, health_check, spawn_prover, string_to_big_int, ProofType, - ProverConfig, - }, - inclusion_json_formatter::BatchInclusionJsonStruct, - inclusion_json_formatter_legacy::BatchInclusionJsonStruct as BatchInclusionJsonStructLegacy, - non_inclusion_json_formatter::BatchNonInclusionJsonStruct, - non_inclusion_json_formatter_legacy::BatchNonInclusionJsonStruct as BatchNonInclusionJsonStructLegacy, - proof_helpers::{compress_proof, deserialize_gnark_proof_json, proof_from_json_struct}, - }, - helpers::bigint_to_u8_32, - inclusion::merkle_inclusion_proof_inputs::{InclusionMerkleProofInputs, InclusionProofInputs}, - inclusion_legacy::merkle_inclusion_proof_inputs::InclusionProofInputs as InclusionProofInputsLegacy, - non_inclusion::merkle_non_inclusion_proof_inputs::{ - get_non_inclusion_proof_inputs, NonInclusionProofInputs, - }, - non_inclusion_legacy::merkle_non_inclusion_proof_inputs::NonInclusionProofInputs as NonInclusionProofInputsLegacy, -}; -use light_sdk::{ - compressed_account::CompressedAccountWithMerkleContext, - event::PublicTransactionEvent, - merkle_context::MerkleContext, - proof::{CompressedProof, ProofRpcResult}, - token::{TokenData, TokenDataWithMerkleContext}, - ADDRESS_MERKLE_TREE_CANOPY_DEPTH, ADDRESS_MERKLE_TREE_HEIGHT, PROGRAM_ID_LIGHT_SYSTEM, - STATE_MERKLE_TREE_CANOPY_DEPTH, STATE_MERKLE_TREE_HEIGHT, - TOKEN_COMPRESSED_ACCOUNT_DISCRIMINATOR, -}; -use light_utils::hashchain::create_hash_chain_from_slice; -use log::warn; -use num_bigint::BigInt; -use num_traits::FromBytes; -use reqwest::Client; -use solana_sdk::pubkey::Pubkey; - -#[derive(Debug)] -pub struct TestIndexer -where - R: RpcConnection + MerkleTreeExt, -{ - pub state_merkle_trees: Vec, - pub address_merkle_trees: Vec, - pub compressed_accounts: Vec, - pub nullified_compressed_accounts: Vec, - pub token_compressed_accounts: Vec, - pub token_nullified_compressed_accounts: Vec, - pub events: Vec, - _rpc: PhantomData, -} - -impl Indexer for TestIndexer -where - R: RpcConnection + MerkleTreeExt, -{ - fn add_event_and_compressed_accounts( - &mut self, - event: &PublicTransactionEvent, - ) -> ( - Vec, - Vec, - ) { - for hash in event.input_compressed_account_hashes.iter() { - let index = self.compressed_accounts.iter().position(|x| { - x.compressed_account - .hash::( - &x.merkle_context.merkle_tree_pubkey, - &x.merkle_context.leaf_index, - ) - .unwrap() - == *hash - }); - if let Some(index) = index { - self.nullified_compressed_accounts - .push(self.compressed_accounts[index].clone()); - self.compressed_accounts.remove(index); - continue; - }; - if index.is_none() { - let index = self - .token_compressed_accounts - .iter() - .position(|x| { - x.compressed_account - .compressed_account - .hash::( - &x.compressed_account.merkle_context.merkle_tree_pubkey, - &x.compressed_account.merkle_context.leaf_index, - ) - .unwrap() - == *hash - }) - .expect("input compressed account not found"); - self.token_nullified_compressed_accounts - .push(self.token_compressed_accounts[index].clone()); - self.token_compressed_accounts.remove(index); - } - } - - let mut compressed_accounts = Vec::new(); - let mut token_compressed_accounts = Vec::new(); - for (i, compressed_account) in event.output_compressed_accounts.iter().enumerate() { - let nullifier_queue_pubkey = self - .state_merkle_trees - .iter() - .find(|x| { - x.accounts.merkle_tree - == event.pubkey_array - [event.output_compressed_accounts[i].merkle_tree_index as usize] - }) - .unwrap() - .accounts - .nullifier_queue; - // if data is some, try to deserialize token data, if it fails, add to compressed_accounts - // if data is none add to compressed_accounts - // new accounts are inserted in front so that the newest accounts are found first - match compressed_account.compressed_account.data.as_ref() { - Some(data) => { - if compressed_account.compressed_account.owner == PROGRAM_ID_LIGHT_SYSTEM - && data.discriminator == TOKEN_COMPRESSED_ACCOUNT_DISCRIMINATOR - { - if let Ok(token_data) = TokenData::deserialize(&mut data.data.as_slice()) { - let token_account = TokenDataWithMerkleContext { - token_data, - compressed_account: CompressedAccountWithMerkleContext { - compressed_account: compressed_account - .compressed_account - .clone(), - merkle_context: MerkleContext { - leaf_index: event.output_leaf_indices[i], - merkle_tree_pubkey: event.pubkey_array[event - .output_compressed_accounts[i] - .merkle_tree_index - as usize], - nullifier_queue_pubkey, - queue_index: None, - }, - }, - }; - token_compressed_accounts.push(token_account.clone()); - self.token_compressed_accounts.insert(0, token_account); - } - } else { - let compressed_account = CompressedAccountWithMerkleContext { - compressed_account: compressed_account.compressed_account.clone(), - merkle_context: MerkleContext { - leaf_index: event.output_leaf_indices[i], - merkle_tree_pubkey: event.pubkey_array[event - .output_compressed_accounts[i] - .merkle_tree_index - as usize], - nullifier_queue_pubkey, - queue_index: None, - }, - }; - compressed_accounts.push(compressed_account.clone()); - self.compressed_accounts.insert(0, compressed_account); - } - } - None => { - let compressed_account = CompressedAccountWithMerkleContext { - compressed_account: compressed_account.compressed_account.clone(), - merkle_context: MerkleContext { - leaf_index: event.output_leaf_indices[i], - merkle_tree_pubkey: event.pubkey_array - [event.output_compressed_accounts[i].merkle_tree_index as usize], - nullifier_queue_pubkey, - queue_index: None, - }, - }; - compressed_accounts.push(compressed_account.clone()); - self.compressed_accounts.insert(0, compressed_account); - } - }; - let merkle_tree = &mut self - .state_merkle_trees - .iter_mut() - .find(|x| { - x.accounts.merkle_tree - == event.pubkey_array - [event.output_compressed_accounts[i].merkle_tree_index as usize] - }) - .unwrap() - .merkle_tree; - merkle_tree - .append( - &compressed_account - .compressed_account - .hash::( - &event.pubkey_array - [event.output_compressed_accounts[i].merkle_tree_index as usize], - &event.output_leaf_indices[i], - ) - .unwrap(), - ) - .expect("insert failed"); - } - - self.events.push(event.clone()); - (compressed_accounts, token_compressed_accounts) - } - - async fn create_proof_for_compressed_accounts( - &mut self, - compressed_accounts: Option<&[[u8; 32]]>, - state_merkle_tree_pubkeys: Option<&[solana_sdk::pubkey::Pubkey]>, - new_addresses: Option<&[[u8; 32]]>, - address_merkle_tree_pubkeys: Option>, - rpc: &mut R, - ) -> ProofRpcResult { - if compressed_accounts.is_some() - && ![1usize, 2usize, 3usize, 4usize, 8usize] - .contains(&compressed_accounts.unwrap().len()) - { - panic!( - "compressed_accounts must be of length 1, 2, 3, 4 or 8 != {}", - compressed_accounts.unwrap().len() - ) - } - if new_addresses.is_some() && ![1usize, 2usize].contains(&new_addresses.unwrap().len()) { - panic!("new_addresses must be of length 1, 2") - } - let client = Client::new(); - let (root_indices, address_root_indices, json_payload) = - match (compressed_accounts, new_addresses) { - (Some(accounts), None) => { - let (payload, payload_legacy, indices) = self - .process_inclusion_proofs(state_merkle_tree_pubkeys.unwrap(), accounts, rpc) - .await; - - if let Some(payload) = payload { - (indices, Vec::new(), payload.to_string()) - } else { - (indices, Vec::new(), payload_legacy.unwrap().to_string()) - } - } - (None, Some(addresses)) => { - let (payload, payload_legacy, indices) = self - .process_non_inclusion_proofs( - address_merkle_tree_pubkeys.unwrap().as_slice(), - addresses, - rpc, - ) - .await; - let payload_string = if let Some(payload) = payload { - payload.to_string() - } else { - payload_legacy.unwrap().to_string() - }; - (Vec::::new(), indices, payload_string) - } - (Some(accounts), Some(addresses)) => { - let (inclusion_payload, inclusion_payload_legacy, inclusion_indices) = self - .process_inclusion_proofs(state_merkle_tree_pubkeys.unwrap(), accounts, rpc) - .await; - - let ( - non_inclusion_payload, - non_inclusion_payload_legacy, - non_inclusion_indices, - ) = self - .process_non_inclusion_proofs( - address_merkle_tree_pubkeys.unwrap().as_slice(), - addresses, - rpc, - ) - .await; - let json_payload = if let Some(non_inclusion_payload) = non_inclusion_payload { - let public_input_hash = BigInt::from_bytes_be( - num_bigint::Sign::Plus, - &create_hash_chain_from_slice(&[ - bigint_to_u8_32( - &string_to_big_int( - &inclusion_payload.as_ref().unwrap().public_input_hash, - ) - .unwrap(), - ) - .unwrap(), - bigint_to_u8_32( - &string_to_big_int(&non_inclusion_payload.public_input_hash) - .unwrap(), - ) - .unwrap(), - ]) - .unwrap(), - ); - - CombinedJsonStruct { - circuit_type: ProofType::Combined.to_string(), - state_tree_height: DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, - address_tree_height: DEFAULT_BATCH_ADDRESS_TREE_HEIGHT, - public_input_hash: big_int_to_string(&public_input_hash), - inclusion: inclusion_payload.unwrap().inputs, - non_inclusion: non_inclusion_payload.inputs, - } - .to_string() - } else if let Some(non_inclusion_payload) = non_inclusion_payload_legacy { - CombinedJsonStructLegacy { - circuit_type: ProofType::Combined.to_string(), - state_tree_height: 26, - address_tree_height: 26, - inclusion: inclusion_payload_legacy.unwrap().inputs, - non_inclusion: non_inclusion_payload.inputs, - } - .to_string() - } else { - panic!("Unsupported tree height") - }; - (inclusion_indices, non_inclusion_indices, json_payload) - } - _ => { - panic!("At least one of compressed_accounts or new_addresses must be provided") - } - }; - - let mut retries = 3; - while retries > 0 { - let response_result = client - .post(format!("{}{}", SERVER_ADDRESS, PROVE_PATH)) - .header("Content-Type", "text/plain; charset=utf-8") - .body(json_payload.clone()) - .send() - .await - .expect("Failed to execute request."); - if response_result.status().is_success() { - let body = response_result.text().await.unwrap(); - let proof_json = deserialize_gnark_proof_json(&body).unwrap(); - let (proof_a, proof_b, proof_c) = proof_from_json_struct(proof_json); - let (proof_a, proof_b, proof_c) = compress_proof(&proof_a, &proof_b, &proof_c); - let root_indices = root_indices.iter().map(|x| Some(*x)).collect(); - return ProofRpcResult { - root_indices, - address_root_indices, - proof: CompressedProof { - a: proof_a, - b: proof_b, - c: proof_c, - }, - }; - } else { - warn!("Error: {}", response_result.text().await.unwrap()); - tokio::time::sleep(Duration::from_secs(1)).await; - retries -= 1; - } - } - panic!("Failed to get proof from server"); - } - - /// Returns compressed accounts owned by the given `owner`. - fn get_compressed_accounts_by_owner( - &self, - owner: &Pubkey, - ) -> Vec { - self.compressed_accounts - .iter() - .filter(|x| x.compressed_account.owner == *owner) - .cloned() - .collect() - } -} - -impl TestIndexer -where - R: RpcConnection + MerkleTreeExt, -{ - pub async fn new( - state_merkle_tree_accounts: &[StateMerkleTreeAccounts], - address_merkle_tree_accounts: &[AddressMerkleTreeAccounts], - inclusion: bool, - non_inclusion: bool, - ) -> Self { - let state_merkle_trees = state_merkle_tree_accounts - .iter() - .map(|accounts| { - let merkle_tree = Box::new(MerkleTree::::new( - STATE_MERKLE_TREE_HEIGHT, - STATE_MERKLE_TREE_CANOPY_DEPTH, - )); - StateMerkleTreeBundle { - accounts: *accounts, - merkle_tree, - rollover_fee: FeeConfig::default().state_merkle_tree_rollover, - } - }) - .collect::>(); - - let address_merkle_trees = address_merkle_tree_accounts - .iter() - .map(|accounts| Self::add_address_merkle_tree_bundle(accounts)) - .collect::>(); - - let mut prover_config = ProverConfig { - circuits: vec![], - run_mode: None, - }; - - if inclusion { - prover_config.circuits.push(ProofType::Inclusion); - } - if non_inclusion { - prover_config.circuits.push(ProofType::NonInclusion); - } - - spawn_prover(true, prover_config).await; - - health_check(20, 1).await; - - Self { - state_merkle_trees, - address_merkle_trees, - compressed_accounts: Vec::new(), - nullified_compressed_accounts: Vec::new(), - token_compressed_accounts: Vec::new(), - token_nullified_compressed_accounts: Vec::new(), - events: Vec::new(), - _rpc: PhantomData, - } - } - - pub fn add_address_merkle_tree_bundle( - accounts: &AddressMerkleTreeAccounts, - // TODO: add config here - ) -> AddressMerkleTreeBundle { - let mut merkle_tree = Box::new( - IndexedMerkleTree::::new( - ADDRESS_MERKLE_TREE_HEIGHT, - ADDRESS_MERKLE_TREE_CANOPY_DEPTH, - ) - .unwrap(), - ); - merkle_tree.init().unwrap(); - let mut indexed_array = Box::>::default(); - indexed_array.init().unwrap(); - AddressMerkleTreeBundle { - merkle_tree, - indexed_array, - accounts: *accounts, - rollover_fee: FeeConfig::default().address_queue_rollover, - } - } - - async fn process_inclusion_proofs( - &self, - merkle_tree_pubkeys: &[Pubkey], - accounts: &[[u8; 32]], - rpc: &mut R, - ) -> ( - Option, - Option, - Vec, - ) { - let mut inclusion_proofs = Vec::new(); - let mut root_indices = Vec::new(); - let mut height = 0; - - for (i, account) in accounts.iter().enumerate() { - let bundle = &self - .state_merkle_trees - .iter() - .find(|x| x.accounts.merkle_tree == merkle_tree_pubkeys[i]) - .unwrap(); - let merkle_tree = &bundle.merkle_tree; - let leaf_index = merkle_tree.get_leaf_index(account).unwrap(); - println!("merkle_tree height {:?}", merkle_tree.height); - let proof = merkle_tree.get_proof_of_leaf(leaf_index, true).unwrap(); - println!("proof length {:?}", proof.len()); - let merkle_tree_account = rpc - .get_account(merkle_tree_pubkeys[i]) - .await - .unwrap() - .unwrap(); - - let discriminator = merkle_tree_account.data[0..8].try_into().unwrap(); - let version = match discriminator { - StateMerkleTreeAccount::DISCRIMINATOR => 1, - BatchedMerkleTreeAccount::DISCRIMINATOR => 2, - _ => panic!("Unsupported discriminator"), - }; - println!("bundle.version {:?}", version); - if height == 0 { - height = merkle_tree.height; - } else { - assert_eq!(height, merkle_tree.height); - } - inclusion_proofs.push(InclusionMerkleProofInputs { - root: BigInt::from_be_bytes(merkle_tree.root().as_slice()), - leaf: BigInt::from_be_bytes(account), - path_index: BigInt::from_be_bytes(leaf_index.to_be_bytes().as_slice()), - path_elements: proof.iter().map(|x| BigInt::from_be_bytes(x)).collect(), - }); - let (root_index, root) = if version == 1 { - let fetched_merkle_tree = - get_concurrent_merkle_tree::( - rpc, - merkle_tree_pubkeys[i], - ) - .await; - // for i in 0..fetched_merkle_tree.roots.len() { - // inf!("roots {:?} {:?}", i, fetched_merkle_tree.roots[i]); - // } - // info!( - // "sequence number {:?}", - // fetched_merkle_tree.sequence_number() - // ); - // info!("root index {:?}", fetched_merkle_tree.root_index()); - // info!("local sequence number {:?}", merkle_tree.sequence_number); - ( - fetched_merkle_tree.root_index() as u32, - fetched_merkle_tree.root(), - ) - } else { - let mut merkle_tree_account = rpc - .get_account(merkle_tree_pubkeys[i]) - .await - .unwrap() - .unwrap(); - let merkle_tree = BatchedMerkleTreeAccount::state_tree_from_bytes_mut( - merkle_tree_account.data.as_mut_slice(), - ) - .unwrap(); - ( - merkle_tree.get_root_index(), - merkle_tree.get_root().unwrap(), - ) - }; - assert_eq!(merkle_tree.root(), root, "Merkle tree root mismatch"); - - root_indices.push(root_index as u16); - } - - let (batch_inclusion_proof_inputs, legacy) = if height - == DEFAULT_BATCH_STATE_TREE_HEIGHT as usize - { - let inclusion_proof_inputs = - InclusionProofInputs::new(inclusion_proofs.as_slice()).unwrap(); - ( - Some(BatchInclusionJsonStruct::from_inclusion_proof_inputs( - &inclusion_proof_inputs, - )), - None, - ) - } else if height == STATE_MERKLE_TREE_HEIGHT { - let inclusion_proof_inputs = InclusionProofInputsLegacy(inclusion_proofs.as_slice()); - ( - None, - Some(BatchInclusionJsonStructLegacy::from_inclusion_proof_inputs( - &inclusion_proof_inputs, - )), - ) - } else { - panic!("Unsupported tree height") - }; - - (batch_inclusion_proof_inputs, legacy, root_indices) - } - - async fn process_non_inclusion_proofs( - &self, - address_merkle_tree_pubkeys: &[Pubkey], - addresses: &[[u8; 32]], - rpc: &mut R, - ) -> ( - Option, - Option, - Vec, - ) { - let mut non_inclusion_proofs = Vec::new(); - let mut address_root_indices = Vec::new(); - let mut tree_heights = Vec::new(); - for (i, address) in addresses.iter().enumerate() { - let address_tree = &self - .address_merkle_trees - .iter() - .find(|x| x.accounts.merkle_tree == address_merkle_tree_pubkeys[i]) - .unwrap(); - tree_heights.push(address_tree.merkle_tree.merkle_tree.height); - let proof_inputs = get_non_inclusion_proof_inputs( - address, - &address_tree.merkle_tree, - &address_tree.indexed_array, - ); - non_inclusion_proofs.push(proof_inputs); - let onchain_address_merkle_tree = rpc - .get_address_merkle_tree(address_merkle_tree_pubkeys[i]) - .await - .unwrap(); - address_root_indices.push(onchain_address_merkle_tree.root_index() as u16); - } - // if tree heights are not the same, panic - if tree_heights.iter().any(|&x| x != tree_heights[0]) { - panic!( - "All address merkle trees must have the same height {:?}", - tree_heights - ); - } - - let (batch_non_inclusion_proof_inputs, batch_non_inclusion_proof_inputs_legacy) = - if tree_heights[0] == 26 { - let non_inclusion_proof_inputs = - NonInclusionProofInputsLegacy::new(non_inclusion_proofs.as_slice()); - ( - None, - Some( - BatchNonInclusionJsonStructLegacy::from_non_inclusion_proof_inputs( - &non_inclusion_proof_inputs, - ), - ), - ) - } else if tree_heights[0] == 40 { - let non_inclusion_proof_inputs = - NonInclusionProofInputs::new(non_inclusion_proofs.as_slice()).unwrap(); - ( - Some( - BatchNonInclusionJsonStruct::from_non_inclusion_proof_inputs( - &non_inclusion_proof_inputs, - ), - ), - None, - ) - } else { - panic!("Unsupported tree height") - }; - ( - batch_non_inclusion_proof_inputs, - batch_non_inclusion_proof_inputs_legacy, - address_root_indices, - ) - } - - /// deserializes an event - /// adds the output_compressed_accounts to the compressed_accounts - /// removes the input_compressed_accounts from the compressed_accounts - /// adds the input_compressed_accounts to the nullified_compressed_accounts - /// deserialiazes token data from the output_compressed_accounts - /// adds the token_compressed_accounts to the token_compressed_accounts - pub fn add_compressed_accounts_with_token_data(&mut self, event: &PublicTransactionEvent) { - self.add_event_and_compressed_accounts(event); - } -} diff --git a/sdk-libs/sdk/Cargo.toml b/sdk-libs/sdk/Cargo.toml index 93a48b9fab..70ae8d8301 100644 --- a/sdk-libs/sdk/Cargo.toml +++ b/sdk-libs/sdk/Cargo.toml @@ -24,45 +24,24 @@ idl-build = ["anchor-lang/idl-build"] legacy = ["account-compression", "light-system-program"] [dependencies] -# Solana -solana-program = { workspace = true } -# Anchor +solana-program = { workspace = true } anchor-lang = { workspace = true } - -# Math and crypto num-bigint = { workspace = true } +borsh = "0.10.0" -aligned-sized = { workspace = true } light-macros = { workspace = true } light-sdk-macros = { workspace = true } -bytemuck = "1.17" light-hasher = { workspace = true, features=["solana"] } light-heap = { workspace = true, optional = true } light-indexed-merkle-tree = { workspace = true } account-compression = { workspace = true , optional = true } light-system-program = { workspace = true , optional = true } -light-concurrent-merkle-tree = { workspace = true } light-utils = { workspace = true } -groth16-solana = "0.0.3" -light-verifier = { workspace = true, features = ["solana"] } -borsh = "0.10.0" [target.'cfg(not(target_os = "solana"))'.dependencies] solana-sdk = { workspace = true } [dev-dependencies] -solana-banks-interface = { workspace = true } -solana-cli-output = { workspace = true } -solana-program-test = { workspace = true } -serde_json = "1.0.133" -reqwest = "0.12" -tokio = { workspace = true } -light-prover-client = { workspace = true } -light-merkle-tree-reference = { workspace = true } light-indexed-merkle-tree = { workspace = true } num-bigint = "0.4.6" -num-traits = "0.2.19" -lazy_static = "1.4.0" -light-hash-set = { workspace = true, features = ["solana"] } -rand = "0.8.5" diff --git a/sdk-libs/sdk/src/proof.rs b/sdk-libs/sdk/src/proof.rs index def6921375..f9c6c8be57 100644 --- a/sdk-libs/sdk/src/proof.rs +++ b/sdk-libs/sdk/src/proof.rs @@ -35,9 +35,27 @@ pub struct CompressedProof { pub c: [u8; 32], } +impl Default for CompressedProof { + fn default() -> Self { + Self { + a: [0; 32], + b: [0; 64], + c: [0; 32], + } + } +} + #[derive(Debug, AnchorDeserialize, AnchorSerialize)] pub struct ProofRpcResult { pub proof: CompressedProof, pub root_indices: Vec>, pub address_root_indices: Vec, } + +#[derive(Debug, Default)] +pub struct BatchedTreeProofRpcResult { + pub proof: Option, + // If none -> proof by index and not included in zkp, else included in zkp + pub root_indices: Vec>, + pub address_root_indices: Vec, +}