diff --git a/forester/src/cli.rs b/forester/src/cli.rs index 1921b636d4..6c9f65702a 100644 --- a/forester/src/cli.rs +++ b/forester/src/cli.rs @@ -1,7 +1,5 @@ use clap::{Parser, Subcommand}; -use crate::forester_stats; - #[derive(Parser)] #[clap(author, version, about, long_about = None)] pub struct Cli { @@ -14,7 +12,6 @@ pub struct Cli { pub enum Commands { Start(StartArgs), Status(StatusArgs), - ForesterStats(forester_stats::Options), } #[derive(Parser, Clone, Debug)] @@ -117,6 +114,13 @@ pub struct StatusArgs { #[arg(long, env = "FORESTER_PUSH_GATEWAY_URL")] pub push_gateway_url: Option, + /// Select to run compressed token program tests. + #[clap(long)] + pub full: bool, + #[clap(long)] + pub protocol_config: bool, + #[clap(long, default_value_t = true)] + pub queue: bool, } impl StartArgs { diff --git a/forester/src/forester_stats.rs b/forester/src/forester_status.rs similarity index 61% rename from forester/src/forester_stats.rs rename to forester/src/forester_status.rs index 5cc0e928d0..34d3d4acf8 100644 --- a/forester/src/forester_stats.rs +++ b/forester/src/forester_status.rs @@ -1,23 +1,20 @@ -use account_compression::{AddressMerkleTreeAccount, QueueAccount, StateMerkleTreeAccount}; use anchor_lang::{AccountDeserialize, Discriminator}; -use clap::Parser; -use light_concurrent_merkle_tree::copy::ConcurrentMerkleTreeCopy; -use light_hash_set::HashSet; -use light_hasher::Poseidon; +use forester_utils::forester_epoch::TreeType; +use light_client::rpc::{RpcConnection, SolanaRpcConnection}; use light_registry::{protocol_config::state::ProtocolConfigPda, EpochPda, ForesterEpochPda}; use solana_sdk::{account::ReadableAccount, commitment_config::CommitmentConfig}; -#[derive(Debug, Parser)] -pub struct Options { - /// Select to run compressed token program tests. - #[clap(long)] - full: bool, - #[clap(long)] - protocol_config: bool, - #[clap(long, default_value_t = true)] - queue: bool, -} +use std::sync::Arc; +use tracing::{debug, warn}; + +use crate::{ + cli::StatusArgs, + metrics::{push_metrics, register_metrics}, + run_queue_info, + tree_data_sync::fetch_trees, + ForesterConfig, +}; -pub fn fetch_foreter_stats(opts: &Options) { +pub async fn fetch_foreter_status(opts: &StatusArgs) { let commitment_config = CommitmentConfig::confirmed(); let rpc_url = std::env::var("RPC_URL") .expect("RPC_URL environment variable not set, export RPC_URL="); @@ -143,57 +140,23 @@ pub fn fetch_foreter_stats(opts: &Options) { if opts.protocol_config { println!("protocol config: {:?}", protocol_config_pdas[0]); } - if opts.queue { - let account_compression_accounts = client - .get_program_accounts(&account_compression::ID) - .expect("Failed to fetch accounts for account compression program."); - for (pubkey, mut account) in account_compression_accounts { - match account.data()[0..8].try_into().unwrap() { - QueueAccount::DISCRIMINATOR => { - unsafe { - let queue = HashSet::from_bytes_copy( - &mut account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); + let config = Arc::new(ForesterConfig::new_for_status(opts).unwrap()); - println!("Queue account: {:?}", pubkey); - let mut num_of_marked_items = 0; - for i in 0..queue.get_capacity() { - if queue.get_unmarked_bucket(i).is_some() { - num_of_marked_items += 1; - } - } - println!( - "queue num of unmarked items: {:?} / {}", - num_of_marked_items, - queue.get_capacity() / 2 // div by 2 because only half of the hash set can be used before tx start to fail - ); - } - } - StateMerkleTreeAccount::DISCRIMINATOR => { - println!("State Merkle tree: {:?}", pubkey); - let merkle_tree = ConcurrentMerkleTreeCopy::::from_bytes_copy( - &account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); - println!( - "State Merkle tree next index {:?}", - merkle_tree.next_index() - ); - } - AddressMerkleTreeAccount::DISCRIMINATOR => { - println!("Address Merkle tree: {:?}", pubkey); - let merkle_tree = ConcurrentMerkleTreeCopy::::from_bytes_copy( - &account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); - println!( - "Address Merkle tree next index {:?}", - merkle_tree.next_index() - ); - } - _ => (), - } - } + if config.general_config.enable_metrics { + register_metrics(); } + + debug!("Fetching trees..."); + debug!("RPC URL: {}", config.external_services.rpc_url); + let rpc = SolanaRpcConnection::new(config.external_services.rpc_url.clone(), None); + let trees = fetch_trees(&rpc).await; + if trees.is_empty() { + warn!("No trees found. Exiting."); + } + run_queue_info(config.clone(), trees.clone(), TreeType::State).await; + run_queue_info(config.clone(), trees.clone(), TreeType::Address).await; + + push_metrics(&config.external_services.pushgateway_url) + .await + .unwrap(); } diff --git a/forester/src/lib.rs b/forester/src/lib.rs index 5cbd6f58ec..baa9e2a10e 100644 --- a/forester/src/lib.rs +++ b/forester/src/lib.rs @@ -4,7 +4,7 @@ pub mod cli; pub mod config; pub mod epoch_manager; pub mod errors; -pub mod forester_stats; +pub mod forester_status; pub mod metrics; pub mod photon_indexer; pub mod pubsub_client; diff --git a/forester/src/main.rs b/forester/src/main.rs index 8800490a00..f5d35060a6 100644 --- a/forester/src/main.rs +++ b/forester/src/main.rs @@ -1,17 +1,15 @@ use clap::Parser; use forester::cli::{Cli, Commands}; use forester::errors::ForesterError; -use forester::metrics::{push_metrics, register_metrics}; +use forester::metrics::register_metrics; use forester::photon_indexer::PhotonIndexer; use forester::telemetry::setup_telemetry; -use forester::tree_data_sync::fetch_trees; -use forester::{forester_stats, run_pipeline, run_queue_info, ForesterConfig}; -use forester_utils::forester_epoch::TreeType; +use forester::{forester_status, run_pipeline, ForesterConfig}; use light_client::rpc::{RpcConnection, SolanaRpcConnection}; use std::sync::Arc; use tokio::signal::ctrl_c; use tokio::sync::{mpsc, oneshot}; -use tracing::{debug, warn}; +use tracing::debug; #[tokio::main] async fn main() -> Result<(), ForesterError> { @@ -54,25 +52,8 @@ async fn main() -> Result<(), ForesterError> { run_pipeline(config, indexer, shutdown_receiver, work_report_sender).await? } Commands::Status(args) => { - let config = Arc::new(ForesterConfig::new_for_status(args)?); - - if config.general_config.enable_metrics { - register_metrics(); - } - - debug!("Fetching trees..."); - debug!("RPC URL: {}", config.external_services.rpc_url); - let rpc = SolanaRpcConnection::new(config.external_services.rpc_url.clone(), None); - let trees = fetch_trees(&rpc).await; - if trees.is_empty() { - warn!("No trees found. Exiting."); - } - run_queue_info(config.clone(), trees.clone(), TreeType::State).await; - run_queue_info(config.clone(), trees.clone(), TreeType::Address).await; - - push_metrics(&config.external_services.pushgateway_url).await?; + forester_status::fetch_foreter_status(args).await; } - Commands::ForesterStats(opts) => forester_stats::fetch_foreter_stats(opts), } Ok(()) } diff --git a/xtask/src/forester_stats.rs b/xtask/src/forester_stats.rs deleted file mode 100644 index 9dcc66d926..0000000000 --- a/xtask/src/forester_stats.rs +++ /dev/null @@ -1,201 +0,0 @@ -use account_compression::{AddressMerkleTreeAccount, QueueAccount, StateMerkleTreeAccount}; -use anchor_lang::{AccountDeserialize, Discriminator}; -use clap::Parser; -use light_concurrent_merkle_tree::copy::ConcurrentMerkleTreeCopy; -use light_hash_set::HashSet; -use light_hasher::Poseidon; -use light_registry::{protocol_config::state::ProtocolConfigPda, EpochPda, ForesterEpochPda}; -use solana_sdk::{account::ReadableAccount, commitment_config::CommitmentConfig}; -#[derive(Debug, Parser)] -pub struct Options { - /// Select to run compressed token program tests. - #[clap(long)] - full: bool, - #[clap(long)] - protocol_config: bool, - #[clap(long, default_value_t = true)] - queue: bool, -} - -pub fn fetch_foreter_stats(opts: Options) -> anyhow::Result<()> { - let commitment_config = CommitmentConfig::confirmed(); - let rpc_url = std::env::var("RPC_URL") - .expect("RPC_URL environment variable not set, export RPC_URL="); - - let client = - solana_client::rpc_client::RpcClient::new_with_commitment(rpc_url, commitment_config); - let registry_accounts = client - .get_program_accounts(&light_registry::ID) - .expect("Failed to fetch accounts for registry program."); - - let mut forester_epoch_pdas = vec![]; - let mut epoch_pdas = vec![]; - let mut protocol_config_pdas = vec![]; - for (_, account) in registry_accounts { - match account.data()[0..8].try_into().unwrap() { - ForesterEpochPda::DISCRIMINATOR => { - let forester_epoch_pda = - ForesterEpochPda::try_deserialize_unchecked(&mut account.data()) - .expect("Failed to deserialize ForesterEpochPda"); - forester_epoch_pdas.push(forester_epoch_pda); - } - EpochPda::DISCRIMINATOR => { - let epoch_pda = EpochPda::try_deserialize_unchecked(&mut account.data()) - .expect("Failed to deserialize EpochPda"); - epoch_pdas.push(epoch_pda); - } - ProtocolConfigPda::DISCRIMINATOR => { - let protocol_config_pda = - ProtocolConfigPda::try_deserialize_unchecked(&mut account.data()) - .expect("Failed to deserialize ProtocolConfigPda"); - protocol_config_pdas.push(protocol_config_pda); - } - _ => (), - } - } - forester_epoch_pdas.sort_by(|a, b| a.epoch.cmp(&b.epoch)); - epoch_pdas.sort_by(|a, b| a.epoch.cmp(&b.epoch)); - let slot = client.get_slot().expect("Failed to fetch slot."); - let current_active_epoch = protocol_config_pdas[0] - .config - .get_current_active_epoch(slot) - .unwrap(); - let current_registration_epoch = protocol_config_pdas[0] - .config - .get_latest_register_epoch(slot) - .unwrap(); - println!("Current active epoch: {:?}", current_active_epoch); - - println!( - "Current registration epoch: {:?}", - current_registration_epoch - ); - - println!( - "Forester registered for latest epoch: {:?}", - forester_epoch_pdas - .iter() - .any(|pda| pda.epoch == current_registration_epoch) - ); - println!( - "Forester registered for active epoch: {:?}", - forester_epoch_pdas - .iter() - .any(|pda| pda.epoch == current_active_epoch) - ); - println!( - "current active epoch progress {:?} / {}", - protocol_config_pdas[0] - .config - .get_current_active_epoch_progress(slot), - protocol_config_pdas[0].config.active_phase_length - ); - println!( - "current active epoch progress {:?}%", - protocol_config_pdas[0] - .config - .get_current_active_epoch_progress(slot) as f64 - / protocol_config_pdas[0].config.active_phase_length as f64 - * 100f64 - ); - println!("Hours until next epoch : {:?} hours", { - // slotduration is 460ms and 1000ms is 1 second and 3600 seconds is 1 hour - protocol_config_pdas[0] - .config - .active_phase_length - .saturating_sub( - protocol_config_pdas[0] - .config - .get_current_active_epoch_progress(slot), - ) - * 460 - / 1000 - / 3600 - }); - let slots_until_next_registration = protocol_config_pdas[0] - .config - .registration_phase_length - .saturating_sub( - protocol_config_pdas[0] - .config - .get_current_active_epoch_progress(slot), - ); - println!( - "Slots until next registration : {:?}", - slots_until_next_registration - ); - println!( - "Hours until next registration : {:?} hours", - // slotduration is 460ms and 1000ms is 1 second and 3600 seconds is 1 hour - slots_until_next_registration * 460 / 1000 / 3600 - ); - if opts.full { - for epoch in &epoch_pdas { - println!("Epoch: {:?}", epoch.epoch); - let registered_foresters_in_epoch = forester_epoch_pdas - .iter() - .filter(|pda| pda.epoch == epoch.epoch); - for forester in registered_foresters_in_epoch { - println!("Forester authority: {:?}", forester.authority); - } - } - } - if opts.protocol_config { - println!("protocol config: {:?}", protocol_config_pdas[0]); - } - if opts.queue { - let account_compression_accounts = client - .get_program_accounts(&account_compression::ID) - .expect("Failed to fetch accounts for account compression program."); - for (pubkey, mut account) in account_compression_accounts { - match account.data()[0..8].try_into().unwrap() { - QueueAccount::DISCRIMINATOR => { - unsafe { - let queue = HashSet::from_bytes_copy( - &mut account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); - - println!("Queue account: {:?}", pubkey); - let mut num_of_marked_items = 0; - for i in 0..queue.get_capacity() { - if queue.get_unmarked_bucket(i).is_some() { - num_of_marked_items += 1; - } - } - println!( - "queue num of unmarked items: {:?} / {}", - num_of_marked_items, - queue.get_capacity() / 2 // div by 2 because only half of the hash set can be used before tx start to fail - ); - } - } - StateMerkleTreeAccount::DISCRIMINATOR => { - println!("State Merkle tree: {:?}", pubkey); - let merkle_tree = ConcurrentMerkleTreeCopy::::from_bytes_copy( - &account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); - println!( - "State Merkle tree next index {:?}", - merkle_tree.next_index() - ); - } - AddressMerkleTreeAccount::DISCRIMINATOR => { - println!("Address Merkle tree: {:?}", pubkey); - let merkle_tree = ConcurrentMerkleTreeCopy::::from_bytes_copy( - &account.data[8 + std::mem::size_of::()..], - ) - .unwrap(); - println!( - "Address Merkle tree next index {:?}", - merkle_tree.next_index() - ); - } - _ => (), - } - } - } - - Ok(()) -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index efc6f9bd8f..b17e6b89e6 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -3,7 +3,6 @@ use clap::{Parser, ValueEnum}; mod bench; mod create_vkeyrs_from_gnark_key; mod fee; -mod forester_stats; mod hash_set; mod type_sizes; mod zero_bytes; @@ -39,8 +38,6 @@ enum Command { Fee, /// Hash set utilities. HashSet(hash_set::HashSetOptions), - /// Hash set utilities. - ForesterStats(forester_stats::Options), } fn main() -> Result<(), anyhow::Error> { @@ -58,6 +55,5 @@ fn main() -> Result<(), anyhow::Error> { Command::Bench(opts) => bench::bench(opts), Command::Fee => fee::fees(), Command::HashSet(opts) => hash_set::hash_set(opts), - Command::ForesterStats(opts) => forester_stats::fetch_foreter_stats(opts), } }