diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 62678b2a2..4aa76361b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -79,3 +79,9 @@ jobs: # TODO: until we have more comprehensive cli parsers, we will need to run tests separately. cargo test sdk_tests -- --nocapture working-directory: rust/integration-tests + + # Run all Tests + - name: Run Sanity Tests + run: | + cargo test regression_tests -- --nocapture + working-directory: rust/integration-tests diff --git a/rust/integration-tests/src/diff_tests/all_tests.rs b/rust/integration-tests/src/diff_tests/all_tests.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/rust/integration-tests/src/lib.rs b/rust/integration-tests/src/lib.rs index 09f62ed28..65b2e6f96 100644 --- a/rust/integration-tests/src/lib.rs +++ b/rust/integration-tests/src/lib.rs @@ -14,8 +14,7 @@ use testcontainers::{ mod diff_test_helper; mod models; -mod scenarios_tests; -#[cfg(test)] +mod sanity_tests; mod sdk_tests; use std::time::Duration; diff --git a/rust/integration-tests/src/sanity_tests/mod.rs b/rust/integration-tests/src/sanity_tests/mod.rs new file mode 100644 index 000000000..421c84bb7 --- /dev/null +++ b/rust/integration-tests/src/sanity_tests/mod.rs @@ -0,0 +1,157 @@ +mod sanity_tests; + +use crate::sdk_tests::run_processor_test; +use aptos_indexer_testing_framework::sdk_test_context::SdkTestContext; +use diesel::PgConnection; +use sdk_processor::processors::{ + account_transactions_processor::AccountTransactionsProcessor, ans_processor::AnsProcessor, + default_processor::DefaultProcessor, events_processor::EventsProcessor, + fungible_asset_processor::FungibleAssetProcessor, objects_processor::ObjectsProcessor, + stake_processor::StakeProcessor, token_v2_processor::TokenV2Processor, + user_transaction_processor::UserTransactionProcessor, +}; +use serde_json::Value; +use std::collections::HashMap; + +/// Wrapper for the different processors to run the tests +#[allow(dead_code)] +pub enum ProcessorWrapper { + EventsProcessor(EventsProcessor), + FungibleAssetProcessor(FungibleAssetProcessor), + AnsProcessor(AnsProcessor), + DefaultProcessor(DefaultProcessor), + ObjectsProcessor(ObjectsProcessor), + StakeProcessor(StakeProcessor), + UserTransactionProcessor(UserTransactionProcessor), + TokenV2Processor(TokenV2Processor), + AccountTransactionsProcessor(AccountTransactionsProcessor), +} + +#[allow(dead_code)] +impl ProcessorWrapper { + async fn run( + self, + test_context: &mut SdkTestContext, + db_values_fn: F, + db_url: String, + diff_flag: bool, + output_path: String, + ) -> anyhow::Result> + where + F: Fn(&mut PgConnection, Vec) -> anyhow::Result> + + Send + + Sync + + 'static, + { + match self { + ProcessorWrapper::EventsProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::FungibleAssetProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::AnsProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::DefaultProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::ObjectsProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::StakeProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::UserTransactionProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::TokenV2Processor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + ProcessorWrapper::AccountTransactionsProcessor(processor) => { + run_processor_test( + test_context, + processor, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + None, + ) + .await + }, + } + } +} diff --git a/rust/integration-tests/src/sanity_tests/sanity_tests.rs b/rust/integration-tests/src/sanity_tests/sanity_tests.rs new file mode 100644 index 000000000..3aa0b0f72 --- /dev/null +++ b/rust/integration-tests/src/sanity_tests/sanity_tests.rs @@ -0,0 +1,216 @@ +#[allow(clippy::needless_return)] +#[cfg(test)] +mod tests { + use crate::{ + diff_test_helper::{ + account_transaction_processor::load_data as load_acc_txn_data, + ans_processor::load_data as load_ans_data, + default_processor::load_data as load_default_data, + event_processor::load_data as load_event_data, + fungible_asset_processor::load_data as load_fungible_asset_data, + objects_processor::load_data as load_object_data, + stake_processor::load_data as load_stake_data, + token_v2_processor::load_data as load_token_v2_data, + user_transaction_processor::load_data as load_ut_data, + }, + sanity_tests::ProcessorWrapper, + sdk_tests::{ + account_transaction_processor_tests::setup_acc_txn_processor_config, + ans_processor_tests::setup_ans_processor_config, + default_processor_tests::setup_default_processor_config, + events_processor_tests::setup_events_processor_config, + fungible_asset_processor_tests::setup_fa_processor_config, + objects_processor_tests::setup_objects_processor_config, setup_test_environment, + stake_processor_tests::setup_stake_processor_config, + token_v2_processor_tests::setup_token_v2_processor_config, + user_transaction_processor_tests::setup_user_txn_processor_config, + }, + }; + use aptos_indexer_test_transactions::{ALL_IMPORTED_MAINNET_TXNS, ALL_IMPORTED_TESTNET_TXNS, ALL_SCRIPTED_TRANSACTIONS}; + use aptos_indexer_testing_framework::{ + cli_parser::get_test_config, database::TestDatabase, sdk_test_context::SdkTestContext, + }; + use diesel::pg::PgConnection; + use sdk_processor::processors::{ + account_transactions_processor::AccountTransactionsProcessor, ans_processor::AnsProcessor, + default_processor::DefaultProcessor, events_processor::EventsProcessor, + fungible_asset_processor::FungibleAssetProcessor, objects_processor::ObjectsProcessor, + stake_processor::StakeProcessor, token_v2_processor::TokenV2Processor, + user_transaction_processor::UserTransactionProcessor, + }; + use serde_json::Value; + use std::collections::HashMap; + + const DEFAULT_OUTPUT_FOLDER: &str = "expected_db_output_files"; + + #[tokio::test] + async fn test_all_testnet_txns_for_all_processors() { + let (diff_flag, custom_output_path) = get_test_config(); + let output_path = custom_output_path + .unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string() + "/imported_testnet_txns"); + + let (db, test_context) = setup_test_environment(ALL_IMPORTED_TESTNET_TXNS).await; + let db_url = db.get_db_url(); + + run_processors_with_test_context(test_context, db_url, diff_flag, output_path).await; + } + + #[tokio::test] + async fn test_all_mainnet_txns_for_all_processors() { + let (diff_flag, custom_output_path) = get_test_config(); + let output_path = custom_output_path + .unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string() + "/imported_testnet_txns"); + + let (db, test_context) = setup_test_environment(ALL_IMPORTED_MAINNET_TXNS).await; + let db_url = db.get_db_url(); + + run_processors_with_test_context(test_context, db_url, diff_flag, output_path).await; + } + + #[tokio::test] + async fn test_all_scripted_txns_for_all_processors() { + let (diff_flag, custom_output_path) = get_test_config(); + let output_path = custom_output_path + .unwrap_or_else(|| DEFAULT_OUTPUT_FOLDER.to_string() + "/imported_testnet_txns"); + + let (db, test_context) = setup_test_environment(ALL_SCRIPTED_TRANSACTIONS).await; + let db_url = db.get_db_url(); + + run_processors_with_test_context(test_context, db_url, diff_flag, output_path).await; + } + + async fn run_processors_with_test_context( + mut test_context: SdkTestContext, + db_url: String, + diff_flag: bool, + output_path: String, + ) { + let processors_map: HashMap = [ + ( + "events_processor".to_string(), + ProcessorWrapper::EventsProcessor( + EventsProcessor::new(setup_events_processor_config(&test_context, &db_url).0) + .await + .expect("Failed to create EventsProcessor"), + ), + ), + ( + "fungible_asset_processor".to_string(), + ProcessorWrapper::FungibleAssetProcessor( + FungibleAssetProcessor::new( + setup_fa_processor_config(&test_context, &db_url).0, + ) + .await + .expect("Failed to create FungibleAssetProcessor"), + ), + ), + ( + "ans_processor".to_string(), + ProcessorWrapper::AnsProcessor( + AnsProcessor::new(setup_ans_processor_config(&test_context, &db_url).0) + .await + .expect("Failed to create AnsProcessor"), + ), + ), + ( + "default_processor".to_string(), + ProcessorWrapper::DefaultProcessor( + DefaultProcessor::new(setup_default_processor_config(&test_context, &db_url).0) + .await + .expect("Failed to create DefaultProcessor"), + ), + ), + ( + "objects_processor".to_string(), + ProcessorWrapper::ObjectsProcessor( + ObjectsProcessor::new(setup_objects_processor_config(&test_context, &db_url).0) + .await + .expect("Failed to create ObjectsProcessor"), + ), + ), + ( + "stake_processor".to_string(), + ProcessorWrapper::StakeProcessor( + StakeProcessor::new(setup_stake_processor_config(&test_context, &db_url).0) + .await + .expect("Failed to create StakeProcessor"), + ), + ), + ( + "user_transactions_processor".to_string(), + ProcessorWrapper::UserTransactionProcessor( + UserTransactionProcessor::new( + setup_user_txn_processor_config(&test_context, &db_url).0, + ) + .await + .expect("Failed to create UserTransactionProcessor"), + ), + ), + ( + "token_v2_processor".to_string(), + ProcessorWrapper::TokenV2Processor( + TokenV2Processor::new( + setup_token_v2_processor_config(&test_context, &db_url).0, + ) + .await + .expect("Failed to create TokenV2Processor"), + ), + ), + ( + "account_transactions_processor".to_string(), + ProcessorWrapper::AccountTransactionsProcessor( + AccountTransactionsProcessor::new( + setup_acc_txn_processor_config(&test_context, &db_url).0, + ) + .await + .expect("Failed to create AccountTransactionProcessor"), + ), + ), + ] + .into_iter() + .collect(); + + // Loop through all processors and run tests + for (processor_name, processor) in processors_map { + let db_values_fn = get_db_values_fn_for_processor(&processor_name); + + match processor + .run( + &mut test_context, + db_values_fn, + db_url.clone(), + diff_flag, + output_path.clone(), + ) + .await + { + Ok(_) => { + println!("Processor ran successfully for {}", processor_name); + }, + Err(e) => { + panic!( + "Error running processor test for {} with error: {}", + processor_name, e + ); + }, + } + } + } + + fn get_db_values_fn_for_processor( + processor_name: &str, + ) -> fn(&mut PgConnection, Vec) -> anyhow::Result> { + match processor_name { + "events_processor" => load_event_data, + "fungible_asset_processor" => load_fungible_asset_data, + "ans_processor" => load_ans_data, + "default_processor" => load_default_data, + "objects_processor" => load_object_data, + "stake_processor" => load_stake_data, + "user_transactions_processor" => load_ut_data, + "token_v2_processor" => load_token_v2_data, + "account_transactions_processor" => load_acc_txn_data, + _ => panic!("Unknown processor: {}", processor_name), + } + } +} diff --git a/rust/integration-tests/src/scenarios_tests/event_processor_tests.rs b/rust/integration-tests/src/scenarios_tests/event_processor_tests.rs deleted file mode 100644 index 727e967b3..000000000 --- a/rust/integration-tests/src/scenarios_tests/event_processor_tests.rs +++ /dev/null @@ -1,205 +0,0 @@ -#[allow(clippy::needless_return)] -#[cfg(test)] -mod test { - use crate::{ScenarioTest, TestContext, TestProcessorConfig, TestType}; - use aptos_indexer_test_transactions::{ - IMPORTED_TESTNET_TXNS_5523474016_VALIDATOR_TXN, - IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER, - IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES, - }; - use diesel::{ - pg::PgConnection, - query_dsl::methods::{FilterDsl, SelectDsl}, - BoolExpressionMethods, ExpressionMethods, QueryResult, RunQueryDsl, - }; - use processor::schema::events::dsl::*; - - const FA_WITHDRAW_EVENT: &str = "0x1::fungible_asset::Withdraw"; - const FA_DEPOSIT_EVENT: &str = "0x1::fungible_asset::Deposit"; - const FEE_STATEMENT_EVENT: &str = "0x1::transaction_fee::FeeStatement"; - const DISTRIBUTE_REWARDS_EVENT: &str = "0x1::stake::DistributeRewardsEvent"; - - // Test Case: Validate the transaction that funds an account A from the faucet parsed into the correct events. - // - Verifies that the faucet funding results in one WithdrawEvent and two DepositEvents. - #[tokio::test] - async fn test_fungible_asset_withdraw_deposit_events() { - let test_context = TestContext::new(&[IMPORTED_TESTNET_TXNS_5992795934_FA_ACTIVITIES]) - .await - .unwrap(); - let processor_config = TestProcessorConfig { - config: processor::processors::ProcessorConfig::EventsProcessor, - }; - let expected_transaction = test_context.transaction_batches[0].clone(); - let test_type = TestType::Scenario(ScenarioTest); - - assert!(test_context - .run( - processor_config, - test_type, - move |conn: &mut PgConnection, version: &str| { - // Load and validate events - let withdraw_events = load_transaction_events( - conn, - version.parse::().unwrap(), - FA_WITHDRAW_EVENT, - ) - .expect("Failed to load WithdrawEvent"); - assert_eq!(withdraw_events.len(), 1); - - validate_event( - &withdraw_events[0], - expected_transaction.version as i64, - "0x0000000000000000000000000000000000000000000000000000000000000000", - 0, - FA_WITHDRAW_EVENT, - ); - - let deposit_events = load_transaction_events( - conn, - version.parse::().unwrap(), - FA_DEPOSIT_EVENT, - ) - .expect("Failed to load DepositEvent"); - assert_eq!(deposit_events.len(), 1); - - validate_event( - &deposit_events[0], - expected_transaction.version as i64, - "0x0000000000000000000000000000000000000000000000000000000000000000", - 0, - FA_DEPOSIT_EVENT, - ); - - Ok(()) - } - ) - .await - .is_ok()); - } - - // Test Case: Validate that a validator transaction generates the correct events. - // - Verifies that a validator transaction results in 20 events, with 19 being DistributeRewardsEvents. - #[tokio::test] - async fn test_validaator_distribute_rewards_events() { - let test_context = TestContext::new(&[IMPORTED_TESTNET_TXNS_5523474016_VALIDATOR_TXN]) - .await - .unwrap(); - let expected_transaction = test_context.transaction_batches[0].clone(); - let processor_config = TestProcessorConfig { - config: processor::processors::ProcessorConfig::EventsProcessor, - }; - let test_type = TestType::Scenario(ScenarioTest); - assert!(test_context - .run( - processor_config, - test_type, - move |conn: &mut PgConnection, version: &str| { - // Load and validate events - let distributed_rewards_events = load_transaction_events( - conn, - version.parse::().unwrap(), - DISTRIBUTE_REWARDS_EVENT, - ) - .expect("Failed to load DistributeRewardsEvents"); - assert_eq!(distributed_rewards_events.len(), 19); - - validate_event( - &distributed_rewards_events[0], - expected_transaction.version as i64, - "0x0a4113560d0b18ba38797f2a899c4b27e0c5b0476be5d8f6be68fba8b1861ed0", - 12, - DISTRIBUTE_REWARDS_EVENT, - ); - - Ok(()) - } - ) - .await - .is_ok()); - } - - // Test Case: Validate that a user-generated script transaction generates the correct event. - // - Verifies that a transaction using a user-defined script results in exactly one FeeStatement event. - #[tokio::test] - async fn test_user_script_transaction_fee_statement_event() { - let test_context = TestContext::new(&[IMPORTED_TESTNET_TXNS_5979639459_COIN_REGISTER]) - .await - .unwrap(); - let expected_transaction = test_context.transaction_batches[0].clone(); - let processor_config = TestProcessorConfig { - config: processor::processors::ProcessorConfig::EventsProcessor, - }; - let test_type = TestType::Scenario(ScenarioTest); - assert!(test_context - .run( - processor_config, - test_type, - move |conn: &mut PgConnection, version: &str| { - // Load and validate events - let actual_events = load_transaction_events( - conn, - version.parse::().unwrap(), - FEE_STATEMENT_EVENT, - ) - .expect("Failed to load FeeStatement events"); - assert_eq!(actual_events.len(), 1); - - validate_event( - &actual_events[0], - expected_transaction.version as i64, - "0x0000000000000000000000000000000000000000000000000000000000000000", - 0, - FEE_STATEMENT_EVENT, - ); - - Ok(()) - } - ) - .await - .is_ok()); - } - - fn load_transaction_events( - conn: &mut PgConnection, - txn_version: i64, - event_type: &str, - ) -> QueryResult> { - // Build the query with the required filters - events - .select(( - transaction_version, - type_, - indexed_type, - account_address, - creation_number, - )) - .filter( - transaction_version - .eq(txn_version) - .and(type_.eq(event_type)), - ) - .load::<(i64, String, String, String, i64)>(conn) - } - - fn validate_event( - actual_event: &(i64, String, String, String, i64), - expected_transaction_version: i64, - expected_account_address: &str, - expected_creation_number: i64, - expected_indexed_type: &str, - ) { - let ( - actual_transaction_version, - actual_type, - actual_indexed_type, - actual_address, - actual_creation_number, - ) = actual_event; - - assert_eq!(*actual_transaction_version, expected_transaction_version); - assert_eq!(actual_type, expected_indexed_type); - assert_eq!(actual_indexed_type, &expected_indexed_type); - assert_eq!(actual_address, expected_account_address); - assert_eq!(actual_creation_number, &expected_creation_number); - } -} diff --git a/rust/integration-tests/src/scenarios_tests/mod.rs b/rust/integration-tests/src/scenarios_tests/mod.rs deleted file mode 100644 index 3d3222304..000000000 --- a/rust/integration-tests/src/scenarios_tests/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod event_processor_tests; diff --git a/rust/integration-tests/src/sdk_tests/account_transaction_processor_tests.rs b/rust/integration-tests/src/sdk_tests/account_transaction_processor_tests.rs index f8cbf8ac3..a5e1b6995 100644 --- a/rust/integration-tests/src/sdk_tests/account_transaction_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/account_transaction_processor_tests.rs @@ -7,7 +7,7 @@ use sdk_processor::config::{ }; use std::collections::HashSet; -pub async fn setup_acc_txn_processor_config( +pub fn setup_acc_txn_processor_config( test_context: &SdkTestContext, db_url: &str, ) -> (IndexerProcessorConfig, &'static str) { @@ -94,7 +94,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = - setup_acc_txn_processor_config(&test_context, &db_url).await; + setup_acc_txn_processor_config(&test_context, &db_url); let acc_txns_processor = AccountTransactionsProcessor::new(indexer_processor_config) .await diff --git a/rust/integration-tests/src/sdk_tests/ans_processor_tests.rs b/rust/integration-tests/src/sdk_tests/ans_processor_tests.rs index a4d925284..b3b472769 100644 --- a/rust/integration-tests/src/sdk_tests/ans_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/ans_processor_tests.rs @@ -10,7 +10,7 @@ use sdk_processor::{ }; use std::collections::HashSet; -pub async fn setup_ans_processor_config( +pub fn setup_ans_processor_config( test_context: &SdkTestContext, db_url: &str, ) -> (IndexerProcessorConfig, &'static str) { @@ -123,7 +123,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = - setup_ans_processor_config(&test_context, &db_url).await; + setup_ans_processor_config(&test_context, &db_url); let ans_processor = AnsProcessor::new(indexer_processor_config) .await diff --git a/rust/integration-tests/src/sdk_tests/default_processor_tests.rs b/rust/integration-tests/src/sdk_tests/default_processor_tests.rs index a420e99e7..673002203 100644 --- a/rust/integration-tests/src/sdk_tests/default_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/default_processor_tests.rs @@ -7,7 +7,7 @@ use sdk_processor::config::{ }; use std::collections::HashSet; -pub async fn setup_default_processor_config( +pub fn setup_default_processor_config( test_context: &SdkTestContext, db_url: &str, ) -> (IndexerProcessorConfig, &'static str) { @@ -102,7 +102,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = - setup_default_processor_config(&test_context, &db_url).await; + setup_default_processor_config(&test_context, &db_url); let default_processor = DefaultProcessor::new(indexer_processor_config) .await diff --git a/rust/integration-tests/src/sdk_tests/events_processor_tests.rs b/rust/integration-tests/src/sdk_tests/events_processor_tests.rs index 08b292564..f6a411157 100644 --- a/rust/integration-tests/src/sdk_tests/events_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/events_processor_tests.rs @@ -7,7 +7,7 @@ use sdk_processor::config::{ }; use std::collections::HashSet; -pub async fn setup_events_processor_config( +pub fn setup_events_processor_config( test_context: &SdkTestContext, db_url: &str, ) -> (IndexerProcessorConfig, &'static str) { @@ -174,7 +174,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, _processor_name) = - setup_events_processor_config(&test_context, &db_url).await; + setup_events_processor_config(&test_context, &db_url); let events_processor = EventsProcessor::new(indexer_processor_config) .await @@ -232,7 +232,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = - setup_events_processor_config(&test_context, &db_url).await; + setup_events_processor_config(&test_context, &db_url); let events_processor = EventsProcessor::new(indexer_processor_config) .await diff --git a/rust/integration-tests/src/sdk_tests/stake_processor_tests.rs b/rust/integration-tests/src/sdk_tests/stake_processor_tests.rs index f9141c130..7577202e0 100644 --- a/rust/integration-tests/src/sdk_tests/stake_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/stake_processor_tests.rs @@ -10,7 +10,7 @@ use sdk_processor::{ }; use std::collections::HashSet; -pub async fn setup_stake_processor_config( +pub fn setup_stake_processor_config( test_context: &SdkTestContext, db_url: &str, ) -> (IndexerProcessorConfig, &'static str) { @@ -139,7 +139,7 @@ mod tests { let db_url = db.get_db_url(); let (indexer_processor_config, processor_name) = - setup_stake_processor_config(&test_context, &db_url).await; + setup_stake_processor_config(&test_context, &db_url); let stake_processor = StakeProcessor::new(indexer_processor_config) .await