diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5cd4f608..4ff91387 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "aptos-indexer-test-transactions" version = "1.0.0" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=60ee0c686c15480b2cbba224c205e03f479bcdbd#60ee0c686c15480b2cbba224c205e03f479bcdbd" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=8bb628129ff48241c650178caf1ff8bf53a44e5e#8bb628129ff48241c650178caf1ff8bf53a44e5e" dependencies = [ - "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=60ee0c686c15480b2cbba224c205e03f479bcdbd)", + "aptos-protos 1.3.1 (git+https://github.com/aptos-labs/aptos-core.git?rev=8bb628129ff48241c650178caf1ff8bf53a44e5e)", "serde_json", ] @@ -309,7 +309,7 @@ dependencies = [ [[package]] name = "aptos-protos" version = "1.3.1" -source = "git+https://github.com/aptos-labs/aptos-core.git?rev=60ee0c686c15480b2cbba224c205e03f479bcdbd#60ee0c686c15480b2cbba224c205e03f479bcdbd" +source = "git+https://github.com/aptos-labs/aptos-core.git?rev=8bb628129ff48241c650178caf1ff8bf53a44e5e#8bb628129ff48241c650178caf1ff8bf53a44e5e" dependencies = [ "futures-core", "pbjson", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 763af200..72c748f9 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -34,7 +34,7 @@ aptos-indexer-processor-sdk = { git = "https://github.com/aptos-labs/aptos-index aptos-indexer-processor-sdk-server-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "e6867c50a2c30ef16ad6f82e02313b2ba5ce361a" } aptos-protos = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "5c48aee129b5a141be2792ffa3d9bd0a1a61c9cb" } aptos-system-utils = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "202bdccff2b2d333a385ae86a4fcf23e89da9f62" } -aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "60ee0c686c15480b2cbba224c205e03f479bcdbd" } +aptos-indexer-test-transactions = { git = "https://github.com/aptos-labs/aptos-core.git", rev = "8bb628129ff48241c650178caf1ff8bf53a44e5e" } aptos-indexer-testing-framework = { git = "https://github.com/aptos-labs/aptos-indexer-processor-sdk.git", rev = "e6867c50a2c30ef16ad6f82e02313b2ba5ce361a" } async-trait = "0.1.53" backtrace = "0.3.58" diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/collections_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/collections_v2.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/collections_v2.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_collections_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_collections_v2.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_collections_v2.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_datas_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_datas_v2.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_datas_v2.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_ownerships_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_ownerships_v2.json new file mode 100644 index 00000000..eeac22ab --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_ownerships_v2.json @@ -0,0 +1,17 @@ +[ + { + "token_data_id": "0x2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8", + "property_version_v1": "0", + "owner_address": "0xe2ebcbacd81584f97b6ba9a458239ea083428d75158c0d42ef38b6379527e99a", + "storage_id": "0xa5e6f84af35d356027c1513b37248bae3968c20720af6fc2d5c58b7c90ffda55", + "amount": "1", + "table_type_v1": "0x3::token::TokenStore", + "token_properties_mutated_v1": {}, + "is_soulbound_v2": null, + "token_standard": "v1", + "is_fungible_v2": null, + "last_transaction_version": 19922017, + "last_transaction_timestamp": "2024-11-28T23:53:41.291148", + "non_transferrable_by_owner": null + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_pending_claims.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_pending_claims.json new file mode 100644 index 00000000..46b9c559 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_pending_claims.json @@ -0,0 +1,18 @@ +[ + { + "token_data_id_hash": "2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8", + "property_version": "0", + "from_address": "0xd77942ad91c35a2d165fdec8f6a28ec48b6bb9f905db2fd0ac8a7e481a9c543e", + "to_address": "0xe2ebcbacd81584f97b6ba9a458239ea083428d75158c0d42ef38b6379527e99a", + "collection_data_id_hash": "84b2198ac10fdbb64bc3474ed0cf184b7bdcd5bc5f098a8858f2e48f379c5fcc", + "creator_address": "0xd77942ad91c35a2d165fdec8f6a28ec48b6bb9f905db2fd0ac8a7e481a9c543e", + "collection_name": "Alice's", + "name": "Alice's first token", + "amount": "0", + "table_handle": "0xc4b3a532f522e5cc021427b4d729938073a6d34949bd397490d4bbf96a4703f1", + "last_transaction_version": 19922017, + "collection_id": "0x84b2198ac10fdbb64bc3474ed0cf184b7bdcd5bc5f098a8858f2e48f379c5fcc", + "last_transaction_timestamp": "2024-11-28T23:53:41.291148", + "token_data_id": "0x2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_v2_metadata.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_v2_metadata.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/current_token_v2_metadata.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_activities_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_activities_v2.json new file mode 100644 index 00000000..8e23ed7e --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_activities_v2.json @@ -0,0 +1,36 @@ +[ + { + "transaction_version": 19922017, + "event_index": 0, + "event_account_address": "0x0000000000000000000000000000000000000000000000000000000000000000", + "token_data_id": "0x2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8", + "property_version_v1": "0", + "type_": "0x3::token::TokenDeposit", + "from_address": null, + "to_address": "0xe2ebcbacd81584f97b6ba9a458239ea083428d75158c0d42ef38b6379527e99a", + "token_amount": "1", + "before_value": null, + "after_value": null, + "entry_function_id_str": "0x3::token_transfers::claim_script", + "token_standard": "v1", + "is_fungible_v2": null, + "transaction_timestamp": "2024-11-28T23:53:41.291148" + }, + { + "transaction_version": 19922017, + "event_index": 1, + "event_account_address": "0x0000000000000000000000000000000000000000000000000000000000000000", + "token_data_id": "0x2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8", + "property_version_v1": "0", + "type_": "0x3::token_transfers::Claim", + "from_address": "0xd77942ad91c35a2d165fdec8f6a28ec48b6bb9f905db2fd0ac8a7e481a9c543e", + "to_address": "0xe2ebcbacd81584f97b6ba9a458239ea083428d75158c0d42ef38b6379527e99a", + "token_amount": "1", + "before_value": null, + "after_value": null, + "entry_function_id_str": "0x3::token_transfers::claim_script", + "token_standard": "v1", + "is_fungible_v2": null, + "transaction_timestamp": "2024-11-28T23:53:41.291148" + } +] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_datas_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_datas_v2.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_datas_v2.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_ownerships_v2.json b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_ownerships_v2.json new file mode 100644 index 00000000..10b25157 --- /dev/null +++ b/rust/integration-tests/sdk_expected_db_output_files/token_v2_processor/test_token_v1_offer_claim_no_table_metadata/token_ownerships_v2.json @@ -0,0 +1,18 @@ +[ + { + "transaction_version": 19922017, + "write_set_change_index": 4, + "token_data_id": "0x2877fd783c3c7957b50941cce1b3b729dae16b5928e0ba0606cd282df2b085c8", + "property_version_v1": "0", + "owner_address": "0xe2ebcbacd81584f97b6ba9a458239ea083428d75158c0d42ef38b6379527e99a", + "storage_id": "0xa5e6f84af35d356027c1513b37248bae3968c20720af6fc2d5c58b7c90ffda55", + "amount": "1", + "table_type_v1": "0x3::token::TokenStore", + "token_properties_mutated_v1": {}, + "is_soulbound_v2": null, + "token_standard": "v1", + "is_fungible_v2": null, + "transaction_timestamp": "2024-11-28T23:53:41.291148", + "non_transferrable_by_owner": null + } +] \ No newline at end of file diff --git a/rust/integration-tests/src/sdk_tests/token_v2_processor_tests.rs b/rust/integration-tests/src/sdk_tests/token_v2_processor_tests.rs index 3bbc2ba1..5b8e8f36 100644 --- a/rust/integration-tests/src/sdk_tests/token_v2_processor_tests.rs +++ b/rust/integration-tests/src/sdk_tests/token_v2_processor_tests.rs @@ -58,6 +58,7 @@ mod sdk_token_v2_processor_tests { }, }; use aptos_indexer_test_transactions::{ + IMPORTED_DEVNET_TXNS_19922017_TOKEN_V1_OFFER_CLAIM, IMPORTED_DEVNET_TXNS_78753831_TOKEN_V1_MINT_TRANSFER_WITH_V2_EVENTS, IMPORTED_DEVNET_TXNS_78753832_TOKEN_V2_MINT_TRANSFER_WITH_V2_EVENTS, IMPORTED_MAINNET_TXNS_1058723093_TOKEN_V1_MINT_WITHDRAW_DEPOSIT_EVENTS, @@ -314,6 +315,15 @@ mod sdk_token_v2_processor_tests { .await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_token_v1_offer_claim_no_table_metadata() { + process_single_transaction( + IMPORTED_DEVNET_TXNS_19922017_TOKEN_V1_OFFER_CLAIM, + Some("test_token_v1_offer_claim_no_table_metadata".to_string()), + ) + .await; + } + /** * This test includes processing for the following: * - Resources diff --git a/rust/processor/src/db/postgres/models/token_models/token_claims.rs b/rust/processor/src/db/postgres/models/token_models/token_claims.rs index f418aede..90b709cd 100644 --- a/rust/processor/src/db/postgres/models/token_models/token_claims.rs +++ b/rust/processor/src/db/postgres/models/token_models/token_claims.rs @@ -6,12 +6,20 @@ #![allow(clippy::unused_unit)] use super::{token_utils::TokenWriteSet, tokens::TableHandleToOwner}; -use crate::{schema::current_token_pending_claims, utils::util::standardize_address}; +use crate::{ + db::postgres::models::token_v2_models::v2_token_activities::TokenActivityHelperV1, + schema::current_token_pending_claims, utils::util::standardize_address, +}; +use ahash::AHashMap; use aptos_protos::transaction::v1::{DeleteTableItem, WriteTableItem}; use bigdecimal::{BigDecimal, Zero}; use field_count::FieldCount; use serde::{Deserialize, Serialize}; +// Map to keep track of the metadata of token offers that were claimed. The key is the token data id of the offer. +// Potentially it'd also be useful to keep track of offers that were canceled. +pub type TokenV1Claimed = AHashMap; + #[derive( Clone, Debug, Deserialize, Eq, FieldCount, Identifiable, Insertable, PartialEq, Serialize, )] @@ -136,6 +144,7 @@ impl CurrentTokenPendingClaim { txn_version: i64, txn_timestamp: chrono::NaiveDateTime, table_handle_to_owner: &TableHandleToOwner, + tokens_claimed: &TokenV1Claimed, ) -> anyhow::Result> { let table_item_data = table_item.data.as_ref().unwrap(); @@ -149,12 +158,27 @@ impl CurrentTokenPendingClaim { }; if let Some(offer) = &maybe_offer { let table_handle = standardize_address(&table_item.handle.to_string()); + let token_data_id = offer.token_id.token_data_id.to_id(); + + // Try to find owner from write resources + let mut maybe_owner_address = table_handle_to_owner + .get(&table_handle) + .map(|table_metadata| table_metadata.get_owner_address()); + + // If table handle isn't in TableHandleToOwner, try to find owner from token v1 claim events + if maybe_owner_address.is_none() { + if let Some(token_claimed) = tokens_claimed.get(&token_data_id) { + maybe_owner_address = token_claimed.from_address.clone(); + } + } - let table_metadata = table_handle_to_owner.get(&table_handle).unwrap_or_else(|| { + let owner_address = maybe_owner_address.unwrap_or_else(|| { panic!( "Missing table handle metadata for claim. \ - Version: {}, table handle for PendingClaims: {}, all metadata: {:?}", - txn_version, table_handle, table_handle_to_owner + Version: {}, table handle for PendingClaims: {}, all metadata: {:?} \ + Missing token data id in token claim event. \ + token_data_id: {}, all token claim events: {:?}", + txn_version, table_handle, table_handle_to_owner, token_data_id, tokens_claimed ) }); @@ -171,7 +195,7 @@ impl CurrentTokenPendingClaim { return Ok(Some(Self { token_data_id_hash, property_version: token_id.property_version, - from_address: table_metadata.get_owner_address(), + from_address: owner_address, to_address: offer.get_to_address(), collection_data_id_hash, creator_address: token_data_id_struct.get_creator_address(), diff --git a/rust/processor/src/db/postgres/models/token_v2_models/v2_token_activities.rs b/rust/processor/src/db/postgres/models/token_v2_models/v2_token_activities.rs index 6121baf0..4be9247c 100644 --- a/rust/processor/src/db/postgres/models/token_v2_models/v2_token_activities.rs +++ b/rust/processor/src/db/postgres/models/token_v2_models/v2_token_activities.rs @@ -9,7 +9,10 @@ use super::v2_token_utils::{TokenStandard, V2TokenEvent}; use crate::{ db::postgres::models::{ object_models::v2_object_utils::ObjectAggregatedDataMapping, - token_models::token_utils::{TokenDataIdType, TokenEvent}, + token_models::{ + token_claims::TokenV1Claimed, + token_utils::{TokenDataIdType, TokenEvent}, + }, }, schema::token_activities_v2, utils::util::standardize_address, @@ -41,7 +44,8 @@ pub struct TokenActivityV2 { } /// A simplified TokenActivity (excluded common fields) to reduce code duplication -struct TokenActivityHelperV1 { +#[derive(Clone, Debug)] +pub struct TokenActivityHelperV1 { pub token_data_id_struct: TokenDataIdType, pub property_version: BigDecimal, pub from_address: Option, @@ -200,6 +204,7 @@ impl TokenActivityV2 { txn_timestamp: chrono::NaiveDateTime, event_index: i64, entry_function_id_str: &Option, + tokens_claimed: &mut TokenV1Claimed, ) -> anyhow::Result> { let event_type = event.type_str.clone(); if let Some(token_event) = &TokenEvent::from_event(&event_type, &event.data, txn_version)? { @@ -290,12 +295,17 @@ impl TokenActivityV2 { to_address: Some(inner.get_to_address()), token_amount: inner.amount.clone(), }, - TokenEvent::ClaimTokenEvent(inner) => TokenActivityHelperV1 { - token_data_id_struct: inner.token_id.token_data_id.clone(), - property_version: inner.token_id.property_version.clone(), - from_address: Some(event_account_address.clone()), - to_address: Some(inner.get_to_address()), - token_amount: inner.amount.clone(), + TokenEvent::ClaimTokenEvent(inner) => { + let token_data_id_struct = inner.token_id.token_data_id.clone(); + let helper = TokenActivityHelperV1 { + token_data_id_struct: token_data_id_struct.clone(), + property_version: inner.token_id.property_version.clone(), + from_address: Some(event_account_address.clone()), + to_address: Some(inner.get_to_address()), + token_amount: inner.amount.clone(), + }; + tokens_claimed.insert(token_data_id_struct.to_id(), helper.clone()); + helper }, TokenEvent::Offer(inner) => TokenActivityHelperV1 { token_data_id_struct: inner.token_id.token_data_id.clone(), @@ -311,12 +321,17 @@ impl TokenActivityV2 { to_address: Some(inner.get_to_address()), token_amount: inner.amount.clone(), }, - TokenEvent::Claim(inner) => TokenActivityHelperV1 { - token_data_id_struct: inner.token_id.token_data_id.clone(), - property_version: inner.token_id.property_version.clone(), - from_address: Some(inner.get_from_address()), - to_address: Some(inner.get_to_address()), - token_amount: inner.amount.clone(), + TokenEvent::Claim(inner) => { + let token_data_id_struct = inner.token_id.token_data_id.clone(); + let helper = TokenActivityHelperV1 { + token_data_id_struct: token_data_id_struct.clone(), + property_version: inner.token_id.property_version.clone(), + from_address: Some(inner.get_from_address()), + to_address: Some(inner.get_to_address()), + token_amount: inner.amount.clone(), + }; + tokens_claimed.insert(token_data_id_struct.to_id(), helper.clone()); + helper }, }; let token_data_id_struct = token_activity_helper.token_data_id_struct; diff --git a/rust/processor/src/processors/token_v2_processor.rs b/rust/processor/src/processors/token_v2_processor.rs index 591ed2ba..c7247c78 100644 --- a/rust/processor/src/processors/token_v2_processor.rs +++ b/rust/processor/src/processors/token_v2_processor.rs @@ -10,7 +10,7 @@ use crate::{ }, resources::{FromWriteResource, V2TokenResource}, token_models::{ - token_claims::CurrentTokenPendingClaim, + token_claims::{CurrentTokenPendingClaim, TokenV1Claimed}, tokens::{CurrentTokenPendingClaimPK, TableHandleToOwner, TableMetadataForToken}, }, token_v2_models::{ @@ -783,6 +783,9 @@ pub async fn parse_v2_token( // Get mint events for token v2 by object let mut tokens_minted: TokenV2Minted = AHashSet::new(); + // Get claim events for token v1 by table handle + let mut tokens_claimed: TokenV1Claimed = AHashMap::new(); + // Loop 1: Need to do a first pass to get all the object addresses and insert them into the helper for wsc in transaction_info.changes.iter() { if let Change::WriteResource(wr) = wsc.change.as_ref().unwrap() { @@ -847,6 +850,7 @@ pub async fn parse_v2_token( // Loop 3: Pass through events to get the burn events and token activities v2 // This needs to be here because we need the metadata parsed in loop 2 for token activities // and burn / transfer events need to come before the next loop + // Also parses token v1 claim events, which will be used in Loop 4 to build the claims table for (index, event) in user_txn.events.iter().enumerate() { if let Some(burn_event) = Burn::from_event(event, txn_version).unwrap() { tokens_burned.insert(burn_event.get_token_address(), burn_event.clone()); @@ -890,6 +894,7 @@ pub async fn parse_v2_token( txn_timestamp, index as i64, &entry_function_id_str, + &mut tokens_claimed, ) .unwrap() { @@ -1053,6 +1058,7 @@ pub async fn parse_v2_token( txn_version, txn_timestamp, table_handle_to_owner, + &tokens_claimed, ) .unwrap() {