add tranche 2 parquet models

rust/processor/src/db/common/models/ans_models/mod.rs

@@ -4,3 +4,6 @@
 pub mod ans_lookup;
 pub mod ans_lookup_v2;
 pub mod ans_utils;
+
+// parquet models
+pub mod parquet_ans_lookup_v2;

rust/processor/src/db/common/models/ans_models/parquet_ans_lookup_v2.rs

@@ -0,0 +1,42 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use field_count::FieldCount;
+use serde::{Deserialize, Serialize};
+
+type Domain = String;
+type Subdomain = String;
+type TokenStandardType = String;
+type RegisteredAddress = String;
+// PK of current_ans_lookup_v2
+type CurrentAnsLookupV2PK = (Domain, Subdomain, TokenStandardType);
+// PK of current_ans_primary_name
+type CurrentAnsPrimaryNameV2PK = (RegisteredAddress, TokenStandardType);
+
+#[derive(Clone, Default, Debug, Deserialize, FieldCount, Serialize)]
+pub struct AnsPrimaryNameV2 {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub registered_address: String,
+    pub token_standard: String,
+    pub domain: Option<String>,
+    pub subdomain: Option<String>,
+    pub token_name: Option<String>,
+    pub is_deleted: bool,
+    pub block_timestamp: chrono::NaiveDateTime,
+}
+
+
+pub struct CurrentAnsPrimaryNameV2 {
+    pub registered_address: String,
+    pub token_standard: String,
+    pub domain: Option<String>,
+    pub subdomain: Option<String>,
+    pub token_name: Option<String>,
+    pub is_deleted: bool,
+    pub last_transaction_version: i64,
+}

rust/processor/src/db/common/models/default_models/mod.rs

@@ -9,6 +9,7 @@ pub mod transactions;
 pub mod write_set_changes;
 
 // parquet models
+pub mod parquet_move_modules;
 pub mod parquet_move_resources;
 pub mod parquet_move_tables;
 pub mod parquet_transactions;

rust/processor/src/db/common/models/default_models/parquet_move_modules.rs

@@ -0,0 +1,104 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+#![allow(clippy::extra_unused_lifetimes)]
+
+use allocative_derive::Allocative;
+use aptos_protos::transaction::v1::{EventSizeInfo, WriteOpSizeInfo};
+use field_count::FieldCount;
+use parquet_derive::ParquetRecordWriter;
+use serde::{Deserialize, Serialize};
+use crate::bq_analytics::generic_parquet_processor::{HasVersion, NamedTable};
+
+#[derive(Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize)]
+pub struct MoveModule {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub block_height: i64,
+    pub name: String,
+    pub address: String,
+    pub bytecode: Option<Vec<u8>>,
+    pub exposed_functions: Option<String>,
+    pub friends: Option<String>,
+    pub structs: Option<String>,
+    pub is_deleted: bool,
+    #[allocative(skip)]
+    pub block_timestamp: chrono::NaiveDateTime,
+    pub txn_total_bytes: i64,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct MoveModuleByteCodeParsed {
+    pub address: String,
+    pub name: String,
+    pub bytecode: Vec<u8>,
+    pub exposed_functions: String,
+    pub friends: String,
+    pub structs: String,
+}
+
+
+#[derive(
+    Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize,
+)]
+pub struct EventSize {
+    pub txn_version: i64,
+    pub event_index: i64,
+    pub type_tag_bytes: i64,
+    pub total_bytes: i64,
+}
+
+impl NamedTable for EventSize {
+    const TABLE_NAME: &'static str = "event_size";
+}
+
+impl HasVersion for EventSize {
+    fn version(&self) -> i64 {
+        self.txn_version
+    }
+}
+
+impl EventSize {
+    pub fn from_event_size_info(info: &EventSizeInfo, txn_version: i64, event_index: i64) -> Self {
+        EventSize {
+            txn_version,
+            event_index,
+            type_tag_bytes: info.type_tag_bytes as i64,
+            total_bytes: info.total_bytes as i64,
+        }
+    }
+}
+
+
+#[derive(
+    Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize,
+)]
+pub struct WriteSetSize {
+    pub txn_version: i64,
+    pub change_index: i64,
+    pub key_bytes: i64,
+    pub value_bytes: i64,
+    pub total_bytes: i64,
+}
+
+impl NamedTable for WriteSetSize {
+    const TABLE_NAME: &'static str = "write_set_size";
+}
+
+impl HasVersion for WriteSetSize {
+    fn version(&self) -> i64 {
+        self.txn_version
+    }
+}
+
+impl WriteSetSize {
+    pub fn from_transaction_info(info: &WriteOpSizeInfo, txn_version: i64, change_index: i64) -> Self {
+        WriteSetSize {
+            txn_version,
+            change_index,
+            key_bytes: info.key_bytes as i64,
+            value_bytes: info.value_bytes as i64,
+            total_bytes: info.key_bytes as i64 + info.value_bytes as i64,
+        }
+    }
+}

rust/processor/src/db/common/models/default_models/parquet_move_tables.rs

@@ -48,6 +48,7 @@ pub struct CurrentTableItem {
     pub last_transaction_version: i64,
     pub is_deleted: bool,
 }
+
 #[derive(Clone, Debug, Deserialize, FieldCount, Serialize)]
 pub struct TableMetadata {
     pub handle: String,

rust/processor/src/db/common/models/fungible_asset_models/mod.rs

@@ -5,3 +5,7 @@ pub mod v2_fungible_asset_activities;
 pub mod v2_fungible_asset_balances;
 pub mod v2_fungible_asset_utils;
 pub mod v2_fungible_metadata;
+
+// parquet models
+pub mod parquet_v2_fungible_asset_balances;// parquet models
+pub mod parquet_coin_supply;

rust/processor/src/db/common/models/fungible_asset_models/parquet_coin_supply.rs

@@ -0,0 +1,37 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use crate::{
+    db::common::models::default_models::move_tables::TableItem,
+    utils::util::{hash_str, APTOS_COIN_TYPE_STR},
+};
+use allocative_derive::Allocative;
+use anyhow::Context;
+use aptos_protos::transaction::v1::WriteTableItem;
+use bigdecimal::{BigDecimal, ToPrimitive};
+use field_count::FieldCount;
+use parquet::data_type::Decimal;
+use parquet_derive::ParquetRecordWriter;
+use serde::{Deserialize, Serialize};
+
+const APTOS_COIN_SUPPLY_TABLE_HANDLE: &str =
+    "0x1b854694ae746cdbd8d44186ca4929b2b337df21d1c74633be19b2710552fdca";
+const APTOS_COIN_SUPPLY_TABLE_KEY: &str =
+    "0x619dc29a0aac8fa146714058e8dd6d2d0f3bdf5f6331907bf91f3acd81e6935";
+
+#[derive(
+    Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize,
+)]
+pub struct CoinSupply {
+    pub txn_version: i64,
+    pub coin_type_hash: String,
+    pub coin_type: String,
+    #[allocative(skip)]
+    pub supply: Vec<u8>,
+    #[allocative(skip)]
+    pub block_timestamp: chrono::NaiveDateTime,
+}

rust/processor/src/db/common/models/fungible_asset_models/parquet_v2_fungible_asset_balances.rs

@@ -0,0 +1,29 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use field_count::FieldCount;
+use std::borrow::Borrow;
+use parquet::data_type::{AsBytes, ByteArray, Decimal};
+use serde::{Serialize, Deserialize, Serializer, Deserializer};
+use serde::de::{self, Visitor};
+use std::fmt;
+use futures_util::TryFutureExt;
+pub type CurrentFungibleAssetBalancePK = String;
+
+#[derive(Clone, Debug, Deserialize, FieldCount, Serialize)]
+pub struct FungibleAssetBalance {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub storage_id: String,
+    pub owner_address: String,
+    pub asset_type: String,
+    pub is_primary: bool,
+    pub is_frozen: bool,
+    pub amount: Vec<u8>,
+    pub block_timestamp: chrono::NaiveDateTime,
+    pub token_standard: String,
+}

rust/processor/src/db/common/models/object_models/mod.rs

@@ -3,3 +3,6 @@
 
 pub mod v2_object_utils;
 pub mod v2_objects;
+
+// parquet models
+pub mod parquet_v2_objects;

rust/processor/src/db/common/models/object_models/parquet_v2_objects.rs

@@ -0,0 +1,33 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use super::v2_object_utils::{CurrentObjectPK, ObjectAggregatedDataMapping};
+use crate::{
+    db::common::models::default_models::move_resources::MoveResource,
+    schema::current_objects,
+    utils::{database::DbPoolConnection, util::standardize_address},
+};
+use ahash::AHashMap;
+use aptos_protos::transaction::v1::{DeleteResource, WriteResource};
+use bigdecimal::{BigDecimal, ToPrimitive};
+use diesel::prelude::*;
+use diesel_async::RunQueryDsl;
+use field_count::FieldCount;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, FieldCount, Serialize)]
+pub struct Object {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub object_address: String,
+    pub owner_address: String,
+    pub state_key_hash: String,
+    pub guid_creation_num: u64,
+    pub allow_ungated_transfer: bool,
+    pub is_deleted: bool,
+    pub untransferrable: bool,
+}

rust/processor/src/db/common/models/token_v2_models/mod.rs

@@ -8,3 +8,8 @@ pub mod v2_token_datas;
 pub mod v2_token_metadata;
 pub mod v2_token_ownerships;
 pub mod v2_token_utils;
+
+// parquet models
+pub mod parquet_v2_collections;
+pub mod parquet_v2_token_datas;
+pub mod parquet_v2_token_ownerships;

rust/processor/src/db/common/models/token_v2_models/parquet_v2_collections.rs

@@ -0,0 +1,35 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use allocative_derive::Allocative;
+use field_count::FieldCount;
+use parquet::data_type::Decimal;
+use parquet_derive::ParquetRecordWriter;
+use serde::{Deserialize, Serialize};
+
+// PK of current_collections_v2, i.e. collection_id
+pub type CurrentCollectionV2PK = String;
+
+#[derive(Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize)]
+pub struct CollectionV2 {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub collection_id: String,
+    pub creator_address: String,
+    pub collection_name: String,
+    pub description: String,
+    pub uri: String,
+    pub current_supply: Vec<u8>,
+    pub max_supply: Option<Vec<u8>>,
+    pub total_minted_v2: Option<Vec<u8>>,
+    pub mutable_description: Option<bool>,
+    pub mutable_uri: Option<bool>,
+    pub table_handle_v1: Option<String>,
+    pub token_standard: String,
+    #[allocative(skip)]
+    pub block_timestamp: chrono::NaiveDateTime,
+}

rust/processor/src/db/common/models/token_v2_models/parquet_v2_token_datas.rs

@@ -0,0 +1,29 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use field_count::FieldCount;
+use serde::{Deserialize, Serialize};
+
+// PK of current_token_datas_v2, i.e. token_data_id
+pub type CurrentTokenDataV2PK = String;
+
+#[derive(Clone, Debug, Deserialize, FieldCount, Serialize)]
+pub struct TokenDataV2 {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub token_data_id: String,
+    pub collection_id: String,
+    pub token_name: String,
+    pub largest_property_version_v1: Option<u64>,
+    pub token_uri: String,
+    pub token_properties: serde_json::Value,
+    pub description: String,
+    pub token_standard: String,
+    pub is_fungible_v2: Option<bool>,
+    pub block_timestamp: chrono::NaiveDateTime,
+    pub is_deleted_v2: Option<bool>,
+}

rust/processor/src/db/common/models/token_v2_models/parquet_v2_token_ownerships.rs

@@ -0,0 +1,34 @@
+// Copyright © Aptos Foundation
+// SPDX-License-Identifier: Apache-2.0
+
+// This is required because a diesel macro makes clippy sad
+#![allow(clippy::extra_unused_lifetimes)]
+#![allow(clippy::unused_unit)]
+
+use allocative_derive::Allocative;
+use bigdecimal::{BigDecimal, ToPrimitive, Zero};
+use field_count::FieldCount;
+use parquet_derive::ParquetRecordWriter;
+use serde::{Deserialize, Serialize};
+
+// PK of current_token_ownerships_v2, i.e. token_data_id, property_version_v1, owner_address, storage_id
+pub type CurrentTokenOwnershipV2PK = (String, BigDecimal, String, String);
+
+#[derive(Allocative, Clone, Debug, Default, Deserialize, FieldCount, ParquetRecordWriter, Serialize)]
+pub struct TokenOwnershipV2 {
+    pub txn_version: i64,
+    pub write_set_change_index: i64,
+    pub token_data_id: String,
+    pub property_version_v1: u64,
+    pub owner_address: Option<String>,
+    pub storage_id: String,
+    pub amount: Vec<u8>,
+    pub table_type_v1: Option<String>,
+    pub token_properties_mutated_v1: Option<String>,
+    pub is_soulbound_v2: Option<bool>,
+    pub token_standard: String,
+    pub is_fungible_v2: Option<bool>,
+    #[allocative(skip)]
+    pub block_timestamp: chrono::NaiveDateTime,
+    pub non_transferrable_by_owner: Option<bool>,
+}