From 8d9a45ad02a046b8f5496677a4b30f86ab09a550 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 13 May 2024 00:53:01 +0100 Subject: [PATCH] [Indexer/TiDB] Standardize prefix lengths for blob indices ## Description Pick standard prefix lengths for well-known types that make good use of details we know about their structure: - Object IDs, addresses and digests are all 32 byte quantities. - Move Identifiers cannot be larger than 128 bytes. - Fully-qualified types are unlikely to be more than 256 bytes (this is the only case where we have to approximate). ## Test plan ``` sui-indexer$ diesel database reset --database-url=$DB --migration-dir='migrations/mysql' sui-indexer$ cargo run \ --no-default-features --features mysql-feature \ -- --db-url $DB --reset-db \ --fullnode-sync-worker \ --rpc-client-url "$MAINNET" ``` Ensure data being filled from mainnet by the indexer is well-formed. --- .../mysql/2024-03-22-052019_events/up.sql | 8 ++--- .../mysql/2024-03-25-203621_objects/up.sql | 34 +++++++++---------- .../2024-03-26-004025_transactions/up.sql | 2 +- .../2024-04-24-180008_checkpoints/up.sql | 2 +- .../mysql/2024-04-24-180418_tx_indices/up.sql | 16 ++++----- .../mysql/2024-04-24-180727_display/up.sql | 2 +- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql index ea46f67373fb9f..bca338b64c1955 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-22-052019_events/up.sql @@ -27,8 +27,8 @@ CREATE TABLE events ) PARTITION BY RANGE (checkpoint_sequence_number) ( PARTITION events_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX events_package ON events (package(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_package_module ON events (package(255), module(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_event_type ON events (event_type(255), tx_sequence_number, event_sequence_number); -CREATE INDEX events_type_package_module_name ON events (event_type_package(128), event_type_module(128), event_type_name(128), tx_sequence_number, event_sequence_number); +CREATE INDEX events_package ON events (package(32), tx_sequence_number, event_sequence_number); +CREATE INDEX events_package_module ON events (package(32), module(128), tx_sequence_number, event_sequence_number); +CREATE INDEX events_event_type ON events (event_type(256), tx_sequence_number, event_sequence_number); +CREATE INDEX events_type_package_module_name ON events (event_type_package(32), event_type_module(128), event_type_name(128), tx_sequence_number, event_sequence_number); CREATE INDEX events_checkpoint_sequence_number ON events (checkpoint_sequence_number); diff --git a/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql index bdefccc67e3427..801b4ed55f0234 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-25-203621_objects/up.sql @@ -32,15 +32,15 @@ CREATE TABLE objects ( df_object_type TEXT, -- object_id in DynamicFieldInfo. df_object_id BLOB, - CONSTRAINT objects_pk PRIMARY KEY (object_id(128)) + CONSTRAINT objects_pk PRIMARY KEY (object_id(32)) ); -- OwnerType: 1: Address, 2: Object, see types.rs -CREATE INDEX objects_owner ON objects (owner_type, owner_id(128)); -CREATE INDEX objects_coin ON objects (owner_id(128), coin_type(128)); +CREATE INDEX objects_owner ON objects (owner_type, owner_id(32)); +CREATE INDEX objects_coin ON objects (owner_id(32), coin_type(256)); CREATE INDEX objects_checkpoint_sequence_number ON objects (checkpoint_sequence_number); -CREATE INDEX objects_package_module_name_full_type ON objects (object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_owner_package_module_name_full_type ON objects (owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_package_module_name_full_type ON objects (object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_owner_package_module_name_full_type ON objects (owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); -- similar to objects table, except that -- 1. the primary key to store multiple object versions and partitions by checkpoint_sequence_number @@ -66,16 +66,16 @@ CREATE TABLE objects_history ( df_name BLOB, df_object_type TEXT, df_object_id BLOB, - CONSTRAINT objects_history_pk PRIMARY KEY (checkpoint_sequence_number, object_id(128), object_version) + CONSTRAINT objects_history_pk PRIMARY KEY (checkpoint_sequence_number, object_id(32), object_version) ) PARTITION BY RANGE (checkpoint_sequence_number) ( PARTITION objects_history_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX objects_history_id_version ON objects_history (object_id(128), object_version, checkpoint_sequence_number); -CREATE INDEX objects_history_owner ON objects_history (checkpoint_sequence_number, owner_type, owner_id(128)); -CREATE INDEX objects_history_coin ON objects_history (checkpoint_sequence_number, owner_id(128), coin_type(128)); -CREATE INDEX objects_history_type ON objects_history (checkpoint_sequence_number, object_type(128)); -CREATE INDEX objects_history_package_module_name_full_type ON objects_history (checkpoint_sequence_number, object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_history_owner_package_module_name_full_type ON objects_history (checkpoint_sequence_number, owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_history_id_version ON objects_history (object_id(32), object_version, checkpoint_sequence_number); +CREATE INDEX objects_history_owner ON objects_history (checkpoint_sequence_number, owner_type, owner_id(32)); +CREATE INDEX objects_history_coin ON objects_history (checkpoint_sequence_number, owner_id(32), coin_type(256)); +CREATE INDEX objects_history_type ON objects_history (checkpoint_sequence_number, object_type(256)); +CREATE INDEX objects_history_package_module_name_full_type ON objects_history (checkpoint_sequence_number, object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_history_owner_package_module_name_full_type ON objects_history (checkpoint_sequence_number, owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); -- snapshot table by folding objects_history table until certain checkpoint, -- effectively the snapshot of objects at the same checkpoint, @@ -99,10 +99,10 @@ CREATE TABLE objects_snapshot ( df_name BLOB, df_object_type TEXT, df_object_id BLOB, - CONSTRAINT objects_snapshot_pk PRIMARY KEY (object_id(128)) + CONSTRAINT objects_snapshot_pk PRIMARY KEY (object_id(32)) ); CREATE INDEX objects_snapshot_checkpoint_sequence_number ON objects_snapshot (checkpoint_sequence_number); -CREATE INDEX objects_snapshot_owner ON objects_snapshot (owner_type, owner_id(128), object_id(128)); -CREATE INDEX objects_snapshot_coin ON objects_snapshot (owner_id(128), coin_type(128), object_id(128)); -CREATE INDEX objects_snapshot_package_module_name_full_type ON objects_snapshot (object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); -CREATE INDEX objects_snapshot_owner_package_module_name_full_type ON objects_snapshot (owner_id(128), object_type_package(128), object_type_module(128), object_type_name(128), object_type(128)); +CREATE INDEX objects_snapshot_owner ON objects_snapshot (owner_type, owner_id(32), object_id(32)); +CREATE INDEX objects_snapshot_coin ON objects_snapshot (owner_id(32), coin_type(256), object_id(32)); +CREATE INDEX objects_snapshot_package_module_name_full_type ON objects_snapshot (object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); +CREATE INDEX objects_snapshot_owner_package_module_name_full_type ON objects_snapshot (owner_id(32), object_type_package(32), object_type_module(128), object_type_name(128), object_type(256)); diff --git a/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql b/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql index 791c8105a0d581..e18f1f368fa421 100644 --- a/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-03-26-004025_transactions/up.sql @@ -23,7 +23,7 @@ CREATE TABLE transactions ( PARTITION transactions_partition_0 VALUES LESS THAN MAXVALUE ); -CREATE INDEX transactions_transaction_digest ON transactions (transaction_digest(255)); +CREATE INDEX transactions_transaction_digest ON transactions (transaction_digest(32)); CREATE INDEX transactions_checkpoint_sequence_number ON transactions (checkpoint_sequence_number); -- only create index for system transactions (0). See types.rs CREATE INDEX transactions_transaction_kind ON transactions (transaction_kind); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql index 76bcce28b1f566..7ab0fa8a3a5cd0 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180008_checkpoints/up.sql @@ -25,4 +25,4 @@ CREATE TABLE checkpoints ); CREATE INDEX checkpoints_epoch ON checkpoints (epoch, sequence_number); -CREATE INDEX checkpoints_digest ON checkpoints (checkpoint_digest(255)); +CREATE INDEX checkpoints_digest ON checkpoints (checkpoint_digest(32)); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql index d1d0e428fdc19d..13feddcd18bf02 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180418_tx_indices/up.sql @@ -4,7 +4,7 @@ CREATE TABLE tx_senders ( tx_sequence_number BIGINT NOT NULL, -- SuiAddress in bytes. sender BLOB NOT NULL, - PRIMARY KEY(sender(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(sender(32), tx_sequence_number, cp_sequence_number) ); CREATE INDEX tx_senders_tx_sequence_number_index ON tx_senders (tx_sequence_number, cp_sequence_number); @@ -14,7 +14,7 @@ CREATE TABLE tx_recipients ( tx_sequence_number BIGINT NOT NULL, -- SuiAddress in bytes. recipient BLOB NOT NULL, - PRIMARY KEY(recipient(255), tx_sequence_number) + PRIMARY KEY(recipient(32), tx_sequence_number) ); CREATE INDEX tx_recipients_tx_sequence_number_index ON tx_recipients (tx_sequence_number, cp_sequence_number); @@ -23,7 +23,7 @@ CREATE TABLE tx_input_objects ( tx_sequence_number BIGINT NOT NULL, -- Object ID in bytes. object_id BLOB NOT NULL, - PRIMARY KEY(object_id(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(object_id(32), tx_sequence_number, cp_sequence_number) ); CREATE TABLE tx_changed_objects ( @@ -31,7 +31,7 @@ CREATE TABLE tx_changed_objects ( tx_sequence_number BIGINT NOT NULL, -- Object Id in bytes. object_id BLOB NOT NULL, - PRIMARY KEY(object_id(255), tx_sequence_number) + PRIMARY KEY(object_id(32), tx_sequence_number) ); CREATE TABLE tx_calls ( @@ -42,16 +42,16 @@ CREATE TABLE tx_calls ( func TEXT NOT NULL, -- 1. Using Primary Key as a unique index. -- 2. Diesel does not like tables with no primary key. - PRIMARY KEY(package(255), tx_sequence_number, cp_sequence_number) + PRIMARY KEY(package(32), tx_sequence_number, cp_sequence_number) ); -CREATE INDEX tx_calls_module ON tx_calls (package(255), module(255), tx_sequence_number, cp_sequence_number); -CREATE INDEX tx_calls_func ON tx_calls (package(255), module(255), func(255), tx_sequence_number, cp_sequence_number); +CREATE INDEX tx_calls_module ON tx_calls (package(32), module(128), tx_sequence_number, cp_sequence_number); +CREATE INDEX tx_calls_func ON tx_calls (package(32), module(128), func(128), tx_sequence_number, cp_sequence_number); CREATE INDEX tx_calls_tx_sequence_number ON tx_calls (tx_sequence_number, cp_sequence_number); CREATE TABLE tx_digests ( tx_digest BLOB NOT NULL, cp_sequence_number BIGINT NOT NULL, tx_sequence_number BIGINT NOT NULL, - PRIMARY KEY(tx_digest(255)) + PRIMARY KEY(tx_digest(32)) ); diff --git a/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql b/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql index f30a9d0171a5e9..82fce6bb86ea5d 100644 --- a/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql +++ b/crates/sui-indexer/migrations/mysql/2024-04-24-180727_display/up.sql @@ -4,5 +4,5 @@ CREATE TABLE display id BLOB NOT NULL, version SMALLINT NOT NULL, bcs MEDIUMBLOB NOT NULL, - primary key (object_type(255)) + primary key (object_type(256)) );