From c3a35f432ee25d0fb45bdf946dfe3d340edb9000 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 6 May 2024 00:57:52 +0100 Subject: [PATCH 1/9] [GraphQL] Leverage `objects_version` table. (#17543) ## Description Use the `objects_version` table to speed up point look-ups (via data loaders) for historical objects (ID + version), and dynamic fields (object look-up bounding version by parent ID). With this change, the restriction of accessing dynamic fields only within the available range is dropped. ## Test plan ``` sui$ cargo nextest run -p sui-graphql-rpc sui$ cargo nextest run -p sui-graphql-e2e-tests --features pg_integration. ``` Perform a query that involves fetching a large number of dynamic fields, which should now be fast. The following example, fetching dynamic fields on a deepbook pool loads 50 dynamic fields in about 5s from cold (which also requires loading packages for resolution), and then 2s from there: ``` query { owner( address: "0x029170bfa0a1677054263424fe4f9960c7cf05d359f6241333994c8830772bdb" ) { dynamicFields { pageInfo { hasNextPage endCursor } nodes { name { type { repr } json } value { ... on MoveValue { type { repr } json } ... on MoveObject { contents { json type { repr } } } } } } } } ``` ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 - #17726 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Dynamic fields can now be looked up on any historical object (not just objects in the available range). - [ ] CLI: - [ ] Rust SDK: --- .../dynamic_fields/dynamic_fields.exp | 12 +- .../src/types/dynamic_field.rs | 9 +- crates/sui-graphql-rpc/src/types/object.rs | 226 +++++++++++++----- crates/sui-graphql-rpc/src/types/owner.rs | 7 +- 4 files changed, 186 insertions(+), 68 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/consistency/dynamic_fields/dynamic_fields.exp b/crates/sui-graphql-e2e-tests/tests/consistency/dynamic_fields/dynamic_fields.exp index afbfbb96ca3a1..51b7b407d25ad 100644 --- a/crates/sui-graphql-e2e-tests/tests/consistency/dynamic_fields/dynamic_fields.exp +++ b/crates/sui-graphql-e2e-tests/tests/consistency/dynamic_fields/dynamic_fields.exp @@ -1166,7 +1166,17 @@ task 34, lines 497-528: Response: { "data": { "parent_version_4": { - "dfAtParentVersion4_outside_range": null + "dfAtParentVersion4_outside_range": { + "name": { + "bcs": "A2RmMQ==", + "type": { + "repr": "0x0000000000000000000000000000000000000000000000000000000000000001::string::String" + } + }, + "value": { + "json": "df1" + } + } }, "parent_version_6": { "dfAtParentVersion6": null diff --git a/crates/sui-graphql-rpc/src/types/dynamic_field.rs b/crates/sui-graphql-rpc/src/types/dynamic_field.rs index ebbc86df505a4..60e05ed610512 100644 --- a/crates/sui-graphql-rpc/src/types/dynamic_field.rs +++ b/crates/sui-graphql-rpc/src/types/dynamic_field.rs @@ -10,7 +10,7 @@ use sui_types::dynamic_field::{derive_dynamic_field_id, DynamicFieldInfo, Dynami use super::available_range::AvailableRange; use super::cursor::{Page, Target}; -use super::object::{self, deserialize_move_struct, Object, ObjectKind, ObjectLookup}; +use super::object::{self, deserialize_move_struct, Object, ObjectKind}; use super::type_filter::ExactTypeFilter; use super::{ base64::Base64, move_object::MoveObject, move_value::MoveValue, sui_address::SuiAddress, @@ -170,9 +170,10 @@ impl DynamicField { let super_ = MoveObject::query( ctx, SuiAddress::from(field_id), - ObjectLookup::LatestAt { - parent_version, - checkpoint_viewed_at, + if let Some(parent_version) = parent_version { + Object::under_parent(parent_version, checkpoint_viewed_at) + } else { + Object::latest_at(checkpoint_viewed_at) }, ) .await?; diff --git a/crates/sui-graphql-rpc/src/types/object.rs b/crates/sui-graphql-rpc/src/types/object.rs index 7243666ef520c..e9177a122057d 100644 --- a/crates/sui-graphql-rpc/src/types/object.rs +++ b/crates/sui-graphql-rpc/src/types/object.rs @@ -34,12 +34,12 @@ use crate::{filter, or_filter}; use async_graphql::connection::{CursorType, Edge}; use async_graphql::dataloader::Loader; use async_graphql::{connection::Connection, *}; -use diesel::{BoolExpressionMethods, ExpressionMethods, QueryDsl}; +use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl, SelectableHelper}; use move_core_types::annotated_value::{MoveStruct, MoveTypeLayout}; use move_core_types::language_storage::StructTag; use serde::{Deserialize, Serialize}; use sui_indexer::models::objects::{StoredDeletedHistoryObject, StoredHistoryObject}; -use sui_indexer::schema::objects_history; +use sui_indexer::schema::{objects_history, objects_version}; use sui_indexer::types::ObjectStatus as NativeObjectStatus; use sui_indexer::types::OwnerType; use sui_types::object::bounded_visitor::BoundedVisitor; @@ -183,9 +183,14 @@ pub(crate) struct AddressOwner { pub(crate) enum ObjectLookup { LatestAt { - /// The parent version to be used as an optional upper bound for the query. Look for the - /// latest version of a child object that is less than or equal to this upper bound. - parent_version: Option, + /// The checkpoint sequence number at which this was viewed at + checkpoint_viewed_at: u64, + }, + + UnderParent { + /// The parent version to be used as an upper bound for the query. Look for the latest + /// version of a child object whose version is less than or equal to this upper bound. + parent_version: u64, /// The checkpoint sequence number at which this was viewed at checkpoint_viewed_at: u64, }, @@ -283,13 +288,21 @@ struct HistoricalKey { checkpoint_viewed_at: u64, } -/// DataLoader key for fetching the latest version of an `Object` as of a consistency cursor. The -/// query can optionally be bounded by a `parent_version` which imposes an additional requirement -/// that the object's version is bounded above by the parent version. +/// DataLoader key for fetching the latest version of an object whose parent object has version +/// `parent_version`, as of `checkpoint_viewed_at`. This look-up can fail to find a valid object if +/// the key is not self-consistent, for example if the `parent_version` is set to a higher version +/// than the object's actual parent as of `checkpoint_viewed_at`. +#[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] +struct ParentVersionKey { + id: SuiAddress, + parent_version: u64, + checkpoint_viewed_at: u64, +} + +/// DataLoader key for fetching the latest version of an `Object` as of a consistency cursor. #[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] struct LatestAtKey { id: SuiAddress, - parent_version: Option, checkpoint_viewed_at: u64, } @@ -807,7 +820,6 @@ impl Object { /// Look-up the latest version of the object as of a given checkpoint. pub(crate) fn latest_at(checkpoint_viewed_at: u64) -> ObjectLookup { ObjectLookup::LatestAt { - parent_version: None, checkpoint_viewed_at, } } @@ -815,8 +827,8 @@ impl Object { /// Look-up the latest version of an object whose version is less than or equal to its parent's /// version, as of a given checkpoint. pub(crate) fn under_parent(parent_version: u64, checkpoint_viewed_at: u64) -> ObjectLookup { - ObjectLookup::LatestAt { - parent_version: Some(parent_version), + ObjectLookup::UnderParent { + parent_version, checkpoint_viewed_at, } } @@ -849,18 +861,30 @@ impl Object { }) .await } - ObjectLookup::LatestAt { + + ObjectLookup::UnderParent { parent_version, checkpoint_viewed_at, } => { loader - .load_one(LatestAtKey { + .load_one(ParentVersionKey { id, parent_version, checkpoint_viewed_at, }) .await } + + ObjectLookup::LatestAt { + checkpoint_viewed_at, + } => { + loader + .load_one(LatestAtKey { + id, + checkpoint_viewed_at, + }) + .await + } } } @@ -1177,7 +1201,8 @@ impl Loader for Db { type Error = Error; async fn load(&self, keys: &[HistoricalKey]) -> Result, Error> { - use objects_history::dsl; + use objects_history::dsl as h; + use objects_version::dsl as v; let id_versions: BTreeSet<_> = keys .iter() @@ -1187,12 +1212,19 @@ impl Loader for Db { let objects: Vec = self .execute(move |conn| { conn.results(move || { - let mut query = dsl::objects_history.into_boxed(); + let mut query = h::objects_history + .inner_join( + v::objects_version.on(v::cp_sequence_number + .eq(h::checkpoint_sequence_number) + .and(v::object_id.eq(h::object_id)) + .and(v::object_version.eq(h::object_version))), + ) + .select(StoredHistoryObject::as_select()) + .into_boxed(); - // TODO: Speed up using an `obj_version` table. for (id, version) in id_versions.iter().cloned() { - query = query - .or_filter(dsl::object_id.eq(id).and(dsl::object_version.eq(version))); + query = + query.or_filter(v::object_id.eq(id).and(v::object_version.eq(version))); } query @@ -1234,17 +1266,20 @@ impl Loader for Db { } #[async_trait::async_trait] -impl Loader for Db { +impl Loader for Db { type Value = Object; type Error = Error; - async fn load(&self, keys: &[LatestAtKey]) -> Result, Error> { + async fn load( + &self, + keys: &[ParentVersionKey], + ) -> Result, Error> { // Group keys by checkpoint viewed at and parent version -- we'll issue a separate query for // each group. #[derive(Eq, PartialEq, Ord, PartialOrd, Clone, Copy)] struct GroupKey { checkpoint_viewed_at: u64, - parent_version: Option, + parent_version: u64, } let mut keys_by_cursor_and_parent_version: BTreeMap<_, BTreeSet<_>> = BTreeMap::new(); @@ -1257,50 +1292,40 @@ impl Loader for Db { keys_by_cursor_and_parent_version .entry(group_key) .or_default() - .insert(key.id); + .insert(key.id.into_vec()); } // Issue concurrent reads for each group of keys. let futures = keys_by_cursor_and_parent_version .into_iter() .map(|(group_key, ids)| { - self.execute_repeatable(move |conn| { - let Some(range) = AvailableRange::result(conn, group_key.checkpoint_viewed_at)? - else { - return Ok::, diesel::result::Error>( - vec![], - ); - }; - - let filter = ObjectFilter { - object_ids: Some(ids.iter().cloned().collect()), - ..Default::default() - }; - - // TODO: Implement queries that use a parent version bound using an - // `obj_version` table. - let apply_parent_bound = |q: RawQuery| { - if let Some(parent_version) = group_key.parent_version { - filter!(q, format!("object_version <= {parent_version}")) - } else { - q - } - }; - - Ok(conn - .results(move || { - build_objects_query( - View::Consistent, - range, - &Page::bounded(ids.len() as u64), - |q| apply_parent_bound(filter.apply(q)), - apply_parent_bound, + self.execute(move |conn| { + let stored: Vec = conn.results(move || { + use objects_history::dsl as h; + use objects_version::dsl as v; + + h::objects_history + .inner_join( + v::objects_version.on(v::cp_sequence_number + .eq(h::checkpoint_sequence_number) + .and(v::object_id.eq(h::object_id)) + .and(v::object_version.eq(h::object_version))), ) + .select(StoredHistoryObject::as_select()) + .filter(v::object_id.eq_any(ids.iter().cloned())) + .filter(v::object_version.le(group_key.parent_version as i64)) + .distinct_on(v::object_id) + .order_by(v::object_id) + .then_order_by(v::object_version.desc()) .into_boxed() - })? - .into_iter() - .map(|r| (group_key, r)) - .collect()) + })?; + + Ok::<_, diesel::result::Error>( + stored + .into_iter() + .map(|stored| (group_key, stored)) + .collect::>(), + ) }) }); @@ -1312,15 +1337,21 @@ impl Loader for Db { for (group_key, stored) in group.map_err(|e| Error::Internal(format!("Failed to fetch objects: {e}")))? { + // This particular object is invalid -- it didn't exist at the checkpoint we are + // viewing at. + if group_key.checkpoint_viewed_at < stored.checkpoint_sequence_number as u64 { + continue; + } + let object = Object::try_from_stored_history_object( stored, group_key.checkpoint_viewed_at, // If `LatestAtKey::parent_version` is set, it must have been correctly // propagated from the `Object::root_version` of some object. - group_key.parent_version, + Some(group_key.parent_version), )?; - let key = LatestAtKey { + let key = ParentVersionKey { id: object.address, checkpoint_viewed_at: group_key.checkpoint_viewed_at, parent_version: group_key.parent_version, @@ -1334,6 +1365,81 @@ impl Loader for Db { } } +#[async_trait::async_trait] +impl Loader for Db { + type Value = Object; + type Error = Error; + + async fn load(&self, keys: &[LatestAtKey]) -> Result, Error> { + // Group keys by checkpoint viewed at -- we'll issue a separate query for each group. + let mut keys_by_cursor_and_parent_version: BTreeMap<_, BTreeSet<_>> = BTreeMap::new(); + + for key in keys { + keys_by_cursor_and_parent_version + .entry(key.checkpoint_viewed_at) + .or_default() + .insert(key.id); + } + + // Issue concurrent reads for each group of keys. + let futures = + keys_by_cursor_and_parent_version + .into_iter() + .map(|(checkpoint_viewed_at, ids)| { + self.execute_repeatable(move |conn| { + let Some(range) = AvailableRange::result(conn, checkpoint_viewed_at)? + else { + return Ok::, diesel::result::Error>( + vec![], + ); + }; + + let filter = ObjectFilter { + object_ids: Some(ids.iter().cloned().collect()), + ..Default::default() + }; + + Ok(conn + .results(move || { + build_objects_query( + View::Consistent, + range, + &Page::bounded(ids.len() as u64), + |q| filter.apply(q), + |q| q, + ) + .into_boxed() + })? + .into_iter() + .map(|r| (checkpoint_viewed_at, r)) + .collect()) + }) + }); + + // Wait for the reads to all finish, and gather them into the result map. + let groups = futures::future::join_all(futures).await; + + let mut results = HashMap::new(); + for group in groups { + for (checkpoint_viewed_at, stored) in + group.map_err(|e| Error::Internal(format!("Failed to fetch objects: {e}")))? + { + let object = + Object::try_from_stored_history_object(stored, checkpoint_viewed_at, None)?; + + let key = LatestAtKey { + id: object.address, + checkpoint_viewed_at, + }; + + results.insert(key, object); + } + } + + Ok(results) + } +} + impl From<&ObjectKind> for ObjectStatus { fn from(kind: &ObjectKind) -> Self { match kind { diff --git a/crates/sui-graphql-rpc/src/types/owner.rs b/crates/sui-graphql-rpc/src/types/owner.rs index 79525ca9e9921..3986a39f6ad23 100644 --- a/crates/sui-graphql-rpc/src/types/owner.rs +++ b/crates/sui-graphql-rpc/src/types/owner.rs @@ -251,9 +251,10 @@ impl Owner { Object::query( ctx, self.address, - object::ObjectLookup::LatestAt { - parent_version: self.root_version, - checkpoint_viewed_at: self.checkpoint_viewed_at, + if let Some(parent_version) = self.root_version { + Object::under_parent(parent_version, self.checkpoint_viewed_at) + } else { + Object::latest_at(self.checkpoint_viewed_at) }, ) .await From eb0f2731f95aed588976c69a475f75eea17f9da3 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:35:30 +0100 Subject: [PATCH 2/9] [GraphQL/MovePackage] Query by ID and version (#17692) ## Description Implement `Query.package` and `MovePackage.atVersion` to query a package at a specific version, using the new fields added to the `packages` table, exposed via some new data loaders. ## Test plan New transactional tests: ``` sui$ cargo nextest run -p sui-graphql-e2e-tests \ --features pg_integration \ -- packages/versioning ``` ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 - #17726 - #17543 - #17692 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Introduce `Query.package` and `MovePackage.atVersion` to query packages at specific versions. - [ ] CLI: - [ ] Rust SDK: --- .../tests/packages/versioning.exp | 318 ++++++++++++++++++ .../tests/packages/versioning.move | 171 ++++++++++ .../schema/current_progress_schema.graphql | 18 + .../sui-graphql-rpc/src/types/move_module.rs | 7 +- .../sui-graphql-rpc/src/types/move_package.rs | 157 ++++++++- crates/sui-graphql-rpc/src/types/object.rs | 11 +- crates/sui-graphql-rpc/src/types/query.rs | 41 ++- .../sui-graphql-rpc/src/types/sui_address.rs | 15 +- .../snapshot_tests__schema_sdl_export.snap | 18 + 9 files changed, 726 insertions(+), 30 deletions(-) create mode 100644 crates/sui-graphql-e2e-tests/tests/packages/versioning.exp create mode 100644 crates/sui-graphql-e2e-tests/tests/packages/versioning.move diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp new file mode 100644 index 0000000000000..4c37560181402 --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp @@ -0,0 +1,318 @@ +processed 9 tasks + +init: +A: object(0,0) + +task 1, lines 6-9: +//# publish --upgradeable --sender A +created: object(1,0), object(1,1) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 5076800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 11-15: +//# upgrade --package P0 --upgrade-capability 1,1 --sender A +created: object(2,0) +mutated: object(0,0), object(1,1) +gas summary: computation_cost: 1000000, storage_cost: 5251600, storage_rebate: 2595780, non_refundable_storage_fee: 26220 + +task 3, lines 17-22: +//# upgrade --package P1 --upgrade-capability 1,1 --sender A +created: object(3,0) +mutated: object(0,0), object(1,1) +gas summary: computation_cost: 1000000, storage_cost: 5426400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 + +task 4, line 24: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 26-45: +//# run-graphql +Response: { + "data": { + "v1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v3": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + } +} + +task 6, lines 47-84: +//# run-graphql +Response: { + "data": { + "v1_from_p1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v1_from_p2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v2_from_p0": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v2_from_p2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v3_from_p0": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + }, + "v3_from_p1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + } +} + +task 7, lines 86-141: +//# run-graphql +Response: { + "data": { + "v1": { + "v1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v3": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + }, + "v2": { + "v1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v3": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + }, + "v3": { + "v1": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + }, + "v2": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + }, + "v3": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + } + } +} + +task 8, lines 143-171: +//# run-graphql +Response: { + "data": { + "v0": null, + "v1": { + "v0": null, + "v4": null + }, + "v4": null + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move new file mode 100644 index 0000000000000..72bdc66db632f --- /dev/null +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move @@ -0,0 +1,171 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 39 --addresses P0=0x0 P1=0x0 P2=0x0 --accounts A --simulator + +//# publish --upgradeable --sender A +module P0::m { + public fun f(): u64 { 42 } +} + +//# upgrade --package P0 --upgrade-capability 1,1 --sender A +module P1::m { + public fun f(): u64 { 42 } + public fun g(): u64 { 43 } +} + +//# upgrade --package P1 --upgrade-capability 1,1 --sender A +module P2::m { + public fun f(): u64 { 42 } + public fun g(): u64 { 43 } + public fun h(): u64 { 44 } +} + +//# create-checkpoint + +//# run-graphql +{ # Test fetching by ID + v1: package(address: "@{P0}") { + module(name: "m") { + functions { nodes { name } } + } + } + + v2: package(address: "@{P1}") { + module(name: "m") { + functions { nodes { name } } + } + } + + v3: package(address: "@{P2}") { + module(name: "m") { + functions { nodes { name } } + } + } +} + +//# run-graphql +{ # Test fetching by version + v1_from_p1: package(address: "@{P1}", version: 1) { + module(name: "m") { + functions { nodes { name } } + } + } + + v1_from_p2: package(address: "@{P2}", version: 1) { + module(name: "m") { + functions { nodes { name } } + } + } + + v2_from_p0: package(address: "@{P0}", version: 2) { + module(name: "m") { + functions { nodes { name } } + } + } + + v2_from_p2: package(address: "@{P2}", version: 2) { + module(name: "m") { + functions { nodes { name } } + } + } + + v3_from_p0: package(address: "@{P0}", version: 3) { + module(name: "m") { + functions { nodes { name } } + } + } + + v3_from_p1: package(address: "@{P1}", version: 3) { + module(name: "m") { + functions { nodes { name } } + } + } +} + +//# run-graphql +{ # Go from one version to another using packageAtVersion + v1: package(address: "@{P1}") { + v1: packageAtVersion(version: 1) { + module(name: "m") { + functions { nodes { name } } + } + } + v2: packageAtVersion(version: 2) { + module(name: "m") { + functions { nodes { name } } + } + } + v3: packageAtVersion(version: 3) { + module(name: "m") { + functions { nodes { name } } + } + } + } + + v2: package(address: "@{P2}") { + v1: packageAtVersion(version: 1) { + module(name: "m") { + functions { nodes { name } } + } + } + v2: packageAtVersion(version: 2) { + module(name: "m") { + functions { nodes { name } } + } + } + v3: packageAtVersion(version: 3) { + module(name: "m") { + functions { nodes { name } } + } + } + } + + v3: package(address: "@{P2}") { + v1: packageAtVersion(version: 1) { + module(name: "m") { + functions { nodes { name } } + } + } + v2: packageAtVersion(version: 2) { + module(name: "m") { + functions { nodes { name } } + } + } + v3: packageAtVersion(version: 3) { + module(name: "m") { + functions { nodes { name } } + } + } + } +} + +//# run-graphql +{ # Fetch out of range versions (should return null) + v0: package(address: "@{P0}", version: 0) { + module(name: "m") { + functions { nodes { name } } + } + } + + # This won't return null, but its inner queries will + v1: package(address: "@{P0}") { + v0: packageAtVersion(version: 0) { + module(name: "m") { + functions { nodes { name } } + } + } + + v4: packageAtVersion(version: 4) { + module(name: "m") { + functions { nodes { name } } + } + } + } + + v4: package(address: "@{P0}", version: 4) { + module(name: "m") { + functions { nodes { name } } + } + } +} diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql index cbde89c2c6fc5..201b62290eabe 100644 --- a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql +++ b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql @@ -2168,6 +2168,11 @@ type MovePackage implements IObject & IOwner { """ bcs: Base64 """ + Fetch another version of this package (the package that shares this package's original ID, + but has the specified `version`). + """ + packageAtVersion(version: Int!): MovePackage + """ A representation of the module called `name` in this package, including the structs and functions it defines. """ @@ -3040,6 +3045,19 @@ type Query { """ object(address: SuiAddress!, version: UInt53): Object """ + The package corresponding to the given address at the (optionally) given version. + + When no version is given, the package is loaded directly from the address given. Otherwise, + the address is translated before loading to point to the package whose original ID matches + the package at `address`, but whose version is `version`. For non-system packages, this may + result in a different address than `address` because different versions of a package, + introduced by upgrades, exist at distinct addresses. + + Note that this interpretation of `version` is different from a historical object read (the + interpretation of `version` for the `object` query). + """ + package(address: SuiAddress!, version: UInt53): MovePackage + """ Look-up an Account by its SuiAddress. """ address(address: SuiAddress!): Address diff --git a/crates/sui-graphql-rpc/src/types/move_module.rs b/crates/sui-graphql-rpc/src/types/move_module.rs index e34ad6c46a8bc..f85d6fe558abc 100644 --- a/crates/sui-graphql-rpc/src/types/move_module.rs +++ b/crates/sui-graphql-rpc/src/types/move_module.rs @@ -15,7 +15,6 @@ use super::datatype::MoveDatatype; use super::move_enum::MoveEnum; use super::move_function::MoveFunction; use super::move_struct::MoveStruct; -use super::object::Object; use super::{base64::Base64, move_package::MovePackage, sui_address::SuiAddress}; #[derive(Clone)] @@ -40,7 +39,7 @@ impl MoveModule { MovePackage::query( ctx, self.storage_id, - Object::latest_at(self.checkpoint_viewed_at), + MovePackage::by_id_at(self.checkpoint_viewed_at), ) .await .extend()? @@ -91,7 +90,7 @@ impl MoveModule { let Some(package) = MovePackage::query( ctx, self.storage_id, - Object::latest_at(checkpoint_viewed_at), + MovePackage::by_id_at(checkpoint_viewed_at), ) .await .extend()? @@ -482,7 +481,7 @@ impl MoveModule { checkpoint_viewed_at: u64, ) -> Result, Error> { let Some(package) = - MovePackage::query(ctx, address, Object::latest_at(checkpoint_viewed_at)).await? + MovePackage::query(ctx, address, MovePackage::by_id_at(checkpoint_viewed_at)).await? else { return Ok(None); }; diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index 1791b0bde32ca..a9dbeb0188f15 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -1,6 +1,8 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::collections::{BTreeSet, HashMap}; + use super::balance::{self, Balance}; use super::base64::Base64; use super::big_int::BigInt; @@ -8,9 +10,7 @@ use super::coin::Coin; use super::cursor::{JsonCursor, Page}; use super::move_module::MoveModule; use super::move_object::MoveObject; -use super::object::{ - self, Object, ObjectFilter, ObjectImpl, ObjectLookup, ObjectOwner, ObjectStatus, -}; +use super::object::{self, Object, ObjectFilter, ObjectImpl, ObjectOwner, ObjectStatus}; use super::owner::OwnerImpl; use super::stake::StakedSui; use super::sui_address::SuiAddress; @@ -19,10 +19,16 @@ use super::transaction_block::{self, TransactionBlock, TransactionBlockFilter}; use super::type_filter::ExactTypeFilter; use super::uint53::UInt53; use crate::consistency::ConsistentNamedCursor; +use crate::data::{DataLoader, Db, DbConnection, QueryExecutor}; use crate::error::Error; +use crate::types::sui_address::addr; use async_graphql::connection::{Connection, CursorType, Edge}; +use async_graphql::dataloader::Loader; use async_graphql::*; +use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl}; +use sui_indexer::schema::packages; use sui_package_resolver::{error::Error as PackageCacheError, Package as ParsedMovePackage}; +use sui_types::is_system_package; use sui_types::{move_package::MovePackage as NativeMovePackage, object::Data}; #[derive(Clone)] @@ -35,6 +41,21 @@ pub(crate) struct MovePackage { pub native: NativeMovePackage, } +/// Filter for a point query of a MovePackage, supporting querying different versions of a package +/// by their version. Note that different versions of the same user package exist at different IDs +/// to each other, so this is different from looking up the historical version of an object. +pub(crate) enum PackageLookup { + /// Get the package at the given address, if it was created before the given checkpoint. + ById { checkpoint_viewed_at: u64 }, + + /// Get the package whose original ID matches the storage ID of the package at the given + /// address, but whose version is `version`. + Versioned { + version: u64, + checkpoint_viewed_at: u64, + }, +} + /// Information used by a package to link to a specific version of its dependency. #[derive(SimpleObject)] struct Linkage { @@ -66,6 +87,18 @@ pub(crate) struct MovePackageDowncastError; pub(crate) type CModule = JsonCursor; +/// DataLoader key for fetching the storage ID of the (user) package that shares an original (aka +/// runtime) ID with the package stored at `package_id`, and whose version is `version`. +/// +/// Note that this is different from looking up the historical version of an object -- the query +/// returns the ID of the package (each version of a user package is at a different ID) -- and it +/// does not work for system packages (whose versions do all reside under the same ID). +#[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] +struct PackageVersionKey { + address: SuiAddress, + version: u64, +} + /// A MovePackage is a kind of Move object that represents code that has been published on chain. /// It exposes information about its modules, type definitions, functions, and dependencies. #[Object] @@ -255,6 +288,22 @@ impl MovePackage { ObjectImpl(&self.super_).bcs().await } + /// Fetch another version of this package (the package that shares this package's original ID, + /// but has the specified `version`). + async fn package_at_version( + &self, + ctx: &Context<'_>, + version: u64, + ) -> Result> { + MovePackage::query( + ctx, + self.super_.address, + MovePackage::by_version(version, self.checkpoint_viewed_at_impl()), + ) + .await + .extend() + } + /// A representation of the module called `name` in this package, including the /// structs and functions it defines. async fn module(&self, name: String) -> Result> { @@ -416,11 +465,53 @@ impl MovePackage { } } + /// Look-up the package by its Storage ID, as of a given checkpoint. + pub(crate) fn by_id_at(checkpoint_viewed_at: u64) -> PackageLookup { + PackageLookup::ById { + checkpoint_viewed_at, + } + } + + /// Look-up a specific version of the package, identified by the storage ID of any version of + /// the package, and the desired version (the actual object loaded might be at a different + /// object ID). + pub(crate) fn by_version(version: u64, checkpoint_viewed_at: u64) -> PackageLookup { + PackageLookup::Versioned { + version, + checkpoint_viewed_at, + } + } + pub(crate) async fn query( ctx: &Context<'_>, address: SuiAddress, - key: ObjectLookup, + key: PackageLookup, ) -> Result, Error> { + let (address, key) = match key { + PackageLookup::ById { + checkpoint_viewed_at, + } => (address, Object::latest_at(checkpoint_viewed_at)), + + PackageLookup::Versioned { + version, + checkpoint_viewed_at, + } => { + if is_system_package(address) { + (address, Object::at_version(version, checkpoint_viewed_at)) + } else { + let DataLoader(loader) = &ctx.data_unchecked(); + let Some(translation) = loader + .load_one(PackageVersionKey { address, version }) + .await? + else { + return Ok(None); + }; + + (translation, Object::latest_at(checkpoint_viewed_at)) + } + } + }; + let Some(object) = Object::query(ctx, address, key).await? else { return Ok(None); }; @@ -431,6 +522,64 @@ impl MovePackage { } } +#[async_trait::async_trait] +impl Loader for Db { + type Value = SuiAddress; + type Error = Error; + + async fn load( + &self, + keys: &[PackageVersionKey], + ) -> Result, Error> { + use packages::dsl; + let other = diesel::alias!(packages as other); + + let id_versions: BTreeSet<_> = keys + .iter() + .map(|k| (k.address.into_vec(), k.version as i64)) + .collect(); + + let stored_packages: Vec<(Vec, i64, Vec)> = self + .execute(move |conn| { + conn.results(|| { + let mut query = dsl::packages + .inner_join(other.on(dsl::original_id.eq(other.field(dsl::original_id)))) + .select(( + dsl::package_id, + other.field(dsl::package_version), + other.field(dsl::package_id), + )) + .into_boxed(); + + for (id, version) in id_versions.iter().cloned() { + query = query.or_filter( + dsl::package_id + .eq(id) + .and(other.field(dsl::package_version).eq(version)), + ); + } + + query + }) + }) + .await + .map_err(|e| Error::Internal(format!("Failed to load packages: {e}")))?; + + let mut result = HashMap::new(); + for (id, version, other_id) in stored_packages { + result.insert( + PackageVersionKey { + address: addr(&id)?, + version: version as u64, + }, + addr(&other_id)?, + ); + } + + Ok(result) + } +} + impl TryFrom<&Object> for MovePackage { type Error = MovePackageDowncastError; diff --git a/crates/sui-graphql-rpc/src/types/object.rs b/crates/sui-graphql-rpc/src/types/object.rs index e9177a122057d..cd3e18709760c 100644 --- a/crates/sui-graphql-rpc/src/types/object.rs +++ b/crates/sui-graphql-rpc/src/types/object.rs @@ -17,6 +17,7 @@ use super::move_object::MoveObject; use super::move_package::MovePackage; use super::owner::OwnerImpl; use super::stake::StakedSui; +use super::sui_address::addr; use super::suins_registration::{DomainFormat, SuinsRegistration}; use super::transaction_block; use super::transaction_block::TransactionBlockFilter; @@ -181,6 +182,7 @@ pub(crate) struct AddressOwner { owner: Option, } +/// Filter for a point query of an Object. pub(crate) enum ObjectLookup { LatestAt { /// The checkpoint sequence number at which this was viewed at @@ -1459,15 +1461,6 @@ impl From<&Object> for OwnerImpl { } } -/// Parse a `SuiAddress` from its stored representation. Failure is an internal error: the -/// database should never contain a malformed address (containing the wrong number of bytes). -fn addr(bytes: impl AsRef<[u8]>) -> Result { - SuiAddress::from_bytes(bytes.as_ref()).map_err(|e| { - let bytes = bytes.as_ref().to_vec(); - Error::Internal(format!("Error deserializing address: {bytes:?}: {e}")) - }) -} - pub(crate) async fn deserialize_move_struct( move_object: &NativeMoveObject, resolver: &PackageResolver, diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index 10a6f4c9a8464..46334df30b1fd 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -12,6 +12,7 @@ use sui_sdk::SuiClient; use sui_types::transaction::{TransactionData, TransactionKind}; use sui_types::{gas_coin::GAS, transaction::TransactionDataAPI, TypeTag}; +use super::move_package::MovePackage; use super::suins_registration::NameService; use super::uint53::UInt53; use super::{ @@ -214,17 +215,37 @@ impl Query { version: Option, ) -> Result> { let Watermark { checkpoint, .. } = *ctx.data()?; + let key = match version { + Some(version) => Object::at_version(version.into(), checkpoint), + None => Object::latest_at(checkpoint), + }; + + Object::query(ctx, address, key).await.extend() + } + + /// The package corresponding to the given address at the (optionally) given version. + /// + /// When no version is given, the package is loaded directly from the address given. Otherwise, + /// the address is translated before loading to point to the package whose original ID matches + /// the package at `address`, but whose version is `version`. For non-system packages, this may + /// result in a different address than `address` because different versions of a package, + /// introduced by upgrades, exist at distinct addresses. + /// + /// Note that this interpretation of `version` is different from a historical object read (the + /// interpretation of `version` for the `object` query). + async fn package( + &self, + ctx: &Context<'_>, + address: SuiAddress, + version: Option, + ) -> Result> { + let Watermark { checkpoint, .. } = *ctx.data()?; + let key = match version { + Some(version) => MovePackage::by_version(version.into(), checkpoint), + None => MovePackage::by_id_at(checkpoint), + }; - match version { - Some(version) => { - Object::query(ctx, address, Object::at_version(version.into(), checkpoint)) - .await - .extend() - } - None => Object::query(ctx, address, Object::latest_at(checkpoint)) - .await - .extend(), - } + MovePackage::query(ctx, address, key).await.extend() } /// Look-up an Account by its SuiAddress. diff --git a/crates/sui-graphql-rpc/src/types/sui_address.rs b/crates/sui-graphql-rpc/src/types/sui_address.rs index 287bf0540e887..3a775e15064d4 100644 --- a/crates/sui-graphql-rpc/src/types/sui_address.rs +++ b/crates/sui-graphql-rpc/src/types/sui_address.rs @@ -3,18 +3,18 @@ use std::str::FromStr; +use crate::error::Error; use async_graphql::*; use move_core_types::account_address::AccountAddress; use serde::{Deserialize, Serialize}; use sui_types::base_types::{ObjectID, SuiAddress as NativeSuiAddress}; -use thiserror::Error; const SUI_ADDRESS_LENGTH: usize = 32; #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Copy)] pub(crate) struct SuiAddress([u8; SUI_ADDRESS_LENGTH]); -#[derive(Error, Debug, Eq, PartialEq)] +#[derive(thiserror::Error, Debug, Eq, PartialEq)] pub(crate) enum FromStrError { #[error("Invalid SuiAddress. Missing 0x prefix.")] NoPrefix, @@ -30,7 +30,7 @@ pub(crate) enum FromStrError { BadHex(char, usize), } -#[derive(Error, Debug, Eq, PartialEq)] +#[derive(thiserror::Error, Debug, Eq, PartialEq)] pub(crate) enum FromVecError { #[error("Expected SuiAddress with {} bytes, received {0}", SUI_ADDRESS_LENGTH)] WrongLength(usize), @@ -161,6 +161,15 @@ impl std::fmt::Display for SuiAddress { } } +/// Parse a `SuiAddress` from its stored representation. Failure is an internal error: the +/// database should never contain a malformed address (containing the wrong number of bytes). +pub(crate) fn addr(bytes: impl AsRef<[u8]>) -> Result { + SuiAddress::from_bytes(bytes.as_ref()).map_err(|e| { + let bytes = bytes.as_ref().to_vec(); + Error::Internal(format!("Error deserializing address: {bytes:?}: {e}")) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap index c82e1181d7f6f..809354db88972 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap @@ -2172,6 +2172,11 @@ type MovePackage implements IObject & IOwner { """ bcs: Base64 """ + Fetch another version of this package (the package that shares this package's original ID, + but has the specified `version`). + """ + packageAtVersion(version: Int!): MovePackage + """ A representation of the module called `name` in this package, including the structs and functions it defines. """ @@ -3044,6 +3049,19 @@ type Query { """ object(address: SuiAddress!, version: UInt53): Object """ + The package corresponding to the given address at the (optionally) given version. + + When no version is given, the package is loaded directly from the address given. Otherwise, + the address is translated before loading to point to the package whose original ID matches + the package at `address`, but whose version is `version`. For non-system packages, this may + result in a different address than `address` because different versions of a package, + introduced by upgrades, exist at distinct addresses. + + Note that this interpretation of `version` is different from a historical object read (the + interpretation of `version` for the `object` query). + """ + package(address: SuiAddress!, version: UInt53): MovePackage + """ Look-up an Account by its SuiAddress. """ address(address: SuiAddress!): Address From bbba33ca8fd8133557c61f86a2393cb2b06b0d15 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:38:54 +0100 Subject: [PATCH 3/9] [GraphQL/MovePackage] Query for latest version (#17693) ## Description Add a new kind of package point look-up to get the latest version of the package at a given ID (or from another `MovePackage`). For system packages, this is analogous to getting the latest version of the object at that ID, but the versions of other packages all exist at different IDs. ## Test plan New transactional tests: ``` sui$ cargo nextest run -p sui-graphql-e2e-tests \ --features pg_integration \ -- packages/versioning ``` ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 - #17726 - #17543 - #17692 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Add `Query.latestPackage` and `MovePackage.latest` for fetching the latest version of a package. - [ ] CLI: - [ ] Rust SDK: --- .../tests/packages/versioning.exp | 147 ++++++++++++++++-- .../tests/packages/versioning.move | 52 +++++++ .../schema/current_progress_schema.graphql | 12 ++ .../sui-graphql-rpc/src/types/move_package.rs | 125 ++++++++++++++- crates/sui-graphql-rpc/src/types/query.rs | 15 ++ .../snapshot_tests__schema_sdl_export.snap | 12 ++ 6 files changed, 351 insertions(+), 12 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp index 4c37560181402..7b8ad9c4c32cc 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp @@ -1,4 +1,4 @@ -processed 9 tasks +processed 14 tasks init: A: object(0,0) @@ -9,23 +9,97 @@ created: object(1,0), object(1,1) mutated: object(0,0) gas summary: computation_cost: 1000000, storage_cost: 5076800, storage_rebate: 0, non_refundable_storage_fee: 0 -task 2, lines 11-15: +task 2, line 11: +//# create-checkpoint +Checkpoint created: 1 + +task 3, lines 13-21: +//# run-graphql +Response: { + "data": { + "latestPackage": { + "version": 1, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + } + } + } +} + +task 4, lines 23-27: //# upgrade --package P0 --upgrade-capability 1,1 --sender A -created: object(2,0) +created: object(4,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5251600, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 3, lines 17-22: +task 5, line 29: +//# create-checkpoint +Checkpoint created: 2 + +task 6, lines 31-39: +//# run-graphql +Response: { + "data": { + "latestPackage": { + "version": 2, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + } + ] + } + } + } + } +} + +task 7, lines 41-46: //# upgrade --package P1 --upgrade-capability 1,1 --sender A -created: object(3,0) +created: object(7,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5426400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 4, line 24: +task 8, line 48: //# create-checkpoint -Checkpoint created: 1 +Checkpoint created: 3 -task 5, lines 26-45: +task 9, lines 50-58: +//# run-graphql +Response: { + "data": { + "latestPackage": { + "version": 3, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } + } + } +} + +task 10, lines 60-97: //# run-graphql Response: { "data": { @@ -38,6 +112,23 @@ Response: { } ] } + }, + "latestPackage": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } } }, "v2": { @@ -52,6 +143,23 @@ Response: { } ] } + }, + "latestPackage": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } } }, "v3": { @@ -69,12 +177,29 @@ Response: { } ] } + }, + "latestPackage": { + "module": { + "functions": { + "nodes": [ + { + "name": "f" + }, + { + "name": "g" + }, + { + "name": "h" + } + ] + } + } } } } } -task 6, lines 47-84: +task 11, lines 99-136: //# run-graphql Response: { "data": { @@ -165,7 +290,7 @@ Response: { } } -task 7, lines 86-141: +task 12, lines 138-193: //# run-graphql Response: { "data": { @@ -304,7 +429,7 @@ Response: { } } -task 8, lines 143-171: +task 13, lines 195-223: //# run-graphql Response: { "data": { diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move index 72bdc66db632f..f4646294722b7 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move @@ -8,12 +8,36 @@ module P0::m { public fun f(): u64 { 42 } } +//# create-checkpoint + +//# run-graphql +{ + latestPackage(address: "@{P0}") { + version + module(name: "m") { + functions { nodes { name } } + } + } +} + //# upgrade --package P0 --upgrade-capability 1,1 --sender A module P1::m { public fun f(): u64 { 42 } public fun g(): u64 { 43 } } +//# create-checkpoint + +//# run-graphql +{ + latestPackage(address: "@{P0}") { + version + module(name: "m") { + functions { nodes { name } } + } + } +} + //# upgrade --package P1 --upgrade-capability 1,1 --sender A module P2::m { public fun f(): u64 { 42 } @@ -23,24 +47,52 @@ module P2::m { //# create-checkpoint +//# run-graphql +{ + latestPackage(address: "@{P0}") { + version + module(name: "m") { + functions { nodes { name } } + } + } +} + //# run-graphql { # Test fetching by ID v1: package(address: "@{P0}") { module(name: "m") { functions { nodes { name } } } + + latestPackage { + module(name: "m") { + functions { nodes { name } } + } + } } v2: package(address: "@{P1}") { module(name: "m") { functions { nodes { name } } } + + latestPackage { + module(name: "m") { + functions { nodes { name } } + } + } } v3: package(address: "@{P2}") { module(name: "m") { functions { nodes { name } } } + + latestPackage { + module(name: "m") { + functions { nodes { name } } + } + } } } diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql index 201b62290eabe..69472d0578045 100644 --- a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql +++ b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql @@ -2173,6 +2173,11 @@ type MovePackage implements IObject & IOwner { """ packageAtVersion(version: Int!): MovePackage """ + Fetch the latest version of this package (the package with the highest `version` that shares + this packages's original ID) + """ + latestPackage: MovePackage! + """ A representation of the module called `name` in this package, including the structs and functions it defines. """ @@ -3058,6 +3063,13 @@ type Query { """ package(address: SuiAddress!, version: UInt53): MovePackage """ + The latest version of the package at `address`. + + This corresponds to the package with the highest `version` that shares its original ID with + the package at `address`. + """ + latestPackage(address: SuiAddress!): MovePackage + """ Look-up an Account by its SuiAddress. """ address(address: SuiAddress!): Address diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index a9dbeb0188f15..1e8490be2465b 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -1,7 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use super::balance::{self, Balance}; use super::base64::Base64; @@ -54,6 +54,10 @@ pub(crate) enum PackageLookup { version: u64, checkpoint_viewed_at: u64, }, + + /// Get the package whose original ID matches the storage ID of the package at the given + /// address, but that has the max version at the given checkpoint. + Latest { checkpoint_viewed_at: u64 }, } /// Information used by a package to link to a specific version of its dependency. @@ -99,6 +103,14 @@ struct PackageVersionKey { version: u64, } +/// DataLoader key for fetching the latest version of a user package: The package with the largest +/// version whose original ID matches the original ID of the package at `address`. +#[derive(Copy, Clone, Hash, Eq, PartialEq, Debug)] +struct LatestKey { + address: SuiAddress, + checkpoint_viewed_at: u64, +} + /// A MovePackage is a kind of Move object that represents code that has been published on chain. /// It exposes information about its modules, type definitions, functions, and dependencies. #[Object] @@ -304,6 +316,19 @@ impl MovePackage { .extend() } + /// Fetch the latest version of this package (the package with the highest `version` that shares + /// this packages's original ID) + async fn latest_package(&self, ctx: &Context<'_>) -> Result { + Ok(MovePackage::query( + ctx, + self.super_.address, + MovePackage::latest_at(self.checkpoint_viewed_at_impl()), + ) + .await + .extend()? + .ok_or_else(|| Error::Internal("No latest version found".to_string()))?) + } + /// A representation of the module called `name` in this package, including the /// structs and functions it defines. async fn module(&self, name: String) -> Result> { @@ -482,6 +507,14 @@ impl MovePackage { } } + /// Look-up the package that shares the same original ID as the package at `address`, but has + /// the latest version, as of the given checkpoint. + pub(crate) fn latest_at(checkpoint_viewed_at: u64) -> PackageLookup { + PackageLookup::Latest { + checkpoint_viewed_at, + } + } + pub(crate) async fn query( ctx: &Context<'_>, address: SuiAddress, @@ -510,6 +543,27 @@ impl MovePackage { (translation, Object::latest_at(checkpoint_viewed_at)) } } + + PackageLookup::Latest { + checkpoint_viewed_at, + } => { + if is_system_package(address) { + (address, Object::latest_at(checkpoint_viewed_at)) + } else { + let DataLoader(loader) = &ctx.data_unchecked(); + let Some(translation) = loader + .load_one(LatestKey { + address, + checkpoint_viewed_at, + }) + .await? + else { + return Ok(None); + }; + + (translation, Object::latest_at(checkpoint_viewed_at)) + } + } }; let Some(object) = Object::query(ctx, address, key).await? else { @@ -580,6 +634,75 @@ impl Loader for Db { } } +#[async_trait::async_trait] +impl Loader for Db { + type Value = SuiAddress; + type Error = Error; + + async fn load(&self, keys: &[LatestKey]) -> Result, Error> { + use packages::dsl; + let other = diesel::alias!(packages as other); + + let mut ids_by_cursor: BTreeMap<_, BTreeSet<_>> = BTreeMap::new(); + for key in keys { + ids_by_cursor + .entry(key.checkpoint_viewed_at) + .or_default() + .insert(key.address.into_vec()); + } + + // Issue concurrent reads for each group of IDs + let futures = ids_by_cursor + .into_iter() + .map(|(checkpoint_viewed_at, ids)| { + self.execute(move |conn| { + let results: Vec<(Vec, Vec)> = conn.results(|| { + let o_original_id = other.field(dsl::original_id); + let o_package_id = other.field(dsl::package_id); + let o_cp_seq_num = other.field(dsl::checkpoint_sequence_number); + let o_version = other.field(dsl::package_version); + + let query = dsl::packages + .inner_join(other.on(dsl::original_id.eq(o_original_id))) + .select((dsl::package_id, o_package_id)) + .filter(dsl::package_id.eq_any(ids.iter().cloned())) + .filter(o_cp_seq_num.le(checkpoint_viewed_at as i64)) + .order_by((dsl::package_id, dsl::original_id, o_version.desc())) + .distinct_on((dsl::package_id, dsl::original_id)); + query + })?; + + Ok::<_, diesel::result::Error>( + results + .into_iter() + .map(|(p, latest)| (checkpoint_viewed_at, p, latest)) + .collect::>(), + ) + }) + }); + + // Wait for the reads to all finish, and gather them into the result map. + let groups = futures::future::join_all(futures).await; + + let mut results = HashMap::new(); + for group in groups { + for (checkpoint_viewed_at, address, latest) in + group.map_err(|e| Error::Internal(format!("Failed to fetch packages: {e}")))? + { + results.insert( + LatestKey { + address: addr(&address)?, + checkpoint_viewed_at, + }, + addr(&latest)?, + ); + } + } + + Ok(results) + } +} + impl TryFrom<&Object> for MovePackage { type Error = MovePackageDowncastError; diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index 46334df30b1fd..5aa55e7334773 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -248,6 +248,21 @@ impl Query { MovePackage::query(ctx, address, key).await.extend() } + /// The latest version of the package at `address`. + /// + /// This corresponds to the package with the highest `version` that shares its original ID with + /// the package at `address`. + async fn latest_package( + &self, + ctx: &Context<'_>, + address: SuiAddress, + ) -> Result> { + let Watermark { checkpoint, .. } = *ctx.data()?; + MovePackage::query(ctx, address, MovePackage::latest_at(checkpoint)) + .await + .extend() + } + /// Look-up an Account by its SuiAddress. async fn address(&self, ctx: &Context<'_>, address: SuiAddress) -> Result> { let Watermark { checkpoint, .. } = *ctx.data()?; diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap index 809354db88972..61cabeeb1de13 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap @@ -2177,6 +2177,11 @@ type MovePackage implements IObject & IOwner { """ packageAtVersion(version: Int!): MovePackage """ + Fetch the latest version of this package (the package with the highest `version` that shares + this packages's original ID) + """ + latestPackage: MovePackage! + """ A representation of the module called `name` in this package, including the structs and functions it defines. """ @@ -3062,6 +3067,13 @@ type Query { """ package(address: SuiAddress!, version: UInt53): MovePackage """ + The latest version of the package at `address`. + + This corresponds to the package with the highest `version` that shares its original ID with + the package at `address`. + """ + latestPackage(address: SuiAddress!): MovePackage + """ Look-up an Account by its SuiAddress. """ address(address: SuiAddress!): Address From 65b74d4d2f47ba0007622e991ddc0410c7579852 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:40:25 +0100 Subject: [PATCH 4/9] [GraphQL/MovePackage] Paginate by checkpoint (#17696) ## Description Adds a query, `Query.packages` for fetching all packages that were introduced within a given checkpoint range. Useful for fetching package contents in bulk, to do local analyses. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-graphql-e2e-tests \ --features pg_integration \ -- packages/versioning ``` Also tested for performance against a large read replica (the query planner quotes a high estimate for the query but the actual results do not take very long to run because queries on many sub-partitions are eliminated). ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 - #17726 - #17543 - #17692 - #17693 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Introduces `Query.packages` for paginating through all packages (optionally bounding by the checkpoint the package was introduced in). - [ ] CLI: - [ ] Rust SDK: --- .../tests/packages/versioning.exp | 204 ++++++++++++++++-- .../tests/packages/versioning.move | 54 +++++ .../schema/current_progress_schema.graphql | 24 +++ crates/sui-graphql-rpc/src/types/event.rs | 12 +- .../sui-graphql-rpc/src/types/move_package.rs | 203 ++++++++++++++++- crates/sui-graphql-rpc/src/types/query.rs | 24 ++- .../snapshot_tests__schema_sdl_export.snap | 24 +++ 7 files changed, 523 insertions(+), 22 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp index 7b8ad9c4c32cc..7a0e6ec3faeed 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp @@ -1,4 +1,4 @@ -processed 14 tasks +processed 15 tasks init: A: object(0,0) @@ -13,7 +13,7 @@ task 2, line 11: //# create-checkpoint Checkpoint created: 1 -task 3, lines 13-21: +task 3, lines 13-28: //# run-graphql Response: { "data": { @@ -28,21 +28,45 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + } + ] } } } -task 4, lines 23-27: +task 4, lines 30-34: //# upgrade --package P0 --upgrade-capability 1,1 --sender A created: object(4,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5251600, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 5, line 29: +task 5, line 36: //# create-checkpoint Checkpoint created: 2 -task 6, lines 31-39: +task 6, lines 38-53: //# run-graphql Response: { "data": { @@ -60,21 +84,49 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] } } } -task 7, lines 41-46: +task 7, lines 55-60: //# upgrade --package P1 --upgrade-capability 1,1 --sender A created: object(7,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5426400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 8, line 48: +task 8, line 62: //# create-checkpoint Checkpoint created: 3 -task 9, lines 50-58: +task 9, lines 64-79: //# run-graphql Response: { "data": { @@ -95,11 +147,43 @@ Response: { ] } } + }, + "packages": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1 + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1 + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1 + }, + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] } } } -task 10, lines 60-97: +task 10, lines 81-118: //# run-graphql Response: { "data": { @@ -199,7 +283,7 @@ Response: { } } -task 11, lines 99-136: +task 11, lines 120-157: //# run-graphql Response: { "data": { @@ -290,7 +374,7 @@ Response: { } } -task 12, lines 138-193: +task 12, lines 159-214: //# run-graphql Response: { "data": { @@ -429,7 +513,7 @@ Response: { } } -task 13, lines 195-223: +task 13, lines 216-244: //# run-graphql Response: { "data": { @@ -441,3 +525,99 @@ Response: { "v4": null } } + +task 14, lines 246-277: +//# run-graphql +Response: { + "data": { + "before": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000002", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000003", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + }, + { + "address": "0x000000000000000000000000000000000000000000000000000000000000dee9", + "version": 1, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 0 + } + } + } + } + ] + }, + "after": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 2 + } + } + } + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 3 + } + } + } + } + ] + }, + "between": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2, + "previousTransactionBlock": { + "effects": { + "checkpoint": { + "sequenceNumber": 2 + } + } + } + } + ] + } + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move index f4646294722b7..694072fb9c445 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move @@ -18,6 +18,13 @@ module P0::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# upgrade --package P0 --upgrade-capability 1,1 --sender A @@ -36,6 +43,13 @@ module P1::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# upgrade --package P1 --upgrade-capability 1,1 --sender A @@ -55,6 +69,13 @@ module P2::m { functions { nodes { name } } } } + + packages(first: 10) { + nodes { + address + version + } + } } //# run-graphql @@ -221,3 +242,36 @@ module P2::m { } } } + +//# run-graphql +{ # Querying packages with checkpoint bounds + before: packages(first: 10, filter: { beforeCheckpoint: 1 }) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } + + after: packages(first: 10, filter: { afterCheckpoint: 1 }) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } + + between: packages(first: 10, filter: { afterCheckpoint: 1, beforeCheckpoint: 3 }) { + nodes { + address + version + previousTransactionBlock { + effects { checkpoint { sequenceNumber } } + } + } + } +} diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql index 69472d0578045..95fca9ce535b8 100644 --- a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql +++ b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql @@ -2201,6 +2201,22 @@ type MovePackage implements IObject & IOwner { moduleBcs: Base64 } +""" +Filter for paginating `MovePackage`s that were created within a range of checkpoints. +""" +input MovePackageCheckpointFilter { + """ + Fetch packages that were published strictly after this checkpoint. Omitting this fetches + packages published since genesis. + """ + afterCheckpoint: UInt53 + """ + Fetch packages that were published strictly before this checkpoint. Omitting this fetches + packages published up to the latest checkpoint (inclusive). + """ + beforeCheckpoint: UInt53 +} + type MovePackageConnection { """ Information to aid in pagination. @@ -3115,6 +3131,14 @@ type Query { """ objects(first: Int, after: String, last: Int, before: String, filter: ObjectFilter): ObjectConnection! """ + The Move packages that exist in the network, optionally filtered to be strictly before + `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + + This query will return all versions of a given user package that appear between the + specified checkpoints, but only records the latest versions of system packages. + """ + packages(first: Int, after: String, last: Int, before: String, filter: MovePackageCheckpointFilter): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """ diff --git a/crates/sui-graphql-rpc/src/types/event.rs b/crates/sui-graphql-rpc/src/types/event.rs index 16284f01618cd..6b7ba8ee8b3c2 100644 --- a/crates/sui-graphql-rpc/src/types/event.rs +++ b/crates/sui-graphql-rpc/src/types/event.rs @@ -143,13 +143,13 @@ impl Event { /// checkpoint sequence numbers as the cursor to determine the correct page of results. The /// query can optionally be further `filter`-ed by the `EventFilter`. /// - /// The `checkpoint_viewed_at` parameter is represents the checkpoint sequence number at which - /// this page was queried for. Each entity returned in the connection will inherit this - /// checkpoint, so that when viewing that entity's state, it will be from the reference of this - /// checkpoint_viewed_at parameter. + /// The `checkpoint_viewed_at` parameter represents the checkpoint sequence number at which this + /// page was queried. Each entity returned in the connection will inherit this checkpoint, so + /// that when viewing that entity's state, it will be as if it is being viewed at this + /// checkpoint. /// - /// If the `Page` is set, then this function will defer to the `checkpoint_viewed_at` in - /// the cursor if they are consistent. + /// The cursors in `page` may also include checkpoint viewed at fields. If these are set, they + /// take precedence over the checkpoint that pagination is being conducted in. pub(crate) async fn paginate( db: &Db, page: Page, diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index 1e8490be2465b..d29612246c305 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -7,7 +7,7 @@ use super::balance::{self, Balance}; use super::base64::Base64; use super::big_int::BigInt; use super::coin::Coin; -use super::cursor::{JsonCursor, Page}; +use super::cursor::{BcsCursor, JsonCursor, Page, RawPaginated, Target}; use super::move_module::MoveModule; use super::move_object::MoveObject; use super::object::{self, Object, ObjectFilter, ObjectImpl, ObjectOwner, ObjectStatus}; @@ -18,14 +18,19 @@ use super::suins_registration::{DomainFormat, SuinsRegistration}; use super::transaction_block::{self, TransactionBlock, TransactionBlockFilter}; use super::type_filter::ExactTypeFilter; use super::uint53::UInt53; -use crate::consistency::ConsistentNamedCursor; +use crate::consistency::{Checkpointed, ConsistentNamedCursor}; use crate::data::{DataLoader, Db, DbConnection, QueryExecutor}; use crate::error::Error; +use crate::raw_query::RawQuery; use crate::types::sui_address::addr; +use crate::{filter, query}; use async_graphql::connection::{Connection, CursorType, Edge}; use async_graphql::dataloader::Loader; use async_graphql::*; -use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl}; +use diesel::prelude::QueryableByName; +use diesel::{BoolExpressionMethods, ExpressionMethods, JoinOnDsl, QueryDsl, Selectable}; +use serde::{Deserialize, Serialize}; +use sui_indexer::models::objects::StoredHistoryObject; use sui_indexer::schema::packages; use sui_package_resolver::{error::Error as PackageCacheError, Package as ParsedMovePackage}; use sui_types::is_system_package; @@ -41,6 +46,18 @@ pub(crate) struct MovePackage { pub native: NativeMovePackage, } +/// Filter for paginating `MovePackage`s that were created within a range of checkpoints. +#[derive(InputObject, Debug, Default, Clone)] +pub(crate) struct MovePackageCheckpointFilter { + /// Fetch packages that were published strictly after this checkpoint. Omitting this fetches + /// packages published since genesis. + pub after_checkpoint: Option, + + /// Fetch packages that were published strictly before this checkpoint. Omitting this fetches + /// packages published up to the latest checkpoint (inclusive). + pub before_checkpoint: Option, +} + /// Filter for a point query of a MovePackage, supporting querying different versions of a package /// by their version. Note that different versions of the same user package exist at different IDs /// to each other, so this is different from looking up the historical version of an object. @@ -87,9 +104,31 @@ struct TypeOrigin { defining_id: SuiAddress, } +/// A wrapper around the stored representation of a package, used to implement pagination-related +/// traits. +#[derive(Selectable, QueryableByName)] +#[diesel(table_name = packages)] +struct StoredHistoryPackage { + original_id: Vec, + #[diesel(embed)] + object: StoredHistoryObject, +} + pub(crate) struct MovePackageDowncastError; pub(crate) type CModule = JsonCursor; +pub(crate) type Cursor = BcsCursor; + +/// The inner struct for the `MovePackage` cursor. The package is identified by the checkpoint it +/// was created in, its original ID, and its version, and the `checkpoint_viewed_at` specifies the +/// checkpoint snapshot that the data came from. +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] +pub(crate) struct PackageCursor { + pub checkpoint_sequence_number: u64, + pub original_id: Vec, + pub package_version: u64, + pub checkpoint_viewed_at: u64, +} /// DataLoader key for fetching the storage ID of the (user) package that shares an original (aka /// runtime) ID with the package stored at `package_id`, and whose version is `version`. @@ -574,6 +613,164 @@ impl MovePackage { Error::Internal(format!("{address} is not a package")) })?)) } + + /// Query the database for a `page` of Move packages. The Page uses the checkpoint sequence + /// number the package was created at, its original ID, and its version as the cursor. The query + /// can optionally be filtered by a bound on the checkpoints the packages were created in. + /// + /// The `checkpoint_viewed_at` parameter represents the checkpoint sequence number at which this + /// page was queried. Each entity returned in the connection will inherit this checkpoint, so + /// that when viewing that entity's state, it will be as if it is being viewed at this + /// checkpoint. + /// + /// The cursors in `page` may also include checkpoint viewed at fields. If these are set, they + /// take precedence over the checkpoint that pagination is being conducted in. + pub(crate) async fn paginate_by_checkpoint( + db: &Db, + page: Page, + filter: Option, + checkpoint_viewed_at: u64, + ) -> Result, Error> { + let cursor_viewed_at = page.validate_cursor_consistency()?; + let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); + + let after_checkpoint: Option = filter + .as_ref() + .and_then(|f| f.after_checkpoint) + .map(|v| v.into()); + + // Clamp the "before checkpoint" bound by "checkpoint viewed at". + let before_checkpoint = filter + .as_ref() + .and_then(|f| f.before_checkpoint) + .map(|v| v.into()) + .unwrap_or(u64::MAX) + .min(checkpoint_viewed_at + 1); + + let (prev, next, results) = db + .execute(move |conn| { + let mut q = query!( + r#" + SELECT + p.original_id, + o.* + FROM + packages p + INNER JOIN + objects_history o + ON + p.package_id = o.object_id + AND p.package_version = o.object_version + AND p.checkpoint_sequence_number = o.checkpoint_sequence_number + "# + ); + + q = filter!( + q, + format!("o.checkpoint_sequence_number < {before_checkpoint}") + ); + if let Some(after) = after_checkpoint { + q = filter!(q, format!("{after} < o.checkpoint_sequence_number")); + } + + page.paginate_raw_query::(conn, checkpoint_viewed_at, q) + }) + .await?; + + let mut conn = Connection::new(prev, next); + + // The "checkpoint viewed at" sets a consistent upper bound for the nested queries. + for stored in results { + let cursor = stored.cursor(checkpoint_viewed_at).encode_cursor(); + let package = + MovePackage::try_from_stored_history_object(stored.object, checkpoint_viewed_at)?; + conn.edges.push(Edge::new(cursor, package)); + } + + Ok(conn) + } + + /// `checkpoint_viewed_at` points to the checkpoint snapshot that this `MovePackage` came from. + /// This is stored in the `MovePackage` so that related fields from the package are read from + /// the same checkpoint (consistently). + pub(crate) fn try_from_stored_history_object( + history_object: StoredHistoryObject, + checkpoint_viewed_at: u64, + ) -> Result { + let object = Object::try_from_stored_history_object( + history_object, + checkpoint_viewed_at, + /* root_version */ None, + )?; + Self::try_from(&object).map_err(|_| Error::Internal("Not a package!".to_string())) + } +} + +impl Checkpointed for Cursor { + fn checkpoint_viewed_at(&self) -> u64 { + self.checkpoint_viewed_at + } +} + +impl RawPaginated for StoredHistoryPackage { + fn filter_ge(cursor: &Cursor, query: RawQuery) -> RawQuery { + filter!( + query, + format!( + "o.checkpoint_sequence_number > {cp} OR (\ + o.checkpoint_sequence_number = {cp} AND + p.original_id > '\\x{id}'::bytea OR (\ + p.original_id = '\\x{id}'::bytea AND \ + p.package_version >= {pv}\ + ))", + cp = cursor.checkpoint_sequence_number, + id = hex::encode(&cursor.original_id), + pv = cursor.package_version, + ) + ) + } + + fn filter_le(cursor: &Cursor, query: RawQuery) -> RawQuery { + filter!( + query, + format!( + "o.checkpoint_sequence_number < {cp} OR (\ + o.checkpoint_sequence_number = {cp} AND + p.original_id < '\\x{id}'::bytea OR (\ + p.original_id = '\\x{id}'::bytea AND \ + p.package_version <= {pv}\ + ))", + cp = cursor.checkpoint_sequence_number, + id = hex::encode(&cursor.original_id), + pv = cursor.package_version, + ) + ) + } + + fn order(asc: bool, query: RawQuery) -> RawQuery { + if asc { + query + .order_by("o.checkpoint_sequence_number ASC") + .order_by("p.original_id ASC") + .order_by("p.package_version ASC") + } else { + query + .order_by("o.checkpoint_sequence_number DESC") + .order_by("p.original_id DESC") + .order_by("p.package_version DESC") + } + } +} + +impl Target for StoredHistoryPackage { + fn cursor(&self, checkpoint_viewed_at: u64) -> Cursor { + Cursor::new(PackageCursor { + checkpoint_sequence_number: self.object.checkpoint_sequence_number as u64, + original_id: self.original_id.clone(), + package_version: self.object.object_version as u64, + checkpoint_viewed_at, + }) + } } #[async_trait::async_trait] diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index 5aa55e7334773..0970a046749c3 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -12,7 +12,7 @@ use sui_sdk::SuiClient; use sui_types::transaction::{TransactionData, TransactionKind}; use sui_types::{gas_coin::GAS, transaction::TransactionDataAPI, TypeTag}; -use super::move_package::MovePackage; +use super::move_package::{self, MovePackage, MovePackageCheckpointFilter}; use super::suins_registration::NameService; use super::uint53::UInt53; use super::{ @@ -435,6 +435,28 @@ impl Query { .extend() } + /// The Move packages that exist in the network, optionally filtered to be strictly before + /// `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + /// + /// This query will return all versions of a given user package that appear between the + /// specified checkpoints, but only records the latest versions of system packages. + async fn packages( + &self, + ctx: &Context<'_>, + first: Option, + after: Option, + last: Option, + before: Option, + filter: Option, + ) -> Result> { + let Watermark { checkpoint, .. } = *ctx.data()?; + + let page = Page::from_params(ctx.data_unchecked(), first, after, last, before)?; + MovePackage::paginate_by_checkpoint(ctx.data_unchecked(), page, filter, checkpoint) + .await + .extend() + } + /// Fetch the protocol config by protocol version (defaults to the latest protocol /// version known to the GraphQL service). async fn protocol_config( diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap index 61cabeeb1de13..3499ea88329cc 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap @@ -2205,6 +2205,22 @@ type MovePackage implements IObject & IOwner { moduleBcs: Base64 } +""" +Filter for paginating `MovePackage`s that were created within a range of checkpoints. +""" +input MovePackageCheckpointFilter { + """ + Fetch packages that were published strictly after this checkpoint. Omitting this fetches + packages published since genesis. + """ + afterCheckpoint: UInt53 + """ + Fetch packages that were published strictly before this checkpoint. Omitting this fetches + packages published up to the latest checkpoint (inclusive). + """ + beforeCheckpoint: UInt53 +} + type MovePackageConnection { """ Information to aid in pagination. @@ -3119,6 +3135,14 @@ type Query { """ objects(first: Int, after: String, last: Int, before: String, filter: ObjectFilter): ObjectConnection! """ + The Move packages that exist in the network, optionally filtered to be strictly before + `beforeCheckpoint` and/or strictly after `afterCheckpoint`. + + This query will return all versions of a given user package that appear between the + specified checkpoints, but only records the latest versions of system packages. + """ + packages(first: Int, after: String, last: Int, before: String, filter: MovePackageCheckpointFilter): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """ From 2a53577e9473239ab4de308369c1c3a7f99d712d Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Fri, 10 May 2024 18:29:57 +0100 Subject: [PATCH 5/9] [GraphQL/MovePackage] Paginate by version (#17697) ## Description Introduce two new queries: `Query.packageVersions` and `MovePackage.versions` for iterating over all the different versions of a given package. This kind of query is useful for understanding package history. These were introduced as a separate query, instead of having a single query for iterating over packages that could optionally take a checkpoint bounds or version bounds because of how system packages interact with the `packages` table: Because system packages are updated in-place, they only have one row in the `packages` table. This makes sense for paginating packages in bulk (e.g. by checkpoint) where the primary aim is to get a snapshot of the packages available at a certain point in time, but doesn't work for answering package version queries for system packages, and it prevents us from creating a combined query. A combined query would also allow someone to create a filter that bounds checkpoints and versions, but doesn't bound the package itself (or would require us to prevent that combination), which is complicated to implement efficiently and not particularly useful. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-graphql-e2e-tests \ --features pg_integration \ -- packages/versioning ``` & Testing against a read replica to make sure system package tests work well, and performance is reasonable. ## Stack - #17686 - #17687 - #17688 - #17689 - #17691 - #17694 - #17695 - #17542 - #17690 - #17543 - #17692 - #17693 - #17696 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: Introduces `Query.packageVersions` and `MovePackage.versions` for paginating over the versions of a particular package. - [ ] CLI: - [ ] Rust SDK: --- .../tests/packages/versioning.exp | 217 ++++++++++++++++-- .../tests/packages/versioning.move | 123 ++++++++++ .../schema/current_progress_schema.graphql | 28 +++ .../sui-graphql-rpc/src/types/move_package.rs | 197 +++++++++++++++- crates/sui-graphql-rpc/src/types/query.rs | 25 +- .../snapshot_tests__schema_sdl_export.snap | 28 +++ 6 files changed, 594 insertions(+), 24 deletions(-) diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp index 7a0e6ec3faeed..7f0e8a7153b98 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.exp @@ -1,4 +1,4 @@ -processed 15 tasks +processed 17 tasks init: A: object(0,0) @@ -13,7 +13,7 @@ task 2, line 11: //# create-checkpoint Checkpoint created: 1 -task 3, lines 13-28: +task 3, lines 13-50: //# run-graphql Response: { "data": { @@ -27,6 +27,35 @@ Response: { } ] } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + } + ] + } + }, + "firstPackage": { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + } + ] } }, "packages": { @@ -56,17 +85,17 @@ Response: { } } -task 4, lines 30-34: +task 4, lines 52-56: //# upgrade --package P0 --upgrade-capability 1,1 --sender A created: object(4,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5251600, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 5, line 36: +task 5, line 58: //# create-checkpoint Checkpoint created: 2 -task 6, lines 38-53: +task 6, lines 60-97: //# run-graphql Response: { "data": { @@ -83,6 +112,43 @@ Response: { } ] } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] + } + }, + "firstPackage": { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] } }, "packages": { @@ -116,17 +182,17 @@ Response: { } } -task 7, lines 55-60: +task 7, lines 99-104: //# upgrade --package P1 --upgrade-capability 1,1 --sender A created: object(7,0) mutated: object(0,0), object(1,1) gas summary: computation_cost: 1000000, storage_cost: 5426400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 -task 8, line 62: +task 8, line 106: //# create-checkpoint Checkpoint created: 3 -task 9, lines 64-79: +task 9, lines 108-145: //# run-graphql Response: { "data": { @@ -146,6 +212,51 @@ Response: { } ] } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] + } + }, + "firstPackage": { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1, + "module": { + "functions": { + "nodes": [ + { + "name": "f" + } + ] + } + }, + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] } }, "packages": { @@ -183,7 +294,7 @@ Response: { } } -task 10, lines 81-118: +task 10, lines 147-184: //# run-graphql Response: { "data": { @@ -283,7 +394,7 @@ Response: { } } -task 11, lines 120-157: +task 11, lines 186-223: //# run-graphql Response: { "data": { @@ -374,7 +485,7 @@ Response: { } } -task 12, lines 159-214: +task 12, lines 225-280: //# run-graphql Response: { "data": { @@ -513,7 +624,7 @@ Response: { } } -task 13, lines 216-244: +task 13, lines 282-310: //# run-graphql Response: { "data": { @@ -526,7 +637,7 @@ Response: { } } -task 14, lines 246-277: +task 14, lines 312-343: //# run-graphql Response: { "data": { @@ -621,3 +732,83 @@ Response: { } } } + +task 15, lines 345-380: +//# run-graphql +Response: { + "data": { + "packageVersions": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] + }, + "after": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + }, + { + "address": "0x0eae57b7a07b0548b1f6b0c309f0692828ff994e9159b541334b25582980631c", + "version": 3 + } + ] + }, + "before": { + "nodes": [ + { + "address": "0x175ae86f2df1eb652d57fbe9e44c7f2d67870d2b6776a4356f30930221b63b88", + "version": 1 + }, + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] + }, + "between": { + "nodes": [ + { + "address": "0x351bc614b36f0f522a64334e4c278d4bfe200234958870c084e0a005f041d681", + "version": 2 + } + ] + } + } +} + +task 16, lines 382-400: +//# run-graphql +Response: { + "data": { + "packageVersions": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + } + ] + }, + "package": { + "packageVersions": { + "nodes": [ + { + "address": "0x0000000000000000000000000000000000000000000000000000000000000001", + "version": 1 + } + ] + } + } + } +} diff --git a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move index 694072fb9c445..b0e0900bbcb59 100644 --- a/crates/sui-graphql-e2e-tests/tests/packages/versioning.move +++ b/crates/sui-graphql-e2e-tests/tests/packages/versioning.move @@ -17,6 +17,28 @@ module P0::m { module(name: "m") { functions { nodes { name } } } + + packageVersions { + nodes { + address + version + } + } + } + + firstPackage: package(address: "@{P0}", version: 1) { + address + version + module(name: "m") { + functions { nodes { name } } + } + + packageVersions { + nodes { + address + version + } + } } packages(first: 10) { @@ -42,6 +64,28 @@ module P1::m { module(name: "m") { functions { nodes { name } } } + + packageVersions { + nodes { + address + version + } + } + } + + firstPackage: package(address: "@{P1}", version: 1) { + address + version + module(name: "m") { + functions { nodes { name } } + } + + packageVersions { + nodes { + address + version + } + } } packages(first: 10) { @@ -68,6 +112,28 @@ module P2::m { module(name: "m") { functions { nodes { name } } } + + packageVersions { + nodes { + address + version + } + } + } + + firstPackage: package(address: "@{P2}", version: 1) { + address + version + module(name: "m") { + functions { nodes { name } } + } + + packageVersions { + nodes { + address + version + } + } } packages(first: 10) { @@ -275,3 +341,60 @@ module P2::m { } } } + +//# run-graphql +{ # Query for versions of a user package + packageVersions(address: "@{P0}") { + nodes { + address + version + } + } + + after: packageVersions(address: "@{P0}", filter: { afterVersion: 1 }) { + nodes { + address + version + } + } + + before: packageVersions(address: "@{P0}", filter: { beforeVersion: 3 }) { + nodes { + address + version + } + } + + between: packageVersions( + address: "@{P0}", + filter: { + afterVersion: 1, + beforeVersion: 3, + }, + ) { + nodes { + address + version + } + } +} + +//# run-graphql +{ # Query for versions of a system package (there will be only one because we + # don't have a way to upgrade system packages in these tests.) + packageVersions(address: "0x1") { + nodes { + address + version + } + } + + package(address: "0x1") { + packageVersions { + nodes { + address + version + } + } + } +} diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql index 95fca9ce535b8..9a134382a7a70 100644 --- a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql +++ b/crates/sui-graphql-rpc/schema/current_progress_schema.graphql @@ -2173,6 +2173,12 @@ type MovePackage implements IObject & IOwner { """ packageAtVersion(version: Int!): MovePackage """ + Fetch all versions of this package (packages that share this package's original ID), + optionally bounding the versions exclusively from below with `afterVersion`, or from above + with `beforeVersion`. + """ + packageVersions(first: Int, after: String, last: Int, before: String, filter: MovePackageVersionFilter): MovePackageConnection! + """ Fetch the latest version of this package (the package with the highest `version` that shares this packages's original ID) """ @@ -2246,6 +2252,22 @@ type MovePackageEdge { cursor: String! } +""" +Filter for paginating versions of a given `MovePackage`. +""" +input MovePackageVersionFilter { + """ + Fetch versions of this package that are strictly newer than this version. Omitting this + fetches versions since the original version. + """ + afterVersion: UInt53 + """ + Fetch versions of this package that are strictly older than this version. Omitting this + fetches versions up to the latest version (inclusive). + """ + beforeVersion: UInt53 +} + """ Description of a struct type, defined in a Move module. """ @@ -3139,6 +3161,12 @@ type Query { """ packages(first: Int, after: String, last: Int, before: String, filter: MovePackageCheckpointFilter): MovePackageConnection! """ + Fetch all versions of package at `address` (packages that share this package's original ID), + optionally bounding the versions exclusively from below with `afterVersion`, or from above + with `beforeVersion`. + """ + packageVersions(first: Int, after: String, last: Int, before: String, address: SuiAddress!, filter: MovePackageVersionFilter): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """ diff --git a/crates/sui-graphql-rpc/src/types/move_package.rs b/crates/sui-graphql-rpc/src/types/move_package.rs index d29612246c305..aba7259ff1a22 100644 --- a/crates/sui-graphql-rpc/src/types/move_package.rs +++ b/crates/sui-graphql-rpc/src/types/move_package.rs @@ -58,6 +58,18 @@ pub(crate) struct MovePackageCheckpointFilter { pub before_checkpoint: Option, } +/// Filter for paginating versions of a given `MovePackage`. +#[derive(InputObject, Debug, Default, Clone)] +pub(crate) struct MovePackageVersionFilter { + /// Fetch versions of this package that are strictly newer than this version. Omitting this + /// fetches versions since the original version. + pub after_version: Option, + + /// Fetch versions of this package that are strictly older than this version. Omitting this + /// fetches versions up to the latest version (inclusive). + pub before_version: Option, +} + /// Filter for a point query of a MovePackage, supporting querying different versions of a package /// by their version. Note that different versions of the same user package exist at different IDs /// to each other, so this is different from looking up the historical version of an object. @@ -355,6 +367,31 @@ impl MovePackage { .extend() } + /// Fetch all versions of this package (packages that share this package's original ID), + /// optionally bounding the versions exclusively from below with `afterVersion`, or from above + /// with `beforeVersion`. + async fn package_versions( + &self, + ctx: &Context<'_>, + first: Option, + after: Option, + last: Option, + before: Option, + filter: Option, + ) -> Result> { + let page = Page::from_params(ctx.data_unchecked(), first, after, last, before)?; + + MovePackage::paginate_by_version( + ctx.data_unchecked(), + page, + self.super_.address, + filter, + self.checkpoint_viewed_at_impl(), + ) + .await + .extend() + } + /// Fetch the latest version of this package (the package with the highest `version` that shares /// this packages's original ID) async fn latest_package(&self, ctx: &Context<'_>) -> Result { @@ -690,6 +727,55 @@ impl MovePackage { Ok(conn) } + /// Query the database for a `page` of Move packages. The Page uses the checkpoint sequence + /// number the package was created at, its original ID, and its version as the cursor. The query + /// is filtered by the ID of a package and will only return packages from the same family + /// (sharing the same original ID as the package whose ID was given), and can optionally be + /// filtered by an upper and lower bound on package version. + /// + /// The `checkpoint_viewed_at` parameter represents the checkpoint sequence number at which this + /// page was queried. Each entity returned in the connection will inherit this checkpoint, so + /// that when viewing that entity's state, it will be as if it is being viewed at this + /// checkpoint. + /// + /// The cursors in `page` may also include checkpoint viewed at fields. If these are set, they + /// take precedence over the checkpoint that pagination is being conducted in. + pub(crate) async fn paginate_by_version( + db: &Db, + page: Page, + package: SuiAddress, + filter: Option, + checkpoint_viewed_at: u64, + ) -> Result, Error> { + let cursor_viewed_at = page.validate_cursor_consistency()?; + let checkpoint_viewed_at = cursor_viewed_at.unwrap_or(checkpoint_viewed_at); + let (prev, next, results) = db + .execute(move |conn| { + page.paginate_raw_query::( + conn, + checkpoint_viewed_at, + if is_system_package(package) { + system_package_version_query(package, filter) + } else { + user_package_version_query(package, filter) + }, + ) + }) + .await?; + + let mut conn = Connection::new(prev, next); + + // The "checkpoint viewed at" sets a consistent upper bound for the nested queries. + for stored in results { + let cursor = stored.cursor(checkpoint_viewed_at).encode_cursor(); + let package = + MovePackage::try_from_stored_history_object(stored.object, checkpoint_viewed_at)?; + conn.edges.push(Edge::new(cursor, package)); + } + + Ok(conn) + } + /// `checkpoint_viewed_at` points to the checkpoint snapshot that this `MovePackage` came from. /// This is stored in the `MovePackage` so that related fields from the package are read from /// the same checkpoint (consistently). @@ -719,9 +805,9 @@ impl RawPaginated for StoredHistoryPackage { format!( "o.checkpoint_sequence_number > {cp} OR (\ o.checkpoint_sequence_number = {cp} AND - p.original_id > '\\x{id}'::bytea OR (\ - p.original_id = '\\x{id}'::bytea AND \ - p.package_version >= {pv}\ + original_id > '\\x{id}'::bytea OR (\ + original_id = '\\x{id}'::bytea AND \ + o.object_version >= {pv}\ ))", cp = cursor.checkpoint_sequence_number, id = hex::encode(&cursor.original_id), @@ -736,9 +822,9 @@ impl RawPaginated for StoredHistoryPackage { format!( "o.checkpoint_sequence_number < {cp} OR (\ o.checkpoint_sequence_number = {cp} AND - p.original_id < '\\x{id}'::bytea OR (\ - p.original_id = '\\x{id}'::bytea AND \ - p.package_version <= {pv}\ + original_id < '\\x{id}'::bytea OR (\ + original_id = '\\x{id}'::bytea AND \ + o.object_version <= {pv}\ ))", cp = cursor.checkpoint_sequence_number, id = hex::encode(&cursor.original_id), @@ -751,13 +837,13 @@ impl RawPaginated for StoredHistoryPackage { if asc { query .order_by("o.checkpoint_sequence_number ASC") - .order_by("p.original_id ASC") - .order_by("p.package_version ASC") + .order_by("original_id ASC") + .order_by("o.object_version ASC") } else { query .order_by("o.checkpoint_sequence_number DESC") - .order_by("p.original_id DESC") - .order_by("p.package_version DESC") + .order_by("original_id DESC") + .order_by("o.object_version DESC") } } } @@ -918,3 +1004,94 @@ impl TryFrom<&Object> for MovePackage { } } } + +/// Query for fetching all the versions of a system package (assumes that `package` has already been +/// verified as a system package). This is an `objects_history` query disguised as a package query. +fn system_package_version_query( + package: SuiAddress, + filter: Option, +) -> RawQuery { + // Query uses a left join to force the query planner to use `objects_version` in the outer loop. + let mut q = query!( + r#" + SELECT + o.object_id AS original_id, + o.* + FROM + objects_version v + LEFT JOIN + objects_history o + ON + v.object_id = o.object_id + AND v.object_version = o.object_version + AND v.cp_sequence_number = o.checkpoint_sequence_number + "# + ); + + q = filter!( + q, + format!( + "v.object_id = '\\x{}'::bytea", + hex::encode(package.into_vec()) + ) + ); + + if let Some(after) = filter.as_ref().and_then(|f| f.after_version) { + let a: u64 = after.into(); + q = filter!(q, format!("v.object_version > {a}")); + } + + if let Some(before) = filter.as_ref().and_then(|f| f.before_version) { + let b: u64 = before.into(); + q = filter!(q, format!("v.object_version < {b}")); + } + + q +} + +/// Query for fetching all the versions of a non-system package (assumes that `package` has already +/// been verified as a system package) +fn user_package_version_query( + package: SuiAddress, + filter: Option, +) -> RawQuery { + let mut q = query!( + r#" + SELECT + p.original_id, + o.* + FROM + packages q + INNER JOIN + packages p + ON + q.original_id = p.original_id + INNER JOIN + objects_history o + ON + p.package_id = o.object_id + AND p.package_version = o.object_version + AND p.checkpoint_sequence_number = o.checkpoint_sequence_number + "# + ); + + q = filter!( + q, + format!( + "q.package_id = '\\x{}'::bytea", + hex::encode(package.into_vec()) + ) + ); + + if let Some(after) = filter.as_ref().and_then(|f| f.after_version) { + let a: u64 = after.into(); + q = filter!(q, format!("p.package_version > {a}")); + } + + if let Some(before) = filter.as_ref().and_then(|f| f.before_version) { + let b: u64 = before.into(); + q = filter!(q, format!("p.package_version < {b}")); + } + + q +} diff --git a/crates/sui-graphql-rpc/src/types/query.rs b/crates/sui-graphql-rpc/src/types/query.rs index 0970a046749c3..1ff209d351a55 100644 --- a/crates/sui-graphql-rpc/src/types/query.rs +++ b/crates/sui-graphql-rpc/src/types/query.rs @@ -12,7 +12,9 @@ use sui_sdk::SuiClient; use sui_types::transaction::{TransactionData, TransactionKind}; use sui_types::{gas_coin::GAS, transaction::TransactionDataAPI, TypeTag}; -use super::move_package::{self, MovePackage, MovePackageCheckpointFilter}; +use super::move_package::{ + self, MovePackage, MovePackageCheckpointFilter, MovePackageVersionFilter, +}; use super::suins_registration::NameService; use super::uint53::UInt53; use super::{ @@ -457,6 +459,27 @@ impl Query { .extend() } + /// Fetch all versions of package at `address` (packages that share this package's original ID), + /// optionally bounding the versions exclusively from below with `afterVersion`, or from above + /// with `beforeVersion`. + async fn package_versions( + &self, + ctx: &Context<'_>, + first: Option, + after: Option, + last: Option, + before: Option, + address: SuiAddress, + filter: Option, + ) -> Result> { + let Watermark { checkpoint, .. } = *ctx.data()?; + + let page = Page::from_params(ctx.data_unchecked(), first, after, last, before)?; + MovePackage::paginate_by_version(ctx.data_unchecked(), page, address, filter, checkpoint) + .await + .extend() + } + /// Fetch the protocol config by protocol version (defaults to the latest protocol /// version known to the GraphQL service). async fn protocol_config( diff --git a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap index 3499ea88329cc..87f5a27061f06 100644 --- a/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap +++ b/crates/sui-graphql-rpc/tests/snapshots/snapshot_tests__schema_sdl_export.snap @@ -2177,6 +2177,12 @@ type MovePackage implements IObject & IOwner { """ packageAtVersion(version: Int!): MovePackage """ + Fetch all versions of this package (packages that share this package's original ID), + optionally bounding the versions exclusively from below with `afterVersion`, or from above + with `beforeVersion`. + """ + packageVersions(first: Int, after: String, last: Int, before: String, filter: MovePackageVersionFilter): MovePackageConnection! + """ Fetch the latest version of this package (the package with the highest `version` that shares this packages's original ID) """ @@ -2250,6 +2256,22 @@ type MovePackageEdge { cursor: String! } +""" +Filter for paginating versions of a given `MovePackage`. +""" +input MovePackageVersionFilter { + """ + Fetch versions of this package that are strictly newer than this version. Omitting this + fetches versions since the original version. + """ + afterVersion: UInt53 + """ + Fetch versions of this package that are strictly older than this version. Omitting this + fetches versions up to the latest version (inclusive). + """ + beforeVersion: UInt53 +} + """ Description of a struct type, defined in a Move module. """ @@ -3143,6 +3165,12 @@ type Query { """ packages(first: Int, after: String, last: Int, before: String, filter: MovePackageCheckpointFilter): MovePackageConnection! """ + Fetch all versions of package at `address` (packages that share this package's original ID), + optionally bounding the versions exclusively from below with `afterVersion`, or from above + with `beforeVersion`. + """ + packageVersions(first: Int, after: String, last: Int, before: String, address: SuiAddress!, filter: MovePackageVersionFilter): MovePackageConnection! + """ Fetch the protocol config by protocol version (defaults to the latest protocol version known to the GraphQL service). """ From cf0f39f4de86c9bb087447de0bdbb0519eeed50d Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:40:43 +0100 Subject: [PATCH 6/9] [chore][GraphQL] Clean-up unused commands (#18287) ## Description Remove commands to generate examples, markdown and schema from the main binary as we do not use them: - Instead of generating examples, we have hand-crafted examples in our docs. Removing this code also removes a test that forces regeneration of a markdown file from these docs (which we also were not using). - We also never used the output from the `generate-schema` sub-command, because the schema was always available as a file, or via introspection commands from the running service. - Logic for gathering examples to test has been moved into the test file, to avoid including test-only code in the main library. ## Test plan CI ## Description Describe the changes or additions included in this PR. ## Test plan How did you test the new or updated feature? ## Stack - #17543 - #17692 - #17693 - #17696 - #17697 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: The GraphQL binary no longer supports generating examples, or exporting its own schema as these commands have been unused for some time. - [ ] CLI: - [ ] Rust SDK: --- Cargo.lock | 7 - Cargo.toml | 1 - crates/sui-graphql-rpc/Cargo.toml | 1 - crates/sui-graphql-rpc/docs/examples.md | 1700 ----------------- crates/sui-graphql-rpc/src/commands.rs | 11 - crates/sui-graphql-rpc/src/examples.rs | 240 --- crates/sui-graphql-rpc/src/lib.rs | 1 - crates/sui-graphql-rpc/src/main.rs | 34 - .../tests/examples_validation_tests.rs | 228 ++- 9 files changed, 134 insertions(+), 2089 deletions(-) delete mode 100644 crates/sui-graphql-rpc/docs/examples.md delete mode 100644 crates/sui-graphql-rpc/src/examples.rs diff --git a/Cargo.lock b/Cargo.lock index c766d515b415e..4449deae1a1d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6716,12 +6716,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "markdown-gen" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8034621d7f1258317ca1dfb9205e3925d27ee4aa2a46620a09c567daf0310562" - [[package]] name = "match_opt" version = "0.1.2" @@ -13353,7 +13347,6 @@ dependencies = [ "insta", "itertools 0.10.5", "lru 0.10.0", - "markdown-gen", "move-binary-format", "move-bytecode-utils", "move-core-types", diff --git a/Cargo.toml b/Cargo.toml index 55f30a7447f6b..78b95b57b41e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -369,7 +369,6 @@ jsonrpsee = { git = "https://github.com/wlmyng/jsonrpsee.git", rev = "b1b3007847 json_to_table = { git = "https://github.com/zhiburt/tabled/", rev = "e449317a1c02eb6b29e409ad6617e5d9eb7b3bd4" } leb128 = "0.2.5" lru = "0.10" -markdown-gen = "1.2.1" match_opt = "0.1.2" miette = { version = "7", features = ["fancy"] } mime = "0.3" diff --git a/crates/sui-graphql-rpc/Cargo.toml b/crates/sui-graphql-rpc/Cargo.toml index 87c8a1571a9ef..12414fd4888c7 100644 --- a/crates/sui-graphql-rpc/Cargo.toml +++ b/crates/sui-graphql-rpc/Cargo.toml @@ -32,7 +32,6 @@ lru.workspace = true move-binary-format.workspace = true move-disassembler.workspace = true move-ir-types.workspace = true -markdown-gen.workspace = true mysten-metrics.workspace = true mysten-network.workspace = true move-core-types.workspace = true diff --git a/crates/sui-graphql-rpc/docs/examples.md b/crates/sui-graphql-rpc/docs/examples.md deleted file mode 100644 index b227665131112..0000000000000 --- a/crates/sui-graphql-rpc/docs/examples.md +++ /dev/null @@ -1,1700 +0,0 @@ -# Sui GraphQL Examples -### [Address](#0) -####   [Address](#0) -####   [Transaction Block Connection](#1) -### [Balance Connection](#1) -####   [Balance Connection](#65535) -### [Chain Id](#2) -####   [Chain Id](#131070) -### [Checkpoint](#3) -####   [At Digest](#196605) -####   [At Seq Num](#196606) -####   [First Two Tx Blocks For Checkpoint](#196607) -####   [Latest Checkpoint](#196608) -####   [Multiple Selections](#196609) -####   [With Timestamp Tx Block Live Objects](#196610) -####   [With Tx Sent Addr Filter](#196611) -### [Checkpoint Connection](#4) -####   [Ascending Fetch](#262140) -####   [First Ten After Checkpoint](#262141) -####   [Last Ten After Checkpoint](#262142) -### [Coin Connection](#5) -####   [Coin Connection](#327675) -### [Coin Metadata](#6) -####   [Coin Metadata](#393210) -### [Epoch](#7) -####   [Latest Epoch](#458745) -####   [Specific Epoch](#458746) -####   [With Checkpoint Connection](#458747) -####   [With Tx Block Connection](#458748) -####   [With Tx Block Connection Latest Epoch](#458749) -### [Event Connection](#8) -####   [Event Connection](#524280) -####   [Filter By Emitting Package Module And Event Type](#524281) -####   [Filter By Sender](#524282) -### [Name Service](#9) -####   [Name Service](#589815) -### [Object](#10) -####   [Object](#655350) -### [Object Connection](#11) -####   [Filter Object Ids](#720885) -####   [Filter On Generic Type](#720886) -####   [Filter On Type](#720887) -####   [Filter Owner](#720888) -####   [Object Connection](#720889) -### [Owner](#12) -####   [Dynamic Field](#786420) -####   [Dynamic Field Connection](#786421) -####   [Dynamic Object Field](#786422) -####   [Owner](#786423) -### [Protocol Configs](#13) -####   [Key Value](#851955) -####   [Key Value Feature Flag](#851956) -####   [Specific Config](#851957) -####   [Specific Feature Flag](#851958) -### [Service Config](#14) -####   [Service Config](#917490) -### [Stake Connection](#15) -####   [Stake Connection](#983025) -### [Sui System State Summary](#16) -####   [Sui System State Summary](#1048560) -### [Transaction Block](#17) -####   [Transaction Block](#1114095) -####   [Transaction Block Kind](#1114096) -### [Transaction Block Connection](#18) -####   [Before After Checkpoint](#1179630) -####   [Changed Object Filter](#1179631) -####   [Input Object Filter](#1179632) -####   [Input Object Sign Addr Filter](#1179633) -####   [Package Filter](#1179634) -####   [Package Module Filter](#1179635) -####   [Package Module Func Filter](#1179636) -####   [Recv Addr Filter](#1179637) -####   [Sign Addr Filter](#1179638) -####   [Tx Ids Filter](#1179639) -####   [Tx Kind Filter](#1179640) -####   [With Defaults Ascending](#1179641) -### [Transaction Block Effects](#19) -####   [Transaction Block Effects](#1245165) -## -## Address -### -### Address -#### Get the address' balance and its coins' id and type - ->
{
->  address(
->    address: "0x5094652429957619e6efa79a404a6714d1126e63f551f4b6c7fb76440f8118c9"
->  ) {
->    address
->    balance {
->      coinType {
->        repr
->      }
->      coinObjectCount
->      totalBalance
->    }
->    coins {
->      nodes {
->        contents {
->          type {
->            repr
->          }
->        }
->      }
->    }
->  }
->}
- -### -### Transaction Block Connection -#### See examples in Query::transactionBlocks as this is similar behavior -#### to the `transactionBlocks` in Query but supports additional -#### `AddressTransactionBlockRelationship` filter -#### Filtering on package where the signer of the TX is the current -#### address and displaying the transaction's sender and the gas price -#### and budget. - ->
# See examples in Query::transactionBlocks as this is similar behavior
-># to the `transactionBlocks` in Query but supports additional
-># `AddressTransactionBlockRelationship` filter
->
-># Filtering on package where the signer of the TX is the current
-># address and displaying the transaction's sender and the gas price
-># and budget.
->query transaction_block_with_relation_filter {
->  address(address: "0x2") {
->    transactionBlocks(relation: SIGN, filter: { function: "0x2" }) {
->      nodes {
->        sender {
->          address
->        }
->        gasInput {
->          gasPrice
->          gasBudget
->        }
->      }
->    }
->  }
->}
- -## -## Balance Connection -### -### Balance Connection -#### Query the balance for objects of type COIN and then for each coin -#### get the coin type, the number of objects, and the total balance - ->
{
->  address(
->    address: "0x5094652429957619e6efa79a404a6714d1126e63f551f4b6c7fb76440f8118c9"
->  ) {
->    balance(
->      type: "0xc060006111016b8a020ad5b33834984a437aaa7d3c74c18e09a95d48aceab08c::coin::COIN"
->    ) {
->      coinObjectCount
->      totalBalance
->    }
->    balances {
->      nodes {
->        coinType {
->          repr
->        }
->        coinObjectCount
->        totalBalance
->      }
->      pageInfo {
->        endCursor
->      }
->    }
->  }
->}
- -## -## Chain Id -### -### Chain Id -#### Returns the chain identifier for the chain that the server is tracking - ->
{
->  chainIdentifier
->}
- -## -## Checkpoint -### -### At Digest -#### Get the checkpoint's information at a particular digest - ->
{
->  checkpoint(id: { digest: "GaDeWEfbSQCQ8FBQHUHVdm4KjrnbgMqEZPuhStoq5njU" }) {
->    digest
->    sequenceNumber
->    validatorSignatures
->    previousCheckpointDigest
->    networkTotalTransactions
->    rollingGasSummary {
->      computationCost
->      storageCost
->      storageRebate
->      nonRefundableStorageFee
->    }
->    epoch {
->      epochId
->      referenceGasPrice
->      startTimestamp
->      endTimestamp
->    }
->  }
->}
- -### -### At Seq Num -#### Get the checkpoint's information at a particular sequence number - ->
{
->  checkpoint(id: { sequenceNumber: 10 }) {
->    digest
->    sequenceNumber
->    validatorSignatures
->    previousCheckpointDigest
->    networkTotalTransactions
->    rollingGasSummary {
->      computationCost
->      storageCost
->      storageRebate
->      nonRefundableStorageFee
->    }
->    epoch {
->      epochId
->      referenceGasPrice
->      startTimestamp
->      endTimestamp
->    }
->  }
->}
- -### -### First Two Tx Blocks For Checkpoint -#### Get data for the first two transaction blocks of checkpoint at sequence number 10 - ->
{
->  checkpoint(id: { sequenceNumber: 10 }) {
->    transactionBlocks(first: 2) {
->      edges {
->        node {
->          kind {
->            __typename
->          }
->          digest
->          sender {
->            address
->          }
->          expiration {
->            epochId
->          }
->        }
->      }
->      pageInfo {
->        startCursor
->        hasNextPage
->        hasPreviousPage
->        endCursor
->      }
->    }
->  }
->}
- -### -### Latest Checkpoint -#### Latest checkpoint's data - ->
{
->  checkpoint {
->    digest
->    sequenceNumber
->    validatorSignatures
->    previousCheckpointDigest
->    networkTotalTransactions
->    rollingGasSummary {
->      computationCost
->      storageCost
->      storageRebate
->      nonRefundableStorageFee
->    }
->    epoch {
->      epochId
->      referenceGasPrice
->      startTimestamp
->      endTimestamp
->    }
->  }
->}
- -### -### Multiple Selections -#### Get the checkpoint at sequence 9769 and show -#### its transactions - ->
{
->  checkpoint(id: { sequenceNumber: 9769 }) {
->    digest
->    sequenceNumber
->    timestamp
->    validatorSignatures
->    previousCheckpointDigest
->    networkTotalTransactions
->    rollingGasSummary {
->      computationCost
->      storageCost
->      storageRebate
->      nonRefundableStorageFee
->    }
->    epoch {
->      epochId
->      liveObjectSetDigest
->    }
->    transactionBlocks {
->      edges {
->        node {
->          digest
->          sender {
->            address
->          }
->          expiration {
->            epochId
->          }
->        }
->      }
->    }
->  }
->}
- -### -### With Timestamp Tx Block Live Objects -#### Latest checkpoint's timestamp, and transaction block data - ->
{
->  checkpoint {
->    digest
->    sequenceNumber
->    timestamp
->    transactionBlocks {
->      edges {
->        node {
->          digest
->          sender {
->            address
->          }
->          expiration {
->            epochId
->          }
->        }
->      }
->    }
->  }
->}
- -### -### With Tx Sent Addr Filter -#### Select checkpoint at sequence number 14830285 for transactions from signAddress - ->
{
->  checkpoint(id: { sequenceNumber: 14830285 }) {
->    digest
->    sequenceNumber
->    timestamp
->    transactionBlocks(
->      filter: {
->        signAddress: "0x0000000000000000000000000000000000000000000000000000000000000000"
->      }
->    ) {
->      edges {
->        node {
->          digest
->          sender {
->            address
->          }
->          expiration {
->            epochId
->          }
->        }
->      }
->    }
->  }
->}
- -## -## Checkpoint Connection -### -### Ascending Fetch -#### Use the checkpoint connection to fetch some default amount of checkpoints in an ascending order - ->
{
->  checkpoints {
->    nodes {
->      digest
->      sequenceNumber
->      validatorSignatures
->      previousCheckpointDigest
->      networkTotalTransactions
->      rollingGasSummary {
->        computationCost
->        storageCost
->        storageRebate
->        nonRefundableStorageFee
->      }
->      epoch {
->        epochId
->        referenceGasPrice
->        startTimestamp
->        endTimestamp
->      }
->    }
->  }
->}
- -### -### First Ten After Checkpoint -#### Fetch the digest and sequence number of the first 10 checkpoints after the cursor, which in this example is set to be checkpoint 0. Note that the cursor is opaque. - ->
{
->  checkpoints(first: 10, after: "eyJjIjoyMjgwMDU4MCwicyI6MH0") {
->    nodes {
->      sequenceNumber
->      digest
->    }
->  }
->}
- -### -### Last Ten After Checkpoint -#### Fetch the digest and the sequence number of the last 20 checkpoints before the cursor - ->
{
->  checkpoints(last: 20, before: "eyJjIjoyMjgwMDY1MSwicyI6MjI4MDA2MzJ9") {
->    nodes {
->      sequenceNumber
->      digest
->    }
->  }
->}
- -## -## Coin Connection -### -### Coin Connection -#### Get last 3 coins owned by `0x0`. - ->
{
->  address(
->    address: "0x0000000000000000000000000000000000000000000000000000000000000000"
->  ) {
->    coins(last: 3) {
->      nodes {
->        coinBalance
->      }
->      pageInfo {
->        endCursor
->        hasNextPage
->      }
->    }
->  }
->}
- -## -## Coin Metadata -### -### Coin Metadata - ->
query CoinMetadata {
->  coinMetadata(coinType: "0x2::sui::SUI") {
->    decimals
->    name
->    symbol
->    description
->    iconUrl
->    supply
->    hasPublicTransfer
->  }
->}
- -## -## Epoch -### -### Latest Epoch -#### Latest epoch, since epoch omitted - ->
{
->  epoch {
->    protocolConfigs {
->      protocolVersion
->    }
->    epochId
->    referenceGasPrice
->    startTimestamp
->    endTimestamp
->  }
->}
- -### -### Specific Epoch -#### Selecting all fields for epoch 100 - ->
{
->  epoch(id: 100) {
->    protocolConfigs {
->      protocolVersion
->    }
->    epochId
->    referenceGasPrice
->    startTimestamp
->    endTimestamp
->    validatorSet {
->      totalStake
->      pendingActiveValidatorsSize
->      stakingPoolMappingsSize
->      inactivePoolsSize
->      validatorCandidatesSize
->      activeValidators {
->        nodes {
->          name
->          description
->          imageUrl
->          projectUrl
->          exchangeRates {
->            storageRebate
->            bcs
->            hasPublicTransfer
->          }
->          exchangeRatesSize
->          stakingPoolActivationEpoch
->          stakingPoolSuiBalance
->          rewardsPool
->          poolTokenBalance
->          pendingStake
->          pendingTotalSuiWithdraw
->          pendingPoolTokenWithdraw
->          votingPower
->          gasPrice
->          commissionRate
->          nextEpochStake
->          nextEpochGasPrice
->          nextEpochCommissionRate
->          atRisk
->        }
->      }
->    }
->  }
->}
- -### -### With Checkpoint Connection - ->
{
->  epoch {
->    checkpoints {
->      nodes {
->        transactionBlocks(first: 10) {
->          pageInfo {
->            hasNextPage
->            endCursor
->          }
->          edges {
->            cursor
->            node {
->              sender {
->                address
->              }
->              effects {
->                gasEffects {
->                  gasObject {
->                    address
->                  }
->                }
->              }
->              gasInput {
->                gasPrice
->                gasBudget
->              }
->            }
->          }
->        }
->      }
->    }
->  }
->}
- -### -### With Tx Block Connection -#### Fetch the first 20 transactions after tx 231220153 (encoded as a -#### cursor) in epoch 97. - ->
{
->  epoch(id: 97) {
->    transactionBlocks(first: 20, after:"eyJjIjoyNjkzMzc3OCwidCI6MjMxMjIwMTUzLCJ0YyI6ODAxMDg4NH0") {
->      pageInfo {
->        hasNextPage
->        endCursor
->      }
->      edges {
->        cursor
->        node {
->          digest
->          sender {
->            address
->          }
->          effects {
->            gasEffects {
->              gasObject {
->                address
->              }
->            }
->          }
->          gasInput {
->            gasPrice
->            gasBudget
->          }
->        }
->      }
->    }
->  }
->}
- -### -### With Tx Block Connection Latest Epoch - ->
{
->  epoch {
->    transactionBlocks(first: 20, after: "eyJjIjoyNjkzMzMyNCwidCI6MTEwMTYxMDQ4MywidGMiOjI2ODUxMjQ4fQ") {
->      pageInfo {
->        hasNextPage
->        endCursor
->      }
->      edges {
->        cursor
->        node {
->          sender {
->            address
->          }
->          effects {
->            gasEffects {
->              gasObject {
->                address
->              }
->            }
->          }
->          gasInput {
->            gasPrice
->            gasBudget
->          }
->        }
->      }
->    }
->  }
->}
- -## -## Event Connection -### -### Event Connection - ->
{
->  events(
->    filter: {
->      eventType: "0x3164fcf73eb6b41ff3d2129346141bd68469964c2d95a5b1533e8d16e6ea6e13::Market::ChangePriceEvent<0x2::sui::SUI>"
->    }
->  ) {
->    nodes {
->      sendingModule {
->        name
->        package { digest }
->      }
->      type {
->        repr
->      }
->      sender {
->        address
->      }
->      timestamp
->      json
->      bcs
->    }
->  }
->}
- -### -### Filter By Emitting Package Module And Event Type - ->
query ByEmittingPackageModuleAndEventType {
->  events(
->    first: 1
->    after: "eyJ0eCI6Njc2MywiZSI6MCwiYyI6MjI4MDA3NDJ9"
->    filter: {
->      emittingModule: "0x3::sui_system",
->      eventType: "0x3::validator::StakingRequestEvent"
->    }
->  ) {
->    pageInfo {
->      hasNextPage
->      endCursor
->    }
->    nodes {
->      sendingModule {
->        name
->      }
->      type {
->        repr
->      }
->      sender {
->        address
->      }
->      timestamp
->      json
->      bcs
->    }
->  }
->}
- -### -### Filter By Sender - ->
query ByTxSender {
->  events(
->    first: 1
->    filter: {
->      sender: "0xdff57c401e125a7e0e06606380560b459a179aacd08ed396d0162d57dbbdadfb"
->    }
->  ) {
->    pageInfo {
->      hasNextPage
->      endCursor
->    }
->    nodes {
->      sendingModule {
->        name
->      }
->      type {
->        repr
->      }
->      sender {
->        address
->      }
->      timestamp
->      json
->      bcs
->    }
->  }
->}
- -## -## Name Service -### -### Name Service - ->
{
->  resolveSuinsAddress(domain: "example.sui") {
->    address
->  }
->  address(
->    address: "0x0b86be5d779fac217b41d484b8040ad5145dc9ba0cba099d083c6cbda50d983e"
->  ) {
->    address
->    balance(type: "0x2::sui::SUI") {
->      coinType {
->        repr
->      }
->      coinObjectCount
->      totalBalance
->    }
->    defaultSuinsName
->  }
->}
- -## -## Object -### -### Object - ->
{
->  object(
->    address: "0x04e20ddf36af412a4096f9014f4a565af9e812db9a05cc40254846cf6ed0ad91"
->  ) {
->    address
->    version
->    digest
->    storageRebate
->    owner {
->      __typename
->      ... on Shared {
->        initialSharedVersion
->      }
->      __typename
->      ... on Parent {
->        parent {
->          address
->        }
->      }
->      __typename
->      ... on AddressOwner {
->        owner {
->          address
->        }
->      }
->    }
->    previousTransactionBlock {
->      digest
->    }
->  }
->}
- -## -## Object Connection -### -### Filter Object Ids -#### Filter on objectIds - ->
{
->  objects(filter: { objectIds: [
->    "0x4bba2c7b9574129c272bca8f58594eba933af8001257aa6e0821ad716030f149"
->  ]}) {
->    edges {
->      node {
->        storageRebate
->        owner {
->          __typename
->          ... on Shared {
->            initialSharedVersion
->          }
->          __typename
->          ... on Parent {
->            parent {
->              address
->            }
->          }
->          __typename
->          ... on AddressOwner {
->            owner {
->              address
->            }
->          }
->        }
->      }
->    }
->  }
->}
- -### -### Filter On Generic Type - ->
{
->  objects(filter: {type: "0x2::coin::Coin"}) {
->    edges {
->      node {
->        asMoveObject {
->          contents {
->            type { repr }
->          }
->        }
->      }
->    }
->  }
->}
- -### -### Filter On Type - ->
{
->  objects(filter: {type: "0x3::staking_pool::StakedSui"}) {
->    edges {
->      node {
->        asMoveObject {
->          contents {
->            type {
->              repr
->            }
->          }
->        }
->      }
->    }
->  }
->}
- -### -### Filter Owner -#### Filter on owner - ->
{
->  objects(filter: {
->    owner: "0x23b7b0e2badb01581ba9b3ab55587d8d9fdae087e0cfc79f2c72af36f5059439"
->  }) {
->    edges {
->      node {
->        storageRebate
->        owner {
->          __typename
->          ... on Shared {
->            initialSharedVersion
->          }
->          __typename
->          ... on Parent {
->            parent {
->              address
->            }
->          }
->          __typename
->          ... on AddressOwner {
->            owner {
->              address
->            }
->          }
->        }
->      }
->    }
->  }
->}
- -### -### Object Connection - ->
{
->  objects {
->    nodes {
->      version
->      digest
->      storageRebate
->      previousTransactionBlock {
->        digest
->        sender { defaultSuinsName }
->        gasInput {
->          gasPrice
->          gasBudget
->        }
->      }
->    }
->    pageInfo {
->      endCursor
->    }
->  }
->}
- -## -## Owner -### -### Dynamic Field - ->
fragment DynamicFieldValueSelection on DynamicFieldValue {
->  ... on MoveValue {
->    type {
->      repr
->    }
->    data
->    __typename
->  }
->  ... on MoveObject {
->    hasPublicTransfer
->    contents {
->      type {
->        repr
->      }
->      data
->    }
->    __typename
->  }
->}
->
->fragment DynamicFieldNameSelection on MoveValue {
->  type {
->    repr
->  }
->  data
->  bcs
->}
->
->fragment DynamicFieldSelect on DynamicField {
->  name {
->    ...DynamicFieldNameSelection
->  }
->  value {
->    ...DynamicFieldValueSelection
->  }
->}
->
->query DynamicField {
->  object(
->    address: "0xb57fba584a700a5bcb40991e1b2e6bf68b0f3896d767a0da92e69de73de226ac"
->  ) {
->    dynamicField(
->      name: {
->        type: "0x2::kiosk::Listing",
->        bcs: "NLArx1UJguOUYmXgNG8Pv8KbKXLjWtCi6i0Yeq1VhfwA",
->      }
->    ) {
->      ...DynamicFieldSelect
->    }
->  }
->}
- -### -### Dynamic Field Connection - ->
fragment DynamicFieldValueSelection on DynamicFieldValue {
->  ... on MoveValue {
->    type {
->      repr
->    }
->    data
->  }
->  ... on MoveObject {
->    hasPublicTransfer
->    contents {
->      type {
->        repr
->      }
->      data
->    }
->  }
->}
->
->fragment DynamicFieldNameSelection on MoveValue {
->  type {
->    repr
->  }
->  data
->  bcs
->}
->
->fragment DynamicFieldSelect on DynamicField {
->  name {
->    ...DynamicFieldNameSelection
->  }
->  value {
->    ...DynamicFieldValueSelection
->  }
->}
->
->query DynamicFields {
->  object(
->    address: "0xb57fba584a700a5bcb40991e1b2e6bf68b0f3896d767a0da92e69de73de226ac"
->  ) {
->    dynamicFields {
->      pageInfo {
->        hasNextPage
->        endCursor
->      }
->      edges {
->        cursor
->        node {
->          ...DynamicFieldSelect
->        }
->      }
->    }
->  }
->}
- -### -### Dynamic Object Field - ->
fragment DynamicFieldValueSelection on DynamicFieldValue {
->  ... on MoveValue {
->    type {
->      repr
->    }
->    data
->    __typename
->  }
->  ... on MoveObject {
->    hasPublicTransfer
->    contents {
->      type {
->        repr
->      }
->      data
->    }
->    __typename
->  }
->}
->
->fragment DynamicFieldNameSelection on MoveValue {
->  type {
->    repr
->  }
->  data
->  bcs
->}
->
->fragment DynamicFieldSelect on DynamicField {
->  name {
->    ...DynamicFieldNameSelection
->  }
->  value {
->    ...DynamicFieldValueSelection
->  }
->}
->
->query DynamicObjectField {
->  object(
->    address: "0xb57fba584a700a5bcb40991e1b2e6bf68b0f3896d767a0da92e69de73de226ac"
->  ) {
->    dynamicObjectField(
->      name: {type: "0x2::kiosk::Item", bcs: "NLArx1UJguOUYmXgNG8Pv8KbKXLjWtCi6i0Yeq1Vhfw="}
->    ) {
->      ...DynamicFieldSelect
->    }
->  }
->}
- -### -### Owner - ->
{
->  owner(
->    address: "0x931f293ce7f65fd5ebe9542653e1fd92fafa03dda563e13b83be35da8a2eecbe"
->  ) {
->    address
->  }
->}
- -## -## Protocol Configs -### -### Key Value -#### Select the key and value of the protocol configuration - ->
{
->  protocolConfig {
->    configs {
->      key
->      value
->    }
->  }
->}
- -### -### Key Value Feature Flag -#### Select the key and value of the feature flag - ->
{
->  protocolConfig {
->    featureFlags {
->      key
->      value
->    }
->  }
->}
- -### -### Specific Config -#### Select the key and value of the specific protocol configuration, in this case `max_move_identifier_len` - ->
{
->  protocolConfig {
->    config(key: "max_move_identifier_len") {
->      key
->      value
->    }
->  }
->}
- -### -### Specific Feature Flag - ->
{
->  protocolConfig {
->    protocolVersion
->    featureFlag(key: "advance_epoch_start_time_in_safe_mode") {
->      value
->    }
->  }
->}
- -## -## Service Config -### -### Service Config -#### Get the configuration of the running service - ->
{
->  serviceConfig {
->    isEnabled(feature: ANALYTICS)
->    enabledFeatures
->    maxQueryDepth
->    maxQueryNodes
->    maxDbQueryCost
->    defaultPageSize
->    maxPageSize
->    requestTimeoutMs
->    maxQueryPayloadSize
->  }
->}
- -## -## Stake Connection -### -### Stake Connection -#### Get all the staked objects for this address and all the active validators at the epoch when the stake became active - ->
{
->  address(
->    address: "0xc0a5b916d0e406ddde11a29558cd91b29c49e644eef597b7424a622955280e1e"
->  ) {
->    address
->    balance(type: "0x2::sui::SUI") {
->      coinType {
->        repr
->      }
->      totalBalance
->    }
->    stakedSuis {
->      nodes {
->        status
->        principal
->        estimatedReward
->        activatedEpoch {
->          epochId
->          referenceGasPrice
->          validatorSet {
->            activeValidators {
->              nodes {
->                name
->                description
->                exchangeRatesSize
->              }
->            }
->            totalStake
->          }
->        }
->        requestedEpoch {
->          epochId
->        }
->      }
->    }
->  }
->}
- -## -## Sui System State Summary -### -### Sui System State Summary -#### Get the latest sui system state data - ->
{
->  epoch {
->    storageFund {
->      totalObjectStorageRebates
->      nonRefundableBalance
->    }
->    safeMode {
->      enabled
->      gasSummary {
->         computationCost
->         storageCost
->         storageRebate
->         nonRefundableStorageFee
->      }
->    }
->    systemStateVersion
->    systemParameters {
->      durationMs
->      stakeSubsidyStartEpoch
->      minValidatorCount
->      maxValidatorCount
->      minValidatorJoiningStake
->      validatorLowStakeThreshold
->      validatorVeryLowStakeThreshold
->      validatorLowStakeGracePeriod
->    }
->    systemStakeSubsidy {
->      balance
->      distributionCounter
->      currentDistributionAmount
->      periodLength
->      decreaseRate
->
->    }
->  }
->}
- -## -## Transaction Block -### -### Transaction Block -#### Get the data for a TransactionBlock by its digest - ->
{
->  transactionBlock(digest: "HvTjk3ELg8gRofmB1GgrpLHBFeA53QKmUKGEuhuypezg") {
->    sender {
->      address
->    }
->    gasInput {
->      gasSponsor {
->        address
->      }
->      gasPayment {
->        nodes {
->          address
->        }
->      }
->      gasPrice
->      gasBudget
->    }
->    kind {
->      __typename
->    }
->    signatures
->    digest
->    expiration {
->      epochId
->    }
->    effects {
->      timestamp
->    }
->  }
->}
- -### -### Transaction Block Kind - ->
{
->  object(
->    address: "0xd6b9c261ab53d636760a104e4ab5f46c2a3e9cda58bd392488fc4efa6e43728c"
->  ) {
->    previousTransactionBlock {
->      sender {
->        address
->      }
->      kind {
->        __typename
->        ... on ConsensusCommitPrologueTransaction {
->          epoch {
->            epochId
->            referenceGasPrice
->          }
->          round
->          commitTimestamp
->          consensusCommitDigest
->        }
->        ... on ChangeEpochTransaction {
->          computationCharge
->          storageCharge
->          startTimestamp
->          storageRebate
->        }
->        ... on GenesisTransaction {
->          objects {
->            nodes { address }
->          }
->        }
->      }
->    }
->  }
->}
- -## -## Transaction Block Connection -### -### Before After Checkpoint -#### Filter on before_ and after_checkpoint. If both are provided, before must be greater than after - ->
{
->  transactionBlocks(
->    filter: { afterCheckpoint: 10, beforeCheckpoint: 20 }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Changed Object Filter -#### Filter on changedObject - ->
{
->  transactionBlocks(
->    filter: {
->      changedObject: "0x0000000000000000000000000000000000000000000000000000000000000006"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Input Object Filter -#### Filter on inputObject - ->
{
->  transactionBlocks(
->    filter: {
->      inputObject: "0x0000000000000000000000000000000000000000000000000000000000000006"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Input Object Sign Addr Filter -#### multiple filters - ->
{
->  transactionBlocks(
->    filter: {
->      inputObject: "0x0000000000000000000000000000000000000000000000000000000000000006"
->      signAddress: "0x0000000000000000000000000000000000000000000000000000000000000000"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      effects {
->        gasEffects {
->          gasObject {
->            address
->          }
->        }
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Package Filter -#### Filtering on package - ->
{
->  transactionBlocks(filter: { function: "0x3" }) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Package Module Filter -#### Filtering on package and module - ->
{
->  transactionBlocks(
->    filter: {
->      function: "0x3::sui_system"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Package Module Func Filter -#### Filtering on package, module and function - ->
{
->  transactionBlocks(
->    filter: {
->      function: "0x3::sui_system::request_withdraw_stake"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Recv Addr Filter -#### Filter on recvAddress - ->
{
->  transactionBlocks(
->    filter: {
->      recvAddress: "0x0000000000000000000000000000000000000000000000000000000000000000"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Sign Addr Filter -#### Filter on signing address - ->
{
->  transactionBlocks(
->    filter: {
->      signAddress: "0x0000000000000000000000000000000000000000000000000000000000000000"
->    }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Tx Ids Filter -#### Filter on transactionIds - ->
{
->  transactionBlocks(
->    filter: { transactionIds: ["DtQ6v6iJW4wMLgadENPUCEUS5t8AP7qvdG5jX84T1akR"] }
->  ) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### Tx Kind Filter -#### Filter on TransactionKind (only SYSTEM_TX or PROGRAMMABLE_TX) - ->
{
->  transactionBlocks(filter: { kind: SYSTEM_TX }) {
->    nodes {
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->  }
->}
- -### -### With Defaults Ascending -#### Fetch some default amount of transactions, ascending - ->
{
->  transactionBlocks {
->    nodes {
->      digest
->      effects {
->        gasEffects {
->          gasObject {
->            version
->            digest
->          }
->          gasSummary {
->            computationCost
->            storageCost
->            storageRebate
->            nonRefundableStorageFee
->          }
->        }
->        errors
->      }
->      sender {
->        address
->      }
->      gasInput {
->        gasPrice
->        gasBudget
->      }
->    }
->    pageInfo {
->      endCursor
->    }
->  }
->}
- -## -## Transaction Block Effects -### -### Transaction Block Effects - ->
{
->  object(
->    address: "0x0bba1e7d907dc2832edfc3bf4468b6deacd9a2df435a35b17e640e135d2d5ddc"
->  ) {
->    version
->    owner {
->      __typename
->      ... on Shared {
->        initialSharedVersion
->      }
->      __typename
->      ... on Parent {
->        parent {
->          address
->        }
->      }
->      __typename
->      ... on AddressOwner {
->        owner {
->          address
->        }
->      }
->    }
->    previousTransactionBlock {
->      effects {
->        status
->        checkpoint {
->          sequenceNumber
->        }
->        lamportVersion
->        gasEffects {
->          gasSummary {
->            computationCost
->            storageCost
->            storageRebate
->            nonRefundableStorageFee
->          }
->        }
->        balanceChanges {
->          nodes {
->            owner {
->              address
->              balance(type: "0x2::sui::SUI") {
->                totalBalance
->              }
->            }
->            amount
->            coinType {
->              repr
->              signature
->              layout
->            }
->          }
->        }
->        dependencies {
->          nodes {
->            sender {
->              address
->            }
->          }
->        }
->      }
->    }
->  }
->}
- diff --git a/crates/sui-graphql-rpc/src/commands.rs b/crates/sui-graphql-rpc/src/commands.rs index f605efd735946..bda0c2f561fab 100644 --- a/crates/sui-graphql-rpc/src/commands.rs +++ b/crates/sui-graphql-rpc/src/commands.rs @@ -13,17 +13,6 @@ use std::path::PathBuf; version )] pub enum Command { - GenerateDocsExamples, - GenerateSchema { - /// Path to output GraphQL schema to, in SDL format. - #[clap(short, long)] - file: Option, - }, - GenerateExamples { - /// Path to output examples docs. - #[clap(short, long)] - file: Option, - }, StartServer { /// The title to display at the top of the page #[clap(short, long)] diff --git a/crates/sui-graphql-rpc/src/examples.rs b/crates/sui-graphql-rpc/src/examples.rs deleted file mode 100644 index 56f86ca428180..0000000000000 --- a/crates/sui-graphql-rpc/src/examples.rs +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::anyhow; -use markdown_gen::markdown::{AsMarkdown, Markdown}; -use std::io::{BufWriter, Read}; -use std::path::PathBuf; - -#[derive(Debug)] -pub struct ExampleQuery { - pub name: String, - pub contents: String, - pub path: PathBuf, -} - -#[derive(Debug)] -pub struct ExampleQueryGroup { - pub name: String, - pub queries: Vec, - pub _path: PathBuf, -} - -const QUERY_EXT: &str = "graphql"; - -fn regularize_string(s: &str) -> String { - // Replace underscore with space and make every word first letter uppercase - s.replace('_', " ") - .split_whitespace() - .map(|word| { - let mut chars = word.chars(); - match chars.next() { - None => String::new(), - Some(f) => f.to_uppercase().chain(chars).collect(), - } - }) - .collect::>() - .join(" ") -} - -pub fn load_examples() -> anyhow::Result> { - let mut buf: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - buf.push("examples"); - - let mut groups = vec![]; - for entry in std::fs::read_dir(buf).map_err(|e| anyhow::anyhow!(e))? { - let entry = entry.map_err(|e| anyhow::anyhow!(e))?; - let path = entry.path(); - let group_name = path - .file_stem() - .ok_or(anyhow::anyhow!("File stem cannot be read"))? - .to_str() - .ok_or(anyhow::anyhow!("File stem cannot be read"))? - .to_string(); - - let mut group = ExampleQueryGroup { - name: group_name.clone(), - queries: vec![], - _path: path.clone(), - }; - - for file in std::fs::read_dir(path).map_err(|e| anyhow::anyhow!(e))? { - assert!(file.is_ok()); - let file = file.map_err(|e| anyhow::anyhow!(e))?; - assert!(file.path().extension().is_some()); - let ext = file - .path() - .extension() - .ok_or(anyhow!("File extension cannot be read"))? - .to_str() - .ok_or(anyhow!("File extension cannot be read to string"))? - .to_string(); - assert_eq!(ext, QUERY_EXT, "wrong file extension for example"); - - let file_path = file.path(); - let query_name = file_path - .file_stem() - .ok_or(anyhow!("File stem cannot be read"))? - .to_str() - .ok_or(anyhow!("File extension cannot be read to string"))? - .to_string(); - - let mut contents = String::new(); - let mut fp = std::fs::File::open(file_path.clone()).map_err(|e| anyhow!(e))?; - fp.read_to_string(&mut contents).map_err(|e| anyhow!(e))?; - group.queries.push(ExampleQuery { - name: query_name, - contents, - path: file_path, - }); - } - group.queries.sort_by(|x, y| x.name.cmp(&y.name)); - - groups.push(group); - } - - groups.sort_by(|x, y| x.name.cmp(&y.name)); - Ok(groups) -} - -/// This generates a markdown page with all the examples, to be used in the docs site -pub fn generate_examples_for_docs() -> anyhow::Result { - let groups = load_examples()?; - - let mut output = BufWriter::new(Vec::new()); - let mut md = Markdown::new(&mut output); - md.write( - r#"--- -title: Examples -description: Query examples for working with the Sui GraphQL RPC. ---- -"#, - )?; - md.write("This page showcases a number of queries to interact with the network. These examples can also be found in the [repository](https://github.com/MystenLabs/sui/tree/main/crates/sui-graphql-rpc/examples). You can use the [interactive online IDE](https://mainnet.sui.io/rpc/graphql) to run these examples.")?; - for group in groups.iter() { - let group_name = regularize_string(&group.name); - md.write(group_name.heading(2)) - .map_err(|e| anyhow::anyhow!(e))?; - for query in group.queries.iter() { - let name = regularize_string(&query.name); - md.write(name.heading(3)).map_err(|e| anyhow::anyhow!(e))?; - let query = query.contents.lines().collect::>().join("\n"); - let content = format!("```graphql\n{}\n```", query); - md.write(content.as_str()).map_err(|e| anyhow::anyhow!(e))?; - } - } - let bytes = output.into_inner().map_err(|e| anyhow::anyhow!(e))?; - Ok(String::from_utf8(bytes) - .map_err(|e| anyhow::anyhow!(e))? - .replace('\\', "")) -} - -pub fn generate_markdown() -> anyhow::Result { - let groups = load_examples()?; - - let mut output = BufWriter::new(Vec::new()); - let mut md = Markdown::new(&mut output); - - md.write("Sui GraphQL Examples".heading(1)) - .map_err(|e| anyhow!(e))?; - - // TODO: reduce multiple loops - // Generate the table of contents - for (id, group) in groups.iter().enumerate() { - let group_name = regularize_string(&group.name); - let group_name_toc = format!("[{}](#{})", group_name, id); - md.write(group_name_toc.heading(3)) - .map_err(|e| anyhow!(e))?; - - for (inner, query) in group.queries.iter().enumerate() { - let inner_id = inner + 0xFFFF * id; - let inner_name = regularize_string(&query.name); - let inner_name_toc = format!("  [{}](#{})", inner_name, inner_id); - md.write(inner_name_toc.heading(4)) - .map_err(|e| anyhow!(e))?; - } - } - - for (id, group) in groups.iter().enumerate() { - let group_name = regularize_string(&group.name); - - let id_tag = format!("", id); - md.write(id_tag.heading(2)) - .map_err(|e| anyhow::anyhow!(e))?; - md.write(group_name.heading(2)) - .map_err(|e| anyhow::anyhow!(e))?; - for (inner, query) in group.queries.iter().enumerate() { - let inner_id = inner + 0xFFFF * id; - let name = regularize_string(&query.name); - - let id_tag = format!("", inner_id); - md.write(id_tag.heading(3)) - .map_err(|e| anyhow::anyhow!(e))?; - md.write(name.heading(3)).map_err(|e| anyhow::anyhow!(e))?; - - // Extract all lines that start with `#` and use them as headers - let mut headers = vec![]; - let mut query_start = 0; - for (idx, line) in query.contents.lines().enumerate() { - let line = line.trim(); - if line.starts_with('#') { - headers.push(line.trim_start_matches('#')); - } else if line.starts_with('{') { - query_start = idx; - break; - } - } - - // Remove headers from query - let query = query - .contents - .lines() - .skip(query_start) - .collect::>() - .join("\n"); - - let content = format!("
{}
", query); - for header in headers { - md.write(header.heading(4)) - .map_err(|e| anyhow::anyhow!(e))?; - } - md.write(content.quote()).map_err(|e| anyhow::anyhow!(e))?; - } - } - let bytes = output.into_inner().map_err(|e| anyhow::anyhow!(e))?; - Ok(String::from_utf8(bytes) - .map_err(|e| anyhow::anyhow!(e))? - .replace('\\', "")) -} - -#[test] -fn test_generate_markdown() { - use similar::*; - use std::fs::File; - - let mut buf: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - buf.push("docs"); - buf.push("examples.md"); - let mut out_file: File = File::open(buf).expect("Could not open examples.md"); - - // Read the current content of `out_file` - let mut current_content = String::new(); - out_file - .read_to_string(&mut current_content) - .expect("Could not read examples.md"); - let new_content: String = generate_markdown().expect("Generating examples markdown failed"); - - if current_content != new_content { - let mut res = vec![]; - let diff = TextDiff::from_lines(¤t_content, &new_content); - for change in diff.iter_all_changes() { - let sign = match change.tag() { - ChangeTag::Delete => "---", - ChangeTag::Insert => "+++", - ChangeTag::Equal => " ", - }; - res.push(format!("{}{}", sign, change)); - } - panic!("Doc examples have changed. Please run `sui-graphql-rpc generate-examples` to update the docs. Diff: {}", res.join("")); - } -} diff --git a/crates/sui-graphql-rpc/src/lib.rs b/crates/sui-graphql-rpc/src/lib.rs index baea0d2ce2ce8..c2f7cd3f8687b 100644 --- a/crates/sui-graphql-rpc/src/lib.rs +++ b/crates/sui-graphql-rpc/src/lib.rs @@ -8,7 +8,6 @@ pub(crate) mod consistency; pub mod context_data; pub(crate) mod data; mod error; -pub mod examples; pub mod extensions; pub(crate) mod functional_group; mod metrics; diff --git a/crates/sui-graphql-rpc/src/main.rs b/crates/sui-graphql-rpc/src/main.rs index 6e552a09e92e8..349ef0f0f74a4 100644 --- a/crates/sui-graphql-rpc/src/main.rs +++ b/crates/sui-graphql-rpc/src/main.rs @@ -9,7 +9,6 @@ use sui_graphql_rpc::commands::Command; use sui_graphql_rpc::config::{ ConnectionConfig, Ide, ServerConfig, ServiceConfig, TxExecFullNodeConfig, Version, }; -use sui_graphql_rpc::server::builder::export_schema; use sui_graphql_rpc::server::graphiql_server::start_graphiql_server; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; @@ -38,39 +37,6 @@ static VERSION: Version = Version { async fn main() { let cmd: Command = Command::parse(); match cmd { - Command::GenerateDocsExamples => { - let mut buf: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // we are looking to put examples content in - // sui/docs/content/references/sui-graphql/examples.mdx - let filename = "docs/content/references/sui-graphql/examples.mdx"; - buf.pop(); - buf.pop(); - buf.push(filename); - let content = sui_graphql_rpc::examples::generate_examples_for_docs() - .expect("Generating examples markdown file for docs failed"); - std::fs::write(buf, content).expect("Writing examples markdown failed"); - println!("Generated the docs example.mdx file and copied it to {filename}."); - } - Command::GenerateSchema { file } => { - let out = export_schema(); - if let Some(file) = file { - println!("Write schema to file: {:?}", file); - std::fs::write(file, &out).unwrap(); - } else { - println!("{}", &out); - } - } - Command::GenerateExamples { file } => { - let new_content: String = sui_graphql_rpc::examples::generate_markdown() - .expect("Generating examples markdown failed"); - let mut buf: PathBuf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - buf.push("docs"); - buf.push("examples.md"); - let file = file.unwrap_or(buf); - - std::fs::write(file.clone(), new_content).expect("Writing examples markdown failed"); - println!("Written examples to file: {:?}", file); - } Command::StartServer { ide_title, db_url, diff --git a/crates/sui-graphql-rpc/tests/examples_validation_tests.rs b/crates/sui-graphql-rpc/tests/examples_validation_tests.rs index fc2a95d21c90b..205c0e1407b5d 100644 --- a/crates/sui-graphql-rpc/tests/examples_validation_tests.rs +++ b/crates/sui-graphql-rpc/tests/examples_validation_tests.rs @@ -3,105 +3,147 @@ #[cfg(feature = "pg_integration")] mod tests { + use anyhow::{anyhow, Context, Result}; use rand::rngs::StdRng; use rand::SeedableRng; use serial_test::serial; use simulacrum::Simulacrum; use std::cmp::max; + use std::collections::BTreeMap; + use std::fs; use std::path::PathBuf; use std::sync::Arc; use sui_graphql_rpc::config::{ConnectionConfig, Limits}; - use sui_graphql_rpc::examples::{load_examples, ExampleQuery, ExampleQueryGroup}; use sui_graphql_rpc::test_infra::cluster::ExecutorCluster; use sui_graphql_rpc::test_infra::cluster::DEFAULT_INTERNAL_DATA_SOURCE_PORT; use tempfile::tempdir; - fn bad_examples() -> ExampleQueryGroup { - ExampleQueryGroup { - name: "bad_examples".to_string(), - queries: vec![ - ExampleQuery { - name: "multiple_queries".to_string(), + struct Example { + contents: String, + path: Option, + } + + fn good_examples() -> Result> { + let examples = PathBuf::from(&env!("CARGO_MANIFEST_DIR")).join("examples"); + + let mut dirs = vec![examples.clone()]; + let mut queries = BTreeMap::new(); + while let Some(dir) = dirs.pop() { + let entries = + fs::read_dir(&dir).with_context(|| format!("Looking in {}", dir.display()))?; + + for entry in entries { + let entry = entry.with_context(|| format!("Entry in {}", dir.display()))?; + let path = entry.path(); + let typ_ = entry + .file_type() + .with_context(|| format!("Metadata for {}", path.display()))?; + + if typ_.is_dir() { + dirs.push(entry.path()); + continue; + } + + if path.ends_with(".graphql") { + let contents = fs::read_to_string(&path) + .with_context(|| format!("Reading {}", path.display()))?; + + let rel_path = path + .strip_prefix(&examples) + .with_context(|| format!("Generating name from {}", path.display()))? + .with_extension(""); + + let name = rel_path + .to_str() + .ok_or_else(|| anyhow!("Generating name from {}", path.display()))?; + + queries.insert( + name.to_string(), + Example { + contents, + path: Some(path), + }, + ); + } + } + } + + Ok(queries) + } + + fn bad_examples() -> BTreeMap { + BTreeMap::from_iter([ + ( + "multiple_queries".to_string(), + Example { contents: "{ chainIdentifier } { chainIdentifier }".to_string(), - path: PathBuf::from("multiple_queries.graphql"), + path: None, }, - ExampleQuery { - name: "malformed".to_string(), + ), + ( + "malformed".to_string(), + Example { contents: "query { }}".to_string(), - path: PathBuf::from("malformed.graphql"), + path: None, }, - ExampleQuery { - name: "invalid".to_string(), + ), + ( + "invalid".to_string(), + Example { contents: "djewfbfo".to_string(), - path: PathBuf::from("invalid.graphql"), + path: None, }, - ExampleQuery { - name: "empty".to_string(), + ), + ( + "empty".to_string(), + Example { contents: " ".to_string(), - path: PathBuf::from("empty.graphql"), + path: None, }, - ], - _path: PathBuf::from("bad_examples"), - } + ), + ]) } - async fn validate_example_query_group( + async fn test_query( cluster: &ExecutorCluster, - group: &ExampleQueryGroup, + name: &str, + query: &Example, max_nodes: &mut u64, max_output_nodes: &mut u64, max_depth: &mut u64, max_payload: &mut u64, ) -> Vec { - let mut errors = vec![]; - for query in &group.queries { - let resp = cluster - .graphql_client - .execute_to_graphql(query.contents.clone(), true, vec![], vec![]) - .await - .unwrap(); - resp.errors().iter().for_each(|err| { - errors.push(format!( - "Query failed: {}: {} at: {}\nError: {}", - group.name, - query.name, - query.path.display(), - err - )) - }); - if resp.errors().is_empty() { - let usage = resp - .usage() - .expect("Usage fetch should succeed") - .unwrap_or_else(|| panic!("Usage should be present for query: {}", query.name)); - - let nodes = *usage.get("inputNodes").unwrap_or_else(|| { - panic!("Node usage should be present for query: {}", query.name) - }); - let output_nodes = *usage.get("outputNodes").unwrap_or_else(|| { - panic!( - "Output node usage should be present for query: {}", - query.name - ) - }); - let depth = *usage.get("depth").unwrap_or_else(|| { - panic!("Depth usage should be present for query: {}", query.name) - }); - let payload = *usage.get("queryPayload").unwrap_or_else(|| { - panic!("Payload usage should be present for query: {}", query.name) - }); - *max_nodes = max(*max_nodes, nodes); - *max_output_nodes = max(*max_output_nodes, output_nodes); - *max_depth = max(*max_depth, depth); - *max_payload = max(*max_payload, payload); - } + let resp = cluster + .graphql_client + .execute_to_graphql(query.contents.clone(), true, vec![], vec![]) + .await + .unwrap(); + + let errors = resp.errors(); + if errors.is_empty() { + let usage = resp + .usage() + .expect("Usage not found") + .expect("Usage not found"); + *max_nodes = max(*max_nodes, usage["inputNodes"]); + *max_output_nodes = max(*max_output_nodes, usage["outputNodes"]); + *max_depth = max(*max_depth, usage["depth"]); + *max_payload = max(*max_payload, usage["queryPayload"]); + return vec![]; } + errors + .into_iter() + .map(|e| match &query.path { + Some(p) => format!("Query {name:?} at {} failed: {e}", p.display()), + None => format!("Query {name:?} failed: {e}"), + }) + .collect() } #[tokio::test] #[serial] - async fn test_single_all_examples_structure_valid() { + async fn good_examples_within_limits() { let rng = StdRng::from_seed([12; 32]); let data_ingestion_path = tempdir().unwrap().into_path(); let mut sim = Simulacrum::new_with_rng(rng); @@ -119,20 +161,20 @@ mod tests { ) .await; - let groups = load_examples().expect("Could not load examples"); - let mut errors = vec![]; - for group in groups { - let group_errors = validate_example_query_group( - &cluster, - &group, - &mut max_nodes, - &mut max_output_nodes, - &mut max_depth, - &mut max_payload, - ) - .await; - errors.extend(group_errors); + for (name, example) in good_examples().expect("Could not load examples") { + errors.extend( + test_query( + &cluster, + &name, + &example, + &mut max_nodes, + &mut max_output_nodes, + &mut max_depth, + &mut max_payload, + ) + .await, + ); } // Check that our examples can run with our usage limits @@ -167,7 +209,7 @@ mod tests { #[tokio::test] #[serial] - async fn test_bad_examples_fail() { + async fn bad_examples_fail() { let rng = StdRng::from_seed([12; 32]); let data_ingestion_path = tempdir().unwrap().into_path(); let mut sim = Simulacrum::new_with_rng(rng); @@ -185,21 +227,19 @@ mod tests { ) .await; - let bad_examples = bad_examples(); - let errors = validate_example_query_group( - &cluster, - &bad_examples, - &mut max_nodes, - &mut max_output_nodes, - &mut max_depth, - &mut max_payload, - ) - .await; + for (name, example) in bad_examples() { + let errors = test_query( + &cluster, + &name, + &example, + &mut max_nodes, + &mut max_output_nodes, + &mut max_depth, + &mut max_payload, + ) + .await; - assert_eq!( - errors.len(), - bad_examples.queries.len(), - "all examples should fail" - ); + assert!(!errors.is_empty(), "Query {name:?} should have failed"); + } } } From 4ee441c02c61c0ee1d5d6699d9caf35f94215339 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:41:40 +0100 Subject: [PATCH 7/9] [chore][GraphQL] Declutter schemas (#18288) ## Description Remove `draft_target_schema.graphql` and promote `current_progress_schema.graphql` to be the canonical schema for the service -- move it to the top-level of the `sui-graphql-rpc` crate to make it easier to find. This is to avoid confusion about source of truth for the GraphQL schema. Because the TS SDK references the schema at multiple GraphQL versions, we will need to cherry-pick this change to release branches when it lands. ## Test plan CI ## Stack - #17543 - #17692 - #17693 - #17696 - #18287 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: The schema file has been moved from `crates/sui-graphql-rpc/schemas/current_progress_schema.graphql` to `crates/sui-graphql-rpc/schema.graphql`. - [ ] CLI: - [ ] Rust SDK: --- ...progress_schema.graphql => schema.graphql} | 0 .../schema/draft_target_schema.graphql | 1588 ----------------- .../sui-graphql-rpc/tests/snapshot_tests.rs | 7 +- docs/site/docusaurus.config.js | 2 +- .../scripts/update-graphql-schemas.ts | 2 +- 5 files changed, 5 insertions(+), 1594 deletions(-) rename crates/sui-graphql-rpc/{schema/current_progress_schema.graphql => schema.graphql} (100%) delete mode 100644 crates/sui-graphql-rpc/schema/draft_target_schema.graphql diff --git a/crates/sui-graphql-rpc/schema/current_progress_schema.graphql b/crates/sui-graphql-rpc/schema.graphql similarity index 100% rename from crates/sui-graphql-rpc/schema/current_progress_schema.graphql rename to crates/sui-graphql-rpc/schema.graphql diff --git a/crates/sui-graphql-rpc/schema/draft_target_schema.graphql b/crates/sui-graphql-rpc/schema/draft_target_schema.graphql deleted file mode 100644 index 733f13eb57bf7..0000000000000 --- a/crates/sui-graphql-rpc/schema/draft_target_schema.graphql +++ /dev/null @@ -1,1588 +0,0 @@ -# Copyright (c) Mysten Labs, Inc. -# SPDX-License-Identifier: Apache-2.0 - -# GraphQL Schema Draft -# -------------------- -# -# This is a draft design of the schema used by the second iteration of -# the RPC service. Note that some elements may not be complete, and -# others may exist in this schema but may not appear in the production -# design initially, or ever. -# -# The source of truth for the actual schema is accessed by querying -# the GraphQL server for its `__schema`. - -schema { - query: Query - subscription: Subscription - mutation: Mutation -} - -type Query { - # First four bytes of the network's genesis checkpoint digest - # (uniquely identifies the network) - chainIdentifier: String! - - # Range of checkpoints that the RPC has data available for (for data - # that can be tied to a particular checkpoint). - availableRange: AvailableRange! - - # Configuration for this RPC service - serviceConfig: ServiceConfig! - - # Simulate running a transaction to inspect its effects without - # committing to them on-chain. - # - # `txBytes` either a `TransactionData` struct or a `TransactionKind` - # struct, BCS-encoded and then Base64-encoded. The expected - # type is controlled by the presence or absence of `txMeta`: If - # present, `txBytes` is assumed to be a `TransactionKind`, if - # absent, then `TransactionData`. - # - # `txMeta` the data that is missing from a `TransactionKind` to make - # a `TransactionData` (sender address and gas information). All - # its fields are nullable: `sender` defaults to `0x0`, if - # `gasObjects` is not present, or is an empty list, it is - # substituted with a mock Coin object, and `gasPrice` defaults to - # the reference gas price. - # - # `skipChecks` optional flag to disable the usual verification - # checks that prevent access to objects that are owned by - # addresses other than the sender, and calling non-public, - # non-entry functions. Defaults to false. - dryRunTransactionBlock( - txBytes: Base64!, - txMeta: TransactionMetadata, - skipChecks: Boolean, - ): DryRunResult - - owner(address: SuiAddress!): Owner - object(address: SuiAddress!, version: Int): Object - address(address: SuiAddress!): Address - type(type: String!): MoveType! - - # Fetch epoch information by ID (defaults to the latest epoch). - epoch(id: Int): Epoch - - # `protocolVersion` defaults to the latest protocol version. - protocolConfig(protocolVersion: Int): ProtocolConfigs - - # Fetch checkpoint information by sequence number or digest - # (defaults to the latest available checkpoint). - checkpoint(id: CheckpointId): Checkpoint - - # Fetch a transaction block by its transaction digest - transactionBlock(digest: String!): TransactionBlock - - coinMetadata(coinType: String!): CoinMetadata - - checkpoints( - first: Int, - after: String, - last: Int, - before: String, - ): CheckpointConnection! - - coins( - first: Int, - after: String, - last: Int, - before: String, - type: String, - ): CoinConnection! - - transactionBlocks( - first: Int, - after: String, - last: Int, - before: String, - filter: TransactionBlockFilter, - ): TransactionBlockConnection! - - events( - first: Int, - after: String, - last: Int, - before: String, - filter: EventFilter, - ): EventConnection! - - objects( - first: Int, - after: String, - last: Int, - before: String, - filter: ObjectFilter, - ): ObjectConnection! - - resolveSuinsAddress(name: String!): Address - - # NB. Will be moved into a private, explorer-specific extension. - networkMetrics: NetworkMetrics - moveCallMetrics: MoveCallMetrics - - allEpochAddressMetrics( - first: Int, - after: String, - last: Int, - before: String, - ): AddressMetricsConnection! -} - -# NB. Add after MVP has stabilised. -# -# Subscriptions use a "push-pull" system: Subscribers are notified -# when there is new data by being sent the cursor pointing after that -# new data. To actually fetch the data, a call must be made to the -# equivalent Connection API: -# -# e.g. When subscription `subscribe { events(filter: F) }` pushes -# cursor `E`. Then -# -# query { events(before: E, filter: F) } -# -# Will start paginating events up to the new data (multiple calls may -# be required if there are multiple pages of information between the -# start and the latest). If the client has already processed some -# prefix, up to cursor `P`, then they can resume with: -# -# query { events(after: P, before: E, filter: F) } -# -# The API for transactions is similar. -type Subscription { - events(filter: EventFilter): String! - transactions(filter: TransactionBlockFilter): String! -} - -type Mutation { - # Execute a transaction, committing its effects on chain. - # - # `txBytes` is a `TransactionData` struct that has been BCS-encoded - # and then Base64-encoded. - # `signatures` are a list of `flag || signature || pubkey` bytes, - # Base64-encoded. - # - # Waits until the transaction has been finalized on chain to return - # its transaction digest. If the transaction could not be - # finalized, returns the errors that prevented it, instead. - executeTransactionBlock( - txBytes: Base64!, - signatures: [Base64!]!, - ): ExecutionResult -} - -# String containing 32B hex-encoded address, with a leading "0x". -# Leading zeroes can be omitted on input but will always appear in -# outputs (SuiAddress in output is guaranteed to be 66 characters -# long). -scalar SuiAddress - -# String representation of an arbitrary width, possibly signed integer -scalar BigInt - -# String containing Base64-encoded binary data. -scalar Base64 - -# ISO-8601 Date and Time -scalar DateTime - -# Arbitrary JSON data -scalar JSON - -# Scalar representing the contents of a Move Value, corresponding to -# the following recursive type: -# -# type MoveData = -# { Number: BigInt } -# | { Bool: bool } -# | { Address: SuiAddress } -# | { UID: SuiAddress } -# | { ID: SuiAddress } -# | { String: string } -# | { Vector: [MoveData] } -# | { Option: MoveData? } -# | { Struct: [{ name: string, value: MoveData }] } -scalar MoveData - -# The signature of a concrete Move Type (a type with all its type -# parameters instantiated with concrete types, that contains no -# references), corresponding to the following recursive type: -# -# type MoveTypeSignature = -# "address" -# | "bool" -# | "u8" | "u16" | ... | "u256" -# | { vector: MoveTypeSignature } -# | { -# struct: { -# package: string, -# module: string, -# type: string, -# typeParameters: [MoveTypeSignature], -# } -# } -scalar MoveTypeSignature - -# The shape of a concrete Move Type (a type with all its type -# parameters instantiated with concrete types), corresponding to the -# following recursive type: -# -# type MoveTypeLayout = -# "address" -# | "bool" -# | "u8" | "u16" | ... | "u256" -# | { vector: MoveTypeLayout } -# | { -# struct: { -# type: string, -# fields: [{ name: string, layout: MoveTypeLayout }], -# } -# } -scalar MoveTypeLayout - -# The shape of an abstract Move Type (a type that can contain free -# type parameters, and can optionally be taken by reference), -# corresponding to the following recursive type: -# -# type OpenMoveTypeSignature = { -# ref: ("&" | "&mut")?, -# body: OpenMoveTypeSignatureBody, -# } -# -# type OpenMoveTypeSignatureBody = -# "address" -# | "bool" -# | "u8" | "u16" | ... | "u256" -# | { vector: OpenMoveTypeSignatureBody } -# | { -# struct: { -# package: string, -# module: string, -# type: string, -# typeParameters: [OpenMoveTypeSignatureBody]? -# } -# } -# | { typeParameter: number } -scalar OpenMoveTypeSignature - -# The extra data required to turn a `TransactionKind` into a -# `TransactionData` in a dry-run. -input TransactionMetadata { - sender: SuiAddress - gasPrice: Int - gasBudget: Int - gasObjects: [ObjectRef!] - gasSponsor: SuiAddress -} - -# A reference to a particular version of an object. -input ObjectRef { - address: SuiAddress! - version: Int! - digest: String! -} - -# Filter either by the digest, or the sequence number, or neither, to -# get the latest checkpoint. -input CheckpointId { - digest: String - sequenceNumber: Int -} - -input ObjectFilter { - # This field is used to specify the type of objects that should be - # include in the query results. - # - # Objects can be filtered by their type's package, package::module, - # or their fully qualified type name. - # - # Generic types can be queried by either the generic type name, e.g. - # `0x2::coin::Coin`, or by the full type name, such as - # `0x2::coin::Coin<0x2::sui::SUI>`. - type: String - - # Filter for live objects by their current owners. - owner: SuiAddress - - # Filter for live objects by their IDs. - objectIds: [SuiAddress!] - - # Filter for live or potentially historical objects by their ID and version. - objectKeys: [ObjectKey!] - - # Enhancement (post-MVP), compound filters. Compound filters are - # exclusive (must be the only filter set if they are used). - any: [ObjectFilter] - all: [ObjectFilter] - not: ObjectFilter -} - -input ObjectKey { - objectId: SuiAddress! - version: Int! -} - -input EventFilter { - sender: SuiAddress - transactionDigest: String - # Enhancement (post-MVP), requires compound filters to be useful. - afterCheckpoint: Int - beforeCheckpoint: Int - - # Events emitted by a particular module. An event is emitted by a - # particular module if some function in the module is called by a - # PTB and emits an event. - # - # Modules can be filtered by their package, or package::module. - emittingModule: String - - # This field is used to specify the type of event emitted. - # - # Events can be filtered by their type's package, package::module, - # or their fully qualified type name. - # - # Generic types can be queried by either the generic type name, e.g. - # `0x2::coin::Coin`, or by the full type name, such as - # `0x2::coin::Coin<0x2::sui::SUI>`. - eventType: String - - # Enhancement (post-MVP), requires compound filters to be useful. - startTime: DateTime - endTime: DateTime - - # Enhancement (post-MVP), compound filters. Compound filters are - # exclusive (must be the only filter set if they are used). - any: [EventFilter] - all: [EventFilter] - not: EventFilter -} - -input TransactionBlockFilter { - # Filter by the function called. Limited to an individual package, - # package::module, or package::module::function. - function: String - - kind: TransactionBlockKindInput - afterCheckpoint: Int - beforeCheckpoint: Int - - signAddress: SuiAddress - sentAddress: SuiAddress - recvAddress: SuiAddress - paidAddress: SuiAddress - - inputObject: SuiAddress - changedObject: SuiAddress - - transactionIDs: [String!] - - # Enhancement (post-MVP), consistency with EventFilter -- timestamp - # comes from checkpoint timestamp. - startTime: DateTime - endTime: DateTime - - # Enhancement (post-MVP), compound filters. Compound filters are - # exclusive (must be the only filter set if they are used). - any: [TransactionBlockFilter] - all: [TransactionBlockFilter] - not: TransactionBlockFilter -} - -input DynamicFieldFilter { - # Filter the type of dynamic field name. - # - # Names can be filtered by their type's package, package::module, or - # their fully qualified type name. - # - # Generic types can be queried by either the generic type name, e.g. - # `0x2::coin::Coin`, or by the full type name, such as - # `0x2::coin::Coin<0x2::sui::SUI>`. - nameType: String - - # Filter the type of dynamic field value. - # - # Values can be filtered by their type's package, package::module, - # or their fully qualified type name. - # - # Generic types can be queried by either the generic type name, e.g. - # `0x2::coin::Coin`, or by the full type name, such as - # `0x2::coin::Coin<0x2::sui::SUI>`. - valueType: String -} - -type AvailableRange { - first: Checkpoint - last: Checkpoint -} - -type ServiceConfig { - availableVersions: [String!] - enabledFeatures: [Feature!] - isEnabled(feature: Feature!): Boolean! - - maxQueryDepth: Int! - maxQueryNodes: Int! - maxOutputNodes: Int! - defaultPageSize: Int! - maxPageSize: Int! - requestTimeoutMs: Int! - maxQueryPayloadSize: Int! -} - -enum Feature { - ANALYTICS - COINS - DYNAMIC_FIELDS - NAME_SERVICE - SUBSCRIPTIONS - SYSTEM_STATE -} - -interface IOwner { - address: SuiAddress! - - objects( - first: Int, - after: String, - last: Int, - before: String, - # Enhancement (post-MVP) relies on compound filters. - filter: ObjectFilter, - ): MoveObjectConnection! - - balance(type: String!): Balance - balances( - first: Int, - after: String, - last: Int, - before: String, - ): BalanceConnection! - - # `type` defaults to `0x2::sui::SUI`. - coins( - first: Int, - after: String, - last: Int, - before: String, - type: String, - ): CoinConnection! - - stakedSuis( - first: Int, - after: String, - last: Int, - before: String, - ): StakedSuiConnection! - - dynamicField(dynamicFieldName: DynamicFieldName!): DynamicField - dynamicObjectField(dynamicFieldName: DynamicFieldName!): DynamicField - dynamicFields( - first: Int, - after: String, - last: Int, - before: String, - # Enhancement (post-MVP) to filter dynamic fields by type. - filter: DynamicFieldFilter, - ): DynamicFieldConnection! - - defaultSuinsName: String - suinsRegistrations( - first: Int, - after: String, - last: Int, - before: String, - ): SuinsRegistrationConnection! -} - -union ObjectOwner = Immutable | Shared | Parent | AddressOwner - -type Immutable { - # Dummy field - _: Boolean -} - -type Shared { - initialSharedVersion: Int! -} - -type Parent { - # Child objects are an implementation-detail of dynamic fields. Only - # another object can be a parent of a child object (not an address). - parent: Object -} - -type AddressOwner { - # The address that owns an object could be an Address, or an Object. - owner: Owner -} - -interface IObject { - version: Int! - digest: String! - owner: ObjectOwner - - previousTransactionBlock: TransactionBlock - storageRebate: BigInt - - display: [DisplayEntry!] - - # Transaction Blocks that sent objects to this object - receivedTransactionBlocks( - first: Int, - after: String, - last: Int, - before: String, - # Enhancement (post-MVP) relies on compound filters. - filter: TransactionBlockFilter, - ): TransactionBlockConnection! - - bcs: Base64 -} - -interface IMoveObject { - contents: MoveValue -} - -# Returned by Object.owner, where we can't disambiguate between -# Address and Object. -type Owner implements IOwner { - asAddress: Address - asObject: Object -} - -type Address implements IOwner { - transactionBlocks( - first: Int, - after: String, - last: Int, - before: String, - relation: AddressTransactionBlockRelationship, - # Enhancement (post-MVP) relies on compound filters. - filter: TransactionBlockFilter, - ): TransactionBlockConnection! -} - -enum AddressTransactionBlockRelationship { - SIGN # Transactions this address has signed - SENT # Transactions that transferred objects from this address - RECV # Transactions that received objects into this address - PAID # Transactions that were paid for by this address -} - -type Object implements IOwner & IObject { - asMoveObject: MoveObject - asMovePackage: MovePackage -} - -type DisplayEntry { - key: String! - value: String - error: String -} - -type Epoch { - epochId: Int! - protocolConfigs: ProtocolConfigs - referenceGasPrice: BigInt - validatorSet: ValidatorSet - - startTimestamp: DateTime! - endTimestamp: DateTime - - totalCheckpoints: BigInt - totalGasFees: BigInt - totalStakeRewards: BigInt - totalStakeSubsidies: BigInt - fundSize: BigInt - netInflow: BigInt - fundInflow: BigInt - fundOutflow: BigInt - - # SystemState fields - storageFund: StorageFund - safeMode: SafeMode - systemStateVersion: BigInt - systemParameters: SystemParameters - systemStakeSubsidy: StakeSubsidy - - checkpoints( - first: Int, - after: String, - last: Int, - before: String, - ): CheckpointConnection! - - transactionBlocks( - first: Int, - after: String, - last: Int, - before: String, - # Enhancement (post-MVP) relies on compound filters. - filter: TransactionBlockFilter, - ): TransactionBlockConnection! -} - -type ProtocolConfigs { - protocolVersion: Int! - featureFlags: [ProtocolConfigFeatureFlag!]! - configs: [ProtocolConfigAttr!]! - config(key: String!): ProtocolConfigAttr - featureFlag(key: String!): ProtocolConfigFeatureFlag -} - -type ProtocolConfigAttr { - key: String! - value: String! -} - -type ProtocolConfigFeatureFlag { - key: String! - value: Boolean! -} - -type SystemParameters { - durationMs: BigInt - stakeSubsidyStartEpoch: Int - - minValidatorCount: Int - maxValidatorCount: Int - - minValidatorJoiningStake: BigInt - validatorLowStakeThreshold: BigInt - validatorVeryLowStakeThreshold: BigInt - validatorLowStakeGracePeriod: Int -} - -type StakeSubsidy { - balance: BigInt - distributionCounter: Int - currentDistributionAmount: BigInt - periodLength: Int - decreaseRate: Int -} - -type ValidatorSet { - totalStake: BigInt - - activeValidators( - first: Int, - after: String, - last: Int, - before: String - ): ValidatorConnection! - - # Indices into `activeValidators` - pendingRemovals: [Int] - - pendingActiveValidators: MoveObject - pendingActiveValidatorsSize: Int - - stakePoolMappings: MoveObject - stakePoolMappingsSize: Int - - inactivePools: MoveObject - inactivePoolsSize: Int - - validatorCandidates: MoveObject - validatorCandidatesSize: Int -} - -type Validator { - address: Address! - - credentials: ValidatorCredentials - nextEpochCredentials: ValidatorCredentials - - name: String - description: String - imageUrl: String - projectUrl: String - - operationCap: MoveObject - stakingPool: MoveObject - - exchangeRates: MoveObject - exchangeRatesSize: Int - - stakingPoolActivationEpoch: Int - stakingPoolSuiBalance: BigInt - rewardsPool: BigInt - poolTokenBalance: BigInt - pendingStake: BigInt - pendingTotalSuiWithdraw: BigInt - pendingPoolTokenWithdraw: BigInt - - votingPower: Int - stakeUnits: Int - gasPrice: BigInt - commissionRate: Int - nextEpochStake: BigInt - nextEpochGasPrice: BigInt - nextEpochCommissionRate: Int - - # The number of epochs for which this validator has been below the - # low stake threshold. - atRisk: Int - - # The other validators this validator has reported - reportRecords: [SuiAddress!] - - apy: Int -} - -type ValidatorCredentials { - protocolPubKey: Base64 - networkPubKey: Base64 - workerPubKey: Base64 - proofOfPossession: Base64 - - netAddress: String - p2pAddreess: String - primaryAddress: String - workerAddress: String -} - -type StorageFund { - totalObjectStorageRebates: BigInt - nonRefundableBalance: BigInt -} - -type SafeMode { - enabled: Boolean - gasSummary: GasCostSummary -} - -type Checkpoint { - digest: String! - sequenceNumber: Int! - - timestamp: DateTime! - validatorSignatures: Base64 - - # Commitments - previousCheckpointDigest: String - liveObjectSetDigest: String - - networkTotalTransactions: Int - rollingGasSummary: GasCostSummary - - epoch: Epoch - - transactionBlocks( - first: Int, - after: String, - last: Int, - before: String, - # Enhancement (post-MVP) relies on compound filters. - filter: TransactionBlockFilter, - ): TransactionBlockConnection! - - # NB. Will be moved into a private, explorer-specific extension. - addressMetrics: AddressMetrics -} - -type TransactionBlock { - digest: String - - sender: Address - gasInput: GasInput - kind: TransactionBlockKind - signatures: [Base64!] - effects: TransactionBlockEffects - - expiration: Epoch - - bcs: Base64 -} - -enum TransactionBlockKindInput { - PROGRAMMABLE_TX - SYSTEM_TX -} - -union TransactionBlockKind = - ConsensusCommitPrologueTransaction - | GenesisTransaction - | ChangeEpochTransaction - | ProgrammableTransactionBlock - | AuthenticatorStateUpdateTransaction - | RandomnessStateUpdateTransaction - | EndOfEpochTransaction - -type ConsensusCommitPrologueTransaction { - epoch: Epoch! - round: Int! - commitTimestamp: DateTime! - consensusCommitDigest: String -} - -type GenesisTransaction { - objects( - first: Int, - after: String, - last: Int, - before: String, - ): ObjectConnection! -} - -type ChangeEpochTransaction { - epoch: Epoch - protocolVersion: Int! - startTimestamp: DateTime! - - storageCharge: BigInt! - computationCharge: BigInt! - storageRebate: BigInt! - nonRefundableStorageFee: BigInt! - - systemPackages( - first: Int, - after: String, - last: Int, - before: String, - ): MovePackageConnection! -} - -type ProgrammableTransactionBlock { - inputs( - first: Int, - after: String, - last: Int, - before: String, - ): TransactionInputConnection! - - transactions( - first: Int, - after: String, - last: Int, - before: String, - ): ProgrammableTransactionConnection! -} - -union TransactionInput = OwnedOrImmutable | SharedInput | Receiving | Pure - -type OwnedOrImmutable { - address: SuiAddress! - version: Int! - digest: String! - object: Object -} - -type SharedInput { - address: SuiAddress! - initialSharedVersion: Int! - mutable: Boolean! -} - -type Receiving { - address: SuiAddress! - version: Int! - digest: String! - object: Object -} - -type Pure { - bytes: Base64! -} - -union TransactionArgument = GasCoin | Input | Result - -type GasCoin { _: Boolean } -type Input { ix: Int! } -type Result { cmd: Int!, ix: Int } - -union ProgrammableTransaction = - MoveCallTransaction - | TransferObjectsTransaction - | SplitCoinTransaction - | MergeCoinsTransaction - | PublishTransaction - | UpgradeTransaction - | MakeMoveVecTransaction - -type MoveCallTransaction { - package: SuiAddress! - module: String! - functionName: String! - function: MoveFunction - typeArguments: [MoveType!]! - arguments: [TransactionArgument!]! -} - -type TransferObjectsTransaction { - objects: [TransactionArgument!]! - address: TransactionArgument! -} - -type SplitCoinsTransaction { - coin: TransactionArgument! - amounts: [TransactionArgument!]! -} - -type MergeCoinsTransaction { - coin: TransactionArgument! - coins: [TransactionArgument!]! -} - -type PublishTransaction { - modules: [Base64!]! - dependencies: [SuiAddress!]! -} - -type UpgradeTransaction { - modules: [Base64!]! - dependencies: [SuiAddress!]! - currentPackage: SuiAddress! - upgradeTicket: TransactionArgument! -} - -type MakeMoveVecTransaction { - type: MoveType - elements: [TransactionArgument!]! -} - -type TransactionBlockEffects { - transactionBlock: TransactionBlock! - status: ExecutionStatus - - errors: String - dependencies( - first: Int, - after: String, - last: Int, - before: String, - ): TransactionBlockConnection! - - lamportVersion: Int - gasEffects: GasEffects - - unchangedSharedObjects( - first: Int, - after: String, - last: Int, - before: String, - ): UnchangedSharedObjectConnection! - - objectChanges( - first: Int, - after: String, - last: Int, - before: String, - ): ObjectChangeConnection! - - balanceChanges( - first: Int, - after: String, - last: Int, - before: String, - ): BalanceChangeConnection! - - timestamp: DateTime - epoch: Epoch - checkpoint: Checkpoint - - events( - first: Int, - after: String, - last: Int, - before: String, - # Extension (post-MVP) relies on compound filters - filter: EventFilter, - ): EventConnection! - - bcs: Base64 -} - -enum ExecutionStatus { - SUCCESS - FAILURE -} - -type GasInput { - gasSponsor: Address - gasPayment( - first: Int, - after: String, - last: Int, - before: String, - ): ObjectConnection! - - gasPrice: BigInt - gasBudget: BigInt -} - -type GasEffects { - gasObject: Coin - gasSummary: GasCostSummary -} - -type GasCostSummary { - computationCost: BigInt - storageCost: BigInt - storageRebate: BigInt - nonRefundableStorageFee: BigInt -} - -union UnchangedSharedObject = SharedObjectRead | SharedObjectDelete - -type SharedObjectRead { - address: SuiAddress! - version: u64! - digest: String! - object: Object -} - -type SharedObjectDelete { - address: SuiAddress! - version: u64! - - # Whether this transaction intended to use this shared object - # mutably or not. - mutable: Boolean! -} - -type ObjectChange { - address: SuiAddress! - - inputState: Object - outputState: Object - - idCreated: Boolean - idDeleted: Boolean -} - -type BalanceChange { - owner: Owner - coinType: MoveType - amount: BigInt -} - -type Event { - # Module that the event was emitted by - sendingModule: MoveModule - - sender: Address - timestamp: DateTime - - type: MoveType! - bcs: Base64! - data: MoveData! - json: JSON! -} - -type Balance { - coinType: MoveType - coinObjectCount: Int - totalBalance: BigInt -} - -type Coin implements IOwner & IObject { - coinBalance: BigInt -} - -type StakedSui implements IOwner & IObject { - stakeStatus: StakeStatus! - requestEpoch: Epoch - activeEpoch: Epoch - principal: BigInt - - # Only available if status is `ACTIVE`. - estimatedReward: BigInt -} - -enum StakeStatus { - PENDING - ACTIVE - UNSTAKED -} - -type CoinMetadata implements IOwner & IObject { - decimals: Int - name: String - symbol: String - description: String - iconUrl: String - supply: BigInt -} - -input DynamicFieldName { - type: String! - bcs: Base64! -} - -type DynamicField { - name: MoveValue - value: DynamicFieldValue -} - -union DynamicFieldValue = MoveObject | MoveValue - -type MoveObject implements IOwner & IObject & IMoveObject { - asCoin: Coin - asStakedSui: StakedSui - asCoinMetadata: CoinMetadata - asSuinsRegistration: SuinsRegistration -} - -type MovePackage implements IOwner & IObject { - module(name: String!): MoveModule - modules( - first: Int, - after: String, - last: Int, - before: String, - ): MoveModuleConnection! - - linkage: [Linkage!] - typeOrigins: [TypeOrigin!] - - moduleBcs: Base64 -} - -type Linkage { - originalId: SuiAddress! - upgradedId: SuiAddress! - version: Int! -} - -type TypeOrigin { - module: String! - struct: String! - definingId: SuiAddress! -} - -enum MoveAbility { - COPY - DROP - STORE - KEY -} - -enum MoveVisibility { - PUBLIC - PRIVATE - FRIEND -} - -type MoveStructTypeParameter { - constraints: [MoveAbility!]! - isPhantom: Boolean! -} - -type MoveFunctionTypeParameter { - constraints: [MoveAbility!]! -} - -type MoveModule { - package: SuiAddress! - name: String! - - fileFormatVersion: Int! - - friends( - first: Int, - after: String, - last: Int, - before: String - ): MoveModuleConnection! - - struct(name: String!): MoveStruct - structs( - first: Int, - after: String, - last: Int, - before: String, - ): MoveStructConnection! - - function(name: String!): MoveFunction - functions( - first: Int, - after: String, - last: Int, - before: String, - ): MoveFunctionConnection! - - bytes: Base64 - disassembly: String -} - -type MoveStruct { - module: MoveModule! - name: String! - abilities: [MoveAbility!] - typeParameters: [MoveStructTypeParameter!] - fields: [MoveField!] -} - -type MoveField { - name: String! - type: OpenMoveType -} - -type MoveFunction { - module: MoveModule! - name: String! - - visibility: MoveVisibility - isEntry: Boolean - - typeParameters: [MoveFunctionTypeParameter!] - parameters: [OpenMoveType!] - return: [OpenMoveType!] -} - -type MoveValue { - type: MoveType! - data: MoveData! - json: JSON! - - bcs: Base64! -} - -# Represents concrete types (no type parameters, no references) -type MoveType { - # Flat representation of the type signature, as a displayable string. - repr: String! - # Structured representation of the type signature. - signature: MoveTypeSignature! - # Structured representation of the "shape" of values that match this type. - layout: MoveTypeLayout! - # The abilities this concrete type has. - abilities: [MoveAbility!]! -} - -# Represents types that could contain references or free type -# parameters. Such types can appear as function parameters, or fields -# in structs. -type OpenMoveType { - # Flat representation of the type signature, as a displayable string. - repr: String! - # Structured representation of the type signature. - signature: OpenMoveTypeSignature! -} - -# Metrics (omitted for brevity) -type NetworkMetrics -type MoveCallMetrics -type AddressMetrics - -# Execution - -# Either TransactionBlockEffects on success, or error on failure. -type ExecutionResult { - effects: TransactionBlockEffects - errors: [String!] -} - -type DryRunResult { - transaction: TransactionBlock - error: String - results: [DryRunEffect!] -} - -type DryRunEffect { - # Changes made to arguments that were mutably borrowed by this - # transaction - mutatedReferences: [DryRunMutation!] - - # Results of this transaction - returnValues: [DryRunReturn!] -} - -type DryRunMutation { - input: TransactionArgument - type: MoveType - bcs: Base64 -} - -type DryRunReturn { - type: MoveType - bcs: Base64 -} - -# Connections - -# Pagination -type PageInfo { - hasNextPage: Boolean! - hasPreviousPage: Boolean! - startCursor: String - endCursor: String -} - -# Checkpoints -type CheckpointConnection { - edges: [CheckpointEdge!]! - nodes: [Checkpoint!]! - pageInfo: PageInfo! -} - -type CheckpointEdge { - cursor: String - node: Checkpoint! -} - -# Balance -type BalanceConnection { - edges: [BalanceEdge!]! - nodes: [Balance!]! - pageInfo: PageInfo! -} - -type BalanceEdge { - cursor: String - node: Balance! -} - -# Coin -type CoinConnection { - edges: [CoinEdge!]! - nodes: [Coin!]! - pageInfo: PageInfo! -} - -type CoinEdge { - cursor: String - node: Coin! -} - -# DynamicField -type DynamicFieldConnection { - edges: [DynamicFieldEdge!]! - nodes: [DynamicField!]! - pageInfo: PageInfo! -} - -type DynamicFieldEdge { - cursor: String - node: DynamicField! -} - -# Object -type ObjectConnection { - edges: [ObjectEdge!]! - nodes: [Object!]! - pageInfo: PageInfo! -} - -type ObjectEdge { - cursor: String - node: Object! -} - -# MoveObject -type MoveObjectConnection { - edges: [MoveObjectEdge!]! - nodes: [MoveObject!]! - pageInfo: PageInfo! -} - -type MoveObjectEdge { - cursor: String - node: MoveObject! -} - -# MovePackage -type MovePackageConnection { - edges: [MovePackageEdge!]! - nodes: [MovePackage!]! - pageInfo: PageInfo! -} - -type MovePackageEdge { - cursor: String - node: MovePackage! -} - -# Event -type EventConnection { - edges: [EventEdge!]! - nodes: [Event!]! - pageInfo: PageInfo! -} - -type EventEdge { - cursor: String - node: Event! -} - -# MoveFunction -type MoveFunctionConnection { - edges: [MoveFunctionEdge!]! - nodes: [MoveFunction!]! - pageInfo: PageInfo! -} - -type MoveFunctionEdge { - cursor: String - node: MoveFunction! -} - -# MoveModuleConnection -type MoveModuleConnection { - edges: [MoveModuleEdge] - nodes: [MoveModule] - pageInfo: PageInfo! -} - -type MoveModuleEdge { - cursor: String - node: MoveModule -} - -# MoveStructConnection -type MoveStructConnection { - edges: [MoveStructEdge!]! - nodes: [MoveStruct!]! - pageInfo: PageInfo! -} - -type MoveStructEdge { - cursor: String - node: MoveStruct! -} - -# TransactionBlockConnection -type TransactionBlockConnection { - totalTransactionBlocks: Int - edges: [TransactionBlockEdge!]! - nodes: [TransactionBlock!]! - pageInfo: PageInfo! -} - -type TransactionBlockEdge { - cursor: String - node: TransactionBlock! -} - -# TransactionInputConnection -type TransactionInputConnection { - edges: [TransactionInputEdge!]! - nodes: [TransactionInput!]! - pageInfo: PageInfo! -} - -type TransactionInputEdge { - cursor: String - node: TransactionInput! -} - -# ProgrammableTransactionConnection -type ProgrammableTransactionConnection { - edges: [ProgrammableTransactionEdge!]! - nodes: [ProgrammableTransaction!]! - pageInfo: PageInfo! -} - -type ProgrammableTransactionEdge { - cursor: String - node: ProgrammableTransaction! -} - -# UnchangedSharedObjectConnection - -type UnchangedSharedObjectConnection { - edges: [UnchangedSharedObjectEdge!]! - nodes: [UnchangedSharedObject!]! - pageInfo: PageInfo! -} - -type UnchangedSharedObjectEdge { - cursor: String - node: UnchangedSharedObject! -} - -# ObjectChangeConnection -type ObjectChangeConnection { - edges: [ObjectChangeEdge!]! - nodes: [ObjectChange!]! - pageInfo: PageInfo! -} - -type ObjectChangeEdge { - cursor: String - node: ObjectChange -} - -# BalanceChangeConnection -type BalanceChangeConnection { - edges: [BalanceChangeEdge!]! - nodes: [BalanceChange!]! - pageInfo: PageInfo! -} - -type BalanceChangeEdge { - cursor: String - node: BalanceChange -} - -# MoveModuleConnection -type MoveModuleConnection { - edges: [MoveModuleEdge!]! - nodes: [MoveModule!]! - pageInfo: PageInfo! -} - -type MoveModuleEdge { - cursor: String - node: MoveModule! -} - -# SuinsRegistrationConnection -type SuinsRegistrationConnection { - edges: [SuinsRegistrationEdge!]! - nodes: [SuinsRegistration!]! - pageInfo: PageInfo! -} - -type SuinsRegistrationEdge { - cursor: String - node: SuinsRegistration -} - -type SuinsRegistration { - """ - Domain name of the SuinsRegistration object - """ - domain: String! - """ - Convert the SuinsRegistration object into a Move object - """ - asMoveObject: MoveObject! -} - -# AddressMetricsConnection -type AddressMetricsConnection { - edges: [AddressMetricEdge!]! - nodes: [AddressMetric!]! - pageInfo: PageInfo! -} - -type AddressMetricEdge { - cursor: String - node: AddressMetrics! -} - -# StakedSuiConnection -type StakedSuiConnection { - edges: [StakedSuiEdge!]! - nodes: [StakedSui!]! - pageInfo: PageInfo! -} - -type StakedSuiEdge { - cursor: String - node: StakedSui! -} - -# ValidatorConnection -type ValidatorConnection { - edges: [ValidatorEdge!]! - nodes: [Validator!]! - pageInfo: PageInfo! -} - -type ValidatorEdge { - cursor: String - node: Validator! -} diff --git a/crates/sui-graphql-rpc/tests/snapshot_tests.rs b/crates/sui-graphql-rpc/tests/snapshot_tests.rs index 30a66934b5deb..dcefef844375d 100644 --- a/crates/sui-graphql-rpc/tests/snapshot_tests.rs +++ b/crates/sui-graphql-rpc/tests/snapshot_tests.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use insta::assert_snapshot; -use std::fs::write; +use std::fs; use std::path::PathBuf; use sui_graphql_rpc::server::builder::export_schema; @@ -11,9 +11,8 @@ fn test_schema_sdl_export() { let sdl = export_schema(); // update the current schema file - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.extend(["schema", "current_progress_schema.graphql"]); - write(path, &sdl).unwrap(); + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("schema.graphql"); + fs::write(path, &sdl).unwrap(); assert_snapshot!(sdl); } diff --git a/docs/site/docusaurus.config.js b/docs/site/docusaurus.config.js index e784d38744f15..5760ff48882b1 100644 --- a/docs/site/docusaurus.config.js +++ b/docs/site/docusaurus.config.js @@ -62,7 +62,7 @@ const config = { "@graphql-markdown/docusaurus", { schema: - "../../crates/sui-graphql-rpc/schema/current_progress_schema.graphql", + "../../crates/sui-graphql-rpc/schema.graphql", rootPath: "../content", // docs will be generated under rootPath/baseURL baseURL: "references/sui-api/sui-graphql/reference", loaders: { diff --git a/sdk/typescript/scripts/update-graphql-schemas.ts b/sdk/typescript/scripts/update-graphql-schemas.ts index a129e7dfe1024..4a3843ede5cbc 100644 --- a/sdk/typescript/scripts/update-graphql-schemas.ts +++ b/sdk/typescript/scripts/update-graphql-schemas.ts @@ -29,7 +29,7 @@ const result = execSync(`git branch --remote --list "origin/releases/sui-graphql minor, patch, branch, - schema: `https://raw.githubusercontent.com/MystenLabs/sui/${branch}/crates/sui-graphql-rpc/schema/current_progress_schema.graphql`, + schema: `https://raw.githubusercontent.com/MystenLabs/sui/${branch}/crates/sui-graphql-rpc/schema.graphql`, } : null; }) From e93152338ac0a1b9f4f4f80a792190ba9972a0fe Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:52:52 +0100 Subject: [PATCH 8/9] [GraphQL] `generate-config` sub-command (#18336) ## Description Add a command for generating a config TOML file for the GraphQL service with all its parameters set to their default values. (We used to have a similar command for the YAML file which we weren't using, but we still use the TOML file). ## Test plan ``` cargo run --bin sui-graphql-rpc -- generate-config /tmp/config.toml ``` ## Stack - #17543 - #17692 - #17693 - #17696 - #18287 - #18288 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [x] GraphQL: New sub-command for `sui-graphql-rpc`, `generate-config` for creating a TOML config with all default values set. - [ ] CLI: - [ ] Rust SDK: --- crates/sui-graphql-rpc/src/commands.rs | 7 +++++++ crates/sui-graphql-rpc/src/main.rs | 13 +++++++++++++ .../sui-graphql-rpc/src/server/graphiql_server.rs | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/sui-graphql-rpc/src/commands.rs b/crates/sui-graphql-rpc/src/commands.rs index bda0c2f561fab..e5166def39f50 100644 --- a/crates/sui-graphql-rpc/src/commands.rs +++ b/crates/sui-graphql-rpc/src/commands.rs @@ -13,6 +13,13 @@ use std::path::PathBuf; version )] pub enum Command { + /// Output a TOML config (suitable for passing into the --config parameter of the start-server + /// command) with all values set to their defaults. + GenerateConfig { + /// Optional path to a file to output to. Prints to stdout if none is provided. + output: Option, + }, + StartServer { /// The title to display at the top of the page #[clap(short, long)] diff --git a/crates/sui-graphql-rpc/src/main.rs b/crates/sui-graphql-rpc/src/main.rs index 349ef0f0f74a4..cedc55b39e72a 100644 --- a/crates/sui-graphql-rpc/src/main.rs +++ b/crates/sui-graphql-rpc/src/main.rs @@ -37,6 +37,19 @@ static VERSION: Version = Version { async fn main() { let cmd: Command = Command::parse(); match cmd { + Command::GenerateConfig { output } => { + let config = ServiceConfig::default(); + let toml = toml::to_string_pretty(&config).expect("Failed to serialize configuration"); + + if let Some(path) = output { + fs::write(&path, toml).unwrap_or_else(|e| { + panic!("Failed to write configuration to {}: {e}", path.display()) + }); + } else { + println!("{}", toml); + } + } + Command::StartServer { ide_title, db_url, diff --git a/crates/sui-graphql-rpc/src/server/graphiql_server.rs b/crates/sui-graphql-rpc/src/server/graphiql_server.rs index d5c2f329ecf7f..7a809c01d85be 100644 --- a/crates/sui-graphql-rpc/src/server/graphiql_server.rs +++ b/crates/sui-graphql-rpc/src/server/graphiql_server.rs @@ -31,7 +31,7 @@ pub async fn start_graphiql_server( version: &Version, cancellation_token: CancellationToken, ) -> Result<(), Error> { - info!("Starting server with config: {:?}", server_config); + info!("Starting server with config: {:#?}", server_config); info!("Server version: {}", version); start_graphiql_server_impl( ServerBuilder::from_config(server_config, version, cancellation_token).await?, From 34b6788241dce9209a377d0a75ae2b116c9ce80e Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 19 Aug 2024 11:53:22 +0100 Subject: [PATCH 9/9] [sui-tool] dump-packages uses GraphQL (#18337) ## Description Replace the original implementation of the dump-packages command (which requires access to an indexer database) with an implementation that reads from a GraphQL service. The former is not readily accessible, but the latter should be. The new tool is also able to run incrementally: Fetching only packages created before a certain checkpoint, or pick up where a previous invocation took off to fetch new packages that were introduced since. ## Test plan Ran a test invocation, on our experimental read replica. With a max page size of 200, I was able to fetch 17000 packages (all the packages at the time the read replica was created) in 3 minutes. ## Stack - #17543 - #17692 - #17693 - #17696 - #18287 - #18288 - #18336 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] Indexer: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- Cargo.lock | 182 +++++++++++++++++-- Cargo.toml | 4 + crates/sui-package-dump/Cargo.toml | 22 +++ crates/sui-package-dump/build.rs | 10 + crates/sui-package-dump/src/client.rs | 39 ++++ crates/sui-package-dump/src/lib.rs | 251 ++++++++++++++++++++++++++ crates/sui-package-dump/src/query.rs | 105 +++++++++++ crates/sui-tool/Cargo.toml | 3 +- crates/sui-tool/src/commands.rs | 26 ++- crates/sui-tool/src/lib.rs | 1 - crates/sui-tool/src/pkg_dump.rs | 122 ------------- 11 files changed, 618 insertions(+), 147 deletions(-) create mode 100644 crates/sui-package-dump/Cargo.toml create mode 100644 crates/sui-package-dump/build.rs create mode 100644 crates/sui-package-dump/src/client.rs create mode 100644 crates/sui-package-dump/src/lib.rs create mode 100644 crates/sui-package-dump/src/query.rs delete mode 100644 crates/sui-tool/src/pkg_dump.rs diff --git a/Cargo.lock b/Cargo.lock index 4449deae1a1d5..37af0287befea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2947,6 +2947,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "counter" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d458e66999348f56fd3ffcfbb7f7951542075ca8359687c703de6500c1ddccd" +dependencies = [ + "num-traits", +] + [[package]] name = "cpp_demangle" version = "0.4.0" @@ -3288,6 +3297,62 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "cynic" +version = "3.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c02b53607e3f21c374f024c2cfc2154e554905bba478e8e09409f10ce3726" +dependencies = [ + "cynic-proc-macros", + "ref-cast", + "reqwest 0.12.5", + "serde", + "serde_json", + "static_assertions", + "thiserror", +] + +[[package]] +name = "cynic-codegen" +version = "3.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c0ec86f960a00ce087e96ff6f073f6ff28b6876d69ce8caa06c03fb4143981c" +dependencies = [ + "counter", + "cynic-parser", + "darling 0.20.3", + "once_cell", + "ouroboros 0.18.4", + "proc-macro2 1.0.78", + "quote 1.0.35", + "strsim 0.10.0", + "syn 2.0.48", + "thiserror", +] + +[[package]] +name = "cynic-parser" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "718f6cd8c54ae5249fd42b0c86639df0100b8a86eea2e5f1b915cde2e1481453" +dependencies = [ + "indexmap 2.2.6", + "lalrpop-util", + "logos", +] + +[[package]] +name = "cynic-proc-macros" +version = "3.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a69ecdf4aa110fed1c0c8de290bc8ccb2835388733cf2f418f0abdf6ff3899" +dependencies = [ + "cynic-codegen", + "darling 0.20.3", + "quote 1.0.35", + "syn 2.0.48", +] + [[package]] name = "darling" version = "0.13.4" @@ -4496,7 +4561,7 @@ dependencies = [ "tokio", "tracing", "walkdir", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -6630,6 +6695,39 @@ dependencies = [ "serde", ] +[[package]] +name = "logos" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1ceb190eb9bdeecdd8f1ad6a71d6d632a50905948771718741b5461fb01e13" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90be66cb7bd40cb5cc2e9cfaf2d1133b04a3d93b72344267715010a466e0915a" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2 1.0.78", + "quote 1.0.35", + "regex-syntax 0.8.2", + "syn 2.0.48", +] + +[[package]] +name = "logos-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45154231e8e96586b39494029e58f12f8ffcb5ecf80333a603a13aa205ea8cbd" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lru" version = "0.7.8" @@ -7308,7 +7406,7 @@ dependencies = [ "move-ir-to-bytecode-syntax", "move-ir-types", "move-symbol-pool", - "ouroboros", + "ouroboros 0.17.2", ] [[package]] @@ -8956,7 +9054,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954" dependencies = [ "aliasable", - "ouroboros_macro", + "ouroboros_macro 0.17.2", + "static_assertions", +] + +[[package]] +name = "ouroboros" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "944fa20996a25aded6b4795c6d63f10014a7a83f8be9828a11860b08c5fc4a67" +dependencies = [ + "aliasable", + "ouroboros_macro 0.18.4", "static_assertions", ] @@ -8973,6 +9082,20 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "ouroboros_macro" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39b0deead1528fd0e5947a8546a9642a9777c25f6e1e26f34c97b204bbb465bd" +dependencies = [ + "heck 0.4.1", + "itertools 0.12.0", + "proc-macro2 1.0.78", + "proc-macro2-diagnostics", + "quote 1.0.35", + "syn 2.0.48", +] + [[package]] name = "output_vt100" version = "0.1.3" @@ -9698,7 +9821,7 @@ dependencies = [ "ctor", "diff", "output_vt100", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -9818,6 +9941,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2 1.0.78", + "quote 1.0.35", + "syn 2.0.48", + "version_check", + "yansi 1.0.1", +] + [[package]] name = "prometheus" version = "0.13.3" @@ -10378,22 +10514,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.14" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c78fb8c9293bcd48ef6fce7b4ca950ceaf21210de6e105a883ee280c0f7b9ed" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.14" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9c0c92af03644e4806106281fe2e068ac5bc0ae74a707266d06ea27bccee5f" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2 1.0.78", "quote 1.0.35", - "syn 1.0.107", + "syn 2.0.48", ] [[package]] @@ -14036,6 +14172,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "sui-package-dump" +version = "1.32.0" +dependencies = [ + "anyhow", + "bcs", + "cynic", + "cynic-codegen", + "fastcrypto", + "move-core-types", + "reqwest 0.12.5", + "serde", + "serde_json", + "sui-types", + "tracing", +] + [[package]] name = "sui-package-management" version = "1.32.0" @@ -14733,7 +14886,6 @@ dependencies = [ "clap", "colored", "comfy-table", - "diesel", "eyre", "fastcrypto", "futures", @@ -14754,8 +14906,8 @@ dependencies = [ "sui-archival", "sui-config", "sui-core", - "sui-indexer", "sui-network", + "sui-package-dump", "sui-protocol-config", "sui-replay", "sui-sdk 1.32.0", @@ -16328,7 +16480,7 @@ dependencies = [ "itertools 0.10.5", "msim", "once_cell", - "ouroboros", + "ouroboros 0.17.2", "proc-macro2 1.0.78", "prometheus", "quote 1.0.35", @@ -17275,6 +17427,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index 78b95b57b41e3..3b3d1836ba1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -130,6 +130,7 @@ members = [ "crates/sui-open-rpc", "crates/sui-open-rpc-macros", "crates/sui-oracle", + "crates/sui-package-dump", "crates/sui-package-management", "crates/sui-package-resolver", "crates/sui-proc-macros", @@ -305,6 +306,8 @@ criterion = { version = "0.5.0", features = [ ] } crossterm = "0.25.0" csv = "1.2.1" +cynic = { version = "3.7.3", features = ["http-reqwest"] } +cynic-codegen = "= 3.7.3" dashmap = "5.5.3" # datatest-stable = "0.1.2" datatest-stable = { git = "https://github.com/nextest-rs/datatest-stable.git", rev = "72db7f6d1bbe36a5407e96b9488a581f763e106f" } @@ -628,6 +631,7 @@ sui-network = { path = "crates/sui-network" } sui-node = { path = "crates/sui-node" } sui-open-rpc = { path = "crates/sui-open-rpc" } sui-open-rpc-macros = { path = "crates/sui-open-rpc-macros" } +sui-package-dump = { path = "crates/sui-package-dump" } sui-package-management = { path = "crates/sui-package-management" } sui-package-resolver = { path = "crates/sui-package-resolver" } sui-proc-macros = { path = "crates/sui-proc-macros" } diff --git a/crates/sui-package-dump/Cargo.toml b/crates/sui-package-dump/Cargo.toml new file mode 100644 index 0000000000000..92632519a9877 --- /dev/null +++ b/crates/sui-package-dump/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sui-package-dump" +version.workspace = true +authors = ["Mysten Labs Result { + Ok(Self { + inner: reqwest::Client::builder() + .user_agent(concat!("sui-package-dump/", env!("CARGO_PKG_VERSION"))) + .build() + .context("Failed to create GraphQL client")?, + url: url.into_url().context("Invalid RPC URL")?, + }) + } + + pub(crate) async fn query(&self, query: Operation) -> Result + where + V: Serialize, + Q: DeserializeOwned + QueryBuilder + 'static, + { + self.inner + .post(self.url.clone()) + .run_graphql(query) + .await + .context("Failed to send GraphQL query")? + .data + .ok_or_else(|| anyhow!("Empty response to query")) + } +} diff --git a/crates/sui-package-dump/src/lib.rs b/crates/sui-package-dump/src/lib.rs new file mode 100644 index 0000000000000..8438a8bec7f88 --- /dev/null +++ b/crates/sui-package-dump/src/lib.rs @@ -0,0 +1,251 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, ensure, Context, Result}; +use client::Client; +use fastcrypto::encoding::{Base64, Encoding}; +use query::{limits, packages, SuiAddress, UInt53}; +use sui_types::object::Object; +use tracing::info; + +mod client; +mod query; + +/// Ensure all packages created before `before_checkpoint` are written to the `output_dir`ectory, +/// from the GraphQL service at `rpc_url`. +/// +/// `output_dir` can be a path to a non-existent directory, in which case it will be created, or an +/// existing empty directory (in which case it will be filled), or an existing directory that has +/// been written to in the past (in which case this invocation will pick back up from where the +/// previous invocation left off). +pub async fn dump( + rpc_url: String, + output_dir: PathBuf, + before_checkpoint: Option, +) -> Result<()> { + ensure_output_directory(&output_dir)?; + + let client = Client::new(rpc_url)?; + let after_checkpoint = read_last_checkpoint(&output_dir)?; + let limit = max_page_size(&client).await?; + let (last_checkpoint, packages) = + fetch_packages(&client, limit, after_checkpoint, before_checkpoint).await?; + + for package in &packages { + let SuiAddress(address) = &package.address; + dump_package(&output_dir, package) + .with_context(|| format!("Failed to dump package {address}"))?; + } + + if let Some(last_checkpoint) = last_checkpoint { + write_last_checkpoint(&output_dir, last_checkpoint)?; + } + + Ok(()) +} + +/// Ensure the output directory exists, either because it already exists as a writable directory, or +/// by creating a new directory. +fn ensure_output_directory(path: impl Into) -> Result<()> { + let path: PathBuf = path.into(); + if !path.exists() { + fs::create_dir_all(&path).context("Making output directory")?; + return Ok(()); + } + + ensure!( + path.is_dir(), + "Output path is not a directory: {}", + path.display() + ); + + let metadata = fs::metadata(&path).context("Getting metadata for output path")?; + + ensure!( + !metadata.permissions().readonly(), + "Output directory is not writable: {}", + path.display() + ); + + Ok(()) +} + +/// Load the last checkpoint that was loaded by a previous run of the tool, if there is a previous +/// run. +fn read_last_checkpoint(output: &Path) -> Result> { + let path = output.join("last-checkpoint"); + if !path.exists() { + return Ok(None); + } + + let content = fs::read_to_string(&path).context("Failed to read last checkpoint")?; + let checkpoint: u64 = + serde_json::from_str(&content).context("Failed to parse last checkpoint")?; + + info!("Resuming download after checkpoint {checkpoint}"); + + Ok(Some(checkpoint)) +} + +/// Write the max checkpoint that we have seen a package from back to the output directory. +fn write_last_checkpoint(output: &Path, checkpoint: u64) -> Result<()> { + let path = output.join("last-checkpoint"); + let content = + serde_json::to_string(&checkpoint).context("Failed to serialize last checkpoint")?; + + fs::write(path, content).context("Failed to write last checkpoint")?; + Ok(()) +} + +/// Read the max page size supported by the GraphQL service. +async fn max_page_size(client: &Client) -> Result { + Ok(client + .query(limits::build()) + .await + .context("Failed to fetch max page size")? + .service_config + .max_page_size) +} + +/// Read all the packages between `after_checkpoint` and `before_checkpoint`, in batches of +/// `page_size` from the `client` connected to a GraphQL service. +/// +/// If `after_checkpoint` is not provided, packages will be read from genesis. If +/// `before_checkpoint` is not provided, packages will be read until the latest checkpoint. +/// +/// Returns the latest checkpoint that was read from in this fetch, and a list of all the packages +/// that were read. +async fn fetch_packages( + client: &Client, + page_size: i32, + after_checkpoint: Option, + before_checkpoint: Option, +) -> Result<(Option, Vec)> { + let packages::Query { + checkpoint: checkpoint_viewed_at, + packages: + packages::MovePackageConnection { + mut page_info, + mut nodes, + }, + } = client + .query(packages::build( + page_size, + None, + after_checkpoint.map(UInt53), + before_checkpoint.map(UInt53), + )) + .await + .with_context(|| "Failed to fetch page 1 of packages.")?; + + for i in 2.. { + if !page_info.has_next_page { + break; + } + + let packages = client + .query(packages::build( + page_size, + page_info.end_cursor, + after_checkpoint.map(UInt53), + before_checkpoint.map(UInt53), + )) + .await + .with_context(|| format!("Failed to fetch page {i} of packages."))? + .packages; + + nodes.extend(packages.nodes); + page_info = packages.page_info; + + info!( + "Fetched page {i} ({} package{} so far).", + nodes.len(), + if nodes.len() == 1 { "" } else { "s" }, + ); + } + + use packages::Checkpoint as C; + let last_checkpoint = match (checkpoint_viewed_at, before_checkpoint) { + ( + Some(C { + sequence_number: UInt53(v), + }), + Some(b), + ) if b > 0 => Some(v.min(b - 1)), + ( + Some(C { + sequence_number: UInt53(c), + }), + _, + ) + | (_, Some(c)) => Some(c), + _ => None, + }; + + Ok((last_checkpoint, nodes)) +} + +/// Write out `pkg` to the `output_dir`ectory, using the package's address and name as the directory +/// name. The following files are written for each directory: +/// +/// - `object.bcs` -- the BCS serialized form of the `Object` type containing the package. +/// +/// - `linkage.json` -- a JSON serialization of the package's linkage table, mapping dependency +/// original IDs to the version of the dependency being depended on and the ID of the object +/// on-chain that contains that version. +/// +/// - `origins.json` -- a JSON serialize of the type origin table, mapping type names contained in +/// this package to the version of the package that first introduced that type. +/// +/// - `*.mv` -- a BCS serialization of each compiled module in the package. +fn dump_package(output_dir: &Path, pkg: &packages::MovePackage) -> Result<()> { + let Some(query::Base64(bcs)) = &pkg.bcs else { + bail!("Missing BCS"); + }; + + let bytes = Base64::decode(bcs).context("Failed to decode BCS")?; + + let object = bcs::from_bytes::(&bytes).context("Failed to deserialize")?; + let id = object.id(); + let Some(package) = object.data.try_as_package() else { + bail!("Not a package"); + }; + + let origins: BTreeMap<_, _> = package + .type_origin_table() + .iter() + .map(|o| { + ( + format!("{}::{}", o.module_name, o.datatype_name), + o.package.to_string(), + ) + }) + .collect(); + + let package_dir = output_dir.join(format!("{}.{}", id, package.version().value())); + fs::create_dir(&package_dir).context("Failed to make output directory")?; + + let linkage_json = serde_json::to_string_pretty(package.linkage_table()) + .context("Failed to serialize linkage")?; + let origins_json = + serde_json::to_string_pretty(&origins).context("Failed to serialize type origins")?; + + fs::write(package_dir.join("object.bcs"), bytes).context("Failed to write object BCS")?; + fs::write(package_dir.join("linkage.json"), linkage_json).context("Failed to write linkage")?; + fs::write(package_dir.join("origins.json"), origins_json) + .context("Failed to write type origins")?; + + for (module_name, module_bytes) in package.serialized_module_map() { + let module_path = package_dir.join(format!("{module_name}.mv")); + fs::write(module_path, module_bytes) + .with_context(|| format!("Failed to write module: {module_name}"))? + } + + Ok(()) +} diff --git a/crates/sui-package-dump/src/query.rs b/crates/sui-package-dump/src/query.rs new file mode 100644 index 0000000000000..a0d2c0ae391d5 --- /dev/null +++ b/crates/sui-package-dump/src/query.rs @@ -0,0 +1,105 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use cynic::Operation; +use cynic::QueryBuilder; + +#[cynic::schema("sui")] +mod schema {} + +#[derive(cynic::Scalar, Debug)] +pub(crate) struct SuiAddress(pub String); + +#[derive(cynic::Scalar, Debug)] +pub(crate) struct Base64(pub String); + +#[derive(cynic::Scalar, Debug)] +pub(crate) struct UInt53(pub u64); + +/// Query types related to GraphQL service limits. +pub(crate) mod limits { + use super::*; + + pub(crate) fn build() -> Operation { + Query::build(()) + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct Query { + pub(crate) service_config: ServiceConfig, + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct ServiceConfig { + pub(crate) max_page_size: i32, + } +} + +/// Query types related to fetching packages. +pub(crate) mod packages { + use super::*; + + pub(crate) fn build( + first: i32, + after: Option, + after_checkpoint: Option, + before_checkpoint: Option, + ) -> Operation { + Query::build(Vars { + first, + after, + filter: Some(MovePackageCheckpointFilter { + after_checkpoint, + before_checkpoint, + }), + }) + } + + #[derive(cynic::QueryVariables, Debug)] + pub(crate) struct Vars { + pub(crate) first: i32, + pub(crate) after: Option, + pub(crate) filter: Option, + } + + #[derive(cynic::InputObject, Debug)] + pub(crate) struct MovePackageCheckpointFilter { + pub(crate) after_checkpoint: Option, + pub(crate) before_checkpoint: Option, + } + + #[derive(cynic::QueryFragment, Debug)] + #[cynic(variables = "Vars")] + pub(crate) struct Query { + pub(crate) checkpoint: Option, + #[arguments( + first: $first, + after: $after, + filter: $filter, + )] + pub(crate) packages: MovePackageConnection, + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct Checkpoint { + pub(crate) sequence_number: UInt53, + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct MovePackageConnection { + pub(crate) page_info: PageInfo, + pub(crate) nodes: Vec, + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct PageInfo { + pub(crate) has_next_page: bool, + pub(crate) end_cursor: Option, + } + + #[derive(cynic::QueryFragment, Debug)] + pub(crate) struct MovePackage { + pub(crate) address: SuiAddress, + pub(crate) bcs: Option, + } +} diff --git a/crates/sui-tool/Cargo.toml b/crates/sui-tool/Cargo.toml index 2db2c5395e241..a19bfaa194b94 100644 --- a/crates/sui-tool/Cargo.toml +++ b/crates/sui-tool/Cargo.toml @@ -13,7 +13,6 @@ bcs.workspace = true clap = { version = "4.1.4", features = ["derive"] } colored.workspace = true comfy-table.workspace = true -diesel.workspace = true eyre.workspace = true futures.workspace = true hex.workspace = true @@ -41,7 +40,6 @@ narwhal-storage.workspace = true narwhal-types.workspace = true sui-config.workspace = true sui-core.workspace = true -sui-indexer.workspace = true sui-network.workspace = true sui-snapshot.workspace = true sui-protocol-config.workspace = true @@ -50,4 +48,5 @@ sui-sdk.workspace = true sui-storage.workspace = true sui-types.workspace = true sui-archival.workspace = true +sui-package-dump.workspace = true bin-version.workspace = true diff --git a/crates/sui-tool/src/commands.rs b/crates/sui-tool/src/commands.rs index 95b39183c96f6..3871143671e3e 100644 --- a/crates/sui-tool/src/commands.rs +++ b/crates/sui-tool/src/commands.rs @@ -5,7 +5,7 @@ use crate::{ check_completed_snapshot, db_tool::{execute_db_tool_command, print_db_all_tables, DbToolCommand}, download_db_snapshot, download_formal_snapshot, dump_checkpoints_from_archive, - get_latest_available_epoch, get_object, get_transaction_block, make_clients, pkg_dump, + get_latest_available_epoch, get_object, get_transaction_block, make_clients, restore_from_db_checkpoint, verify_archive, verify_archive_by_checksum, ConciseObjectOutput, GroupedObjectOutput, SnapshotVerifyMode, VerboseObjectOutput, }; @@ -177,21 +177,26 @@ pub enum ToolCommand { max_content_length: usize, }, - /// Download all packages to the local filesystem from an indexer database. Each package gets - /// its own sub-directory, named for its ID on-chain, containing two metadata files - /// (linkage.json and origins.json) as well as a file for every module it contains. Each module - /// file is named for its module name, with a .mv suffix, and contains Move bytecode (suitable - /// for passing into a disassembler). + /// Download all packages to the local filesystem from a GraphQL service. Each package gets its + /// own sub-directory, named for its ID on-chain and version containing two metadata files + /// (linkage.json and origins.json), a file containing the overall object and a file for every + /// module it contains. Each module file is named for its module name, with a .mv suffix, and + /// contains Move bytecode (suitable for passing into a disassembler). #[command(name = "dump-packages")] DumpPackages { - /// Connection information for the Indexer's Postgres DB. + /// Connection information for a GraphQL service. #[clap(long, short)] - db_url: String, + rpc_url: String, /// Path to a non-existent directory that can be created and filled with package information. #[clap(long, short)] output_dir: PathBuf, + /// Only fetch packages that were created before this checkpoint (given by its sequence + /// number). + #[clap(long)] + before_checkpoint: Option, + /// If false (default), log level will be overridden to "off", and output will be reduced to /// necessary status information. #[clap(short, long = "verbose")] @@ -633,8 +638,9 @@ impl ToolCommand { } } ToolCommand::DumpPackages { - db_url, + rpc_url, output_dir, + before_checkpoint, verbose, } => { if !verbose { @@ -643,7 +649,7 @@ impl ToolCommand { .expect("Failed to update log level"); } - pkg_dump::dump(db_url, output_dir).await?; + sui_package_dump::dump(rpc_url, output_dir, before_checkpoint).await?; } ToolCommand::DumpValidators { genesis, concise } => { let genesis = Genesis::load(genesis).unwrap(); diff --git a/crates/sui-tool/src/lib.rs b/crates/sui-tool/src/lib.rs index 574eb87a64f1b..b4290dc48b510 100644 --- a/crates/sui-tool/src/lib.rs +++ b/crates/sui-tool/src/lib.rs @@ -71,7 +71,6 @@ use typed_store::rocks::MetricConf; pub mod commands; pub mod db_tool; -pub mod pkg_dump; #[derive( Clone, Serialize, Deserialize, Debug, PartialEq, Copy, PartialOrd, Ord, Eq, ValueEnum, Default, diff --git a/crates/sui-tool/src/pkg_dump.rs b/crates/sui-tool/src/pkg_dump.rs deleted file mode 100644 index bd78cbf2b4b87..0000000000000 --- a/crates/sui-tool/src/pkg_dump.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::BTreeMap, - fs, - path::{Path, PathBuf}, - time::Duration, -}; - -use anyhow::{anyhow, ensure, Context, Result}; -use diesel::{ - r2d2::{ConnectionManager, Pool}, - PgConnection, RunQueryDsl, -}; -use sui_indexer::{models::packages::StoredPackage, schema::packages}; -use sui_types::{base_types::SuiAddress, move_package::MovePackage}; -use tracing::info; - -type PgPool = Pool>; - -pub(crate) async fn dump(db_url: String, output_dir: PathBuf) -> Result<()> { - ensure_output_directory(&output_dir)?; - - let conn = ConnectionManager::::new(db_url); - let pool = Pool::builder() - .max_size(1) - .connection_timeout(Duration::from_secs(30)) - .build(conn) - .context("Failed to create connection pool.")?; - - info!("Querying Indexer..."); - let pkgs = query_packages(&pool)?; - let total = pkgs.len(); - - let mut progress = 0; - for (i, pkg) in pkgs.into_iter().enumerate() { - let pct = (100 * i) / total; - if pct % 5 == 0 && pct > progress { - info!("Dumping packages ({total}): {pct: >3}%"); - progress = pct; - } - - let id = SuiAddress::from_bytes(&pkg.package_id).context("Parsing package ID")?; - dump_package(&output_dir, id, &pkg.move_package) - .with_context(|| format!("Dumping package: {id}"))?; - } - - info!("Dumping packages ({total}): 100%, Done."); - Ok(()) -} - -/// Ensure the output directory exists, either because it already exists as an empty, writable -/// directory, or by creating a new directory. -fn ensure_output_directory(path: impl Into) -> Result<()> { - let path: PathBuf = path.into(); - if path.exists() { - ensure!( - path.is_dir(), - "Output path is not a directory: {}", - path.display() - ); - ensure!( - path.read_dir().is_ok_and(|mut d| d.next().is_none()), - "Output directory is not empty: {}", - path.display(), - ); - - let metadata = fs::metadata(&path).context("Getting metadata for output path")?; - - ensure!( - !metadata.permissions().readonly(), - "Output directory is not writable: {}", - path.display() - ) - } else { - fs::create_dir_all(&path).context("Making output directory")?; - } - - Ok(()) -} - -fn query_packages(pool: &PgPool) -> Result> { - let mut conn = pool - .get() - .map_err(|e| anyhow!("Failed to get connection: {e}"))?; - Ok(packages::dsl::packages.load::(&mut conn)?) -} - -fn dump_package(output_dir: &Path, id: SuiAddress, pkg: &[u8]) -> Result<()> { - let package = bcs::from_bytes::(pkg).context("Deserializing")?; - let origins: BTreeMap<_, _> = package - .type_origin_table() - .iter() - .map(|o| { - ( - format!("{}::{}", o.module_name, o.datatype_name), - o.package.to_string(), - ) - }) - .collect(); - - let package_dir = output_dir.join(format!("{}.{}", id, package.version().value())); - fs::create_dir(&package_dir).context("Making output directory")?; - - let linkage_json = - serde_json::to_string_pretty(package.linkage_table()).context("Serializing linkage")?; - let origins_json = - serde_json::to_string_pretty(&origins).context("Serializing type origins")?; - - fs::write(package_dir.join("package.bcs"), pkg).context("Writing package BCS")?; - fs::write(package_dir.join("linkage.json"), linkage_json).context("Writing linkage")?; - fs::write(package_dir.join("origins.json"), origins_json).context("Writing type origins")?; - - for (module_name, module_bytes) in package.serialized_module_map() { - let module_path = package_dir.join(format!("{module_name}.mv")); - fs::write(module_path, module_bytes) - .with_context(|| format!("Writing module: {module_name}"))? - } - - Ok(()) -}