Skip to content

Commit

Permalink
[sui-tool] dump-packages uses GraphQL
Browse files Browse the repository at this point in the history
## Description

Replace the original implementation of the dump-packages
command (which requires access to an indexer database) with an
implementation that reads from a GraphQL service. The former is not
readily accessible, but the latter should be.

The new tool is also able to run incrementally: Fetching only packages
created before a certain checkpoint, or pick up where a previous
invocation took off to fetch new packages that were introduced since.

## Test plan

Ran a test invocation, on our experimental read replica. With a max
page size of 200, I was able to fetch 17000 packages (all the packages
at the time the read replica was created) in 3 minutes.
  • Loading branch information
amnn committed Aug 19, 2024
1 parent 8ac5874 commit 657e435
Show file tree
Hide file tree
Showing 11 changed files with 618 additions and 147 deletions.
182 changes: 170 additions & 12 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ members = [
"crates/sui-open-rpc",
"crates/sui-open-rpc-macros",
"crates/sui-oracle",
"crates/sui-package-dump",
"crates/sui-package-management",
"crates/sui-package-resolver",
"crates/sui-proc-macros",
Expand Down Expand Up @@ -305,6 +306,8 @@ criterion = { version = "0.5.0", features = [
] }
crossterm = "0.25.0"
csv = "1.2.1"
cynic = { version = "3.7.3", features = ["http-reqwest"] }
cynic-codegen = "= 3.7.3"
dashmap = "5.5.3"
# datatest-stable = "0.1.2"
datatest-stable = { git = "https://github.com/nextest-rs/datatest-stable.git", rev = "72db7f6d1bbe36a5407e96b9488a581f763e106f" }
Expand Down Expand Up @@ -628,6 +631,7 @@ sui-network = { path = "crates/sui-network" }
sui-node = { path = "crates/sui-node" }
sui-open-rpc = { path = "crates/sui-open-rpc" }
sui-open-rpc-macros = { path = "crates/sui-open-rpc-macros" }
sui-package-dump = { path = "crates/sui-package-dump" }
sui-package-management = { path = "crates/sui-package-management" }
sui-package-resolver = { path = "crates/sui-package-resolver" }
sui-proc-macros = { path = "crates/sui-proc-macros" }
Expand Down
22 changes: 22 additions & 0 deletions crates/sui-package-dump/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
[package]
name = "sui-package-dump"
version.workspace = true
authors = ["Mysten Labs <[email protected]"]
license = "Apache-2.0"
publish = false
edition = "2021"

[dependencies]
anyhow.workspace = true
bcs.workspace = true
cynic.workspace = true
fastcrypto.workspace = true
move-core-types.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
sui-types.workspace = true
tracing.workspace = true

[build-dependencies]
cynic-codegen.workspace = true
10 changes: 10 additions & 0 deletions crates/sui-package-dump/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Copyright (c) Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

fn main() {
cynic_codegen::register_schema("sui")
.from_sdl_file("../sui-graphql-rpc/schema.graphql")
.unwrap()
.as_default()
.unwrap();
}
39 changes: 39 additions & 0 deletions crates/sui-package-dump/src/client.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright (c) Mysten Labs, Inc.
// SPDX-License-Identifier: Apache-2.0

use anyhow::{anyhow, Context, Result};
use cynic::{http::ReqwestExt, Operation, QueryBuilder};
use reqwest::IntoUrl;
use serde::{de::DeserializeOwned, Serialize};

pub(crate) struct Client {
inner: reqwest::Client,
url: reqwest::Url,
}

impl Client {
/// Create a new GraphQL client, talking to a Sui GraphQL service at `url`.
pub(crate) fn new(url: impl IntoUrl) -> Result<Self> {
Ok(Self {
inner: reqwest::Client::builder()
.user_agent(concat!("sui-package-dump/", env!("CARGO_PKG_VERSION")))
.build()
.context("Failed to create GraphQL client")?,
url: url.into_url().context("Invalid RPC URL")?,
})
}

pub(crate) async fn query<Q, V>(&self, query: Operation<Q, V>) -> Result<Q>
where
V: Serialize,
Q: DeserializeOwned + QueryBuilder<V> + 'static,
{
self.inner
.post(self.url.clone())
.run_graphql(query)
.await
.context("Failed to send GraphQL query")?
.data
.ok_or_else(|| anyhow!("Empty response to query"))
}
}
Loading

0 comments on commit 657e435

Please sign in to comment.