From 404ff36d7cd5350fa2aa1c9f6c9921c720c443b3 Mon Sep 17 00:00:00 2001 From: Phuong Nguyen Date: Fri, 13 Jan 2023 18:15:18 -0800 Subject: [PATCH] Added Builder for Networks --- README.md | 4 ++ examples/manually-spawned-sandbox.md | 28 +++++++++ workspaces/src/error/impls.rs | 20 +++--- workspaces/src/network/builder.rs | 92 ++++++++++++++++++++++++++++ workspaces/src/network/mod.rs | 1 + workspaces/src/network/sandbox.rs | 60 +++++++++++------- workspaces/src/network/server.rs | 69 ++++++++++++++++----- workspaces/src/worker/impls.rs | 9 ++- workspaces/src/worker/mod.rs | 5 +- workspaces/tests/deploy.rs | 39 +++++++++++- 10 files changed, 272 insertions(+), 55 deletions(-) create mode 100644 examples/manually-spawned-sandbox.md create mode 100644 workspaces/src/network/builder.rs diff --git a/README.md b/README.md index dd8a3622..3f5fa3ea 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,10 @@ async fn test_contract() -> anyhow::Result<()> { For a full example, take a look at [workspaces/tests/deploy_project.rs](https://github.com/near/workspaces-rs/blob/main/workspaces/tests/deploy_project.rs). +### Other Features + +Other features can be directly found in the `examples/` folder, with some documentation outlining how they can be used. + ### Environment Variables These environment variables will be useful if there was ever a snag hit: diff --git a/examples/manually-spawned-sandbox.md b/examples/manually-spawned-sandbox.md new file mode 100644 index 00000000..791873d6 --- /dev/null +++ b/examples/manually-spawned-sandbox.md @@ -0,0 +1,28 @@ +# Utilizing custom sandbox node + +This example will show us how to spin up a sandbox node of our own choosing. Follow the guide in https://github.com/near/sandbox to download it. This is mainly needed if a user wants to manage their own node and/or not require each test to spin up a new node each time. + +Then initialize the chain via `init` and run it: + +```sh +near-sandbox --home ${MY_HOME_DIRECTORY} init +near-sandbox --home ${MY_HOME_DIRECTORY} run +``` + +This will launch the chain onto `localhost:3030` by default. The `${MY_HOME_DIRECTORY}` is a path of our choosing here and this will be needed when running the workspaces code later on. In the following example, we had it set to `/home/user/.near-sandbox-home`. + +In workspaces, to connect to our manually launched node, all we have to do is add a few additional parameters to `workspaces::sandbox()`: + +```rs +#[tokio::main] +fn main() { + let worker = workspaces::sandbox() + .rpc_addr("http://localhost:3030") + .home_dir("/home/user/.near-sandbox-home") + .await?; + + Ok(()) +} +``` + +Then afterwards, we can continue performing our tests as we normally would if workspaces has spawned its own sandbox process. diff --git a/workspaces/src/error/impls.rs b/workspaces/src/error/impls.rs index 432c2de6..1e1cf9d4 100644 --- a/workspaces/src/error/impls.rs +++ b/workspaces/src/error/impls.rs @@ -6,14 +6,6 @@ use crate::result::ExecutionFailure; use super::{Error, ErrorKind, ErrorRepr, RpcErrorCode, SandboxErrorCode}; impl ErrorKind { - pub(crate) fn full(self, msg: T, error: E) -> Error - where - E: Into>, - T: Into>, - { - Error::full(self, msg, error) - } - pub(crate) fn custom(self, error: E) -> Error where E: Into>, @@ -144,6 +136,13 @@ impl std::error::Error for Error { } impl SandboxErrorCode { + pub(crate) fn message(self, msg: T) -> Error + where + T: Into>, + { + Error::message(ErrorKind::Sandbox(self), msg) + } + pub(crate) fn custom(self, error: E) -> Error where E: Into>, @@ -151,11 +150,12 @@ impl SandboxErrorCode { Error::custom(ErrorKind::Sandbox(self), error) } - pub(crate) fn message(self, msg: T) -> Error + pub(crate) fn full(self, msg: T, error: E) -> Error where T: Into>, + E: Into>, { - Error::message(ErrorKind::Sandbox(self), msg) + Error::full(ErrorKind::Sandbox(self), msg, error) } } diff --git a/workspaces/src/network/builder.rs b/workspaces/src/network/builder.rs new file mode 100644 index 00000000..1cc433ff --- /dev/null +++ b/workspaces/src/network/builder.rs @@ -0,0 +1,92 @@ +use std::future::{Future, IntoFuture}; +use std::marker::PhantomData; +use std::path::PathBuf; + +use crate::network::Sandbox; +use crate::{Network, Worker}; + +pub(crate) type BoxFuture<'a, T> = std::pin::Pin + Send + 'a>>; + +/// This trait provides a way to construct Networks out of a single builder. Currently +/// not planned to offer this trait outside, since the custom networks can just construct +/// themselves however they want utilizing `Worker::new` like so: +/// ```ignore +/// Worker::new(CustomNetwork { +/// ... // fields +/// }) +/// ``` +#[async_trait::async_trait] +pub(crate) trait FromNetworkBuilder: Sized { + async fn from_builder<'a>(build: NetworkBuilder<'a, Self>) -> crate::result::Result; +} + +/// Builder for Networks. Only usable with workspaces provided Networks. +// Note, this is currently the aggregated state for all network types you can have since +// I didn't want to add additional reading complexity with another trait that associates the +// Network state. +pub struct NetworkBuilder<'a, T> { + pub(crate) name: &'a str, + pub(crate) rpc_addr: Option, + pub(crate) home_dir: Option, + _network: PhantomData, +} + +impl<'a, T> IntoFuture for NetworkBuilder<'a, T> +where + T: FromNetworkBuilder + Network + Send + 'a, +{ + type Output = crate::result::Result>; + type IntoFuture = BoxFuture<'a, Self::Output>; + + fn into_future(self) -> Self::IntoFuture { + let fut = async { + let network = FromNetworkBuilder::from_builder(self).await?; + Ok(Worker::new(network)) + }; + Box::pin(fut) + } +} + +impl<'a, T> NetworkBuilder<'a, T> { + pub(crate) fn new(name: &'a str) -> Self { + Self { + name, + rpc_addr: None, + home_dir: None, + _network: PhantomData, + } + } + + /// Sets the RPC addr for this network. Useful for setting the Url to a different RPC + /// node than the default one provided by near.org. This enables certain features that + /// the default node doesn't provide such as getting beyond the data cap when downloading + /// state from the network. + /// Example: + /// ``` + /// async fn connect_custom() -> anyhow::Result<()> { + /// let worker = workspaces::testnet() + /// .rpc_addr("https://rpc.testnet.near.org") + /// .await?; + /// Ok(()) + /// } + /// ``` + /// + /// Note that, for sandbox, we also are required to specify `home_dir` + pub fn rpc_addr(mut self, addr: &str) -> Self { + self.rpc_addr = Some(addr.into()); + self + } +} + +// So far, only Sandbox makes use of home_dir. +impl NetworkBuilder<'_, Sandbox> { + /// Specify at which location the home_dir of the manually spawned sandbox node is at. + /// We are expected to init our own sandbox before running this builder. To learn more + /// about initalizing and starting our own sandbox, go to [near-sandbox](https://github.com/near/sandbox). + /// Also required to set the home directory where all the chain data lives. This is + /// the `my_home_folder` we passed into `near-sandbox --home {my_home_folder} init`. + pub fn home_dir(mut self, home_dir: impl AsRef) -> Self { + self.home_dir = Some(home_dir.as_ref().into()); + self + } +} diff --git a/workspaces/src/network/mod.rs b/workspaces/src/network/mod.rs index eee289eb..a9fd8fb7 100644 --- a/workspaces/src/network/mod.rs +++ b/workspaces/src/network/mod.rs @@ -10,6 +10,7 @@ mod sandbox; mod server; mod testnet; +pub(crate) mod builder; pub(crate) mod variants; pub(crate) use variants::DEV_ACCOUNT_SEED; diff --git a/workspaces/src/network/sandbox.rs b/workspaces/src/network/sandbox.rs index 0fbba364..b628fe1a 100644 --- a/workspaces/src/network/sandbox.rs +++ b/workspaces/src/network/sandbox.rs @@ -6,6 +6,7 @@ use near_jsonrpc_client::methods::sandbox_fast_forward::RpcSandboxFastForwardReq use near_jsonrpc_client::methods::sandbox_patch_state::RpcSandboxPatchStateRequest; use near_primitives::state_record::StateRecord; +use super::builder::{FromNetworkBuilder, NetworkBuilder}; use super::{AllowDevAccountCreation, NetworkClient, NetworkInfo, TopLevelAccountCreator}; use crate::error::SandboxErrorCode; use crate::network::server::SandboxServer; @@ -26,19 +27,52 @@ const DEFAULT_DEPOSIT: Balance = 100 * NEAR_BASE; /// /// [`workspaces::sandbox`]: crate::sandbox pub struct Sandbox { - server: SandboxServer, + pub(crate) server: SandboxServer, client: Client, info: Info, } impl Sandbox { pub(crate) fn root_signer(&self) -> Result { - let path = self.server.home_dir.path().join("validator_key.json"); + let path = self.server.home_dir.join("validator_key.json"); InMemorySigner::from_file(&path) } +} + +impl std::fmt::Debug for Sandbox { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("Sandbox") + .field("root_id", &self.info.root_id) + .field("rpc_url", &self.info.rpc_url) + .field("rpc_port", &self.server.rpc_port()) + .field("net_port", &self.server.net_port()) + .finish() + } +} + +#[async_trait] +impl FromNetworkBuilder for Sandbox { + async fn from_builder<'a>(build: NetworkBuilder<'a, Self>) -> Result { + // Check the conditions of the provided rpc_url and home_dir + let mut server = match (build.rpc_addr, build.home_dir) { + // Connect to a provided sandbox: + (Some(rpc_url), Some(home_dir)) => SandboxServer::connect(rpc_url, home_dir).await?, + + // Spawn a new sandbox since rpc_url and home_dir weren't specified: + (None, None) => SandboxServer::run_new().await?, + + // Missing inputted paramters for sandbox: + (Some(rpc_url), None) => { + return Err(SandboxErrorCode::InitFailure + .message(format!("Custom rpc_url={rpc_url} requires home_dir set."))); + } + (None, Some(home_dir)) => { + return Err(SandboxErrorCode::InitFailure.message(format!( + "Custom home_dir={home_dir:?} requires rpc_url set." + ))); + } + }; - pub(crate) async fn new() -> Result { - let mut server = SandboxServer::run_new().await?; let client = Client::new(&server.rpc_addr()); client.wait_for_rpc().await?; @@ -49,7 +83,7 @@ impl Sandbox { server.unlock_lockfiles()?; let info = Info { - name: "sandbox".to_string(), + name: build.name.into(), root_id: AccountId::from_str("test.near").unwrap(), keystore_path: PathBuf::from(".near-credentials/sandbox/"), rpc_url: server.rpc_addr(), @@ -61,22 +95,6 @@ impl Sandbox { info, }) } - - /// Port being used by sandbox server for RPC requests. - pub(crate) fn rpc_port(&self) -> u16 { - self.server.rpc_port - } -} - -impl std::fmt::Debug for Sandbox { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.debug_struct("Sandbox") - .field("root_id", &self.info.root_id) - .field("rpc_url", &self.info.rpc_url) - .field("rpc_port", &self.server.rpc_port) - .field("net_port", &self.server.net_port) - .finish() - } } impl AllowDevAccountCreation for Sandbox {} diff --git a/workspaces/src/network/server.rs b/workspaces/src/network/server.rs index 69f35fcb..7ad4b6b2 100644 --- a/workspaces/src/network/server.rs +++ b/workspaces/src/network/server.rs @@ -1,4 +1,5 @@ use std::fs::File; +use std::path::PathBuf; use crate::error::{ErrorKind, SandboxErrorCode}; use crate::result::Result; @@ -6,11 +7,14 @@ use crate::result::Result; use async_process::Child; use fs2::FileExt; use portpicker::pick_unused_port; +use reqwest::Url; use tempfile::TempDir; use tracing::info; use near_sandbox_utils as sandbox; +pub const DEFAULT_RPC_URL: &str = "http://localhost"; + /// Acquire an unused port and lock it for the duration until the sandbox server has /// been started. fn acquire_unused_port() -> Result<(u16, File)> { @@ -40,37 +44,59 @@ async fn init_home_dir() -> Result { } pub struct SandboxServer { - pub(crate) rpc_port: u16, - pub(crate) net_port: u16, - pub(crate) home_dir: TempDir, + pub(crate) home_dir: PathBuf, + rpc_addr: Url, + net_port: Option, rpc_port_lock: Option, net_port_lock: Option, process: Option, } impl SandboxServer { + /// Connect a sandbox server that's already been running, provided we know the rpc_addr + /// and home_dir pointing to the sandbox process. + pub(crate) async fn connect(rpc_addr: String, home_dir: PathBuf) -> Result { + let rpc_addr = Url::parse(&rpc_addr).map_err(|e| { + SandboxErrorCode::InitFailure.full(format!("Invalid rpc_url={rpc_addr}"), e) + })?; + Ok(Self { + home_dir, + rpc_addr, + net_port: None, + rpc_port_lock: None, + net_port_lock: None, + process: None, + }) + } + + /// Run a new SandboxServer, spawning the sandbox node in the process. pub(crate) async fn run_new() -> Result { // Supress logs for the sandbox binary by default: supress_sandbox_logs_if_required(); + let home_dir = init_home_dir().await?.into_path(); + // Configure `$home_dir/config.json` to our liking. Sandbox requires extra settings + // for the best user experience, and being able to offer patching large state payloads. + crate::network::config::set_sandbox_configs(&home_dir)?; + // Try running the server with the follow provided rpc_ports and net_ports let (rpc_port, rpc_port_lock) = acquire_unused_port()?; let (net_port, net_port_lock) = acquire_unused_port()?; - let home_dir = init_home_dir().await?; + let rpc_addr = format!("{}:{}", DEFAULT_RPC_URL, rpc_port); + // This is guaranteed to be a valid URL, since this is using the default URL. + let rpc_addr = Url::parse(&rpc_addr).unwrap(); - // Configure `$home_dir/config.json` to our liking. Sandbox requires extra settings - // for the best user experience, and being able to offer patching large state payloads. - crate::network::config::set_sandbox_configs(&home_dir)?; + info!(target: "workspaces", "Starting up sandbox at localhost:{}", rpc_port); let child = sandbox::run(&home_dir, rpc_port, net_port) .map_err(|e| SandboxErrorCode::RunFailure.custom(e))?; info!(target: "workspaces", "Started up sandbox at localhost:{} with pid={:?}", rpc_port, child.id()); Ok(Self { - rpc_port, - net_port, home_dir, + rpc_addr, + net_port: Some(net_port), rpc_port_lock: Some(rpc_port_lock), net_port_lock: Some(net_port_lock), process: Some(child), @@ -83,7 +109,10 @@ impl SandboxServer { if let Some(rpc_port_lock) = self.rpc_port_lock.take() { rpc_port_lock.unlock().map_err(|e| { ErrorKind::Io.full( - format!("failed to unlock lockfile for rpc_port={}", self.rpc_port), + format!( + "failed to unlock lockfile for rpc_port={:?}", + self.rpc_port() + ), e, ) })?; @@ -91,7 +120,7 @@ impl SandboxServer { if let Some(net_port_lock) = self.net_port_lock.take() { net_port_lock.unlock().map_err(|e| { ErrorKind::Io.full( - format!("failed to unlock lockfile for net_port={}", self.net_port), + format!("failed to unlock lockfile for net_port={:?}", self.net_port), e, ) })?; @@ -100,8 +129,16 @@ impl SandboxServer { Ok(()) } + pub fn rpc_port(&self) -> Option { + self.rpc_addr.port() + } + + pub fn net_port(&self) -> Option { + self.net_port + } + pub fn rpc_addr(&self) -> String { - format!("http://localhost:{}", self.rpc_port) + self.rpc_addr.to_string() } } @@ -111,12 +148,13 @@ impl Drop for SandboxServer { return; } + let rpc_port = self.rpc_port(); let child = self.process.as_mut().unwrap(); info!( target: "workspaces", - "Cleaning up sandbox: port={}, pid={}", - self.rpc_port, + "Cleaning up sandbox: port={:?}, pid={}", + rpc_port, child.id() ); @@ -124,9 +162,6 @@ impl Drop for SandboxServer { .kill() .map_err(|e| format!("Could not cleanup sandbox due to: {:?}", e)) .unwrap(); - - // Unlock the ports just in case they have not been preemptively done. - self.unlock_lockfiles().unwrap(); } } diff --git a/workspaces/src/worker/impls.rs b/workspaces/src/worker/impls.rs index 9651b970..bea7029c 100644 --- a/workspaces/src/worker/impls.rs +++ b/workspaces/src/worker/impls.rs @@ -237,7 +237,12 @@ impl Worker { } /// The port being used by RPC - pub fn rpc_port(&self) -> u16 { - self.workspace.rpc_port() + pub fn rpc_port(&self) -> Option { + self.workspace.server.rpc_port() + } + + /// Get the address the client is using to connect to the RPC of the network. + pub fn rpc_addr(&self) -> String { + self.workspace.server.rpc_addr() } } diff --git a/workspaces/src/worker/mod.rs b/workspaces/src/worker/mod.rs index 094d7ce0..889dfac0 100644 --- a/workspaces/src/worker/mod.rs +++ b/workspaces/src/worker/mod.rs @@ -3,6 +3,7 @@ mod impls; use std::fmt; use std::sync::Arc; +use crate::network::builder::NetworkBuilder; use crate::network::{Betanet, Mainnet, Sandbox, Testnet}; use crate::{Network, Result}; @@ -42,8 +43,8 @@ impl fmt::Debug for Worker { } /// Spin up a new sandbox instance, and grab a [`Worker`] that interacts with it. -pub async fn sandbox() -> Result> { - Ok(Worker::new(Sandbox::new().await?)) +pub fn sandbox() -> NetworkBuilder<'static, Sandbox> { + NetworkBuilder::new("sandbox") } /// Connect to the [testnet](https://explorer.testnet.near.org/) network, and grab diff --git a/workspaces/tests/deploy.rs b/workspaces/tests/deploy.rs index 8028695b..09685074 100644 --- a/workspaces/tests/deploy.rs +++ b/workspaces/tests/deploy.rs @@ -2,6 +2,9 @@ use serde::{Deserialize, Serialize}; use test_log::test; +use workspaces::network::Sandbox; +use workspaces::Worker; + const NFT_WASM_FILEPATH: &str = "../examples/res/non_fungible_token.wasm"; const EXPECTED_NFT_METADATA: &str = r#"{ "spec": "nft-1.0.0", @@ -28,9 +31,7 @@ fn expected() -> NftMetadata { serde_json::from_str(EXPECTED_NFT_METADATA).unwrap() } -#[test(tokio::test)] -async fn test_dev_deploy() -> anyhow::Result<()> { - let worker = workspaces::sandbox().await?; +async fn deploy_and_assert(worker: Worker) -> anyhow::Result<()> { let wasm = std::fs::read(NFT_WASM_FILEPATH)?; let contract = worker.dev_deploy(&wasm).await?; @@ -46,6 +47,38 @@ async fn test_dev_deploy() -> anyhow::Result<()> { let actual: NftMetadata = contract.view("nft_metadata").await?.json()?; assert_eq!(actual, expected()); + Ok(()) +} + +#[test(tokio::test)] +async fn test_dev_deploy() -> anyhow::Result<()> { + let worker = workspaces::sandbox().await?; + deploy_and_assert(worker).await?; + Ok(()) +} + +#[test(tokio::test)] +async fn test_manually_spawned_deploy() -> anyhow::Result<()> { + let rpc_port = + portpicker::pick_unused_port().ok_or_else(|| anyhow::anyhow!("no free ports"))?; + let net_port = + portpicker::pick_unused_port().ok_or_else(|| anyhow::anyhow!("no free ports"))?; + let mut home_dir = std::env::temp_dir(); + home_dir.push(format!("test-sandbox-{}", rpc_port)); + + // intialize chain data with supplied home dir + let output = near_sandbox_utils::init(&home_dir)?.output().await?; + tracing::info!(target: "workspaces-test", "sandbox-init: {:?}", output); + + let mut child = near_sandbox_utils::run(&home_dir, rpc_port, net_port)?; + + // connect to local sandbox node + let worker = workspaces::sandbox() + .rpc_addr(&format!("http://localhost:{}", rpc_port)) + .home_dir(home_dir) + .await?; + deploy_and_assert(worker).await?; + child.kill()?; Ok(()) }