Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Autonomi cli #2152

Merged
merged 5 commits into from
Oct 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
resolver = "2"
members = [
"autonomi",
"autonomi_cli",
"evmlib",
"evm_testnet",
# "sn_auditor",
Expand Down
1 change: 1 addition & 0 deletions autonomi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ curv = { version = "0.10.1", package = "sn_curv", default-features = false, feat
eip2333 = { version = "0.2.1", package = "sn_bls_ckd" }
const-hex = "1.12.0"
evmlib = { path = "../evmlib", version = "0.1" }
hex = "~0.4.3"
libp2p = "0.54.1"
rand = "0.8.5"
rmp-serde = "1.1.1"
Expand Down
42 changes: 42 additions & 0 deletions autonomi/src/client/address.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
// Copyright 2024 MaidSafe.net limited.
//
// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3.
// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed
// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use xor_name::XorName;

#[derive(Debug, thiserror::Error)]
pub enum DataError {
#[error("Invalid XorName")]
InvalidXorName,
#[error("Input address is not a hex string")]
InvalidHexString,
}

pub fn str_to_xorname(addr: &str) -> Result<XorName, DataError> {
let bytes = hex::decode(addr).map_err(|_| DataError::InvalidHexString)?;
let xor = XorName(bytes.try_into().map_err(|_| DataError::InvalidXorName)?);
Ok(xor)
}

pub fn xorname_to_str(addr: XorName) -> String {
hex::encode(addr)
}

#[cfg(test)]
mod test {
use super::*;
use xor_name::XorName;

#[test]
fn test_xorname_to_str() {
let rng = &mut rand::thread_rng();
let xorname = XorName::random(rng);
let str = xorname_to_str(xorname);
let xorname2 = str_to_xorname(&str).expect("Failed to convert back to xorname");
assert_eq!(xorname, xorname2);
}
}
27 changes: 25 additions & 2 deletions autonomi/src/client/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use evmlib::common::{QuoteHash, QuotePayment, TxHash};
use evmlib::wallet::Wallet;
use libp2p::futures;
use rand::{thread_rng, Rng};
use sn_evm::ProofOfPayment;
use sn_evm::{Amount, AttoTokens, ProofOfPayment};
use sn_networking::PutRecordCfg;
use sn_networking::{GetRecordCfg, Network, NetworkError, PayeeQuote, VerificationKind};
use sn_protocol::{
Expand Down Expand Up @@ -54,7 +54,9 @@ pub enum PayError {
#[error("Could not simultaneously fetch store costs: {0:?}")]
JoinError(JoinError),
#[error("Wallet error: {0:?}")]
WalletError(#[from] wallet::Error),
EvmWalletError(#[from] wallet::Error),
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
}

/// Errors that can occur during the get operation.
Expand Down Expand Up @@ -184,6 +186,27 @@ impl Client {
Ok(map_xor_name)
}

pub(crate) async fn cost(
&mut self,
data: Bytes,
) -> Result<AttoTokens, PayError> {
let now = std::time::Instant::now();
let (data_map_chunk, chunks) = encrypt(data)?;

tracing::debug!("Encryption took: {:.2?}", now.elapsed());

let map_xor_name = *data_map_chunk.address().xorname();
let mut content_addrs = vec![map_xor_name];

for chunk in &chunks {
content_addrs.push(*chunk.name());
}

let cost_map = self.get_store_quotes(content_addrs.into_iter()).await?;
let total_cost = AttoTokens::from_atto(cost_map.iter().map(|(_, quote)| quote.2.cost.as_atto()).sum::<Amount>());
Ok(total_cost)
}

pub(crate) async fn pay(
&mut self,
content_addrs: impl Iterator<Item = XorName>,
Expand Down
55 changes: 52 additions & 3 deletions autonomi/src/client/files.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use crate::client::data::{GetError, PutError};
use crate::client::Client;
use crate::self_encryption::encrypt;
use bytes::Bytes;
use evmlib::wallet::Wallet;
use serde::{Deserialize, Serialize};
use sn_evm::{Amount, AttoTokens};
use std::collections::HashMap;
use std::path::PathBuf;
use walkdir::WalkDir;
Expand Down Expand Up @@ -61,12 +62,59 @@ impl Client {
Ok(data)
}

/// Get the cost to upload a file/dir to the network.
/// quick and dirty implementation, please refactor once files are cleanly implemented
pub async fn file_cost(
&mut self,
path: &PathBuf,
) -> Result<AttoTokens, UploadError> {
let mut map = HashMap::new();
let mut total_cost = Amount::ZERO;

for entry in WalkDir::new(path) {
let entry = entry?;

if !entry.file_type().is_file() {
continue;
}

let path = entry.path().to_path_buf();
tracing::info!("Cost for file: {path:?}");

let data = tokio::fs::read(&path).await?;
let file_bytes = Bytes::from(data);
let file_cost = self.cost(file_bytes.clone()).await.expect("TODO");

total_cost += file_cost.as_atto();

// re-do encryption to get the correct map xorname here
// this code needs refactor
let now = std::time::Instant::now();
let (data_map_chunk, _) = encrypt(file_bytes).expect("TODO");
tracing::debug!("Encryption took: {:.2?}", now.elapsed());
let map_xor_name = *data_map_chunk.address().xorname();
let data_map_xorname = FilePointer {
data_map: map_xor_name,
created_at: 0,
modified_at: 0,
};

map.insert(path, data_map_xorname);
}

let root = Root { map };
let root_serialized = rmp_serde::to_vec(&root).expect("TODO");

let cost = self.cost(Bytes::from(root_serialized)).await.expect("TODO");
Ok(cost)
}

/// Upload a directory to the network. The directory is recursively walked.
#[cfg(feature = "fs")]
pub async fn upload_from_dir(
&mut self,
path: PathBuf,
wallet: &Wallet,
wallet: &sn_evm::EvmWallet,
) -> Result<(Root, XorName), UploadError> {
let mut map = HashMap::new();

Expand All @@ -79,6 +127,7 @@ impl Client {

let path = entry.path().to_path_buf();
tracing::info!("Uploading file: {path:?}");
println!("Uploading file: {path:?}");
let file = upload_from_file(self, path.clone(), wallet).await?;

map.insert(path, file);
Expand All @@ -96,7 +145,7 @@ impl Client {
async fn upload_from_file(
client: &mut Client,
path: PathBuf,
wallet: &Wallet,
wallet: &sn_evm::EvmWallet,
) -> Result<FilePointer, UploadError> {
let data = tokio::fs::read(path).await?;
let data = Bytes::from(data);
Expand Down
2 changes: 2 additions & 0 deletions autonomi/src/client/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
pub mod address;

#[cfg(feature = "data")]
pub mod data;
#[cfg(feature = "files")]
Expand Down
3 changes: 3 additions & 0 deletions autonomi/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ pub mod client;
#[cfg(feature = "data")]
mod self_encryption;

pub use sn_evm::EvmWallet as Wallet;
pub use sn_evm::EvmNetwork as Network;

#[doc(no_inline)] // Place this under 'Re-exports' in the docs.
pub use bytes::Bytes;
#[doc(no_inline)] // Place this under 'Re-exports' in the docs.
Expand Down
82 changes: 82 additions & 0 deletions autonomi/tests/evm/file.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#[cfg(feature = "evm-payments")]
mod test {

use crate::common;
use crate::common::{evm_network_from_env, evm_wallet_from_env_or_default};
use autonomi::Client;
use bytes::Bytes;
use eyre::bail;
use std::time::Duration;
use tokio::time::sleep;

#[tokio::test]
async fn file() -> Result<(), Box<dyn std::error::Error>> {
common::enable_logging();

let network = evm_network_from_env();
let mut client = Client::connect(&[]).await.unwrap();
let mut wallet = evm_wallet_from_env_or_default(network);

// let data = common::gen_random_data(1024 * 1024 * 1000);
// let user_key = common::gen_random_data(32);

let (root, addr) = client
.upload_from_dir("tests/file/test_dir".into(), &mut wallet)
.await?;

sleep(Duration::from_secs(10)).await;

let root_fetched = client.fetch_root(addr).await?;

assert_eq!(
root.map, root_fetched.map,
"root fetched should match root put"
);

Ok(())
}

#[cfg(feature = "vault")]
#[tokio::test]
async fn file_into_vault() -> eyre::Result<()> {
common::enable_logging();

let network = evm_network_from_env();

let mut client = Client::connect(&[])
.await?
.with_vault_entropy(Bytes::from("at least 32 bytes of entropy here"))?;

let mut wallet = evm_wallet_from_env_or_default(network);

let (root, addr) = client
.upload_from_dir("tests/file/test_dir".into(), &mut wallet)
.await?;
sleep(Duration::from_secs(2)).await;

let root_fetched = client.fetch_root(addr).await?;

assert_eq!(
root.map, root_fetched.map,
"root fetched should match root put"
);

// now assert over the stored account packet
let new_client = Client::connect(&[])
.await?
.with_vault_entropy(Bytes::from("at least 32 bytes of entropy here"))?;

if let Some(ap) = new_client.fetch_and_decrypt_vault().await? {
let ap_root_fetched = Client::deserialise_root(ap)?;

assert_eq!(
root.map, ap_root_fetched.map,
"root fetched should match root put"
);
} else {
bail!("No account packet found");
}

Ok(())
}
}
33 changes: 33 additions & 0 deletions autonomi_cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
[package]
name = "autonomi_cli"
version = "0.1.0"
edition = "2021"

[features]
default = ["metrics"]
local-discovery = ["sn_peers_acquisition/local-discovery"]
metrics = ["sn_logging/process-metrics"]
network-contacts = ["sn_peers_acquisition/network-contacts"]

[dependencies]
autonomi = { path = "../autonomi", version = "0.1.0", features = ["data", "files"] }
clap = { version = "4.2.1", features = ["derive"] }
color-eyre = "~0.6"
dirs-next = "~2.0.0"
indicatif = { version = "0.17.5", features = ["tokio"] }
tokio = { version = "1.32.0", features = [
"io-util",
"macros",
"parking_lot",
"rt",
"sync",
"time",
"fs",
] }
tracing = { version = "~0.1.26" }
sn_peers_acquisition = { path = "../sn_peers_acquisition", version = "0.5.0" }
sn_build_info = { path = "../sn_build_info", version = "0.1.11" }
sn_logging = { path = "../sn_logging", version = "0.2.33" }

[lints]
workspace = true
27 changes: 27 additions & 0 deletions autonomi_cli/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# A CLI for the Autonomi Network

```
Usage: autonomi_cli [OPTIONS] <COMMAND>
Commands:
file Operations related to file handling
register Operations related to register management
vault Operations related to vault management
help Print this message or the help of the given subcommand(s)
Options:
--log-output-dest <LOG_OUTPUT_DEST>
Specify the logging output destination. [default: data-dir]
--log-format <LOG_FORMAT>
Specify the logging format.
--peer <multiaddr>
Peer(s) to use for bootstrap, in a 'multiaddr' format containing the peer ID [env: SAFE_PEERS=]
--timeout <CONNECTION_TIMEOUT>
The maximum duration to wait for a connection to the network before timing out
-x, --no-verify
Prevent verification of data storage on the network
-h, --help
Print help (see more with '--help')
-V, --version
Print version
```
Loading
Loading