Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(autonomi): restructure modules and private/public naming #2499

Merged
merged 14 commits into from
Dec 6, 2024
Merged
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions ant-cli/src/access/user_data.rs
Original file line number Diff line number Diff line change
@@ -10,8 +10,7 @@ use std::collections::HashMap;

use autonomi::client::{
address::{addr_to_str, str_to_addr},
archive::ArchiveAddr,
archive_private::PrivateArchiveAccess,
files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr},
registers::{RegisterAddress, RegisterSecretKey},
vault::UserData,
};
13 changes: 8 additions & 5 deletions ant-cli/src/actions/download.rs
Original file line number Diff line number Diff line change
@@ -8,7 +8,10 @@

use super::get_progress_bar;
use autonomi::{
client::{address::str_to_addr, archive::ArchiveAddr, archive_private::PrivateArchiveAccess},
client::{
address::str_to_addr,
files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr},
},
Client,
};
use color_eyre::{
@@ -40,15 +43,15 @@ async fn download_private(
client: &mut Client,
) -> Result<()> {
let archive = client
.private_archive_get(private_address)
.archive_get(private_address)
.await
.wrap_err("Failed to fetch data from address")?;

let progress_bar = get_progress_bar(archive.iter().count() as u64)?;
let mut all_errs = vec![];
for (path, access, _meta) in archive.iter() {
progress_bar.println(format!("Fetching file: {path:?}..."));
let bytes = match client.private_data_get(access.clone()).await {
let bytes = match client.data_get(access.clone()).await {
Ok(bytes) => bytes,
Err(e) => {
let err = format!("Failed to fetch file {path:?}: {e}");
@@ -86,15 +89,15 @@ async fn download_public(
client: &mut Client,
) -> Result<()> {
let archive = client
.archive_get(address)
.archive_get_public(address)
.await
.wrap_err("Failed to fetch data from address")?;

let progress_bar = get_progress_bar(archive.iter().count() as u64)?;
let mut all_errs = vec![];
for (path, addr, _meta) in archive.iter() {
progress_bar.println(format!("Fetching file: {path:?}..."));
let bytes = match client.data_get(*addr).await {
let bytes = match client.data_get_public(*addr).await {
Ok(bytes) => bytes,
Err(e) => {
let err = format!("Failed to fetch file {path:?}: {e}");
11 changes: 8 additions & 3 deletions ant-cli/src/commands/file.rs
Original file line number Diff line number Diff line change
@@ -53,16 +53,21 @@ pub async fn upload(file: &str, public: bool, peers: Vec<Multiaddr>) -> Result<(
let local_addr;
let archive = if public {
let xor_name = client
.dir_upload(dir_path, &wallet)
.dir_upload_public(dir_path, &wallet)
.await
.wrap_err("Failed to upload file")?;
local_addr = addr_to_str(xor_name);
local_addr.clone()
} else {
let private_data_access = client
.private_dir_upload(dir_path, &wallet)
let private_archive = client
.dir_upload(dir_path, &wallet)
.await
.wrap_err("Failed to upload file")?;
let private_data_access = client
.archive_put(private_archive, (&wallet).into())
.await
.wrap_err("Failed to upload private archive")?;

local_addr = private_data_access.address();
private_data_access.to_hex()
};
4 changes: 2 additions & 2 deletions ant-node/tests/data_with_churn.rs
Original file line number Diff line number Diff line change
@@ -338,7 +338,7 @@ fn store_chunks_task(
let mut retries = 1;
loop {
match client
.data_put(random_data.clone(), (&wallet).into())
.data_put_public(random_data.clone(), (&wallet).into())
.await
.inspect_err(|err| {
println!("Error to put chunk: {err:?}");
@@ -537,7 +537,7 @@ async fn query_content(client: &Client, net_addr: &NetworkAddress) -> Result<()>
Ok(())
}
NetworkAddress::ChunkAddress(addr) => {
client.data_get(*addr.xorname()).await?;
client.data_get_public(*addr.xorname()).await?;
Ok(())
}
_other => Ok(()), // we don't create/store any other type of content in this test yet
4 changes: 2 additions & 2 deletions ant-node/tests/storage_payments.rs
Original file line number Diff line number Diff line change
@@ -205,7 +205,7 @@
// let _upload_stats = uploader.start_upload().await?;

// let mut files_download = FilesDownload::new(files_api);
// let _ = files_download.download_file(file_addr, None).await?;
// let _ = files_download.file_download_public(file_addr, None).await?;

// Ok(())
// }
@@ -252,7 +252,7 @@
// let mut files_download = FilesDownload::new(files_api);
// assert!(
// matches!(
// files_download.download_file(content_addr, None).await,
// files_download.file_download_public(content_addr, None).await,
// Err(ClientError::Network(NetworkError::GetRecordError(
// GetRecordError::RecordNotFound
// )))
2 changes: 1 addition & 1 deletion ant-node/tests/verify_data_location.rs
Original file line number Diff line number Diff line change
@@ -351,7 +351,7 @@ async fn store_chunks(

let random_bytes = Bytes::from(random_bytes);

client.data_put(random_bytes, wallet.into()).await?;
client.data_put_public(random_bytes, wallet.into()).await?;

uploaded_chunks_count += 1;

66 changes: 29 additions & 37 deletions autonomi/README.md
Original file line number Diff line number Diff line change
@@ -28,14 +28,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {

// Put and fetch data.
let data_addr = client
.data_put(Bytes::from("Hello, World"), (&wallet).into())
.data_put_public(Bytes::from("Hello, World"), (&wallet).into())
.await?;
let _data_fetched = client.data_get(data_addr).await?;
let _data_fetched = client.data_get_public(data_addr).await?;

// Put and fetch directory from local file system.
let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?;
let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?;
client
.dir_download(dir_addr, "files/downloaded".into())
.dir_download_public(dir_addr, "files/downloaded".into())
.await?;

Ok(())
@@ -53,52 +53,44 @@ let wallet = Wallet::new_from_private_key(EvmNetwork::new_custom("<rpc URL>", "<

## Running tests

### Using a local EVM testnet
To run the tests, we can run a local network:

1. If you haven't, install Foundry, to be able to run Anvil
nodes: https://book.getfoundry.sh/getting-started/installation
2. Run a local EVM node:
1. Run a local EVM node:
> Note: To run the EVM node, Foundry is required to be installed: https://book.getfoundry.sh/getting-started/installation

```sh
cargo run --bin=evm-testnet
```

3. Run a local network with the `local` feature and use the local evm node.

```sh
cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address=<ETHEREUM_ADDRESS> evm-local
```
```sh
cargo run --bin evm-testnet
```

4. Then run the tests with the `local` feature and pass the EVM params again:
2. Run a local network with the `local` feature and use the local EVM node.
```sh
cargo run --bin antctl --features local -- local run --build --clean --rewards-address <ETHEREUM_ADDRESS> evm-local
```

```sh
EVM_NETWORK=local cargo test --package autonomi --features=local
# Or with logs
RUST_LOG=autonomi EVM_NETWORK=local cargo test --package autonomi --features local -- --nocapture
```
3. Then run the tests with the `local` feature and pass the EVM params again:
```sh
EVM_NETWORK=local cargo test --features local --package autonomi
```

### Using a live testnet or mainnet

Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and
point it to a live network.
Using the hardcoded `Arbitrum One` option as an example, but you can also use the command flags of the steps above and point it to a live network.

1. Run a local network with the `local` feature:

```sh
cargo run --bin=antctl --features=local -- local run --build --clean --rewards-address=<ETHEREUM_ADDRESS> evm-arbitrum-one
cargo run --bin antctl --features local -- local run --build --clean --rewards-address <ETHEREUM_ADDRESS> evm-arbitrum-one
```

2. Then run the tests with the `local` feature. Make sure that the wallet of the private key you pass has enough gas and
payment tokens on the network (in this case Arbitrum One):
2. Then pass the private key of the wallet, and ensure it has enough gas and payment tokens on the network (in this case Arbitrum One):

```sh
EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY=<PRIVATE_KEY> cargo test --package=autonomi --features=local
EVM_NETWORK=arbitrum-one EVM_PRIVATE_KEY=<PRIVATE_KEY> cargo test --package autonomi --features local
```

## Using funds from the Deployer Wallet

You can use the `Deployer wallet private key` printed in the EVM node output to
initialise a wallet from with almost infinite gas and payment tokens. Example:
You can use the `Deployer wallet private key` printed in the EVM node output to initialise a wallet from with almost infinite gas and payment tokens. Example:

```rust
let rpc_url = "http://localhost:54370/";
@@ -107,25 +99,25 @@ let data_payments_address = "0x8464135c8F25Da09e49BC8782676a84730C318bC";
let private_key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80";

let network = Network::Custom(CustomNetwork::new(
rpc_url,
payment_token_address,
data_payments_address,
rpc_url,
payment_token_address,
data_payments_address,
));

let deployer_wallet = Wallet::new_from_private_key(network, private_key).unwrap();
let receiving_wallet = Wallet::new_with_random_wallet(network);

// Send 10 payment tokens (atto)
let _ = deployer_wallet
.transfer_tokens(receiving_wallet.address(), Amount::from(10))
.await;
.transfer_tokens(receiving_wallet.address(), Amount::from(10))
.await;
```

Alternatively, you can provide the wallet address that should own all the gas and payment tokens to the EVM testnet
startup command using the `--genesis-wallet` flag:

```sh
cargo run --bin evm-testnet -- --genesis-wallet=<ETHEREUM_ADDRESS>
cargo run --bin evm-testnet -- --genesis-wallet <ETHEREUM_ADDRESS>
```

```shell
28 changes: 14 additions & 14 deletions autonomi/README_PYTHON.md
Original file line number Diff line number Diff line change
@@ -26,11 +26,11 @@ payment = PaymentOption.wallet(wallet)

# Upload data
data = b"Hello, Safe Network!"
addr = client.data_put(data, payment)
addr = client.data_put_public(data, payment)
print(f"Data uploaded to: {addr}")

# Download data
retrieved = client.data_get(addr)
retrieved = client.data_get_public(addr)
print(f"Retrieved: {retrieved.decode()}")
```

@@ -40,10 +40,10 @@ print(f"Retrieved: {retrieved.decode()}")

- `Client`: Main interface to the Autonomi network
- `connect(peers: List[str])`: Connect to network nodes
- `data_put(data: bytes, payment: PaymentOption)`: Upload data
- `data_get(addr: str)`: Download data
- `private_data_put(data: bytes, payment: PaymentOption)`: Store private data
- `private_data_get(access: PrivateDataAccess)`: Retrieve private data
- `data_put_public(data: bytes, payment: PaymentOption)`: Upload data
- `data_get_public(addr: str)`: Download data
- `data_put(data: bytes, payment: PaymentOption)`: Store private data
- `data_get(access: DataMapChunk)`: Retrieve private data
- `register_generate_key()`: Generate register key

- `Wallet`: Ethereum wallet management
@@ -56,16 +56,16 @@ print(f"Retrieved: {retrieved.decode()}")

#### Private Data

- `PrivateDataAccess`: Handle private data storage
- `DataMapChunk`: Handle private data storage
- `from_hex(hex: str)`: Create from hex string
- `to_hex()`: Convert to hex string
- `address()`: Get short reference address

```python
# Private data example
access = client.private_data_put(secret_data, payment)
access = client.data_put(secret_data, payment)
print(f"Private data stored at: {access.to_hex()}")
retrieved = client.private_data_get(access)
retrieved = client.data_get(access)
```

#### Registers
@@ -117,15 +117,15 @@ data, content_type = client.fetch_and_decrypt_vault(vault_key)
def handle_data_operations(client, payment):
# Upload text
text_data = b"Hello, Safe Network!"
text_addr = client.data_put(text_data, payment)
text_addr = client.data_put_public(text_data, payment)

# Upload binary data
with open("image.jpg", "rb") as f:
image_data = f.read()
image_addr = client.data_put(image_data, payment)
image_addr = client.data_put_public(image_data, payment)

# Download and verify
downloaded = client.data_get(text_addr)
downloaded = client.data_get_public(text_addr)
assert downloaded == text_data
```

@@ -138,11 +138,11 @@ def handle_private_data(client, payment):
data = json.dumps(secret).encode()

# Store privately
access = client.private_data_put(data, payment)
access = client.data_put(data, payment)
print(f"Access token: {access.to_hex()}")

# Retrieve
retrieved = client.private_data_get(access)
retrieved = client.data_get(access)
secret = json.loads(retrieved.decode())
```

4 changes: 2 additions & 2 deletions autonomi/examples/autonomi_advanced.py
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ def connect_to_network(peers: list[str]) -> Client:

def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str:
try:
addr = client.data_put(data, payment)
addr = client.data_put_public(data, payment)
print(f"Successfully uploaded data to: {addr}")
return addr
except Exception as e:
@@ -34,7 +34,7 @@ def upload_data(client: Client, data: bytes, payment: PaymentOption) -> str:

def download_data(client: Client, addr: str) -> bytes:
try:
data = client.data_get(addr)
data = client.data_get_public(addr)
print(f"Successfully downloaded {len(data)} bytes")
return data
except Exception as e:
8 changes: 4 additions & 4 deletions autonomi/examples/autonomi_data_registers.py
Original file line number Diff line number Diff line change
@@ -7,22 +7,22 @@ def handle_data_operations(client: Client, payment: PaymentOption):

# Upload some text data
text_data = b"Hello, Safe Network!"
text_addr = client.data_put(text_data, payment)
text_addr = client.data_put_public(text_data, payment)
print(f"Text data uploaded to: {text_addr}")

# Upload binary data (like an image)
with open("example.jpg", "rb") as f:
image_data = f.read()
image_addr = client.data_put(image_data, payment)
image_addr = client.data_put_public(image_data, payment)
print(f"Image uploaded to: {image_addr}")

# Download and verify data
downloaded_text = client.data_get(text_addr)
downloaded_text = client.data_get_public(text_addr)
assert downloaded_text == text_data, "Text data verification failed!"
print("Text data verified successfully")

# Download and save image
downloaded_image = client.data_get(image_addr)
downloaded_image = client.data_get_public(image_addr)
with open("downloaded_example.jpg", "wb") as f:
f.write(downloaded_image)
print("Image downloaded successfully")
6 changes: 3 additions & 3 deletions autonomi/examples/autonomi_example.py
Original file line number Diff line number Diff line change
@@ -21,17 +21,17 @@ def main():

# Upload some data
data = b"Hello, Safe Network!"
addr = client.data_put(data, payment)
addr = client.data_put_public(data, payment)
print(f"Data uploaded to address: {addr}")

# Download the data back
downloaded = client.data_get(addr)
downloaded = client.data_get_public(addr)
print(f"Downloaded data: {downloaded.decode()}")

# You can also upload files
with open("example.txt", "rb") as f:
file_data = f.read()
file_addr = client.data_put(file_data, payment)
file_addr = client.data_put_public(file_data, payment)
print(f"File uploaded to address: {file_addr}")

if __name__ == "__main__":
4 changes: 2 additions & 2 deletions autonomi/examples/autonomi_private_data.py
Original file line number Diff line number Diff line change
@@ -10,12 +10,12 @@ def __init__(self, client: Client, wallet: Wallet):

def store_private_data(self, data: bytes) -> str:
"""Store data privately and return its address"""
addr = self.client.private_data_put(data, self.payment)
addr = self.client.data_put(data, self.payment)
return addr

def retrieve_private_data(self, addr: str) -> bytes:
"""Retrieve privately stored data"""
return self.client.private_data_get(addr)
return self.client.data_get(addr)

def create_shared_register(self, name: str, initial_value: bytes,
allowed_writers: List[str]) -> str:
6 changes: 3 additions & 3 deletions autonomi/examples/autonomi_private_encryption.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from autonomi_client import (
Client, Wallet, PaymentOption, PrivateDataAccess,
Client, Wallet, PaymentOption, DataMapChunk,
encrypt, hash_to_short_string
)
import json
@@ -16,12 +16,12 @@ def demonstrate_private_data(client: Client, payment: PaymentOption):
data_bytes = json.dumps(secret_data).encode()

# Store it privately
access = client.private_data_put(data_bytes, payment)
access = client.data_put(data_bytes, payment)
print(f"Stored private data, access token: {access.to_hex()}")
print(f"Short reference: {access.address()}")

# Retrieve it
retrieved_bytes = client.private_data_get(access)
retrieved_bytes = client.data_get(access)
retrieved_data = json.loads(retrieved_bytes.decode())
print(f"Retrieved private data: {retrieved_data}")

14 changes: 7 additions & 7 deletions autonomi/examples/basic.py
Original file line number Diff line number Diff line change
@@ -22,15 +22,15 @@ def main():

# Upload public data
data = b"Hello World!"
addr = client.data_put(data, wallet)
addr = client.data_put_public(data, wallet)
print(f"Uploaded public data to: {addr}")
retrieved = client.data_get(addr)
retrieved = client.data_get_public(addr)
print(f"Retrieved public data: {retrieved}")

# Upload private data
private_access = client.private_data_put(b"Secret message", wallet)
private_access = client.data_put(b"Secret message", wallet)
print(f"Private data access: {private_access}")
private_data = client.private_data_get(private_access)
private_data = client.data_get(private_access)
print(f"Retrieved private data: {private_data}")

# Create register
@@ -40,7 +40,7 @@ def main():
print(f"Register values: {reg_values}")

# Upload file/directory
file_addr = client.file_upload("./test_data", wallet)
file_addr = client.file_upload_public("./test_data", wallet)
print(f"Uploaded files to: {file_addr}")
client.file_download(file_addr, "./downloaded_data")
print("Downloaded files")
@@ -58,9 +58,9 @@ def main():
print(f"Retrieved user data: {retrieved_data}")

# Private directory operations
private_dir_access = client.private_dir_upload("./test_data", wallet)
private_dir_access = client.dir_upload("./test_data", wallet)
print(f"Uploaded private directory, access: {private_dir_access}")
client.private_dir_download(private_dir_access, "./downloaded_private")
client.dir_download(private_dir_access, "./downloaded_private")
print("Downloaded private directory")

# External signer example
10 changes: 6 additions & 4 deletions autonomi/examples/put_and_dir_upload.rs
Original file line number Diff line number Diff line change
@@ -10,14 +10,16 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {

// Put and fetch data.
let data_addr = client
.data_put(Bytes::from("Hello, World"), (&wallet).into())
.data_put_public(Bytes::from("Hello, World"), (&wallet).into())
.await?;
let _data_fetched = client.data_get(data_addr).await?;
let _data_fetched = client.data_get_public(data_addr).await?;

// Put and fetch directory from local file system.
let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?;
let dir_addr = client
.dir_upload_public("files/to/upload".into(), &wallet)
.await?;
client
.dir_download(dir_addr, "files/downloaded".into())
.dir_download_public(dir_addr, "files/downloaded".into())
.await?;

Ok(())
4 changes: 2 additions & 2 deletions autonomi/python/autonomi_client/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, PrivateDataAccess, encrypt
from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt

__all__ = [
"Client",
"Wallet",
"PaymentOption",
"VaultSecretKey",
"UserData",
"PrivateDataAccess",
"DataMapChunk",
"encrypt"
]
239 changes: 239 additions & 0 deletions autonomi/src/client/data/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
// Copyright 2024 MaidSafe.net limited.
//
// This SAFE Network Software is licensed to you under The General Public License (GPL), version 3.
// Unless required by applicable law or agreed to in writing, the SAFE Network Software distributed
// under the GPL Licence is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use std::hash::{DefaultHasher, Hash, Hasher};
use std::sync::LazyLock;

use ant_evm::{Amount, EvmWalletError};
use ant_networking::NetworkError;
use ant_protocol::storage::Chunk;
use ant_protocol::NetworkAddress;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use xor_name::XorName;

use crate::client::payment::PaymentOption;
use crate::client::{ClientEvent, UploadSummary};
use crate::{self_encryption::encrypt, Client};

pub mod public;

/// Number of chunks to upload in parallel.
///
/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable.
pub(crate) static CHUNK_UPLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("Chunk upload batch size: {}", batch_size);
batch_size
});

/// Number of chunks to download in parallel.
///
/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable.
pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("Chunk download batch size: {}", batch_size);
batch_size
});

/// Number of retries to upload chunks.
pub(crate) const RETRY_ATTEMPTS: usize = 3;

/// Raw Data Address (points to a DataMap)
pub type DataAddr = XorName;
/// Raw Chunk Address (points to a [`Chunk`])
pub type ChunkAddr = XorName;

/// Errors that can occur during the put operation.
#[derive(Debug, thiserror::Error)]
pub enum PutError {
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("A network error occurred.")]
Network(#[from] NetworkError),
#[error("Error occurred during cost estimation.")]
CostError(#[from] CostError),
#[error("Error occurred during payment.")]
PayError(#[from] PayError),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("A wallet error occurred.")]
Wallet(#[from] ant_evm::EvmError),
#[error("The vault owner key does not match the client's public key")]
VaultBadOwner,
#[error("Payment unexpectedly invalid for {0:?}")]
PaymentUnexpectedlyInvalid(NetworkAddress),
}

/// Errors that can occur during the pay operation.
#[derive(Debug, thiserror::Error)]
pub enum PayError {
#[error("Wallet error: {0:?}")]
EvmWalletError(#[from] EvmWalletError),
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Cost error: {0:?}")]
Cost(#[from] CostError),
}

/// Errors that can occur during the get operation.
#[derive(Debug, thiserror::Error)]
pub enum GetError {
#[error("Could not deserialize data map.")]
InvalidDataMap(rmp_serde::decode::Error),
#[error("Failed to decrypt data.")]
Decryption(crate::self_encryption::Error),
#[error("Failed to deserialize")]
Deserialization(#[from] rmp_serde::decode::Error),
#[error("General networking error: {0:?}")]
Network(#[from] NetworkError),
#[error("General protocol error: {0:?}")]
Protocol(#[from] ant_protocol::Error),
}

/// Errors that can occur during the cost calculation.
#[derive(Debug, thiserror::Error)]
pub enum CostError {
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Could not get store quote for: {0:?} after several retries")]
CouldNotGetStoreQuote(XorName),
#[error("Could not get store costs: {0:?}")]
CouldNotGetStoreCosts(NetworkError),
#[error("Failed to serialize {0}")]
Serialization(String),
}

/// Private data on the network can be accessed with this
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DataMapChunk(Chunk);

impl DataMapChunk {
pub fn to_hex(&self) -> String {
hex::encode(self.0.value())
}

pub fn from_hex(hex: &str) -> Result<Self, hex::FromHexError> {
let data = hex::decode(hex)?;
Ok(Self(Chunk::new(Bytes::from(data))))
}

/// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side.
pub fn address(&self) -> String {
hash_to_short_string(&self.to_hex())
}
}

fn hash_to_short_string(input: &str) -> String {
let mut hasher = DefaultHasher::new();
input.hash(&mut hasher);
let hash_value = hasher.finish();
hash_value.to_string()
}

impl Client {
/// Fetch a blob of (private) data from the network
pub async fn data_get(&self, data_map: DataMapChunk) -> Result<Bytes, GetError> {
info!(
"Fetching private data from Data Map {:?}",
data_map.0.address()
);
let data = self.fetch_from_data_map_chunk(data_map.0.value()).await?;

debug!("Successfully fetched a blob of private data from the network");
Ok(data)
}

/// Upload a piece of private data to the network. This data will be self-encrypted.
/// The [`DataMapChunk`] is not uploaded to the network, keeping the data private.
///
/// Returns the [`DataMapChunk`] containing the map to the encrypted chunks.
pub async fn data_put(
&self,
data: Bytes,
payment_option: PaymentOption,
) -> Result<DataMapChunk, PutError> {
let now = ant_networking::target_arch::Instant::now();
let (data_map_chunk, chunks) = encrypt(data)?;
debug!("Encryption took: {:.2?}", now.elapsed());

// Pay for all chunks
let xor_names: Vec<_> = chunks.iter().map(|chunk| *chunk.name()).collect();
info!("Paying for {} addresses", xor_names.len());
let receipt = self
.pay_for_content_addrs(xor_names.into_iter(), payment_option)
.await
.inspect_err(|err| error!("Error paying for data: {err:?}"))?;

// Upload the chunks with the payments
debug!("Uploading {} chunks", chunks.len());

let mut failed_uploads = self
.upload_chunks_with_retries(chunks.iter().collect(), &receipt)
.await;

// Return the last chunk upload error
if let Some(last_chunk_fail) = failed_uploads.pop() {
tracing::error!(
"Error uploading chunk ({:?}): {:?}",
last_chunk_fail.0.address(),
last_chunk_fail.1
);
return Err(last_chunk_fail.1);
}

let record_count = chunks.len();

// Reporting
if let Some(channel) = self.client_event_sender.as_ref() {
let tokens_spent = receipt
.values()
.map(|proof| proof.quote.cost.as_atto())
.sum::<Amount>();

let summary = UploadSummary {
record_count,
tokens_spent,
};
if let Err(err) = channel.send(ClientEvent::UploadComplete(summary)).await {
error!("Failed to send client event: {err:?}");
}
}

Ok(DataMapChunk(data_map_chunk))
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_hex() {
let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello")));
let hex = data_map.to_hex();
let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex");
assert_eq!(data_map, data_map2);
}
}
107 changes: 4 additions & 103 deletions autonomi/src/client/data.rs → autonomi/src/client/data/public.rs
Original file line number Diff line number Diff line change
@@ -10,124 +10,25 @@ use bytes::Bytes;
use libp2p::kad::Quorum;

use std::collections::{HashMap, HashSet};
use std::sync::LazyLock;
use xor_name::XorName;

use crate::client::payment::PaymentOption;
use crate::client::utils::process_tasks_with_max_concurrency;
use crate::client::{ClientEvent, UploadSummary};
use crate::{self_encryption::encrypt, Client};
use ant_evm::ProofOfPayment;
use ant_evm::{Amount, AttoTokens};
use ant_evm::{EvmWalletError, ProofOfPayment};
use ant_networking::{GetRecordCfg, NetworkError};
use ant_protocol::{
storage::{try_deserialize_record, Chunk, ChunkAddress, RecordHeader, RecordKind},
NetworkAddress,
};

/// Number of chunks to upload in parallel.
/// Can be overridden by the `CHUNK_UPLOAD_BATCH_SIZE` environment variable.
pub static CHUNK_UPLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("CHUNK_UPLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("Chunk upload batch size: {}", batch_size);
batch_size
});

/// Number of retries to upload chunks.
pub const RETRY_ATTEMPTS: usize = 3;

/// Number of chunks to download in parallel.
/// Can be overridden by the `CHUNK_DOWNLOAD_BATCH_SIZE` environment variable.
pub static CHUNK_DOWNLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("CHUNK_DOWNLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("Chunk download batch size: {}", batch_size);
batch_size
});

/// Raw Data Address (points to a DataMap)
pub type DataAddr = XorName;
/// Raw Chunk Address (points to a [`Chunk`])
pub type ChunkAddr = XorName;

/// Errors that can occur during the put operation.
#[derive(Debug, thiserror::Error)]
pub enum PutError {
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("A network error occurred.")]
Network(#[from] NetworkError),
#[error("Error occurred during cost estimation.")]
CostError(#[from] CostError),
#[error("Error occurred during payment.")]
PayError(#[from] PayError),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("A wallet error occurred.")]
Wallet(#[from] ant_evm::EvmError),
#[error("The vault owner key does not match the client's public key")]
VaultBadOwner,
#[error("Payment unexpectedly invalid for {0:?}")]
PaymentUnexpectedlyInvalid(NetworkAddress),
}

/// Errors that can occur during the pay operation.
#[derive(Debug, thiserror::Error)]
pub enum PayError {
#[error("Wallet error: {0:?}")]
EvmWalletError(#[from] EvmWalletError),
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Cost error: {0:?}")]
Cost(#[from] CostError),
}

/// Errors that can occur during the get operation.
#[derive(Debug, thiserror::Error)]
pub enum GetError {
#[error("Could not deserialize data map.")]
InvalidDataMap(rmp_serde::decode::Error),
#[error("Failed to decrypt data.")]
Decryption(crate::self_encryption::Error),
#[error("Failed to deserialize")]
Deserialization(#[from] rmp_serde::decode::Error),
#[error("General networking error: {0:?}")]
Network(#[from] NetworkError),
#[error("General protocol error: {0:?}")]
Protocol(#[from] ant_protocol::Error),
}

/// Errors that can occur during the cost calculation.
#[derive(Debug, thiserror::Error)]
pub enum CostError {
#[error("Failed to self-encrypt data.")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Could not get store quote for: {0:?} after several retries")]
CouldNotGetStoreQuote(XorName),
#[error("Could not get store costs: {0:?}")]
CouldNotGetStoreCosts(NetworkError),
#[error("Failed to serialize {0}")]
Serialization(String),
}
use super::*;

impl Client {
/// Fetch a blob of data from the network
pub async fn data_get(&self, addr: DataAddr) -> Result<Bytes, GetError> {
pub async fn data_get_public(&self, addr: DataAddr) -> Result<Bytes, GetError> {
info!("Fetching data from Data Address: {addr:?}");
let data_map_chunk = self.chunk_get(addr).await?;
let data = self
@@ -141,7 +42,7 @@ impl Client {
/// Upload a piece of data to the network.
/// Returns the Data Address at which the data was stored.
/// This data is publicly accessible.
pub async fn data_put(
pub async fn data_put_public(
&self,
data: Bytes,
payment_option: PaymentOption,
131 changes: 0 additions & 131 deletions autonomi/src/client/data_private.rs

This file was deleted.

Original file line number Diff line number Diff line change
@@ -13,25 +13,62 @@ use std::{

use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH};

use super::{
archive::{Metadata, RenameError},
data::{GetError, PutError},
data_private::PrivateDataAccess,
use crate::{
client::{
data::{DataMapChunk, GetError, PutError},
payment::PaymentOption,
},
Client,
};
use crate::client::payment::PaymentOption;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use thiserror::Error;

/// The address of a private archive
/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data
pub type PrivateArchiveAccess = PrivateDataAccess;
/// Private archive data map, allowing access to the [`PrivateArchive`] data.
pub type PrivateArchiveAccess = DataMapChunk;

/// A private archive of files that containing file paths, their metadata and the files data maps
/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address.
#[derive(Error, Debug, PartialEq, Eq)]
pub enum RenameError {
#[error("File not found in archive: {0}")]
FileNotFound(PathBuf),
}

/// Metadata for a file in an archive. Time values are UNIX timestamps.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Metadata {
/// When the file was (last) uploaded to the network.
pub uploaded: u64,
/// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS.
pub created: u64,
/// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS.
pub modified: u64,
/// File size in bytes
pub size: u64,
}

impl Metadata {
/// Create a new metadata struct with the current time as uploaded, created and modified.
pub fn new_with_size(size: u64) -> Self {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::from_secs(0))
.as_secs();

Self {
uploaded: now,
created: now,
modified: now,
size,
}
}
}

/// Directory structure mapping filepaths to their data maps and metadata.
///
/// The data maps are stored within this structure instead of uploading them to the network, keeping the data private.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct PrivateArchive {
map: HashMap<PathBuf, (PrivateDataAccess, Metadata)>,
map: HashMap<PathBuf, (DataMapChunk, Metadata)>,
}

impl PrivateArchive {
@@ -62,7 +99,7 @@ impl PrivateArchive {

/// Add a file to a local archive
/// Note that this does not upload the archive to the network
pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) {
pub fn add_file(&mut self, path: PathBuf, data_map: DataMapChunk, meta: Metadata) {
self.map.insert(path.clone(), (data_map, meta));
debug!("Added a new file to the archive, path: {:?}", path);
}
@@ -76,23 +113,24 @@ impl PrivateArchive {
}

/// List all data addresses of the files in the archive
pub fn addresses(&self) -> Vec<PrivateDataAccess> {
pub fn addresses(&self) -> Vec<DataMapChunk> {
self.map
.values()
.map(|(data_map, _)| data_map.clone())
.collect()
}

/// Iterate over the archive items
/// Returns an iterator over (PathBuf, SecretDataMap, Metadata)
pub fn iter(&self) -> impl Iterator<Item = (&PathBuf, &PrivateDataAccess, &Metadata)> {
/// Iterate over the archive items.
///
/// Returns an iterator over ([`PathBuf`], [`DataMapChunk`], [`Metadata`])
pub fn iter(&self) -> impl Iterator<Item = (&PathBuf, &DataMapChunk, &Metadata)> {
self.map
.iter()
.map(|(path, (data_map, meta))| (path, data_map, meta))
}

/// Get the underlying map
pub fn map(&self) -> &HashMap<PathBuf, (PrivateDataAccess, Metadata)> {
pub fn map(&self) -> &HashMap<PathBuf, (DataMapChunk, Metadata)> {
&self.map
}

@@ -113,25 +151,25 @@ impl PrivateArchive {
}

impl Client {
/// Fetch a private archive from the network
pub async fn private_archive_get(
/// Fetch a [`PrivateArchive`] from the network
pub async fn archive_get(
&self,
addr: PrivateArchiveAccess,
) -> Result<PrivateArchive, GetError> {
let data = self.private_data_get(addr).await?;
let data = self.data_get(addr).await?;
Ok(PrivateArchive::from_bytes(data)?)
}

/// Upload a private archive to the network
pub async fn private_archive_put(
/// Upload a [`PrivateArchive`] to the network
pub async fn archive_put(
&self,
archive: PrivateArchive,
payment_option: PaymentOption,
) -> Result<PrivateArchiveAccess, PutError> {
let bytes = archive
.into_bytes()
.map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?;
let result = self.private_data_put(bytes, payment_option).await;
let result = self.data_put(bytes, payment_option).await;
debug!("Uploaded private archive {archive:?} to the network and address is {result:?}");
result
}
Original file line number Diff line number Diff line change
@@ -13,65 +13,31 @@ use std::{

use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH};

use super::{
data::{CostError, DataAddr, GetError, PutError},
Client,
};
use ant_evm::{AttoTokens, EvmWallet};
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use xor_name::XorName;

/// The address of an archive on the network. Points to an [`Archive`].
pub type ArchiveAddr = XorName;

use thiserror::Error;
use super::archive::Metadata;
use crate::{
client::{
data::{CostError, DataAddr, GetError, PutError},
files::archive::RenameError,
},
Client,
};

#[derive(Error, Debug, PartialEq, Eq)]
pub enum RenameError {
#[error("File not found in archive: {0}")]
FileNotFound(PathBuf),
}
/// The address of a public archive on the network. Points to an [`PublicArchive`].
pub type ArchiveAddr = XorName;

/// An archive of files that containing file paths, their metadata and the files data addresses
/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address.
/// Archives are public meaning anyone can read the data in the archive. For private archives use [`crate::client::archive_private::PrivateArchive`].
/// Public variant of [`crate::client::files::archive::PrivateArchive`]. Differs in that data maps of files are uploaded
/// to the network, of which the addresses are stored in this archive.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct Archive {
pub struct PublicArchive {
map: HashMap<PathBuf, (DataAddr, Metadata)>,
}

/// Metadata for a file in an archive. Time values are UNIX timestamps.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Metadata {
/// When the file was (last) uploaded to the network.
pub uploaded: u64,
/// File creation time on local file system. See [`std::fs::Metadata::created`] for details per OS.
pub created: u64,
/// Last file modification time taken from local file system. See [`std::fs::Metadata::modified`] for details per OS.
pub modified: u64,
/// File size in bytes
pub size: u64,
}

impl Metadata {
/// Create a new metadata struct with the current time as uploaded, created and modified.
pub fn new_with_size(size: u64) -> Self {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or(Duration::from_secs(0))
.as_secs();

Self {
uploaded: now,
created: now,
modified: now,
size,
}
}
}

impl Archive {
impl PublicArchive {
/// Create a new emtpy local archive
/// Note that this does not upload the archive to the network
pub fn new() -> Self {
@@ -131,8 +97,8 @@ impl Archive {
}

/// Deserialize from bytes.
pub fn from_bytes(data: Bytes) -> Result<Archive, rmp_serde::decode::Error> {
let root: Archive = rmp_serde::from_slice(&data[..])?;
pub fn from_bytes(data: Bytes) -> Result<PublicArchive, rmp_serde::decode::Error> {
let root: PublicArchive = rmp_serde::from_slice(&data[..])?;

Ok(root)
}
@@ -157,13 +123,13 @@ impl Client {
/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?];
/// let client = Client::connect(&peers).await?;
/// let archive = client.archive_get(ArchiveAddr::random(&mut rand::thread_rng())).await?;
/// let archive = client.archive_get_public(ArchiveAddr::random(&mut rand::thread_rng())).await?;
/// # Ok(())
/// # }
/// ```
pub async fn archive_get(&self, addr: ArchiveAddr) -> Result<Archive, GetError> {
let data = self.data_get(addr).await?;
Ok(Archive::from_bytes(data)?)
pub async fn archive_get_public(&self, addr: ArchiveAddr) -> Result<PublicArchive, GetError> {
let data = self.data_get_public(addr).await?;
Ok(PublicArchive::from_bytes(data)?)
}

/// Upload an archive to the network
@@ -173,34 +139,34 @@ impl Client {
/// Create simple archive containing `file.txt` pointing to random XOR name.
///
/// ```no_run
/// # use autonomi::client::{Client, data::DataAddr, archive::{Archive, ArchiveAddr, Metadata}};
/// # use autonomi::client::{Client, data::DataAddr, archive::{PublicArchive, ArchiveAddr, Metadata}};
/// # use std::path::PathBuf;
/// # #[tokio::main]
/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// # let peers = ["/ip4/127.0.0.1/udp/1234/quic-v1".parse()?];
/// # let client = Client::connect(&peers).await?;
/// # let wallet = todo!();
/// let mut archive = Archive::new();
/// let mut archive = PublicArchive::new();
/// archive.add_file(PathBuf::from("file.txt"), DataAddr::random(&mut rand::thread_rng()), Metadata::new_with_size(0));
/// let address = client.archive_put(archive, &wallet).await?;
/// let address = client.archive_put_public(archive, &wallet).await?;
/// # Ok(())
/// # }
/// ```
pub async fn archive_put(
pub async fn archive_put_public(
&self,
archive: Archive,
archive: PublicArchive,
wallet: &EvmWallet,
) -> Result<ArchiveAddr, PutError> {
let bytes = archive
.into_bytes()
.map_err(|e| PutError::Serialization(format!("Failed to serialize archive: {e:?}")))?;
let result = self.data_put(bytes, wallet.into()).await;
let result = self.data_put_public(bytes, wallet.into()).await;
debug!("Uploaded archive {archive:?} to the network and the address is {result:?}");
result
}

/// Get the cost to upload an archive
pub async fn archive_cost(&self, archive: Archive) -> Result<AttoTokens, CostError> {
pub async fn archive_cost(&self, archive: PublicArchive) -> Result<AttoTokens, CostError> {
let bytes = archive
.into_bytes()
.map_err(|e| CostError::Serialization(format!("Failed to serialize archive: {e:?}")))?;
110 changes: 77 additions & 33 deletions autonomi/src/client/fs_private.rs → autonomi/src/client/files/fs.rs
Original file line number Diff line number Diff line change
@@ -14,26 +14,81 @@
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use crate::client::data::{CostError, DataMapChunk, GetError, PutError};
use crate::client::utils::process_tasks_with_max_concurrency;
use crate::client::Client;
use ant_evm::EvmWallet;
use bytes::Bytes;
use std::path::PathBuf;
use std::{path::PathBuf, sync::LazyLock};

use super::archive_private::{PrivateArchive, PrivateArchiveAccess};
use super::data_private::PrivateDataAccess;
use super::fs::{DownloadError, UploadError};
use super::archive::{PrivateArchive, PrivateArchiveAccess};

use super::fs::FILE_UPLOAD_BATCH_SIZE;
/// Number of files to upload in parallel.
///
/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable.
pub static FILE_UPLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("File upload batch size: {}", batch_size);
batch_size
});

/// Errors that can occur during the file upload operation.
#[derive(Debug, thiserror::Error)]
pub enum UploadError {
#[error("Failed to recursively traverse directory")]
WalkDir(#[from] walkdir::Error),
#[error("Input/output failure")]
IoError(#[from] std::io::Error),
#[error("Failed to upload file")]
PutError(#[from] PutError),
#[error("Failed to fetch file")]
GetError(#[from] GetError),
#[error("Failed to serialize")]
Serialization(#[from] rmp_serde::encode::Error),
#[error("Failed to deserialize")]
Deserialization(#[from] rmp_serde::decode::Error),
}

/// Errors that can occur during the download operation.
#[derive(Debug, thiserror::Error)]
pub enum DownloadError {
#[error("Failed to download file")]
GetError(#[from] GetError),
#[error("IO failure")]
IoError(#[from] std::io::Error),
}

/// Errors that can occur during the file cost calculation.
#[derive(Debug, thiserror::Error)]
pub enum FileCostError {
#[error("Cost error: {0}")]
Cost(#[from] CostError),
#[error("IO failure")]
IoError(#[from] std::io::Error),
#[error("Serialization error")]
Serialization(#[from] rmp_serde::encode::Error),
#[error("Self encryption error")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Walkdir error")]
WalkDir(#[from] walkdir::Error),
}

impl Client {
/// Download a private file from network to local file system
pub async fn private_file_download(
pub async fn file_download(
&self,
data_access: PrivateDataAccess,
data_access: DataMapChunk,
to_dest: PathBuf,
) -> Result<(), DownloadError> {
let data = self.private_data_get(data_access).await?;
let data = self.data_get(data_access).await?;
if let Some(parent) = to_dest.parent() {
tokio::fs::create_dir_all(parent).await?;
debug!("Created parent directories for {to_dest:?}");
@@ -44,27 +99,26 @@ impl Client {
}

/// Download a private directory from network to local file system
pub async fn private_dir_download(
pub async fn dir_download(
&self,
archive_access: PrivateArchiveAccess,
to_dest: PathBuf,
) -> Result<(), DownloadError> {
let archive = self.private_archive_get(archive_access).await?;
let archive = self.archive_get(archive_access).await?;
for (path, addr, _meta) in archive.iter() {
self.private_file_download(addr.clone(), to_dest.join(path))
.await?;
self.file_download(addr.clone(), to_dest.join(path)).await?;
}
debug!("Downloaded directory to {to_dest:?}");
Ok(())
}

/// Upload a private directory to the network. The directory is recursively walked.
/// Reads all files, splits into chunks, uploads chunks, uploads private archive, returns [`PrivateArchiveAccess`] (pointing to the private archive)
pub async fn private_dir_upload(
/// Upload a directory to the network. The directory is recursively walked and each file is uploaded to the network.
/// The data maps of these (private) files are not uploaded but returned within the [`PrivateArchive`] return type.
pub async fn dir_upload(
&self,
dir_path: PathBuf,
wallet: &EvmWallet,
) -> Result<PrivateArchiveAccess, UploadError> {
) -> Result<PrivateArchive, UploadError> {
info!("Uploading directory as private: {dir_path:?}");
let start = tokio::time::Instant::now();

@@ -76,10 +130,10 @@ impl Client {
continue;
}

let metadata = super::fs::metadata_from_entry(&entry);
let metadata = super::fs_public::metadata_from_entry(&entry);
let path = entry.path().to_path_buf();
upload_tasks.push(async move {
let file = self.private_file_upload(path.clone(), wallet).await;
let file = self.file_upload(path.clone(), wallet).await;
(path, metadata, file)
});
}
@@ -103,35 +157,25 @@ impl Client {
}
}

// upload archive
let archive_serialized = archive.into_bytes()?;
let arch_addr = self
.private_data_put(archive_serialized, wallet.into())
.await?;

info!(
"Complete private archive upload completed in {:?}",
start.elapsed()
);
#[cfg(feature = "loud")]
println!("Upload completed in {:?}", start.elapsed());
Ok(arch_addr)
Ok(archive)
}

/// Upload a private file to the network.
/// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap)
async fn private_file_upload(
/// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap)
async fn file_upload(
&self,
path: PathBuf,
wallet: &EvmWallet,
) -> Result<PrivateDataAccess, UploadError> {
) -> Result<DataMapChunk, UploadError> {
info!("Uploading file: {path:?}");
#[cfg(feature = "loud")]
println!("Uploading file: {path:?}");

let data = tokio::fs::read(path).await?;
let data = Bytes::from(data);
let addr = self.private_data_put(data, wallet.into()).await?;
let addr = self.data_put(data, wallet.into()).await?;
debug!("Uploaded file successfully in the privateAchive: {addr:?}");
Ok(addr)
}
Original file line number Diff line number Diff line change
@@ -6,84 +6,26 @@
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use crate::client::archive::Metadata;
use crate::client::data::CostError;
use crate::client::data::DataAddr;
use crate::client::files::archive::Metadata;
use crate::client::utils::process_tasks_with_max_concurrency;
use crate::client::Client;
use ant_evm::EvmWallet;
use ant_networking::target_arch::{Duration, SystemTime};
use bytes::Bytes;
use std::path::PathBuf;
use std::sync::LazyLock;

use super::archive::{Archive, ArchiveAddr};
use super::data::{DataAddr, GetError, PutError};

/// Number of files to upload in parallel.
/// Can be overridden by the `FILE_UPLOAD_BATCH_SIZE` environment variable.
pub static FILE_UPLOAD_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
let batch_size = std::env::var("FILE_UPLOAD_BATCH_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
* 8,
);
info!("File upload batch size: {}", batch_size);
batch_size
});

/// Errors that can occur during the file upload operation.
#[derive(Debug, thiserror::Error)]
pub enum UploadError {
#[error("Failed to recursively traverse directory")]
WalkDir(#[from] walkdir::Error),
#[error("Input/output failure")]
IoError(#[from] std::io::Error),
#[error("Failed to upload file")]
PutError(#[from] PutError),
#[error("Failed to fetch file")]
GetError(#[from] GetError),
#[error("Failed to serialize")]
Serialization(#[from] rmp_serde::encode::Error),
#[error("Failed to deserialize")]
Deserialization(#[from] rmp_serde::decode::Error),
}

/// Errors that can occur during the download operation.
#[derive(Debug, thiserror::Error)]
pub enum DownloadError {
#[error("Failed to download file")]
GetError(#[from] GetError),
#[error("IO failure")]
IoError(#[from] std::io::Error),
}

/// Errors that can occur during the file cost calculation.
#[derive(Debug, thiserror::Error)]
pub enum FileCostError {
#[error("Cost error: {0}")]
Cost(#[from] CostError),
#[error("IO failure")]
IoError(#[from] std::io::Error),
#[error("Serialization error")]
Serialization(#[from] rmp_serde::encode::Error),
#[error("Self encryption error")]
SelfEncryption(#[from] crate::self_encryption::Error),
#[error("Walkdir error")]
WalkDir(#[from] walkdir::Error),
}
use super::archive_public::{ArchiveAddr, PublicArchive};
use super::fs::*;

impl Client {
/// Download file from network to local file system
pub async fn file_download(
pub async fn file_download_public(
&self,
data_addr: DataAddr,
to_dest: PathBuf,
) -> Result<(), DownloadError> {
let data = self.data_get(data_addr).await?;
let data = self.data_get_public(data_addr).await?;
if let Some(parent) = to_dest.parent() {
tokio::fs::create_dir_all(parent).await?;
debug!("Created parent directories {parent:?} for {to_dest:?}");
@@ -94,15 +36,15 @@ impl Client {
}

/// Download directory from network to local file system
pub async fn dir_download(
pub async fn dir_download_public(
&self,
archive_addr: ArchiveAddr,
to_dest: PathBuf,
) -> Result<(), DownloadError> {
let archive = self.archive_get(archive_addr).await?;
let archive = self.archive_get_public(archive_addr).await?;
debug!("Downloaded archive for the directory from the network at {archive_addr:?}");
for (path, addr, _meta) in archive.iter() {
self.file_download(*addr, to_dest.join(path)).await?;
self.file_download_public(*addr, to_dest.join(path)).await?;
}
debug!(
"All files in the directory downloaded to {:?} from the network address {:?}",
@@ -114,7 +56,7 @@ impl Client {

/// Upload a directory to the network. The directory is recursively walked.
/// Reads all files, splits into chunks, uploads chunks, uploads datamaps, uploads archive, returns ArchiveAddr (pointing to the archive)
pub async fn dir_upload(
pub async fn dir_upload_public(
&self,
dir_path: PathBuf,
wallet: &EvmWallet,
@@ -133,7 +75,7 @@ impl Client {
let metadata = metadata_from_entry(&entry);
let path = entry.path().to_path_buf();
upload_tasks.push(async move {
let file = self.file_upload(path.clone(), wallet).await;
let file = self.file_upload_public(path.clone(), wallet).await;
(path, metadata, file)
});
}
@@ -146,7 +88,7 @@ impl Client {
uploads.len(),
start.elapsed()
);
let mut archive = Archive::new();
let mut archive = PublicArchive::new();
for (path, metadata, maybe_file) in uploads.into_iter() {
match maybe_file {
Ok(file) => archive.add_file(path, file, metadata),
@@ -159,7 +101,9 @@ impl Client {

// upload archive
let archive_serialized = archive.into_bytes()?;
let arch_addr = self.data_put(archive_serialized, wallet.into()).await?;
let arch_addr = self
.data_put_public(archive_serialized, wallet.into())
.await?;

info!("Complete archive upload completed in {:?}", start.elapsed());
#[cfg(feature = "loud")]
@@ -170,7 +114,7 @@ impl Client {

/// Upload a file to the network.
/// Reads file, splits into chunks, uploads chunks, uploads datamap, returns DataAddr (pointing to the datamap)
async fn file_upload(
async fn file_upload_public(
&self,
path: PathBuf,
wallet: &EvmWallet,
@@ -181,15 +125,15 @@ impl Client {

let data = tokio::fs::read(path.clone()).await?;
let data = Bytes::from(data);
let addr = self.data_put(data, wallet.into()).await?;
let addr = self.data_put_public(data, wallet.into()).await?;
debug!("File {path:?} uploaded to the network at {addr:?}");
Ok(addr)
}

/// Get the cost to upload a file/dir to the network.
/// quick and dirty implementation, please refactor once files are cleanly implemented
pub async fn file_cost(&self, path: &PathBuf) -> Result<ant_evm::AttoTokens, FileCostError> {
let mut archive = Archive::new();
let mut archive = PublicArchive::new();
let mut total_cost = ant_evm::Amount::ZERO;

for entry in walkdir::WalkDir::new(path) {
8 changes: 8 additions & 0 deletions autonomi/src/client/files/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
pub mod archive;
pub mod archive_public;
#[cfg(feature = "fs")]
#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
pub mod fs;
#[cfg(feature = "fs")]
#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
pub mod fs_public;
10 changes: 1 addition & 9 deletions autonomi/src/client/mod.rs
Original file line number Diff line number Diff line change
@@ -12,19 +12,11 @@
pub mod address;
pub mod payment;

pub mod archive;
pub mod archive_private;
pub mod data;
pub mod data_private;
#[cfg(feature = "external-signer")]
#[cfg_attr(docsrs, doc(cfg(feature = "external-signer")))]
pub mod external_signer;
#[cfg(feature = "fs")]
#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
pub mod fs;
#[cfg(feature = "fs")]
#[cfg_attr(docsrs, doc(cfg(feature = "fs")))]
pub mod fs_private;
pub mod files;
#[cfg(feature = "registers")]
#[cfg_attr(docsrs, doc(cfg(feature = "registers")))]
pub mod registers;
4 changes: 2 additions & 2 deletions autonomi/src/client/vault/user_data.rs
Original file line number Diff line number Diff line change
@@ -8,10 +8,10 @@

use std::collections::HashMap;

use crate::client::archive::ArchiveAddr;
use crate::client::archive_private::PrivateArchiveAccess;
use crate::client::data::GetError;
use crate::client::data::PutError;
use crate::client::files::archive::PrivateArchiveAccess;
use crate::client::files::archive_public::ArchiveAddr;
use crate::client::payment::PaymentOption;
use crate::client::registers::RegisterAddress;
use crate::client::vault::VaultError;
49 changes: 25 additions & 24 deletions autonomi/src/client/wasm.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use super::address::{addr_to_str, str_to_addr};
#[cfg(feature = "vault")]
use super::vault::UserData;
use crate::client::data_private::PrivateDataAccess;
use crate::client::data::DataMapChunk;
use crate::client::payment::Receipt;
use ant_protocol::storage::Chunk;
use libp2p::Multiaddr;
@@ -100,22 +100,22 @@ impl JsClient {
#[wasm_bindgen(js_name = putData)]
pub async fn put_data(&self, data: Vec<u8>, wallet: &JsWallet) -> Result<String, JsError> {
let data = crate::Bytes::from(data);
let xorname = self.0.data_put(data, (&wallet.0).into()).await?;
let xorname = self.0.data_put_public(data, (&wallet.0).into()).await?;

Ok(addr_to_str(xorname))
}

/// Upload private data to the network.
///
/// Returns the `PrivateDataAccess` chunk of the data.
/// Returns the `DataMapChunk` chunk of the data.
#[wasm_bindgen(js_name = putPrivateData)]
pub async fn put_private_data(
&self,
data: Vec<u8>,
wallet: &JsWallet,
) -> Result<JsValue, JsError> {
let data = crate::Bytes::from(data);
let private_data_access = self.0.private_data_put(data, (&wallet.0).into()).await?;
let private_data_access = self.0.data_put(data, (&wallet.0).into()).await?;
let js_value = serde_wasm_bindgen::to_value(&private_data_access)?;

Ok(js_value)
@@ -124,7 +124,7 @@ impl JsClient {
/// Upload private data to the network.
/// Uses a `Receipt` as payment.
///
/// Returns the `PrivateDataAccess` chunk of the data.
/// Returns the `DataMapChunk` chunk of the data.
#[wasm_bindgen(js_name = putPrivateDataWithReceipt)]
pub async fn put_private_data_with_receipt(
&self,
@@ -133,7 +133,7 @@ impl JsClient {
) -> Result<JsValue, JsError> {
let data = crate::Bytes::from(data);
let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?;
let private_data_access = self.0.private_data_put(data, receipt.into()).await?;
let private_data_access = self.0.data_put(data, receipt.into()).await?;
let js_value = serde_wasm_bindgen::to_value(&private_data_access)?;

Ok(js_value)
@@ -143,17 +143,17 @@ impl JsClient {
#[wasm_bindgen(js_name = getData)]
pub async fn get_data(&self, addr: String) -> Result<Vec<u8>, JsError> {
let addr = str_to_addr(&addr)?;
let data = self.0.data_get(addr).await?;
let data = self.0.data_get_public(addr).await?;

Ok(data.to_vec())
}

/// Fetch the data from the network.
#[wasm_bindgen(js_name = getPrivateData)]
pub async fn get_private_data(&self, private_data_access: JsValue) -> Result<Vec<u8>, JsError> {
let private_data_access: PrivateDataAccess =
let private_data_access: DataMapChunk =
serde_wasm_bindgen::from_value(private_data_access)?;
let data = self.0.private_data_get(private_data_access).await?;
let data = self.0.data_get(private_data_access).await?;

Ok(data.to_vec())
}
@@ -171,15 +171,14 @@ impl JsClient {
mod archive {
use super::*;
use crate::client::{
address::str_to_addr,
archive::{Archive, Metadata},
address::str_to_addr, files::archive::Metadata, files::archive_public::PublicArchive,
};
use std::path::PathBuf;
use wasm_bindgen::JsError;

/// Structure mapping paths to data addresses.
#[wasm_bindgen(js_name = Archive)]
pub struct JsArchive(Archive);
pub struct JsArchive(PublicArchive);

/// Create new metadata with the current time as uploaded, created and modified.
///
@@ -201,7 +200,7 @@ mod archive {
/// Create a new archive.
#[wasm_bindgen(constructor)]
pub fn new() -> Self {
Self(Archive::new())
Self(PublicArchive::new())
}

/// Add a new file to the archive.
@@ -249,7 +248,7 @@ mod archive {
#[wasm_bindgen(js_name = getArchive)]
pub async fn get_archive(&self, addr: String) -> Result<JsArchive, JsError> {
let addr = str_to_addr(&addr)?;
let archive = self.0.archive_get(addr).await?;
let archive = self.0.archive_get_public(addr).await?;
let archive = JsArchive(archive);

Ok(archive)
@@ -264,7 +263,10 @@ mod archive {
archive: &JsArchive,
wallet: &JsWallet,
) -> Result<String, JsError> {
let addr = self.0.archive_put(archive.0.clone(), &wallet.0).await?;
let addr = self
.0
.archive_put_public(archive.0.clone(), &wallet.0)
.await?;

Ok(addr_to_str(addr))
}
@@ -273,9 +275,8 @@ mod archive {

mod archive_private {
use super::*;
use crate::client::archive::Metadata;
use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess};
use crate::client::data_private::PrivateDataAccess;
use crate::client::data::DataMapChunk;
use crate::client::files::archive::{Metadata, PrivateArchive, PrivateArchiveAccess};
use crate::client::payment::Receipt;
use std::path::PathBuf;
use wasm_bindgen::{JsError, JsValue};
@@ -301,7 +302,7 @@ mod archive_private {
metadata: JsValue,
) -> Result<(), JsError> {
let path = PathBuf::from(path);
let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?;
let data_map: DataMapChunk = serde_wasm_bindgen::from_value(data_map)?;
let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?;
self.0.add_file(path, data_map, metadata);

@@ -332,7 +333,7 @@ mod archive_private {
) -> Result<JsPrivateArchive, JsError> {
let private_archive_access: PrivateArchiveAccess =
serde_wasm_bindgen::from_value(private_archive_access)?;
let archive = self.0.private_archive_get(private_archive_access).await?;
let archive = self.0.archive_get(private_archive_access).await?;
let archive = JsPrivateArchive(archive);

Ok(archive)
@@ -349,7 +350,7 @@ mod archive_private {
) -> Result<JsValue, JsError> {
let private_archive_access = self
.0
.private_archive_put(archive.0.clone(), (&wallet.0).into())
.archive_put(archive.0.clone(), (&wallet.0).into())
.await?;

let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?;
@@ -371,7 +372,7 @@ mod archive_private {

let private_archive_access = self
.0
.private_archive_put(archive.0.clone(), receipt.into())
.archive_put(archive.0.clone(), receipt.into())
.await?;

let js_value = serde_wasm_bindgen::to_value(&private_archive_access)?;
@@ -385,7 +386,7 @@ mod archive_private {
mod vault {
use super::*;
use crate::client::address::addr_to_str;
use crate::client::archive_private::PrivateArchiveAccess;
use crate::client::files::archive::PrivateArchiveAccess;
use crate::client::payment::Receipt;
use crate::client::vault::key::blst_to_blsttc;
use crate::client::vault::key::derive_secret_key_from_seed;
@@ -691,7 +692,7 @@ mod external_signer {
) -> Result<String, JsError> {
let data = crate::Bytes::from(data);
let receipt: Receipt = serde_wasm_bindgen::from_value(receipt)?;
let xorname = self.0.data_put(data, receipt.into()).await?;
let xorname = self.0.data_put_public(data, receipt.into()).await?;
Ok(addr_to_str(xorname))
}
}
10 changes: 5 additions & 5 deletions autonomi/src/lib.rs
Original file line number Diff line number Diff line change
@@ -22,12 +22,12 @@
//! let wallet = Wallet::new_from_private_key(Default::default(), key)?;
//!
//! // Put and fetch data.
//! let data_addr = client.data_put(Bytes::from("Hello, World"), (&wallet).into()).await?;
//! let _data_fetched = client.data_get(data_addr).await?;
//! let data_addr = client.data_put_public(Bytes::from("Hello, World"), (&wallet).into()).await?;
//! let _data_fetched = client.data_get_public(data_addr).await?;
//!
//! // Put and fetch directory from local file system.
//! let dir_addr = client.dir_upload("files/to/upload".into(), &wallet).await?;
//! client.dir_download(dir_addr, "files/downloaded".into()).await?;
//! let dir_addr = client.dir_upload_public("files/to/upload".into(), &wallet).await?;
//! client.dir_download_public(dir_addr, "files/downloaded".into()).await?;
//!
//! Ok(())
//! }
@@ -79,7 +79,7 @@ pub use bytes::Bytes;
pub use libp2p::Multiaddr;

#[doc(inline)]
pub use client::Client;
pub use client::{files::archive::PrivateArchive, Client};

#[cfg(feature = "extension-module")]
mod python;
39 changes: 17 additions & 22 deletions autonomi/src/python.rs
Original file line number Diff line number Diff line change
@@ -2,9 +2,8 @@
#![allow(non_local_definitions)]

use crate::client::{
archive::ArchiveAddr,
archive_private::PrivateArchiveAccess,
data_private::PrivateDataAccess,
data::DataMapChunk,
files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr},
payment::PaymentOption as RustPaymentOption,
vault::{UserData, VaultSecretKey},
Client as RustClient,
@@ -39,40 +38,36 @@ impl PyClient {
Ok(Self { inner: client })
}

fn private_data_put(
&self,
data: Vec<u8>,
payment: &PyPaymentOption,
) -> PyResult<PyPrivateDataAccess> {
fn data_put(&self, data: Vec<u8>, payment: &PyPaymentOption) -> PyResult<PyDataMapChunk> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let access = rt
.block_on(
self.inner
.private_data_put(Bytes::from(data), payment.inner.clone()),
.data_put(Bytes::from(data), payment.inner.clone()),
)
.map_err(|e| {
pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}"))
})?;

Ok(PyPrivateDataAccess { inner: access })
Ok(PyDataMapChunk { inner: access })
}

fn private_data_get(&self, access: &PyPrivateDataAccess) -> PyResult<Vec<u8>> {
fn data_get(&self, access: &PyDataMapChunk) -> PyResult<Vec<u8>> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let data = rt
.block_on(self.inner.private_data_get(access.inner.clone()))
.block_on(self.inner.data_get(access.inner.clone()))
.map_err(|e| {
pyo3::exceptions::PyValueError::new_err(format!("Failed to get private data: {e}"))
})?;
Ok(data.to_vec())
}

fn data_put(&self, data: Vec<u8>, payment: &PyPaymentOption) -> PyResult<String> {
fn data_put_public(&self, data: Vec<u8>, payment: &PyPaymentOption) -> PyResult<String> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let addr = rt
.block_on(
self.inner
.data_put(bytes::Bytes::from(data), payment.inner.clone()),
.data_put_public(bytes::Bytes::from(data), payment.inner.clone()),
)
.map_err(|e| {
pyo3::exceptions::PyValueError::new_err(format!("Failed to put data: {e}"))
@@ -81,13 +76,13 @@ impl PyClient {
Ok(crate::client::address::addr_to_str(addr))
}

fn data_get(&self, addr: &str) -> PyResult<Vec<u8>> {
fn data_get_public(&self, addr: &str) -> PyResult<Vec<u8>> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let addr = crate::client::address::str_to_addr(addr).map_err(|e| {
pyo3::exceptions::PyValueError::new_err(format!("Invalid address: {e}"))
})?;

let data = rt.block_on(self.inner.data_get(addr)).map_err(|e| {
let data = rt.block_on(self.inner.data_get_public(addr)).map_err(|e| {
pyo3::exceptions::PyValueError::new_err(format!("Failed to get data: {e}"))
})?;

@@ -299,17 +294,17 @@ impl PyUserData {
}
}

#[pyclass(name = "PrivateDataAccess")]
#[pyclass(name = "DataMapChunk")]
#[derive(Clone)]
pub(crate) struct PyPrivateDataAccess {
inner: PrivateDataAccess,
pub(crate) struct PyDataMapChunk {
inner: DataMapChunk,
}

#[pymethods]
impl PyPrivateDataAccess {
impl PyDataMapChunk {
#[staticmethod]
fn from_hex(hex: &str) -> PyResult<Self> {
PrivateDataAccess::from_hex(hex)
DataMapChunk::from_hex(hex)
.map(|access| Self { inner: access })
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex: {e}")))
}
@@ -347,7 +342,7 @@ fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_class::<PyPaymentOption>()?;
m.add_class::<PyVaultSecretKey>()?;
m.add_class::<PyUserData>()?;
m.add_class::<PyPrivateDataAccess>()?;
m.add_class::<PyDataMapChunk>()?;
m.add_function(wrap_pyfunction!(encrypt, m)?)?;
Ok(())
}
19 changes: 5 additions & 14 deletions autonomi/tests/external_signer.rs
Original file line number Diff line number Diff line change
@@ -4,9 +4,8 @@ use alloy::network::TransactionBuilder;
use alloy::providers::Provider;
use ant_evm::{QuoteHash, TxHash};
use ant_logging::LogBuilder;
use autonomi::client::archive::Metadata;
use autonomi::client::archive_private::PrivateArchive;
use autonomi::client::external_signer::encrypt_data;
use autonomi::client::files::archive::{Metadata, PrivateArchive};
use autonomi::client::payment::Receipt;
use autonomi::client::vault::user_data::USER_DATA_VAULT_CONTENT_IDENTIFIER;
use autonomi::client::vault::VaultSecretKey;
@@ -111,9 +110,7 @@ async fn external_signer_put() -> eyre::Result<()> {

sleep(Duration::from_secs(5)).await;

let private_data_access = client
.private_data_put(data.clone(), receipt.into())
.await?;
let private_data_access = client.data_put(data.clone(), receipt.into()).await?;

let mut private_archive = PrivateArchive::new();
private_archive.add_file(
@@ -128,9 +125,7 @@ async fn external_signer_put() -> eyre::Result<()> {

sleep(Duration::from_secs(5)).await;

let private_archive_access = client
.private_archive_put(private_archive, receipt.into())
.await?;
let private_archive_access = client.archive_put(private_archive, receipt.into()).await?;

let vault_key = VaultSecretKey::random();

@@ -174,19 +169,15 @@ async fn external_signer_put() -> eyre::Result<()> {
.expect("No private archive present in the UserData")
.clone();

let fetched_private_archive = client
.private_archive_get(fetched_private_archive_access)
.await?;
let fetched_private_archive = client.archive_get(fetched_private_archive_access).await?;

let (_, (fetched_private_file_access, _)) = fetched_private_archive
.map()
.iter()
.next()
.expect("No file present in private archive");

let fetched_private_file = client
.private_data_get(fetched_private_file_access.clone())
.await?;
let fetched_private_file = client.data_get(fetched_private_file_access.clone()).await?;

assert_eq!(
fetched_private_file, data,
11 changes: 6 additions & 5 deletions autonomi/tests/fs.rs
Original file line number Diff line number Diff line change
@@ -30,13 +30,13 @@ async fn dir_upload_download() -> Result<()> {
let wallet = get_funded_wallet();

let addr = client
.dir_upload("tests/file/test_dir".into(), &wallet)
.dir_upload_public("tests/file/test_dir".into(), &wallet)
.await?;

sleep(Duration::from_secs(10)).await;

client
.dir_download(addr, "tests/file/test_dir_fetched".into())
.dir_download_public(addr, "tests/file/test_dir_fetched".into())
.await?;

// compare the two directories
@@ -86,11 +86,11 @@ async fn file_into_vault() -> Result<()> {
let client_sk = bls::SecretKey::random();

let addr = client
.dir_upload("tests/file/test_dir".into(), &wallet)
.dir_upload_public("tests/file/test_dir".into(), &wallet)
.await?;
sleep(Duration::from_secs(2)).await;

let archive = client.archive_get(addr).await?;
let archive = client.archive_get_public(addr).await?;
let set_version = 0;
client
.write_bytes_to_vault(
@@ -106,7 +106,8 @@ async fn file_into_vault() -> Result<()> {

let (ap, got_version) = new_client.fetch_and_decrypt_vault(&client_sk).await?;
assert_eq!(set_version, got_version);
let ap_archive_fetched = autonomi::client::archive::Archive::from_bytes(ap)?;
let ap_archive_fetched =
autonomi::client::files::archive_public::PublicArchive::from_bytes(ap)?;

assert_eq!(
archive, ap_archive_fetched,
4 changes: 2 additions & 2 deletions autonomi/tests/put.rs
Original file line number Diff line number Diff line change
@@ -21,11 +21,11 @@ async fn put() -> Result<()> {
let wallet = get_funded_wallet();
let data = gen_random_data(1024 * 1024 * 10);

let addr = client.data_put(data.clone(), wallet.into()).await?;
let addr = client.data_put_public(data.clone(), wallet.into()).await?;

sleep(Duration::from_secs(10)).await;

let data_fetched = client.data_get(addr).await?;
let data_fetched = client.data_get_public(addr).await?;
assert_eq!(data, data_fetched, "data fetched should match data put");

Ok(())
4 changes: 2 additions & 2 deletions autonomi/tests/wasm.rs
Original file line number Diff line number Diff line change
@@ -25,11 +25,11 @@ async fn put() -> Result<(), Box<dyn std::error::Error>> {
let wallet = get_funded_wallet();
let data = gen_random_data(1024 * 1024 * 10);

let addr = client.data_put(data.clone(), wallet.into()).await?;
let addr = client.data_put_public(data.clone(), wallet.into()).await?;

sleep(Duration::from_secs(10)).await;

let data_fetched = client.data_get(addr).await?;
let data_fetched = client.data_get_public(addr).await?;
assert_eq!(data, data_fetched, "data fetched should match data put");

Ok(())