From 5b0102590685951e5dc4cdc96aa1ce3e75cf1877 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sat, 27 Nov 2021 18:26:11 -0900 Subject: [PATCH 01/12] parallelize decoding metadata --- Cargo.lock | 1 + Cargo.toml | 1 + src/decode.rs | 49 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c355598..69b9769e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1555,6 +1555,7 @@ dependencies = [ "bs58 0.4.0", "glob", "metaplex-token-metadata", + "rayon", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 01ead909..27de1eb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ borsh = "0.9.1" bs58 = "0.4.0" glob = "0.3.0" metaplex-token-metadata = "0.0.1" +rayon = "1.5.1" reqwest = "0.11.5" serde = "1.0.130" serde_json = "1.0.68" diff --git a/src/decode.rs b/src/decode.rs index 54a1c172..42cb2640 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result as AnyResult}; use metaplex_token_metadata::state::{Key, Metadata}; +use rayon::prelude::*; use serde::Serialize; use serde_json::{json, Value}; use solana_client::rpc_client::RpcClient; @@ -27,23 +28,57 @@ pub fn decode_metadata_all( let file = File::open(json_file)?; let mint_accounts: Vec = serde_json::from_reader(file)?; - for mint_account in &mint_accounts { + mint_accounts.par_iter().for_each(|mint_account| { let metadata = match decode(client, mint_account) { Ok(m) => m, Err(err) => match err { DecodeError::MissingAccount(account) => { println!("No account data found for mint account: {}!", account); - continue; + return; + } + err => { + println!( + "Failed to decode metadata for mint account: {}, error: {}", + mint_account, err + ); + return; } - _ => return Err(anyhow!(err)), }, }; - let json_metadata = decode_to_json(metadata)?; + let json_metadata = match decode_to_json(metadata) { + Ok(j) => j, + Err(err) => { + println!( + "Failed to decode metadata to JSON for mint account: {}, error: {}", + mint_account, err + ); + return; + } + }; - let mut file = File::create(format!("{}/{}.json", output, mint_account))?; - serde_json::to_writer(&mut file, &json_metadata)?; - } + let mut file = match File::create(format!("{}/{}.json", output, mint_account)) { + Ok(f) => f, + Err(err) => { + println!( + "Failed to create JSON file for mint account: {}, error: {}", + mint_account, err + ); + return; + } + }; + + match serde_json::to_writer(&mut file, &json_metadata) { + Ok(_) => (), + Err(err) => { + println!( + "Failed to write JSON file for mint account: {}, error: {}", + mint_account, err + ); + return; + } + } + }); Ok(()) } From 63a1355563590e2aa846aa90157075847b23fc95 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 10:57:55 -0900 Subject: [PATCH 02/12] parallelize minting --- Cargo.lock | 1 + Cargo.toml | 1 + src/mint.rs | 35 ++++++++++++++++++++--------------- src/process_subcommands.rs | 2 +- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69b9769e..3f97ffd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1555,6 +1555,7 @@ dependencies = [ "bs58 0.4.0", "glob", "metaplex-token-metadata", + "num_cpus", "rayon", "reqwest", "serde", diff --git a/Cargo.toml b/Cargo.toml index 27de1eb0..8b1a2bfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ borsh = "0.9.1" bs58 = "0.4.0" glob = "0.3.0" metaplex-token-metadata = "0.0.1" +num_cpus = "1.13.0" rayon = "1.5.1" reqwest = "0.11.5" serde = "1.0.130" diff --git a/src/mint.rs b/src/mint.rs index 5e4c29da..87436ed1 100644 --- a/src/mint.rs +++ b/src/mint.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use glob::glob; use metaplex_token_metadata::instruction::{create_master_edition, create_metadata_accounts}; +use rayon::prelude::*; use solana_client::rpc_client::RpcClient; use solana_sdk::{ pubkey::Pubkey, @@ -32,30 +33,34 @@ pub fn mint_list( let path = Path::new(&list_dir).join("*.json"); let pattern = path.to_str().ok_or(anyhow!("Invalid directory path"))?; - for res in glob(pattern)? { - match res { - Ok(path) => { - let file_path = path.to_str().ok_or(anyhow!("Invalid directory path"))?; - mint_one( - client, - &keypair, - &receiver, - file_path.to_string(), - immutable, - )?; - } - Err(e) => return Err(anyhow!("GlobError on path: {}", e)), + let (paths, errors): (Vec<_>, Vec<_>) = glob(pattern)?.into_iter().partition(Result::is_ok); + + let paths: Vec<_> = paths.into_iter().map(Result::unwrap).collect(); + let errors: Vec<_> = errors.into_iter().map(Result::unwrap_err).collect(); + + paths.par_iter().for_each(|path| { + match mint_one(client, &keypair, &receiver, path, immutable) { + Ok(_) => (), + Err(e) => println!("Failed to mint {:?}: {}", &path, e), + } + }); + + // TODO: handle errors in a better way and log instead of print. + if !errors.is_empty() { + println!("Failed to read some of the files with the following errors:"); + for error in errors { + println!("{}", error); } } Ok(()) } -pub fn mint_one( +pub fn mint_one>( client: &RpcClient, keypair: &String, receiver: &Option, - nft_data_file: String, + nft_data_file: P, immutable: bool, ) -> Result<()> { let keypair = parse_keypair(&keypair)?; diff --git a/src/process_subcommands.rs b/src/process_subcommands.rs index ab2ae946..bf33960b 100644 --- a/src/process_subcommands.rs +++ b/src/process_subcommands.rs @@ -26,7 +26,7 @@ pub fn process_mint(client: &RpcClient, commands: MintSubcommands) -> Result<()> receiver, nft_data_file, immutable, - } => mint_one(&client, &keypair, &receiver, nft_data_file, immutable), + } => mint_one(&client, &keypair, &receiver, &nft_data_file, immutable), MintSubcommands::List { keypair, receiver, From bce2be3135a85a621e8cfbc86011010c4bce39b9 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 11:20:38 -0900 Subject: [PATCH 03/12] parallelize signing --- src/sign.rs | 52 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/sign.rs b/src/sign.rs index fdbfe4ef..e20229eb 100644 --- a/src/sign.rs +++ b/src/sign.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use metaplex_token_metadata::{ instruction::sign_metadata, state::Metadata, ID as METAPLEX_PROGRAM_ID, }; +use rayon::prelude::*; use solana_client::rpc_client::RpcClient; use solana_program::borsh::try_from_slice_unchecked; use solana_sdk::{ @@ -10,7 +11,11 @@ use solana_sdk::{ signer::{keypair::Keypair, Signer}, transaction::Transaction, }; -use std::{fs::File, str::FromStr}; +use std::{ + fs::File, + str::FromStr, + sync::{Arc, Mutex}, +}; use crate::decode::get_metadata_pda; use crate::parse::{is_only_one_option, parse_keypair}; @@ -74,8 +79,15 @@ pub fn sign_mint_accounts( creator: &Keypair, mint_accounts: Vec, ) -> Result<()> { - for mint_account in mint_accounts { - let account_pubkey = Pubkey::from_str(&mint_account)?; + mint_accounts.par_iter().for_each(|mint_account| { + let account_pubkey = match Pubkey::from_str(&mint_account) { + Ok(pubkey) => pubkey, + Err(err) => { + eprintln!("Invalid public key: {}, error: {}", mint_account, err); + return; + } + }; + let metadata_pubkey = get_metadata_pda(account_pubkey); // Try to sign all accounts, print any errors that crop up. @@ -83,7 +95,7 @@ pub fn sign_mint_accounts( Ok(sig) => println!("{}", sig), Err(e) => println!("{}", e), } - } + }); Ok(()) } @@ -95,11 +107,18 @@ pub fn sign_candy_machine_accounts( ) -> Result<()> { let accounts = get_cm_creator_accounts(client, candy_machine_id)?; - let mut signed_at_least_one_account = false; - // Only sign accounts that have not been signed yet - for (metadata_pubkey, account) in &accounts { - let metadata: Metadata = try_from_slice_unchecked(&account.data.clone())?; + let signed_at_least_one_account = Arc::new(Mutex::new(false)); + + accounts.par_iter().for_each(|(metadata_pubkey, account)| { + let signed_at_least_one_account = signed_at_least_one_account.clone(); + let metadata: Metadata = match try_from_slice_unchecked(&account.data.clone()) { + Ok(metadata) => metadata, + Err(_) => { + println!("Account {} has no metadata", metadata_pubkey); + return; + } + }; if let Some(creators) = metadata.data.creators { // Check whether the specific creator has already signed the account @@ -111,19 +130,26 @@ pub fn sign_candy_machine_accounts( ); println!("Signing..."); - let sig = sign(client, &signing_creator, *metadata_pubkey)?; + let sig = match sign(client, &signing_creator, *metadata_pubkey) { + Ok(sig) => sig, + Err(e) => { + println!("Error signing: {}", e); + return; + } + }; + println!("{}", sig); - signed_at_least_one_account = true; + *signed_at_least_one_account.lock().unwrap() = true; } } } else { // No creators for that token, nothing to sign. - continue; + return; } - } + }); - if !signed_at_least_one_account { + if !*signed_at_least_one_account.lock().unwrap() { println!("No unverified metadata for this creator and candy machine."); return Ok(()); } From f16c3dbf2086a23692c2035052598427a3eede74 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 12:43:10 -0900 Subject: [PATCH 04/12] parallelize snapshot holders --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/snapshot.rs | 64 ++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f97ffd5..68c562f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,7 +1547,7 @@ dependencies = [ [[package]] name = "metaboss" -version = "0.2.2" +version = "0.3.0-beta" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 8b1a2bfe..755ceb31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "metaboss" -version = "0.2.2" +version = "0.3.0-beta" edition = "2018" [dependencies] diff --git a/src/snapshot.rs b/src/snapshot.rs index 849066ca..e0b76dbe 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use metaplex_token_metadata::state::Metadata; use metaplex_token_metadata::ID as TOKEN_METADATA_PROGRAM_ID; +use rayon::prelude::*; use serde::Serialize; use solana_account_decoder::{ parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount}, @@ -18,7 +19,11 @@ use solana_sdk::{ pubkey::Pubkey, }; use spl_token::ID as TOKEN_PROGRAM_ID; -use std::{fs::File, str::FromStr}; +use std::{ + fs::File, + str::FromStr, + sync::{Arc, Mutex}, +}; use crate::constants::*; use crate::parse::is_only_one_option; @@ -111,27 +116,64 @@ pub fn snapshot_holders( )); }; - let mut nft_holders: Vec = Vec::new(); + let nft_holders: Arc>> = Arc::new(Mutex::new(Vec::new())); - for (metadata_pubkey, account) in accounts { - let metadata: Metadata = try_from_slice_unchecked(&account.data)?; + // for (metadata_pubkey, account) in accounts { + accounts.par_iter().for_each(|(metadata_pubkey, account)| { + let nft_holders = nft_holders.clone(); + + let metadata: Metadata = match try_from_slice_unchecked(&account.data) { + Ok(metadata) => metadata, + Err(_) => { + println!("Account {} has no metadata", metadata_pubkey); + return; + } + }; - let token_accounts = get_holder_token_accounts(client, metadata.mint.to_string())?; + let token_accounts = match get_holder_token_accounts(client, metadata.mint.to_string()) { + Ok(token_accounts) => token_accounts, + Err(_) => { + println!("Account {} has no token accounts", metadata_pubkey); + return; + } + }; for (associated_token_address, account) in token_accounts { - let data = parse_account_data( + let data = match parse_account_data( &metadata.mint, &TOKEN_PROGRAM_ID, &account.data, Some(AccountAdditionalData { spl_token_decimals: Some(0), }), - )?; - let amount = parse_token_amount(&data)?; + ) { + Ok(data) => data, + Err(err) => { + println!("Account {} has no data: {}", associated_token_address, err); + return; + } + }; + + let amount = match parse_token_amount(&data) { + Ok(amount) => amount, + Err(err) => { + println!( + "Account {} has no amount: {}", + associated_token_address, err + ); + return; + } + }; // Only include current holder of the NFT. if amount == 1 { - let owner_wallet = parse_owner(&data)?; + let owner_wallet = match parse_owner(&data) { + Ok(owner_wallet) => owner_wallet, + Err(err) => { + println!("Account {} has no owner: {}", associated_token_address, err); + return; + } + }; let associated_token_address = associated_token_address.to_string(); let holder = Holder { owner_wallet, @@ -139,10 +181,10 @@ pub fn snapshot_holders( mint_account: metadata.mint.to_string(), metadata_account: metadata_pubkey.to_string(), }; - nft_holders.push(holder); + nft_holders.lock().unwrap().push(holder); } } - } + }); let prefix = if let Some(update_authority) = update_authority { update_authority From c74ac0ca003e9f50d2d82c3ec35a515c3d9a65da Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 12:45:59 -0900 Subject: [PATCH 05/12] parallelize set_update_authority_all --- src/update_metadata.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/update_metadata.rs b/src/update_metadata.rs index 36bfa272..a622b0e8 100644 --- a/src/update_metadata.rs +++ b/src/update_metadata.rs @@ -1,5 +1,6 @@ use anyhow::Result; use metaplex_token_metadata::instruction::update_metadata_accounts; +use rayon::prelude::*; use solana_client::rpc_client::RpcClient; use solana_sdk::{pubkey::Pubkey, signer::Signer, transaction::Transaction}; use std::{fs::File, str::FromStr}; @@ -170,7 +171,8 @@ pub fn set_update_authority_all( let file = File::open(json_file)?; let items: Vec = serde_json::from_reader(file)?; - for item in items.iter() { + // for item in items.iter() { + items.par_iter().for_each(|item| { println!("Updating metadata for mint account: {}", item); // If someone uses a json list that contains a mint account that has already @@ -181,6 +183,7 @@ pub fn set_update_authority_all( println!("Error occurred! {}", error) } }; - } + }); + Ok(()) } From b80f6f3c3782e20020aeb9b513c0ed5ee7c2b2c0 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 13:59:47 -0900 Subject: [PATCH 06/12] add update_data_all function; change error print to stderr --- src/data.rs | 6 +++ src/decode.rs | 10 ++-- src/mint.rs | 6 +-- src/opt.rs | 11 ++++ src/process_subcommands.rs | 5 +- src/sign.rs | 4 +- src/snapshot.rs | 10 ++-- src/update_metadata.rs | 102 ++++++++++++++++++++++++++++++++----- 8 files changed, 126 insertions(+), 28 deletions(-) diff --git a/src/data.rs b/src/data.rs index 4aeb22c6..339827f5 100644 --- a/src/data.rs +++ b/src/data.rs @@ -15,6 +15,12 @@ pub struct NFTData { pub creators: Option>, } +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateNFTData { + pub mint_account: String, + pub nft_data: NFTData, +} + #[derive(Debug, Serialize, Deserialize)] pub struct NFTCreator { pub address: String, diff --git a/src/decode.rs b/src/decode.rs index 42cb2640..0f9b38a8 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -33,11 +33,11 @@ pub fn decode_metadata_all( Ok(m) => m, Err(err) => match err { DecodeError::MissingAccount(account) => { - println!("No account data found for mint account: {}!", account); + eprintln!("No account data found for mint account: {}!", account); return; } err => { - println!( + eprintln!( "Failed to decode metadata for mint account: {}, error: {}", mint_account, err ); @@ -49,7 +49,7 @@ pub fn decode_metadata_all( let json_metadata = match decode_to_json(metadata) { Ok(j) => j, Err(err) => { - println!( + eprintln!( "Failed to decode metadata to JSON for mint account: {}, error: {}", mint_account, err ); @@ -60,7 +60,7 @@ pub fn decode_metadata_all( let mut file = match File::create(format!("{}/{}.json", output, mint_account)) { Ok(f) => f, Err(err) => { - println!( + eprintln!( "Failed to create JSON file for mint account: {}, error: {}", mint_account, err ); @@ -71,7 +71,7 @@ pub fn decode_metadata_all( match serde_json::to_writer(&mut file, &json_metadata) { Ok(_) => (), Err(err) => { - println!( + eprintln!( "Failed to write JSON file for mint account: {}, error: {}", mint_account, err ); diff --git a/src/mint.rs b/src/mint.rs index 87436ed1..83c525c9 100644 --- a/src/mint.rs +++ b/src/mint.rs @@ -41,15 +41,15 @@ pub fn mint_list( paths.par_iter().for_each(|path| { match mint_one(client, &keypair, &receiver, path, immutable) { Ok(_) => (), - Err(e) => println!("Failed to mint {:?}: {}", &path, e), + Err(e) => eprintln!("Failed to mint {:?}: {}", &path, e), } }); // TODO: handle errors in a better way and log instead of print. if !errors.is_empty() { - println!("Failed to read some of the files with the following errors:"); + eprintln!("Failed to read some of the files with the following errors:"); for error in errors { - println!("{}", error); + eprintln!("{}", error); } } diff --git a/src/opt.rs b/src/opt.rs index 66776a27..a80c6be2 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -253,6 +253,17 @@ pub enum UpdateSubcommands { #[structopt(short, long)] new_data_file: String, }, + /// UPdate the data struct on a list of NFTs + #[structopt(name = "data-all")] + DataAll { + /// Path to the creator's keypair file + #[structopt(short, long)] + keypair: String, + + /// Path to directory containing JSON files with new data + #[structopt(short, long)] + data_dir: String, + }, /// Update the metadata URI, keeping the rest of the data the same #[structopt(name = "uri")] Uri { diff --git a/src/process_subcommands.rs b/src/process_subcommands.rs index bf33960b..ea2bc853 100644 --- a/src/process_subcommands.rs +++ b/src/process_subcommands.rs @@ -42,7 +42,10 @@ pub fn process_update(client: &RpcClient, commands: UpdateSubcommands) -> Result keypair, account, new_data_file, - } => update_data(&client, &keypair, &account, &new_data_file), + } => update_data_one(&client, &keypair, &account, &new_data_file), + UpdateSubcommands::DataAll { keypair, data_dir } => { + update_data_all(&client, &keypair, &data_dir) + } UpdateSubcommands::Uri { keypair, account, diff --git a/src/sign.rs b/src/sign.rs index e20229eb..184a1d91 100644 --- a/src/sign.rs +++ b/src/sign.rs @@ -115,7 +115,7 @@ pub fn sign_candy_machine_accounts( let metadata: Metadata = match try_from_slice_unchecked(&account.data.clone()) { Ok(metadata) => metadata, Err(_) => { - println!("Account {} has no metadata", metadata_pubkey); + eprintln!("Account {} has no metadata", metadata_pubkey); return; } }; @@ -133,7 +133,7 @@ pub fn sign_candy_machine_accounts( let sig = match sign(client, &signing_creator, *metadata_pubkey) { Ok(sig) => sig, Err(e) => { - println!("Error signing: {}", e); + eprintln!("Error signing: {}", e); return; } }; diff --git a/src/snapshot.rs b/src/snapshot.rs index e0b76dbe..ac5b5d01 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -125,7 +125,7 @@ pub fn snapshot_holders( let metadata: Metadata = match try_from_slice_unchecked(&account.data) { Ok(metadata) => metadata, Err(_) => { - println!("Account {} has no metadata", metadata_pubkey); + eprintln!("Account {} has no metadata", metadata_pubkey); return; } }; @@ -133,7 +133,7 @@ pub fn snapshot_holders( let token_accounts = match get_holder_token_accounts(client, metadata.mint.to_string()) { Ok(token_accounts) => token_accounts, Err(_) => { - println!("Account {} has no token accounts", metadata_pubkey); + eprintln!("Account {} has no token accounts", metadata_pubkey); return; } }; @@ -149,7 +149,7 @@ pub fn snapshot_holders( ) { Ok(data) => data, Err(err) => { - println!("Account {} has no data: {}", associated_token_address, err); + eprintln!("Account {} has no data: {}", associated_token_address, err); return; } }; @@ -157,7 +157,7 @@ pub fn snapshot_holders( let amount = match parse_token_amount(&data) { Ok(amount) => amount, Err(err) => { - println!( + eprintln!( "Account {} has no amount: {}", associated_token_address, err ); @@ -170,7 +170,7 @@ pub fn snapshot_holders( let owner_wallet = match parse_owner(&data) { Ok(owner_wallet) => owner_wallet, Err(err) => { - println!("Account {} has no owner: {}", associated_token_address, err); + eprintln!("Account {} has no owner: {}", associated_token_address, err); return; } }; diff --git a/src/update_metadata.rs b/src/update_metadata.rs index a622b0e8..7535bdd3 100644 --- a/src/update_metadata.rs +++ b/src/update_metadata.rs @@ -1,33 +1,111 @@ -use anyhow::Result; -use metaplex_token_metadata::instruction::update_metadata_accounts; +use anyhow::{anyhow, Result}; +use glob::glob; +use metaplex_token_metadata::{instruction::update_metadata_accounts, state::Data}; use rayon::prelude::*; use solana_client::rpc_client::RpcClient; -use solana_sdk::{pubkey::Pubkey, signer::Signer, transaction::Transaction}; -use std::{fs::File, str::FromStr}; +use solana_sdk::{ + pubkey::Pubkey, + signer::{keypair::Keypair, Signer}, + transaction::Transaction, +}; +use std::{fs::File, path::Path, str::FromStr}; use crate::constants::*; -use crate::data::NFTData; +use crate::data::{NFTData, UpdateNFTData}; use crate::decode::{decode, get_metadata_pda}; use crate::parse::{convert_local_to_remote_data, parse_keypair}; -pub fn update_data( +pub fn update_data_one( client: &RpcClient, keypair: &String, mint_account: &String, json_file: &String, ) -> Result<()> { let keypair = parse_keypair(keypair)?; + let f = File::open(json_file)?; + let new_data: NFTData = serde_json::from_reader(f)?; + + let data = convert_local_to_remote_data(new_data)?; + + update_data(client, &keypair, mint_account, data)?; + + Ok(()) +} + +pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) -> Result<()> { + let keypair = parse_keypair(keypair)?; + + let path = Path::new(&data_dir).join("*.json"); + let pattern = path.to_str().ok_or(anyhow!("Invalid directory path"))?; + + let (paths, errors): (Vec<_>, Vec<_>) = glob(pattern)?.into_iter().partition(Result::is_ok); + + let paths: Vec<_> = paths.into_iter().map(Result::unwrap).collect(); + let errors: Vec<_> = errors.into_iter().map(Result::unwrap_err).collect(); + + paths.par_iter().for_each(|path| { + let f = match File::open(path) { + Ok(f) => f, + Err(e) => { + eprintln!("Failed to open file: {:?} error: {}", path, e); + return; + } + }; + + let update_nft_data: UpdateNFTData = match serde_json::from_reader(f) { + Ok(data) => data, + Err(e) => { + eprintln!( + "Failed to parse JSON data from file: {:?} error: {}", + path, e + ); + return; + } + }; + + let data = match convert_local_to_remote_data(update_nft_data.nft_data) { + Ok(data) => data, + Err(e) => { + eprintln!( + "Failed to convert local data to remote data: {:?} error: {}", + path, e + ); + return; + } + }; + + match update_data(client, &keypair, &update_nft_data.mint_account, data) { + Ok(_) => (), + Err(e) => { + eprintln!("Failed to update data: {:?} error: {}", path, e); + return; + } + } + }); + + // TODO: handle errors in a better way and log instead of print. + if !errors.is_empty() { + eprintln!("Failed to read some of the files with the following errors:"); + for error in errors { + eprintln!("{}", error); + } + } + + Ok(()) +} + +pub fn update_data( + client: &RpcClient, + keypair: &Keypair, + mint_account: &String, + data: Data, +) -> Result<()> { let program_id = Pubkey::from_str(METAPLEX_PROGRAM_ID)?; let mint_pubkey = Pubkey::from_str(mint_account)?; let metadata_account = get_metadata_pda(mint_pubkey); - let f = File::open(json_file)?; - let new_data: NFTData = serde_json::from_reader(f)?; - let update_authority = keypair.pubkey(); - let data = convert_local_to_remote_data(new_data)?; - let ix = update_metadata_accounts( program_id, metadata_account, @@ -40,7 +118,7 @@ pub fn update_data( let tx = Transaction::new_signed_with_payer( &[ix], Some(&update_authority), - &[&keypair], + &[keypair], recent_blockhash, ); From 641a489e7e30b1d659a940f1133e988c94c2db6b Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 14:37:24 -0900 Subject: [PATCH 07/12] add update_uri_all --- src/data.rs | 6 ++++++ src/opt.rs | 11 +++++++++++ src/process_subcommands.rs | 5 ++++- src/update_metadata.rs | 39 ++++++++++++++++++++++++++++++++++---- 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/data.rs b/src/data.rs index 339827f5..87a21de8 100644 --- a/src/data.rs +++ b/src/data.rs @@ -21,6 +21,12 @@ pub struct UpdateNFTData { pub nft_data: NFTData, } +#[derive(Debug, Serialize, Deserialize)] +pub struct UpdateUriData { + pub mint_account: String, + pub new_uri: String, +} + #[derive(Debug, Serialize, Deserialize)] pub struct NFTCreator { pub address: String, diff --git a/src/opt.rs b/src/opt.rs index a80c6be2..ba00ae5c 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -279,4 +279,15 @@ pub enum UpdateSubcommands { #[structopt(short = "u", long)] new_uri: String, }, + /// Update the metadata URI on a list of mint accounts + #[structopt(name = "uri")] + UriAll { + /// Path to the creator's keypair file + #[structopt(short, long)] + keypair: String, + + /// JSON file with list of mint accounts and new URIs + #[structopt(short = "u", long)] + json_file: String, + }, } diff --git a/src/process_subcommands.rs b/src/process_subcommands.rs index ea2bc853..c4784b6c 100644 --- a/src/process_subcommands.rs +++ b/src/process_subcommands.rs @@ -50,7 +50,10 @@ pub fn process_update(client: &RpcClient, commands: UpdateSubcommands) -> Result keypair, account, new_uri, - } => update_uri(&client, &keypair, &account, &new_uri), + } => update_uri_one(&client, &keypair, &account, &new_uri), + UpdateSubcommands::UriAll { keypair, json_file } => { + update_uri_all(&client, &keypair, &json_file) + } } } diff --git a/src/update_metadata.rs b/src/update_metadata.rs index 7535bdd3..035ec529 100644 --- a/src/update_metadata.rs +++ b/src/update_metadata.rs @@ -11,7 +11,7 @@ use solana_sdk::{ use std::{fs::File, path::Path, str::FromStr}; use crate::constants::*; -use crate::data::{NFTData, UpdateNFTData}; +use crate::data::{NFTData, UpdateNFTData, UpdateUriData}; use crate::decode::{decode, get_metadata_pda}; use crate::parse::{convert_local_to_remote_data, parse_keypair}; @@ -128,15 +128,46 @@ pub fn update_data( Ok(()) } -pub fn update_uri( +pub fn update_uri_one( client: &RpcClient, keypair: &String, mint_account: &String, new_uri: &String, ) -> Result<()> { let keypair = parse_keypair(keypair)?; - let program_id = Pubkey::from_str(METAPLEX_PROGRAM_ID)?; + + update_uri(client, &keypair, &mint_account, new_uri)?; + + Ok(()) +} + +pub fn update_uri_all(client: &RpcClient, keypair: &String, json_file: &String) -> Result<()> { + let keypair = parse_keypair(keypair)?; + + let f = File::open(json_file)?; + let update_uris: Vec = serde_json::from_reader(f)?; + + update_uris.par_iter().for_each(|data| { + match update_uri(client, &keypair, &data.mint_account, &data.new_uri) { + Ok(_) => (), + Err(e) => { + eprintln!("Failed to update uri: {:?} error: {}", data, e); + return; + } + } + }); + + Ok(()) +} + +pub fn update_uri( + client: &RpcClient, + keypair: &Keypair, + mint_account: &String, + new_uri: &String, +) -> Result<()> { let mint_pubkey = Pubkey::from_str(mint_account)?; + let program_id = Pubkey::from_str(METAPLEX_PROGRAM_ID)?; let update_authority = keypair.pubkey(); let metadata_account = get_metadata_pda(mint_pubkey); @@ -158,7 +189,7 @@ pub fn update_uri( let tx = Transaction::new_signed_with_payer( &[ix], Some(&update_authority), - &[&keypair], + &[keypair], recent_blockhash, ); From e516c72d834284a083e65317adb41e836c84f0bc Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 14:46:16 -0900 Subject: [PATCH 08/12] update README --- README.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4eeed9a8..31e5045e 100644 --- a/README.md +++ b/README.md @@ -450,7 +450,9 @@ metaboss update data --keypair --account --new- The JSON file should include all the fields of the metadata `Data` struct and should match `creator` `verified` bools for existing creators. E.g. if your NFT was minted by the Metaplex Candy Machine program, and you wish to keep your candy machine as a verified creator _you must add the candy machine to your creators array with `verified` set to `true`_. -**Make sure you understand how the Metaplex Metadata `Data` struct works and how this command will affect your NFT. Always test on `devnet` before running on mainnet. ** +Note: The on-chain `Data` struct is *different* than the external metadata stored at the link in the `uri` field so make you understand the difference before running this command. + +**Make sure you understand how the Metaplex Metadata `Data` struct works and how this command will affect your NFT. Always test on `devnet` before running on mainnet.** ```json { @@ -480,6 +482,53 @@ The JSON file should include all the fields of the metadata `Data` struct and sh Outputs a TxId to the command line so you can check the result. +#### Update Data All + +Update the `Data` struct on a list of NFTs from JSON files. + +##### Usage + +```bash +metaboss update data-all --keypair --data-dir +``` + +Each JSON file in the data directory should include the mint account and all the fields of the metadata `Data` struct and should match `creator` `verified` bools for existing creators. E.g. if your NFT was minted by the Metaplex Candy Machine program, and you wish to keep your candy machine as a verified creator _you must add the candy machine to your creators array with `verified` set to `true`_. + +Note: The on-chain `Data` struct is *different* than the external metadata stored at the link in the `uri` field so make you understand the difference before running this command. + +**Make sure you understand how the Metaplex Metadata `Data` struct works and how this command will affect your NFT. Always test on `devnet` before running on mainnet.** + +```json +{ + "mint_account": "CQNKXw1rw2eWwi812Exk4cKUjKuomZ2156STGRyXd2Mp", + "nft_data": + { + "name": "FerrisCrab #4", + "symbol": "FERRIS", + "uri": "https://arweave.net/N36gZYJ6PEH8OE11i0MppIbPG4VXKV4iuQw1zaq3rls", + "seller_fee_basis_points": 100, + "creators": [ + { + "address": "", + "verified": true, + "share": 0 + }, + { + "address": "", + "verified": true, + "share": 50 + }, + { + "address": "42NevAWA6A8m9prDvZRUYReQmhNC3NtSZQNFUppPJDRB", + "verified": false, + "share": 50 + } + } +} +``` + +Outputs a TxId to the command line so you can check the result. + #### Update URI Update the metadata URI, keeping the rest of the `Data` struct the same. @@ -489,3 +538,26 @@ Update the metadata URI, keeping the rest of the `Data` struct the same. ```bash metaboss update uri --keypair --account --new-uri ``` + +#### Update URI All + +Update the metadata URI for a list of mint accounts, keeping the rest of the `Data` struct the same. + +##### Usage + +```bash +metaboss update uri-all --keypair --json-file +``` + +```json +[ + { + "mint_account": "xZ43...", + "new_uri": "https://arweave.net/N36gZYJ6PEH8OE11i0MppIbPG4VXKV4iuQw1zaq3rls" + }, + { + "mint_account": "71bk...", + "new_uri": "https://arweave.net/FPGAv1XnyZidnqquOdEbSY6_ES735ckcDTdaAtI7GFw" + } +] +``` \ No newline at end of file From 192e4e5f9aaa64d2b2276e34880c5ef816fb7b9f Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Sun, 28 Nov 2021 22:31:40 -0900 Subject: [PATCH 09/12] add rate limiter for public apis and add to decode --- Cargo.lock | 8 ++++++++ Cargo.toml | 2 ++ src/constants.rs | 21 +++++++++++++++++++++ src/decode.rs | 11 +++++++++++ src/lib.rs | 1 + src/limiter.rs | 23 +++++++++++++++++++++++ src/main.rs | 22 +++++++++++++++++----- 7 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 src/limiter.rs diff --git a/Cargo.lock b/Cargo.lock index 68c562f7..49211791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1554,8 +1554,10 @@ dependencies = [ "borsh", "bs58 0.4.0", "glob", + "lazy_static", "metaplex-token-metadata", "num_cpus", + "ratelimit", "rayon", "reqwest", "serde", @@ -2156,6 +2158,12 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "ratelimit" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4777eb47471c2a42bee8b553b22b8e5c496f657dc6f8b8e29bd69662f31e7e" + [[package]] name = "rayon" version = "1.5.1" diff --git a/Cargo.toml b/Cargo.toml index 755ceb31..cc9549fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,8 +8,10 @@ anyhow = "1.0.44" borsh = "0.9.1" bs58 = "0.4.0" glob = "0.3.0" +lazy_static = "1.4.0" metaplex-token-metadata = "0.0.1" num_cpus = "1.13.0" +ratelimit = "0.4.4" rayon = "1.5.1" reqwest = "0.11.5" serde = "1.0.130" diff --git a/src/constants.rs b/src/constants.rs index 7f7aadb1..40f30a1c 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,3 +1,6 @@ +use lazy_static::lazy_static; +use std::sync::RwLock; + pub const MAX_NAME_LENGTH: usize = 32; pub const MAX_URI_LENGTH: usize = 200; pub const MAX_SYMBOL_LENGTH: usize = 10; @@ -7,3 +10,21 @@ pub const METAPLEX_PROGRAM_ID: &'static str = "metaqbxxUerdq28cj1RbAWkYQm3ybzjb6 pub const CANDY_MACHINE_PROGRAM_ID: &'static str = "cndyAnrLdpjq1Ssp1z8xxDsB8dxe7u4HL5Nxi2K5WXZ"; pub const DEFAULT_RPC_DELAY_MS: u64 = 300; + +pub const PUBLIC_RPC_URLS: &'static [&'static str] = &[ + "https://api.devnet.solana.com", + "https://api.testnet.solana.com", + "https://api.mainnet-beta.solana.com", + "https://solana-api.projectserum.com", +]; + +pub const MAX_REQUESTS: u64 = 40; +pub const TIME_PER_MAX_REQUESTS_NS: u64 = 10_000_000_000; +pub const TIME_BUFFER_NS: u32 = 50_000_000; + +// Delay in milliseconds between RPC requests +pub const RATE_LIMIT: u64 = 500; + +lazy_static! { + pub static ref USE_RATE_LIMIT: RwLock = RwLock::new(false); +} diff --git a/src/decode.rs b/src/decode.rs index 0f9b38a8..8004b8bb 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -11,6 +11,7 @@ use std::str::FromStr; use crate::constants::*; use crate::errors::*; +use crate::limiter::create_rate_limiter; use crate::parse::is_only_one_option; #[derive(Debug, Serialize)] @@ -27,8 +28,18 @@ pub fn decode_metadata_all( ) -> AnyResult<()> { let file = File::open(json_file)?; let mint_accounts: Vec = serde_json::from_reader(file)?; + let use_rate_limit = *USE_RATE_LIMIT.read().unwrap(); + let handle = create_rate_limiter(); + + println!("Decoding accounts..."); mint_accounts.par_iter().for_each(|mint_account| { + let mut handle = handle.clone(); + + if use_rate_limit { + handle.wait(); + } + let metadata = match decode(client, mint_account) { Ok(m) => m, Err(err) => match err { diff --git a/src/lib.rs b/src/lib.rs index a2c268a3..39ff98fb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ pub mod constants; pub mod data; pub mod decode; pub mod errors; +pub mod limiter; pub mod mint; pub mod opt; pub mod parse; diff --git a/src/limiter.rs b/src/limiter.rs new file mode 100644 index 00000000..5af78508 --- /dev/null +++ b/src/limiter.rs @@ -0,0 +1,23 @@ +use ratelimit::Handle; +use std::{thread, time::Duration}; + +use crate::constants::*; + +pub fn create_rate_limiter() -> Handle { + let num_cpus = num_cpus::get(); + + let mut limiter = ratelimit::Builder::new() + .capacity(num_cpus as u32) + .quantum(1) + .interval(Duration::new( + 0, + (TIME_PER_MAX_REQUESTS_NS / MAX_REQUESTS) as u32 + TIME_BUFFER_NS, + )) + .build(); + + let handle = limiter.make_handle(); + thread::spawn(move || { + limiter.run(); + }); + handle +} diff --git a/src/main.rs b/src/main.rs index 4e9bb5f9..01eeb793 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,12 @@ use anyhow::Result; +use metaboss::constants::PUBLIC_RPC_URLS; use solana_client::rpc_client::RpcClient; use solana_sdk::commitment_config::CommitmentConfig; use std::str::FromStr; use std::time::Duration; use structopt::StructOpt; +use metaboss::constants::*; use metaboss::opt::*; use metaboss::parse::parse_solana_config; use metaboss::process_subcommands::*; @@ -15,10 +17,10 @@ fn main() -> Result<()> { let (mut rpc, commitment) = if let Some(config) = sol_config { (config.json_rpc_url, config.commitment) } else { - ( - "https://api.devnet.solana.com".to_string(), - "confirmed".to_string(), - ) + eprintln!( + "Could not find a valid Solana-CLI config file. Please specify a RPC manually with '-r' or set up your Solana-CLI config file." + ); + std::process::exit(1); }; let options = Opt::from_args(); @@ -26,8 +28,18 @@ fn main() -> Result<()> { if let Some(cli_rpc) = options.rpc { rpc = cli_rpc.clone(); } - let commitment = CommitmentConfig::from_str(&commitment)?; + // Set rate limiting if the user specified a public RPC. + if PUBLIC_RPC_URLS.contains(&rpc.as_str()) { + eprintln!( + r#" + WARNING: Using a public RPC URL is not recommended for heavy tasks as you will be rate-limited and suffer a performance hit. + Please use a private RPC endpoint for best performance results."# + ); + *USE_RATE_LIMIT.write().unwrap() = true; + } + + let commitment = CommitmentConfig::from_str(&commitment)?; let timeout = Duration::from_secs(options.timeout); let client = RpcClient::new_with_timeout_and_commitment(rpc, timeout, commitment); From 46489a5e25188fc84b95f463dea5619a0f9ec9e1 Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Mon, 29 Nov 2021 11:56:22 -0900 Subject: [PATCH 10/12] add progress bar to decode; fix errors --- Cargo.lock | 24 ++++++++++- Cargo.toml | 1 + src/decode.rs | 108 +++++++++++++++++++++++++++----------------------- src/errors.rs | 6 ++- 4 files changed, 86 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49211791..6fd23aae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1303,7 +1303,20 @@ checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" dependencies = [ "console 0.14.1", "lazy_static", - "number_prefix", + "number_prefix 0.3.0", + "regex", +] + +[[package]] +name = "indicatif" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +dependencies = [ + "console 0.14.1", + "lazy_static", + "number_prefix 0.4.0", + "rayon", "regex", ] @@ -1554,6 +1567,7 @@ dependencies = [ "borsh", "bs58 0.4.0", "glob", + "indicatif 0.16.2", "lazy_static", "metaplex-token-metadata", "num_cpus", @@ -1760,6 +1774,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.26.2" @@ -2660,7 +2680,7 @@ dependencies = [ "bincode", "bs58 0.3.1", "clap", - "indicatif", + "indicatif 0.15.0", "jsonrpc-core", "log", "net2", diff --git a/Cargo.toml b/Cargo.toml index cc9549fa..48228d39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ anyhow = "1.0.44" borsh = "0.9.1" bs58 = "0.4.0" glob = "0.3.0" +indicatif = { version = "0.16.2", features = ["rayon"] } lazy_static = "1.4.0" metaplex-token-metadata = "0.0.1" num_cpus = "1.13.0" diff --git a/src/decode.rs b/src/decode.rs index 8004b8bb..cca55aa8 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result as AnyResult}; +use indicatif::ParallelProgressIterator; use metaplex_token_metadata::state::{Key, Metadata}; use rayon::prelude::*; use serde::Serialize; @@ -33,63 +34,70 @@ pub fn decode_metadata_all( let handle = create_rate_limiter(); println!("Decoding accounts..."); - mint_accounts.par_iter().for_each(|mint_account| { - let mut handle = handle.clone(); + mint_accounts + .par_iter() + .progress() + .for_each(|mint_account| { + let mut handle = handle.clone(); + + if use_rate_limit { + handle.wait(); + } - if use_rate_limit { - handle.wait(); - } + let metadata = match decode(client, mint_account) { + Ok(m) => m, + Err(err) => match err { + DecodeError::ClientError(kind) => { + eprintln!("Client Error: {}!", kind); + return; + } + DecodeError::PubkeyParseFailed(address) => { + eprintln!("Failed to parse pubkey from mint address: {}", address); + return; + } + err => { + eprintln!( + "Failed to decode metadata for mint account: {}, error: {}", + mint_account, err + ); + return; + } + }, + }; + + let json_metadata = match decode_to_json(metadata) { + Ok(j) => j, + Err(err) => { + eprintln!( + "Failed to decode metadata to JSON for mint account: {}, error: {}", + mint_account, err + ); + return; + } + }; - let metadata = match decode(client, mint_account) { - Ok(m) => m, - Err(err) => match err { - DecodeError::MissingAccount(account) => { - eprintln!("No account data found for mint account: {}!", account); + let mut file = match File::create(format!("{}/{}.json", output, mint_account)) { + Ok(f) => f, + Err(err) => { + eprintln!( + "Failed to create JSON file for mint account: {}, error: {}", + mint_account, err + ); return; } - err => { + }; + + match serde_json::to_writer(&mut file, &json_metadata) { + Ok(_) => (), + Err(err) => { eprintln!( - "Failed to decode metadata for mint account: {}, error: {}", + "Failed to write JSON file for mint account: {}, error: {}", mint_account, err ); return; } - }, - }; - - let json_metadata = match decode_to_json(metadata) { - Ok(j) => j, - Err(err) => { - eprintln!( - "Failed to decode metadata to JSON for mint account: {}, error: {}", - mint_account, err - ); - return; - } - }; - - let mut file = match File::create(format!("{}/{}.json", output, mint_account)) { - Ok(f) => f, - Err(err) => { - eprintln!( - "Failed to create JSON file for mint account: {}, error: {}", - mint_account, err - ); - return; } - }; - - match serde_json::to_writer(&mut file, &json_metadata) { - Ok(_) => (), - Err(err) => { - eprintln!( - "Failed to write JSON file for mint account: {}, error: {}", - mint_account, err - ); - return; - } - } - }); + }); Ok(()) } @@ -126,14 +134,14 @@ pub fn decode_metadata( pub fn decode(client: &RpcClient, mint_account: &String) -> Result { let pubkey = match Pubkey::from_str(&mint_account) { Ok(pubkey) => pubkey, - Err(_) => return Err(DecodeError::PubkeyParseFailed), + Err(_) => return Err(DecodeError::PubkeyParseFailed(mint_account.clone())), }; let metadata_pda = get_metadata_pda(pubkey); let account_data = match client.get_account_data(&metadata_pda) { Ok(data) => data, - Err(_) => { - return Err(DecodeError::MissingAccount(mint_account.to_string())); + Err(err) => { + return Err(DecodeError::ClientError(err.kind)); } }; diff --git a/src/errors.rs b/src/errors.rs index fbbdd281..9ccd9b64 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,3 +1,4 @@ +use solana_client::client_error::ClientErrorKind; use std::io; use thiserror::Error; @@ -6,8 +7,11 @@ pub enum DecodeError { #[error("no account data found")] MissingAccount(String), + #[error("failed to get account data")] + ClientError(ClientErrorKind), + #[error("failed to parse string into Pubkey")] - PubkeyParseFailed, + PubkeyParseFailed(String), #[error("failed to decode metadata")] DecodeMetadataFailed(String), From 0bd95ac0a678ec1c50800dfc91270f432d78fdbd Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Mon, 29 Nov 2021 12:06:21 -0900 Subject: [PATCH 11/12] add progressbar to all par_iter calls --- src/mint.rs | 3 +- src/sign.rs | 109 ++++++++++++++++++++------------------- src/snapshot.rs | 113 ++++++++++++++++++++++------------------- src/update_metadata.rs | 8 +-- 4 files changed, 126 insertions(+), 107 deletions(-) diff --git a/src/mint.rs b/src/mint.rs index 83c525c9..a68e5f01 100644 --- a/src/mint.rs +++ b/src/mint.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; use glob::glob; +use indicatif::ParallelProgressIterator; use metaplex_token_metadata::instruction::{create_master_edition, create_metadata_accounts}; use rayon::prelude::*; use solana_client::rpc_client::RpcClient; @@ -38,7 +39,7 @@ pub fn mint_list( let paths: Vec<_> = paths.into_iter().map(Result::unwrap).collect(); let errors: Vec<_> = errors.into_iter().map(Result::unwrap_err).collect(); - paths.par_iter().for_each(|path| { + paths.par_iter().progress().for_each(|path| { match mint_one(client, &keypair, &receiver, path, immutable) { Ok(_) => (), Err(e) => eprintln!("Failed to mint {:?}: {}", &path, e), diff --git a/src/sign.rs b/src/sign.rs index 184a1d91..36c5fb5b 100644 --- a/src/sign.rs +++ b/src/sign.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use indicatif::ParallelProgressIterator; use metaplex_token_metadata::{ instruction::sign_metadata, state::Metadata, ID as METAPLEX_PROGRAM_ID, }; @@ -79,23 +80,26 @@ pub fn sign_mint_accounts( creator: &Keypair, mint_accounts: Vec, ) -> Result<()> { - mint_accounts.par_iter().for_each(|mint_account| { - let account_pubkey = match Pubkey::from_str(&mint_account) { - Ok(pubkey) => pubkey, - Err(err) => { - eprintln!("Invalid public key: {}, error: {}", mint_account, err); - return; - } - }; + mint_accounts + .par_iter() + .progress() + .for_each(|mint_account| { + let account_pubkey = match Pubkey::from_str(&mint_account) { + Ok(pubkey) => pubkey, + Err(err) => { + eprintln!("Invalid public key: {}, error: {}", mint_account, err); + return; + } + }; - let metadata_pubkey = get_metadata_pda(account_pubkey); + let metadata_pubkey = get_metadata_pda(account_pubkey); - // Try to sign all accounts, print any errors that crop up. - match sign(client, &creator, metadata_pubkey) { - Ok(sig) => println!("{}", sig), - Err(e) => println!("{}", e), - } - }); + // Try to sign all accounts, print any errors that crop up. + match sign(client, &creator, metadata_pubkey) { + Ok(sig) => println!("{}", sig), + Err(e) => println!("{}", e), + } + }); Ok(()) } @@ -110,44 +114,47 @@ pub fn sign_candy_machine_accounts( // Only sign accounts that have not been signed yet let signed_at_least_one_account = Arc::new(Mutex::new(false)); - accounts.par_iter().for_each(|(metadata_pubkey, account)| { - let signed_at_least_one_account = signed_at_least_one_account.clone(); - let metadata: Metadata = match try_from_slice_unchecked(&account.data.clone()) { - Ok(metadata) => metadata, - Err(_) => { - eprintln!("Account {} has no metadata", metadata_pubkey); - return; - } - }; - - if let Some(creators) = metadata.data.creators { - // Check whether the specific creator has already signed the account - for creator in creators { - if creator.address == signing_creator.pubkey() && !creator.verified { - println!( - "Found creator unverified for mint account: {}", - metadata.mint - ); - println!("Signing..."); - - let sig = match sign(client, &signing_creator, *metadata_pubkey) { - Ok(sig) => sig, - Err(e) => { - eprintln!("Error signing: {}", e); - return; - } - }; - - println!("{}", sig); - - *signed_at_least_one_account.lock().unwrap() = true; + accounts + .par_iter() + .progress() + .for_each(|(metadata_pubkey, account)| { + let signed_at_least_one_account = signed_at_least_one_account.clone(); + let metadata: Metadata = match try_from_slice_unchecked(&account.data.clone()) { + Ok(metadata) => metadata, + Err(_) => { + eprintln!("Account {} has no metadata", metadata_pubkey); + return; } + }; + + if let Some(creators) = metadata.data.creators { + // Check whether the specific creator has already signed the account + for creator in creators { + if creator.address == signing_creator.pubkey() && !creator.verified { + println!( + "Found creator unverified for mint account: {}", + metadata.mint + ); + println!("Signing..."); + + let sig = match sign(client, &signing_creator, *metadata_pubkey) { + Ok(sig) => sig, + Err(e) => { + eprintln!("Error signing: {}", e); + return; + } + }; + + println!("{}", sig); + + *signed_at_least_one_account.lock().unwrap() = true; + } + } + } else { + // No creators for that token, nothing to sign. + return; } - } else { - // No creators for that token, nothing to sign. - return; - } - }); + }); if !*signed_at_least_one_account.lock().unwrap() { println!("No unverified metadata for this creator and candy machine."); diff --git a/src/snapshot.rs b/src/snapshot.rs index ac5b5d01..9a3390d1 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use indicatif::ParallelProgressIterator; use metaplex_token_metadata::state::Metadata; use metaplex_token_metadata::ID as TOKEN_METADATA_PROGRAM_ID; use rayon::prelude::*; @@ -66,6 +67,7 @@ pub fn snapshot_mints( )); } + println!("Getting accounts..."); let accounts = if let Some(ref update_authority) = update_authority { get_mints_by_update_authority(client, &update_authority)? } else if let Some(ref candy_machine_id) = candy_machine_id { @@ -76,6 +78,7 @@ pub fn snapshot_mints( )); }; + println!("Getting metadata and writing to file..."); let mut mint_accounts: Vec = Vec::new(); for (_, account) in accounts { @@ -106,6 +109,7 @@ pub fn snapshot_holders( candy_machine_id: &Option, output: &String, ) -> Result<()> { + println!("Getting accounts..."); let accounts = if let Some(update_authority) = update_authority { get_mints_by_update_authority(client, update_authority)? } else if let Some(candy_machine_id) = candy_machine_id { @@ -116,75 +120,80 @@ pub fn snapshot_holders( )); }; + println!("Finding current holders..."); let nft_holders: Arc>> = Arc::new(Mutex::new(Vec::new())); // for (metadata_pubkey, account) in accounts { - accounts.par_iter().for_each(|(metadata_pubkey, account)| { - let nft_holders = nft_holders.clone(); - - let metadata: Metadata = match try_from_slice_unchecked(&account.data) { - Ok(metadata) => metadata, - Err(_) => { - eprintln!("Account {} has no metadata", metadata_pubkey); - return; - } - }; - - let token_accounts = match get_holder_token_accounts(client, metadata.mint.to_string()) { - Ok(token_accounts) => token_accounts, - Err(_) => { - eprintln!("Account {} has no token accounts", metadata_pubkey); - return; - } - }; - - for (associated_token_address, account) in token_accounts { - let data = match parse_account_data( - &metadata.mint, - &TOKEN_PROGRAM_ID, - &account.data, - Some(AccountAdditionalData { - spl_token_decimals: Some(0), - }), - ) { - Ok(data) => data, - Err(err) => { - eprintln!("Account {} has no data: {}", associated_token_address, err); + accounts + .par_iter() + .progress() + .for_each(|(metadata_pubkey, account)| { + let nft_holders = nft_holders.clone(); + + let metadata: Metadata = match try_from_slice_unchecked(&account.data) { + Ok(metadata) => metadata, + Err(_) => { + eprintln!("Account {} has no metadata", metadata_pubkey); return; } }; - let amount = match parse_token_amount(&data) { - Ok(amount) => amount, - Err(err) => { - eprintln!( - "Account {} has no amount: {}", - associated_token_address, err - ); + let token_accounts = match get_holder_token_accounts(client, metadata.mint.to_string()) + { + Ok(token_accounts) => token_accounts, + Err(_) => { + eprintln!("Account {} has no token accounts", metadata_pubkey); return; } }; - // Only include current holder of the NFT. - if amount == 1 { - let owner_wallet = match parse_owner(&data) { - Ok(owner_wallet) => owner_wallet, + for (associated_token_address, account) in token_accounts { + let data = match parse_account_data( + &metadata.mint, + &TOKEN_PROGRAM_ID, + &account.data, + Some(AccountAdditionalData { + spl_token_decimals: Some(0), + }), + ) { + Ok(data) => data, Err(err) => { - eprintln!("Account {} has no owner: {}", associated_token_address, err); + eprintln!("Account {} has no data: {}", associated_token_address, err); return; } }; - let associated_token_address = associated_token_address.to_string(); - let holder = Holder { - owner_wallet, - associated_token_address, - mint_account: metadata.mint.to_string(), - metadata_account: metadata_pubkey.to_string(), + + let amount = match parse_token_amount(&data) { + Ok(amount) => amount, + Err(err) => { + eprintln!( + "Account {} has no amount: {}", + associated_token_address, err + ); + return; + } }; - nft_holders.lock().unwrap().push(holder); + + // Only include current holder of the NFT. + if amount == 1 { + let owner_wallet = match parse_owner(&data) { + Ok(owner_wallet) => owner_wallet, + Err(err) => { + eprintln!("Account {} has no owner: {}", associated_token_address, err); + return; + } + }; + let associated_token_address = associated_token_address.to_string(); + let holder = Holder { + owner_wallet, + associated_token_address, + mint_account: metadata.mint.to_string(), + metadata_account: metadata_pubkey.to_string(), + }; + nft_holders.lock().unwrap().push(holder); + } } - } - }); + }); let prefix = if let Some(update_authority) = update_authority { update_authority diff --git a/src/update_metadata.rs b/src/update_metadata.rs index 035ec529..05c4f91c 100644 --- a/src/update_metadata.rs +++ b/src/update_metadata.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; use glob::glob; +use indicatif::ParallelProgressIterator; use metaplex_token_metadata::{instruction::update_metadata_accounts, state::Data}; use rayon::prelude::*; use solana_client::rpc_client::RpcClient; @@ -43,7 +44,8 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) let paths: Vec<_> = paths.into_iter().map(Result::unwrap).collect(); let errors: Vec<_> = errors.into_iter().map(Result::unwrap_err).collect(); - paths.par_iter().for_each(|path| { + println!("Updating..."); + paths.par_iter().progress().for_each(|path| { let f = match File::open(path) { Ok(f) => f, Err(e) => { @@ -280,8 +282,8 @@ pub fn set_update_authority_all( let file = File::open(json_file)?; let items: Vec = serde_json::from_reader(file)?; - // for item in items.iter() { - items.par_iter().for_each(|item| { + println!("Setting update_authority..."); + items.par_iter().progress().for_each(|item| { println!("Updating metadata for mint account: {}", item); // If someone uses a json list that contains a mint account that has already From 2aa78b81c0f4b1db31ec800442af409606efc1be Mon Sep 17 00:00:00 2001 From: Samuel Vanderwaal Date: Tue, 30 Nov 2021 12:54:53 -0900 Subject: [PATCH 12/12] setup basic logging --- Cargo.lock | 19 +++++++++++++++++-- Cargo.toml | 2 ++ src/decode.rs | 21 +++++++++++++++------ src/lib.rs | 1 + src/main.rs | 30 +++++++++++++++++++++++------- src/mint.rs | 10 ++++++---- src/opt.rs | 4 ++++ src/sign.rs | 26 +++++++++++++++++--------- src/snapshot.rs | 21 +++++++++++++-------- src/spinner.rs | 28 ++++++++++++++++++++++++++++ src/update_metadata.rs | 26 ++++++++++++++++---------- 11 files changed, 142 insertions(+), 46 deletions(-) create mode 100644 src/spinner.rs diff --git a/Cargo.lock b/Cargo.lock index 6fd23aae..693c0bfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "failure" version = "0.1.8" @@ -1566,9 +1579,11 @@ dependencies = [ "assert_cmd", "borsh", "bs58 0.4.0", + "env_logger 0.9.0", "glob", "indicatif 0.16.2", "lazy_static", + "log", "metaplex-token-metadata", "num_cpus", "ratelimit", @@ -2822,7 +2837,7 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0397bbbbf03e7a40db4d722d417d876e8b6cecb0cc4f757dbd82313bca33e54" dependencies = [ - "env_logger", + "env_logger 0.8.4", "lazy_static", "log", ] @@ -2844,7 +2859,7 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2af9cec93596c79479874772d974105c6ea388858f431d8defaa3f42d9da3c" dependencies = [ - "env_logger", + "env_logger 0.8.4", "gethostname", "lazy_static", "log", diff --git a/Cargo.toml b/Cargo.toml index 48228d39..624d8fd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,11 @@ edition = "2018" anyhow = "1.0.44" borsh = "0.9.1" bs58 = "0.4.0" +env_logger = "0.9.0" glob = "0.3.0" indicatif = { version = "0.16.2", features = ["rayon"] } lazy_static = "1.4.0" +log = "0.4.14" metaplex-token-metadata = "0.0.1" num_cpus = "1.13.0" ratelimit = "0.4.4" diff --git a/src/decode.rs b/src/decode.rs index cca55aa8..713b03c4 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result as AnyResult}; use indicatif::ParallelProgressIterator; +use log::{debug, error, info}; use metaplex_token_metadata::state::{Key, Metadata}; use rayon::prelude::*; use serde::Serialize; @@ -33,6 +34,7 @@ pub fn decode_metadata_all( let handle = create_rate_limiter(); + info!("Decoding accounts..."); println!("Decoding accounts..."); mint_accounts .par_iter() @@ -44,19 +46,20 @@ pub fn decode_metadata_all( handle.wait(); } + debug!("Decoding metadata for mint account: {}", mint_account); let metadata = match decode(client, mint_account) { Ok(m) => m, Err(err) => match err { DecodeError::ClientError(kind) => { - eprintln!("Client Error: {}!", kind); + error!("Client Error: {}!", kind); return; } DecodeError::PubkeyParseFailed(address) => { - eprintln!("Failed to parse pubkey from mint address: {}", address); + error!("Failed to parse pubkey from mint address: {}", address); return; } err => { - eprintln!( + error!( "Failed to decode metadata for mint account: {}, error: {}", mint_account, err ); @@ -65,10 +68,14 @@ pub fn decode_metadata_all( }, }; + debug!( + "Converting metadata into JSON for mint account {}", + mint_account + ); let json_metadata = match decode_to_json(metadata) { Ok(j) => j, Err(err) => { - eprintln!( + error!( "Failed to decode metadata to JSON for mint account: {}, error: {}", mint_account, err ); @@ -76,10 +83,11 @@ pub fn decode_metadata_all( } }; + debug!("Creating file for mint account: {}", mint_account); let mut file = match File::create(format!("{}/{}.json", output, mint_account)) { Ok(f) => f, Err(err) => { - eprintln!( + error!( "Failed to create JSON file for mint account: {}, error: {}", mint_account, err ); @@ -87,10 +95,11 @@ pub fn decode_metadata_all( } }; + debug!("Writing to file for mint account: {}", mint_account); match serde_json::to_writer(&mut file, &json_metadata) { Ok(_) => (), Err(err) => { - eprintln!( + error!( "Failed to write JSON file for mint account: {}, error: {}", mint_account, err ); diff --git a/src/lib.rs b/src/lib.rs index 39ff98fb..6bf41de7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,4 +9,5 @@ pub mod parse; pub mod process_subcommands; pub mod sign; pub mod snapshot; +pub mod spinner; pub mod update_metadata; diff --git a/src/main.rs b/src/main.rs index 01eeb793..cdd83158 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,9 @@ +#[macro_use] +extern crate log; + use anyhow::Result; +use env_logger::{Builder, Target}; +use log::LevelFilter; use metaboss::constants::PUBLIC_RPC_URLS; use solana_client::rpc_client::RpcClient; use solana_sdk::commitment_config::CommitmentConfig; @@ -11,30 +16,40 @@ use metaboss::opt::*; use metaboss::parse::parse_solana_config; use metaboss::process_subcommands::*; +fn setup_logging(log_level: String) -> Result<()> { + let level = LevelFilter::from_str(log_level.as_str())?; + Builder::new() + .filter_level(level) + .target(Target::Stdout) + .init(); + Ok(()) +} + fn main() -> Result<()> { + let options = Opt::from_args(); + + setup_logging(options.log_level)?; + let sol_config = parse_solana_config(); let (mut rpc, commitment) = if let Some(config) = sol_config { (config.json_rpc_url, config.commitment) } else { - eprintln!( + error!( "Could not find a valid Solana-CLI config file. Please specify a RPC manually with '-r' or set up your Solana-CLI config file." ); std::process::exit(1); }; - let options = Opt::from_args(); - if let Some(cli_rpc) = options.rpc { rpc = cli_rpc.clone(); } // Set rate limiting if the user specified a public RPC. if PUBLIC_RPC_URLS.contains(&rpc.as_str()) { - eprintln!( - r#" - WARNING: Using a public RPC URL is not recommended for heavy tasks as you will be rate-limited and suffer a performance hit. - Please use a private RPC endpoint for best performance results."# + warn!( + "Using a public RPC URL is not recommended for heavy tasks as you will be rate-limited and suffer a performance hit. + Please use a private RPC endpoint for best performance results." ); *USE_RATE_LIMIT.write().unwrap() = true; } @@ -54,5 +69,6 @@ fn main() -> Result<()> { } => process_snapshot(&client, snapshot_subcommands)?, } + println!("Done!"); Ok(()) } diff --git a/src/mint.rs b/src/mint.rs index a68e5f01..c779f0a3 100644 --- a/src/mint.rs +++ b/src/mint.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use glob::glob; use indicatif::ParallelProgressIterator; +use log::{error, info}; use metaplex_token_metadata::instruction::{create_master_edition, create_metadata_accounts}; use rayon::prelude::*; use solana_client::rpc_client::RpcClient; @@ -42,15 +43,15 @@ pub fn mint_list( paths.par_iter().progress().for_each(|path| { match mint_one(client, &keypair, &receiver, path, immutable) { Ok(_) => (), - Err(e) => eprintln!("Failed to mint {:?}: {}", &path, e), + Err(e) => error!("Failed to mint {:?}: {}", &path, e), } }); - // TODO: handle errors in a better way and log instead of print. + // TODO: handle errors in a better way. if !errors.is_empty() { - eprintln!("Failed to read some of the files with the following errors:"); + error!("Failed to read some of the files with the following errors:"); for error in errors { - eprintln!("{}", error); + error!("{}", error); } } @@ -77,6 +78,7 @@ pub fn mint_one>( let (tx_id, mint_account) = mint(client, keypair, receiver, nft_data, immutable)?; println!("Tx id: {:?}\nMint account: {:?}", tx_id, mint_account); + info!("Tx id: {:?}\nMint account: {:?}", tx_id, mint_account); Ok(()) } diff --git a/src/opt.rs b/src/opt.rs index ba00ae5c..75002585 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -11,6 +11,10 @@ pub struct Opt { #[structopt(short, long, default_value = "60")] pub timeout: u64, + /// Log level + #[structopt(short, long, default_value = "warn")] + pub log_level: String, + #[structopt(subcommand)] pub cmd: Command, } diff --git a/src/sign.rs b/src/sign.rs index 36c5fb5b..63a051c5 100644 --- a/src/sign.rs +++ b/src/sign.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; use indicatif::ParallelProgressIterator; +use log::{error, info}; use metaplex_token_metadata::{ instruction::sign_metadata, state::Metadata, ID as METAPLEX_PROGRAM_ID, }; @@ -28,8 +29,14 @@ pub fn sign_one(client: &RpcClient, keypair: String, account: String) -> Result< let metadata_pubkey = get_metadata_pda(account_pubkey); + info!( + "Signing metadata: {} with creator: {}", + metadata_pubkey, + &creator.pubkey() + ); let sig = sign(client, &creator, metadata_pubkey)?; - println!("{}", sig); + info!("Tx sig: {}", sig); + println!("Tx sig: {}", sig); Ok(()) } @@ -87,7 +94,7 @@ pub fn sign_mint_accounts( let account_pubkey = match Pubkey::from_str(&mint_account) { Ok(pubkey) => pubkey, Err(err) => { - eprintln!("Invalid public key: {}, error: {}", mint_account, err); + error!("Invalid public key: {}, error: {}", mint_account, err); return; } }; @@ -96,8 +103,8 @@ pub fn sign_mint_accounts( // Try to sign all accounts, print any errors that crop up. match sign(client, &creator, metadata_pubkey) { - Ok(sig) => println!("{}", sig), - Err(e) => println!("{}", e), + Ok(sig) => info!("{}", sig), + Err(e) => error!("{}", e), } }); @@ -122,7 +129,7 @@ pub fn sign_candy_machine_accounts( let metadata: Metadata = match try_from_slice_unchecked(&account.data.clone()) { Ok(metadata) => metadata, Err(_) => { - eprintln!("Account {} has no metadata", metadata_pubkey); + error!("Account {} has no metadata", metadata_pubkey); return; } }; @@ -131,21 +138,21 @@ pub fn sign_candy_machine_accounts( // Check whether the specific creator has already signed the account for creator in creators { if creator.address == signing_creator.pubkey() && !creator.verified { - println!( + info!( "Found creator unverified for mint account: {}", metadata.mint ); - println!("Signing..."); + info!("Signing..."); let sig = match sign(client, &signing_creator, *metadata_pubkey) { Ok(sig) => sig, Err(e) => { - eprintln!("Error signing: {}", e); + error!("Error signing: {}", e); return; } }; - println!("{}", sig); + info!("{}", sig); *signed_at_least_one_account.lock().unwrap() = true; } @@ -157,6 +164,7 @@ pub fn sign_candy_machine_accounts( }); if !*signed_at_least_one_account.lock().unwrap() { + info!("No unverified metadata for this creator and candy machine."); println!("No unverified metadata for this creator and candy machine."); return Ok(()); } diff --git a/src/snapshot.rs b/src/snapshot.rs index 9a3390d1..1071b7dd 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; use indicatif::ParallelProgressIterator; +use log::{error, info}; use metaplex_token_metadata::state::Metadata; use metaplex_token_metadata::ID as TOKEN_METADATA_PROGRAM_ID; use rayon::prelude::*; @@ -28,6 +29,7 @@ use std::{ use crate::constants::*; use crate::parse::is_only_one_option; +use crate::spinner::*; #[derive(Debug, Serialize, Clone)] struct Holder { @@ -67,7 +69,7 @@ pub fn snapshot_mints( )); } - println!("Getting accounts..."); + let spinner = create_spinner("Getting accounts..."); let accounts = if let Some(ref update_authority) = update_authority { get_mints_by_update_authority(client, &update_authority)? } else if let Some(ref candy_machine_id) = candy_machine_id { @@ -77,7 +79,9 @@ pub fn snapshot_mints( "Please specify either a candy machine id or an update authority, but not both." )); }; + spinner.finish(); + info!("Getting metadata and writing to file..."); println!("Getting metadata and writing to file..."); let mut mint_accounts: Vec = Vec::new(); @@ -109,7 +113,7 @@ pub fn snapshot_holders( candy_machine_id: &Option, output: &String, ) -> Result<()> { - println!("Getting accounts..."); + let spinner = create_spinner("Getting accounts..."); let accounts = if let Some(update_authority) = update_authority { get_mints_by_update_authority(client, update_authority)? } else if let Some(candy_machine_id) = candy_machine_id { @@ -119,11 +123,12 @@ pub fn snapshot_holders( "Must specify either --update-authority or --candy-machine-id" )); }; + spinner.finish_with_message("Getting accounts...Done!"); + info!("Finding current holders..."); println!("Finding current holders..."); let nft_holders: Arc>> = Arc::new(Mutex::new(Vec::new())); - // for (metadata_pubkey, account) in accounts { accounts .par_iter() .progress() @@ -133,7 +138,7 @@ pub fn snapshot_holders( let metadata: Metadata = match try_from_slice_unchecked(&account.data) { Ok(metadata) => metadata, Err(_) => { - eprintln!("Account {} has no metadata", metadata_pubkey); + error!("Account {} has no metadata", metadata_pubkey); return; } }; @@ -142,7 +147,7 @@ pub fn snapshot_holders( { Ok(token_accounts) => token_accounts, Err(_) => { - eprintln!("Account {} has no token accounts", metadata_pubkey); + error!("Account {} has no token accounts", metadata_pubkey); return; } }; @@ -158,7 +163,7 @@ pub fn snapshot_holders( ) { Ok(data) => data, Err(err) => { - eprintln!("Account {} has no data: {}", associated_token_address, err); + error!("Account {} has no data: {}", associated_token_address, err); return; } }; @@ -166,7 +171,7 @@ pub fn snapshot_holders( let amount = match parse_token_amount(&data) { Ok(amount) => amount, Err(err) => { - eprintln!( + error!( "Account {} has no amount: {}", associated_token_address, err ); @@ -179,7 +184,7 @@ pub fn snapshot_holders( let owner_wallet = match parse_owner(&data) { Ok(owner_wallet) => owner_wallet, Err(err) => { - eprintln!("Account {} has no owner: {}", associated_token_address, err); + error!("Account {} has no owner: {}", associated_token_address, err); return; } }; diff --git a/src/spinner.rs b/src/spinner.rs new file mode 100644 index 00000000..dd69056c --- /dev/null +++ b/src/spinner.rs @@ -0,0 +1,28 @@ +use indicatif::{ProgressBar, ProgressStyle}; + +pub fn create_spinner(msg: &'static str) -> ProgressBar { + let spinner = ProgressBar::new_spinner(); + spinner.enable_steady_tick(10); + spinner.set_style( + ProgressStyle::default_spinner() + .tick_strings(&["▹▹▹▹▹", "▸▹▹▹▹", "▹▸▹▹▹", "▹▹▸▹▹", "▹▹▹▸▹", "▹▹▹▹▸", ""]) + .template("{spinner:.blue} {msg}"), + ); + spinner.set_message(msg); + spinner +} + +pub fn create_alt_spinner(msg: &'static str) -> ProgressBar { + let spinner = ProgressBar::new_spinner(); + spinner.enable_steady_tick(80); + spinner.set_style( + ProgressStyle::default_spinner() + .tick_strings(&[ + "[ ]", "[= ]", "[== ]", "[=== ]", "[ ===]", "[ ==]", "[ =]", "[ ]", + "[ =]", "[ ==]", "[ ===]", "[====]", "[=== ]", "[== ]", "[= ]", + ]) + .template("{spinner:.blue} {msg}"), + ); + spinner.set_message(msg); + spinner +} diff --git a/src/update_metadata.rs b/src/update_metadata.rs index 05c4f91c..12537df7 100644 --- a/src/update_metadata.rs +++ b/src/update_metadata.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use glob::glob; use indicatif::ParallelProgressIterator; +use log::{error, info}; use metaplex_token_metadata::{instruction::update_metadata_accounts, state::Data}; use rayon::prelude::*; use solana_client::rpc_client::RpcClient; @@ -44,12 +45,13 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) let paths: Vec<_> = paths.into_iter().map(Result::unwrap).collect(); let errors: Vec<_> = errors.into_iter().map(Result::unwrap_err).collect(); + info!("Updating..."); println!("Updating..."); paths.par_iter().progress().for_each(|path| { let f = match File::open(path) { Ok(f) => f, Err(e) => { - eprintln!("Failed to open file: {:?} error: {}", path, e); + error!("Failed to open file: {:?} error: {}", path, e); return; } }; @@ -57,7 +59,7 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) let update_nft_data: UpdateNFTData = match serde_json::from_reader(f) { Ok(data) => data, Err(e) => { - eprintln!( + error!( "Failed to parse JSON data from file: {:?} error: {}", path, e ); @@ -68,7 +70,7 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) let data = match convert_local_to_remote_data(update_nft_data.nft_data) { Ok(data) => data, Err(e) => { - eprintln!( + error!( "Failed to convert local data to remote data: {:?} error: {}", path, e ); @@ -79,7 +81,7 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) match update_data(client, &keypair, &update_nft_data.mint_account, data) { Ok(_) => (), Err(e) => { - eprintln!("Failed to update data: {:?} error: {}", path, e); + error!("Failed to update data: {:?} error: {}", path, e); return; } } @@ -87,9 +89,9 @@ pub fn update_data_all(client: &RpcClient, keypair: &String, data_dir: &String) // TODO: handle errors in a better way and log instead of print. if !errors.is_empty() { - eprintln!("Failed to read some of the files with the following errors:"); + error!("Failed to read some of the files with the following errors:"); for error in errors { - eprintln!("{}", error); + error!("{}", error); } } @@ -125,6 +127,7 @@ pub fn update_data( ); let sig = client.send_and_confirm_transaction(&tx)?; + info!("Tx sig: {:?}", sig); println!("Tx sig: {:?}", sig); Ok(()) @@ -153,7 +156,7 @@ pub fn update_uri_all(client: &RpcClient, keypair: &String, json_file: &String) match update_uri(client, &keypair, &data.mint_account, &data.new_uri) { Ok(_) => (), Err(e) => { - eprintln!("Failed to update uri: {:?} error: {}", data, e); + error!("Failed to update uri: {:?} error: {}", data, e); return; } } @@ -196,6 +199,7 @@ pub fn update_uri( ); let sig = client.send_and_confirm_transaction(&tx)?; + info!("Tx sig: {:?}", sig); println!("Tx sig: {:?}", sig); Ok(()) @@ -231,6 +235,7 @@ pub fn set_primary_sale_happened( ); let sig = client.send_and_confirm_transaction(&tx)?; + info!("Tx sig: {:?}", sig); println!("Tx sig: {:?}", sig); Ok(()) @@ -268,6 +273,7 @@ pub fn set_update_authority( ); let sig = client.send_and_confirm_transaction(&tx)?; + info!("Tx sig: {:?}", sig); println!("Tx sig: {:?}", sig); Ok(()) @@ -282,16 +288,16 @@ pub fn set_update_authority_all( let file = File::open(json_file)?; let items: Vec = serde_json::from_reader(file)?; - println!("Setting update_authority..."); + info!("Setting update_authority..."); items.par_iter().progress().for_each(|item| { - println!("Updating metadata for mint account: {}", item); + info!("Updating metadata for mint account: {}", item); // If someone uses a json list that contains a mint account that has already // been updated this will throw an error. We print that error and continue let _ = match set_update_authority(client, keypair, &item, &new_update_authority) { Ok(_) => {} Err(error) => { - println!("Error occurred! {}", error) + error!("Error occurred! {}", error) } }; });