Add option to load accounts from file

This introduces the `--clone-from-file` option for
solana-test-validator. It allows specifying any number of files
(without extension) containing account info and data, which will be
loaded at genesis. This is similar to `--bpf-program` for programs
loading.

The files will be searched for in the CWD or in `tests/fixtures`.

Example: `solana-test-validator --clone-from-file SRM_token USD_token`

.gitignore

@@ -4,6 +4,7 @@
 /solana-metrics/
 /solana-metrics.tar.bz2
 /target/
+/test-ledger/
 
 **/*.rs.bk
 .cargo

cli-output/src/cli_output.rs

@@ -99,7 +99,7 @@ impl OutputFormat {
 pub struct CliAccount {
     #[serde(flatten)]
     pub keyed_account: RpcKeyedAccount,
-    #[serde(skip_serializing)]
+    #[serde(skip_serializing, skip_deserializing)]
     pub use_lamports_unit: bool,
 }
 

docs/src/developing/test-validator.md

@@ -14,6 +14,7 @@ starts a full-featured, single-node cluster on the developer's workstation.
 - Direct [on-chain program](on-chain-programs/overview) deployment
   (`--bpf-program ...`)
 - Clone accounts from a public cluster, including programs (`--clone ...`)
+- Load accounts from files
 - Configurable transaction history retention (`--limit-ledger-size ...`)
 - Configurable epoch length (`--slots-per-epoch ...`)
 - Jump to an arbitrary slot (`--warp-slot ...`)

test-validator/Cargo.toml

@@ -13,6 +13,9 @@ edition = "2021"
 [dependencies]
 base64 = "0.12.3"
 log = "0.4.14"
+serde_derive = "1.0.103"
+serde_json = "1.0.72"
+solana-cli-output = { path = "../cli-output", version = "=1.10.0" }
 solana-client = { path = "../client", version = "=1.10.0" }
 solana-core = { path = "../core", version = "=1.10.0" }
 solana-gossip = { path = "../gossip", version = "=1.10.0" }

test-validator/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::integer_arithmetic)]
 use {
     log::*,
+    solana_cli_output::CliAccount,
     solana_client::rpc_client::RpcClient,
     solana_core::{
         tower_storage::TowerStorage,
@@ -36,9 +37,11 @@ use {
     solana_streamer::socket::SocketAddrSpace,
     std::{
         collections::HashMap,
-        fs::remove_dir_all,
+        fs::{remove_dir_all, File},
+        io::Read,
         net::{IpAddr, Ipv4Addr, SocketAddr},
         path::{Path, PathBuf},
+        str::FromStr,
         sync::{Arc, RwLock},
         thread::sleep,
         time::Duration,
@@ -203,6 +206,37 @@ impl TestValidatorGenesis {
         self
     }
 
+    pub fn add_accounts_with_path(&mut self, filenames: Vec<&str>) -> &mut Self {
+        for filename in filenames {
+            let account_info_path = solana_program_test::find_file(&format!("{}.json", filename))
+                .unwrap_or_else(|| {
+                    error!("Unable to locate {}", filename);
+                    solana_core::validator::abort();
+                });
+            let mut file = File::open(account_info_path).unwrap();
+            let mut account_info_raw = String::new();
+            file.read_to_string(&mut account_info_raw).unwrap();
+
+            let result: serde_json::Result<CliAccount> = serde_json::from_str(&account_info_raw);
+            let account_info = match result {
+                Err(err) => {
+                    error!("Unable to deserialize {}: {}", filename, err);
+                    solana_core::validator::abort();
+                }
+                Ok(deserialized) => deserialized,
+            };
+            let address = Pubkey::from_str(account_info.keyed_account.pubkey.as_str()).unwrap();
+            let account = account_info
+                .keyed_account
+                .account
+                .decode::<AccountSharedData>()
+                .unwrap();
+
+            self.add_account(address, account);
+        }
+        self
+    }
+
     /// Add an account to the test environment with the account data in the provided `filename`
     pub fn add_account_with_file_data(
         &mut self,

validator/src/bin/solana-test-validator.rs

@@ -247,6 +247,17 @@ fn main() {
                      If the ledger already exists then this parameter is silently ignored",
                 ),
         )
+        .arg(
+            Arg::with_name("clone_account_from_file")
+                .long("clone-from-file")
+                .value_name("PATH")
+                .takes_value(true)
+                .multiple(true)
+                .help(
+                    "Loads an account from the provided JSON file. \
+                     If the ledger already exists then this parameter is silently ignored",
+                ),
+        )
         .arg(
             Arg::with_name("warp_slot")
                 .required(false)
@@ -394,7 +405,7 @@ fn main() {
         faucet_port,
     ));
 
-    let mut programs = vec![];
+    let mut programs_to_load = vec![];
     if let Some(values) = matches.values_of("bpf_program") {
         let values: Vec<&str> = values.collect::<Vec<_>>();
         for address_program in values.chunks(2) {
@@ -417,7 +428,7 @@ fn main() {
                         exit(1);
                     }
 
-                    programs.push(ProgramInfo {
+                    programs_to_load.push(ProgramInfo {
                         program_id: address,
                         loader: solana_sdk::bpf_loader::id(),
                         program_path,
@@ -428,7 +439,14 @@ fn main() {
         }
     }
 
-    let clone_accounts: HashSet<_> = pubkeys_of(&matches, "clone_account")
+    let mut accounts_to_load: Vec<&str> = matches
+        .values_of("clone_account_from_file")
+        .unwrap()
+        .collect::<Vec<_>>();
+    accounts_to_load.sort_unstable();
+    accounts_to_load.dedup();
+
+    let accounts_to_clone: HashSet<_> = pubkeys_of(&matches, "clone_account")
         .map(|v| v.into_iter().collect())
         .unwrap_or_default();
 
@@ -490,6 +508,7 @@ fn main() {
         for (name, long) in &[
             ("bpf_program", "--bpf-program"),
             ("clone_account", "--clone"),
+            ("clone_account_from_file", "--clone-from-file"),
             ("mint_address", "--mint"),
             ("slots_per_epoch", "--slots-per-epoch"),
             ("faucet_sol", "--faucet-sol"),
@@ -555,11 +574,12 @@ fn main() {
         })
         .bpf_jit(!matches.is_present("no_bpf_jit"))
         .rpc_port(rpc_port)
-        .add_programs_with_path(&programs);
+        .add_programs_with_path(&programs_to_load)
+        .add_accounts_with_path(accounts_to_load);
 
-    if !clone_accounts.is_empty() {
+    if !accounts_to_clone.is_empty() {
         genesis.clone_accounts(
-            clone_accounts,
+            accounts_to_clone,
             cluster_rpc_client
                 .as_ref()
                 .expect("bug: --url argument missing?"),