diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index d93b9e29d13be7..493a8b22c9d0ae 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -68,7 +68,7 @@ use { rent_collector::RentCollector, sorted_storages::SortedStorages, storable_accounts::StorableAccounts, - u64_align, + u64_align, utils, verify_accounts_hash_in_background::VerifyAccountsHashInBackground, }, blake3::traits::digest::Digest, @@ -1197,82 +1197,6 @@ impl AccountStorageEntry { } } -/// To allow generating a bank snapshot directory with full state information, we need to -/// hardlink account appendvec files from the runtime operation directory to a snapshot -/// hardlink directory. This is to create the run/ and snapshot sub directories for an -/// account_path provided by the user. These two sub directories are on the same file -/// system partition to allow hard-linking. -pub fn create_accounts_run_and_snapshot_dirs( - account_dir: impl AsRef, -) -> std::io::Result<(PathBuf, PathBuf)> { - let run_path = account_dir.as_ref().join("run"); - let snapshot_path = account_dir.as_ref().join("snapshot"); - if (!run_path.is_dir()) || (!snapshot_path.is_dir()) { - // If the "run/" or "snapshot" sub directories do not exist, the directory may be from - // an older version for which the appendvec files are at this directory. Clean up - // them first. - // This will be done only once when transitioning from an old image without run directory - // to this new version using run and snapshot directories. - // The run/ content cleanup will be done at a later point. The snapshot/ content persists - // across the process boot, and will be purged by the account_background_service. - if fs::remove_dir_all(&account_dir).is_err() { - delete_contents_of_path(&account_dir); - } - fs::create_dir_all(&run_path)?; - fs::create_dir_all(&snapshot_path)?; - } - - Ok((run_path, snapshot_path)) -} - -/// For all account_paths, create the run/ and snapshot/ sub directories. -/// If an account_path directory does not exist, create it. -/// It returns (account_run_paths, account_snapshot_paths) or error -pub fn create_all_accounts_run_and_snapshot_dirs( - account_paths: &[PathBuf], -) -> std::io::Result<(Vec, Vec)> { - let mut run_dirs = Vec::with_capacity(account_paths.len()); - let mut snapshot_dirs = Vec::with_capacity(account_paths.len()); - for account_path in account_paths { - // create the run/ and snapshot/ sub directories for each account_path - let (run_dir, snapshot_dir) = create_accounts_run_and_snapshot_dirs(account_path)?; - run_dirs.push(run_dir); - snapshot_dirs.push(snapshot_dir); - } - Ok((run_dirs, snapshot_dirs)) -} - -/// Delete the files and subdirectories in a directory. -/// This is useful if the process does not have permission -/// to delete the top level directory it might be able to -/// delete the contents of that directory. -pub fn delete_contents_of_path(path: impl AsRef) { - match fs::read_dir(&path) { - Err(err) => { - warn!( - "Failed to delete contents of '{}': could not read dir: {err}", - path.as_ref().display(), - ) - } - Ok(dir_entries) => { - for entry in dir_entries.flatten() { - let sub_path = entry.path(); - let result = if sub_path.is_dir() { - fs::remove_dir_all(&sub_path) - } else { - fs::remove_file(&sub_path) - }; - if let Err(err) = result { - warn!( - "Failed to delete contents of '{}': {err}", - sub_path.display(), - ); - } - } - } - } -} - pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec, Vec)> { let temp_dirs: IoResult> = (0..count).map(|_| TempDir::new()).collect(); let temp_dirs = temp_dirs?; @@ -1280,7 +1204,8 @@ pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec, Vec> = temp_dirs .iter() .map(|temp_dir| { - create_accounts_run_and_snapshot_dirs(temp_dir).map(|(run_dir, _snapshot_dir)| run_dir) + utils::create_accounts_run_and_snapshot_dirs(temp_dir) + .map(|(run_dir, _snapshot_dir)| run_dir) }) .collect(); let paths = paths?; diff --git a/accounts-db/src/lib.rs b/accounts-db/src/lib.rs index 61cfcdaccb194a..74fdb8627193ee 100644 --- a/accounts-db/src/lib.rs +++ b/accounts-db/src/lib.rs @@ -46,6 +46,7 @@ pub mod storable_accounts; pub mod tiered_storage; pub mod transaction_error_metrics; pub mod transaction_results; +pub mod utils; mod verify_accounts_hash_in_background; pub mod waitable_condvar; diff --git a/accounts-db/src/utils.rs b/accounts-db/src/utils.rs new file mode 100644 index 00000000000000..1e3a6855570ec0 --- /dev/null +++ b/accounts-db/src/utils.rs @@ -0,0 +1,119 @@ +use { + log::*, + std::{ + fs, + path::{Path, PathBuf}, + }, +}; + +pub const ACCOUNTS_RUN_DIR: &str = "run"; +pub const ACCOUNTS_SNAPSHOT_DIR: &str = "snapshot"; + +/// For all account_paths, create the run/ and snapshot/ sub directories. +/// If an account_path directory does not exist, create it. +/// It returns (account_run_paths, account_snapshot_paths) or error +pub fn create_all_accounts_run_and_snapshot_dirs( + account_paths: &[PathBuf], +) -> std::io::Result<(Vec, Vec)> { + let mut run_dirs = Vec::with_capacity(account_paths.len()); + let mut snapshot_dirs = Vec::with_capacity(account_paths.len()); + for account_path in account_paths { + // create the run/ and snapshot/ sub directories for each account_path + let (run_dir, snapshot_dir) = create_accounts_run_and_snapshot_dirs(account_path)?; + run_dirs.push(run_dir); + snapshot_dirs.push(snapshot_dir); + } + Ok((run_dirs, snapshot_dirs)) +} + +/// To allow generating a bank snapshot directory with full state information, we need to +/// hardlink account appendvec files from the runtime operation directory to a snapshot +/// hardlink directory. This is to create the run/ and snapshot sub directories for an +/// account_path provided by the user. These two sub directories are on the same file +/// system partition to allow hard-linking. +pub fn create_accounts_run_and_snapshot_dirs( + account_dir: impl AsRef, +) -> std::io::Result<(PathBuf, PathBuf)> { + let run_path = account_dir.as_ref().join(ACCOUNTS_RUN_DIR); + let snapshot_path = account_dir.as_ref().join(ACCOUNTS_SNAPSHOT_DIR); + if (!run_path.is_dir()) || (!snapshot_path.is_dir()) { + // If the "run/" or "snapshot" sub directories do not exist, the directory may be from + // an older version for which the appendvec files are at this directory. Clean up + // them first. + // This will be done only once when transitioning from an old image without run directory + // to this new version using run and snapshot directories. + // The run/ content cleanup will be done at a later point. The snapshot/ content persists + // across the process boot, and will be purged by the account_background_service. + if fs::remove_dir_all(&account_dir).is_err() { + delete_contents_of_path(&account_dir); + } + fs::create_dir_all(&run_path)?; + fs::create_dir_all(&snapshot_path)?; + } + + Ok((run_path, snapshot_path)) +} + +/// Delete the files and subdirectories in a directory. +/// This is useful if the process does not have permission +/// to delete the top level directory it might be able to +/// delete the contents of that directory. +pub fn delete_contents_of_path(path: impl AsRef) { + match fs::read_dir(&path) { + Err(err) => { + warn!( + "Failed to delete contents of '{}': could not read dir: {err}", + path.as_ref().display(), + ) + } + Ok(dir_entries) => { + for entry in dir_entries.flatten() { + let sub_path = entry.path(); + let result = if sub_path.is_dir() { + fs::remove_dir_all(&sub_path) + } else { + fs::remove_file(&sub_path) + }; + if let Err(err) = result { + warn!( + "Failed to delete contents of '{}': {err}", + sub_path.display(), + ); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, tempfile::TempDir}; + + #[test] + pub fn test_create_all_accounts_run_and_snapshot_dirs() { + let (_tmp_dirs, account_paths): (Vec, Vec) = (0..4) + .map(|_| { + let tmp_dir = tempfile::TempDir::new().unwrap(); + let account_path = tmp_dir.path().join("accounts"); + (tmp_dir, account_path) + }) + .unzip(); + + // create the `run/` and `snapshot/` dirs, and ensure they're there + let (account_run_paths, account_snapshot_paths) = + create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap(); + account_run_paths.iter().all(|path| path.is_dir()); + account_snapshot_paths.iter().all(|path| path.is_dir()); + + // delete a `run/` and `snapshot/` dir, then re-create it + let account_path_first = account_paths.first().unwrap(); + delete_contents_of_path(account_path_first); + assert!(account_path_first.exists()); + assert!(!account_path_first.join(ACCOUNTS_RUN_DIR).exists()); + assert!(!account_path_first.join(ACCOUNTS_SNAPSHOT_DIR).exists()); + + _ = create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap(); + account_run_paths.iter().all(|path| path.is_dir()); + account_snapshot_paths.iter().all(|path| path.is_dir()); + } +} diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 1476c2df06adbc..a5142ea2a3d65d 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -3,7 +3,9 @@ use { clap::{value_t, value_t_or_exit, values_t_or_exit, ArgMatches}, crossbeam_channel::unbounded, log::*, - solana_accounts_db::hardened_unpack::open_genesis_config, + solana_accounts_db::{ + hardened_unpack::open_genesis_config, utils::create_all_accounts_run_and_snapshot_dirs, + }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, validator::BlockVerificationMethod, }, @@ -35,8 +37,8 @@ use { snapshot_config::SnapshotConfig, snapshot_hash::StartingSnapshotHashes, snapshot_utils::{ - self, clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs, - move_and_async_delete_path_contents, SnapshotError, + self, clean_orphaned_account_snapshot_dirs, move_and_async_delete_path_contents, + SnapshotError, }, }, solana_sdk::{ @@ -68,7 +70,7 @@ pub(crate) enum LoadAndProcessLedgerError { CleanOrphanedAccountSnapshotDirectories(#[source] SnapshotError), #[error("failed to create all run and snapshot directories: {0}")] - CreateAllAccountsRunAndSnapshotDirectories(#[source] SnapshotError), + CreateAllAccountsRunAndSnapshotDirectories(#[source] std::io::Error), #[error("custom accounts path is not supported with seconday blockstore access")] CustomAccountsPathUnsupported(#[source] BlockstoreError), diff --git a/local-cluster/src/integration_tests.rs b/local-cluster/src/integration_tests.rs index 26d87d0d39ad85..db394cd394adbd 100644 --- a/local-cluster/src/integration_tests.rs +++ b/local-cluster/src/integration_tests.rs @@ -17,7 +17,7 @@ use { validator_configs::*, }, log::*, - solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs, + solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs, solana_core::{ consensus::{tower_storage::FileTowerStorage, Tower, SWITCH_FORK_THRESHOLD}, validator::{is_snapshot_config_valid, ValidatorConfig}, diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index d180a4abaf0e3a..9d1b483d85fdd3 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -6,7 +6,7 @@ use { }, itertools::izip, log::*, - solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs, + solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs, solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, solana_core::{ consensus::tower_storage::FileTowerStorage, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 74039f1a64a330..aa919e75f0366d 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -7,7 +7,7 @@ use { rand::seq::IteratorRandom, serial_test::serial, solana_accounts_db::{ - accounts_db::create_accounts_run_and_snapshot_dirs, hardened_unpack::open_genesis_config, + hardened_unpack::open_genesis_config, utils::create_accounts_run_and_snapshot_dirs, }, solana_client::thin_client::ThinClient, solana_core::{ diff --git a/runtime/src/snapshot_bank_utils.rs b/runtime/src/snapshot_bank_utils.rs index 1f734fb32b70be..5494eb1beb716c 100644 --- a/runtime/src/snapshot_bank_utils.rs +++ b/runtime/src/snapshot_bank_utils.rs @@ -13,11 +13,10 @@ use { snapshot_hash::SnapshotHash, snapshot_package::{AccountsPackage, AccountsPackageKind, SnapshotKind, SnapshotPackage}, snapshot_utils::{ - self, archive_snapshot_package, delete_contents_of_path, - deserialize_snapshot_data_file, deserialize_snapshot_data_files, get_bank_snapshot_dir, - get_highest_bank_snapshot_post, get_highest_full_snapshot_archive_info, - get_highest_incremental_snapshot_archive_info, get_snapshot_file_name, - get_storages_to_serialize, hard_link_storages_to_snapshot, + self, archive_snapshot_package, deserialize_snapshot_data_file, + deserialize_snapshot_data_files, get_bank_snapshot_dir, get_highest_bank_snapshot_post, + get_highest_full_snapshot_archive_info, get_highest_incremental_snapshot_archive_info, + get_snapshot_file_name, get_storages_to_serialize, hard_link_storages_to_snapshot, rebuild_storages_from_snapshot_dir, serialize_snapshot_data_file, verify_and_unarchive_snapshots, verify_unpacked_snapshots_dir_and_version, AddBankSnapshotError, ArchiveFormat, BankSnapshotInfo, BankSnapshotType, SnapshotError, @@ -36,6 +35,7 @@ use { accounts_hash::AccountsHash, accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, + utils::delete_contents_of_path, }, solana_measure::{measure, measure::Measure}, solana_sdk::{ @@ -1259,9 +1259,9 @@ mod tests { bank_forks::BankForks, genesis_utils, snapshot_utils::{ - clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs, - create_tmp_accounts_dir_for_tests, get_bank_snapshots, get_bank_snapshots_post, - get_bank_snapshots_pre, get_highest_bank_snapshot, purge_bank_snapshot, + clean_orphaned_account_snapshot_dirs, create_tmp_accounts_dir_for_tests, + get_bank_snapshots, get_bank_snapshots_post, get_bank_snapshots_pre, + get_highest_bank_snapshot, purge_bank_snapshot, purge_bank_snapshots_older_than_slot, purge_incomplete_bank_snapshots, purge_old_bank_snapshots, purge_old_bank_snapshots_at_startup, snapshot_storage_rebuilder::get_slot_and_append_vec_id, ArchiveFormat, @@ -2091,34 +2091,6 @@ mod tests { assert_eq!(snapshot.slot, 1); } - #[test] - pub fn test_create_all_accounts_run_and_snapshot_dirs() { - let (_tmp_dirs, account_paths): (Vec, Vec) = (0..4) - .map(|_| { - let tmp_dir = tempfile::TempDir::new().unwrap(); - let account_path = tmp_dir.path().join("accounts"); - (tmp_dir, account_path) - }) - .unzip(); - - // create the `run/` and `snapshot/` dirs, and ensure they're there - let (account_run_paths, account_snapshot_paths) = - create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap(); - account_run_paths.iter().all(|path| path.is_dir()); - account_snapshot_paths.iter().all(|path| path.is_dir()); - - // delete a `run/` and `snapshot/` dir, then re-create it - let account_path_first = account_paths.first().unwrap(); - delete_contents_of_path(account_path_first); - assert!(account_path_first.exists()); - assert!(!account_path_first.join("run").exists()); - assert!(!account_path_first.join("snapshot").exists()); - - _ = create_all_accounts_run_and_snapshot_dirs(&account_paths).unwrap(); - account_run_paths.iter().all(|path| path.is_dir()); - account_snapshot_paths.iter().all(|path| path.is_dir()); - } - #[test] fn test_clean_orphaned_account_snapshot_dirs() { let genesis_config = GenesisConfig::default(); diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index d1a04b5b77fc1c..371c12512a67a1 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -19,11 +19,12 @@ use { regex::Regex, solana_accounts_db::{ account_storage::AccountStorageMap, - accounts_db::{self, AccountStorageEntry, AtomicAppendVecId}, + accounts_db::{AccountStorageEntry, AtomicAppendVecId}, accounts_file::AccountsFileError, append_vec::AppendVec, hardened_unpack::{self, ParallelSelector, UnpackError}, shared_buffer_reader::{SharedBuffer, SharedBufferReader}, + utils::delete_contents_of_path, }, solana_measure::{measure, measure::Measure}, solana_sdk::{clock::Slot, hash::Hash}, @@ -44,7 +45,10 @@ use { thiserror::Error, }; #[cfg(feature = "dev-context-only-utils")] -use {hardened_unpack::UnpackedAppendVecMap, rayon::prelude::*}; +use { + hardened_unpack::UnpackedAppendVecMap, rayon::prelude::*, + solana_accounts_db::utils::create_accounts_run_and_snapshot_dirs, +}; mod archive_format; pub mod snapshot_storage_rebuilder; @@ -538,14 +542,6 @@ pub fn create_and_canonicalize_directories(directories: &[PathBuf]) -> Result) { - accounts_db::delete_contents_of_path(path) -} - /// Moves and asynchronously deletes the contents of a directory to avoid blocking on it. /// The directory is re-created after the move, and should now be empty. pub fn move_and_async_delete_path_contents(path: impl AsRef) { @@ -1173,17 +1169,6 @@ fn check_deserialize_file_consumed( Ok(()) } -/// For all account_paths, create the run/ and snapshot/ sub directories. -/// If an account_path directory does not exist, create it. -/// It returns (account_run_paths, account_snapshot_paths) or error -pub fn create_all_accounts_run_and_snapshot_dirs( - account_paths: &[PathBuf], -) -> Result<(Vec, Vec)> { - accounts_db::create_all_accounts_run_and_snapshot_dirs(account_paths).map_err(|err| { - SnapshotError::IoWithSource(err, "Unable to create account run and snapshot directories") - }) -} - /// Return account path from the appendvec path after checking its format. fn get_account_path_from_appendvec_path(appendvec_path: &Path) -> Option { let run_path = appendvec_path.parent()?; @@ -2083,9 +2068,7 @@ pub fn verify_snapshot_archive( ) { let temp_dir = tempfile::TempDir::new().unwrap(); let unpack_dir = temp_dir.path(); - let unpack_account_dir = accounts_db::create_accounts_run_and_snapshot_dirs(unpack_dir) - .unwrap() - .0; + let unpack_account_dir = create_accounts_run_and_snapshot_dirs(unpack_dir).unwrap().0; untar_snapshot_in( snapshot_archive, unpack_dir, @@ -2263,9 +2246,7 @@ pub fn should_take_incremental_snapshot( #[cfg(feature = "dev-context-only-utils")] pub fn create_tmp_accounts_dir_for_tests() -> (TempDir, PathBuf) { let tmp_dir = tempfile::TempDir::new().unwrap(); - let account_dir = accounts_db::create_accounts_run_and_snapshot_dirs(&tmp_dir) - .unwrap() - .0; + let account_dir = create_accounts_run_and_snapshot_dirs(&tmp_dir).unwrap().0; (tmp_dir, account_dir) } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 9f994eee9a19df..f041d80e6148e3 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -4,9 +4,9 @@ use { crossbeam_channel::Receiver, log::*, solana_accounts_db::{ - accounts_db::{create_accounts_run_and_snapshot_dirs, AccountsDbConfig}, - accounts_index::AccountsIndexConfig, + accounts_db::AccountsDbConfig, accounts_index::AccountsIndexConfig, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + utils::create_accounts_run_and_snapshot_dirs, }, solana_cli_output::CliAccount, solana_client::rpc_request::MAX_MULTIPLE_ACCOUNTS, diff --git a/validator/src/main.rs b/validator/src/main.rs index 0877c50869af52..986a38929494fc 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -14,6 +14,7 @@ use { AccountsIndexConfig, IndexLimitMb, }, partitioned_rewards::TestPartitionedEpochRewards, + utils::create_all_accounts_run_and_snapshot_dirs, }, solana_clap_utils::input_parsers::{keypair_of, keypairs_of, pubkey_of, value_of}, solana_core::{ @@ -48,8 +49,7 @@ use { snapshot_bank_utils::DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, snapshot_config::{SnapshotConfig, SnapshotUsage}, snapshot_utils::{ - self, create_all_accounts_run_and_snapshot_dirs, create_and_canonicalize_directories, - ArchiveFormat, SnapshotVersion, + self, create_and_canonicalize_directories, ArchiveFormat, SnapshotVersion, }, }, solana_sdk::{