Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Uses AccountHash in tiered storage #33763

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions accounts-db/src/account_storage/meta.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,10 @@ impl<'storage> StoredAccountMeta<'storage> {
}

pub fn hash(&self) -> &'storage AccountHash {
let hash = match self {
Self::AppendVec(av) => av.hash(),
Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH.0),
};
bytemuck::cast_ref(hash)
match self {
Self::AppendVec(av) => bytemuck::cast_ref(av.hash()),
Self::Hot(hot) => hot.hash().unwrap_or(&DEFAULT_ACCOUNT_HASH),
}
}

pub fn stored_size(&self) -> usize {
Expand Down
7 changes: 4 additions & 3 deletions accounts-db/src/tiered_storage/byte_block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ impl ByteBlockReader {
mod tests {
use {
super::*,
crate::accounts_hash::AccountHash,
solana_sdk::{hash::Hash, stake_history::Epoch},
};

Expand Down Expand Up @@ -311,7 +312,7 @@ mod tests {
// prepare a vector of optional fields that contains all combinations
// of Some and None.
for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for account_hash in [None, Some(AccountHash(Hash::new_unique()))] {
some_count += rent_epoch.iter().count() + account_hash.iter().count();

opt_fields_vec.push(AccountMetaOptionalFields {
Expand Down Expand Up @@ -351,10 +352,10 @@ mod tests {
offset += std::mem::size_of::<Epoch>();
}
if let Some(expected_hash) = opt_fields.account_hash {
let hash = read_type::<Hash>(&decoded_buffer, offset).unwrap();
let hash = read_type::<AccountHash>(&decoded_buffer, offset).unwrap();
assert_eq!(hash, &expected_hash);
verified_count += 1;
offset += std::mem::size_of::<Hash>();
offset += std::mem::size_of::<AccountHash>();
}
}

Expand Down
31 changes: 18 additions & 13 deletions accounts-db/src/tiered_storage/hot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,22 @@
//! The account meta and related structs for hot accounts.

use {
crate::tiered_storage::{
byte_block,
footer::{AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter},
index::AccountIndexFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
mmap_utils::get_type,
TieredStorageFormat, TieredStorageResult,
crate::{
accounts_hash::AccountHash,
tiered_storage::{
byte_block,
footer::{
AccountBlockFormat, AccountMetaFormat, OwnersBlockFormat, TieredStorageFooter,
},
index::AccountIndexFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
mmap_utils::get_type,
TieredStorageFormat, TieredStorageResult,
},
},
memmap2::{Mmap, MmapOptions},
modular_bitfield::prelude::*,
solana_sdk::{hash::Hash, stake_history::Epoch},
solana_sdk::stake_history::Epoch,
std::{fs::OpenOptions, option::Option, path::Path},
};

Expand Down Expand Up @@ -152,13 +157,13 @@ impl TieredAccountMeta for HotAccountMeta {

/// Returns the account hash by parsing the specified account block. None
/// will be returned if this account does not persist this optional field.
fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a Hash> {
fn account_hash<'a>(&self, account_block: &'a [u8]) -> Option<&'a AccountHash> {
self.flags()
.has_account_hash()
.then(|| {
let offset = self.optional_fields_offset(account_block)
+ AccountMetaOptionalFields::account_hash_offset(self.flags());
byte_block::read_type::<Hash>(account_block, offset)
byte_block::read_type::<AccountHash>(account_block, offset)
})
.flatten()
}
Expand Down Expand Up @@ -239,9 +244,9 @@ pub mod tests {
index::AccountIndexFormat,
meta::{AccountMetaFlags, AccountMetaOptionalFields, TieredAccountMeta},
},
::solana_sdk::{hash::Hash, pubkey::Pubkey, stake_history::Epoch},
memoffset::offset_of,
rand::Rng,
solana_sdk::{hash::Hash, pubkey::Pubkey, stake_history::Epoch},
tempfile::TempDir,
};

Expand Down Expand Up @@ -304,7 +309,7 @@ pub mod tests {

let optional_fields = AccountMetaOptionalFields {
rent_epoch: Some(TEST_RENT_EPOCH),
account_hash: Some(Hash::new_unique()),
account_hash: Some(AccountHash(Hash::new_unique())),
};

let flags = AccountMetaFlags::new_from(&optional_fields);
Expand All @@ -331,7 +336,7 @@ pub mod tests {

let optional_fields = AccountMetaOptionalFields {
rent_epoch: Some(TEST_RENT_EPOCH),
account_hash: Some(Hash::new_unique()),
account_hash: Some(AccountHash(Hash::new_unique())),
};

let flags = AccountMetaFlags::new_from(&optional_fields);
Expand Down
24 changes: 13 additions & 11 deletions accounts-db/src/tiered_storage/meta.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#![allow(dead_code)]
//! The account meta and related structs for the tiered storage.
use {
::solana_sdk::{hash::Hash, stake_history::Epoch},
modular_bitfield::prelude::*,
crate::accounts_hash::AccountHash, modular_bitfield::prelude::*,
solana_sdk::stake_history::Epoch,
};

/// The struct that handles the account meta flags.
Expand Down Expand Up @@ -65,7 +65,7 @@ pub trait TieredAccountMeta: Sized {

/// Returns the account hash by parsing the specified account block. None
/// will be returned if this account does not persist this optional field.
fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a Hash>;
fn account_hash<'a>(&self, _account_block: &'a [u8]) -> Option<&'a AccountHash>;

/// Returns the offset of the optional fields based on the specified account
/// block.
Expand Down Expand Up @@ -98,14 +98,16 @@ pub struct AccountMetaOptionalFields {
/// the epoch at which its associated account will next owe rent
pub rent_epoch: Option<Epoch>,
/// the hash of its associated account
pub account_hash: Option<Hash>,
pub account_hash: Option<AccountHash>,
}

impl AccountMetaOptionalFields {
/// The size of the optional fields in bytes (excluding the boolean flags).
pub fn size(&self) -> usize {
self.rent_epoch.map_or(0, |_| std::mem::size_of::<Epoch>())
+ self.account_hash.map_or(0, |_| std::mem::size_of::<Hash>())
+ self
.account_hash
.map_or(0, |_| std::mem::size_of::<AccountHash>())
}

/// Given the specified AccountMetaFlags, returns the size of its
Expand All @@ -116,7 +118,7 @@ impl AccountMetaOptionalFields {
fields_size += std::mem::size_of::<Epoch>();
}
if flags.has_account_hash() {
fields_size += std::mem::size_of::<Hash>();
fields_size += std::mem::size_of::<AccountHash>();
}

fields_size
Expand All @@ -142,7 +144,7 @@ impl AccountMetaOptionalFields {

#[cfg(test)]
pub mod tests {
use super::*;
use {super::*, solana_sdk::hash::Hash};

#[test]
fn test_account_meta_flags_new() {
Expand Down Expand Up @@ -194,7 +196,7 @@ pub mod tests {
let test_epoch = 5432312;

for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for account_hash in [None, Some(AccountHash(Hash::new_unique()))] {
update_and_verify_flags(&AccountMetaOptionalFields {
rent_epoch,
account_hash,
Expand All @@ -208,15 +210,15 @@ pub mod tests {
let test_epoch = 5432312;

for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for account_hash in [None, Some(AccountHash(Hash::new_unique()))] {
let opt_fields = AccountMetaOptionalFields {
rent_epoch,
account_hash,
};
assert_eq!(
opt_fields.size(),
rent_epoch.map_or(0, |_| std::mem::size_of::<Epoch>())
+ account_hash.map_or(0, |_| std::mem::size_of::<Hash>())
+ account_hash.map_or(0, |_| std::mem::size_of::<AccountHash>())
);
assert_eq!(
opt_fields.size(),
Expand All @@ -233,7 +235,7 @@ pub mod tests {
let test_epoch = 5432312;

for rent_epoch in [None, Some(test_epoch)] {
for account_hash in [None, Some(Hash::new_unique())] {
for account_hash in [None, Some(AccountHash(Hash::new_unique()))] {
let rent_epoch_offset = 0;
let account_hash_offset =
rent_epoch_offset + rent_epoch.as_ref().map(std::mem::size_of_val).unwrap_or(0);
Expand Down
17 changes: 10 additions & 7 deletions accounts-db/src/tiered_storage/readable.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
use {
crate::tiered_storage::{
footer::{AccountMetaFormat, TieredStorageFooter},
hot::HotStorageReader,
meta::TieredAccountMeta,
TieredStorageResult,
crate::{
accounts_hash::AccountHash,
tiered_storage::{
footer::{AccountMetaFormat, TieredStorageFooter},
hot::HotStorageReader,
meta::TieredAccountMeta,
TieredStorageResult,
},
},
solana_sdk::{account::ReadableAccount, hash::Hash, pubkey::Pubkey, stake_history::Epoch},
solana_sdk::{account::ReadableAccount, pubkey::Pubkey, stake_history::Epoch},
std::path::Path,
};

Expand All @@ -32,7 +35,7 @@ impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file,
}

/// Returns the hash of this account.
pub fn hash(&self) -> Option<&'accounts_file Hash> {
pub fn hash(&self) -> Option<&'accounts_file AccountHash> {
self.meta.account_hash(self.account_block)
}

Expand Down