Struct AccountsDb

pub struct AccountsDb {

    pub accounts_index: AccountsIndex<AccountInfo, AccountInfo>,
    pub ancient_append_vec_offset: Option<i64>,
    pub skip_initial_hash_calc: bool,
    pub storage: AccountStorage,
    pub test_skip_rewrites_but_include_in_bank_hash: bool,
    pub accounts_cache: AccountsCache,
    pub next_id: AtomicAppendVecId,
    pub shrink_candidate_slots: Mutex<IntSet<Slot>>,
    pub write_version: AtomicU64,
    pub paths: Vec<PathBuf>,
    pub shrink_paths: RwLock<Option<Vec<PathBuf>>>,
    pub temp_paths: Option<Vec<TempDir>>,
    pub thread_pool: ThreadPool,
    pub thread_pool_clean: ThreadPool,
    pub stats: AccountsStats,
    pub shrink_stats: ShrinkStats,
    pub cluster_type: Option<ClusterType>,
    pub account_indexes: AccountSecondaryIndexes,
    pub verify_accounts_hash_in_bg: VerifyAccountsHashInBackground,
    pub log_dead_slots: AtomicBool,
    pub partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig,
    pub epoch_accounts_hash_manager: EpochAccountsHashManager,
    /* private fields */
}

Fields

accounts_index: AccountsIndex<AccountInfo, AccountInfo>

Keeps tracks of index into AppendVec on a per slot basis

ancient_append_vec_offset: Option<i64>

Some(offset) iff we want to squash old append vecs together into ‘ancient append vecs’ Some(offset) means for slots up to (max_slot - (slots_per_epoch - ‘offset’)), put them in ancient append vecs

skip_initial_hash_calc: bool

true iff we want to skip the initial hash calculation on startup

storage: AccountStorage

test_skip_rewrites_but_include_in_bank_hash: bool

true if this client should skip rewrites but still include those rewrites in the bank hash as if rewrites had occurred.

accounts_cache: AccountsCache

next_id: AtomicAppendVecId

distribute the accounts across storage lists

shrink_candidate_slots: Mutex<IntSet<Slot>>

Set of shrinkable stores organized by map of slot to append_vec_id

write_version: AtomicU64

paths: Vec<PathBuf>

Set of storage paths to pick from

shrink_paths: RwLock<Option<Vec<PathBuf>>>

temp_paths: Option<Vec<TempDir>>

Directory of paths this accounts_db needs to hold/remove

thread_pool: ThreadPool

Thread pool used for par_iter

thread_pool_clean: ThreadPool

stats: AccountsStats

shrink_stats: ShrinkStats

cluster_type: Option<ClusterType>

account_indexes: AccountSecondaryIndexes

verify_accounts_hash_in_bg: VerifyAccountsHashInBackground

log_dead_slots: AtomicBool

Used to disable logging dead slots during removal. allow disabling noisy log

partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig

this will live here until the feature for partitioned epoch rewards is activated. At that point, this and other code can be deleted.

epoch_accounts_hash_manager: EpochAccountsHashManager

the full accounts hash calculation as of a predetermined block height ‘N’ to be included in the bank hash at a predetermined block height ‘M’ The cadence is once per epoch, all nodes calculate a full accounts hash as of a known slot calculated using ‘N’ Some time later (to allow for slow calculation time), the bank hash at a slot calculated using ‘M’ includes the full accounts hash. Thus, the state of all accounts on a validator is known to be correct at least once per epoch.

Implementations

impl AccountsDb

pub fn notify_account_restore_from_snapshot(&self)

Notify the plugins of of account data when AccountsDb is restored from a snapshot. The data is streamed in the reverse order of the slots so that an account is only streamed once. At a slot, if the accounts is updated multiple times only the last write (with highest write_version) is notified.

pub fn notify_account_at_accounts_update<P>( &self, slot: Slot, account: &AccountSharedData, txn: &Option<&SanitizedTransaction>, pubkey: &Pubkey, write_version_producer: &mut P, )

where P: Iterator<Item = u64>,

impl AccountsDb

pub const DEFAULT_ACCOUNTS_HASH_CACHE_DIR: &'static str = "accounts_hash_cache"

pub fn default_for_tests() -> Self

pub fn new_single_for_tests() -> Self

pub fn new_for_tests(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self

pub fn new_with_config( paths: Vec<PathBuf>, cluster_type: &ClusterType, account_indexes: AccountSecondaryIndexes, shrink_ratio: AccountShrinkThreshold, accounts_db_config: Option<AccountsDbConfig>, accounts_update_notifier: Option<AccountsUpdateNotifier>, exit: Arc<AtomicBool>, ) -> Self

pub fn set_shrink_paths(&self, paths: Vec<PathBuf>)

pub fn file_size(&self) -> u64

pub fn get_base_working_path(&self) -> PathBuf

Get the base working directory

pub fn expected_cluster_type(&self) -> ClusterType

pub fn purge_keys_exact<'a, C>( &'a self, pubkey_to_slot_set: impl Iterator<Item = &'a (Pubkey, C)>, ) -> (Vec<(Slot, AccountInfo)>, HashSet<Pubkey>)

where C: Contains<'a, Slot> + 'a,

pub fn clean_accounts_for_tests(&self)

Call clean_accounts() with the common parameters that tests/benches use.

pub fn clean_accounts( &self, max_clean_root_inclusive: Option<Slot>, is_startup: bool, last_full_snapshot_slot: Option<Slot>, epoch_schedule: &EpochSchedule, )

pub fn get_unique_accounts_from_storage<'a>( &self, store: &'a Arc<AccountStorageEntry>, ) -> GetUniqueAccountsResult<'a>

get all accounts in all the storages passed in for duplicate pubkeys, the account with the highest write_value is returned

pub fn mark_dirty_dead_stores( &self, slot: Slot, add_dirty_stores: bool, shrink_in_progress: Option<ShrinkInProgress<'_>>, shrink_can_be_active: bool, ) -> Vec<Arc<AccountStorageEntry>>

get stores for ‘slot’ Drop ‘shrink_in_progress’, which will cause the old store to be removed from the storage map. For ‘shrink_in_progress’.‘old_storage’ which is not retained, insert in ‘dead_storages’ and optionally ‘dirty_stores’ This is the end of the life cycle of shrink_in_progress.

pub fn drop_or_recycle_stores( &self, dead_storages: Vec<Arc<AccountStorageEntry>>, stats: &ShrinkStats, )

pub fn get_store_for_shrink( &self, slot: Slot, aligned_total: u64, ) -> ShrinkInProgress<'_>

return a store that can contain ‘aligned_total’ bytes

pub fn shrink_ancient_slots(&self, epoch_schedule: &EpochSchedule)

get a sorted list of slots older than an epoch squash those slots into ancient append vecs

pub fn shrink_candidate_slots(&self, epoch_schedule: &EpochSchedule) -> usize

pub fn shrink_all_slots( &self, is_startup: bool, last_full_snapshot_slot: Option<Slot>, epoch_schedule: &EpochSchedule, )

pub fn scan_accounts<F>( &self, ancestors: &Ancestors, bank_id: BankId, scan_func: F, config: &ScanConfig, ) -> ScanResult<()>

where F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>),

pub fn unchecked_scan_accounts<F>( &self, metric_name: &'static str, ancestors: &Ancestors, scan_func: F, config: &ScanConfig, )

where F: FnMut(&Pubkey, LoadedAccount<'_>, Slot),

pub fn range_scan_accounts<F, R>( &self, metric_name: &'static str, ancestors: &Ancestors, range: R, config: &ScanConfig, scan_func: F, )

where F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>), R: RangeBounds<Pubkey> + Debug,

Only guaranteed to be safe when called from rent collection

pub fn index_scan_accounts<F>( &self, ancestors: &Ancestors, bank_id: BankId, index_key: IndexKey, scan_func: F, config: &ScanConfig, ) -> ScanResult<bool>

where F: FnMut(Option<(&Pubkey, AccountSharedData, Slot)>),

pub fn scan_account_storage<R, B>( &self, slot: Slot, cache_map_func: impl Fn(LoadedAccount<'_>) -> Option<R> + Sync, storage_scan_func: impl Fn(&B, LoadedAccount<'_>) + Sync, ) -> ScanStorageResult<R, B>

where R: Send, B: Send + Default + Sync,

Scan a specific slot through all the account storage

pub fn insert_default_bank_hash_stats(&self, slot: Slot, parent_slot: Slot)

Insert a default bank hash stats for slot

This fn is called when creating a new bank from parent.

pub fn load( &self, ancestors: &Ancestors, pubkey: &Pubkey, load_hint: LoadHint, ) -> Option<(AccountSharedData, Slot)>

pub fn account_matches_owners( &self, ancestors: &Ancestors, account: &Pubkey, owners: &[Pubkey], ) -> Result<usize, MatchAccountOwnerError>

Return Ok(index_of_matching_owner) if the account owner at offset is one of the pubkeys in owners. Return Err(MatchAccountOwnerError::NoMatch) if the account has 0 lamports or the owner is not one of the pubkeys in owners. Return Err(MatchAccountOwnerError::UnableToLoad) if the account could not be accessed.

pub fn load_account_into_read_cache( &self, ancestors: &Ancestors, pubkey: &Pubkey, )

load the account with pubkey into the read only accounts cache. The goal is to make subsequent loads (which caller expects to occur) to find the account quickly.

pub fn load_with_fixed_root( &self, ancestors: &Ancestors, pubkey: &Pubkey, ) -> Option<(AccountSharedData, Slot)>

note this returns None for accounts with zero lamports

pub fn flush_read_only_cache_for_tests(&self)

remove all entries from the read only accounts cache useful for benches/tests

pub fn load_account_hash( &self, ancestors: &Ancestors, pubkey: &Pubkey, max_root: Option<Slot>, load_hint: LoadHint, ) -> Option<AccountHash>

pub fn page_align(size: u64) -> u64

pub fn enable_bank_drop_callback(&self)

pub fn purge_slot( &self, slot: Slot, bank_id: BankId, is_serialized_with_abs: bool, )

This should only be called after the Bank::drop() runs in bank.rs, See BANK_DROP_SAFETY comment below for more explanation.

• is_serialized_with_abs - indicates whehter this call runs sequentially with all other accounts_db relevant calls, such as shrinking, purging etc., in account background service.

pub fn purge_slots_from_cache_and_store<'a>( &self, removed_slots: impl Iterator<Item = &'a Slot> + Clone, purge_stats: &PurgeStats, log_accounts: bool, )

Purges every slot in removed_slots from both the cache and storage. This includes entries in the accounts index, cache entries, and any backing storage entries.

pub fn remove_unrooted_slots(&self, remove_slots: &[(Slot, BankId)])

pub fn hash_account<T: ReadableAccount>( account: &T, pubkey: &Pubkey, ) -> AccountHash

pub fn mark_slot_frozen(&self, slot: Slot)

pub fn expire_old_recycle_stores(&self)

pub fn flush_accounts_cache( &self, force_flush: bool, requested_flush_root: Option<Slot>, )

pub fn combine_multiple_slots_into_one_at_startup( path: &Path, id: AppendVecId, slot: Slot, slot_stores: &HashMap<AppendVecId, Arc<AccountStorageEntry>>, ) -> Arc<AccountStorageEntry>

1.13 and some 1.14 could produce legal snapshots with more than 1 append vec per slot. This is now illegal at runtime in the validator. However, there is a clear path to be able to support this. So, combine all accounts from ‘slot_stores’ into a new storage and return it. This runs prior to the storages being put in AccountsDb.storage

pub fn checked_iterative_sum_for_capitalization( total_cap: u64, new_cap: u64, ) -> u64

pub fn checked_sum_for_capitalization<T: Iterator<Item = u64>>( balances: T, ) -> u64

pub fn calculate_accounts_hash_from_index( &self, max_slot: Slot, config: &CalcAccountsHashConfig<'_>, ) -> Result<(AccountsHash, u64), AccountsHashVerificationError>

pub fn update_accounts_hash_for_tests( &self, slot: Slot, ancestors: &Ancestors, debug_verify: bool, is_startup: bool, ) -> (AccountsHash, u64)

This is only valid to call from tests. run the accounts hash calculation and store the results

pub fn calculate_accounts_hash( &self, data_source: CalcAccountsHashDataSource, slot: Slot, config: &CalcAccountsHashConfig<'_>, ) -> Result<(AccountsHash, u64), AccountsHashVerificationError>

pub fn update_accounts_hash_with_verify( &self, data_source: CalcAccountsHashDataSource, debug_verify: bool, slot: Slot, ancestors: &Ancestors, expected_capitalization: Option<u64>, epoch_schedule: &EpochSchedule, rent_collector: &RentCollector, is_startup: bool, ) -> (AccountsHash, u64)

run the accounts hash calculation and store the results

pub fn update_accounts_hash( &self, config: &CalcAccountsHashConfig<'_>, storages: &SortedStorages<'_>, slot: Slot, stats: HashStats, ) -> Result<(AccountsHash, u64), AccountsHashVerificationError>

Calculate the full accounts hash for storages and save the results at slot

pub fn update_incremental_accounts_hash( &self, config: &CalcAccountsHashConfig<'_>, storages: &SortedStorages<'_>, slot: Slot, stats: HashStats, ) -> Result<(IncrementalAccountsHash, u64), AccountsHashVerificationError>

Calculate the incremental accounts hash for storages and save the results at slot

pub fn set_accounts_hash_from_snapshot( &mut self, slot: Slot, accounts_hash: SerdeAccountsHash, capitalization: u64, ) -> Option<(AccountsHash, u64)>

After deserializing a snapshot, set the accounts hash for the new AccountsDb

pub fn get_accounts_hash(&self, slot: Slot) -> Option<(AccountsHash, u64)>

Get the accounts hash for slot

pub fn get_accounts_hashes(&self) -> HashMap<Slot, (AccountsHash, u64)>

Get all accounts hashes

pub fn set_incremental_accounts_hash( &self, slot: Slot, incremental_accounts_hash: (IncrementalAccountsHash, u64), ) -> Option<(IncrementalAccountsHash, u64)>

Set the incremental accounts hash for slot

returns the previous incremental accounts hash for slot

pub fn set_incremental_accounts_hash_from_snapshot( &mut self, slot: Slot, incremental_accounts_hash: SerdeIncrementalAccountsHash, capitalization: u64, ) -> Option<(IncrementalAccountsHash, u64)>

After deserializing a snapshot, set the incremental accounts hash for the new AccountsDb

pub fn get_incremental_accounts_hash( &self, slot: Slot, ) -> Option<(IncrementalAccountsHash, u64)>

Get the incremental accounts hash for slot

pub fn get_incremental_accounts_hashes( &self, ) -> HashMap<Slot, (IncrementalAccountsHash, u64)>

Get all incremental accounts hashes

pub fn purge_old_accounts_hashes(&self, last_full_snapshot_slot: Slot)

Purge accounts hashes that are older than last_full_snapshot_slot

Should only be called by AccountsHashVerifier, since it consumes the accounts hashes and knows which ones are still needed.

pub fn calculate_accounts_hash_from_storages( &self, config: &CalcAccountsHashConfig<'_>, storages: &SortedStorages<'_>, stats: HashStats, ) -> Result<(AccountsHash, u64), AccountsHashVerificationError>

pub fn calculate_incremental_accounts_hash( &self, config: &CalcAccountsHashConfig<'_>, storages: &SortedStorages<'_>, stats: HashStats, ) -> Result<(IncrementalAccountsHash, u64), AccountsHashVerificationError>

Calculate the incremental accounts hash

This calculation is intended to be used by incremental snapshots, and thus differs from a “full” accounts hash in a few ways:

• Zero-lamport accounts are included in the hash because zero-lamport accounts are also included in the incremental snapshot. This ensures reconstructing the AccountsDb is still correct when using this incremental accounts hash.
• storages must be the same as the ones going into the incremental snapshot.

pub fn verify_accounts_hash_and_lamports( &self, slot: Slot, total_lamports: u64, base: Option<(Slot, u64)>, config: VerifyAccountsHashAndLamportsConfig<'_>, ) -> Result<(), AccountsHashVerificationError>

Verify accounts hash at startup (or tests)

Calculate accounts hash(es) and compare them to the values set at startup. If base is None, only calculates the full accounts hash for [0, slot]. If base is Some, calculate the full accounts hash for [0, base slot] and then calculate the incremental accounts hash for (base slot, slot].

pub fn get_pubkey_hash_for_slot( &self, slot: Slot, ) -> (Vec<(Pubkey, AccountHash)>, u64, Measure)

helper to return

1. pubkey, hash pairs for the slot
2. us spent scanning
3. Measure started when we began accumulating

pub fn get_pubkey_hash_account_for_slot( &self, slot: Slot, ) -> Vec<PubkeyHashAccount>

Return all of the accounts for a given slot

pub fn calculate_accounts_delta_hash(&self, slot: Slot) -> AccountsDeltaHash

Wrapper function to calculate accounts delta hash for slot (only used for testing and benchmarking.)

As part of calculating the accounts delta hash, get a list of accounts modified this slot (aka dirty pubkeys) and add them to self.uncleaned_pubkeys for future cleaning.

pub fn calculate_accounts_delta_hash_internal( &self, slot: Slot, ignore: Option<Pubkey>, skipped_rewrites: HashMap<Pubkey, AccountHash>, ) -> AccountsDeltaHash

Calculate accounts delta hash for slot

As part of calculating the accounts delta hash, get a list of accounts modified this slot (aka dirty pubkeys) and add them to self.uncleaned_pubkeys for future cleaning.

pub fn set_accounts_delta_hash_from_snapshot( &mut self, slot: Slot, accounts_delta_hash: SerdeAccountsDeltaHash, ) -> Option<AccountsDeltaHash>

After deserializing a snapshot, set the accounts delta hash for the new AccountsDb

pub fn get_accounts_delta_hash(&self, slot: Slot) -> Option<AccountsDeltaHash>

Get the accounts delta hash for slot in the accounts_delta_hashes map

pub fn update_bank_hash_stats_from_snapshot( &mut self, slot: Slot, stats: BankHashStats, ) -> Option<BankHashStats>

When reconstructing AccountsDb from a snapshot, insert the bank_hash_stats into the internal bank hash stats map.

This fn is only called when loading from a snapshot, which means AccountsDb is new and its bank hash stats map is unpopulated. Except for slot 0.

Slot 0 is a special case. When a new AccountsDb is created–like when loading from a snapshot–the bank hash stats map is populated with a default entry at slot 0. Remove the default entry at slot 0, and then insert the new value at slot.

pub fn get_bank_hash_stats(&self, slot: Slot) -> Option<BankHashStats>

Get the bank hash stats for slot in the bank_hash_stats map

pub fn store_cached<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( &self, accounts: impl StorableAccounts<'a, T>, transactions: Option<&'a [Option<&'a SanitizedTransaction>]>, )

pub fn store_uncached( &self, slot: Slot, accounts: &[(&Pubkey, &AccountSharedData)], )

Store the account update. only called by tests

pub fn store_accounts_frozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( &self, accounts: impl StorableAccounts<'a, T>, hashes: Option<Vec<impl Borrow<AccountHash>>>, storage: &Arc<AccountStorageEntry>, write_version_producer: Option<Box<dyn Iterator<Item = StoredMetaWriteVersion>>>, reclaim: StoreReclaims, ) -> StoreAccountsTiming

pub fn add_root(&self, slot: Slot) -> AccountsAddRootTiming

pub fn get_snapshot_storages( &self, requested_slots: impl RangeBounds<Slot> + Sync, ) -> (Vec<Arc<AccountStorageEntry>>, Vec<Slot>)

Get storages to use for snapshots, for the requested slots

pub fn generate_index( &self, limit_load_slot_count_from_snapshot: Option<usize>, verify: bool, genesis_config: &GenesisConfig, ) -> IndexGenerationInfo

pub fn print_accounts_stats(&self, label: &str)

pub fn print_count_and_status(&self, label: &str)

Trait Implementations

impl AbiExample for AccountsDb

fn example() -> Self

impl Debug for AccountsDb

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.