use std::{
ops::Deref,
sync::atomic::{AtomicBool, Ordering},
time::Instant,
};
use gix_features::progress::{DynNestedProgress, MessageLevel, Progress};
use crate::{
pack,
store::verify::integrity::{IndexStatistics, SingleOrMultiStatistics},
types::IndexAndPacks,
};
pub mod integrity {
use std::{marker::PhantomData, path::PathBuf};
use crate::pack;
pub type Options<F> = pack::index::verify::integrity::Options<F>;
#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
pub enum Error {
#[error(transparent)]
MultiIndexIntegrity(#[from] pack::index::traverse::Error<pack::multi_index::verify::integrity::Error>),
#[error(transparent)]
IndexIntegrity(#[from] pack::index::traverse::Error<pack::index::verify::integrity::Error>),
#[error(transparent)]
IndexOpen(#[from] pack::index::init::Error),
#[error(transparent)]
LooseObjectStoreIntegrity(#[from] crate::loose::verify::integrity::Error),
#[error(transparent)]
MultiIndexOpen(#[from] pack::multi_index::init::Error),
#[error(transparent)]
PackOpen(#[from] pack::data::init::Error),
#[error(transparent)]
InitializeODB(#[from] crate::store::load_index::Error),
#[error("The disk on state changed while performing the operation, and we observed the change.")]
NeedsRetryDueToChangeOnDisk,
}
#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct LooseObjectStatistics {
pub path: PathBuf,
pub statistics: crate::loose::verify::integrity::Statistics,
}
#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[allow(missing_docs)]
pub enum SingleOrMultiStatistics {
Single(pack::index::traverse::Statistics),
Multi(Vec<(PathBuf, pack::index::traverse::Statistics)>),
}
#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct IndexStatistics {
pub path: PathBuf,
pub statistics: SingleOrMultiStatistics,
}
pub struct Outcome {
pub loose_object_stores: Vec<LooseObjectStatistics>,
pub index_statistics: Vec<IndexStatistics>,
}
#[derive(Debug, Copy, Clone)]
pub enum ProgressId {
VerifyLooseObjectDbPath,
VerifyIndex(PhantomData<gix_pack::index::verify::integrity::ProgressId>),
VerifyMultiIndex(PhantomData<gix_pack::multi_index::verify::integrity::ProgressId>),
}
impl From<ProgressId> for gix_features::progress::Id {
fn from(v: ProgressId) -> Self {
match v {
ProgressId::VerifyLooseObjectDbPath => *b"VISP",
ProgressId::VerifyMultiIndex(_) => *b"VIMI",
ProgressId::VerifyIndex(_) => *b"VISI",
}
}
}
}
impl super::Store {
pub fn verify_integrity<C, F>(
&self,
progress: &mut dyn DynNestedProgress,
should_interrupt: &AtomicBool,
options: integrity::Options<F>,
) -> Result<integrity::Outcome, integrity::Error>
where
C: pack::cache::DecodeEntry,
F: Fn() -> C + Send + Clone,
{
let _span = gix_features::trace::coarse!("gix_odb:Store::verify_integrity()");
let mut index = self.index.load();
if !index.is_initialized() {
self.consolidate_with_disk_state(true, false)?;
index = self.index.load();
assert!(
index.is_initialized(),
"BUG: after consolidating successfully, we have an initialized index"
)
}
progress.init(
Some(index.slot_indices.len()),
gix_features::progress::count("pack indices"),
);
let mut statistics = Vec::new();
let index_check_message = |path: &std::path::Path| {
format!(
"Checking integrity: {}",
path.file_name()
.map_or_else(Default::default, std::ffi::OsStr::to_string_lossy)
)
};
gix_features::trace::detail!("verify indices").into_scope(|| {
for slot_index in &index.slot_indices {
let slot = &self.files[*slot_index];
if slot.generation.load(Ordering::SeqCst) != index.generation {
return Err(integrity::Error::NeedsRetryDueToChangeOnDisk);
}
let files = slot.files.load();
let files = Option::as_ref(&files).ok_or(integrity::Error::NeedsRetryDueToChangeOnDisk)?;
let start = Instant::now();
let (mut child_progress, num_objects, index_path) = match files {
IndexAndPacks::Index(bundle) => {
let index;
let index = match bundle.index.loaded() {
Some(index) => index.deref(),
None => {
index = pack::index::File::at(bundle.index.path(), self.object_hash)?;
&index
}
};
let pack;
let data = match bundle.data.loaded() {
Some(pack) => pack.deref(),
None => {
pack = pack::data::File::at(bundle.data.path(), self.object_hash)?;
&pack
}
};
let mut child_progress = progress.add_child_with_id(
"verify index".into(),
integrity::ProgressId::VerifyIndex(Default::default()).into(),
);
let outcome = index.verify_integrity(
Some(pack::index::verify::PackContext {
data,
options: options.clone(),
}),
&mut child_progress,
should_interrupt,
)?;
statistics.push(IndexStatistics {
path: bundle.index.path().to_owned(),
statistics: SingleOrMultiStatistics::Single(
outcome
.pack_traverse_statistics
.expect("pack provided so there are stats"),
),
});
(child_progress, index.num_objects(), index.path().to_owned())
}
IndexAndPacks::MultiIndex(bundle) => {
let index;
let index = match bundle.multi_index.loaded() {
Some(index) => index.deref(),
None => {
index = pack::multi_index::File::at(bundle.multi_index.path())?;
&index
}
};
let mut child_progress = progress.add_child_with_id(
"verify multi-index".into(),
integrity::ProgressId::VerifyMultiIndex(Default::default()).into(),
);
let outcome = index.verify_integrity(&mut child_progress, should_interrupt, options.clone())?;
let index_dir = bundle.multi_index.path().parent().expect("file in a directory");
statistics.push(IndexStatistics {
path: Default::default(),
statistics: SingleOrMultiStatistics::Multi(
outcome
.pack_traverse_statistics
.into_iter()
.zip(index.index_names())
.map(|(statistics, index_name)| (index_dir.join(index_name), statistics))
.collect(),
),
});
(child_progress, index.num_objects(), index.path().to_owned())
}
};
child_progress.set_name(index_check_message(&index_path));
child_progress.show_throughput_with(
start,
num_objects as usize,
gix_features::progress::count("objects").expect("set"),
MessageLevel::Success,
);
progress.inc();
}
Ok(())
})?;
progress.init(
Some(index.loose_dbs.len()),
gix_features::progress::count("loose object stores"),
);
let mut loose_object_stores = Vec::new();
gix_features::trace::detail!("verify loose ODBs").into_scope(
|| -> Result<_, crate::loose::verify::integrity::Error> {
for loose_db in &*index.loose_dbs {
let out = loose_db
.verify_integrity(
&mut progress.add_child_with_id(
loose_db.path().display().to_string(),
integrity::ProgressId::VerifyLooseObjectDbPath.into(),
),
should_interrupt,
)
.map(|statistics| integrity::LooseObjectStatistics {
path: loose_db.path().to_owned(),
statistics,
})?;
loose_object_stores.push(out);
}
Ok(())
},
)?;
Ok(integrity::Outcome {
loose_object_stores,
index_statistics: statistics,
})
}
}