1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
use crate::{
deserialize_from, parse_append_vec_name, AccountsDbFields, AppendVec, AppendVecIterator,
DeserializableVersionedBank, Result, SerializableAccountStorageEntry, SnapshotError,
SnapshotExtractor,
};
use log::info;
use std::fs::File;
use std::io::{BufReader, Read};
use std::path::{Component, Path};
use std::pin::Pin;
use std::time::Instant;
use tar::{Archive, Entries, Entry};
pub struct ArchiveSnapshotExtractor<Source>
where
Source: Read + Unpin + 'static,
{
accounts_db_fields: AccountsDbFields<SerializableAccountStorageEntry>,
_archive: Pin<Box<Archive<zstd::Decoder<'static, BufReader<Source>>>>>,
entries: Option<Entries<'static, zstd::Decoder<'static, BufReader<Source>>>>,
}
impl<Source> SnapshotExtractor for ArchiveSnapshotExtractor<Source>
where
Source: Read + Unpin + 'static,
{
fn iter(&mut self) -> AppendVecIterator<'_> {
Box::new(self.unboxed_iter())
}
}
impl<Source> ArchiveSnapshotExtractor<Source>
where
Source: Read + Unpin + 'static,
{
pub fn from_reader(source: Source) -> Result<Self> {
let tar_stream = zstd::stream::read::Decoder::new(source)?;
let mut archive = Box::pin(Archive::new(tar_stream));
let archive_static = unsafe { &mut *((&mut *archive) as *mut Archive<_>) };
let mut entries = archive_static.entries()?;
let mut snapshot_file: Option<Entry<_>> = None;
for entry in entries.by_ref() {
let entry = entry?;
let path = entry.path()?;
if Self::is_snapshot_manifest_file(&path) {
snapshot_file = Some(entry);
break;
} else if Self::is_appendvec_file(&path) {
return Err(SnapshotError::UnexpectedAppendVec);
}
}
let snapshot_file = snapshot_file.ok_or(SnapshotError::NoSnapshotManifest)?;
let snapshot_file_path = snapshot_file.path()?.as_ref().to_path_buf();
info!("Opening snapshot manifest: {:?}", &snapshot_file_path);
let mut snapshot_file = BufReader::new(snapshot_file);
let pre_unpack = Instant::now();
let versioned_bank: DeserializableVersionedBank = deserialize_from(&mut snapshot_file)?;
drop(versioned_bank);
let versioned_bank_post_time = Instant::now();
let accounts_db_fields: AccountsDbFields<SerializableAccountStorageEntry> =
deserialize_from(&mut snapshot_file)?;
let accounts_db_fields_post_time = Instant::now();
drop(snapshot_file);
info!(
"Read bank fields in {:?}",
versioned_bank_post_time - pre_unpack
);
info!(
"Read accounts DB fields in {:?}",
accounts_db_fields_post_time - versioned_bank_post_time
);
Ok(ArchiveSnapshotExtractor {
_archive: archive,
accounts_db_fields,
entries: Some(entries),
})
}
fn unboxed_iter(&mut self) -> impl Iterator<Item = Result<AppendVec>> + '_ {
self.entries
.take()
.into_iter()
.flatten()
.filter_map(|entry| {
let mut entry = match entry {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
let path = match entry.path() {
Ok(x) => x,
Err(e) => return Some(Err(e.into())),
};
let (slot, id) = path.file_name().and_then(parse_append_vec_name)?;
Some(self.process_entry(&mut entry, slot, id))
})
}
fn process_entry(
&self,
entry: &mut Entry<'static, zstd::Decoder<'static, BufReader<Source>>>,
slot: u64,
id: u64,
) -> Result<AppendVec> {
let known_vecs = self
.accounts_db_fields
.0
.get(&slot)
.map(|v| &v[..])
.unwrap_or(&[]);
let known_vec = known_vecs.iter().find(|entry| entry.id == (id as usize));
let known_vec = match known_vec {
None => return Err(SnapshotError::UnexpectedAppendVec),
Some(v) => v,
};
Ok(AppendVec::new_from_reader(
entry,
known_vec.accounts_current_len,
)?)
}
fn is_snapshot_manifest_file(path: &Path) -> bool {
let mut components = path.components();
if components.next() != Some(Component::Normal("snapshots".as_ref())) {
return false;
}
let slot_number_str_1 = match components.next() {
Some(Component::Normal(slot)) => slot,
_ => return false,
};
if slot_number_str_1
.to_str()
.and_then(|s| s.parse::<u64>().ok())
.is_none()
{
return false;
}
let slot_number_str_2 = match components.next() {
Some(Component::Normal(slot)) => slot,
_ => return false,
};
components.next().is_none() && slot_number_str_1 == slot_number_str_2
}
fn is_appendvec_file(path: &Path) -> bool {
let mut components = path.components();
if components.next() != Some(Component::Normal("accounts".as_ref())) {
return false;
}
let name = match components.next() {
Some(Component::Normal(c)) => c,
_ => return false,
};
components.next().is_none() && parse_append_vec_name(name).is_some()
}
}
impl ArchiveSnapshotExtractor<File> {
pub fn open(path: &Path) -> Result<Self> {
Self::from_reader(File::open(path)?)
}
}