wasmer_package/package/volume/
fs.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    fs::File,
4    io::Read,
5    path::{Path, PathBuf},
6};
7
8use anyhow::{Context, Error};
9use shared_buffer::OwnedBuffer;
10
11use webc::{
12    sanitize_path,
13    v3::{
14        self,
15        write::{DirEntry, Directory, FileEntry},
16    },
17    AbstractVolume, Metadata, PathSegment, PathSegments, Timestamps, ToPathSegments,
18};
19
20use crate::package::Strictness;
21
22use super::WasmerPackageVolume;
23
24/// A lazily loaded volume in a Wasmer package.
25///
26/// Note that it is the package resolver's role to interpret a package's
27/// [`crate::metadata::annotations::FileSystemMappings`]. A [`Volume`] contains
28/// directories as they were when the package was published.
29#[derive(Debug, Clone, PartialEq)]
30pub struct FsVolume {
31    /// Name of the volume
32    name: String,
33    /// A pre-computed set of intermediate directories that are needed to allow
34    /// access to the whitelisted files and directories.
35    intermediate_directories: BTreeSet<PathBuf>,
36    /// Specific files that this volume has access to.
37    metadata_files: BTreeSet<PathBuf>,
38    /// Directories that allow the user to access anything inside them.
39    mapped_directories: BTreeSet<PathBuf>,
40    /// The base directory all [`PathSegments`] will be resolved relative to.
41    base_dir: PathBuf,
42}
43
44impl FsVolume {
45    /// The name of the volume used to store metadata files.
46    pub(crate) const METADATA: &'static str = "metadata";
47
48    /// Create a new metadata volume.
49    pub(crate) fn new_metadata(
50        manifest: &wasmer_config::package::Manifest,
51        base_dir: impl Into<PathBuf>,
52    ) -> Result<Self, Error> {
53        let base_dir = base_dir.into();
54        let mut files = BTreeSet::new();
55
56        // check if manifest.package is None
57        if let Some(package) = &manifest.package {
58            if let Some(license_file) = &package.license_file {
59                files.insert(base_dir.join(license_file));
60            }
61
62            if let Some(readme) = &package.readme {
63                files.insert(base_dir.join(readme));
64            }
65        }
66
67        for module in &manifest.modules {
68            if let Some(bindings) = &module.bindings {
69                let bindings_files = bindings.referenced_files(&base_dir)?;
70                files.extend(bindings_files);
71            }
72        }
73
74        Ok(FsVolume::new_with_intermediate_dirs(
75            FsVolume::METADATA.to_string(),
76            base_dir,
77            files,
78            BTreeSet::new(),
79        ))
80    }
81
82    pub(crate) fn new_assets(
83        manifest: &wasmer_config::package::Manifest,
84        base_dir: &Path,
85    ) -> Result<BTreeMap<String, Self>, Error> {
86        // Create asset volumes
87        let dirs: BTreeSet<_> = manifest
88            .fs
89            .values()
90            .map(|path| base_dir.join(path))
91            .collect();
92
93        for path in &dirs {
94            // Perform a basic sanity check to make sure the directories exist.
95            let _ = std::fs::metadata(path).with_context(|| {
96                format!("Unable to get the metadata for \"{}\"", path.display())
97            })?;
98        }
99
100        let mut volumes = BTreeMap::new();
101        for entry in manifest.fs.values() {
102            let name = entry
103                .to_str()
104                .ok_or_else(|| anyhow::anyhow!("Failed to convert path to str"))?;
105
106            let name = sanitize_path(name);
107
108            let mut dirs = BTreeSet::new();
109            let dir = base_dir.join(entry);
110            dirs.insert(dir);
111
112            volumes.insert(
113                name.clone(),
114                FsVolume::new(
115                    name.to_string(),
116                    base_dir.to_path_buf(),
117                    BTreeSet::new(),
118                    dirs,
119                ),
120            );
121        }
122
123        Ok(volumes)
124    }
125
126    pub(crate) fn new_with_intermediate_dirs(
127        name: String,
128        base_dir: PathBuf,
129        whitelisted_files: BTreeSet<PathBuf>,
130        whitelisted_directories: BTreeSet<PathBuf>,
131    ) -> Self {
132        let mut intermediate_directories: BTreeSet<PathBuf> = whitelisted_files
133            .iter()
134            .filter_map(|p| p.parent())
135            .chain(whitelisted_directories.iter().map(|p| p.as_path()))
136            .flat_map(|dir| dir.ancestors())
137            .filter(|dir| dir.starts_with(&base_dir))
138            .map(|dir| dir.to_path_buf())
139            .collect();
140
141        // The base directory is always accessible (even if its contents isn't)
142        intermediate_directories.insert(base_dir.clone());
143
144        FsVolume {
145            name,
146            intermediate_directories,
147            metadata_files: whitelisted_files,
148            mapped_directories: whitelisted_directories,
149            base_dir,
150        }
151    }
152
153    pub(crate) fn new(
154        name: String,
155        base_dir: PathBuf,
156        whitelisted_files: BTreeSet<PathBuf>,
157        whitelisted_directories: BTreeSet<PathBuf>,
158    ) -> Self {
159        FsVolume {
160            name,
161            intermediate_directories: BTreeSet::new(),
162            metadata_files: whitelisted_files,
163            mapped_directories: whitelisted_directories,
164            base_dir,
165        }
166    }
167
168    fn is_accessible(&self, path: &Path) -> bool {
169        self.intermediate_directories.contains(path)
170            || self.metadata_files.contains(path)
171            || self
172                .mapped_directories
173                .iter()
174                .any(|dir| path.starts_with(dir))
175    }
176
177    fn resolve(&self, path: &PathSegments) -> Option<PathBuf> {
178        let resolved = if let Some(dir) = &self.mapped_directories.first() {
179            resolve(dir, path)
180        } else {
181            resolve(&self.base_dir, path)
182        };
183
184        let accessible = self.is_accessible(&resolved);
185        accessible.then_some(resolved)
186    }
187
188    /// Returns the name of the volume
189    pub fn name(&self) -> &str {
190        self.name.as_str()
191    }
192
193    /// Read a file from the volume.
194    pub fn read_file(&self, path: &PathSegments) -> Option<OwnedBuffer> {
195        let path = self.resolve(path)?;
196        let mut f = File::open(path).ok()?;
197
198        // First we try to mmap it
199        if let Ok(mmapped) = OwnedBuffer::from_file(&f) {
200            return Some(mmapped);
201        }
202
203        // otherwise, fall back to reading the file's contents into memory
204        let mut buffer = Vec::new();
205        f.read_to_end(&mut buffer).ok()?;
206        Some(OwnedBuffer::from_bytes(buffer))
207    }
208
209    /// Read the contents of a directory.
210    #[allow(clippy::type_complexity)]
211    pub fn read_dir(
212        &self,
213        path: &PathSegments,
214    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
215        let resolved = self.resolve(path)?;
216
217        let walker = ignore::WalkBuilder::new(&resolved)
218            .require_git(true)
219            .add_custom_ignore_filename(".wasmerignore")
220            .follow_links(false)
221            .max_depth(Some(1))
222            .build();
223
224        let mut entries = Vec::new();
225
226        for entry in walker {
227            let entry = entry.ok()?;
228            // Walk returns the root dir as well, we don't want to process it
229            if entry.depth() == 0 {
230                continue;
231            }
232
233            let entry = entry.path();
234
235            if !self.is_accessible(entry) {
236                continue;
237            }
238
239            let segment: PathSegment = entry.file_name()?.to_str()?.parse().ok()?;
240
241            let path = path.join(segment.clone());
242            let metadata = self.metadata(&path)?;
243            entries.push((segment, None, metadata));
244        }
245
246        entries.sort_by_key(|k| k.0.clone());
247
248        Some(entries)
249    }
250
251    /// Get the metadata for a particular item.
252    pub fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
253        let path = self.resolve(path)?;
254        let meta = path.metadata().ok()?;
255
256        let timestamps = Timestamps::from_metadata(&meta).unwrap();
257
258        if meta.is_dir() {
259            Some(Metadata::Dir {
260                timestamps: Some(timestamps),
261            })
262        } else if meta.is_file() {
263            Some(Metadata::File {
264                length: meta.len().try_into().ok()?,
265                timestamps: Some(timestamps),
266            })
267        } else {
268            None
269        }
270    }
271
272    pub(crate) fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
273        if self.name() == "metadata" {
274            let mut root = Directory::default();
275
276            for file_path in self.metadata_files.iter() {
277                if !file_path.exists() || !file_path.is_file() {
278                    if strictness.is_strict() {
279                        anyhow::bail!("{} does not exist", file_path.display());
280                    }
281
282                    // ignore missing metadata
283                    continue;
284                }
285                let path = file_path.strip_prefix(&self.base_dir)?;
286                let path = PathBuf::from("/").join(path);
287                let segments = path.to_path_segments()?;
288                let segments: Vec<_> = segments.iter().collect();
289
290                let file_entry = DirEntry::File(FileEntry::from_path(file_path)?);
291
292                let mut curr_dir = &mut root;
293                for (index, segment) in segments.iter().enumerate() {
294                    if segments.len() == 1 {
295                        curr_dir.children.insert((*segment).clone(), file_entry);
296                        break;
297                    } else {
298                        if index == segments.len() - 1 {
299                            curr_dir.children.insert((*segment).clone(), file_entry);
300                            break;
301                        }
302
303                        let curr_entry = curr_dir
304                            .children
305                            .entry((*segment).clone())
306                            .or_insert(DirEntry::Dir(Directory::default()));
307                        let DirEntry::Dir(dir) = curr_entry else {
308                            unreachable!()
309                        };
310
311                        curr_dir = dir;
312                    }
313                }
314            }
315
316            Ok(root)
317        } else {
318            let paths: Vec<_> = self.mapped_directories.iter().cloned().collect();
319            directory_tree(paths, &self.base_dir, strictness)
320        }
321    }
322}
323
324impl AbstractVolume for FsVolume {
325    fn read_file(&self, path: &PathSegments) -> Option<(OwnedBuffer, Option<[u8; 32]>)> {
326        self.read_file(path).map(|c| (c, None))
327    }
328
329    fn read_dir(
330        &self,
331        path: &PathSegments,
332    ) -> Option<Vec<(PathSegment, Option<[u8; 32]>, Metadata)>> {
333        self.read_dir(path)
334    }
335
336    fn metadata(&self, path: &PathSegments) -> Option<Metadata> {
337        self.metadata(path)
338    }
339}
340
341impl WasmerPackageVolume for FsVolume {
342    fn as_directory_tree(&self, strictness: Strictness) -> Result<Directory<'_>, Error> {
343        self.as_directory_tree(strictness)
344    }
345}
346
347/// Resolve a [`PathSegments`] to its equivalent path on disk.
348fn resolve(base_dir: &Path, path: &PathSegments) -> PathBuf {
349    let mut resolved = base_dir.to_path_buf();
350    for segment in path.iter() {
351        resolved.push(segment.as_str());
352    }
353
354    resolved
355}
356
357/// Given a list of absolute paths, create a directory tree relative to some
358/// base directory.
359fn directory_tree(
360    paths: impl IntoIterator<Item = PathBuf>,
361    base_dir: &Path,
362    strictness: Strictness,
363) -> Result<Directory<'static>, Error> {
364    let paths: Vec<_> = paths.into_iter().collect();
365    let mut root = Directory::default();
366
367    for path in paths {
368        if path.is_file() {
369            let dir_entry = v3::write::DirEntry::File(v3::write::FileEntry::from_path(&path)?);
370            let path = path.strip_prefix(base_dir)?;
371            let path_segment = PathSegment::try_from(path.as_os_str())?;
372
373            if root.children.insert(path_segment, dir_entry).is_some() {
374                println!("Warning: {path:?} already exists. Overriding the old entry");
375            }
376        } else {
377            match webc::v3::write::Directory::from_path_with_ignore(&path) {
378                Ok(dir) => {
379                    for (path, child) in dir.children {
380                        root.children.insert(path.clone(), child);
381                    }
382                }
383                Err(e) => {
384                    let e = Error::from(e);
385                    let error = e.context(format!(
386                        "Unable to add \"{}\" to the directory tree",
387                        path.display()
388                    ));
389                    strictness.on_error(&path, error)?;
390                }
391            }
392        }
393    }
394
395    Ok(root)
396}
397
398#[cfg(test)]
399mod tests {
400    use tempfile::TempDir;
401    use wasmer_config::package::Manifest;
402
403    use super::*;
404
405    #[test]
406    fn metadata_volume() {
407        let temp = TempDir::new().unwrap();
408        let wasmer_toml = r#"
409            [package]
410            name = "some/package"
411            version = "0.0.0"
412            description = ""
413            license-file = "./path/to/LICENSE"
414            readme = "README.md"
415
416            [[module]]
417            name = "asdf"
418            source = "asdf.wasm"
419            abi = "none"
420            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
421        "#;
422        let wasmer_toml_path = temp.path().join("wasmer.toml");
423        std::fs::write(&wasmer_toml_path, wasmer_toml.as_bytes()).unwrap();
424        let license_dir = temp.path().join("path").join("to");
425        std::fs::create_dir_all(&license_dir).unwrap();
426        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
427        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
428        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
429        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
430        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
431
432        let volume = FsVolume::new_metadata(&manifest, temp.path().to_path_buf()).unwrap();
433
434        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
435        let expected = [
436            PathSegment::parse("README.md").unwrap(),
437            PathSegment::parse("asdf.wai").unwrap(),
438            PathSegment::parse("browser.wai").unwrap(),
439            PathSegment::parse("path").unwrap(),
440        ];
441
442        for i in 0..expected.len() {
443            assert_eq!(entries[i].0, expected[i]);
444            assert!(entries[i].2.timestamps().is_some());
445        }
446
447        let license: PathSegments = "/path/to/LICENSE".parse().unwrap();
448        assert_eq!(
449            String::from_utf8(volume.read_file(&license).unwrap().into()).unwrap(),
450            "license"
451        );
452    }
453
454    #[test]
455    fn asset_volume() {
456        let temp = TempDir::new().unwrap();
457        let wasmer_toml = r#"
458            [package]
459            name = "some/package"
460            version = "0.0.0"
461            description = ""
462            license_file = "./path/to/LICENSE"
463            readme = "README.md"
464
465            [[module]]
466            name = "asdf"
467            source = "asdf.wasm"
468            abi = "none"
469            bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] }
470
471            [fs]
472            "/etc" = "etc"
473        "#;
474        let license_dir = temp.path().join("path").join("to");
475        std::fs::create_dir_all(&license_dir).unwrap();
476        std::fs::write(license_dir.join("LICENSE"), "license").unwrap();
477        std::fs::write(temp.path().join("README.md"), "readme").unwrap();
478        std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap();
479        std::fs::write(temp.path().join("browser.wai"), "imports").unwrap();
480
481        let etc = temp.path().join("etc");
482        let share = etc.join("share");
483        std::fs::create_dir_all(&share).unwrap();
484
485        std::fs::write(etc.join(".wasmerignore"), b"ignore_me").unwrap();
486        std::fs::write(etc.join(".hidden"), "anything, really").unwrap();
487        std::fs::write(etc.join("ignore_me"), "I should be ignored").unwrap();
488        std::fs::write(share.join("package.1"), "man page").unwrap();
489        std::fs::write(share.join("ignore_me"), "I should be ignored too").unwrap();
490
491        let manifest: Manifest = toml::from_str(wasmer_toml).unwrap();
492
493        let volume = FsVolume::new_assets(&manifest, temp.path()).unwrap();
494
495        let volume = &volume["/etc"];
496
497        let entries = volume.read_dir(&PathSegments::ROOT).unwrap();
498        let expected = [PathSegment::parse("share").unwrap()];
499
500        for i in 0..expected.len() {
501            assert_eq!(entries[i].0, expected[i]);
502            assert!(entries[i].2.timestamps().is_some());
503        }
504
505        let man_page: PathSegments = "/share/package.1".parse().unwrap();
506        assert_eq!(
507            String::from_utf8(volume.read_file(&man_page).unwrap().into()).unwrap(),
508            "man page"
509        );
510    }
511}
wasmer_package/package/volume/fs.rs

wasmer_package/package/volume/
fs.rs
