1use crate::mmap::MmapType;
9use crate::threadconditions::ThreadConditions;
10pub use crate::threadconditions::{NotifyLocation, WaiterError};
11use crate::trap::Trap;
12use crate::{mmap::Mmap, store::MaybeInstanceOwned, vmcontext::VMMemoryDefinition};
13use more_asserts::assert_ge;
14use std::cell::UnsafeCell;
15use std::convert::TryInto;
16use std::ptr::NonNull;
17use std::rc::Rc;
18use std::slice;
19use std::sync::RwLock;
20use std::time::Duration;
21use wasmer_types::{Bytes, MemoryError, MemoryStyle, MemoryType, Pages, WASM_PAGE_SIZE};
22
23#[derive(Debug)]
25struct WasmMmap {
26 alloc: Mmap,
28 size: Pages,
30 vm_memory_definition: MaybeInstanceOwned<VMMemoryDefinition>,
32}
33
34impl WasmMmap {
35 fn get_vm_memory_definition(&self) -> NonNull<VMMemoryDefinition> {
36 self.vm_memory_definition.as_ptr()
37 }
38
39 fn size(&self) -> Pages {
40 unsafe {
41 let md_ptr = self.get_vm_memory_definition();
42 let md = md_ptr.as_ref();
43 Bytes::from(md.current_length).try_into().unwrap()
44 }
45 }
46
47 fn grow(&mut self, delta: Pages, conf: VMMemoryConfig) -> Result<Pages, MemoryError> {
48 if delta.0 == 0 {
50 return Ok(self.size);
51 }
52
53 let new_pages = self
54 .size
55 .checked_add(delta)
56 .ok_or(MemoryError::CouldNotGrow {
57 current: self.size,
58 attempted_delta: delta,
59 })?;
60 let prev_pages = self.size;
61
62 if let Some(maximum) = conf.maximum {
63 if new_pages > maximum {
64 return Err(MemoryError::CouldNotGrow {
65 current: self.size,
66 attempted_delta: delta,
67 });
68 }
69 }
70
71 if new_pages > Pages::max_value() {
75 return Err(MemoryError::CouldNotGrow {
77 current: self.size,
78 attempted_delta: delta,
79 });
80 }
81
82 let delta_bytes = delta.bytes().0;
83 let prev_bytes = prev_pages.bytes().0;
84 let new_bytes = new_pages.bytes().0;
85
86 if new_bytes > self.alloc.len() - conf.offset_guard_size {
87 let guard_bytes = conf.offset_guard_size;
90 let request_bytes =
91 new_bytes
92 .checked_add(guard_bytes)
93 .ok_or_else(|| MemoryError::CouldNotGrow {
94 current: new_pages,
95 attempted_delta: Bytes(guard_bytes).try_into().unwrap(),
96 })?;
97
98 let mut new_mmap =
99 Mmap::accessible_reserved(new_bytes, request_bytes, None, MmapType::Private)
100 .map_err(MemoryError::Region)?;
101
102 let copy_len = self.alloc.len() - conf.offset_guard_size;
103 new_mmap.as_mut_slice()[..copy_len].copy_from_slice(&self.alloc.as_slice()[..copy_len]);
104
105 self.alloc = new_mmap;
106 } else if delta_bytes > 0 {
107 self.alloc
109 .make_accessible(prev_bytes, delta_bytes)
110 .map_err(MemoryError::Region)?;
111 }
112
113 self.size = new_pages;
114
115 unsafe {
117 let mut md_ptr = self.vm_memory_definition.as_ptr();
118 let md = md_ptr.as_mut();
119 md.current_length = new_pages.bytes().0;
120 md.base = self.alloc.as_mut_ptr() as _;
121 }
122
123 Ok(prev_pages)
124 }
125
126 fn grow_at_least(&mut self, min_size: u64, conf: VMMemoryConfig) -> Result<(), MemoryError> {
129 let cur_size = self.size.bytes().0 as u64;
130 if cur_size < min_size {
131 let growth = min_size - cur_size;
132 let growth_pages = ((growth - 1) / WASM_PAGE_SIZE as u64) + 1;
133 self.grow(Pages(growth_pages as u32), conf)?;
134 }
135
136 Ok(())
137 }
138
139 fn reset(&mut self) -> Result<(), MemoryError> {
141 self.size.0 = 0;
142 Ok(())
143 }
144
145 pub fn copy(&mut self) -> Result<Self, MemoryError> {
148 let mem_length = self.size.bytes().0;
149 let mut alloc = self
150 .alloc
151 .copy(Some(mem_length))
152 .map_err(MemoryError::Generic)?;
153 let base_ptr = alloc.as_mut_ptr();
154 Ok(Self {
155 vm_memory_definition: MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(
156 VMMemoryDefinition {
157 base: base_ptr,
158 current_length: mem_length,
159 },
160 ))),
161 alloc,
162 size: self.size,
163 })
164 }
165}
166
167#[derive(Debug, Clone)]
169struct VMMemoryConfig {
170 maximum: Option<Pages>,
172 memory: MemoryType,
174 style: MemoryStyle,
176 offset_guard_size: usize,
179}
180
181impl VMMemoryConfig {
182 fn ty(&self, minimum: Pages) -> MemoryType {
183 let mut out = self.memory;
184 out.minimum = minimum;
185
186 out
187 }
188
189 fn style(&self) -> MemoryStyle {
190 self.style
191 }
192}
193
194#[derive(Debug)]
196pub struct VMOwnedMemory {
197 mmap: WasmMmap,
199 config: VMMemoryConfig,
201}
202
203unsafe impl Send for VMOwnedMemory {}
204unsafe impl Sync for VMOwnedMemory {}
205
206impl VMOwnedMemory {
207 pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
212 unsafe { Self::new_internal(memory, style, None, None, MmapType::Private) }
213 }
214
215 pub fn new_with_file(
222 memory: &MemoryType,
223 style: &MemoryStyle,
224 backing_file: std::path::PathBuf,
225 memory_type: MmapType,
226 ) -> Result<Self, MemoryError> {
227 unsafe { Self::new_internal(memory, style, None, Some(backing_file), memory_type) }
228 }
229
230 pub unsafe fn from_definition(
238 memory: &MemoryType,
239 style: &MemoryStyle,
240 vm_memory_location: NonNull<VMMemoryDefinition>,
241 ) -> Result<Self, MemoryError> {
242 Self::new_internal(
243 memory,
244 style,
245 Some(vm_memory_location),
246 None,
247 MmapType::Private,
248 )
249 }
250
251 pub unsafe fn from_definition_with_file(
261 memory: &MemoryType,
262 style: &MemoryStyle,
263 vm_memory_location: NonNull<VMMemoryDefinition>,
264 backing_file: Option<std::path::PathBuf>,
265 memory_type: MmapType,
266 ) -> Result<Self, MemoryError> {
267 Self::new_internal(
268 memory,
269 style,
270 Some(vm_memory_location),
271 backing_file,
272 memory_type,
273 )
274 }
275
276 unsafe fn new_internal(
278 memory: &MemoryType,
279 style: &MemoryStyle,
280 vm_memory_location: Option<NonNull<VMMemoryDefinition>>,
281 backing_file: Option<std::path::PathBuf>,
282 memory_type: MmapType,
283 ) -> Result<Self, MemoryError> {
284 if memory.minimum > Pages::max_value() {
285 return Err(MemoryError::MinimumMemoryTooLarge {
286 min_requested: memory.minimum,
287 max_allowed: Pages::max_value(),
288 });
289 }
290 if let Some(max) = memory.maximum {
292 if max > Pages::max_value() {
293 return Err(MemoryError::MaximumMemoryTooLarge {
294 max_requested: max,
295 max_allowed: Pages::max_value(),
296 });
297 }
298 if max < memory.minimum {
299 return Err(MemoryError::InvalidMemory {
300 reason: format!(
301 "the maximum ({} pages) is less than the minimum ({} pages)",
302 max.0, memory.minimum.0
303 ),
304 });
305 }
306 }
307
308 let offset_guard_bytes = style.offset_guard_size() as usize;
309
310 let minimum_pages = match style {
311 MemoryStyle::Dynamic { .. } => memory.minimum,
312 MemoryStyle::Static { bound, .. } => {
313 assert_ge!(*bound, memory.minimum);
314 *bound
315 }
316 };
317 let minimum_bytes = minimum_pages.bytes().0;
318 let request_bytes = minimum_bytes.checked_add(offset_guard_bytes).unwrap();
319 let mapped_pages = memory.minimum;
320 let mapped_bytes = mapped_pages.bytes();
321
322 let mut alloc =
323 Mmap::accessible_reserved(mapped_bytes.0, request_bytes, backing_file, memory_type)
324 .map_err(MemoryError::Region)?;
325
326 let base_ptr = alloc.as_mut_ptr();
327 let mem_length = memory
328 .minimum
329 .bytes()
330 .0
331 .max(alloc.as_slice_accessible().len());
332 let mmap = WasmMmap {
333 vm_memory_definition: if let Some(mem_loc) = vm_memory_location {
334 {
335 let mut ptr = mem_loc;
336 let md = ptr.as_mut();
337 md.base = base_ptr;
338 md.current_length = mem_length;
339 }
340 MaybeInstanceOwned::Instance(mem_loc)
341 } else {
342 MaybeInstanceOwned::Host(Box::new(UnsafeCell::new(VMMemoryDefinition {
343 base: base_ptr,
344 current_length: mem_length,
345 })))
346 },
347 alloc,
348 size: Bytes::from(mem_length).try_into().unwrap(),
349 };
350
351 Ok(Self {
352 mmap,
353 config: VMMemoryConfig {
354 maximum: memory.maximum,
355 offset_guard_size: offset_guard_bytes,
356 memory: *memory,
357 style: *style,
358 },
359 })
360 }
361
362 pub fn to_shared(self) -> VMSharedMemory {
364 VMSharedMemory {
365 mmap: Rc::new(RwLock::new(self.mmap)),
366 config: self.config,
367 conditions: ThreadConditions::new(),
368 }
369 }
370
371 pub fn copy(&mut self) -> Result<Self, MemoryError> {
373 Ok(Self {
374 mmap: self.mmap.copy()?,
375 config: self.config.clone(),
376 })
377 }
378}
379
380impl LinearMemory for VMOwnedMemory {
381 fn ty(&self) -> MemoryType {
383 let minimum = self.mmap.size();
384 self.config.ty(minimum)
385 }
386
387 fn size(&self) -> Pages {
389 self.mmap.size()
390 }
391
392 fn style(&self) -> MemoryStyle {
394 self.config.style()
395 }
396
397 fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
402 self.mmap.grow(delta, self.config.clone())
403 }
404
405 fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> {
408 self.mmap.grow_at_least(min_size, self.config.clone())
409 }
410
411 fn reset(&mut self) -> Result<(), MemoryError> {
413 self.mmap.reset()?;
414 Ok(())
415 }
416
417 fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
419 self.mmap.vm_memory_definition.as_ptr()
420 }
421
422 fn try_clone(&self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
424 Err(MemoryError::MemoryNotShared)
425 }
426
427 fn copy(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
429 let forked = Self::copy(self)?;
430 Ok(Box::new(forked))
431 }
432}
433
434#[derive(Debug, Clone)]
436pub struct VMSharedMemory {
437 mmap: Rc<RwLock<WasmMmap>>,
439 config: VMMemoryConfig,
441 conditions: ThreadConditions,
443}
444
445unsafe impl Send for VMSharedMemory {}
446unsafe impl Sync for VMSharedMemory {}
447
448impl VMSharedMemory {
449 pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
454 Ok(VMOwnedMemory::new(memory, style)?.to_shared())
455 }
456
457 pub fn new_with_file(
464 memory: &MemoryType,
465 style: &MemoryStyle,
466 backing_file: std::path::PathBuf,
467 memory_type: MmapType,
468 ) -> Result<Self, MemoryError> {
469 Ok(VMOwnedMemory::new_with_file(memory, style, backing_file, memory_type)?.to_shared())
470 }
471
472 pub unsafe fn from_definition(
480 memory: &MemoryType,
481 style: &MemoryStyle,
482 vm_memory_location: NonNull<VMMemoryDefinition>,
483 ) -> Result<Self, MemoryError> {
484 Ok(VMOwnedMemory::from_definition(memory, style, vm_memory_location)?.to_shared())
485 }
486
487 pub unsafe fn from_definition_with_file(
497 memory: &MemoryType,
498 style: &MemoryStyle,
499 vm_memory_location: NonNull<VMMemoryDefinition>,
500 backing_file: Option<std::path::PathBuf>,
501 memory_type: MmapType,
502 ) -> Result<Self, MemoryError> {
503 Ok(VMOwnedMemory::from_definition_with_file(
504 memory,
505 style,
506 vm_memory_location,
507 backing_file,
508 memory_type,
509 )?
510 .to_shared())
511 }
512
513 pub fn copy(&mut self) -> Result<Self, MemoryError> {
515 let mut guard = self.mmap.write().unwrap();
516 Ok(Self {
517 mmap: Rc::new(RwLock::new(guard.copy()?)),
518 config: self.config.clone(),
519 conditions: ThreadConditions::new(),
520 })
521 }
522}
523
524impl LinearMemory for VMSharedMemory {
525 fn ty(&self) -> MemoryType {
527 let minimum = {
528 let guard = self.mmap.read().unwrap();
529 guard.size()
530 };
531 self.config.ty(minimum)
532 }
533
534 fn size(&self) -> Pages {
536 let guard = self.mmap.read().unwrap();
537 guard.size()
538 }
539
540 fn style(&self) -> MemoryStyle {
542 self.config.style()
543 }
544
545 fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
550 let mut guard = self.mmap.write().unwrap();
551 guard.grow(delta, self.config.clone())
552 }
553
554 fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> {
557 let mut guard = self.mmap.write().unwrap();
558 guard.grow_at_least(min_size, self.config.clone())
559 }
560
561 fn reset(&mut self) -> Result<(), MemoryError> {
563 let mut guard = self.mmap.write().unwrap();
564 guard.reset()?;
565 Ok(())
566 }
567
568 fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
570 let guard = self.mmap.read().unwrap();
571 guard.vm_memory_definition.as_ptr()
572 }
573
574 fn try_clone(&self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
576 Ok(Box::new(self.clone()))
577 }
578
579 fn copy(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
581 let forked = Self::copy(self)?;
582 Ok(Box::new(forked))
583 }
584
585 fn do_wait(
587 &mut self,
588 dst: NotifyLocation,
589 timeout: Option<Duration>,
590 ) -> Result<u32, WaiterError> {
591 self.conditions.do_wait(dst, timeout)
592 }
593
594 fn do_notify(&mut self, dst: NotifyLocation, count: u32) -> u32 {
596 self.conditions.do_notify(dst, count)
597 }
598
599 fn thread_conditions(&self) -> Option<&ThreadConditions> {
600 Some(&self.conditions)
601 }
602}
603
604impl From<VMOwnedMemory> for VMMemory {
605 fn from(mem: VMOwnedMemory) -> Self {
606 Self(Box::new(mem))
607 }
608}
609
610impl From<VMSharedMemory> for VMMemory {
611 fn from(mem: VMSharedMemory) -> Self {
612 Self(Box::new(mem))
613 }
614}
615
616#[derive(Debug)]
618pub struct VMMemory(pub Box<dyn LinearMemory + 'static>);
619
620impl From<Box<dyn LinearMemory + 'static>> for VMMemory {
621 fn from(mem: Box<dyn LinearMemory + 'static>) -> Self {
622 Self(mem)
623 }
624}
625
626impl LinearMemory for VMMemory {
627 fn ty(&self) -> MemoryType {
629 self.0.ty()
630 }
631
632 fn size(&self) -> Pages {
634 self.0.size()
635 }
636
637 fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError> {
642 self.0.grow(delta)
643 }
644
645 fn grow_at_least(&mut self, min_size: u64) -> Result<(), MemoryError> {
648 self.0.grow_at_least(min_size)
649 }
650
651 fn reset(&mut self) -> Result<(), MemoryError> {
653 self.0.reset()?;
654 Ok(())
655 }
656
657 fn style(&self) -> MemoryStyle {
659 self.0.style()
660 }
661
662 fn vmmemory(&self) -> NonNull<VMMemoryDefinition> {
664 self.0.vmmemory()
665 }
666
667 fn try_clone(&self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
669 self.0.try_clone()
670 }
671
672 unsafe fn initialize_with_data(&self, start: usize, data: &[u8]) -> Result<(), Trap> {
674 self.0.initialize_with_data(start, data)
675 }
676
677 fn copy(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
679 self.0.copy()
680 }
681
682 fn do_wait(
684 &mut self,
685 dst: NotifyLocation,
686 timeout: Option<Duration>,
687 ) -> Result<u32, WaiterError> {
688 self.0.do_wait(dst, timeout)
689 }
690
691 fn do_notify(&mut self, dst: NotifyLocation, count: u32) -> u32 {
693 self.0.do_notify(dst, count)
694 }
695
696 fn thread_conditions(&self) -> Option<&ThreadConditions> {
697 self.0.thread_conditions()
698 }
699}
700
701impl VMMemory {
702 pub fn new(memory: &MemoryType, style: &MemoryStyle) -> Result<Self, MemoryError> {
708 Ok(if memory.shared {
709 Self(Box::new(VMSharedMemory::new(memory, style)?))
710 } else {
711 Self(Box::new(VMOwnedMemory::new(memory, style)?))
712 })
713 }
714
715 pub fn get_runtime_size(&self) -> u32 {
717 self.0.size().0
718 }
719
720 pub unsafe fn from_definition(
728 memory: &MemoryType,
729 style: &MemoryStyle,
730 vm_memory_location: NonNull<VMMemoryDefinition>,
731 ) -> Result<Self, MemoryError> {
732 Ok(if memory.shared {
733 Self(Box::new(VMSharedMemory::from_definition(
734 memory,
735 style,
736 vm_memory_location,
737 )?))
738 } else {
739 Self(Box::new(VMOwnedMemory::from_definition(
740 memory,
741 style,
742 vm_memory_location,
743 )?))
744 })
745 }
746
747 pub fn from_custom<IntoVMMemory>(memory: IntoVMMemory) -> Self
752 where
753 IntoVMMemory: Into<Self>,
754 {
755 memory.into()
756 }
757
758 pub fn copy(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError> {
760 LinearMemory::copy(self)
761 }
762}
763
764#[doc(hidden)]
765pub unsafe fn initialize_memory_with_data(
767 memory: &VMMemoryDefinition,
768 start: usize,
769 data: &[u8],
770) -> Result<(), Trap> {
771 let mem_slice = slice::from_raw_parts_mut(memory.base, memory.current_length);
772 let end = start + data.len();
773 let to_init = &mut mem_slice[start..end];
774 to_init.copy_from_slice(data);
775
776 Ok(())
777}
778
779pub trait LinearMemory
781where
782 Self: std::fmt::Debug + Send,
783{
784 fn ty(&self) -> MemoryType;
786
787 fn size(&self) -> Pages;
789
790 fn style(&self) -> MemoryStyle;
792
793 fn grow(&mut self, delta: Pages) -> Result<Pages, MemoryError>;
798
799 fn grow_at_least(&mut self, _min_size: u64) -> Result<(), MemoryError> {
802 Err(MemoryError::UnsupportedOperation {
803 message: "grow_at_least() is not supported".to_string(),
804 })
805 }
806
807 fn reset(&mut self) -> Result<(), MemoryError> {
809 Err(MemoryError::UnsupportedOperation {
810 message: "reset() is not supported".to_string(),
811 })
812 }
813
814 fn vmmemory(&self) -> NonNull<VMMemoryDefinition>;
816
817 fn try_clone(&self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError>;
819
820 #[doc(hidden)]
821 unsafe fn initialize_with_data(&self, start: usize, data: &[u8]) -> Result<(), Trap> {
825 let memory = self.vmmemory().as_ref();
826
827 initialize_memory_with_data(memory, start, data)
828 }
829
830 fn copy(&mut self) -> Result<Box<dyn LinearMemory + 'static>, MemoryError>;
832
833 fn do_wait(
836 &mut self,
837 _dst: NotifyLocation,
838 _timeout: Option<Duration>,
839 ) -> Result<u32, WaiterError> {
840 Err(WaiterError::Unimplemented)
841 }
842
843 fn do_notify(&mut self, _dst: NotifyLocation, _count: u32) -> u32 {
845 0
846 }
847
848 fn thread_conditions(&self) -> Option<&ThreadConditions> {
852 None
853 }
854}