1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
use std::{any::Any, num::NonZeroUsize};
#[cfg(feature = "gc")]
mod enabled;
#[cfg(feature = "gc")]
pub use enabled::*;
#[cfg(not(feature = "gc"))]
mod disabled;
#[cfg(not(feature = "gc"))]
pub use disabled::*;
mod gc_ref;
mod gc_runtime;
mod host_data;
mod i31;
pub use gc_ref::*;
pub use gc_runtime::*;
pub use host_data::*;
pub use i31::*;
use crate::GcHeapAllocationIndex;
use anyhow::{bail, Result};
use wasmtime_environ::StackMap;
/// Used by the runtime to lookup information about a module given a
/// program counter value.
pub trait ModuleInfoLookup {
/// Lookup the module information from a program counter value.
fn lookup(&self, pc: usize) -> Option<&dyn ModuleInfo>;
}
/// Used by the runtime to query module information.
pub trait ModuleInfo {
/// Lookup the stack map at a program counter value.
fn lookup_stack_map(&self, pc: usize) -> Option<&StackMap>;
}
/// GC-related data that is one-to-one with a `wasmtime::Store`.
///
/// Contains everything we need to do collections, invoke barriers, etc...
///
/// In general, exposes a very similar interface to `GcHeap`, but fills in some
/// of the context arguments for callers (such as the `ExternRefHostDataTable`)
/// since they are all stored together inside `GcStore`.
pub struct GcStore {
/// This GC heap's allocation index (primarily used for integrating with the
/// pooling allocator).
pub allocation_index: GcHeapAllocationIndex,
/// The actual GC heap.
pub gc_heap: Box<dyn GcHeap>,
/// The `externref` host data table for this GC heap.
pub host_data_table: ExternRefHostDataTable,
}
impl GcStore {
/// Create a new `GcStore`.
pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {
let host_data_table = ExternRefHostDataTable::default();
Self {
allocation_index,
gc_heap,
host_data_table,
}
}
/// Perform garbage collection within this heap.
pub fn gc(&mut self, roots: GcRootsIter<'_>) {
let mut collection = self.gc_heap.gc(roots, &mut self.host_data_table);
collection.collect();
}
/// Asynchronously perform garbage collection within this heap.
#[cfg(feature = "async")]
pub async fn gc_async(&mut self, roots: GcRootsIter<'_>) {
let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
collect_async(collection).await;
}
/// Clone a GC reference, calling GC write barriers as necessary.
pub fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
if gc_ref.is_i31() {
gc_ref.unchecked_copy()
} else {
self.gc_heap.clone_gc_ref(gc_ref)
}
}
/// Write the `source` GC reference into the `destination` slot, performing
/// write barriers as necessary.
pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
// If neither the source nor destination actually point to a GC object
// (that is, they are both either null or `i31ref`s) then we can skip
// the GC barrier.
if destination.as_ref().map_or(true, |d| d.is_i31())
&& source.as_ref().map_or(true, |s| s.is_i31())
{
*destination = source.map(|s| s.unchecked_copy());
return;
}
self.gc_heap
.write_gc_ref(&mut self.host_data_table, destination, source);
}
/// Drop the given GC reference, performing drop barriers as necessary.
pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
if !gc_ref.is_i31() {
self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);
}
}
/// Hook to call whenever a GC reference is about to be exposed to Wasm.
pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) {
if !gc_ref.is_i31() {
self.gc_heap.expose_gc_ref_to_wasm(gc_ref);
}
}
/// Allocate a new `externref`.
///
/// Returns:
///
/// * `Ok(Ok(_))`: Successfully allocated the `externref`.
///
/// * `Ok(Err(value))`: Failed to allocate the `externref`, but doing a GC
/// and then trying again may succeed. Returns the given `value` as the
/// error payload.
///
/// * `Err(_)`: Unrecoverable allocation failure.
pub fn alloc_externref(
&mut self,
value: Box<dyn Any + Send + Sync>,
) -> Result<Result<VMExternRef, Box<dyn Any + Send + Sync>>> {
let host_data_id = self.host_data_table.alloc(value);
match self.gc_heap.alloc_externref(host_data_id)? {
Some(x) => Ok(Ok(x)),
None => Ok(Err(self.host_data_table.dealloc(host_data_id))),
}
}
/// Get a shared borrow of the given `externref`'s host data.
///
/// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
/// associated with a different heap is memory safe but will lead to general
/// incorrectness such as panics and wrong results.
pub fn externref_host_data(&self, externref: &VMExternRef) -> &(dyn Any + Send + Sync) {
let host_data_id = self.gc_heap.externref_host_data(externref);
self.host_data_table.get(host_data_id)
}
/// Get a mutable borrow of the given `externref`'s host data.
///
/// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
/// associated with a different heap is memory safe but will lead to general
/// incorrectness such as panics and wrong results.
pub fn externref_host_data_mut(
&mut self,
externref: &VMExternRef,
) -> &mut (dyn Any + Send + Sync) {
let host_data_id = self.gc_heap.externref_host_data(externref);
self.host_data_table.get_mut(host_data_id)
}
}
/// Get a no-op GC heap for when GC is disabled (either statically at compile
/// time or dynamically due to it being turned off in the `wasmtime::Config`).
pub fn disabled_gc_heap() -> Box<dyn GcHeap> {
return Box::new(DisabledGcHeap);
struct DisabledGcHeap;
unsafe impl GcHeap for DisabledGcHeap {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn enter_no_gc_scope(&mut self) {}
fn exit_no_gc_scope(&mut self) {}
fn header(&self, _gc_ref: &VMGcRef) -> &VMGcHeader {
unreachable!()
}
fn clone_gc_ref(&mut self, _gc_ref: &VMGcRef) -> VMGcRef {
unreachable!()
}
fn write_gc_ref(
&mut self,
_host_data_table: &mut ExternRefHostDataTable,
_destination: &mut Option<VMGcRef>,
_source: Option<&VMGcRef>,
) {
unreachable!()
}
fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) {
unreachable!()
}
fn need_gc_before_entering_wasm(&self, _num_gc_refs: NonZeroUsize) -> bool {
unreachable!()
}
fn alloc_externref(
&mut self,
_host_data: ExternRefHostDataId,
) -> Result<Option<VMExternRef>> {
bail!(
"GC support disabled either in the `Config` or at compile time \
because the `gc` cargo feature was not enabled"
)
}
fn externref_host_data(&self, _externref: &VMExternRef) -> ExternRefHostDataId {
unreachable!()
}
fn gc<'a>(
&'a mut self,
_roots: GcRootsIter<'a>,
_host_data_table: &'a mut ExternRefHostDataTable,
) -> Box<dyn GarbageCollection<'a> + 'a> {
return Box::new(NoGc);
struct NoGc;
impl<'a> GarbageCollection<'a> for NoGc {
fn collect_increment(&mut self) -> GcProgress {
GcProgress::Complete
}
}
}
unsafe fn vmctx_gc_heap_base(&self) -> *mut u8 {
std::ptr::null_mut()
}
unsafe fn vmctx_gc_heap_bound(&self) -> usize {
0
}
unsafe fn vmctx_gc_heap_data(&self) -> *mut u8 {
std::ptr::null_mut()
}
#[cfg(feature = "pooling-allocator")]
fn reset(&mut self) {}
}
}