1use crate::{
22 host::HostState,
23 instance_wrapper::{EntryPoint, InstanceWrapper, MemoryWrapper},
24 util::{self, replace_strategy_if_broken},
25};
26
27use parking_lot::Mutex;
28use sc_allocator::{AllocationStats, FreeingBumpHeapAllocator};
29use sc_executor_common::{
30 error::{Error, Result, WasmError},
31 runtime_blob::RuntimeBlob,
32 util::checked_range,
33 wasm_runtime::{HeapAllocStrategy, WasmInstance, WasmModule},
34};
35use sp_runtime_interface::unpack_ptr_and_len;
36use sp_wasm_interface::{HostFunctions, Pointer, WordSize};
37use std::{
38 path::{Path, PathBuf},
39 sync::{
40 atomic::{AtomicBool, Ordering},
41 Arc,
42 },
43};
44use wasmtime::{AsContext, Engine, Memory};
45
46const MAX_INSTANCE_COUNT: u32 = 64;
47
48#[derive(Default)]
49pub(crate) struct StoreData {
50 pub(crate) host_state: Option<HostState>,
52 pub(crate) memory: Option<Memory>,
54}
55
56impl StoreData {
57 pub fn host_state_mut(&mut self) -> Option<&mut HostState> {
59 self.host_state.as_mut()
60 }
61
62 pub fn memory(&self) -> Memory {
64 self.memory.expect("memory is always set; qed")
65 }
66}
67
68pub(crate) type Store = wasmtime::Store<StoreData>;
69
70enum Strategy {
71 RecreateInstance(InstanceCreator),
72}
73
74struct InstanceCreator {
75 engine: Engine,
76 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
77 instance_counter: Arc<InstanceCounter>,
78}
79
80impl InstanceCreator {
81 fn instantiate(&mut self) -> Result<InstanceWrapper> {
82 InstanceWrapper::new(&self.engine, &self.instance_pre, self.instance_counter.clone())
83 }
84}
85
86pub(crate) struct ReleaseInstanceHandle {
88 counter: Arc<InstanceCounter>,
89}
90
91impl Drop for ReleaseInstanceHandle {
92 fn drop(&mut self) {
93 {
94 let mut counter = self.counter.counter.lock();
95 *counter = counter.saturating_sub(1);
96 }
97
98 self.counter.wait_for_instance.notify_one();
99 }
100}
101
102#[derive(Default)]
109pub(crate) struct InstanceCounter {
110 counter: Mutex<u32>,
111 wait_for_instance: parking_lot::Condvar,
112}
113
114impl InstanceCounter {
115 pub fn acquire_instance(self: Arc<Self>) -> ReleaseInstanceHandle {
122 let mut counter = self.counter.lock();
123
124 while *counter >= MAX_INSTANCE_COUNT {
125 self.wait_for_instance.wait(&mut counter);
126 }
127 *counter += 1;
128
129 ReleaseInstanceHandle { counter: self.clone() }
130 }
131}
132
133pub struct WasmtimeRuntime {
136 engine: Engine,
137 instance_pre: Arc<wasmtime::InstancePre<StoreData>>,
138 instantiation_strategy: InternalInstantiationStrategy,
139 instance_counter: Arc<InstanceCounter>,
140}
141
142impl WasmModule for WasmtimeRuntime {
143 fn new_instance(&self) -> Result<Box<dyn WasmInstance>> {
144 let strategy = match self.instantiation_strategy {
145 InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator {
146 engine: self.engine.clone(),
147 instance_pre: self.instance_pre.clone(),
148 instance_counter: self.instance_counter.clone(),
149 }),
150 };
151
152 Ok(Box::new(WasmtimeInstance { strategy }))
153 }
154}
155
156pub struct WasmtimeInstance {
159 strategy: Strategy,
160}
161
162impl WasmtimeInstance {
163 fn call_impl(
164 &mut self,
165 method: &str,
166 data: &[u8],
167 allocation_stats: &mut Option<AllocationStats>,
168 ) -> Result<Vec<u8>> {
169 match &mut self.strategy {
170 Strategy::RecreateInstance(ref mut instance_creator) => {
171 let mut instance_wrapper = instance_creator.instantiate()?;
172 let heap_base = instance_wrapper.extract_heap_base()?;
173 let entrypoint = instance_wrapper.resolve_entrypoint(method)?;
174 let allocator = FreeingBumpHeapAllocator::new(heap_base);
175
176 perform_call(data, &mut instance_wrapper, entrypoint, allocator, allocation_stats)
177 },
178 }
179 }
180}
181
182impl WasmInstance for WasmtimeInstance {
183 fn call_with_allocation_stats(
184 &mut self,
185 method: &str,
186 data: &[u8],
187 ) -> (Result<Vec<u8>>, Option<AllocationStats>) {
188 let mut allocation_stats = None;
189 let result = self.call_impl(method, data, &mut allocation_stats);
190 (result, allocation_stats)
191 }
192}
193
194fn setup_wasmtime_caching(
198 cache_path: &Path,
199 config: &mut wasmtime::Config,
200) -> std::result::Result<(), String> {
201 use std::fs;
202
203 let wasmtime_cache_root = cache_path.join("wasmtime");
204 fs::create_dir_all(&wasmtime_cache_root)
205 .map_err(|err| format!("cannot create the dirs to cache: {}", err))?;
206
207 let wasmtime_cache_root = wasmtime_cache_root
209 .canonicalize()
210 .map_err(|err| format!("failed to canonicalize the path: {}", err))?;
211
212 let cache_config_path = wasmtime_cache_root.join("cache-config.toml");
214 let config_content = format!(
215 "\
216[cache]
217enabled = true
218directory = \"{cache_dir}\"
219",
220 cache_dir = wasmtime_cache_root.display()
221 );
222 fs::write(&cache_config_path, config_content)
223 .map_err(|err| format!("cannot write the cache config: {}", err))?;
224
225 config
226 .cache_config_load(cache_config_path)
227 .map_err(|err| format!("failed to parse the config: {:#}", err))?;
228
229 Ok(())
230}
231
232fn common_config(semantics: &Semantics) -> std::result::Result<wasmtime::Config, WasmError> {
233 let mut config = wasmtime::Config::new();
234 config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize);
235 config.cranelift_nan_canonicalization(semantics.canonicalize_nans);
236
237 #[allow(deprecated)]
240 config.cranelift_use_egraphs(false);
241
242 let profiler = match std::env::var_os("WASMTIME_PROFILING_STRATEGY") {
243 Some(os_string) if os_string == "jitdump" => wasmtime::ProfilingStrategy::JitDump,
244 None => wasmtime::ProfilingStrategy::None,
245 Some(_) => {
246 static UNKNOWN_PROFILING_STRATEGY: AtomicBool = AtomicBool::new(false);
248 if !UNKNOWN_PROFILING_STRATEGY.swap(true, Ordering::Relaxed) {
250 log::warn!("WASMTIME_PROFILING_STRATEGY is set to unknown value, ignored.");
251 }
252 wasmtime::ProfilingStrategy::None
253 },
254 };
255 config.profiler(profiler);
256
257 let native_stack_max = match semantics.deterministic_stack_limit {
258 Some(DeterministicStackLimit { native_stack_max, .. }) => native_stack_max,
259
260 None => 1024 * 1024,
265 };
266
267 config.max_wasm_stack(native_stack_max as usize);
268
269 config.parallel_compilation(semantics.parallel_compilation);
270
271 config.wasm_reference_types(semantics.wasm_reference_types);
274 config.wasm_simd(semantics.wasm_simd);
275 config.wasm_bulk_memory(semantics.wasm_bulk_memory);
276 config.wasm_multi_value(semantics.wasm_multi_value);
277 config.wasm_multi_memory(false);
278 config.wasm_threads(false);
279 config.wasm_memory64(false);
280
281 let (use_pooling, use_cow) = match semantics.instantiation_strategy {
282 InstantiationStrategy::PoolingCopyOnWrite => (true, true),
283 InstantiationStrategy::Pooling => (true, false),
284 InstantiationStrategy::RecreateInstanceCopyOnWrite => (false, true),
285 InstantiationStrategy::RecreateInstance => (false, false),
286 };
287
288 const WASM_PAGE_SIZE: u64 = 65536;
289
290 config.memory_init_cow(use_cow);
291 config.memory_guaranteed_dense_image_size(match semantics.heap_alloc_strategy {
292 HeapAllocStrategy::Dynamic { maximum_pages } =>
293 maximum_pages.map(|p| p as u64 * WASM_PAGE_SIZE).unwrap_or(u64::MAX),
294 HeapAllocStrategy::Static { .. } => u64::MAX,
295 });
296
297 if use_pooling {
298 const MAX_WASM_PAGES: u64 = 0x10000;
299
300 let memory_pages = match semantics.heap_alloc_strategy {
301 HeapAllocStrategy::Dynamic { maximum_pages } =>
302 maximum_pages.map(|p| p as u64).unwrap_or(MAX_WASM_PAGES),
303 HeapAllocStrategy::Static { .. } => MAX_WASM_PAGES,
304 };
305
306 let mut pooling_config = wasmtime::PoolingAllocationConfig::default();
307 pooling_config
308 .max_unused_warm_slots(4)
309 .instance_size(128 * 1024)
317 .instance_table_elements(8192)
318 .instance_memory_pages(memory_pages)
319 .instance_tables(1)
321 .instance_memories(1)
322 .instance_count(MAX_INSTANCE_COUNT);
325
326 config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pooling_config));
327 }
328
329 Ok(config)
330}
331
332#[derive(Clone)]
358pub struct DeterministicStackLimit {
359 pub logical_max: u32,
364 pub native_stack_max: u32,
375}
376
377#[non_exhaustive]
387#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
388pub enum InstantiationStrategy {
389 PoolingCopyOnWrite,
394
395 RecreateInstanceCopyOnWrite,
398
399 Pooling,
402
403 RecreateInstance,
405}
406
407enum InternalInstantiationStrategy {
408 Builtin,
409}
410
411#[derive(Clone)]
412pub struct Semantics {
413 pub instantiation_strategy: InstantiationStrategy,
415
416 pub deterministic_stack_limit: Option<DeterministicStackLimit>,
427
428 pub canonicalize_nans: bool,
441
442 pub parallel_compilation: bool,
444
445 pub heap_alloc_strategy: HeapAllocStrategy,
447
448 pub wasm_multi_value: bool,
450
451 pub wasm_bulk_memory: bool,
453
454 pub wasm_reference_types: bool,
456
457 pub wasm_simd: bool,
459}
460
461#[derive(Clone)]
462pub struct Config {
463 pub allow_missing_func_imports: bool,
468
469 pub cache_path: Option<PathBuf>,
471
472 pub semantics: Semantics,
474}
475
476enum CodeSupplyMode<'a> {
477 Fresh(RuntimeBlob),
479
480 Precompiled(&'a Path),
488
489 PrecompiledBytes(&'a [u8]),
494}
495
496pub fn create_runtime<H>(
502 blob: RuntimeBlob,
503 config: Config,
504) -> std::result::Result<WasmtimeRuntime, WasmError>
505where
506 H: HostFunctions,
507{
508 unsafe { do_create_runtime::<H>(CodeSupplyMode::Fresh(blob), config) }
510}
511
512pub unsafe fn create_runtime_from_artifact<H>(
529 compiled_artifact_path: &Path,
530 config: Config,
531) -> std::result::Result<WasmtimeRuntime, WasmError>
532where
533 H: HostFunctions,
534{
535 do_create_runtime::<H>(CodeSupplyMode::Precompiled(compiled_artifact_path), config)
536}
537
538pub unsafe fn create_runtime_from_artifact_bytes<H>(
554 compiled_artifact_bytes: &[u8],
555 config: Config,
556) -> std::result::Result<WasmtimeRuntime, WasmError>
557where
558 H: HostFunctions,
559{
560 do_create_runtime::<H>(CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes), config)
561}
562
563unsafe fn do_create_runtime<H>(
568 code_supply_mode: CodeSupplyMode<'_>,
569 mut config: Config,
570) -> std::result::Result<WasmtimeRuntime, WasmError>
571where
572 H: HostFunctions,
573{
574 replace_strategy_if_broken(&mut config.semantics.instantiation_strategy);
575
576 let mut wasmtime_config = common_config(&config.semantics)?;
577 if let Some(ref cache_path) = config.cache_path {
578 if let Err(reason) = setup_wasmtime_caching(cache_path, &mut wasmtime_config) {
579 log::warn!(
580 "failed to setup wasmtime cache. Performance may degrade significantly: {}.",
581 reason,
582 );
583 }
584 }
585
586 let engine = Engine::new(&wasmtime_config)
587 .map_err(|e| WasmError::Other(format!("cannot create the wasmtime engine: {:#}", e)))?;
588
589 let (module, instantiation_strategy) = match code_supply_mode {
590 CodeSupplyMode::Fresh(blob) => {
591 let blob = prepare_blob_for_compilation(blob, &config.semantics)?;
592 let serialized_blob = blob.clone().serialize();
593
594 let module = wasmtime::Module::new(&engine, &serialized_blob)
595 .map_err(|e| WasmError::Other(format!("cannot create module: {:#}", e)))?;
596
597 match config.semantics.instantiation_strategy {
598 InstantiationStrategy::Pooling |
599 InstantiationStrategy::PoolingCopyOnWrite |
600 InstantiationStrategy::RecreateInstance |
601 InstantiationStrategy::RecreateInstanceCopyOnWrite =>
602 (module, InternalInstantiationStrategy::Builtin),
603 }
604 },
605 CodeSupplyMode::Precompiled(compiled_artifact_path) => {
606 let module = wasmtime::Module::deserialize_file(&engine, compiled_artifact_path)
611 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
612
613 (module, InternalInstantiationStrategy::Builtin)
614 },
615 CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes) => {
616 let module = wasmtime::Module::deserialize(&engine, compiled_artifact_bytes)
621 .map_err(|e| WasmError::Other(format!("cannot deserialize module: {:#}", e)))?;
622
623 (module, InternalInstantiationStrategy::Builtin)
624 },
625 };
626
627 let mut linker = wasmtime::Linker::new(&engine);
628 crate::imports::prepare_imports::<H>(&mut linker, &module, config.allow_missing_func_imports)?;
629
630 let instance_pre = linker
631 .instantiate_pre(&module)
632 .map_err(|e| WasmError::Other(format!("cannot preinstantiate module: {:#}", e)))?;
633
634 Ok(WasmtimeRuntime {
635 engine,
636 instance_pre: Arc::new(instance_pre),
637 instantiation_strategy,
638 instance_counter: Default::default(),
639 })
640}
641
642fn prepare_blob_for_compilation(
643 mut blob: RuntimeBlob,
644 semantics: &Semantics,
645) -> std::result::Result<RuntimeBlob, WasmError> {
646 if let Some(DeterministicStackLimit { logical_max, .. }) = semantics.deterministic_stack_limit {
647 blob = blob.inject_stack_depth_metering(logical_max)?;
648 }
649
650 blob.convert_memory_import_into_export()?;
655 blob.setup_memory_according_to_heap_alloc_strategy(semantics.heap_alloc_strategy)?;
656
657 Ok(blob)
658}
659
660pub fn prepare_runtime_artifact(
663 blob: RuntimeBlob,
664 semantics: &Semantics,
665) -> std::result::Result<Vec<u8>, WasmError> {
666 let mut semantics = semantics.clone();
667 replace_strategy_if_broken(&mut semantics.instantiation_strategy);
668
669 let blob = prepare_blob_for_compilation(blob, &semantics)?;
670
671 let engine = Engine::new(&common_config(&semantics)?)
672 .map_err(|e| WasmError::Other(format!("cannot create the engine: {:#}", e)))?;
673
674 engine
675 .precompile_module(&blob.serialize())
676 .map_err(|e| WasmError::Other(format!("cannot precompile module: {:#}", e)))
677}
678
679fn perform_call(
680 data: &[u8],
681 instance_wrapper: &mut InstanceWrapper,
682 entrypoint: EntryPoint,
683 mut allocator: FreeingBumpHeapAllocator,
684 allocation_stats: &mut Option<AllocationStats>,
685) -> Result<Vec<u8>> {
686 let (data_ptr, data_len) = inject_input_data(instance_wrapper, &mut allocator, data)?;
687
688 let host_state = HostState::new(allocator);
689
690 instance_wrapper.store_mut().data_mut().host_state = Some(host_state);
692
693 let ret = entrypoint
694 .call(instance_wrapper.store_mut(), data_ptr, data_len)
695 .map(unpack_ptr_and_len);
696
697 let host_state = instance_wrapper.store_mut().data_mut().host_state.take().expect(
699 "the host state is always set before calling into WASM so it can't be None here; qed",
700 );
701 *allocation_stats = Some(host_state.allocation_stats());
702
703 let (output_ptr, output_len) = ret?;
704 let output = extract_output_data(instance_wrapper, output_ptr, output_len)?;
705
706 Ok(output)
707}
708
709fn inject_input_data(
710 instance: &mut InstanceWrapper,
711 allocator: &mut FreeingBumpHeapAllocator,
712 data: &[u8],
713) -> Result<(Pointer<u8>, WordSize)> {
714 let mut ctx = instance.store_mut();
715 let memory = ctx.data().memory();
716 let data_len = data.len() as WordSize;
717 let data_ptr = allocator.allocate(&mut MemoryWrapper(&memory, &mut ctx), data_len)?;
718 util::write_memory_from(instance.store_mut(), data_ptr, data)?;
719 Ok((data_ptr, data_len))
720}
721
722fn extract_output_data(
723 instance: &InstanceWrapper,
724 output_ptr: u32,
725 output_len: u32,
726) -> Result<Vec<u8>> {
727 let ctx = instance.store();
728
729 let memory_size = ctx.as_context().data().memory().data_size(ctx);
735 if checked_range(output_ptr as usize, output_len as usize, memory_size).is_none() {
736 Err(Error::OutputExceedsBounds)?
737 }
738 let mut output = vec![0; output_len as usize];
739
740 util::read_memory_into(ctx, Pointer::new(output_ptr), &mut output)?;
741 Ok(output)
742}