From f7712b3f620ab2c6a7df49a7dc3faab50456d170 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Wed, 22 Oct 2025 18:07:06 -0700 Subject: [PATCH 1/4] Debug: add some infrastructure for catching traps, and handle traps properly in Pulley. This is a followup to #11895 where I had disabled a test that failed to emit a debug event for a hostcall-generated trap on a divide-by-zero in Pulley. This PR allows that test to pass, and brings Pulley back to parity with native Cranelift in debug support currently. This was a bit of a "start to pull the thread and the entire finished mechanism materializes" PR; happy to consider ways to split it up if needed. In short, disabling signal-based traps on a Pulley configuration still relies on Pulley opcodes (e.g., divide) actually trapping, in a way that looks more like a "native ISA trap"; so I had to start to build out the actual trap-handling mechanisms. In any case, this will all be needed for followup work soon that will handle traps on native platforms (redirecting from signals by injecting calls), so this is not a distraction. This PR includes, ranked in decreasing order of "may scare other Wasmtime maintainers" score: - A raw `NonNull` in the `CallThreadState`, with a long comment about provenance and mut-borrow exclusivity. This is needed right now to allow the interpreter to invoke the debug event handler, but will soon be needed when injecting hostcalls on signals, because a signal context also has no state available from the Wasm code other than what is in TLS. Hence, we need a way to get the store back from the Wasm when we do something that is "morally a hostcall" at a trapping instruction. I do believe this is sound, or at least close to it if not (please scrutinize carefully!); the basic idea is that the Wasm acts as an opaque blob in the middle, and the pointer comes out of it one way or another (the normal way, as the first arg to a hostcall, or the weird way, via TLS and the CallThreadState during a trap). Exclusive ownership is still clear at any given point and only one `&mut` ever exists in the current frame at a time. That said, I haven't tested with miri yet. This does require careful thought about the Wasm compilation, too; we need the moral equivalent of a `&mut self` reborrow as-if we were making a hostcall on each trapping instruction. It turns out that we already treat them as memory-fence instructions, so nothing loaded from the store can be moved or cached across them, and I've added a comment now about how this is load-bearing. - Updates to `CallThreadState`'s "exit state", normally set by the exit trampoline, that we now also set when we invoke a debug event handler during a trap context[^1] so that `Store::debug_frames` properly sees the current activation. This is a little more awkward than it could be because we store the *trampoline* FP, not last Wasm FP, and there is no trampoline frame in this case, so I've added a flag and some conditionals. I'm happy to refactor instead to go (back) to storing the last Wasm FP instead, with the extra load in the exit trampoline to compute that. - A whole bunch of plumbing, creating a large but mechanical diff, in the code translator to actually add debug tags on all traps and calls to `raise`. It turns out that once I got all of the above working in Pulley, the test disagreed about current Wasm PC between native and Pulley, and Pulley was right; native was getting it wrong because the `raise` libcall was sunk to the bottom in a cold block and, without tags, we scanned backward to pick up the last Wasm PC in the function. This new plumbing and addition of tags in all the appropriate places fixes that. [^1]: I keep saying "during a trap context" here, but to avoid any signal-safety scares, note that when this is done for native signals in a followup PR, we will inject a hostcall by modifying stack/register state and returning from the actual signal context, so it really is as-if we did a hostcall from a trapping instruction. --- cranelift/codegen/src/inst_predicates.rs | 6 + crates/cranelift/src/bounds_checks.rs | 26 +- crates/cranelift/src/func_environ.rs | 209 ++++++++++---- crates/cranelift/src/func_environ/gc.rs | 7 +- .../cranelift/src/func_environ/gc/disabled.rs | 16 ++ .../cranelift/src/func_environ/gc/enabled.rs | 103 +++++-- .../src/func_environ/gc/enabled/drc.rs | 57 ++-- .../src/func_environ/gc/enabled/null.rs | 32 ++- .../src/translate/code_translator.rs | 122 +++++--- crates/cranelift/src/translate/table.rs | 4 +- crates/wasmtime/src/runtime/debug.rs | 3 +- crates/wasmtime/src/runtime/func.rs | 6 + crates/wasmtime/src/runtime/trap.rs | 1 + crates/wasmtime/src/runtime/vm/interpreter.rs | 2 + .../wasmtime/src/runtime/vm/traphandlers.rs | 261 +++++++++++++----- .../src/runtime/vm/traphandlers/backtrace.rs | 3 + crates/wasmtime/src/runtime/vm/vmcontext.rs | 29 +- tests/all/debug.rs | 15 +- 18 files changed, 674 insertions(+), 228 deletions(-) diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs index 7b42e1728663..c40a835b048a 100644 --- a/cranelift/codegen/src/inst_predicates.rs +++ b/cranelift/codegen/src/inst_predicates.rs @@ -150,6 +150,12 @@ pub fn has_memory_fence_semantics(op: Opcode) -> bool { | Opcode::Debugtrap | Opcode::SequencePoint => true, Opcode::Call | Opcode::CallIndirect | Opcode::TryCall | Opcode::TryCallIndirect => true, + // N.B.: this is *load-bearing for borrow safety and + // provenance in Wasmtime*. A trapping op can potentially + // cause an implicit hostcall, and that hostcall implicitly + // mutably borrows Wasmtime's Store. So we can't allow alias + // anslysis to cross trapping opcodes; they are implicitly + // as-if they called the host. op if op.can_trap() => true, _ => false, } diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rs index c0dc9ad664dd..2806e21c3976 100644 --- a/crates/cranelift/src/bounds_checks.rs +++ b/crates/cranelift/src/bounds_checks.rs @@ -22,7 +22,7 @@ use crate::{ Reachability, func_environ::FuncEnvironment, - translate::{HeapData, TargetEnvironment}, + translate::{FuncTranslationStacks, HeapData, TargetEnvironment}, }; use Reachability::*; use cranelift_codegen::{ @@ -84,12 +84,15 @@ pub fn bounds_check_and_compute_addr( index: ir::Value, bounds_check: BoundsCheck, trap: ir::TrapCode, + stacks: &FuncTranslationStacks, ) -> Reachability { match bounds_check { BoundsCheck::StaticOffset { offset, access_size, - } => bounds_check_field_access(builder, env, heap, index, offset, access_size, trap), + } => { + bounds_check_field_access(builder, env, heap, index, offset, access_size, trap, stacks) + } #[cfg(feature = "gc")] BoundsCheck::StaticObjectField { @@ -113,6 +116,7 @@ pub fn bounds_check_and_compute_addr( 0, object_size, trap, + stacks, ) { Reachable(v) => v, u @ Unreachable => return u, @@ -123,7 +127,7 @@ pub fn bounds_check_and_compute_addr( } // Otherwise, bounds check just this one field's access. - bounds_check_field_access(builder, env, heap, index, offset, access_size, trap) + bounds_check_field_access(builder, env, heap, index, offset, access_size, trap, stacks) } // Compute the index of the end of the object, bounds check that and get @@ -148,6 +152,7 @@ pub fn bounds_check_and_compute_addr( 0, 0, trap, + stacks, ) { Reachable(v) => v, u @ Unreachable => return u, @@ -177,6 +182,7 @@ fn bounds_check_field_access( offset: u32, access_size: u8, trap: ir::TrapCode, + stacks: &FuncTranslationStacks, ) -> Reachability { let pointer_bit_width = u16::try_from(env.pointer_type().bits()).unwrap(); let bound_gv = heap.bound; @@ -298,7 +304,7 @@ fn bounds_check_field_access( // max_memory_size`, since we will end up being out-of-bounds regardless // of the given `index`. env.before_unconditionally_trapping_memory_access(builder); - env.trap(builder, trap); + env.trap(builder, trap, stacks); return Unreachable; } @@ -308,7 +314,7 @@ fn bounds_check_field_access( // native pointer type anyway, so this is an unconditional trap. if pointer_bit_width < 64 && offset_and_size >= (1 << pointer_bit_width) { env.before_unconditionally_trapping_memory_access(builder); - env.trap(builder, trap); + env.trap(builder, trap, stacks); return Unreachable; } @@ -430,6 +436,7 @@ fn bounds_check_field_access( AddrPcc::static32(heap.pcc_memory_type, memory_reservation), oob, trap, + stacks, )); } @@ -464,6 +471,7 @@ fn bounds_check_field_access( AddrPcc::dynamic(heap.pcc_memory_type, bound_gv), oob, trap, + stacks, )); } @@ -513,6 +521,7 @@ fn bounds_check_field_access( AddrPcc::dynamic(heap.pcc_memory_type, bound_gv), oob, trap, + stacks, )); } @@ -558,6 +567,7 @@ fn bounds_check_field_access( AddrPcc::dynamic(heap.pcc_memory_type, bound_gv), oob, trap, + stacks, )); } @@ -575,7 +585,7 @@ fn bounds_check_field_access( builder.func.dfg.facts[access_size_val] = Some(Fact::constant(pointer_bit_width, offset_and_size)); } - let adjusted_index = env.uadd_overflow_trap(builder, index, access_size_val, trap); + let adjusted_index = env.uadd_overflow_trap(builder, index, access_size_val, trap, stacks); if pcc { builder.func.dfg.facts[adjusted_index] = Some(Fact::value_offset( pointer_bit_width, @@ -603,6 +613,7 @@ fn bounds_check_field_access( AddrPcc::dynamic(heap.pcc_memory_type, bound_gv), oob, trap, + stacks, )) } @@ -756,9 +767,10 @@ fn explicit_check_oob_condition_and_compute_addr( // in bounds (and therefore we can proceed). oob_condition: ir::Value, trap: ir::TrapCode, + stacks: &FuncTranslationStacks, ) -> ir::Value { if let OobBehavior::ExplicitTrap = oob_behavior { - env.trapnz(builder, oob_condition, trap); + env.trapnz(builder, oob_condition, trap, stacks); } let addr_ty = env.pointer_type(); diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 52208e7d908a..5188e519f176 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -835,6 +835,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { table_index: TableIndex, index: ir::Value, cold_blocks: bool, + stacks: &FuncTranslationStacks, ) -> ir::Value { let pointer_type = self.pointer_type(); let table_data = self.get_or_create_table(builder.func, table_index); @@ -843,7 +844,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { // contents, we check for a null entry here, and // if null, we take a slow-path that invokes a // libcall. - let (table_entry_addr, flags) = table_data.prepare_table_addr(self, builder, index); + let (table_entry_addr, flags) = table_data.prepare_table_addr(self, builder, index, stacks); let value = builder.ins().load(pointer_type, flags, table_entry_addr, 0); if !self.tunables.table_lazy_init { @@ -1028,6 +1029,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder: &mut FunctionBuilder, trap_cond: ir::Value, trap: ir::TrapCode, + stacks: &FuncTranslationStacks, ) { assert!(!self.clif_instruction_traps_enabled()); @@ -1043,17 +1045,22 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder.seal_block(continuation_block); builder.switch_to_block(trap_block); - self.trap(builder, trap); + self.trap(builder, trap, stacks); builder.switch_to_block(continuation_block); } /// Helper used when `!self.clif_instruction_traps_enabled()` is enabled to /// test whether the divisor is zero. - fn guard_zero_divisor(&mut self, builder: &mut FunctionBuilder, rhs: ir::Value) { + fn guard_zero_divisor( + &mut self, + builder: &mut FunctionBuilder, + rhs: ir::Value, + stacks: &FuncTranslationStacks, + ) { if self.clif_instruction_traps_enabled() { return; } - self.trapz(builder, rhs, ir::TrapCode::INTEGER_DIVISION_BY_ZERO); + self.trapz(builder, rhs, ir::TrapCode::INTEGER_DIVISION_BY_ZERO, stacks); } /// Helper used when `!self.clif_instruction_traps_enabled()` is enabled to @@ -1063,11 +1070,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder: &mut FunctionBuilder, lhs: ir::Value, rhs: ir::Value, + stacks: &FuncTranslationStacks, ) { if self.clif_instruction_traps_enabled() { return; } - self.trapz(builder, rhs, ir::TrapCode::INTEGER_DIVISION_BY_ZERO); + self.trapz(builder, rhs, ir::TrapCode::INTEGER_DIVISION_BY_ZERO, stacks); let ty = builder.func.dfg.value_type(rhs); let minus_one = builder.ins().iconst(ty, -1); @@ -1082,7 +1090,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ); let lhs_is_int_min = builder.ins().icmp(IntCC::Equal, lhs, int_min); let is_integer_overflow = builder.ins().band(rhs_is_minus_one, lhs_is_int_min); - self.conditionally_trap(builder, is_integer_overflow, ir::TrapCode::INTEGER_OVERFLOW); + self.conditionally_trap( + builder, + is_integer_overflow, + ir::TrapCode::INTEGER_OVERFLOW, + stacks, + ); } /// Helper used when `!self.clif_instruction_traps_enabled()` is enabled to @@ -1093,6 +1106,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { ty: ir::Type, val: ir::Value, signed: bool, + stacks: &FuncTranslationStacks, ) { assert!(!self.clif_instruction_traps_enabled()); let val_ty = builder.func.dfg.value_type(val); @@ -1102,19 +1116,24 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder.ins().fpromote(F64, val) }; let isnan = builder.ins().fcmp(FloatCC::NotEqual, val, val); - self.trapnz(builder, isnan, ir::TrapCode::BAD_CONVERSION_TO_INTEGER); + self.trapnz( + builder, + isnan, + ir::TrapCode::BAD_CONVERSION_TO_INTEGER, + stacks, + ); let val = self.trunc_f64(builder, val); let (lower_bound, upper_bound) = f64_cvt_to_int_bounds(signed, ty.bits()); let lower_bound = builder.ins().f64const(lower_bound); let too_small = builder .ins() .fcmp(FloatCC::LessThanOrEqual, val, lower_bound); - self.trapnz(builder, too_small, ir::TrapCode::INTEGER_OVERFLOW); + self.trapnz(builder, too_small, ir::TrapCode::INTEGER_OVERFLOW, stacks); let upper_bound = builder.ins().f64const(upper_bound); let too_large = builder .ins() .fcmp(FloatCC::GreaterThanOrEqual, val, upper_bound); - self.trapnz(builder, too_large, ir::TrapCode::INTEGER_OVERFLOW); + self.trapnz(builder, too_large, ir::TrapCode::INTEGER_OVERFLOW, stacks); } /// Get the `ir::Type` for a `VMSharedTypeIndex`. @@ -2052,6 +2071,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { table_index, callee, cold_blocks, + self.stack, ); // If necessary, check the signature. @@ -2151,10 +2171,12 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { self.builder, funcref_ptr, crate::TRAP_INDIRECT_CALL_TO_NULL, + self.stack, ); } } - self.env.trap(self.builder, crate::TRAP_BAD_SIGNATURE); + self.env + .trap(self.builder, crate::TRAP_BAD_SIGNATURE, self.stack); return CheckIndirectCallTypeSignature::StaticTrap; } } @@ -2164,7 +2186,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { WasmHeapType::NoFunc => { assert!(table.ref_type.nullable); self.env - .trap(self.builder, crate::TRAP_INDIRECT_CALL_TO_NULL); + .trap(self.builder, crate::TRAP_INDIRECT_CALL_TO_NULL, self.stack); return CheckIndirectCallTypeSignature::StaticTrap; } @@ -2207,8 +2229,12 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { if self.env.clif_memory_traps_enabled() { mem_flags = mem_flags.with_trap_code(Some(crate::TRAP_INDIRECT_CALL_TO_NULL)); } else { - self.env - .trapz(self.builder, funcref_ptr, crate::TRAP_INDIRECT_CALL_TO_NULL); + self.env.trapz( + self.builder, + funcref_ptr, + crate::TRAP_INDIRECT_CALL_TO_NULL, + self.stack, + ); } let callee_sig_id = self.env @@ -2232,7 +2258,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { .icmp(IntCC::Equal, callee_sig_id, caller_sig_id) }; self.env - .trapz(self.builder, matches, crate::TRAP_BAD_SIGNATURE); + .trapz(self.builder, matches, crate::TRAP_BAD_SIGNATURE, self.stack); CheckIndirectCallTypeSignature::Runtime } @@ -2290,7 +2316,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { callee_flags = callee_flags.with_trap_code(callee_load_trap_code); } else { if let Some(trap) = callee_load_trap_code { - self.env.trapz(self.builder, callee, trap); + self.env.trapz(self.builder, callee, trap, self.stack); } } let func_addr = self.builder.ins().load( @@ -2587,6 +2613,7 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, table_index: TableIndex, index: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { let table = self.module.tables[table_index]; let table_data = self.get_or_create_table(builder.func, table_index); @@ -2594,24 +2621,30 @@ impl FuncEnvironment<'_> { match heap_ty.top() { // GC-managed types. WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { - let (src, flags) = table_data.prepare_table_addr(self, builder, index); + let (src, flags) = table_data.prepare_table_addr(self, builder, index, stacks); gc::gc_compiler(self)?.translate_read_gc_reference( self, builder, table.ref_type, src, flags, + stacks, ) } // Function types. - WasmHeapTopType::Func => { - Ok(self.get_or_init_func_ref_table_elem(builder, table_index, index, false)) - } + WasmHeapTopType::Func => Ok(self.get_or_init_func_ref_table_elem( + builder, + table_index, + index, + false, + stacks, + )), // Continuation types. WasmHeapTopType::Cont => { - let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + let (elem_addr, flags) = + table_data.prepare_table_addr(self, builder, index, stacks); Ok(builder.ins().load( stack_switching::fatpointer::fatpointer_type(self), flags, @@ -2628,6 +2661,7 @@ impl FuncEnvironment<'_> { table_index: TableIndex, value: ir::Value, index: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { let table = self.module.tables[table_index]; let table_data = self.get_or_create_table(builder.func, table_index); @@ -2635,7 +2669,7 @@ impl FuncEnvironment<'_> { match heap_ty.top() { // GC-managed types. WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { - let (dst, flags) = table_data.prepare_table_addr(self, builder, index); + let (dst, flags) = table_data.prepare_table_addr(self, builder, index, stacks); gc::gc_compiler(self)?.translate_write_gc_reference( self, builder, @@ -2643,12 +2677,14 @@ impl FuncEnvironment<'_> { dst, value, flags, + stacks, ) } // Function types. WasmHeapTopType::Func => { - let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + let (elem_addr, flags) = + table_data.prepare_table_addr(self, builder, index, stacks); // Set the "initialized bit". See doc-comment on // `FUNCREF_INIT_BIT` in // crates/environ/src/ref_bits.rs for details. @@ -2667,7 +2703,8 @@ impl FuncEnvironment<'_> { // Continuation types. WasmHeapTopType::Cont => { - let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + let (elem_addr, flags) = + table_data.prepare_table_addr(self, builder, index, stacks); builder.ins().store(flags, value, elem_addr, 0); Ok(()) } @@ -2732,11 +2769,12 @@ impl FuncEnvironment<'_> { &mut self, builder: &mut FunctionBuilder, i31ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { // TODO: If we knew we have a `(ref i31)` here, instead of maybe a `(ref // null i31)`, we could omit the `trapz`. But plumbing that type info // from `wasmparser` and through to here is a bit funky. - self.trapz(builder, i31ref, crate::TRAP_NULL_REFERENCE); + self.trapz(builder, i31ref, crate::TRAP_NULL_REFERENCE, stacks); Ok(builder.ins().sshr_imm(i31ref, 1)) } @@ -2744,11 +2782,12 @@ impl FuncEnvironment<'_> { &mut self, builder: &mut FunctionBuilder, i31ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { // TODO: If we knew we have a `(ref i31)` here, instead of maybe a `(ref // null i31)`, we could omit the `trapz`. But plumbing that type info // from `wasmparser` and through to here is a bit funky. - self.trapz(builder, i31ref, crate::TRAP_NULL_REFERENCE); + self.trapz(builder, i31ref, crate::TRAP_NULL_REFERENCE, stacks); Ok(builder.ins().ushr_imm(i31ref, 1)) } @@ -2765,16 +2804,18 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, struct_type_index: TypeIndex, fields: StructFieldsVec, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_struct_new(self, builder, struct_type_index, &fields) + gc::translate_struct_new(self, builder, struct_type_index, &fields, stacks) } pub fn translate_struct_new_default( &mut self, builder: &mut FunctionBuilder, struct_type_index: TypeIndex, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_struct_new_default(self, builder, struct_type_index) + gc::translate_struct_new_default(self, builder, struct_type_index, stacks) } pub fn translate_struct_get( @@ -2784,6 +2825,7 @@ impl FuncEnvironment<'_> { field_index: u32, struct_ref: ir::Value, extension: Option, + stacks: &FuncTranslationStacks, ) -> WasmResult { gc::translate_struct_get( self, @@ -2792,6 +2834,7 @@ impl FuncEnvironment<'_> { field_index, struct_ref, extension, + stacks, ) } @@ -2802,6 +2845,7 @@ impl FuncEnvironment<'_> { field_index: u32, struct_ref: ir::Value, value: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { gc::translate_struct_set( self, @@ -2810,6 +2854,7 @@ impl FuncEnvironment<'_> { field_index, struct_ref, value, + stacks, ) } @@ -2818,8 +2863,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, tag_index: TagIndex, exn_ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult> { - gc::translate_exn_unbox(self, builder, tag_index, exn_ref) + gc::translate_exn_unbox(self, builder, tag_index, exn_ref, stacks) } pub fn translate_exn_throw( @@ -2828,8 +2874,9 @@ impl FuncEnvironment<'_> { tag_index: TagIndex, args: &[ir::Value], handlers: impl IntoIterator, Block)>, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { - gc::translate_exn_throw(self, builder, tag_index, args, handlers) + gc::translate_exn_throw(self, builder, tag_index, args, handlers, stacks) } pub fn translate_exn_throw_ref( @@ -2837,8 +2884,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, exnref: ir::Value, handlers: impl IntoIterator, Block)>, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { - gc::translate_exn_throw_ref(self, builder, exnref, handlers) + gc::translate_exn_throw_ref(self, builder, exnref, handlers, stacks) } pub fn translate_array_new( @@ -2847,8 +2895,9 @@ impl FuncEnvironment<'_> { array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_array_new(self, builder, array_type_index, elem, len) + gc::translate_array_new(self, builder, array_type_index, elem, len, stacks) } pub fn translate_array_new_default( @@ -2856,8 +2905,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_array_new_default(self, builder, array_type_index, len) + gc::translate_array_new_default(self, builder, array_type_index, len, stacks) } pub fn translate_array_new_fixed( @@ -2865,8 +2915,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, array_type_index: TypeIndex, elems: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_array_new_fixed(self, builder, array_type_index, elems) + gc::translate_array_new_fixed(self, builder, array_type_index, elems, stacks) } pub fn translate_array_new_data( @@ -2941,8 +2992,18 @@ impl FuncEnvironment<'_> { index: ir::Value, value: ir::Value, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { - gc::translate_array_fill(self, builder, array_type_index, array, index, value, len) + gc::translate_array_fill( + self, + builder, + array_type_index, + array, + index, + value, + len, + stacks, + ) } pub fn translate_array_init_data( @@ -3013,8 +3074,9 @@ impl FuncEnvironment<'_> { &mut self, builder: &mut FunctionBuilder, array: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_array_len(self, builder, array) + gc::translate_array_len(self, builder, array, stacks) } pub fn translate_array_get( @@ -3024,8 +3086,17 @@ impl FuncEnvironment<'_> { array: ir::Value, index: ir::Value, extension: Option, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_array_get(self, builder, array_type_index, array, index, extension) + gc::translate_array_get( + self, + builder, + array_type_index, + array, + index, + extension, + stacks, + ) } pub fn translate_array_set( @@ -3035,8 +3106,9 @@ impl FuncEnvironment<'_> { array: ir::Value, index: ir::Value, value: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { - gc::translate_array_set(self, builder, array_type_index, array, index, value) + gc::translate_array_set(self, builder, array_type_index, array, index, value, stacks) } pub fn translate_ref_test( @@ -3045,8 +3117,9 @@ impl FuncEnvironment<'_> { test_ty: WasmRefType, gc_ref: ir::Value, gc_ref_ty: WasmRefType, + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc::translate_ref_test(self, builder, test_ty, gc_ref, gc_ref_ty) + gc::translate_ref_test(self, builder, test_ty, gc_ref, gc_ref_ty, stacks) } pub fn translate_ref_null( @@ -3111,6 +3184,7 @@ impl FuncEnvironment<'_> { &mut self, builder: &mut FunctionBuilder<'_>, global_index: GlobalIndex, + stacks: &FuncTranslationStacks, ) -> WasmResult { match self.get_or_create_global(builder.func, global_index) { GlobalVariable::Memory { gv, offset, ty } => { @@ -3151,6 +3225,7 @@ impl FuncEnvironment<'_> { } else { ir::MemFlags::trusted().with_readonly().with_can_move() }, + stacks, ) } } @@ -3161,6 +3236,7 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder<'_>, global_index: GlobalIndex, val: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { match self.get_or_create_global(builder.func, global_index) { GlobalVariable::Memory { gv, offset, ty } => { @@ -3197,6 +3273,7 @@ impl FuncEnvironment<'_> { src, val, ir::MemFlags::trusted(), + stacks, )? } } @@ -3805,7 +3882,7 @@ impl FuncEnvironment<'_> { pub fn before_translate_function( &mut self, builder: &mut FunctionBuilder, - _state: &FuncTranslationStacks, + state: &FuncTranslationStacks, ) -> WasmResult<()> { // If an explicit stack limit is requested, emit one here at the start // of the function. @@ -3813,7 +3890,7 @@ impl FuncEnvironment<'_> { let limit = builder.ins().global_value(self.pointer_type(), gv); let sp = builder.ins().get_stack_pointer(self.pointer_type()); let overflow = builder.ins().icmp(IntCC::UnsignedLessThan, sp, limit); - self.conditionally_trap(builder, overflow, ir::TrapCode::STACK_OVERFLOW); + self.conditionally_trap(builder, overflow, ir::TrapCode::STACK_OVERFLOW, state); } // Additionally we initialize `fuel_var` if it will get used. @@ -4383,7 +4460,12 @@ impl FuncEnvironment<'_> { &*self.isa } - pub fn trap(&mut self, builder: &mut FunctionBuilder, trap: ir::TrapCode) { + pub fn trap( + &mut self, + builder: &mut FunctionBuilder, + trap: ir::TrapCode, + stacks: &FuncTranslationStacks, + ) { match ( self.clif_instruction_traps_enabled(), crate::clif_trap_to_env_trap(trap), @@ -4403,31 +4485,45 @@ impl FuncEnvironment<'_> { let trap_code = builder.ins().iconst(I8, i64::from(trap as u8)); builder.ins().call(libcall, &[vmctx, trap_code]); let raise = self.builtin_functions.raise(&mut builder.func); - builder.ins().call(raise, &[vmctx]); + let call = builder.ins().call(raise, &[vmctx]); + let tags = self.debug_tags(stacks, builder.srcloc()); + builder.func.debug_tags.set(call, tags); builder.ins().trap(TRAP_INTERNAL_ASSERT); } } } - pub fn trapz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { + pub fn trapz( + &mut self, + builder: &mut FunctionBuilder, + value: ir::Value, + trap: ir::TrapCode, + stacks: &FuncTranslationStacks, + ) { if self.clif_instruction_traps_enabled() { builder.ins().trapz(value, trap); } else { let ty = builder.func.dfg.value_type(value); let zero = builder.ins().iconst(ty, 0); let cmp = builder.ins().icmp(IntCC::Equal, value, zero); - self.conditionally_trap(builder, cmp, trap); + self.conditionally_trap(builder, cmp, trap, stacks); } } - pub fn trapnz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { + pub fn trapnz( + &mut self, + builder: &mut FunctionBuilder, + value: ir::Value, + trap: ir::TrapCode, + stacks: &FuncTranslationStacks, + ) { if self.clif_instruction_traps_enabled() { builder.ins().trapnz(value, trap); } else { let ty = builder.func.dfg.value_type(value); let zero = builder.ins().iconst(ty, 0); let cmp = builder.ins().icmp(IntCC::NotEqual, value, zero); - self.conditionally_trap(builder, cmp, trap); + self.conditionally_trap(builder, cmp, trap, stacks); } } @@ -4437,12 +4533,13 @@ impl FuncEnvironment<'_> { lhs: ir::Value, rhs: ir::Value, trap: ir::TrapCode, + stacks: &FuncTranslationStacks, ) -> ir::Value { if self.clif_instruction_traps_enabled() { builder.ins().uadd_overflow_trap(lhs, rhs, trap) } else { let (ret, overflow) = builder.ins().uadd_overflow(lhs, rhs); - self.conditionally_trap(builder, overflow, trap); + self.conditionally_trap(builder, overflow, trap, stacks); ret } } @@ -4452,8 +4549,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, lhs: ir::Value, rhs: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { - self.guard_signed_divide(builder, lhs, rhs); + self.guard_signed_divide(builder, lhs, rhs, stacks); builder.ins().sdiv(lhs, rhs) } @@ -4462,8 +4560,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, lhs: ir::Value, rhs: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { - self.guard_zero_divisor(builder, rhs); + self.guard_zero_divisor(builder, rhs, stacks); builder.ins().udiv(lhs, rhs) } @@ -4472,8 +4571,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, lhs: ir::Value, rhs: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { - self.guard_zero_divisor(builder, rhs); + self.guard_zero_divisor(builder, rhs, stacks); builder.ins().srem(lhs, rhs) } @@ -4482,8 +4582,9 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, lhs: ir::Value, rhs: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { - self.guard_zero_divisor(builder, rhs); + self.guard_zero_divisor(builder, rhs, stacks); builder.ins().urem(lhs, rhs) } @@ -4492,11 +4593,12 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, ty: ir::Type, val: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { // NB: for now avoid translating this entire instruction to CLIF and // just do it in a libcall. if !self.clif_instruction_traps_enabled() { - self.guard_fcvt_to_int(builder, ty, val, true); + self.guard_fcvt_to_int(builder, ty, val, true, stacks); } builder.ins().fcvt_to_sint(ty, val) } @@ -4506,9 +4608,10 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, ty: ir::Type, val: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { if !self.clif_instruction_traps_enabled() { - self.guard_fcvt_to_int(builder, ty, val, false); + self.guard_fcvt_to_int(builder, ty, val, false, stacks); } builder.ins().fcvt_to_uint(ty, val) } diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index 87cc59a212e3..5f1fb4593707 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -5,7 +5,7 @@ //! to have just a single `cfg(feature = "gc")` for the whole crate, which //! selects between these two implementations. -use crate::func_environ::FuncEnvironment; +use crate::{func_environ::FuncEnvironment, translate::FuncTranslationStacks}; use cranelift_codegen::ir; use cranelift_frontend::FunctionBuilder; use wasmtime_environ::{GcTypeLayouts, TagIndex, TypeIndex, WasmRefType, WasmResult}; @@ -52,6 +52,7 @@ pub trait GcCompiler { builder: &mut FunctionBuilder<'_>, array_type_index: TypeIndex, init: ArrayInit<'_>, + stacks: &FuncTranslationStacks, ) -> WasmResult; /// Emit code to allocate a new struct. @@ -65,6 +66,7 @@ pub trait GcCompiler { builder: &mut FunctionBuilder<'_>, struct_type_index: TypeIndex, fields: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult; /// Emit code to allocate a new exception object. @@ -83,6 +85,7 @@ pub trait GcCompiler { fields: &[ir::Value], instance_id: ir::Value, tag: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult; /// Emit a read barrier for when we are cloning a GC reference onto the Wasm @@ -124,6 +127,7 @@ pub trait GcCompiler { ty: WasmRefType, src: ir::Value, flags: ir::MemFlags, + stacks: &FuncTranslationStacks, ) -> WasmResult; /// Emit a write barrier for when we are writing a GC reference over another @@ -166,6 +170,7 @@ pub trait GcCompiler { dst: ir::Value, new_val: ir::Value, flags: ir::MemFlags, + stacks: &FuncTranslationStacks, ) -> WasmResult<()>; } diff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rs index 9a0fb4e036ff..4ee24a8dab67 100644 --- a/crates/cranelift/src/func_environ/gc/disabled.rs +++ b/crates/cranelift/src/func_environ/gc/disabled.rs @@ -2,6 +2,7 @@ use super::GcCompiler; use crate::func_environ::{Extension, FuncEnvironment}; +use crate::translate::FuncTranslationStacks; use cranelift_codegen::ir; use cranelift_frontend::FunctionBuilder; use smallvec::SmallVec; @@ -24,6 +25,7 @@ pub fn translate_struct_new( _builder: &mut FunctionBuilder<'_>, _struct_type_index: TypeIndex, _fields: &[ir::Value], + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -32,6 +34,7 @@ pub fn translate_struct_new_default( _func_env: &mut FuncEnvironment<'_>, _builder: &mut FunctionBuilder<'_>, _struct_type_index: TypeIndex, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -43,6 +46,7 @@ pub fn translate_struct_get( _field_index: u32, _struct_ref: ir::Value, _extension: Option, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -54,6 +58,7 @@ pub fn translate_struct_set( _field_index: u32, _struct_ref: ir::Value, _new_val: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { disabled() } @@ -63,6 +68,7 @@ pub fn translate_exn_unbox( _builder: &mut FunctionBuilder<'_>, _tag_index: TagIndex, _exn_ref: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult> { disabled() } @@ -73,6 +79,7 @@ pub fn translate_exn_throw( _tag_index: TagIndex, _args: &[ir::Value], _handlers: impl IntoIterator, ir::Block)>, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { disabled() } @@ -82,6 +89,7 @@ pub fn translate_exn_throw_ref( _builder: &mut FunctionBuilder<'_>, _exnref: ir::Value, _handlers: impl IntoIterator, ir::Block)>, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { disabled() } @@ -92,6 +100,7 @@ pub fn translate_array_new( _array_type_index: TypeIndex, _elem: ir::Value, _len: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -101,6 +110,7 @@ pub fn translate_array_new_default( _builder: &mut FunctionBuilder, _array_type_index: TypeIndex, _len: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -110,6 +120,7 @@ pub fn translate_array_new_fixed( _builder: &mut FunctionBuilder, _array_type_index: TypeIndex, _elems: &[ir::Value], + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -122,6 +133,7 @@ pub fn translate_array_fill( _index: ir::Value, _value: ir::Value, _n: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { disabled() } @@ -130,6 +142,7 @@ pub fn translate_array_len( _func_env: &mut FuncEnvironment<'_>, _builder: &mut FunctionBuilder, _array: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -141,6 +154,7 @@ pub fn translate_array_get( _array: ir::Value, _index: ir::Value, _extension: Option, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } @@ -152,6 +166,7 @@ pub fn translate_array_set( _array: ir::Value, _index: ir::Value, _value: ir::Value, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { disabled() } @@ -162,6 +177,7 @@ pub fn translate_ref_test( _test_ty: WasmRefType, _val: ir::Value, _val_ty: WasmRefType, + _stacks: &FuncTranslationStacks, ) -> WasmResult { disabled() } diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs index dc780f5e2aa9..68016347e537 100644 --- a/crates/cranelift/src/func_environ/gc/enabled.rs +++ b/crates/cranelift/src/func_environ/gc/enabled.rs @@ -1,7 +1,7 @@ use super::{ArrayInit, GcCompiler}; use crate::bounds_checks::BoundsCheck; use crate::func_environ::{Extension, FuncEnvironment}; -use crate::translate::{Heap, HeapData, StructFieldsVec, TargetEnvironment}; +use crate::translate::{FuncTranslationStacks, Heap, HeapData, StructFieldsVec, TargetEnvironment}; use crate::{Reachability, TRAP_INTERNAL_ASSERT}; use cranelift_codegen::ir::immediates::Offset32; use cranelift_codegen::ir::{ @@ -106,6 +106,7 @@ fn read_field_at_addr( ty: WasmStorageType, addr: ir::Value, extension: Option, + stacks: &FuncTranslationStacks, ) -> WasmResult { assert_eq!(extension.is_none(), matches!(ty, WasmStorageType::Val(_))); assert_eq!( @@ -128,7 +129,7 @@ fn read_field_at_addr( WasmValType::Ref(r) => match r.heap_type.top() { WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => { gc_compiler(func_env)? - .translate_read_gc_reference(func_env, builder, r, addr, flags)? + .translate_read_gc_reference(func_env, builder, r, addr, flags, stacks)? } WasmHeapTopType::Func => { let expected_ty = match r.heap_type { @@ -233,6 +234,7 @@ fn write_field_at_addr( field_ty: WasmStorageType, field_addr: ir::Value, new_val: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { // Data inside GC objects is always little endian. let flags = ir::MemFlags::trusted().with_endianness(ir::Endianness::Little); @@ -248,8 +250,9 @@ fn write_field_at_addr( write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?; } WasmStorageType::Val(WasmValType::Ref(r)) => { - gc_compiler(func_env)? - .translate_write_gc_reference(func_env, builder, r, field_addr, new_val, flags)?; + gc_compiler(func_env)?.translate_write_gc_reference( + func_env, builder, r, field_addr, new_val, flags, stacks, + )?; } WasmStorageType::Val(_) => { assert_eq!( @@ -267,8 +270,9 @@ pub fn translate_struct_new( builder: &mut FunctionBuilder<'_>, struct_type_index: TypeIndex, fields: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult { - gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields) + gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields, stacks) } fn default_value( @@ -304,6 +308,7 @@ pub fn translate_struct_new_default( func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, struct_type_index: TypeIndex, + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_ty = func_env.module.types[struct_type_index].unwrap_module_type_index(); let struct_ty = func_env.types.unwrap_struct(interned_ty)?; @@ -312,7 +317,7 @@ pub fn translate_struct_new_default( .iter() .map(|f| default_value(&mut builder.cursor(), func_env, &f.element_type)) .collect::(); - gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields) + gc_compiler(func_env)?.alloc_struct(func_env, builder, struct_type_index, &fields, stacks) } pub fn translate_struct_get( @@ -322,6 +327,7 @@ pub fn translate_struct_get( field_index: u32, struct_ref: ir::Value, extension: Option, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!( "translate_struct_get({struct_type_index:?}, {field_index:?}, {struct_ref:?}, {extension:?})" @@ -330,7 +336,7 @@ pub fn translate_struct_get( // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that // type info from `wasmparser` and through to here is a bit funky. - func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE, stacks); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index(); @@ -351,6 +357,7 @@ pub fn translate_struct_get( access_size: u8::try_from(field_size).unwrap(), object_size: struct_size, }, + stacks, ); let result = read_field_at_addr( @@ -359,6 +366,7 @@ pub fn translate_struct_get( field_ty.element_type, field_addr, extension, + stacks, ); log::trace!("translate_struct_get(..) -> {result:?}"); result @@ -371,13 +379,14 @@ pub fn translate_struct_set( field_index: u32, struct_ref: ir::Value, new_val: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { log::trace!( "translate_struct_set({struct_type_index:?}, {field_index:?}, struct_ref: {struct_ref:?}, new_val: {new_val:?})" ); // TODO: See comment in `translate_struct_get` about the `trapz`. - func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE, stacks); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index(); @@ -398,6 +407,7 @@ pub fn translate_struct_set( access_size: u8::try_from(field_size).unwrap(), object_size: struct_size, }, + stacks, ); write_field_at_addr( @@ -406,6 +416,7 @@ pub fn translate_struct_set( field_ty.element_type, field_addr, new_val, + stacks, )?; log::trace!("translate_struct_set: finished"); @@ -417,6 +428,7 @@ pub fn translate_exn_unbox( builder: &mut FunctionBuilder<'_>, tag_index: TagIndex, exn_ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult> { log::trace!("translate_exn_unbox({tag_index:?}, {exn_ref:?})"); @@ -453,9 +465,10 @@ pub fn translate_exn_unbox( access_size: u8::try_from(field_size).unwrap(), object_size: exn_size, }, + stacks, ); - let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None)?; + let value = read_field_at_addr(func_env, builder, field_ty, field_addr, None, stacks)?; result.push(value); } @@ -469,6 +482,7 @@ pub fn translate_exn_throw( tag_index: TagIndex, args: &[ir::Value], handlers: impl IntoIterator, Block)>, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { let (instance_id, defined_tag_id) = func_env.get_instance_and_tag(builder, tag_index); let exnref = gc_compiler(func_env)?.alloc_exn( @@ -478,8 +492,9 @@ pub fn translate_exn_throw( args, instance_id, defined_tag_id, + stacks, )?; - translate_exn_throw_ref(func_env, builder, exnref, handlers) + translate_exn_throw_ref(func_env, builder, exnref, handlers, stacks) } pub fn translate_exn_throw_ref( @@ -487,6 +502,7 @@ pub fn translate_exn_throw_ref( builder: &mut FunctionBuilder<'_>, exnref: ir::Value, handlers: impl IntoIterator, Block)>, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { let builtin = func_env.builtin_functions.throw_ref(builder.func); let sig = builder.func.dfg.ext_funcs[builtin].signature; @@ -520,7 +536,7 @@ pub fn translate_exn_throw_ref( builder.switch_to_block(continuation); builder.seal_block(continuation); - func_env.trap(builder, crate::TRAP_UNREACHABLE); + func_env.trap(builder, crate::TRAP_UNREACHABLE, stacks); Ok(()) } @@ -531,6 +547,7 @@ pub fn translate_array_new( array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})"); let result = gc_compiler(func_env)?.alloc_array( @@ -538,6 +555,7 @@ pub fn translate_array_new( builder, array_type_index, ArrayInit::Fill { elem, len }, + stacks, )?; log::trace!("translate_array_new(..) -> {result:?}"); Ok(result) @@ -548,6 +566,7 @@ pub fn translate_array_new_default( builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_array_new_default({array_type_index:?}, {len:?})"); @@ -559,6 +578,7 @@ pub fn translate_array_new_default( builder, array_type_index, ArrayInit::Fill { elem, len }, + stacks, )?; log::trace!("translate_array_new_default(..) -> {result:?}"); Ok(result) @@ -569,6 +589,7 @@ pub fn translate_array_new_fixed( builder: &mut FunctionBuilder, array_type_index: TypeIndex, elems: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_array_new_fixed({array_type_index:?}, {elems:?})"); let result = gc_compiler(func_env)?.alloc_array( @@ -576,6 +597,7 @@ pub fn translate_array_new_fixed( builder, array_type_index, ArrayInit::Elems(elems), + stacks, )?; log::trace!("translate_array_new_fixed(..) -> {result:?}"); Ok(result) @@ -757,19 +779,26 @@ pub fn translate_array_fill( index: ir::Value, value: ir::Value, n: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { log::trace!( "translate_array_fill({array_type_index:?}, {array_ref:?}, {index:?}, {value:?}, {n:?})" ); - let len = translate_array_len(func_env, builder, array_ref)?; + let len = translate_array_len(func_env, builder, array_ref, stacks)?; // Check that the full range of elements we want to fill is within bounds. - let end_index = func_env.uadd_overflow_trap(builder, index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + let end_index = + func_env.uadd_overflow_trap(builder, index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS, stacks); let out_of_bounds = builder .ins() .icmp(IntCC::UnsignedGreaterThan, end_index, len); - func_env.trapnz(builder, out_of_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapnz( + builder, + out_of_bounds, + crate::TRAP_ARRAY_OUT_OF_BOUNDS, + stacks, + ); // Get the address of the first element we want to fill. let interned_type_index = func_env.module.types[array_type_index].unwrap_module_type_index(); @@ -787,6 +816,7 @@ pub fn translate_array_fill( offset: obj_offset, object_size: obj_size, }, + stacks, ); // Calculate the end address, just after the filled region. @@ -809,7 +839,7 @@ pub fn translate_array_fill( .unwrap_array(interned_type_index)? .0 .element_type; - write_field_at_addr(func_env, builder, elem_ty, elem_addr, value) + write_field_at_addr(func_env, builder, elem_ty, elem_addr, value, stacks) }, ); log::trace!("translate_array_fill(..) -> {result:?}"); @@ -820,10 +850,11 @@ pub fn translate_array_len( func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder, array_ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_array_len({array_ref:?})"); - func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE, stacks); let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); let len_field = func_env.prepare_gc_ref_access( @@ -835,6 +866,7 @@ pub fn translate_array_len( offset: len_offset, access_size: u8::try_from(ir::types::I32.bytes()).unwrap(), }, + stacks, ); let result = builder.ins().load( ir::types::I32, @@ -915,6 +947,7 @@ fn array_elem_addr( array_type_index: ModuleInternedTypeIndex, array_ref: ir::Value, index: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { // First, assert that `index < array.length`. // @@ -926,10 +959,10 @@ fn array_elem_addr( // code in `bounds_check.rs` to implement these bounds checks. That is all // planned, but not yet implemented. - let len = translate_array_len(func_env, builder, array_ref).unwrap(); + let len = translate_array_len(func_env, builder, array_ref, stacks).unwrap(); let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len); - func_env.trapz(builder, in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapz(builder, in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS, stacks); // Compute the size (in bytes) of the whole array object. let ArraySizeInfo { @@ -969,6 +1002,7 @@ fn array_elem_addr( offset: offset_in_array, object_size: obj_size, }, + stacks, ) } @@ -979,16 +1013,24 @@ pub fn translate_array_get( array_ref: ir::Value, index: ir::Value, extension: Option, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})"); let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index(); - let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index); + let elem_addr = array_elem_addr( + func_env, + builder, + array_type_index, + array_ref, + index, + stacks, + ); let array_ty = func_env.types.unwrap_array(array_type_index)?; let elem_ty = array_ty.0.element_type; - let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension)?; + let result = read_field_at_addr(func_env, builder, elem_ty, elem_addr, extension, stacks)?; log::trace!("translate_array_get(..) -> {result:?}"); Ok(result) } @@ -1000,16 +1042,24 @@ pub fn translate_array_set( array_ref: ir::Value, index: ir::Value, value: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})"); let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index(); - let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index); + let elem_addr = array_elem_addr( + func_env, + builder, + array_type_index, + array_ref, + index, + stacks, + ); let array_ty = func_env.types.unwrap_array(array_type_index)?; let elem_ty = array_ty.0.element_type; - write_field_at_addr(func_env, builder, elem_ty, elem_addr, value)?; + write_field_at_addr(func_env, builder, elem_ty, elem_addr, value, stacks)?; log::trace!("translate_array_set: finished"); Ok(()) @@ -1021,6 +1071,7 @@ pub fn translate_ref_test( test_ty: WasmRefType, val: ir::Value, val_ty: WasmRefType, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_ref_test({test_ty:?}, {val:?})"); @@ -1147,6 +1198,7 @@ pub fn translate_ref_test( access_size: wasmtime_environ::VM_GC_KIND_SIZE, object_size: wasmtime_environ::VM_GC_HEADER_SIZE, }, + stacks, ); let actual_kind = builder.ins().load( ir::types::I32, @@ -1204,6 +1256,7 @@ pub fn translate_ref_test( offset: wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET, access_size: func_env.offsets.size_of_vmshared_type_index(), }, + stacks, ); let actual_shared_ty = builder.ins().load( ir::types::I32, @@ -1278,6 +1331,7 @@ fn emit_array_size( builder: &mut FunctionBuilder<'_>, array_layout: &GcArrayLayout, len: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { let base_size = builder .ins() @@ -1300,7 +1354,7 @@ fn emit_array_size( .ins() .imul_imm(len, i64::from(array_layout.elem_size)); let high_bits = builder.ins().ushr_imm(elems_size_64, 32); - func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE, stacks); let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64); // And if adding the base size and elements size overflows, then the @@ -1310,6 +1364,7 @@ fn emit_array_size( base_size, elems_size, crate::TRAP_ALLOCATION_TOO_LARGE, + stacks, ); size @@ -1486,6 +1541,7 @@ impl FuncEnvironment<'_> { builder: &mut FunctionBuilder, gc_ref: ir::Value, bounds_check: BoundsCheck, + stacks: &FuncTranslationStacks, ) -> ir::Value { log::trace!("prepare_gc_ref_access({gc_ref:?}, {bounds_check:?})"); assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32); @@ -1499,6 +1555,7 @@ impl FuncEnvironment<'_> { gc_ref, bounds_check, crate::TRAP_INTERNAL_ASSERT, + stacks, ) { Reachability::Reachable(v) => v, Reachability::Unreachable => { diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rs index d0abb2ee53b6..4422853b67c6 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs @@ -2,7 +2,7 @@ //! barriers. use super::*; -use crate::translate::TargetEnvironment; +use crate::translate::{FuncTranslationStacks, TargetEnvironment}; use crate::{TRAP_INTERNAL_ASSERT, func_environ::FuncEnvironment}; use cranelift_codegen::ir::condcodes::IntCC; use cranelift_codegen::ir::{self, InstBuilder}; @@ -28,6 +28,7 @@ impl DrcCompiler { func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder, gc_ref: ir::Value, + stacks: &FuncTranslationStacks, ) -> ir::Value { let offset = func_env.offsets.vm_drc_header_ref_count(); let pointer = func_env.prepare_gc_ref_access( @@ -37,6 +38,7 @@ impl DrcCompiler { offset, access_size: u8::try_from(ir::types::I64.bytes()).unwrap(), }, + stacks, ); builder .ins() @@ -53,6 +55,7 @@ impl DrcCompiler { builder: &mut FunctionBuilder, gc_ref: ir::Value, new_ref_count: ir::Value, + stacks: &FuncTranslationStacks, ) { let offset = func_env.offsets.vm_drc_header_ref_count(); let pointer = func_env.prepare_gc_ref_access( @@ -62,6 +65,7 @@ impl DrcCompiler { offset, access_size: u8::try_from(ir::types::I64.bytes()).unwrap(), }, + stacks, ); builder .ins() @@ -80,11 +84,12 @@ impl DrcCompiler { builder: &mut FunctionBuilder, gc_ref: ir::Value, delta: i64, + stacks: &FuncTranslationStacks, ) -> ir::Value { debug_assert!(delta == -1 || delta == 1); - let old_ref_count = self.load_ref_count(func_env, builder, gc_ref); + let old_ref_count = self.load_ref_count(func_env, builder, gc_ref, stacks); let new_ref_count = builder.ins().iadd_imm(old_ref_count, delta); - self.store_ref_count(func_env, builder, gc_ref, new_ref_count); + self.store_ref_count(func_env, builder, gc_ref, new_ref_count, stacks); new_ref_count } @@ -99,6 +104,7 @@ impl DrcCompiler { builder: &mut FunctionBuilder<'_>, gc_ref: ir::Value, reserved: ir::Value, + stacks: &FuncTranslationStacks, ) { debug_assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32); debug_assert_eq!(builder.func.dfg.value_type(reserved), ir::types::I32); @@ -112,11 +118,11 @@ impl DrcCompiler { .load(ir::types::I32, ir::MemFlags::trusted(), head, 0); // Update our object's header to point to `next` and consider itself part of the list. - self.set_next_over_approximated_stack_root(func_env, builder, gc_ref, next); - self.set_in_over_approximated_stack_roots_bit(func_env, builder, gc_ref, reserved); + self.set_next_over_approximated_stack_root(func_env, builder, gc_ref, next, stacks); + self.set_in_over_approximated_stack_roots_bit(func_env, builder, gc_ref, reserved, stacks); // Increment our ref count because the list is logically holding a strong reference. - self.mutate_ref_count(func_env, builder, gc_ref, 1); + self.mutate_ref_count(func_env, builder, gc_ref, 1, stacks); // Commit this object as the new head of the list. builder @@ -149,6 +155,7 @@ impl DrcCompiler { builder: &mut FunctionBuilder<'_>, gc_ref: ir::Value, next: ir::Value, + stacks: &FuncTranslationStacks, ) { debug_assert_eq!(builder.func.dfg.value_type(gc_ref), ir::types::I32); debug_assert_eq!(builder.func.dfg.value_type(next), ir::types::I32); @@ -161,6 +168,7 @@ impl DrcCompiler { .vm_drc_header_next_over_approximated_stack_root(), access_size: u8::try_from(ir::types::I32.bytes()).unwrap(), }, + stacks, ); builder.ins().store(ir::MemFlags::trusted(), next, ptr, 0); } @@ -173,13 +181,14 @@ impl DrcCompiler { builder: &mut FunctionBuilder<'_>, gc_ref: ir::Value, old_reserved_bits: ir::Value, + stacks: &FuncTranslationStacks, ) { let in_set_bit = builder.ins().iconst( ir::types::I32, i64::from(wasmtime_environ::drc::HEADER_IN_OVER_APPROX_LIST_BIT), ); let new_reserved = builder.ins().bor(old_reserved_bits, in_set_bit); - self.set_reserved_bits(func_env, builder, gc_ref, new_reserved); + self.set_reserved_bits(func_env, builder, gc_ref, new_reserved, stacks); } /// Update the reserved bits in a `VMDrcHeader`. @@ -189,6 +198,7 @@ impl DrcCompiler { builder: &mut FunctionBuilder<'_>, gc_ref: ir::Value, new_reserved: ir::Value, + stacks: &FuncTranslationStacks, ) { let ptr = func_env.prepare_gc_ref_access( builder, @@ -197,6 +207,7 @@ impl DrcCompiler { offset: func_env.offsets.vm_gc_header_reserved_bits(), access_size: u8::try_from(ir::types::I32.bytes()).unwrap(), }, + stacks, ); builder .ins() @@ -211,6 +222,7 @@ impl DrcCompiler { field_addr: ir::Value, ty: WasmStorageType, val: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { // Data inside GC objects is always little endian. let flags = ir::MemFlags::trusted().with_endianness(ir::Endianness::Little); @@ -222,7 +234,9 @@ impl DrcCompiler { write_func_ref_at_addr(func_env, builder, r, flags, field_addr, val)?; } WasmStorageType::Val(WasmValType::Ref(r)) => { - self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags)?; + self.translate_init_gc_reference( + func_env, builder, r, field_addr, val, flags, stacks, + )?; } WasmStorageType::I8 => { assert_eq!(builder.func.dfg.value_type(val), ir::types::I32); @@ -259,6 +273,7 @@ impl DrcCompiler { dst: ir::Value, new_val: ir::Value, flags: ir::MemFlags, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { let (ref_ty, needs_stack_map) = func_env.reference_type(ty.heap_type); debug_assert!(needs_stack_map); @@ -331,7 +346,7 @@ impl DrcCompiler { builder.switch_to_block(inc_ref_block); builder.seal_block(inc_ref_block); log::trace!("DRC initialization barrier: increment the ref count of the initial value"); - self.mutate_ref_count(func_env, builder, new_val, 1); + self.mutate_ref_count(func_env, builder, new_val, 1, stacks); builder.ins().jump(continue_block, &[]); // Join point after we're done with the GC barrier: do the actual store @@ -388,6 +403,7 @@ impl GcCompiler for DrcCompiler { builder: &mut FunctionBuilder<'_>, array_type_index: TypeIndex, init: super::ArrayInit<'_>, + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.types[array_type_index].unwrap_module_type_index(); @@ -402,7 +418,7 @@ impl GcCompiler for DrcCompiler { // First, compute the array's total size from its base size, element // size, and length. let len = init.len(&mut builder.cursor()); - let size = emit_array_size(func_env, builder, &array_layout, len); + let size = emit_array_size(func_env, builder, &array_layout, len, stacks); // Second, now that we have the array object's total size, call the // `gc_alloc_raw` builtin libcall to allocate the array. @@ -440,7 +456,7 @@ impl GcCompiler for DrcCompiler { size, elems_addr, |func_env, builder, elem_ty, elem_addr, val| { - self.init_field(func_env, builder, elem_addr, elem_ty, val) + self.init_field(func_env, builder, elem_addr, elem_ty, val, stacks) }, )?; Ok(array_ref) @@ -452,6 +468,7 @@ impl GcCompiler for DrcCompiler { builder: &mut FunctionBuilder<'_>, struct_type_index: TypeIndex, field_vals: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index(); @@ -489,7 +506,7 @@ impl GcCompiler for DrcCompiler { raw_ptr_to_struct, field_vals, |func_env, builder, ty, field_addr, val| { - self.init_field(func_env, builder, field_addr, ty, val) + self.init_field(func_env, builder, field_addr, ty, val, stacks) }, )?; @@ -504,6 +521,7 @@ impl GcCompiler for DrcCompiler { field_vals: &[ir::Value], instance_id: ir::Value, tag: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.tags[tag_index] .exception @@ -543,7 +561,7 @@ impl GcCompiler for DrcCompiler { raw_ptr_to_exn, field_vals, |func_env, builder, ty, field_addr, val| { - self.init_field(func_env, builder, field_addr, ty, val) + self.init_field(func_env, builder, field_addr, ty, val, stacks) }, )?; @@ -557,6 +575,7 @@ impl GcCompiler for DrcCompiler { instance_id_addr, WasmStorageType::Val(WasmValType::I32), instance_id, + stacks, )?; let tag_addr = builder .ins() @@ -567,6 +586,7 @@ impl GcCompiler for DrcCompiler { tag_addr, WasmStorageType::Val(WasmValType::I32), tag, + stacks, )?; Ok(exn_ref) @@ -579,6 +599,7 @@ impl GcCompiler for DrcCompiler { ty: WasmRefType, src: ir::Value, flags: ir::MemFlags, + stacks: &FuncTranslationStacks, ) -> WasmResult { log::trace!("translate_read_gc_reference({ty:?}, {src:?}, {flags:?})"); @@ -695,6 +716,7 @@ impl GcCompiler for DrcCompiler { offset: func_env.offsets.vm_gc_header_reserved_bits(), access_size: u8::try_from(ir::types::I32.bytes()).unwrap(), }, + stacks, ); let reserved = builder .ins() @@ -715,7 +737,7 @@ impl GcCompiler for DrcCompiler { log::trace!( "DRC read barrier: push the object onto the over-approximated-stack-roots list" ); - self.push_onto_over_approximated_stack_roots(func_env, builder, gc_ref, reserved); + self.push_onto_over_approximated_stack_roots(func_env, builder, gc_ref, reserved, stacks); builder.ins().jump(continue_block, &[]); // Join point after we're done with the GC barrier. @@ -733,6 +755,7 @@ impl GcCompiler for DrcCompiler { dst: ir::Value, new_val: ir::Value, flags: ir::MemFlags, + stacks: &FuncTranslationStacks, ) -> WasmResult<()> { assert!(ty.is_vmgcref_type()); @@ -859,7 +882,7 @@ impl GcCompiler for DrcCompiler { builder.switch_to_block(inc_ref_block); log::trace!("DRC write barrier: increment new ref's ref count"); builder.seal_block(inc_ref_block); - self.mutate_ref_count(func_env, builder, new_val, 1); + self.mutate_ref_count(func_env, builder, new_val, 1, stacks); builder.ins().jump(check_old_val_block, &[]); // Block to store the new value into `dst` and then check whether the @@ -885,7 +908,7 @@ impl GcCompiler for DrcCompiler { log::trace!( "DRC write barrier: decrement old ref's ref count and check for zero ref count" ); - let ref_count = self.load_ref_count(func_env, builder, old_val); + let ref_count = self.load_ref_count(func_env, builder, old_val, stacks); let new_ref_count = builder.ins().iadd_imm(ref_count, -1); let old_val_needs_drop = builder.ins().icmp_imm(IntCC::Equal, new_ref_count, 0); builder.ins().brif( @@ -915,7 +938,7 @@ impl GcCompiler for DrcCompiler { builder.switch_to_block(store_dec_ref_block); builder.seal_block(store_dec_ref_block); log::trace!("DRC write barrier: store decremented ref count into old ref"); - self.store_ref_count(func_env, builder, old_val, new_ref_count); + self.store_ref_count(func_env, builder, old_val, new_ref_count, stacks); builder.ins().jump(continue_block, &[]); // Join point after we're done with the GC barrier. diff --git a/crates/cranelift/src/func_environ/gc/enabled/null.rs b/crates/cranelift/src/func_environ/gc/enabled/null.rs index a0e938d55c0f..d8d6df126528 100644 --- a/crates/cranelift/src/func_environ/gc/enabled/null.rs +++ b/crates/cranelift/src/func_environ/gc/enabled/null.rs @@ -6,6 +6,7 @@ use super::*; use crate::func_environ::FuncEnvironment; +use crate::translate::FuncTranslationStacks; use cranelift_codegen::ir::{self, InstBuilder}; use cranelift_frontend::FunctionBuilder; use wasmtime_environ::VMSharedTypeIndex; @@ -44,6 +45,7 @@ impl NullCompiler { ty: Option, size: ir::Value, align: ir::Value, + stacks: &FuncTranslationStacks, ) -> (ir::Value, ir::Value) { log::trace!("emit_inline_alloc(kind={kind:?}, ty={ty:?}, size={size}, align={align})"); @@ -64,7 +66,7 @@ impl NullCompiler { .ins() .iconst(ir::types::I32, i64::from(VMGcKind::MASK)); let masked = builder.ins().band(size, mask); - func_env.trapnz(builder, masked, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, masked, crate::TRAP_ALLOCATION_TOO_LARGE, stacks); // Load the bump "pointer" (it is actually an index into the GC heap, // not a raw pointer). @@ -94,6 +96,7 @@ impl NullCompiler { next, align_minus_one, crate::TRAP_ALLOCATION_TOO_LARGE, + stacks, ); let not_align_minus_one = builder.ins().bnot(align_minus_one); let aligned = builder @@ -101,8 +104,13 @@ impl NullCompiler { .band(next_plus_align_minus_one, not_align_minus_one); // Check whether the allocation fits in the heap space we have left. - let end_of_object = - func_env.uadd_overflow_trap(builder, aligned, size, crate::TRAP_ALLOCATION_TOO_LARGE); + let end_of_object = func_env.uadd_overflow_trap( + builder, + aligned, + size, + crate::TRAP_ALLOCATION_TOO_LARGE, + stacks, + ); let uext_end_of_object = uextend_i32_to_pointer_type(builder, pointer_type, end_of_object); let bound = func_env.get_gc_heap_bound(builder); let is_in_bounds = builder.ins().icmp( @@ -187,6 +195,7 @@ impl GcCompiler for NullCompiler { builder: &mut FunctionBuilder<'_>, array_type_index: TypeIndex, init: super::ArrayInit<'_>, + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.types[array_type_index].unwrap_module_type_index(); @@ -201,7 +210,7 @@ impl GcCompiler for NullCompiler { // First, compute the array's total size from its base size, element // size, and length. let len = init.len(&mut builder.cursor()); - let size = emit_array_size(func_env, builder, &array_layout, len); + let size = emit_array_size(func_env, builder, &array_layout, len, stacks); // Next, allocate the array. assert!(align.is_power_of_two()); @@ -213,6 +222,7 @@ impl GcCompiler for NullCompiler { Some(interned_type_index), size, align, + stacks, ); // Write the array's length into its field. @@ -237,7 +247,7 @@ impl GcCompiler for NullCompiler { size, elems_addr, |func_env, builder, elem_ty, elem_addr, val| { - write_field_at_addr(func_env, builder, elem_ty, elem_addr, val) + write_field_at_addr(func_env, builder, elem_ty, elem_addr, val, stacks) }, )?; @@ -250,6 +260,7 @@ impl GcCompiler for NullCompiler { builder: &mut FunctionBuilder<'_>, struct_type_index: TypeIndex, field_vals: &[ir::Value], + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index(); @@ -274,6 +285,7 @@ impl GcCompiler for NullCompiler { Some(interned_type_index), struct_size_val, align, + stacks, ); // Initialize the struct's fields. @@ -288,7 +300,7 @@ impl GcCompiler for NullCompiler { raw_struct_pointer, field_vals, |func_env, builder, ty, field_addr, val| { - write_field_at_addr(func_env, builder, ty, field_addr, val) + write_field_at_addr(func_env, builder, ty, field_addr, val, stacks) }, )?; @@ -303,6 +315,7 @@ impl GcCompiler for NullCompiler { field_vals: &[ir::Value], instance_id: ir::Value, tag: ir::Value, + stacks: &FuncTranslationStacks, ) -> WasmResult { let interned_type_index = func_env.module.tags[tag_index] .exception @@ -326,6 +339,7 @@ impl GcCompiler for NullCompiler { Some(interned_type_index), exn_size_val, align, + stacks, ); // Initialize the exception object's fields. @@ -340,7 +354,7 @@ impl GcCompiler for NullCompiler { raw_exn_pointer, field_vals, |func_env, builder, ty, field_addr, val| { - write_field_at_addr(func_env, builder, ty, field_addr, val) + write_field_at_addr(func_env, builder, ty, field_addr, val, stacks) }, )?; @@ -354,6 +368,7 @@ impl GcCompiler for NullCompiler { WasmStorageType::Val(WasmValType::I32), instance_id_addr, instance_id, + stacks, )?; let tag_addr = builder .ins() @@ -364,6 +379,7 @@ impl GcCompiler for NullCompiler { WasmStorageType::Val(WasmValType::I32), tag_addr, tag, + stacks, )?; Ok(exn_ref) @@ -376,6 +392,7 @@ impl GcCompiler for NullCompiler { _ty: WasmRefType, src: ir::Value, flags: ir::MemFlags, + _stacks: &FuncTranslationStacks, ) -> WasmResult { // NB: Don't use `unbarriered_load_gc_ref` here because we don't need to // mark the value as requiring inclusion in stack maps. @@ -390,6 +407,7 @@ impl GcCompiler for NullCompiler { dst: ir::Value, new_val: ir::Value, flags: ir::MemFlags, + _stacks: &FuncTranslationStacks, ) -> WasmResult<()> { unbarriered_store_gc_ref(builder, ty.heap_type, dst, new_val, flags) } diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index b7855e3f2746..d55406eb67d6 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -185,7 +185,7 @@ pub fn translate_operator( ***********************************************************************************/ Operator::GlobalGet { global_index } => { let global_index = GlobalIndex::from_u32(*global_index); - let val = environ.translate_global_get(builder, global_index)?; + let val = environ.translate_global_get(builder, global_index, stack)?; stack.push1(val); } Operator::GlobalSet { global_index } => { @@ -195,7 +195,7 @@ pub fn translate_operator( if builder.func.dfg.value_type(val).is_vector() { val = optionally_bitcast_vector(val, I8X16, builder); } - environ.translate_global_set(builder, global_index, val)?; + environ.translate_global_set(builder, global_index, val, stack)?; } /********************************* Stack misc *************************************** * `drop`, `nop`, `unreachable` and `select`. @@ -230,7 +230,7 @@ pub fn translate_operator( // We do nothing } Operator::Unreachable => { - environ.trap(builder, crate::TRAP_UNREACHABLE); + environ.trap(builder, crate::TRAP_UNREACHABLE, stack); stack.reachable = false; } /***************************** Control flow blocks ********************************** @@ -622,14 +622,20 @@ pub fn translate_operator( let tag_index = TagIndex::from_u32(*tag_index); let arity = environ.tag_param_arity(tag_index); let args = stack.peekn(arity); - environ.translate_exn_throw(builder, tag_index, args, stack.handlers.handlers())?; + environ.translate_exn_throw( + builder, + tag_index, + args, + stack.handlers.handlers(), + stack, + )?; stack.popn(arity); stack.reachable = false; } Operator::ThrowRef => { let exnref = stack.pop1(); - environ.translate_exn_throw_ref(builder, exnref, stack.handlers.handlers())?; + environ.translate_exn_throw_ref(builder, exnref, stack.handlers.handlers(), stack)?; stack.reachable = false; } @@ -1083,19 +1089,19 @@ pub fn translate_operator( } Operator::I64TruncF64S | Operator::I64TruncF32S => { let val = stack.pop1(); - stack.push1(environ.translate_fcvt_to_sint(builder, I64, val)); + stack.push1(environ.translate_fcvt_to_sint(builder, I64, val, stack)); } Operator::I32TruncF64S | Operator::I32TruncF32S => { let val = stack.pop1(); - stack.push1(environ.translate_fcvt_to_sint(builder, I32, val)); + stack.push1(environ.translate_fcvt_to_sint(builder, I32, val, stack)); } Operator::I64TruncF64U | Operator::I64TruncF32U => { let val = stack.pop1(); - stack.push1(environ.translate_fcvt_to_uint(builder, I64, val)); + stack.push1(environ.translate_fcvt_to_uint(builder, I64, val, stack)); } Operator::I32TruncF64U | Operator::I32TruncF32U => { let val = stack.pop1(); - stack.push1(environ.translate_fcvt_to_uint(builder, I32, val)); + stack.push1(environ.translate_fcvt_to_uint(builder, I32, val, stack)); } Operator::I64TruncSatF64S | Operator::I64TruncSatF32S => { let val = stack.pop1(); @@ -1222,19 +1228,19 @@ pub fn translate_operator( } Operator::I32DivS | Operator::I64DivS => { let (arg1, arg2) = stack.pop2(); - stack.push1(environ.translate_sdiv(builder, arg1, arg2)); + stack.push1(environ.translate_sdiv(builder, arg1, arg2, stack)); } Operator::I32DivU | Operator::I64DivU => { let (arg1, arg2) = stack.pop2(); - stack.push1(environ.translate_udiv(builder, arg1, arg2)); + stack.push1(environ.translate_udiv(builder, arg1, arg2, stack)); } Operator::I32RemS | Operator::I64RemS => { let (arg1, arg2) = stack.pop2(); - stack.push1(environ.translate_srem(builder, arg1, arg2)); + stack.push1(environ.translate_srem(builder, arg1, arg2, stack)); } Operator::I32RemU | Operator::I64RemU => { let (arg1, arg2) = stack.pop2(); - stack.push1(environ.translate_urem(builder, arg1, arg2)); + stack.push1(environ.translate_urem(builder, arg1, arg2, stack)); } Operator::F32Min | Operator::F64Min => { let (arg1, arg2) = stack.pop2(); @@ -1325,7 +1331,13 @@ pub fn translate_operator( } else { let index_type = environ.heaps()[heap].index_type(); let offset = builder.ins().iconst(index_type, memarg.offset as i64); - environ.uadd_overflow_trap(builder, addr, offset, ir::TrapCode::HEAP_OUT_OF_BOUNDS) + environ.uadd_overflow_trap( + builder, + addr, + offset, + ir::TrapCode::HEAP_OUT_OF_BOUNDS, + stack, + ) }; // `fn translate_atomic_wait` can inspect the type of `expected` to figure out what // code it needs to generate, if it wants. @@ -1349,7 +1361,13 @@ pub fn translate_operator( } else { let index_type = environ.heaps()[heap].index_type(); let offset = builder.ins().iconst(index_type, memarg.offset as i64); - environ.uadd_overflow_trap(builder, addr, offset, ir::TrapCode::HEAP_OUT_OF_BOUNDS) + environ.uadd_overflow_trap( + builder, + addr, + offset, + ir::TrapCode::HEAP_OUT_OF_BOUNDS, + stack, + ) }; let res = environ.translate_atomic_notify( builder, @@ -1606,13 +1624,13 @@ pub fn translate_operator( Operator::TableGet { table: index } => { let table_index = TableIndex::from_u32(*index); let index = stack.pop1(); - stack.push1(environ.translate_table_get(builder, table_index, index)?); + stack.push1(environ.translate_table_get(builder, table_index, index, stack)?); } Operator::TableSet { table: index } => { let table_index = TableIndex::from_u32(*index); let value = stack.pop1(); let index = stack.pop1(); - environ.translate_table_set(builder, table_index, value, index)?; + environ.translate_table_set(builder, table_index, value, index, stack)?; } Operator::TableCopy { dst_table: dst_table_index, @@ -2555,7 +2573,7 @@ pub fn translate_operator( unreachable!("validation") }; let is_null = environ.translate_ref_is_null(builder.cursor(), r, *r_ty)?; - environ.trapnz(builder, is_null, crate::TRAP_NULL_REFERENCE); + environ.trapnz(builder, is_null, crate::TRAP_NULL_REFERENCE, stack); stack.push1(r); } @@ -2566,12 +2584,12 @@ pub fn translate_operator( } Operator::I31GetS => { let i31ref = stack.pop1(); - let val = environ.translate_i31_get_s(builder, i31ref)?; + let val = environ.translate_i31_get_s(builder, i31ref, stack)?; stack.push1(val); } Operator::I31GetU => { let i31ref = stack.pop1(); - let val = environ.translate_i31_get_u(builder, i31ref)?; + let val = environ.translate_i31_get_u(builder, i31ref, stack)?; stack.push1(val); } @@ -2580,13 +2598,15 @@ pub fn translate_operator( let arity = environ.struct_fields_len(struct_type_index)?; let fields: StructFieldsVec = stack.peekn(arity).iter().copied().collect(); stack.popn(arity); - let struct_ref = environ.translate_struct_new(builder, struct_type_index, fields)?; + let struct_ref = + environ.translate_struct_new(builder, struct_type_index, fields, stack)?; stack.push1(struct_ref); } Operator::StructNewDefault { struct_type_index } => { let struct_type_index = TypeIndex::from_u32(*struct_type_index); - let struct_ref = environ.translate_struct_new_default(builder, struct_type_index)?; + let struct_ref = + environ.translate_struct_new_default(builder, struct_type_index, stack)?; stack.push1(struct_ref); } @@ -2603,6 +2623,7 @@ pub fn translate_operator( *field_index, struct_ref, val, + stack, )?; } @@ -2618,6 +2639,7 @@ pub fn translate_operator( *field_index, struct_ref, Some(Extension::Sign), + stack, )?; stack.push1(val); } @@ -2634,6 +2656,7 @@ pub fn translate_operator( *field_index, struct_ref, Some(Extension::Zero), + stack, )?; stack.push1(val); } @@ -2650,6 +2673,7 @@ pub fn translate_operator( *field_index, struct_ref, None, + stack, )?; stack.push1(val); } @@ -2657,13 +2681,15 @@ pub fn translate_operator( Operator::ArrayNew { array_type_index } => { let array_type_index = TypeIndex::from_u32(*array_type_index); let (elem, len) = stack.pop2(); - let array_ref = environ.translate_array_new(builder, array_type_index, elem, len)?; + let array_ref = + environ.translate_array_new(builder, array_type_index, elem, len, stack)?; stack.push1(array_ref); } Operator::ArrayNewDefault { array_type_index } => { let array_type_index = TypeIndex::from_u32(*array_type_index); let len = stack.pop1(); - let array_ref = environ.translate_array_new_default(builder, array_type_index, len)?; + let array_ref = + environ.translate_array_new_default(builder, array_type_index, len, stack)?; stack.push1(array_ref); } Operator::ArrayNewFixed { @@ -2673,7 +2699,8 @@ pub fn translate_operator( let array_type_index = TypeIndex::from_u32(*array_type_index); let array_size = usize::try_from(*array_size).unwrap(); let elems = stack.peekn(array_size); - let array_ref = environ.translate_array_new_fixed(builder, array_type_index, elems)?; + let array_ref = + environ.translate_array_new_fixed(builder, array_type_index, elems, stack)?; stack.popn(array_size); stack.push1(array_ref); } @@ -2730,7 +2757,15 @@ pub fn translate_operator( Operator::ArrayFill { array_type_index } => { let array_type_index = TypeIndex::from_u32(*array_type_index); let (array, index, val, len) = stack.pop4(); - environ.translate_array_fill(builder, array_type_index, array, index, val, len)?; + environ.translate_array_fill( + builder, + array_type_index, + array, + index, + val, + len, + stack, + )?; } Operator::ArrayInitData { array_type_index, @@ -2768,14 +2803,20 @@ pub fn translate_operator( } Operator::ArrayLen => { let array = stack.pop1(); - let len = environ.translate_array_len(builder, array)?; + let len = environ.translate_array_len(builder, array, stack)?; stack.push1(len); } Operator::ArrayGet { array_type_index } => { let array_type_index = TypeIndex::from_u32(*array_type_index); let (array, index) = stack.pop2(); - let elem = - environ.translate_array_get(builder, array_type_index, array, index, None)?; + let elem = environ.translate_array_get( + builder, + array_type_index, + array, + index, + None, + stack, + )?; stack.push1(elem); } Operator::ArrayGetS { array_type_index } => { @@ -2787,6 +2828,7 @@ pub fn translate_operator( array, index, Some(Extension::Sign), + stack, )?; stack.push1(elem); } @@ -2799,13 +2841,14 @@ pub fn translate_operator( array, index, Some(Extension::Zero), + stack, )?; stack.push1(elem); } Operator::ArraySet { array_type_index } => { let array_type_index = TypeIndex::from_u32(*array_type_index); let (array, index, elem) = stack.pop3(); - environ.translate_array_set(builder, array_type_index, array, index, elem)?; + environ.translate_array_set(builder, array_type_index, array, index, elem, stack)?; } Operator::RefEq => { let (r1, r2) = stack.pop2(); @@ -2827,6 +2870,7 @@ pub fn translate_operator( }, r, *r_ty, + stack, )?; stack.push1(result); } @@ -2844,6 +2888,7 @@ pub fn translate_operator( }, r, *r_ty, + stack, )?; stack.push1(result); } @@ -2861,8 +2906,9 @@ pub fn translate_operator( }, r, *r_ty, + stack, )?; - environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE); + environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE, stack); stack.push1(r); } Operator::RefCastNullable { hty } => { @@ -2879,8 +2925,9 @@ pub fn translate_operator( }, r, *r_ty, + stack, )?; - environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE); + environ.trapz(builder, cast_okay, crate::TRAP_CAST_FAILURE, stack); stack.push1(r); } Operator::BrOnCast { @@ -2894,7 +2941,7 @@ pub fn translate_operator( }; let to_ref_type = environ.convert_ref_type(*to_ref_type)?; - let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty)?; + let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty, stack)?; let (cast_succeeds_block, inputs) = translate_br_if_args(*relative_depth, stack); let cast_fails_block = builder.create_block(); @@ -2928,7 +2975,7 @@ pub fn translate_operator( }; let to_ref_type = environ.convert_ref_type(*to_ref_type)?; - let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty)?; + let cast_is_okay = environ.translate_ref_test(builder, to_ref_type, r, *r_ty, stack)?; let (cast_fails_block, inputs) = translate_br_if_args(*relative_depth, stack); let cast_succeeds_block = builder.create_block(); @@ -3453,6 +3500,7 @@ fn prepare_addr( access_size, }, ir::TrapCode::HEAP_OUT_OF_BOUNDS, + stack, ), // If the offset doesn't fit within a u32, then we can't pass it @@ -3490,6 +3538,7 @@ fn prepare_addr( index, offset, ir::TrapCode::HEAP_OUT_OF_BOUNDS, + stack, ); bounds_check_and_compute_addr( builder, @@ -3501,6 +3550,7 @@ fn prepare_addr( access_size, }, ir::TrapCode::HEAP_OUT_OF_BOUNDS, + stack, ) } }; @@ -3560,7 +3610,7 @@ fn align_atomic_addr( .ins() .band_imm(effective_addr, i64::from(loaded_bytes - 1)); let f = builder.ins().icmp_imm(IntCC::NotEqual, misalignment, 0); - environ.trapnz(builder, f, crate::TRAP_HEAP_MISALIGNED); + environ.trapnz(builder, f, crate::TRAP_HEAP_MISALIGNED, stack); } } @@ -4379,7 +4429,7 @@ fn create_catch_block( if let Some(tag) = tag { let tag = TagIndex::from_u32(tag); - params.extend(environ.translate_exn_unbox(builder, tag, exn_ref)?); + params.extend(environ.translate_exn_unbox(builder, tag, exn_ref, stacks)?); } if is_ref { params.push(exn_ref); diff --git a/crates/cranelift/src/translate/table.rs b/crates/cranelift/src/translate/table.rs index 4d01da7802c9..55fbc8fe4744 100644 --- a/crates/cranelift/src/translate/table.rs +++ b/crates/cranelift/src/translate/table.rs @@ -1,4 +1,5 @@ use crate::func_environ::FuncEnvironment; +use crate::translate::FuncTranslationStacks; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::{self, InstBuilder, condcodes::IntCC, immediates::Imm64}; use cranelift_codegen::isa::TargetIsa; @@ -62,6 +63,7 @@ impl TableData { env: &mut FuncEnvironment<'_>, pos: &mut FunctionBuilder, mut index: ir::Value, + stacks: &FuncTranslationStacks, ) -> (ir::Value, ir::MemFlags) { let index_ty = pos.func.dfg.value_type(index); let addr_ty = env.pointer_type(); @@ -78,7 +80,7 @@ impl TableData { .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound); if !spectre_mitigations_enabled { - env.trapnz(pos, oob, crate::TRAP_TABLE_OUT_OF_BOUNDS); + env.trapnz(pos, oob, crate::TRAP_TABLE_OUT_OF_BOUNDS, stacks); } // Convert `index` to `addr_ty`. diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rs index 70bcef9526a0..1e681564e3cd 100644 --- a/crates/wasmtime/src/runtime/debug.rs +++ b/crates/wasmtime/src/runtime/debug.rs @@ -43,10 +43,11 @@ impl<'a, T> StoreContextMut<'a, T> { // `StoreOpaque`), which owns all active stacks in the // store. We do not provide any API that could mutate the // frames that we are walking on the `DebugFrameCursor`. + let is_trapping_frame = unsafe { *self.0.vm_store_context().last_wasm_exit_was_trap.get() }; let iter = unsafe { CurrentActivationBacktrace::new(self) }; let mut view = DebugFrameCursor { iter, - is_trapping_frame: false, + is_trapping_frame, frames: vec![], current: None, }; diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 7de4f2229be4..f1046a1e5230 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1525,7 +1525,9 @@ pub(crate) struct EntryStoreContext { /// `VMStoreContext` when exiting Wasm. pub stack_limit: Option, pub last_wasm_exit_pc: usize, + pub last_wasm_exit_was_trap: bool, pub last_wasm_exit_trampoline_fp: usize, + pub last_wasm_exit_trap_fp: usize, pub last_wasm_entry_fp: usize, pub last_wasm_entry_sp: usize, pub last_wasm_entry_trap_handler: usize, @@ -1625,6 +1627,8 @@ impl EntryStoreContext { last_wasm_exit_trampoline_fp: *(*vm_store_context) .last_wasm_exit_trampoline_fp .get(), + last_wasm_exit_was_trap: *(*vm_store_context).last_wasm_exit_was_trap.get(), + last_wasm_exit_trap_fp: *(*vm_store_context).last_wasm_exit_trap_fp.get(), last_wasm_entry_fp: *(*vm_store_context).last_wasm_entry_fp.get(), last_wasm_entry_sp: *(*vm_store_context).last_wasm_entry_sp.get(), last_wasm_entry_trap_handler: *(*vm_store_context) @@ -1649,6 +1653,8 @@ impl EntryStoreContext { *(*self.vm_store_context).last_wasm_exit_trampoline_fp.get() = self.last_wasm_exit_trampoline_fp; + *(*self.vm_store_context).last_wasm_exit_trap_fp.get() = self.last_wasm_exit_trap_fp; + *(*self.vm_store_context).last_wasm_exit_was_trap.get() = self.last_wasm_exit_was_trap; *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc; *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp; *(*self.vm_store_context).last_wasm_entry_sp.get() = self.last_wasm_entry_sp; diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs index 3089e4c4ef5f..49fee82b27c0 100644 --- a/crates/wasmtime/src/runtime/trap.rs +++ b/crates/wasmtime/src/runtime/trap.rs @@ -99,6 +99,7 @@ pub(crate) fn from_runtime_box( crate::runtime::vm::TrapReason::User(error) => (error, None), crate::runtime::vm::TrapReason::Jit { pc, + fp: _, faulting_addr, trap, } => { diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index f67e088d101a..f2e11218f847 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -471,6 +471,8 @@ impl InterpreterRef<'_> { TrapKind::StackOverflow => Trap::StackOverflow, }; s.set_jit_trap(regs, None, trap); + log::trace!("about to invoke debug event from interpreter"); + s.debug_event_from_interpreter(); s.entry_trap_handler() } None => { diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 353f8e608347..2b573d213bb7 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -18,7 +18,7 @@ pub use self::signals::*; #[cfg(feature = "gc")] use crate::ThrownException; use crate::runtime::module::lookup_code; -use crate::runtime::store::{ExecutorRef, StoreOpaque}; +use crate::runtime::store::ExecutorRef; use crate::runtime::vm::sys::traphandlers; use crate::runtime::vm::{InterpreterRef, VMContext, VMStore, VMStoreContext, f32x4, f64x2, i8x16}; #[cfg(feature = "debug")] @@ -367,6 +367,10 @@ pub enum TrapReason { /// the trapping address to a trap code. pc: usize, + /// The FP register from when this trap originated. + #[allow(dead_code, reason = "used in debug configuration")] + fp: usize, + /// If the trap was a memory-related trap such as SIGSEGV then this /// field will contain the address of the inaccessible data. /// @@ -546,6 +550,56 @@ mod call_thread_state { pub(crate) vm_store_context: NonNull, pub(crate) unwinder: &'static dyn Unwind, + /// Raw pointer to the `*mut dyn VMStore` running in this + /// activation, to be used *only* when re-entering the host + /// during a trap. + /// + /// This is a very tricky ownership/provenance dance. When + /// control is in the Wasm code itself, the store is + /// completely owned by the Wasm. It passes ownership back + /// during hostcalls via mutable reborrow (as with any call + /// with a `&mut self` in Rust). That's all well and good for + /// explicit calls. + /// + /// When a trap occurs, however, we can also think of the + /// ownership passing as-if the trapping instruction were a + /// hostcall with a `&mut dyn VMStore` parameter. This works + /// *as long as* all possibly trapping points in compiled code + /// act as if they invalidate any other held borrows into the + /// store. + /// + /// It turns out that we generally enforce this in compiled + /// guest code in Cranelift: any `can_trap` opcode returns + /// `true` from `has_memory_fence_semantics()` (see + /// corresponding comment there). This is enough to ensure + /// that the compiler treats every trapping op as-if it were a + /// hostcall, which clobbers all memory state; so from the + /// Wasm code's point of view, it is safely reborrowing the + /// Store and passing it "somewhere" on every trap. The + /// plumbing for that "passing" goes through this field, but + /// that is an implementation detail. When control comes back + /// out of the Wasm activation, we clear this field; the + /// invocation itself takes a mutable borrow of the store, so + /// safety is preserved on the caller side as well. In other + /// words, the provenance is something like + /// + /// ```plain + /// + /// host (caller side) with `&mut dyn VMStore` + /// / \ + /// (param into / (this field) + /// entry trampoline) \ + /// | | + /// ~~~~~~~ (wasm code) ~~~~~~ + /// | | + /// libcall trap + /// ``` + /// + /// with only *one* of those paths dynamically taken at any + /// given time. + #[cfg(all(feature = "debug", feature = "pulley"))] + pub(crate) raw_store: NonNull, + pub(super) prev: Cell, // The state of the runtime for the *previous* `CallThreadState` for @@ -570,7 +624,7 @@ mod call_thread_state { impl CallThreadState { #[inline] pub(super) fn new( - store: &mut StoreOpaque, + store: &mut dyn VMStore, old_state: *mut EntryStoreContext, ) -> CallThreadState { CallThreadState { @@ -582,6 +636,8 @@ mod call_thread_state { #[cfg(feature = "coredump")] capture_coredump: store.engine().config().coredump_on_trap, vm_store_context: store.vm_store_context_ptr(), + #[cfg(all(feature = "debug", feature = "pulley"))] + raw_store: NonNull::from_mut(store), prev: Cell::new(ptr::null()), old_state, } @@ -594,10 +650,18 @@ mod call_thread_state { /// Requires that the saved last Wasm trampoline FP points to /// a valid trampoline frame, or is null. pub unsafe fn old_last_wasm_exit_fp(&self) -> usize { - let trampoline_fp = unsafe { (&*self.old_state).last_wasm_exit_trampoline_fp }; - // SAFETY: `trampoline_fp` is either a valid FP from an - // active trampoline frame or is null. - unsafe { VMStoreContext::wasm_exit_fp_from_trampoline_fp(trampoline_fp) } + // SAFETY: saved fields adhere to invariants: if + // `was_trap` is set, `trap_fp` is valid, otherwise + // `trampoline_fp` is either a valid FP from an active + // trampoline frame or is null. + unsafe { + if (*self.old_state).last_wasm_exit_was_trap { + (*self.old_state).last_wasm_exit_trap_fp + } else { + let trampoline_fp = (*self.old_state).last_wasm_exit_trampoline_fp; + VMStoreContext::wasm_exit_fp_from_trampoline_fp(trampoline_fp) + } + } } /// Get the saved PC upon exit from Wasm for the previous `CallThreadState`. @@ -678,6 +742,14 @@ mod call_thread_state { &cx.last_wasm_exit_pc, &mut (*self.old_state).last_wasm_exit_pc, ); + swap( + &cx.last_wasm_exit_was_trap, + &mut (*self.old_state).last_wasm_exit_was_trap, + ); + swap( + &cx.last_wasm_exit_trap_fp, + &mut (*self.old_state).last_wasm_exit_trap_fp, + ); swap( &cx.last_wasm_entry_fp, &mut (*self.old_state).last_wasm_entry_fp, @@ -826,73 +898,7 @@ impl CallThreadState { unsafe fn unwind(&self, store: &mut dyn VMStore) { #[allow(unused_mut, reason = "only mutated in `debug` configuration")] let mut unwind = self.unwind.replace(UnwindState::None); - - #[cfg(feature = "debug")] - { - let result = match &unwind { - UnwindState::UnwindToWasm(_) => { - assert!(store.as_store_opaque().has_pending_exception()); - let exn = store - .as_store_opaque() - .pending_exception_owned_rooted() - .expect("exception should be set when we are throwing"); - store.block_on_debug_handler(crate::DebugEvent::CaughtExceptionThrown(exn)) - } - - UnwindState::UnwindToHost { - reason: UnwindReason::Trap(TrapReason::Exception), - .. - } => { - use crate::store::AsStoreOpaque; - let exn = store - .as_store_opaque() - .pending_exception_owned_rooted() - .expect("exception should be set when we are throwing"); - store.block_on_debug_handler(crate::DebugEvent::UncaughtExceptionThrown( - exn.clone(), - )) - } - UnwindState::UnwindToHost { - reason: UnwindReason::Trap(TrapReason::Wasm(trap)), - .. - } => store.block_on_debug_handler(crate::DebugEvent::Trap(*trap)), - UnwindState::UnwindToHost { - reason: UnwindReason::Trap(TrapReason::User(err)), - .. - } => store.block_on_debug_handler(crate::DebugEvent::HostcallError(err)), - - UnwindState::UnwindToHost { - reason: UnwindReason::Trap(TrapReason::Jit { .. }), - .. - } => { - // JIT traps not handled yet. - Ok(()) - } - #[cfg(all(feature = "std", panic = "unwind"))] - UnwindState::UnwindToHost { - reason: UnwindReason::Panic(_), - .. - } => { - // We don't invoke any debugger hook when we're - // unwinding due to a Rust (host-side) panic. - Ok(()) - } - - UnwindState::None => unreachable!(), - }; - - // If the debugger invocation itself resulted in an `Err` - // (which can only come from the `block_on` hitting a - // failure mode), we need to override our unwind as-if - // were handling a host error. - if let Err(err) = result { - unwind = UnwindState::UnwindToHost { - reason: UnwindReason::Trap(TrapReason::User(err)), - backtrace: None, - coredump_stack: None, - }; - } - } + self.debug_event_on_unwind(store, &mut unwind); match unwind { UnwindState::UnwindToHost { .. } => { @@ -937,6 +943,114 @@ impl CallThreadState { } } + /// From the Pulley interpreter, perform a fiber suspend for a + /// debug event handler after setting the unwind state with + /// `set_jit_trap`. + #[cfg(all(feature = "debug", feature = "pulley"))] + pub(crate) fn debug_event_from_interpreter(&self) { + let mut unwind = self.unwind.replace(UnwindState::None); + // SAFETY: this is invoked only within a trapping context when + // we have received control back from the Wasm code. See the + // provenance diagram and comments on `self.raw_store` for + // more details. + let store = unsafe { self.raw_store.as_ptr().as_mut().unwrap() }; + self.debug_event_on_unwind(store, &mut unwind); + self.unwind.set(unwind); + } + + /// Suspend from the current fiber, blocking on an async debug + /// callback hook, if any, if the `unwind` state merits a debug + /// event. + #[cfg(feature = "debug")] + fn debug_event_on_unwind(&self, store: &mut dyn VMStore, unwind: &mut UnwindState) { + let result = match unwind { + UnwindState::UnwindToWasm(_) => { + assert!(store.as_store_opaque().has_pending_exception()); + let exn = store + .as_store_opaque() + .pending_exception_owned_rooted() + .expect("exception should be set when we are throwing"); + store.block_on_debug_handler(crate::DebugEvent::CaughtExceptionThrown(exn)) + } + + UnwindState::UnwindToHost { + reason: UnwindReason::Trap(TrapReason::Exception), + .. + } => { + use crate::store::AsStoreOpaque; + let exn = store + .as_store_opaque() + .pending_exception_owned_rooted() + .expect("exception should be set when we are throwing"); + store + .block_on_debug_handler(crate::DebugEvent::UncaughtExceptionThrown(exn.clone())) + } + UnwindState::UnwindToHost { + reason: UnwindReason::Trap(TrapReason::Wasm(trap)), + .. + } => store.block_on_debug_handler(crate::DebugEvent::Trap(*trap)), + UnwindState::UnwindToHost { + reason: UnwindReason::Trap(TrapReason::User(err)), + .. + } => store.block_on_debug_handler(crate::DebugEvent::HostcallError(err)), + + UnwindState::UnwindToHost { + reason: UnwindReason::Trap(TrapReason::Jit { pc, fp, trap, .. }), + .. + } => self.with_trap_exit_state(*pc, *fp, |_| { + store.block_on_debug_handler(crate::DebugEvent::Trap(*trap)) + }), + #[cfg(all(feature = "std", panic = "unwind"))] + UnwindState::UnwindToHost { + reason: UnwindReason::Panic(_), + .. + } => { + // We don't invoke any debugger hook when we're + // unwinding due to a Rust (host-side) panic. + Ok(()) + } + + UnwindState::None => unreachable!(), + }; + + // If the debugger invocation itself resulted in an `Err` + // (which can only come from the `block_on` hitting a + // failure mode), we need to override our unwind as-if + // were handling a host error. + if let Err(err) = result { + *unwind = UnwindState::UnwindToHost { + reason: UnwindReason::Trap(TrapReason::User(err)), + backtrace: None, + coredump_stack: None, + }; + } + } + + #[cfg(not(feature = "debug"))] + pub(crate) fn debug_event_on_unwind( + &self, + _store: &mut dyn VMStore, + _unwind: &mut UnwindState, + ) { + } + + /// Set up our state according to a trap exit back to the host. + #[cfg(feature = "debug")] + fn with_trap_exit_state R>(&self, pc: usize, fp: usize, f: F) -> R { + unsafe { + let cx = self.vm_store_context.as_ref(); + *cx.last_wasm_exit_pc.get() = pc; + *cx.last_wasm_exit_trap_fp.get() = fp; + *cx.last_wasm_exit_was_trap.get() = true; + } + let result = f(self); + unsafe { + let cx = self.vm_store_context.as_ref(); + *cx.last_wasm_exit_was_trap.get() = false; + } + result + } + pub(crate) fn entry_trap_handler(&self) -> Handler { unsafe { let vm_store_context = self.vm_store_context.as_ref(); @@ -1053,6 +1167,7 @@ impl CallThreadState { self.unwind.set(UnwindState::UnwindToHost { reason: UnwindReason::Trap(TrapReason::Jit { pc, + fp, faulting_addr, trap, }), diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index be9403e97f88..99592736e46d 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -366,6 +366,9 @@ impl<'a, T: 'static> CurrentActivationBacktrace<'a, T> { let exit_pc = unsafe { *(*vm_store_context).last_wasm_exit_pc.get() }; let exit_fp = unsafe { (*vm_store_context).last_wasm_exit_fp() }; let trampoline_fp = unsafe { *(*vm_store_context).last_wasm_entry_fp.get() }; + log::trace!( + "activation backtrace: exit_pc {exit_pc:x} exit_fp {exit_fp:x} entry_fp {trampoline_fp:x}" + ); let inner: Box> = if exit_fp == 0 { // No activations on this Store; return an empty iterator. Box::new(core::iter::empty()) diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index ef714422d688..df28bf060c31 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1188,6 +1188,25 @@ pub struct VMStoreContext { /// situation while this field is read it'll never classify a fault as an /// guard page fault. pub async_guard_range: Range<*mut u8>, + + /// The last Wasm exit to host was due to a trap. + /// + /// This is set whenever we update the exit state *from the host* + /// when handling a trap. It allows us to interpret the exit PC + /// correctly -- that is, either pointing *to* a trapping + /// instruction, or *after* a call (a single PC could be both + /// after a call and at a trapping instruction!). + /// + /// It is set *only* from host code, but is kept here alongside + /// the other last-exit state for consistency. + pub last_wasm_exit_was_trap: UnsafeCell, + + /// The last Wasm exit to host, if via trap, had this FP in guest + /// code (*not* in a trampoline). + /// + /// Either this field or `last_wasm_exit_trampoline_fp` is set, + /// never both. This field is always set from the host. + pub last_wasm_exit_trap_fp: UnsafeCell, } impl VMStoreContext { @@ -1210,8 +1229,12 @@ impl VMStoreContext { // will be writing our store when we have control), and the // helper function's safety condition is the same as ours. unsafe { - let trampoline_fp = *self.last_wasm_exit_trampoline_fp.get(); - Self::wasm_exit_fp_from_trampoline_fp(trampoline_fp) + if *self.last_wasm_exit_was_trap.get() { + *self.last_wasm_exit_trap_fp.get() + } else { + let trampoline_fp = *self.last_wasm_exit_trampoline_fp.get(); + Self::wasm_exit_fp_from_trampoline_fp(trampoline_fp) + } } } @@ -1271,6 +1294,8 @@ impl Default for VMStoreContext { stack_chain: UnsafeCell::new(VMStackChain::Absent), async_guard_range: ptr::null_mut()..ptr::null_mut(), store_data: VmPtr::dangling(), + last_wasm_exit_was_trap: UnsafeCell::new(false), + last_wasm_exit_trap_fp: UnsafeCell::new(0), } } } diff --git a/tests/all/debug.rs b/tests/all/debug.rs index 904b0b33c6b7..3eab430e7466 100644 --- a/tests/all/debug.rs +++ b/tests/all/debug.rs @@ -363,12 +363,6 @@ async fn caught_exception_events() -> anyhow::Result<()> { #[tokio::test] #[cfg_attr(miri, ignore)] -#[cfg(any( - target_arch = "x86_64", - target_arch = "aarch64", - target_arch = "s390x", - target_arch = "riscv64" -))] async fn hostcall_trap_events() -> anyhow::Result<()> { let _ = env_logger::try_init(); @@ -392,7 +386,14 @@ async fn hostcall_trap_events() -> anyhow::Result<()> { debug_event_checker!( D, store, { 0 ; - wasmtime::DebugEvent::Trap(wasmtime_environ::Trap::IntegerDivisionByZero) => {} + wasmtime::DebugEvent::Trap(wasmtime_environ::Trap::IntegerDivisionByZero) => { + let mut stack = store.debug_frames().unwrap(); + assert!(!stack.done()); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().0.as_u32(), 0); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().1, 37); + stack.move_to_parent(); + assert!(stack.done()); + } } ); From 7fd1f60e7caa12cd02d1acd2d64fc23b5dfb7f75 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 23 Oct 2025 13:27:42 -0700 Subject: [PATCH 2/4] Refactor Wasm activation exit state to have one FP only: Wasm FP, not trampoline FP. --- crates/cranelift/src/compiler.rs | 12 +-- crates/environ/src/vmoffsets.rs | 12 +-- crates/wasmtime/src/runtime/func.rs | 14 +--- crates/wasmtime/src/runtime/vm/throw.rs | 2 +- .../wasmtime/src/runtime/vm/traphandlers.rs | 23 +---- .../src/runtime/vm/traphandlers/backtrace.rs | 4 +- crates/wasmtime/src/runtime/vm/vmcontext.rs | 84 ++----------------- 7 files changed, 31 insertions(+), 120 deletions(-) diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index 0c001f6b18d4..c2af9cb9a52c 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -1620,15 +1620,17 @@ fn save_last_wasm_exit_fp_and_pc( ptr: &impl PtrSize, limits: Value, ) { - // Save the trampoline FP to the limits. Exception unwind needs - // this so that it can know the SP (bottom of frame) for the very - // last Wasm frame. + // Save the Wasm frame exit FP to the limits. We have the + // trampoline FP here; load the next FP in the chain. let trampoline_fp = builder.ins().get_frame_pointer(pointer_type); + let wasm_fp = builder + .ins() + .load(pointer_type, MemFlags::trusted(), trampoline_fp, 0); builder.ins().store( MemFlags::trusted(), - trampoline_fp, + wasm_fp, limits, - ptr.vmstore_context_last_wasm_exit_trampoline_fp(), + ptr.vmstore_context_last_wasm_exit_fp(), ); // Finally save the Wasm return address to the limits. diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 60c88a2c520e..15795ca6aa98 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -196,26 +196,26 @@ pub trait PtrSize { /// Return the offset of the `gc_heap.base` field within a `VMStoreContext`. fn vmstore_context_gc_heap_base(&self) -> u8 { let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_base(); - debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp()); + debug_assert!(offset < self.vmstore_context_last_wasm_exit_fp()); offset } /// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`. fn vmstore_context_gc_heap_current_length(&self) -> u8 { let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_current_length(); - debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp()); + debug_assert!(offset < self.vmstore_context_last_wasm_exit_fp()); offset } - /// Return the offset of the `last_wasm_exit_trampoline_fp` field - /// of `VMStoreContext`. - fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 { + /// Return the offset of the `last_wasm_exit_fp` field of + /// `VMStoreContext`. + fn vmstore_context_last_wasm_exit_fp(&self) -> u8 { self.vmstore_context_gc_heap() + self.size_of_vmmemory_definition() } /// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`. fn vmstore_context_last_wasm_exit_pc(&self) -> u8 { - self.vmstore_context_last_wasm_exit_trampoline_fp() + self.size() + self.vmstore_context_last_wasm_exit_fp() + self.size() } /// Return the offset of the `last_wasm_entry_sp` field of `VMStoreContext`. diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index f1046a1e5230..a8cdc8a411b1 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1525,9 +1525,8 @@ pub(crate) struct EntryStoreContext { /// `VMStoreContext` when exiting Wasm. pub stack_limit: Option, pub last_wasm_exit_pc: usize, + pub last_wasm_exit_fp: usize, pub last_wasm_exit_was_trap: bool, - pub last_wasm_exit_trampoline_fp: usize, - pub last_wasm_exit_trap_fp: usize, pub last_wasm_entry_fp: usize, pub last_wasm_entry_sp: usize, pub last_wasm_entry_trap_handler: usize, @@ -1624,11 +1623,8 @@ impl EntryStoreContext { Self { stack_limit, last_wasm_exit_pc: *(*vm_store_context).last_wasm_exit_pc.get(), - last_wasm_exit_trampoline_fp: *(*vm_store_context) - .last_wasm_exit_trampoline_fp - .get(), + last_wasm_exit_fp: *(*vm_store_context).last_wasm_exit_fp.get(), last_wasm_exit_was_trap: *(*vm_store_context).last_wasm_exit_was_trap.get(), - last_wasm_exit_trap_fp: *(*vm_store_context).last_wasm_exit_trap_fp.get(), last_wasm_entry_fp: *(*vm_store_context).last_wasm_entry_fp.get(), last_wasm_entry_sp: *(*vm_store_context).last_wasm_entry_sp.get(), last_wasm_entry_trap_handler: *(*vm_store_context) @@ -1651,11 +1647,9 @@ impl EntryStoreContext { *(&*self.vm_store_context).stack_limit.get() = limit; } - *(*self.vm_store_context).last_wasm_exit_trampoline_fp.get() = - self.last_wasm_exit_trampoline_fp; - *(*self.vm_store_context).last_wasm_exit_trap_fp.get() = self.last_wasm_exit_trap_fp; - *(*self.vm_store_context).last_wasm_exit_was_trap.get() = self.last_wasm_exit_was_trap; *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc; + *(*self.vm_store_context).last_wasm_exit_fp.get() = self.last_wasm_exit_fp; + *(*self.vm_store_context).last_wasm_exit_was_trap.get() = self.last_wasm_exit_was_trap; *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp; *(*self.vm_store_context).last_wasm_entry_sp.get() = self.last_wasm_entry_sp; *(*self.vm_store_context).last_wasm_entry_trap_handler.get() = diff --git a/crates/wasmtime/src/runtime/vm/throw.rs b/crates/wasmtime/src/runtime/vm/throw.rs index ba5105fe112f..75cc5c42064f 100644 --- a/crates/wasmtime/src/runtime/vm/throw.rs +++ b/crates/wasmtime/src/runtime/vm/throw.rs @@ -32,7 +32,7 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option { let (exit_pc, exit_fp, entry_fp) = unsafe { ( *nogc.vm_store_context().last_wasm_exit_pc.get(), - nogc.vm_store_context().last_wasm_exit_fp(), + *nogc.vm_store_context().last_wasm_exit_fp.get(), *nogc.vm_store_context().last_wasm_entry_fp.get(), ) }; diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 2b573d213bb7..774c989420a2 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -650,18 +650,7 @@ mod call_thread_state { /// Requires that the saved last Wasm trampoline FP points to /// a valid trampoline frame, or is null. pub unsafe fn old_last_wasm_exit_fp(&self) -> usize { - // SAFETY: saved fields adhere to invariants: if - // `was_trap` is set, `trap_fp` is valid, otherwise - // `trampoline_fp` is either a valid FP from an active - // trampoline frame or is null. - unsafe { - if (*self.old_state).last_wasm_exit_was_trap { - (*self.old_state).last_wasm_exit_trap_fp - } else { - let trampoline_fp = (*self.old_state).last_wasm_exit_trampoline_fp; - VMStoreContext::wasm_exit_fp_from_trampoline_fp(trampoline_fp) - } - } + unsafe { (*self.old_state).last_wasm_exit_fp } } /// Get the saved PC upon exit from Wasm for the previous `CallThreadState`. @@ -735,8 +724,8 @@ mod call_thread_state { unsafe { let cx = self.vm_store_context.as_ref(); swap( - &cx.last_wasm_exit_trampoline_fp, - &mut (*self.old_state).last_wasm_exit_trampoline_fp, + &cx.last_wasm_exit_fp, + &mut (*self.old_state).last_wasm_exit_fp, ); swap( &cx.last_wasm_exit_pc, @@ -746,10 +735,6 @@ mod call_thread_state { &cx.last_wasm_exit_was_trap, &mut (*self.old_state).last_wasm_exit_was_trap, ); - swap( - &cx.last_wasm_exit_trap_fp, - &mut (*self.old_state).last_wasm_exit_trap_fp, - ); swap( &cx.last_wasm_entry_fp, &mut (*self.old_state).last_wasm_entry_fp, @@ -1040,7 +1025,7 @@ impl CallThreadState { unsafe { let cx = self.vm_store_context.as_ref(); *cx.last_wasm_exit_pc.get() = pc; - *cx.last_wasm_exit_trap_fp.get() = fp; + *cx.last_wasm_exit_fp.get() = fp; *cx.last_wasm_exit_was_trap.get() = true; } let result = f(self); diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 99592736e46d..f34282c9b41b 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -175,7 +175,7 @@ impl Backtrace { // through the Wasm-to-host trampoline. None => unsafe { let pc = *(*vm_store_context).last_wasm_exit_pc.get(); - let fp = (*vm_store_context).last_wasm_exit_fp(); + let fp = *(*vm_store_context).last_wasm_exit_fp.get(); (pc, fp) }, }; @@ -364,7 +364,7 @@ impl<'a, T: 'static> CurrentActivationBacktrace<'a, T> { // Get the initial exit FP, exit PC, and entry FP. let vm_store_context = store.0.vm_store_context(); let exit_pc = unsafe { *(*vm_store_context).last_wasm_exit_pc.get() }; - let exit_fp = unsafe { (*vm_store_context).last_wasm_exit_fp() }; + let exit_fp = unsafe { *(*vm_store_context).last_wasm_exit_fp.get() }; let trampoline_fp = unsafe { *(*vm_store_context).last_wasm_entry_fp.get() }; log::trace!( "activation backtrace: exit_pc {exit_pc:x} exit_fp {exit_fp:x} entry_fp {trampoline_fp:x}" diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index df28bf060c31..2e11ae69bc60 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1110,8 +1110,8 @@ pub struct VMStoreContext { /// The `VMMemoryDefinition` for this store's GC heap. pub gc_heap: VMMemoryDefinition, - /// The value of the frame pointer register in the trampoline used - /// to call from Wasm to the host. + /// The value of the frame pointer register in the Wasm frame that + /// called from Wasm to the host. /// /// Maintained by our Wasm-to-host trampoline, and cleared just /// before calling into Wasm in `catch_traps`. @@ -1120,13 +1120,8 @@ pub struct VMStoreContext { /// to the host. /// /// Used to find the start of a contiguous sequence of Wasm frames - /// when walking the stack. Note that we record the FP of the - /// *trampoline*'s frame, not the last Wasm frame, because we need - /// to know the SP (bottom of frame) of the last Wasm frame as - /// well in case we need to resume to an exception handler in that - /// frame. The FP of the last Wasm frame can be recovered by - /// loading the saved FP value at this FP address. - pub last_wasm_exit_trampoline_fp: UnsafeCell, + /// when walking the stack. + pub last_wasm_exit_fp: UnsafeCell, /// The last Wasm program counter before we called from Wasm to the host. /// @@ -1200,70 +1195,6 @@ pub struct VMStoreContext { /// It is set *only* from host code, but is kept here alongside /// the other last-exit state for consistency. pub last_wasm_exit_was_trap: UnsafeCell, - - /// The last Wasm exit to host, if via trap, had this FP in guest - /// code (*not* in a trampoline). - /// - /// Either this field or `last_wasm_exit_trampoline_fp` is set, - /// never both. This field is always set from the host. - pub last_wasm_exit_trap_fp: UnsafeCell, -} - -impl VMStoreContext { - /// From the current saved trampoline FP, get the FP of the last - /// Wasm frame. If the current saved trampoline FP is null, return - /// null. - /// - /// We store only the trampoline FP, because (i) we need the - /// trampoline FP, so we know the size (bottom) of the last Wasm - /// frame; and (ii) the last Wasm frame, just above the trampoline - /// frame, can be recovered via the FP chain. - /// - /// # Safety - /// - /// This function requires that the `last_wasm_exit_trampoline_fp` - /// field either points to an active trampoline frame or is a null - /// pointer. - pub(crate) unsafe fn last_wasm_exit_fp(&self) -> usize { - // SAFETY: the unsafe cell is safe to load (no other threads - // will be writing our store when we have control), and the - // helper function's safety condition is the same as ours. - unsafe { - if *self.last_wasm_exit_was_trap.get() { - *self.last_wasm_exit_trap_fp.get() - } else { - let trampoline_fp = *self.last_wasm_exit_trampoline_fp.get(); - Self::wasm_exit_fp_from_trampoline_fp(trampoline_fp) - } - } - } - - /// From any saved trampoline FP, get the FP of the last Wasm - /// frame. If the given trampoline FP is null, return null. - /// - /// This differs from `last_wasm_exit_fp()` above in that it - /// allows accessing activations further up the stack as well, - /// e.g. via `CallThreadState::old_state`. - /// - /// # Safety - /// - /// This function requires that the provided FP value is valid, - /// and points to an active trampoline frame, or is null. - /// - /// This function depends on the invariant that on all supported - /// architectures, we store the previous FP value under the - /// current FP. This is a property of our ABI that we control and - /// ensure. - pub(crate) unsafe fn wasm_exit_fp_from_trampoline_fp(trampoline_fp: usize) -> usize { - if trampoline_fp != 0 { - // SAFETY: We require that trampoline_fp points to a valid - // frame, which will (by definition) contain an old FP value - // that we can load. - unsafe { *(trampoline_fp as *const usize) } - } else { - 0 - } - } } // The `VMStoreContext` type is a pod-type with no destructor, and we don't @@ -1286,7 +1217,7 @@ impl Default for VMStoreContext { base: NonNull::dangling().into(), current_length: AtomicUsize::new(0), }, - last_wasm_exit_trampoline_fp: UnsafeCell::new(0), + last_wasm_exit_fp: UnsafeCell::new(0), last_wasm_exit_pc: UnsafeCell::new(0), last_wasm_entry_fp: UnsafeCell::new(0), last_wasm_entry_sp: UnsafeCell::new(0), @@ -1295,7 +1226,6 @@ impl Default for VMStoreContext { async_guard_range: ptr::null_mut()..ptr::null_mut(), store_data: VmPtr::dangling(), last_wasm_exit_was_trap: UnsafeCell::new(false), - last_wasm_exit_trap_fp: UnsafeCell::new(0), } } } @@ -1335,8 +1265,8 @@ mod test_vmstore_context { usize::from(offsets.ptr.vmstore_context_gc_heap_current_length()) ); assert_eq!( - offset_of!(VMStoreContext, last_wasm_exit_trampoline_fp), - usize::from(offsets.ptr.vmstore_context_last_wasm_exit_trampoline_fp()) + offset_of!(VMStoreContext, last_wasm_exit_fp), + usize::from(offsets.ptr.vmstore_context_last_wasm_exit_fp()) ); assert_eq!( offset_of!(VMStoreContext, last_wasm_exit_pc), From f2f3e2b9268ca432f9c0bc657d00fc856849ac6a Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 23 Oct 2025 13:47:54 -0700 Subject: [PATCH 3/4] Update disas tests after trampoline change from trampoline exit FP to Wasm exit FP. --- tests/disas/debug-exceptions.wat | 6 ++-- tests/disas/epoch-interruption-x86.wat | 4 +-- tests/disas/exceptions.wat | 6 ++-- tests/disas/gc/struct-new-stack-map.wat | 2 +- tests/disas/pulley/epoch-simple.wat | 2 +- .../disas/riscv64-component-builtins-asm.wat | 2 +- tests/disas/riscv64-component-builtins.wat | 29 ++++++++++--------- tests/disas/trunc.wat | 18 ++++++------ tests/disas/trunc32.wat | 18 ++++++------ .../aarch64/call_indirect/call_indirect.wat | 4 +-- .../winch/aarch64/call_indirect/local_arg.wat | 2 +- .../disas/winch/x64/atomic/notify/notify.wat | 2 +- .../winch/x64/atomic/notify/notify_offset.wat | 2 +- tests/disas/winch/x64/atomic/wait/wait32.wat | 2 +- .../winch/x64/atomic/wait/wait32_offset.wat | 2 +- tests/disas/winch/x64/atomic/wait/wait64.wat | 2 +- .../winch/x64/atomic/wait/wait64_offset.wat | 2 +- .../winch/x64/call_indirect/call_indirect.wat | 4 +-- .../winch/x64/call_indirect/local_arg.wat | 2 +- tests/disas/winch/x64/epoch/func.wat | 2 +- tests/disas/winch/x64/epoch/loop.wat | 4 +-- .../winch/x64/f32_ceil/f32_ceil_param.wat | 2 +- .../winch/x64/f32_floor/f32_floor_param.wat | 2 +- .../x64/f32_nearest/f32_nearest_param.wat | 2 +- .../winch/x64/f32_trunc/f32_trunc_param.wat | 2 +- .../winch/x64/f64_ceil/f64_ceil_param.wat | 2 +- .../winch/x64/f64_floor/f64_floor_param.wat | 2 +- .../x64/f64_nearest/f64_nearest_param.wat | 2 +- .../winch/x64/f64_trunc/f64_trunc_param.wat | 2 +- tests/disas/winch/x64/fuel/call.wat | 4 +-- tests/disas/winch/x64/fuel/func.wat | 2 +- tests/disas/winch/x64/fuel/loop.wat | 4 +-- tests/disas/winch/x64/load/grow_load.wat | 2 +- tests/disas/winch/x64/table/fill.wat | 4 +-- tests/disas/winch/x64/table/get.wat | 2 +- tests/disas/winch/x64/table/grow.wat | 2 +- .../disas/winch/x64/table/init_copy_drop.wat | 20 ++++++------- tests/disas/winch/x64/table/set.wat | 2 +- 38 files changed, 88 insertions(+), 87 deletions(-) diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat index 540226aba234..bc94e4fc1af4 100644 --- a/tests/disas/debug-exceptions.wat +++ b/tests/disas/debug-exceptions.wat @@ -40,14 +40,14 @@ ;; ╰─╼ debug frame state (before next inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 64, slot at FP-0xb0, locals , stack ;; ldur x2, [sp, #0x10] ;; ╰─╼ debug frame state (before next inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 66, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; bl #0x31c +;; bl #0x328 ;; 60: mov x21, x2 ;; mov w3, #0x4000000 ;; mov w4, #2 ;; mov w5, #0x28 ;; mov w6, #8 ;; ldur x2, [sp, #0x10] -;; bl #0x2a8 +;; bl #0x2b0 ;; 7c: ldur x11, [sp, #0x10] ;; ldr x0, [x11, #8] ;; ldr x5, [x0, #0x18] @@ -62,7 +62,7 @@ ;; str w3, [x4, w2, uxtw] ;; mov x3, x2 ;; ldur x2, [sp, #0x10] -;; bl #0x354 +;; bl #0x364 ;; ├─╼ exception frame offset: SP = FP - 0xb0 ;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0xbc ;; b8: .byte 0x1f, 0xc1, 0x00, 0x00 diff --git a/tests/disas/epoch-interruption-x86.wat b/tests/disas/epoch-interruption-x86.wat index 043114a2b2cd..7ba7c7666c1a 100644 --- a/tests/disas/epoch-interruption-x86.wat +++ b/tests/disas/epoch-interruption-x86.wat @@ -28,12 +28,12 @@ ;; jae 0x64 ;; jmp 0x46 ;; 57: movq %r15, %rdi -;; callq 0xe1 +;; callq 0xe3 ;; jmp 0x46 ;; 64: movq 8(%r13), %rax ;; cmpq %rax, %r11 ;; jb 0x46 ;; 71: movq %r15, %rdi -;; callq 0xe1 +;; callq 0xe3 ;; jmp 0x46 ;; 7e: ud2 diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat index c125c8b60cdd..209547d854d7 100644 --- a/tests/disas/exceptions.wat +++ b/tests/disas/exceptions.wat @@ -33,14 +33,14 @@ ;; movq %rdi, %r12 ;; movq %rcx, %r13 ;; movq %rdx, %r15 -;; callq 0x3b2 +;; callq 0x3ba ;; movq %rax, %r14 ;; movl $0x4000000, %esi ;; movl $3, %edx ;; movl $0x30, %ecx ;; movl $8, %r8d ;; movq %r12, %rdi -;; callq 0x34f +;; callq 0x353 ;; movq 8(%r12), %r8 ;; movq 0x18(%r8), %r8 ;; movl %eax, %r9d @@ -54,7 +54,7 @@ ;; movq %rax, %rsi ;; movq %r12, %rdi ;; movq %r12, (%rsp) -;; callq 0x3de +;; callq 0x3e9 ;; ud2 ;; ud2 ;; diff --git a/tests/disas/gc/struct-new-stack-map.wat b/tests/disas/gc/struct-new-stack-map.wat index a58654548d21..8df655d16325 100644 --- a/tests/disas/gc/struct-new-stack-map.wat +++ b/tests/disas/gc/struct-new-stack-map.wat @@ -30,7 +30,7 @@ ;; movl $0x28, %ecx ;; movl $8, %r8d ;; movq %rdi, %r13 -;; callq 0x12f +;; callq 0x133 ;; movq 8(%r13), %rdx ;; ╰─╼ stack_map: frame_size=48, frame_offsets=[0] ;; movq 0x18(%rdx), %rdx diff --git a/tests/disas/pulley/epoch-simple.wat b/tests/disas/pulley/epoch-simple.wat index 947f92812732..6322437bb456 100644 --- a/tests/disas/pulley/epoch-simple.wat +++ b/tests/disas/pulley/epoch-simple.wat @@ -14,5 +14,5 @@ ;; br_if_xulteq64 x7, x6, 0x9 // target = 0x26 ;; 24: pop_frame ;; ret -;; 26: call 0x6f // target = 0x95 +;; 26: call 0x76 // target = 0x9c ;; 2b: jump -0x7 // target = 0x24 diff --git a/tests/disas/riscv64-component-builtins-asm.wat b/tests/disas/riscv64-component-builtins-asm.wat index dd6a465899e7..b67cc9ff5daa 100644 --- a/tests/disas/riscv64-component-builtins-asm.wat +++ b/tests/disas/riscv64-component-builtins-asm.wat @@ -21,7 +21,7 @@ ;; mv s1, a1 ;; mv a3, a2 ;; ld a4, 0x10(a0) -;; mv a5, s0 +;; ld a5, 0(s0) ;; sd a5, 0x28(a4) ;; ld a5, 8(s0) ;; sd a5, 0x30(a4) diff --git a/tests/disas/riscv64-component-builtins.wat b/tests/disas/riscv64-component-builtins.wat index 4a1102fed04a..407777f1c80f 100644 --- a/tests/disas/riscv64-component-builtins.wat +++ b/tests/disas/riscv64-component-builtins.wat @@ -17,25 +17,26 @@ ;; block0(v0: i64, v1: i64, v2: i32): ;; v3 = load.i64 notrap aligned v0+16 ;; v4 = get_frame_pointer.i64 -;; store notrap aligned v4, v3+40 -;; v5 = get_return_address.i64 -;; store notrap aligned v5, v3+48 -;; v8 = load.i64 notrap aligned readonly v0+8 -;; v9 = load.i64 notrap aligned readonly v8+16 -;; v6 = iconst.i32 0 -;; v10 = call_indirect sig0, v9(v0, v6, v6, v2) ; v6 = 0, v6 = 0 -;; v11 = iconst.i64 -1 -;; v12 = icmp ne v10, v11 ; v11 = -1 -;; brif v12, block2, block1 +;; v5 = load.i64 notrap aligned v4 +;; store notrap aligned v5, v3+40 +;; v6 = get_return_address.i64 +;; store notrap aligned v6, v3+48 +;; v9 = load.i64 notrap aligned readonly v0+8 +;; v10 = load.i64 notrap aligned readonly v9+16 +;; v7 = iconst.i32 0 +;; v11 = call_indirect sig0, v10(v0, v7, v7, v2) ; v7 = 0, v7 = 0 +;; v12 = iconst.i64 -1 +;; v13 = icmp ne v11, v12 ; v12 = -1 +;; brif v13, block2, block1 ;; ;; block1 cold: -;; v13 = load.i64 notrap aligned readonly v1+16 -;; v14 = load.i64 notrap aligned readonly v13+408 -;; call_indirect sig1, v14(v1) +;; v14 = load.i64 notrap aligned readonly v1+16 +;; v15 = load.i64 notrap aligned readonly v14+408 +;; call_indirect sig1, v15(v1) ;; trap user1 ;; ;; block2: -;; brif.i64 v10, block3, block4 +;; brif.i64 v11, block3, block4 ;; ;; block3: ;; jump block4 diff --git a/tests/disas/trunc.wat b/tests/disas/trunc.wat index e6acff281e3d..a97bec5e0ed6 100644 --- a/tests/disas/trunc.wat +++ b/tests/disas/trunc.wat @@ -24,7 +24,7 @@ ;; jne 0x101 ;; 39: movq %r14, %rdi ;; movdqu (%rsp), %xmm0 -;; callq 0x245 +;; callq 0x247 ;; movabsq $13830554455654793216, %rax ;; movq %rax, %xmm6 ;; ucomisd %xmm0, %xmm6 @@ -55,27 +55,27 @@ ;; retq ;; d3: movl $6, %esi ;; d8: movq %r14, %rdi -;; db: callq 0x271 +;; db: callq 0x276 ;; e0: movq %r14, %rdi -;; e3: callq 0x2a1 +;; e3: callq 0x2a9 ;; e8: ud2 ;; ea: movl $6, %esi ;; ef: movq %r14, %rdi -;; f2: callq 0x271 +;; f2: callq 0x276 ;; f7: movq %r14, %rdi -;; fa: callq 0x2a1 +;; fa: callq 0x2a9 ;; ff: ud2 ;; 101: movl $8, %esi ;; 106: movq %r14, %rdi -;; 109: callq 0x271 +;; 109: callq 0x276 ;; 10e: movq %r14, %rdi -;; 111: callq 0x2a1 +;; 111: callq 0x2a9 ;; 116: ud2 ;; 118: xorl %esi, %esi ;; 11a: movq %r14, %rdi -;; 11d: callq 0x271 +;; 11d: callq 0x276 ;; 122: movq %r14, %rdi -;; 125: callq 0x2a1 +;; 125: callq 0x2a9 ;; 12a: ud2 ;; 12c: ud2 ;; 12e: ud2 diff --git a/tests/disas/trunc32.wat b/tests/disas/trunc32.wat index 8e5e2e8a631b..2b719dfe183c 100644 --- a/tests/disas/trunc32.wat +++ b/tests/disas/trunc32.wat @@ -26,7 +26,7 @@ ;; jp 0xf6 ;; jne 0xf6 ;; 46: movq %r12, %rdi -;; callq 0x243 +;; callq 0x245 ;; movabsq $13830554455654793216, %r8 ;; movq %r8, %xmm1 ;; ucomisd %xmm0, %xmm1 @@ -56,27 +56,27 @@ ;; retq ;; c8: movl $6, %esi ;; cd: movq %r12, %rdi -;; d0: callq 0x26f +;; d0: callq 0x274 ;; d5: movq %r12, %rdi -;; d8: callq 0x29f +;; d8: callq 0x2a7 ;; dd: ud2 ;; df: movl $6, %esi ;; e4: movq %r12, %rdi -;; e7: callq 0x26f +;; e7: callq 0x274 ;; ec: movq %r12, %rdi -;; ef: callq 0x29f +;; ef: callq 0x2a7 ;; f4: ud2 ;; f6: movl $8, %esi ;; fb: movq %r12, %rdi -;; fe: callq 0x26f +;; fe: callq 0x274 ;; 103: movq %r12, %rdi -;; 106: callq 0x29f +;; 106: callq 0x2a7 ;; 10b: ud2 ;; 10d: xorl %esi, %esi ;; 10f: movq %r12, %rdi -;; 112: callq 0x26f +;; 112: callq 0x274 ;; 117: movq %r12, %rdi -;; 11a: callq 0x29f +;; 11a: callq 0x2a7 ;; 11f: ud2 ;; 121: ud2 ;; 123: ud2 diff --git a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat index 731043475f29..c5b0b27f7364 100644 --- a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat +++ b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat @@ -85,7 +85,7 @@ ;; mov x0, x9 ;; mov x1, #0 ;; ldur w2, [x28] -;; bl #0x3e4 +;; bl #0x3f0 ;; e0: add x28, x28, #4 ;; mov sp, x28 ;; ldur x9, [x28, #0x14] @@ -153,7 +153,7 @@ ;; mov x0, x9 ;; mov x1, #0 ;; ldur w2, [x28, #0xc] -;; bl #0x3e4 +;; bl #0x3f0 ;; 1f0: add x28, x28, #0xc ;; mov sp, x28 ;; add x28, x28, #4 diff --git a/tests/disas/winch/aarch64/call_indirect/local_arg.wat b/tests/disas/winch/aarch64/call_indirect/local_arg.wat index b91b748f9ff9..3d546753ca91 100644 --- a/tests/disas/winch/aarch64/call_indirect/local_arg.wat +++ b/tests/disas/winch/aarch64/call_indirect/local_arg.wat @@ -91,7 +91,7 @@ ;; mov x0, x9 ;; mov x1, #0 ;; ldur w2, [x28] -;; bl #0x404 +;; bl #0x3fc ;; 120: add x28, x28, #4 ;; mov sp, x28 ;; ldur x9, [x28, #0x14] diff --git a/tests/disas/winch/x64/atomic/notify/notify.wat b/tests/disas/winch/x64/atomic/notify/notify.wat index 4f8e4e0e3d87..ea9cd72baf24 100644 --- a/tests/disas/winch/x64/atomic/notify/notify.wat +++ b/tests/disas/winch/x64/atomic/notify/notify.wat @@ -27,7 +27,7 @@ ;; movl $0, %esi ;; movq 8(%rsp), %rdx ;; movl 4(%rsp), %ecx -;; callq 0x175 +;; callq 0x178 ;; addq $4, %rsp ;; addq $0xc, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/atomic/notify/notify_offset.wat b/tests/disas/winch/x64/atomic/notify/notify_offset.wat index 5f5fd6c8d477..118128338629 100644 --- a/tests/disas/winch/x64/atomic/notify/notify_offset.wat +++ b/tests/disas/winch/x64/atomic/notify/notify_offset.wat @@ -28,7 +28,7 @@ ;; movl $0, %esi ;; movq 8(%rsp), %rdx ;; movl 4(%rsp), %ecx -;; callq 0x17c +;; callq 0x17f ;; addq $4, %rsp ;; addq $0xc, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/atomic/wait/wait32.wat b/tests/disas/winch/x64/atomic/wait/wait32.wat index 2d018f2f7020..370c737bffad 100644 --- a/tests/disas/winch/x64/atomic/wait/wait32.wat +++ b/tests/disas/winch/x64/atomic/wait/wait32.wat @@ -30,7 +30,7 @@ ;; movq 0x18(%rsp), %rdx ;; movl 0x14(%rsp), %ecx ;; movq 0xc(%rsp), %r8 -;; callq 0x182 +;; callq 0x185 ;; addq $0xc, %rsp ;; addq $0x14, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat index bca69bb45ea4..df22377fdbf7 100644 --- a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat +++ b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat @@ -34,7 +34,7 @@ ;; movq 0x18(%rsp), %rdx ;; movl 0x14(%rsp), %ecx ;; movq 0xc(%rsp), %r8 -;; callq 0x189 +;; callq 0x18c ;; addq $0xc, %rsp ;; addq $0x14, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/atomic/wait/wait64.wat b/tests/disas/winch/x64/atomic/wait/wait64.wat index 92e9279f341d..81e3e669f958 100644 --- a/tests/disas/winch/x64/atomic/wait/wait64.wat +++ b/tests/disas/winch/x64/atomic/wait/wait64.wat @@ -29,7 +29,7 @@ ;; movq 0x18(%rsp), %rdx ;; movq 0x10(%rsp), %rcx ;; movq 8(%rsp), %r8 -;; callq 0x17a +;; callq 0x17d ;; addq $8, %rsp ;; addq $0x18, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat index 60278a7855fd..ccfc59b65eaf 100644 --- a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat +++ b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat @@ -33,7 +33,7 @@ ;; movq 0x18(%rsp), %rdx ;; movq 0x10(%rsp), %rcx ;; movq 8(%rsp), %r8 -;; callq 0x181 +;; callq 0x184 ;; addq $8, %rsp ;; addq $0x18, %rsp ;; movq 8(%rsp), %r14 diff --git a/tests/disas/winch/x64/call_indirect/call_indirect.wat b/tests/disas/winch/x64/call_indirect/call_indirect.wat index d6ac1776bdeb..7ec91a542ab5 100644 --- a/tests/disas/winch/x64/call_indirect/call_indirect.wat +++ b/tests/disas/winch/x64/call_indirect/call_indirect.wat @@ -76,7 +76,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 8(%rsp), %edx -;; callq 0x337 +;; callq 0x33e ;; addq $8, %rsp ;; addq $4, %rsp ;; movq 0x1c(%rsp), %r14 @@ -128,7 +128,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 4(%rsp), %edx -;; callq 0x337 +;; callq 0x33e ;; addq $4, %rsp ;; addq $4, %rsp ;; movq 0x20(%rsp), %r14 diff --git a/tests/disas/winch/x64/call_indirect/local_arg.wat b/tests/disas/winch/x64/call_indirect/local_arg.wat index a28a35c29d74..9925e43dd644 100644 --- a/tests/disas/winch/x64/call_indirect/local_arg.wat +++ b/tests/disas/winch/x64/call_indirect/local_arg.wat @@ -72,7 +72,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 8(%rsp), %edx -;; callq 0x32b +;; callq 0x327 ;; addq $8, %rsp ;; addq $4, %rsp ;; movq 0x1c(%rsp), %r14 diff --git a/tests/disas/winch/x64/epoch/func.wat b/tests/disas/winch/x64/epoch/func.wat index 87a203e4b34a..8d9197338527 100644 --- a/tests/disas/winch/x64/epoch/func.wat +++ b/tests/disas/winch/x64/epoch/func.wat @@ -23,7 +23,7 @@ ;; cmpq %rcx, %rdx ;; jb 0x54 ;; 47: movq %r14, %rdi -;; callq 0x13b +;; callq 0x13d ;; movq 8(%rsp), %r14 ;; addq $0x10, %rsp ;; popq %rbp diff --git a/tests/disas/winch/x64/epoch/loop.wat b/tests/disas/winch/x64/epoch/loop.wat index 77ebfad29d5d..ee9c1baf770c 100644 --- a/tests/disas/winch/x64/epoch/loop.wat +++ b/tests/disas/winch/x64/epoch/loop.wat @@ -25,7 +25,7 @@ ;; cmpq %rcx, %rdx ;; jb 0x54 ;; 47: movq %r14, %rdi -;; callq 0x165 +;; callq 0x167 ;; movq 8(%rsp), %r14 ;; movq 0x18(%r14), %rdx ;; movq (%rdx), %rdx @@ -34,7 +34,7 @@ ;; cmpq %rcx, %rdx ;; jb 0x79 ;; 6c: movq %r14, %rdi -;; callq 0x165 +;; callq 0x167 ;; movq 8(%rsp), %r14 ;; jmp 0x54 ;; 7e: addq $0x10, %rsp diff --git a/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat b/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat index 69d565b6d88a..a1408fee0777 100644 --- a/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat +++ b/tests/disas/winch/x64/f32_ceil/f32_ceil_param.wat @@ -26,7 +26,7 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movss 0xc(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f32_floor/f32_floor_param.wat b/tests/disas/winch/x64/f32_floor/f32_floor_param.wat index f9db481766f2..8f229d8dd115 100644 --- a/tests/disas/winch/x64/f32_floor/f32_floor_param.wat +++ b/tests/disas/winch/x64/f32_floor/f32_floor_param.wat @@ -26,7 +26,7 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movss 0xc(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat b/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat index adc7fa0af72c..531ae020f6e5 100644 --- a/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat +++ b/tests/disas/winch/x64/f32_nearest/f32_nearest_param.wat @@ -26,7 +26,7 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movss 0xc(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat b/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat index 9aebb3411a50..bf2b1a63a857 100644 --- a/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat +++ b/tests/disas/winch/x64/f32_trunc/f32_trunc_param.wat @@ -26,7 +26,7 @@ ;; subq $0xc, %rsp ;; movq %r14, %rdi ;; movss 0xc(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat b/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat index cdc1b87b18ec..79aeb031e0f5 100644 --- a/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat +++ b/tests/disas/winch/x64/f64_ceil/f64_ceil_param.wat @@ -26,7 +26,7 @@ ;; subq $8, %rsp ;; movq %r14, %rdi ;; movsd 8(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f64_floor/f64_floor_param.wat b/tests/disas/winch/x64/f64_floor/f64_floor_param.wat index 2b4debf2fd5d..5465998a203c 100644 --- a/tests/disas/winch/x64/f64_floor/f64_floor_param.wat +++ b/tests/disas/winch/x64/f64_floor/f64_floor_param.wat @@ -26,7 +26,7 @@ ;; subq $8, %rsp ;; movq %r14, %rdi ;; movsd 8(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat b/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat index e78d9a01fe34..ca3cfe650e33 100644 --- a/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat +++ b/tests/disas/winch/x64/f64_nearest/f64_nearest_param.wat @@ -26,7 +26,7 @@ ;; subq $8, %rsp ;; movq %r14, %rdi ;; movsd 8(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat b/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat index 138fff26475a..093fc46dbe56 100644 --- a/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat +++ b/tests/disas/winch/x64/f64_trunc/f64_trunc_param.wat @@ -26,7 +26,7 @@ ;; subq $8, %rsp ;; movq %r14, %rdi ;; movsd 8(%rsp), %xmm0 -;; callq 0xdc +;; callq 0xde ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/fuel/call.wat b/tests/disas/winch/x64/fuel/call.wat index efd069f7115c..5f554bb0dec1 100644 --- a/tests/disas/winch/x64/fuel/call.wat +++ b/tests/disas/winch/x64/fuel/call.wat @@ -28,7 +28,7 @@ ;; cmpq $0, %rcx ;; jl 0x5e ;; 51: movq %r14, %rdi -;; callq 0x1f5 +;; callq 0x1f7 ;; movq 8(%rsp), %r14 ;; movq 8(%r14), %rax ;; movq (%rax), %r11 @@ -74,7 +74,7 @@ ;; cmpq $0, %rcx ;; jl 0x10e ;; 101: movq %r14, %rdi -;; callq 0x1f5 +;; callq 0x1f7 ;; movq 8(%rsp), %r14 ;; addq $0x10, %rsp ;; popq %rbp diff --git a/tests/disas/winch/x64/fuel/func.wat b/tests/disas/winch/x64/fuel/func.wat index 14ab5b247150..bef9b6adf2c0 100644 --- a/tests/disas/winch/x64/fuel/func.wat +++ b/tests/disas/winch/x64/fuel/func.wat @@ -24,7 +24,7 @@ ;; cmpq $0, %rcx ;; jl 0x5e ;; 51: movq %r14, %rdi -;; callq 0x145 +;; callq 0x147 ;; movq 8(%rsp), %r14 ;; addq $0x10, %rsp ;; popq %rbp diff --git a/tests/disas/winch/x64/fuel/loop.wat b/tests/disas/winch/x64/fuel/loop.wat index 3487f1061f1a..3c29e8815dd4 100644 --- a/tests/disas/winch/x64/fuel/loop.wat +++ b/tests/disas/winch/x64/fuel/loop.wat @@ -26,14 +26,14 @@ ;; cmpq $0, %rcx ;; jl 0x5e ;; 51: movq %r14, %rdi -;; callq 0x179 +;; callq 0x17b ;; movq 8(%rsp), %r14 ;; movq 8(%r14), %rcx ;; movq (%rcx), %rcx ;; cmpq $0, %rcx ;; jl 0x7c ;; 6f: movq %r14, %rdi -;; callq 0x179 +;; callq 0x17b ;; movq 8(%rsp), %r14 ;; movq 8(%r14), %rax ;; movq (%rax), %r11 diff --git a/tests/disas/winch/x64/load/grow_load.wat b/tests/disas/winch/x64/load/grow_load.wat index f3ed3d2d7836..206968c2d1f1 100644 --- a/tests/disas/winch/x64/load/grow_load.wat +++ b/tests/disas/winch/x64/load/grow_load.wat @@ -65,7 +65,7 @@ ;; movq %r14, %rdi ;; movl 0xc(%rsp), %esi ;; movl $0, %edx -;; callq 0x2e2 +;; callq 0x2e5 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x58(%rsp), %r14 diff --git a/tests/disas/winch/x64/table/fill.wat b/tests/disas/winch/x64/table/fill.wat index a58d41451a60..bd663b63e6ea 100644 --- a/tests/disas/winch/x64/table/fill.wat +++ b/tests/disas/winch/x64/table/fill.wat @@ -113,7 +113,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 0xc(%rsp), %edx -;; callq 0x4ee +;; callq 0x4f7 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x28(%rsp), %r14 @@ -133,7 +133,7 @@ ;; movl 0xc(%rsp), %edx ;; movq 4(%rsp), %rcx ;; movl (%rsp), %r8d -;; callq 0x51a +;; callq 0x526 ;; addq $0x10, %rsp ;; movq 0x28(%rsp), %r14 ;; addq $0x30, %rsp diff --git a/tests/disas/winch/x64/table/get.wat b/tests/disas/winch/x64/table/get.wat index 03febe2ed81d..f47517eb7999 100644 --- a/tests/disas/winch/x64/table/get.wat +++ b/tests/disas/winch/x64/table/get.wat @@ -65,7 +65,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 0xc(%rsp), %edx -;; callq 0x2ef +;; callq 0x2f8 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/table/grow.wat b/tests/disas/winch/x64/table/grow.wat index a6dfa4e1a530..86be26a28c6a 100644 --- a/tests/disas/winch/x64/table/grow.wat +++ b/tests/disas/winch/x64/table/grow.wat @@ -30,7 +30,7 @@ ;; movl $0, %esi ;; movl $0xa, %edx ;; movq 8(%rsp), %rcx -;; callq 0x178 +;; callq 0x180 ;; addq $8, %rsp ;; addq $8, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat index dab3c0e6ac37..f4fed0127c88 100644 --- a/tests/disas/winch/x64/table/init_copy_drop.wat +++ b/tests/disas/winch/x64/table/init_copy_drop.wat @@ -142,11 +142,11 @@ ;; movl $7, %ecx ;; movl $0, %r8d ;; movl $4, %r9d -;; callq 0x94a +;; callq 0x95a ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $1, %esi -;; callq 0x995 +;; callq 0x9a9 ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -154,11 +154,11 @@ ;; movl $0xf, %ecx ;; movl $1, %r8d ;; movl $3, %r9d -;; callq 0x94a +;; callq 0x95a ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $3, %esi -;; callq 0x995 +;; callq 0x9a9 ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -166,7 +166,7 @@ ;; movl $0x14, %ecx ;; movl $0xf, %r8d ;; movl $5, %r9d -;; callq 0x8ff +;; callq 0x90b ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -174,7 +174,7 @@ ;; movl $0x15, %ecx ;; movl $0x1d, %r8d ;; movl $1, %r9d -;; callq 0x8ff +;; callq 0x90b ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -182,7 +182,7 @@ ;; movl $0x18, %ecx ;; movl $0xa, %r8d ;; movl $1, %r9d -;; callq 0x8ff +;; callq 0x90b ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -190,7 +190,7 @@ ;; movl $0xd, %ecx ;; movl $0xb, %r8d ;; movl $4, %r9d -;; callq 0x8ff +;; callq 0x90b ;; movq 8(%rsp), %r14 ;; movq %r14, %rdi ;; movl $0, %esi @@ -198,7 +198,7 @@ ;; movl $0x13, %ecx ;; movl $0x14, %r8d ;; movl $5, %r9d -;; callq 0x8ff +;; callq 0x90b ;; movq 8(%rsp), %r14 ;; addq $0x10, %rsp ;; popq %rbp @@ -243,7 +243,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 0xc(%rsp), %edx -;; callq 0x9c0 +;; callq 0x9d7 ;; addq $0xc, %rsp ;; addq $4, %rsp ;; movq 0x18(%rsp), %r14 diff --git a/tests/disas/winch/x64/table/set.wat b/tests/disas/winch/x64/table/set.wat index 2ae1255aee14..9252b525f030 100644 --- a/tests/disas/winch/x64/table/set.wat +++ b/tests/disas/winch/x64/table/set.wat @@ -109,7 +109,7 @@ ;; movq %r14, %rdi ;; movl $0, %esi ;; movl 8(%rsp), %edx -;; callq 0x4ba +;; callq 0x4c0 ;; addq $8, %rsp ;; addq $4, %rsp ;; movq 0x1c(%rsp), %r14 From 05fca4bd5e1e8ad692e6f8536971dc622a5bbf49 Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 23 Oct 2025 17:47:24 -0700 Subject: [PATCH 4/4] Debug: implement call injection to invoke debug event handlers at signal-based traps. This repurposes the code from #11826 to "inject calls": when in a signal handler, we can update the register state to redirect execution upon signal-handler return to a special hand-written trampoline, and this trampoline can save all registers and enter the host, just as if a hostcall had occurred. --- crates/wasmtime/src/runtime/store.rs | 5 + crates/wasmtime/src/runtime/vm.rs | 7 + crates/wasmtime/src/runtime/vm/interpreter.rs | 29 ++- .../src/runtime/vm/sys/unix/signals.rs | 124 +++++++-- .../wasmtime/src/runtime/vm/traphandlers.rs | 155 +++++++----- .../runtime/vm/traphandlers/inject_call.rs | 237 ++++++++++++++++++ .../vm/traphandlers/inject_call/aarch64.rs | 98 ++++++++ .../vm/traphandlers/inject_call/riscv64.rs | 233 +++++++++++++++++ .../vm/traphandlers/inject_call/s390x.rs | 184 ++++++++++++++ .../vm/traphandlers/inject_call/x86_64.rs | 127 ++++++++++ crates/wasmtime/src/runtime/vm/vmcontext.rs | 90 +++++++ pulley/src/interp.rs | 5 + tests/all/debug.rs | 109 +++++--- 13 files changed, 1281 insertions(+), 122 deletions(-) create mode 100644 crates/wasmtime/src/runtime/vm/traphandlers/inject_call.rs create mode 100644 crates/wasmtime/src/runtime/vm/traphandlers/inject_call/aarch64.rs create mode 100644 crates/wasmtime/src/runtime/vm/traphandlers/inject_call/riscv64.rs create mode 100644 crates/wasmtime/src/runtime/vm/traphandlers/inject_call/s390x.rs create mode 100644 crates/wasmtime/src/runtime/vm/traphandlers/inject_call/x86_64.rs diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index b38e4f806d42..b5f00aa93400 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -2764,6 +2764,11 @@ unsafe impl VMStore for StoreInner { Ok(()) } } + + #[cfg(feature = "debug")] + fn has_debug_handler(&self) -> bool { + self.debug_handler.is_some() + } } impl StoreInner { diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 247cea7ac1d0..a37011961b6c 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -232,6 +232,13 @@ pub unsafe trait VMStore: 'static { /// Invoke a debug handler, if present, at a debug event. #[cfg(feature = "debug")] fn block_on_debug_handler(&mut self, event: crate::DebugEvent) -> anyhow::Result<()>; + + /// Is a debug handler registered? This allows for the hook call + /// above to be conditionalized, saving a little effort + /// constructing the event and/or doing other work (such as + /// injecting hostcalls on traps) if not. + #[cfg(feature = "debug")] + fn has_debug_handler(&self) -> bool; } impl Deref for dyn VMStore + '_ { diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index f2e11218f847..add21200bc4b 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -2,6 +2,7 @@ use crate::runtime::vm::vmcontext::VMArrayCallNative; use crate::runtime::vm::{ StoreBox, TrapRegisters, TrapTest, VMContext, VMOpaqueContext, f32x4, f64x2, i8x16, tls, }; +use crate::vm::TrapResumeArgs; use crate::{Engine, ValRaw}; use core::marker; use core::ptr::NonNull; @@ -457,8 +458,13 @@ impl InterpreterRef<'_> { let regs = TrapRegisters { pc: pc.as_ptr() as usize, fp: self.vm().fp() as usize, + sp: self.vm().sp() as usize, + // N.B.: We do not do call injection with Pulley traps, so + // we don't need these values. + arg0: 0, + arg1: 0, }; - let handler = tls::with(|s| { + let (handler, args) = tls::with(|s| { let s = s.unwrap(); match kind { Some(kind) => { @@ -472,8 +478,15 @@ impl InterpreterRef<'_> { }; s.set_jit_trap(regs, None, trap); log::trace!("about to invoke debug event from interpreter"); - s.debug_event_from_interpreter(); - s.entry_trap_handler() + + // SAFETY: this is invoked only within a trapping context when + // we have received control back from the Wasm code. See the + // provenance diagram and comments on `self.raw_store` for + // more details. + let store = unsafe { s.vm_store_context.as_ref().raw_store_mut() }; + s.invoke_debug_event(store); + + (s.entry_trap_handler(), TrapResumeArgs::Handler(0, 0)) } None => { match s.test_if_trap(regs, None, |_| false) { @@ -489,13 +502,19 @@ impl InterpreterRef<'_> { // Trap was handled, yay! Configure interpreter state // to resume at the exception handler. - TrapTest::Trap(handler) => handler, + TrapTest::Trap(handler, args) => (handler, args), } } } }); unsafe { - self.resume_to_exception_handler(&handler, 0, 0); + let (arg1, arg2) = match args { + TrapResumeArgs::Handler(arg1, arg2) => (arg1, arg2), + TrapResumeArgs::Call(..) => { + panic!("Cannot set call registers in Pulley resume path") + } + }; + self.resume_to_exception_handler(&handler, arg1, arg2); } self.take_resume_at_pc() } diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/signals.rs b/crates/wasmtime/src/runtime/vm/sys/unix/signals.rs index be7fb6c89976..81200b8bb2dc 100644 --- a/crates/wasmtime/src/runtime/vm/sys/unix/signals.rs +++ b/crates/wasmtime/src/runtime/vm/sys/unix/signals.rs @@ -1,7 +1,7 @@ //! Trap handling on Unix based on POSIX signals. use crate::prelude::*; -use crate::runtime::vm::traphandlers::{TrapRegisters, TrapTest, tls}; +use crate::runtime::vm::traphandlers::{TrapRegisters, TrapResumeArgs, TrapTest, tls}; use std::cell::RefCell; use std::io; use std::mem; @@ -179,9 +179,9 @@ unsafe extern "C" fn trap_handler( false } TrapTest::HandledByEmbedder => true, - TrapTest::Trap(handler) => { + TrapTest::Trap(handler, args) => { unsafe { - store_handler_in_ucontext(context, &handler); + store_handler_in_ucontext(context, &handler, &args); } true } @@ -244,18 +244,32 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra TrapRegisters { pc: cx.uc_mcontext.gregs[libc::REG_RIP as usize] as usize, fp: cx.uc_mcontext.gregs[libc::REG_RBP as usize] as usize, + sp: cx.uc_mcontext.gregs[libc::REG_RSP as usize] as usize, + arg0: cx.uc_mcontext.gregs[libc::REG_RDI as usize] as usize, + arg1: cx.uc_mcontext.gregs[libc::REG_RSI as usize] as usize, } } else if #[cfg(all(target_os = "linux", target_arch = "x86"))] { let cx = unsafe { &*(cx as *const libc::ucontext_t) }; TrapRegisters { pc: cx.uc_mcontext.gregs[libc::REG_EIP as usize] as usize, fp: cx.uc_mcontext.gregs[libc::REG_EBP as usize] as usize, + sp: cx.uc_mcontext.gregs[libc::REG_ESP as usize] as usize, + // We don't do trap call injection on x86-32, so we + // don't need these values. If we did, we'd have to + // invent a register calling convention to save off + // reg values, pass them into our custom stub, and + // restore them later. + arg0: 0, + arg1: 0, } } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] { let cx = unsafe { &*(cx as *const libc::ucontext_t) }; TrapRegisters { pc: cx.uc_mcontext.pc as usize, fp: cx.uc_mcontext.regs[29] as usize, + sp: cx.uc_mcontext.sp as usize, + arg0: cx.uc_mcontext.regs[0] as usize, + arg1: cx.uc_mcontext.regs[1] as usize, } } else if #[cfg(all(target_os = "linux", target_arch = "s390x"))] { // On s390x, SIGILL and SIGFPE are delivered with the PSW address @@ -277,6 +291,9 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra TrapRegisters { pc: (cx.uc_mcontext.psw.addr - trap_offset) as usize, fp: *(cx.uc_mcontext.gregs[15] as *const usize), + sp: cx.uc_mcontext.gregs[15] as usize, + arg0: cx.uc_mcontext.gregs[2] as usize, + arg1: cx.uc_mcontext.gregs[3] as usize, } } } else if #[cfg(all(target_vendor = "apple", target_arch = "x86_64"))] { @@ -285,6 +302,9 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra TrapRegisters { pc: (*cx.uc_mcontext).__ss.__rip as usize, fp: (*cx.uc_mcontext).__ss.__rbp as usize, + sp: (*cx.uc_mcontext).__ss.__rsp as usize, + arg0: (*cx.uc_mcontext).__ss.__rdi as usize, + arg1: (*cx.uc_mcontext).__ss.__rsi as usize, } } } else if #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] { @@ -293,6 +313,9 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra TrapRegisters { pc: (*cx.uc_mcontext).__ss.__pc as usize, fp: (*cx.uc_mcontext).__ss.__fp as usize, + sp: (*cx.uc_mcontext).__ss.__sp as usize, + arg0: (*cx.uc_mcontext).__ss.__x[0] as usize, + arg1: (*cx.uc_mcontext).__ss.__x[1] as usize, } } } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] { @@ -300,30 +323,47 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra TrapRegisters { pc: cx.uc_mcontext.mc_rip as usize, fp: cx.uc_mcontext.mc_rbp as usize, + sp: cx.uc_mcontext.mc_rsp as usize, + arg0: cx.uc_mcontext.mc_rdi as usize, + arg1: cx.uc_mcontext.mc_rsi as usize, } } else if #[cfg(all(target_os = "linux", target_arch = "riscv64"))] { let cx = unsafe { &*(cx as *const libc::ucontext_t) }; TrapRegisters { pc: cx.uc_mcontext.__gregs[libc::REG_PC] as usize, fp: cx.uc_mcontext.__gregs[libc::REG_S0] as usize, + sp: cx.uc_mcontext.__gregs[libc::REG_SP] as usize, + arg0: cx.uc_mcontext.__gregs[libc::REG_A0 + 0] as usize, + arg1: cx.uc_mcontext.__gregs[libc::REG_A0 + 1] as usize, } } else if #[cfg(all(target_os = "freebsd", target_arch = "aarch64"))] { let cx = unsafe { &*(cx as *const libc::mcontext_t) }; TrapRegisters { pc: cx.mc_gpregs.gp_elr as usize, fp: cx.mc_gpregs.gp_x[29] as usize, + sp: cx.mc_gpregs.gp_sp as usize, + arg0: cx.mc_gpregs.gp_x[0] as usize, + arg1: cx.mc_gpregs.gp_x[1] as usize, } } else if #[cfg(all(target_os = "openbsd", target_arch = "x86_64"))] { let cx = unsafe { &*(cx as *const libc::ucontext_t) }; TrapRegisters { pc: cx.sc_rip as usize, fp: cx.sc_rbp as usize, + sp: cx.sc_rsp as usize, + arg0: cx.sc_rdi as usize, + arg1: cx.sc_rsi as usize, } } else if #[cfg(all(target_os = "linux", target_arch = "arm"))] { let cx = unsafe { &*(cx as *const libc::ucontext_t) }; TrapRegisters { pc: cx.uc_mcontext.arm_pc as usize, fp: cx.uc_mcontext.arm_fp as usize, + sp: cx.uc_mcontext.arm_sp as usize, + // We don't do call injection on arm32, so we don't + // need these values. + arg0: 0, + arg1: 0, } } else { compile_error!("unsupported platform"); @@ -334,28 +374,52 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra /// Updates the siginfo context stored in `cx` to resume to `handler` up on /// resumption while returning from the signal handler. -unsafe fn store_handler_in_ucontext(cx: *mut libc::c_void, handler: &Handler) { +unsafe fn store_handler_in_ucontext( + cx: *mut libc::c_void, + handler: &Handler, + args: &TrapResumeArgs, +) { cfg_if::cfg_if! { if #[cfg(all(any(target_os = "linux", target_os = "android", target_os = "illumos"), target_arch = "x86_64"))] { let cx = unsafe { cx.cast::().as_mut().unwrap() }; cx.uc_mcontext.gregs[libc::REG_RIP as usize] = handler.pc as _; cx.uc_mcontext.gregs[libc::REG_RSP as usize] = handler.sp as _; cx.uc_mcontext.gregs[libc::REG_RBP as usize] = handler.fp as _; - cx.uc_mcontext.gregs[libc::REG_RAX as usize] = 0; - cx.uc_mcontext.gregs[libc::REG_RDX as usize] = 0; + match *args { + TrapResumeArgs::Handler(a, b) => { + cx.uc_mcontext.gregs[libc::REG_RAX as usize] = a as _; + cx.uc_mcontext.gregs[libc::REG_RDX as usize] = b as _; + } + TrapResumeArgs::Call(a, b) => { + cx.uc_mcontext.gregs[libc::REG_RDI as usize] = a as _; + cx.uc_mcontext.gregs[libc::REG_RSI as usize] = b as _; + } + } } else if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "aarch64"))] { let cx = unsafe { cx.cast::().as_mut().unwrap() }; cx.uc_mcontext.pc = handler.pc as _; cx.uc_mcontext.sp = handler.sp as _; cx.uc_mcontext.regs[29] = handler.fp as _; - cx.uc_mcontext.regs[0] = 0; - cx.uc_mcontext.regs[1] = 0; + match *args { + TrapResumeArgs::Handler(a, b) | TrapResumeArgs::Call(a, b) => { + cx.uc_mcontext.regs[0] = a as _; + cx.uc_mcontext.regs[1] = b as _; + } + } } else if #[cfg(all(target_os = "linux", target_arch = "s390x"))] { let cx = unsafe { cx.cast::().as_mut().unwrap() }; cx.uc_mcontext.psw.addr = handler.pc as _; cx.uc_mcontext.gregs[15] = handler.sp as _; - cx.uc_mcontext.gregs[6] = 0; - cx.uc_mcontext.gregs[7] = 0; + match *args { + TrapResumeArgs::Handler(a, b) => { + cx.uc_mcontext.gregs[6] = a as _; + cx.uc_mcontext.gregs[7] = b as _; + } + TrapResumeArgs::Call(a, b) => { + cx.uc_mcontext.gregs[2] = a as _; + cx.uc_mcontext.gregs[3] = b as _; + } + } } else if #[cfg(all(target_vendor = "apple", target_arch = "x86_64"))] { unsafe { let cx = cx.cast::().as_mut().unwrap(); @@ -363,8 +427,16 @@ unsafe fn store_handler_in_ucontext(cx: *mut libc::c_void, handler: &Handler) { cx.__ss.__rip = handler.pc as _; cx.__ss.__rsp = handler.sp as _; cx.__ss.__rbp = handler.fp as _; - cx.__ss.__rax = 0; - cx.__ss.__rdx = 0; + match *args { + TrapResumeArgs::Handler(a, b) => { + cx.__ss.__rax = a as _; + cx.__ss.__rdx = b as _; + } + TrapResumeArgs::Call(a, b) => { + cx.__ss.__rdi = a as _; + cx.__ss.__rsi = b as _; + } + } } } else if #[cfg(all(target_vendor = "apple", target_arch = "aarch64"))] { unsafe { @@ -373,23 +445,39 @@ unsafe fn store_handler_in_ucontext(cx: *mut libc::c_void, handler: &Handler) { cx.__ss.__pc = handler.pc as _; cx.__ss.__sp = handler.sp as _; cx.__ss.__fp = handler.fp as _; - cx.__ss.__x[0] = 0; - cx.__ss.__x[1] = 0; + match *args { + TrapResumeArgs::Handler(a, b) | TrapResumeArgs::Call(a, b) => { + cx.__ss.__x[0] = a as _; + cx.__ss.__x[1] = b as _; + } + } } } else if #[cfg(all(target_os = "freebsd", target_arch = "x86_64"))] { let cx = unsafe { cx.cast::().as_mut().unwrap() }; cx.uc_mcontext.mc_rip = handler.pc as _; cx.uc_mcontext.mc_rbp = handler.fp as _; cx.uc_mcontext.mc_rsp = handler.sp as _; - cx.uc_mcontext.mc_rax = 0; - cx.uc_mcontext.mc_rdx = 0; + match *args { + TrapResumeArgs::Handler(a, b) => { + cx.uc_mcontext.mc_rax = a as _; + cx.uc_mcontext.mc_rdx = b as _; + } + TrapResumeArgs::Call(a, b) => { + cx.uc_mcontext.mc_rdi = a as _; + cx.uc_mcontext.mc_rsi = b as _; + } + } } else if #[cfg(all(target_os = "linux", target_arch = "riscv64"))] { let cx = unsafe { cx.cast::().as_mut().unwrap() }; cx.uc_mcontext.__gregs[libc::REG_PC] = handler.pc as _; cx.uc_mcontext.__gregs[libc::REG_S0] = handler.fp as _; cx.uc_mcontext.__gregs[libc::REG_SP] = handler.sp as _; - cx.uc_mcontext.__gregs[libc::REG_A0] = 0; - cx.uc_mcontext.__gregs[libc::REG_A0 + 1] = 0; + match *args { + TrapResumeArgs::Handler(a, b) | TrapResumeArgs::Call(a, b) => { + cx.uc_mcontext.__gregs[libc::REG_A0] = a as _; + cx.uc_mcontext.__gregs[libc::REG_A0 + 1] = b as _; + } + } } else { compile_error!("unsupported platform"); panic!(); diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 774c989420a2..789b657b013d 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -15,6 +15,11 @@ mod signals; #[cfg(all(has_native_signals))] pub use self::signals::*; +#[cfg(all(has_native_signals))] +mod inject_call; +#[cfg(all(has_native_signals))] +pub use self::inject_call::*; + #[cfg(feature = "gc")] use crate::ThrownException; use crate::runtime::module::lookup_code; @@ -43,9 +48,29 @@ pub use self::tls::{AsyncWasmCallState, PreviousAsyncWasmCallState}; pub use traphandlers::SignalHandler; +#[allow( + dead_code, + reason = "some registers not used when debug trap handling is disabled" +)] +#[derive(Clone, Copy, Debug)] pub(crate) struct TrapRegisters { pub pc: usize, pub fp: usize, + pub sp: usize, + pub arg0: usize, + pub arg1: usize, +} + +#[allow( + dead_code, + reason = "some registers not used when debug trap handling is disabled" +)] +#[derive(Clone, Copy, Debug)] +pub(crate) enum TrapResumeArgs { + /// Exception ABI argument values. + Handler(usize, usize), + /// Ordinary call ABI argument values. + Call(usize, usize), } /// Return value from `test_if_trap`. @@ -57,7 +82,7 @@ pub(crate) enum TrapTest { #[cfg_attr(miri, expect(dead_code, reason = "using #[cfg] too unergonomic"))] HandledByEmbedder, /// This is a wasm trap, it needs to be handled. - Trap(Handler), + Trap(Handler, TrapResumeArgs), } fn lazy_per_thread_init() { @@ -430,12 +455,26 @@ where { let caller = store.0.default_caller(); + // Set the raw store pointer (`VMStoreContext::raw_store`) + // here, just before entry, so that it has correct + // provenance. See comment on that field for more detials. + #[cfg(feature = "debug")] + unsafe { + *store.0.vm_store_context_mut().raw_store.get() = + Some(NonNull::from(store.0 as &mut dyn VMStore)); + } + let result = CallThreadState::new(store.0, old_state).with(|_cx| match store.0.executor() { ExecutorRef::Interpreter(r) => closure(caller, Some(r)), #[cfg(has_host_compiler_backend)] ExecutorRef::Native => closure(caller, None), }); + #[cfg(feature = "debug")] + unsafe { + *store.0.vm_store_context_mut().raw_store.get() = None; + } + match result { Ok(x) => Ok(x), #[cfg(feature = "gc")] @@ -550,56 +589,6 @@ mod call_thread_state { pub(crate) vm_store_context: NonNull, pub(crate) unwinder: &'static dyn Unwind, - /// Raw pointer to the `*mut dyn VMStore` running in this - /// activation, to be used *only* when re-entering the host - /// during a trap. - /// - /// This is a very tricky ownership/provenance dance. When - /// control is in the Wasm code itself, the store is - /// completely owned by the Wasm. It passes ownership back - /// during hostcalls via mutable reborrow (as with any call - /// with a `&mut self` in Rust). That's all well and good for - /// explicit calls. - /// - /// When a trap occurs, however, we can also think of the - /// ownership passing as-if the trapping instruction were a - /// hostcall with a `&mut dyn VMStore` parameter. This works - /// *as long as* all possibly trapping points in compiled code - /// act as if they invalidate any other held borrows into the - /// store. - /// - /// It turns out that we generally enforce this in compiled - /// guest code in Cranelift: any `can_trap` opcode returns - /// `true` from `has_memory_fence_semantics()` (see - /// corresponding comment there). This is enough to ensure - /// that the compiler treats every trapping op as-if it were a - /// hostcall, which clobbers all memory state; so from the - /// Wasm code's point of view, it is safely reborrowing the - /// Store and passing it "somewhere" on every trap. The - /// plumbing for that "passing" goes through this field, but - /// that is an implementation detail. When control comes back - /// out of the Wasm activation, we clear this field; the - /// invocation itself takes a mutable borrow of the store, so - /// safety is preserved on the caller side as well. In other - /// words, the provenance is something like - /// - /// ```plain - /// - /// host (caller side) with `&mut dyn VMStore` - /// / \ - /// (param into / (this field) - /// entry trampoline) \ - /// | | - /// ~~~~~~~ (wasm code) ~~~~~~ - /// | | - /// libcall trap - /// ``` - /// - /// with only *one* of those paths dynamically taken at any - /// given time. - #[cfg(all(feature = "debug", feature = "pulley"))] - pub(crate) raw_store: NonNull, - pub(super) prev: Cell, // The state of the runtime for the *previous* `CallThreadState` for @@ -636,8 +625,6 @@ mod call_thread_state { #[cfg(feature = "coredump")] capture_coredump: store.engine().config().coredump_on_trap, vm_store_context: store.vm_store_context_ptr(), - #[cfg(all(feature = "debug", feature = "pulley"))] - raw_store: NonNull::from_mut(store), prev: Cell::new(ptr::null()), old_state, } @@ -881,10 +868,9 @@ impl CallThreadState { /// skip all Rust destructors on the stack, if there are any, for native /// executors as `Handler::resume` will be used. unsafe fn unwind(&self, store: &mut dyn VMStore) { - #[allow(unused_mut, reason = "only mutated in `debug` configuration")] - let mut unwind = self.unwind.replace(UnwindState::None); - self.debug_event_on_unwind(store, &mut unwind); + self.invoke_debug_event(store); + let unwind = self.unwind.replace(UnwindState::None); match unwind { UnwindState::UnwindToHost { .. } => { self.unwind.set(unwind); @@ -928,17 +914,11 @@ impl CallThreadState { } } - /// From the Pulley interpreter, perform a fiber suspend for a - /// debug event handler after setting the unwind state with - /// `set_jit_trap`. - #[cfg(all(feature = "debug", feature = "pulley"))] - pub(crate) fn debug_event_from_interpreter(&self) { + /// Invoke a debug event handler after setting the unwind state + /// with `set_jit_trap`. + #[cfg(all(feature = "debug"))] + pub(crate) fn invoke_debug_event(&self, store: &mut dyn VMStore) { let mut unwind = self.unwind.replace(UnwindState::None); - // SAFETY: this is invoked only within a trapping context when - // we have received control back from the Wasm code. See the - // provenance diagram and comments on `self.raw_store` for - // more details. - let store = unsafe { self.raw_store.as_ptr().as_mut().unwrap() }; self.debug_event_on_unwind(store, &mut unwind); self.unwind.set(unwind); } @@ -948,6 +928,10 @@ impl CallThreadState { /// event. #[cfg(feature = "debug")] fn debug_event_on_unwind(&self, store: &mut dyn VMStore, unwind: &mut UnwindState) { + if !store.has_debug_handler() { + return; + } + let result = match unwind { UnwindState::UnwindToWasm(_) => { assert!(store.as_store_opaque().has_pending_exception()); @@ -1134,11 +1118,44 @@ impl CallThreadState { return TrapTest::NotWasm; }; - // If all that passed then this is indeed a wasm trap, so return the - // `Handler` setup in the original wasm frame. + // If all that passed then this is indeed a wasm trap; we will + // handle it either directly or by injecting a call to a + // trap-hostcall. + self.set_jit_trap(regs, faulting_addr, trap); + + // If a debug session is active on the store, then we will + // inject a hostcall so that we can invoke the debug event + // handler. + #[cfg(all(feature = "debug", has_native_signals))] + { + // SAFETY: We are in a trapping context, without another + // `&mut dyn VMStore` or equivalent present anywhere. This + // is like a mutable reborrow of the store from the + // trapping code, as-if the trapping opcode were a + // hostcall. See provenance/safety comment on `raw_store` + // for more. + let vm_store = unsafe { self.vm_store_context.as_ref().raw_store_mut() }; + + if vm_store.has_debug_handler() { + let mut pc = regs.pc; + let mut arg0 = regs.arg0; + let mut arg1 = regs.arg1; + let vm_store_context = vm_store.as_store_opaque().vm_store_context_mut(); + vm_store_context.inject_trap_handler_hostcall(&mut pc, &mut arg0, &mut arg1); + let handler = Handler { + pc: pc, + sp: regs.sp, + fp: regs.fp, + }; + return TrapTest::Trap(handler, TrapResumeArgs::Call(arg0, arg1)); + } + } + + // Otherwise: return the `Handler` setup in the original wasm + // frame so that the signal handler resumes to it. let entry_handler = self.entry_trap_handler(); - TrapTest::Trap(entry_handler) + TrapTest::Trap(entry_handler, TrapResumeArgs::Handler(0, 0)) } pub(crate) fn set_jit_trap( diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/inject_call.rs b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call.rs new file mode 100644 index 000000000000..c3169d86100a --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call.rs @@ -0,0 +1,237 @@ +//! Functionality to convert signals to synthetic calls. +//! +//! When handling a signal that occurs *synchronously*, i.e., +//! intentionally as a result of an instruction in generated code, we +//! sometimes wish to convert the signal to behavior as-if the +//! faulting instruction had been a call. This may seem somewhat odd, +//! but it comes in extremely useful for: +//! +//! - Allowing introspection of trapping state when a trap is detected +//! as a caught signal (e.g. an out-of-bounds Wasm memory access) +//! before the stack is unwound (or fiber stack is thrown away). +//! +//! - Allowing "resumable traps" for debugging breakpoints: a trap +//! instruction (e.g. `ud2` on x86-64 or `brk` on aarch64) can be +//! converted to a "yield to debugger" hostcall, and we can even +//! return and continue by incrementing PC beyond the instruction. +//! +//! We do not require space on the stack, and we don't inject a stack +//! frame (because Windows vectored exception handling won't allow us +//! to do so: the exception handler runs on the same stack already, so +//! its frame is in the way). We also have to preserve all register +//! state, because we don't want to treat all potentially-trapping +//! instructions as clobbering a bunch of registers. Rather, our +//! trampoline should save all state. +//! +//! Because we're injecting a new state value on trap (a new PC, to +//! our trampoline), and because we need to save all existing state, +//! something has to give: we don't have anywhere in the register +//! state to save the existing PC of the trap location. Instead, what +//! we do is we make use of the fact that we have access to the Store +//! here: we save off the trapping PC, and just overwrite PC in the +//! signal-frame context to redirect. The trampoline will then call +//! into host code and that host code will return the original PC; the +//! trampoline will restore all state it saved on the stack, and +//! resume to that PC. + +use crate::vm::VMStoreContext; + +/// State for a hostcall invocation created in response to a signal. +/// +/// This state is created when we perform the state mutation within +/// the signal handler, and is consumed when we return to the injected +/// trampoline so that it can resume to the original code. +pub struct InjectedCallState { + /// Saved PC. + pub pc: usize, + /// Saved first argument register. + arg0: usize, + /// Saved second argument register. + arg1: usize, +} + +impl InjectedCallState { + fn inject( + pc: &mut usize, + arg0: &mut usize, + arg1: &mut usize, + trampoline: usize, + trampoline_arg: usize, + ) -> InjectedCallState { + assert!(SUPPORTED_ARCH); + + log::trace!( + "inject: orig pc {:x} at stack slot {:x}; injecting {:x}", + *pc, + pc as *const _ as usize, + injected_call_trampoline as usize + ); + // Save off the PC at the trapping location, and update it to + // point to our trampoline. + let orig_pc = core::mem::replace(pc, injected_call_trampoline as usize); + let saved_arg0 = core::mem::replace(arg0, trampoline); + let saved_arg1 = core::mem::replace(arg1, trampoline_arg); + // Save the original state to restore in the stack frame while + // the injected call runs. + InjectedCallState { + // Note: we don't yet support resumable traps, but that + // will be implemented soon; when it is, then we will + // actually return to this PC. + pc: orig_pc, + arg0: saved_arg0, + arg1: saved_arg1, + } + } + + fn restore(self, pc: &mut usize, arg0: &mut usize, arg1: &mut usize) { + log::trace!( + "restore: pc slot at {:x} gets {:x}", + pc as *const _ as usize, + self.pc + ); + *pc = self.pc; + *arg0 = self.arg0; + *arg1 = self.arg1; + } +} + +impl VMStoreContext { + /// From a VMStoreContext in a trap context, inject a call to the + /// trap-handler hostcall. + /// + /// `pc`, `arg0`, and `arg1` are mutable borrows to register + /// values in the signal register context corresponding to the + /// program counter and first and second function argument + /// registers on the current platform. + pub(crate) fn inject_trap_handler_hostcall( + &mut self, + pc: &mut usize, + arg0: &mut usize, + arg1: &mut usize, + ) { + let vmctx_raw_ptr = self as *mut _ as usize; + let handler = injected_trap_handler as usize; + let state = InjectedCallState::inject(pc, arg0, arg1, handler, vmctx_raw_ptr); + let old = core::mem::replace(self.injected_call_state.get_mut(), Some(state)); + assert!(old.is_none()); + } + + /// From the trap-handler hostcall, restore state needed to return + /// from the hostcall. + /// + /// `pc`, `arg0`, and `arg1` are mutable borrows to register + /// values in the tramopline's register-save frame to be updated. + pub(crate) fn trap_handler_hostcall_fixup( + &mut self, + pc: &mut usize, + arg0: &mut usize, + arg1: &mut usize, + ) { + let state = core::mem::replace(self.injected_call_state.get_mut(), None) + .expect("Saved register state must be present"); + state.restore(pc, arg0, arg1); + } +} + +cfg_if::cfg_if! { + if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub const SUPPORTED_ARCH: bool = true; + pub(crate) use aarch64::*; + } else if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub const SUPPORTED_ARCH: bool = true; + pub(crate) use x86_64::*; + } else if #[cfg(target_arch = "s390x")] { + mod s390x; + pub const SUPPORTED_ARCH: bool = true; + pub(crate) use s390x::*; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + pub const SUPPORTED_ARCH: bool = true; + pub(crate) use riscv64::*; + } else { + pub(crate) use unsupported::*; + } +} + +#[allow( + dead_code, + reason = "expected to have dead code in some configurations" +)] +mod unsupported { + pub const SUPPORTED_ARCH: bool = false; + pub(crate) fn injected_call_trampoline() -> ! { + unreachable!() + } +} + +/// This handler is invoked directly from the stub injected by the +/// signal handler on Wasm code when debugging is enabled; from its +/// perspective, it has been called in a Wasm context, i.e. without a +/// normal VM exit trampoline. The debug yield reason will have +/// already been filled in; this handler's only job is to restore the +/// register state used to inject the stub call, then suspend the +/// fiber, and then return when it resumes. +unsafe extern "C" fn injected_trap_handler( + vm_store_context: *mut VMStoreContext, + orig_pc: *mut u8, + orig_arg0: *mut u8, + orig_arg1: *mut u8, +) { + log::trace!( + "injected trap handler running; orig_pc = {:x}", + orig_pc as usize + ); + let vm_store_context = unsafe { + vm_store_context + .as_mut() + .expect("null VMStoreContext pointer") + }; + let orig_pc: &mut usize = unsafe { &mut *orig_pc.cast::() }; + let orig_arg0: &mut usize = unsafe { &mut *orig_arg0.cast::() }; + let orig_arg1: &mut usize = unsafe { &mut *orig_arg1.cast::() }; + vm_store_context.trap_handler_hostcall_fixup(orig_pc, orig_arg0, orig_arg1); + + // SAFETY: we have a valid VMStoreContext, so we can use its Store + // backpointer. This will be the only store reference held in this + // context so it is valid to derive. + let store = unsafe { vm_store_context.raw_store_mut() }; + super::tls::with(|s| { + let s = s.expect("Trap context must have a CallThreadState"); + s.invoke_debug_event(store); + }); + + // Now perform the original trap resume to the entry trap + // handler. Note: once we support resumable traps, this is where + // we will instead update `orig_pc` to skip over a break + // instruction and return if resuming. + + // SAFETY: we are in a trap context, and all entries into Wasm + // that would have trapped will set an entry trap handler. + unsafe { + injected_trap_terminate(); + } +} + +/// Perform the original resumption to the entry trap resume point. +/// +/// # Safety +/// +/// We must be in a trap context with a valid entry trap handler ste. +unsafe extern "C" fn injected_trap_terminate() -> ! { + let handler = super::tls::with(|state| { + let state = state.expect("there must be an active CallThreadState"); + state.entry_trap_handler() + }); + log::trace!("injected_trap_terminate about to invoke original entry handler"); + unsafe { handler.resume_tailcc(0, 0) } +} + +const _: () = { + #[used] + static USED1: unsafe extern "C" fn(*mut VMStoreContext, *mut u8, *mut u8, *mut u8) = + injected_trap_handler; + #[used] + static USED2: unsafe extern "C" fn() -> ! = injected_trap_terminate; +}; diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/aarch64.rs b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/aarch64.rs new file mode 100644 index 000000000000..631db1f6b993 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/aarch64.rs @@ -0,0 +1,98 @@ +//! aarch64 injected-call trampoline. +//! +//! see [`super::x86_64`] for documentation on how the +//! architecture-specific trampolines are used. + +use core::arch::naked_asm; + +#[unsafe(naked)] +pub(crate) unsafe extern "C" fn injected_call_trampoline(_hostcall: usize, _store: usize) { + naked_asm!( + " + // Fake return address and saved FP. + stp fp, xzr, [sp, #-16]! + mov fp, sp + + // Saved GPRs. + stp x0, x1, [sp, #-16]! + stp x2, x3, [sp, #-16]! + stp x4, x5, [sp, #-16]! + stp x6, x7, [sp, #-16]! + stp x8, x9, [sp, #-16]! + stp x10, x11, [sp, #-16]! + stp x12, x13, [sp, #-16]! + stp x14, x15, [sp, #-16]! + stp x16, x17, [sp, #-16]! + stp x18, x19, [sp, #-16]! + stp x20, x21, [sp, #-16]! + stp x22, x23, [sp, #-16]! + stp x24, x25, [sp, #-16]! + stp x26, x27, [sp, #-16]! + stp x28, xzr, [sp, #-16]! + + // Saved float/vector registers. + stp q0, q1, [sp, #-32]! + stp q2, q3, [sp, #-32]! + stp q4, q5, [sp, #-32]! + stp q6, q7, [sp, #-32]! + stp q8, q9, [sp, #-32]! + stp q10, q11, [sp, #-32]! + stp q12, q13, [sp, #-32]! + stp q14, q15, [sp, #-32]! + stp q16, q17, [sp, #-32]! + stp q18, q19, [sp, #-32]! + stp q20, q21, [sp, #-32]! + stp q22, q23, [sp, #-32]! + stp q24, q25, [sp, #-32]! + stp q26, q27, [sp, #-32]! + stp q28, q29, [sp, #-32]! + stp q30, q31, [sp, #-32]! + + mov x4, x0 + mov x0, x1 + add x1, sp, #(16 * 32 + 15 * 16 + 8) // saved LR. + add x2, sp, #(16 * 32 + 14 * 16 + 0) // saved X0. + add x3, sp, #(16 * 32 + 14 * 16 + 8) // saved X1. + blr x4 + + ldp q30, q31, [sp], #32 + ldp q28, q29, [sp], #32 + ldp q26, q27, [sp], #32 + ldp q24, q25, [sp], #32 + ldp q22, q23, [sp], #32 + ldp q20, q21, [sp], #32 + ldp q18, q19, [sp], #32 + ldp q16, q17, [sp], #32 + ldp q14, q15, [sp], #32 + ldp q12, q13, [sp], #32 + ldp q10, q11, [sp], #32 + ldp q8, q9, [sp], #32 + ldp q6, q7, [sp], #32 + ldp q4, q5, [sp], #32 + ldp q2, q3, [sp], #32 + ldp q0, q1, [sp], #32 + + ldp x28, x29, [sp], #16 + ldp x26, x27, [sp], #16 + ldp x24, x25, [sp], #16 + ldp x22, x23, [sp], #16 + ldp x20, x21, [sp], #16 + ldp x18, x19, [sp], #16 + ldp x16, x17, [sp], #16 + ldp x14, x15, [sp], #16 + ldp x12, x13, [sp], #16 + ldp x10, x11, [sp], #16 + ldp x8, x9, [sp], #16 + ldp x6, x7, [sp], #16 + ldp x4, x5, [sp], #16 + ldp x2, x3, [sp], #16 + ldp x0, x1, [sp], #16 + ldp fp, lr, [sp], #16 + + // N.B.: this leaves LR clobbered; but Cranelift + // code interrupted by a signal will already have + // unconditionally saved LR in its frame. + ret + ", + ); +} diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/riscv64.rs b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/riscv64.rs new file mode 100644 index 000000000000..bf3a5f92c371 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/riscv64.rs @@ -0,0 +1,233 @@ +//! riscv64 injected-call trampoline. +//! +//! see [`super::x86_64`] for documentation on how the +//! architecture-specific trampolines are used. + +use core::arch::naked_asm; + +#[unsafe(naked)] +pub(crate) unsafe extern "C" fn injected_call_trampoline(_hostcall: usize, _store: usize) { + naked_asm!( + " + // Note that we assume the V extension on platforms that + // support debugging. To do otherwise, we'd need two + // versions of this trampoline and pick the right one at + // runtime depending on the engine configuration. + .attribute arch, \"rv64gc\" + .option push + .option arch, +zve32x + + addi sp, sp, -16 + sd zero, 8(sp) // Fake return address. + sd fp, 0(sp) + mv fp, sp + + // 28 (other) X regs: 8 * 28 = 224 + addi sp, sp, -224 + + sd gp, 216(sp) + sd tp, 208(sp) + sd t0, 200(sp) + sd t1, 192(sp) + sd t2, 184(sp) + // s0 is fp (saved above). + sd s1, 176(sp) + sd a0, 168(sp) + sd a1, 160(sp) + sd a2, 152(sp) + sd a3, 144(sp) + sd a4, 136(sp) + sd a5, 128(sp) + sd a6, 120(sp) + sd a7, 112(sp) + sd s2, 104(sp) + sd s3, 96(sp) + sd s4, 88(sp) + sd s5, 80(sp) + sd s6, 72(sp) + sd s7, 64(sp) + sd s8, 56(sp) + sd s9, 48(sp) + sd s10, 40(sp) + sd s11, 32(sp) + sd t3, 24(sp) + sd t4, 16(sp) + sd t5, 8(sp) + sd t6, 0(sp) + + // 32 F regs: 8 * 32 = 256 + addi sp, sp, -256 + fsd f0, 248(sp) + fsd f1, 240(sp) + fsd f2, 232(sp) + fsd f3, 224(sp) + fsd f4, 216(sp) + fsd f5, 208(sp) + fsd f6, 200(sp) + fsd f7, 192(sp) + fsd f8, 184(sp) + fsd f9, 176(sp) + fsd f10, 168(sp) + fsd f11, 160(sp) + fsd f12, 152(sp) + fsd f13, 144(sp) + fsd f14, 136(sp) + fsd f15, 128(sp) + fsd f16, 120(sp) + fsd f17, 112(sp) + fsd f18, 104(sp) + fsd f19, 96(sp) + fsd f20, 88(sp) + fsd f21, 80(sp) + fsd f22, 72(sp) + fsd f23, 64(sp) + fsd f24, 56(sp) + fsd f25, 48(sp) + fsd f26, 40(sp) + fsd f27, 32(sp) + fsd f28, 24(sp) + fsd f29, 16(sp) + fsd f30, 8(sp) + fsd f31, 0(sp) + + // V extension state save/restore. + // See the Linux kernel context-switching implementation at + // https://github.com/torvalds/linux/blob/98906f9d850e4882004749eccb8920649dc98456/arch/riscv/include/asm/vector.h + // for a good source on all of this. + + // First save the VSTART, VTYPE, VL, VCSR control registers. + addi sp, sp, -32 + csrr t0, 0x8 // VSTART + csrr t1, 0xc21 // VTYPE + csrr t2, 0xc20 // VL + csrr t3, 0xf // VCSR + sd t0, 24(sp) + sd t1, 16(sp) + sd t2, 8(sp) + sd t3, 0(sp) + + // 32 V regs, saving 128 bits because that is + // all we use in Cranelift-compiled code. + + // Set the VLEN and VTYPE registers so that we have grouping + // by 8 in the V register bank. + vsetivli zero, 16, e8, m8, ta, ma // 16 elems, 8-bit elems, 8-reg groups, + // tail-agnostic, mask-agnostic + addi sp, sp, -128 + vse8.v v0, 0(sp) + addi sp, sp, -128 + vse8.v v8, 0(sp) + addi sp, sp, -128 + vse8.v v16, 0(sp) + addi sp, sp, -128 + vse8.v v24, 0(sp) + + mv t0, a0 + mv a0, a1 + addi a1, sp, (512 + 32 + 256 + 224 + 8) // saved RA. + addi a2, sp, (512 + 32 + 256 + 168) // saved A0. + addi a3, sp, (512 + 32 + 256 + 160) // saved A1. + jalr t0 + + // Set up V state again for our restore. + vsetivli zero, 16, e8, m8, ta, ma + vle8.v v24, 0(sp) + addi sp, sp, 128 + vle8.v v16, 0(sp) + addi sp, sp, 128 + vle8.v v8, 0(sp) + addi sp, sp, 128 + vle8.v v0, 0(sp) + addi sp, sp, 128 + + // Restore VSTART, VTYPE, VL, VCSR. + ld t0, 24(sp) + ld t1, 16(sp) + ld t2, 8(sp) + ld t3, 0(sp) + csrw 0x8, t0 // VSTART + vsetvl zero, t2, t1 + csrw 0xf, t3 // VCSR + addi sp, sp, 32 + + // Restore F regs. + fld f0, 248(sp) + fld f1, 240(sp) + fld f2, 232(sp) + fld f3, 224(sp) + fld f4, 216(sp) + fld f5, 208(sp) + fld f6, 200(sp) + fld f7, 192(sp) + fld f8, 184(sp) + fld f9, 176(sp) + fld f10, 168(sp) + fld f11, 160(sp) + fld f12, 152(sp) + fld f13, 144(sp) + fld f14, 136(sp) + fld f15, 128(sp) + fld f16, 120(sp) + fld f17, 112(sp) + fld f18, 104(sp) + fld f19, 96(sp) + fld f20, 88(sp) + fld f21, 80(sp) + fld f22, 72(sp) + fld f23, 64(sp) + fld f24, 56(sp) + fld f25, 48(sp) + fld f26, 40(sp) + fld f27, 32(sp) + fld f28, 24(sp) + fld f29, 16(sp) + fld f30, 8(sp) + fld f31, 0(sp) + addi sp, sp, 256 + + // Restore X regs. + ld gp, 216(sp) + ld tp, 208(sp) + ld t0, 200(sp) + ld t1, 192(sp) + ld t2, 184(sp) + // s0 is fp (restored below). + ld s1, 176(sp) + ld a0, 168(sp) + ld a1, 160(sp) + ld a2, 152(sp) + ld a3, 144(sp) + ld a4, 136(sp) + ld a5, 128(sp) + ld a6, 120(sp) + ld a7, 112(sp) + ld s2, 104(sp) + ld s3, 96(sp) + ld s4, 88(sp) + ld s5, 80(sp) + ld s6, 72(sp) + ld s7, 64(sp) + ld s8, 56(sp) + ld s9, 48(sp) + ld s10, 40(sp) + ld s11, 32(sp) + ld t3, 24(sp) + ld t4, 16(sp) + ld t5, 8(sp) + ld t6, 0(sp) + addi sp, sp, 224 + + // Restore FP and RA. + ld fp, 0(sp) + ld ra, 8(sp) + addi sp, sp, 16 + + // N.B.: this leaves RA clobbered; but Cranelift + // code interrupted by a signal will already have + // unconditionally saved LR in its frame. + ret + + .option pop + ", + ); +} diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/s390x.rs b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/s390x.rs new file mode 100644 index 000000000000..7c1f733737fa --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/s390x.rs @@ -0,0 +1,184 @@ +//! s390x injected-call trampoline. +//! +//! see [`super::x86_64`] for documentation on how the +//! architecture-specific trampolines are used. + +use core::arch::naked_asm; + +#[unsafe(naked)] +pub(crate) unsafe extern "C" fn injected_call_trampoline(_hostcall: usize, _store: usize) { + naked_asm!( + " + // Unlike an ordinary function in the s390x calling convention, + // we do not enter here with a stack frame already allocated for + // us; rather, we have to assume everything at SP (%r15) and up is + // user-controlled, and we must allocate our own frame. We also save + // all registers, not only volatiles. + + aghi %r15, -128 + stmg %r0, %r15, 0(%r15) + + aghi %r15, -512 + // We want to save vector registers, but the default Rust target + // s390x-unknown-linux-gnu does not have the vector extension + // enabled by default, even though we generate vector code with + // Cranelift just fine. So we're going to manually encode instructions + // below with byte literals (eek!). Manually verified with + // `s390x-linux-gnu-as` + `s390x-linux-gnu-objdump`. + + // vst %v0, 0(%r15) + .word 0xe700, 0xf000, 0x000e + // vst %v1, 16(%r15) + .word 0xe710, 0xf010, 0x000e + // vst %v2, 32(%r15) + .word 0xe720, 0xf020, 0x000e + // vst %v3, 48(%r15) + .word 0xe730, 0xf030, 0x000e + // vst %v4, 64(%r15) + .word 0xe740, 0xf040, 0x000e + // vst %v5, 80(%r15) + .word 0xe750, 0xf050, 0x000e + // vst %v6, 96(%r15) + .word 0xe760, 0xf060, 0x000e + // vst %v7, 112(%r15) + .word 0xe770, 0xf070, 0x000e + // vst %v8, 128(%r15) + .word 0xe780, 0xf080, 0x000e + // vst %v9, 144(%r15) + .word 0xe790, 0xf090, 0x000e + // vst %v10, 160(%r15) + .word 0xe7a0, 0xf0a0, 0x000e + // vst %v11, 176(%r15) + .word 0xe7b0, 0xf0b0, 0x000e + // vst %v12, 192(%r15) + .word 0xe7c0, 0xf0c0, 0x000e + // vst %v13, 208(%r15) + .word 0xe7d0, 0xf0d0, 0x000e + // vst %v14, 224(%r15) + .word 0xe7e0, 0xf0e0, 0x000e + // vst %v15, 240(%r15) + .word 0xe7f0, 0xf0f0, 0x000e + // vst %v16, 256(%r15) + .word 0xe700, 0xf100, 0x080e + // vst %v17, 272(%r15) + .word 0xe710, 0xf110, 0x080e + // vst %v18, 288(%r15) + .word 0xe720, 0xf120, 0x080e + // vst %v19, 304(%r15) + .word 0xe730, 0xf130, 0x080e + // vst %v20, 320(%r15) + .word 0xe740, 0xf140, 0x080e + // vst %v21, 336(%r15) + .word 0xe750, 0xf150, 0x080e + // vst %v22, 352(%r15) + .word 0xe760, 0xf160, 0x080e + // vst %v23, 368(%r15) + .word 0xe770, 0xf170, 0x080e + // vst %v24, 384(%r15) + .word 0xe780, 0xf180, 0x080e + // vst %v25, 400(%r15) + .word 0xe790, 0xf190, 0x080e + // vst %v26, 416(%r15) + .word 0xe7a0, 0xf1a0, 0x080e + // vst %v27, 432(%r15) + .word 0xe7b0, 0xf1b0, 0x080e + // vst %v28, 448(%r15) + .word 0xe7c0, 0xf1c0, 0x080e + // vst %v29, 464(%r15) + .word 0xe7d0, 0xf1d0, 0x080e + // vst %v30, 480(%r15) + .word 0xe7e0, 0xf1e0, 0x080e + // vst %v31, 496(%r15) + .word 0xe7f0, 0xf1f0, 0x080e + + // Create frame for callee. + aghi %r15, -160 + lgr %r1, %r2 // Arg 0: next callee. + lgr %r2, %r3 // Arg 1: callee's first arg. + lgr %r3, %r15 + // Address of saved return address. We use r1: it is the spilltmp + // in Cranelift-compiled code and will not be relied upon to be + // saved across any trapping instruction. + aghi %r3, 160 + 512 + 1*8 + lgr %r4, %r15 + aghi %r4, 160 + 512 + 2*8 // Address of saved r2 (arg 0). + lgr %r5, %r15 + aghi %r5, 160 + 512 + 3*8 // Address of saved r3 (arg 1). + basr %r14, %r1 + aghi %r15, 160 + + // vl %v0, 0(%r15) + .word 0xe700, 0xf000, 0x0006 + // vl %v1, 16(%r15) + .word 0xe710, 0xf010, 0x0006 + // vl %v2, 32(%r15) + .word 0xe720, 0xf020, 0x0006 + // vl %v3, 48(%r15) + .word 0xe730, 0xf030, 0x0006 + // vl %v4, 64(%r15) + .word 0xe740, 0xf040, 0x0006 + // vl %v5, 80(%r15) + .word 0xe750, 0xf050, 0x0006 + // vl %v6, 96(%r15) + .word 0xe760, 0xf060, 0x0006 + // vl %v7, 112(%r15) + .word 0xe770, 0xf070, 0x0006 + // vl %v8, 128(%r15) + .word 0xe780, 0xf080, 0x0006 + // vl %v9, 144(%r15) + .word 0xe790, 0xf090, 0x0006 + // vl %v10, 160(%r15) + .word 0xe7a0, 0xf0a0, 0x0006 + // vl %v11, 176(%r15) + .word 0xe7b0, 0xf0b0, 0x0006 + // vl %v12, 192(%r15) + .word 0xe7c0, 0xf0c0, 0x0006 + // vl %v13, 208(%r15) + .word 0xe7d0, 0xf0d0, 0x0006 + // vl %v14, 224(%r15) + .word 0xe7e0, 0xf0e0, 0x0006 + // vl %v15, 240(%r15) + .word 0xe7f0, 0xf0f0, 0x0006 + // vl %v16, 256(%r15) + .word 0xe700, 0xf100, 0x0806 + // vl %v17, 272(%r15) + .word 0xe710, 0xf110, 0x0806 + // vl %v18, 288(%r15) + .word 0xe720, 0xf120, 0x0806 + // vl %v19, 304(%r15) + .word 0xe730, 0xf130, 0x0806 + // vl %v20, 320(%r15) + .word 0xe740, 0xf140, 0x0806 + // vl %v21, 336(%r15) + .word 0xe750, 0xf150, 0x0806 + // vl %v22, 352(%r15) + .word 0xe760, 0xf160, 0x0806 + // vl %v23, 368(%r15) + .word 0xe770, 0xf170, 0x0806 + // vl %v24, 384(%r15) + .word 0xe780, 0xf180, 0x0806 + // vl %v25, 400(%r15) + .word 0xe790, 0xf190, 0x0806 + // vl %v26, 416(%r15) + .word 0xe7a0, 0xf1a0, 0x0806 + // vl %v27, 432(%r15) + .word 0xe7b0, 0xf1b0, 0x0806 + // vl %v28, 448(%r15) + .word 0xe7c0, 0xf1c0, 0x0806 + // vl %v29, 464(%r15) + .word 0xe7d0, 0xf1d0, 0x0806 + // vl %v30, 480(%r15) + .word 0xe7e0, 0xf1e0, 0x0806 + // vl %v31, 496(%r15) + .word 0xe7f0, 0xf1f0, 0x0806 + + aghi %r15, 512 + + lmg %r0, %r15, 0(%r15) + // No need to add 128 to SP (%r15); we restored it + // with the load-multiple above. + + br %r1 + ", + ); +} diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/x86_64.rs b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/x86_64.rs new file mode 100644 index 000000000000..d03494ae6f23 --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/traphandlers/inject_call/x86_64.rs @@ -0,0 +1,127 @@ +//! x86-64 injected-call trampoline. +//! +//! The purpose of this trampoline is to save all registers to the +//! stack (which we know will be valid at the point that a +//! trap-causing signal occurs in Wasm code) and invoke a +//! hostcall. + +use core::arch::naked_asm; + +#[unsafe(naked)] +pub(crate) unsafe extern "C" fn injected_call_trampoline(_hostcall: usize, _store: usize) { + naked_asm!( + " + // When control reaches here, we have just returned from a signal + // handler after rewriting PC to point to this trampoline but updating + // no other register state. We will have interrupted an instruction known + // to cause traps, but it is not treated as a register-clobbering + // instruction; thus, we need to take care to save all registers. + // + // We can assume the stack has enough space for this state-saving. We + // ensure this via our implementation of stack limit checks in + // Cranelift-compiled code. + // + // The signal handler will have placed the address of the hostcall + // in our first argument register (rdi), and the store value in our + // second argument register (rsi). These are necessary because + // we otherwise know nothing about user state here and e.g. where to + // find the store and look up trampolines. + // + // We call the hostcall trampoline with pointers to the places + // where we saved PC (rip) and the argument registers (rdi/rsi), so + // it can fill back in the original values before we return. + + // Push a fake return address; it will be filled in by the hostcall. + push 0 + // This is an ordinary frame as seen by stack-walks. + push rbp + + // Save all GPRs excep rbp/rsp (saved above and by normal stack + // discipline, respectively). + push rax + push rbx + push rcx + push rdx + push rdi + push rsi + push r8 + push r9 + push r10 + push r11 + push r12 + push r13 + push r14 + push r15 + + // N.B.: we don't save rflags; Cranelift-compiled code + // never assumes it is saved across instructions outside of + // flag-generation / flag-consumption pairs, and the only + // resumable traps we are interested in are not flags-related. + + sub rsp, 256 // enough for all 16 XMM registers. + movdqu [rsp + 0 * 16], xmm0 + movdqu [rsp + 1 * 16], xmm1 + movdqu [rsp + 2 * 16], xmm2 + movdqu [rsp + 3 * 16], xmm3 + movdqu [rsp + 4 * 16], xmm4 + movdqu [rsp + 5 * 16], xmm5 + movdqu [rsp + 6 * 16], xmm6 + movdqu [rsp + 7 * 16], xmm7 + movdqu [rsp + 8 * 16], xmm8 + movdqu [rsp + 9 * 16], xmm9 + movdqu [rsp + 10 * 16], xmm10 + movdqu [rsp + 11 * 16], xmm11 + movdqu [rsp + 12 * 16], xmm12 + movdqu [rsp + 13 * 16], xmm13 + movdqu [rsp + 14 * 16], xmm14 + movdqu [rsp + 15 * 16], xmm15 + + // Move host-entry trampoline call target to rax, and generate + // parameters as addresses to fill in original PC, RDI, and RSI. + // Note that the trampoline is called with `tail` calling convention. + mov rax, rdi + mov rdi, rsi // vmctx. + lea rsi, [rsp + 16 * 16 + 15 * 8] // saved RIP above (to restore). + lea rdx, [rsp + 16 * 16 + 9 * 8] // saved RDI above (to restore). + lea rcx, [rsp + 16 * 16 + 8 * 8] // saved RSI above (to restore). + call rax + + // Now restore everything and return normally. + movdqu xmm0, [rsp + 0 * 16] + movdqu xmm1, [rsp + 1 * 16] + movdqu xmm2, [rsp + 2 * 16] + movdqu xmm3, [rsp + 3 * 16] + movdqu xmm4, [rsp + 4 * 16] + movdqu xmm5, [rsp + 5 * 16] + movdqu xmm6, [rsp + 6 * 16] + movdqu xmm7, [rsp + 7 * 16] + movdqu xmm8, [rsp + 8 * 16] + movdqu xmm9, [rsp + 9 * 16] + movdqu xmm10, [rsp + 10 * 16] + movdqu xmm11, [rsp + 11 * 16] + movdqu xmm12, [rsp + 12 * 16] + movdqu xmm13, [rsp + 13 * 16] + movdqu xmm14, [rsp + 14 * 16] + movdqu xmm15, [rsp + 15 * 16] + add rsp, 256 + + pop r15 + pop r14 + pop r13 + pop r12 + pop r11 + pop r10 + pop r9 + pop r8 + pop rsi + pop rdi + pop rdx + pop rcx + pop rbx + pop rax + + pop rbp + ret + ", + ); +} diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 2e11ae69bc60..43658e9888c7 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -7,6 +7,10 @@ pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16}; use crate::store::StoreOpaque; +#[cfg(all(has_native_signals, feature = "debug"))] +use crate::vm::InjectedCallState; +#[cfg(all(feature = "debug"))] +use crate::vm::VMStore; use crate::vm::stack_switching::VMStackChain; use core::cell::UnsafeCell; use core::ffi::c_void; @@ -1195,6 +1199,67 @@ pub struct VMStoreContext { /// It is set *only* from host code, but is kept here alongside /// the other last-exit state for consistency. pub last_wasm_exit_was_trap: UnsafeCell, + + /// Register state to restore into signal frame for injected trap + /// hostcalls. + /// + /// When `VMStoreContext::inject_trap_handler_hostcall()` is used + /// from within a signal handler to redirect execution to a + /// hostcall on trap, we overwrite some signal register state to + /// give context to the hostcall. Those overwritten register + /// values are saved here, and placed back into a trampoline + /// register-save frame so that they are restored properly if we + /// resume the original guest code. + #[cfg(all(feature = "debug", has_native_signals))] + pub injected_call_state: UnsafeCell>, + + /// Raw pointer to the `*mut dyn VMStore` running in this + /// activation, to be used *only* when re-entering the host during + /// a trap (from native code using call injection, or from the + /// interpreter). + /// + /// This is a very tricky ownership/provenance dance. When control + /// is in the Wasm code itself, the store is completely owned by + /// the Wasm. It passes ownership back during hostcalls via + /// mutable reborrow (as with any call with a `&mut self` in + /// Rust). That's all well and good for explicit calls. + /// + /// When a trap occurs, however, we can also think of the + /// ownership passing as-if the trapping instruction were a + /// hostcall with a `&mut dyn VMStore` parameter. This works *as + /// long as* all possibly trapping points in compiled code act as + /// if they invalidate any other held borrows into the store. + /// + /// It turns out that we generally enforce this in compiled guest + /// code in Cranelift: any `can_trap` opcode returns `true` from + /// `has_memory_fence_semantics()` (see corresponding comment + /// there). This is enough to ensure that the compiler treats + /// every trapping op as-if it were a hostcall, which clobbers all + /// memory state; so from the Wasm code's point of view, it is + /// safely reborrowing the Store and passing it "somewhere" on + /// every trap. The plumbing for that "passing" goes through this + /// field, but that is an implementation detail. When control + /// comes back out of the Wasm activation, we clear this field; + /// the invocation itself takes a mutable borrow of the store, so + /// safety is preserved on the caller side as well. In other + /// words, the provenance is something like + /// + /// ```plain + /// + /// host (caller side) with `&mut dyn VMStore` + /// / \ + /// (param into / (this field) + /// entry trampoline) \ + /// | | + /// ~~~~~~~ (wasm code) ~~~~~~ + /// | | + /// libcall trap + /// ``` + /// + /// with only *one* of those paths dynamically taken at any given + /// time. + #[cfg(all(feature = "debug"))] + pub(crate) raw_store: UnsafeCell>>, } // The `VMStoreContext` type is a pod-type with no destructor, and we don't @@ -1226,6 +1291,31 @@ impl Default for VMStoreContext { async_guard_range: ptr::null_mut()..ptr::null_mut(), store_data: VmPtr::dangling(), last_wasm_exit_was_trap: UnsafeCell::new(false), + #[cfg(all(feature = "debug", has_native_signals))] + injected_call_state: UnsafeCell::new(None), + #[cfg(feature = "debug")] + raw_store: UnsafeCell::new(None), + } + } +} + +impl VMStoreContext { + #[cfg(feature = "debug")] + /// Get a mutable borrow to the `dyn VMStore` from this `VMStoreContext`. + /// + /// This must *only* be used from within a trap context, where the + /// `VMContext`/`VMStoreContext` flow out of Wasm code but no + /// other live borrow of the store exists. This is a way to + /// recover the latter from the former in contexts where we don't + /// have an explicit store borrow by way of hostcall + /// arguments. Note that the Wasm code must expect this by acting + /// as if a mutable borrow of the store is possible at the point + /// that the trap occurs. + #[cfg(feature = "debug")] + pub(crate) unsafe fn raw_store_mut(&self) -> &mut dyn VMStore { + unsafe { + let raw = (*self.raw_store.get()).expect("Raw store pointer must be set"); + raw.as_ptr().as_mut().expect("must be non-null") } } } diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs index f394822aae26..57d8db666d3d 100644 --- a/pulley/src/interp.rs +++ b/pulley/src/interp.rs @@ -204,6 +204,11 @@ impl Vm { self.state.fp } + /// Returns the current `sp` register value. + pub fn sp(&self) -> *mut u8 { + self.state.x_regs[XReg::sp.index()].get_ptr() + } + /// Returns the current `lr` register value. pub fn lr(&self) -> *mut u8 { self.state.lr diff --git a/tests/all/debug.rs b/tests/all/debug.rs index 3eab430e7466..c3f94268e571 100644 --- a/tests/all/debug.rs +++ b/tests/all/debug.rs @@ -366,33 +366,75 @@ async fn caught_exception_events() -> anyhow::Result<()> { async fn hostcall_trap_events() -> anyhow::Result<()> { let _ = env_logger::try_init(); + // Test both hostcall-based traps and native traps. + for signals_based_traps in [false, true] { + let (module, mut store) = get_module_and_store( + |config| { + config.async_support(true); + config.wasm_exceptions(true); + config.signals_based_traps(signals_based_traps); + }, + r#" + (module + (func (export "main") + i32.const 0 + i32.const 0 + i32.div_u + drop)) + "#, + )?; + + debug_event_checker!( + D, store, + { 0 ; + wasmtime::DebugEvent::Trap(wasmtime_environ::Trap::IntegerDivisionByZero) => { + let mut stack = store.debug_frames().unwrap(); + assert!(!stack.done()); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().0.as_u32(), 0); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().1, 37); + stack.move_to_parent(); + assert!(stack.done()); + } + } + ); + + let (handler, counter) = D::new_and_counter(); + store.set_debug_handler(handler); + + let instance = Instance::new_async(&mut store, &module, &[]).await?; + let func = instance.get_func(&mut store, "main").unwrap(); + let mut results = []; + let result = func.call_async(&mut store, &[], &mut results).await; + assert!(result.is_err()); // Uncaught trap. + assert_eq!(counter.load(Ordering::Relaxed), 1); + } + + Ok(()) +} + +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn hostcall_error_events() -> anyhow::Result<()> { + let _ = env_logger::try_init(); + let (module, mut store) = get_module_and_store( |config| { config.async_support(true); config.wasm_exceptions(true); - // Force hostcall-based traps. - config.signals_based_traps(false); }, r#" (module + (import "" "do_a_trap" (func)) (func (export "main") - i32.const 0 - i32.const 0 - i32.div_u - drop)) + call 0)) "#, )?; debug_event_checker!( D, store, { 0 ; - wasmtime::DebugEvent::Trap(wasmtime_environ::Trap::IntegerDivisionByZero) => { - let mut stack = store.debug_frames().unwrap(); - assert!(!stack.done()); - assert_eq!(stack.wasm_function_index_and_pc().unwrap().0.as_u32(), 0); - assert_eq!(stack.wasm_function_index_and_pc().unwrap().1, 37); - stack.move_to_parent(); - assert!(stack.done()); + wasmtime::DebugEvent::HostcallError(e) => { + assert!(format!("{e:?}").contains("secret error message")); } } ); @@ -400,19 +442,24 @@ async fn hostcall_trap_events() -> anyhow::Result<()> { let (handler, counter) = D::new_and_counter(); store.set_debug_handler(handler); - let instance = Instance::new_async(&mut store, &module, &[]).await?; + let do_a_trap = Func::wrap( + &mut store, + |_caller: Caller<'_, ()>| -> anyhow::Result<()> { + Err(anyhow::anyhow!("secret error message")) + }, + ); + let instance = Instance::new_async(&mut store, &module, &[Extern::Func(do_a_trap)]).await?; let func = instance.get_func(&mut store, "main").unwrap(); let mut results = []; let result = func.call_async(&mut store, &[], &mut results).await; assert!(result.is_err()); // Uncaught trap. assert_eq!(counter.load(Ordering::Relaxed), 1); - Ok(()) } #[tokio::test] #[cfg_attr(miri, ignore)] -async fn hostcall_error_events() -> anyhow::Result<()> { +async fn hostcall_trap_out_of_bounds_signals() -> anyhow::Result<()> { let _ = env_logger::try_init(); let (module, mut store) = get_module_and_store( @@ -421,18 +468,25 @@ async fn hostcall_error_events() -> anyhow::Result<()> { config.wasm_exceptions(true); }, r#" - (module - (import "" "do_a_trap" (func)) - (func (export "main") - call 0)) - "#, + (module + (memory $m 1 1) + (func (export "main") + i32.const 0x1_0000 + i32.load $m + drop)) + "#, )?; debug_event_checker!( D, store, { 0 ; - wasmtime::DebugEvent::HostcallError(e) => { - assert!(format!("{e:?}").contains("secret error message")); + wasmtime::DebugEvent::Trap(wasmtime_environ::Trap::MemoryOutOfBounds) => { + let mut stack = store.debug_frames().unwrap(); + assert!(!stack.done()); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().0.as_u32(), 0); + assert_eq!(stack.wasm_function_index_and_pc().unwrap().1, 43); + stack.move_to_parent(); + assert!(stack.done()); } } ); @@ -440,17 +494,12 @@ async fn hostcall_error_events() -> anyhow::Result<()> { let (handler, counter) = D::new_and_counter(); store.set_debug_handler(handler); - let do_a_trap = Func::wrap( - &mut store, - |_caller: Caller<'_, ()>| -> anyhow::Result<()> { - Err(anyhow::anyhow!("secret error message")) - }, - ); - let instance = Instance::new_async(&mut store, &module, &[Extern::Func(do_a_trap)]).await?; + let instance = Instance::new_async(&mut store, &module, &[]).await?; let func = instance.get_func(&mut store, "main").unwrap(); let mut results = []; let result = func.call_async(&mut store, &[], &mut results).await; assert!(result.is_err()); // Uncaught trap. assert_eq!(counter.load(Ordering::Relaxed), 1); + Ok(()) }