Introduce debuginfo to statements in MIR

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1146,7 +1146,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 let opt_assignment_rhs_span =
                     self.find_assignments(local).first().map(|&location| {
                         if let Some(mir::Statement {
-                            source_info: _,
                             kind:
                                 mir::StatementKind::Assign(box (
                                     _,

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -33,7 +33,21 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
     ) {
         // FIXME(tempdragon): Not sure if this is correct, probably wrong but still keep it here.
         #[cfg(feature = "master")]
-        _variable_alloca.set_location(_dbg_loc);
+        val.set_location(_dbg_loc);
+    }
+
+    fn dbg_var_value(
+        &mut self,
+        dbg_var: Self::DIVariable,
+        dbg_loc: Self::DILocation,
+        value: Self::Value,
+        direct_offset: Size,
+        // NB: each offset implies a deref (i.e. they're steps in a pointer chain).
+        indirect_offsets: &[Size],
+        // Byte range in the `dbg_var` covered by this fragment,
+        // if this is a fragment of a composite `DIVariable`.
+        fragment: Option<Range<Size>>,
+    ) {
     }
 
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {

compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs

@@ -35,3 +35,6 @@ declare_constant!(DW_OP_plus_uconst: u64);
 /// Double-checked by a static assertion in `RustWrapper.cpp`.
 #[allow(non_upper_case_globals)]
 pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000;
+// It describes the actual value of a source variable which might not exist in registers or in memory.
+#[allow(non_upper_case_globals)]
+pub(crate) const DW_OP_stack_value: u64 = 0x9f;

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -190,7 +190,6 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
 
         unsafe {
-            // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`.
             llvm::LLVMRustDIBuilderInsertDeclareAtEnd(
                 DIB(self.cx()),
                 variable_alloca,
@@ -203,6 +202,53 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
     }
 
+    fn dbg_var_value(
+        &mut self,
+        dbg_var: &'ll DIVariable,
+        dbg_loc: &'ll DILocation,
+        value: Self::Value,
+        direct_offset: Size,
+        indirect_offsets: &[Size],
+        fragment: Option<Range<Size>>,
+    ) {
+        use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst, DW_OP_stack_value};
+
+        // Convert the direct and indirect offsets and fragment byte range to address ops.
+        let mut addr_ops = SmallVec::<[u64; 8]>::new();
+
+        if direct_offset.bytes() > 0 {
+            addr_ops.push(DW_OP_plus_uconst);
+            addr_ops.push(direct_offset.bytes() as u64);
+            addr_ops.push(DW_OP_stack_value);
+        }
+        for &offset in indirect_offsets {
+            addr_ops.push(DW_OP_deref);
+            if offset.bytes() > 0 {
+                addr_ops.push(DW_OP_plus_uconst);
+                addr_ops.push(offset.bytes() as u64);
+            }
+        }
+        if let Some(fragment) = fragment {
+            // `DW_OP_LLVM_fragment` takes as arguments the fragment's
+            // offset and size, both of them in bits.
+            addr_ops.push(DW_OP_LLVM_fragment);
+            addr_ops.push(fragment.start.bits() as u64);
+            addr_ops.push((fragment.end - fragment.start).bits() as u64);
+        }
+
+        unsafe {
+            llvm::LLVMRustDIBuilderInsertDbgValueAtEnd(
+                DIB(self.cx()),
+                value,
+                dbg_var,
+                addr_ops.as_ptr(),
+                addr_ops.len() as c_uint,
+                dbg_loc,
+                self.llbb(),
+            );
+        }
+    }
+
     fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) {
         unsafe {
             llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, dbg_loc);

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2322,6 +2322,16 @@ unsafe extern "C" {
         InsertAtEnd: &'a BasicBlock,
     );
 
+    pub(crate) fn LLVMRustDIBuilderInsertDbgValueAtEnd<'a>(
+        Builder: &DIBuilder<'a>,
+        Val: &'a Value,
+        VarInfo: &'a DIVariable,
+        AddrOps: *const u64,
+        AddrOpsCount: c_uint,
+        DL: &'a DILocation,
+        InsertAtEnd: &'a BasicBlock,
+    );
+
     pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>(
         Builder: &DIBuilder<'a>,
         Name: *const c_char,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1295,6 +1295,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             for statement in &data.statements {
                 self.codegen_statement(bx, statement);
             }
+            self.codegen_stmt_debuginfos(bx, &data.after_last_stmt_debuginfos);
 
             let merging_succ = self.codegen_terminator(bx, bb, data.terminator());
             if let MergingSucc::False = merging_succ {

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -253,6 +253,51 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         spill_slot
     }
 
+    pub(crate) fn debug_new_value_to_local(
+        &self,
+        bx: &mut Bx,
+        local: mir::Local,
+        base: PlaceValue<Bx::Value>,
+        layout: TyAndLayout<'tcx>,
+        projection: &[mir::PlaceElem<'tcx>],
+    ) {
+        let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full;
+        if !full_debug_info {
+            return;
+        }
+
+        let vars = match &self.per_local_var_debug_info {
+            Some(per_local) => &per_local[local],
+            None => return,
+        };
+
+        for var in vars.iter().cloned() {
+            self.debug_new_value_to_local_as_var(bx, base, layout, projection, var);
+        }
+    }
+
+    fn debug_new_value_to_local_as_var(
+        &self,
+        bx: &mut Bx,
+        base: PlaceValue<Bx::Value>,
+        layout: TyAndLayout<'tcx>,
+        projection: &[mir::PlaceElem<'tcx>],
+        var: PerLocalVarDebugInfo<'tcx, Bx::DIVariable>,
+    ) {
+        let Some(dbg_var) = var.dbg_var else { return };
+        let Some(dbg_loc) = self.dbg_loc(var.source_info) else { return };
+        let DebugInfoOffset { direct_offset, indirect_offsets, result: _ } =
+            calculate_debuginfo_offset(bx, projection, layout);
+        bx.dbg_var_value(
+            dbg_var,
+            dbg_loc,
+            base.llval,
+            direct_offset,
+            &indirect_offsets,
+            var.fragment,
+        );
+    }
+
     /// Apply debuginfo and/or name, after creating the `alloca` for a local,
     /// or initializing the local with an operand (whichever applies).
     pub(crate) fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) {

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -1,13 +1,17 @@
-use rustc_middle::mir::{self, NonDivergingIntrinsic};
-use rustc_middle::span_bug;
+use rustc_middle::mir::{self, NonDivergingIntrinsic, StmtDebugInfo};
+use rustc_middle::{bug, span_bug};
 use tracing::instrument;
 
 use super::{FunctionCx, LocalRef};
+use crate::common::TypeKind;
+use crate::mir::operand::OperandValue;
+use crate::mir::place::PlaceRef;
 use crate::traits::*;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[instrument(level = "debug", skip(self, bx))]
     pub(crate) fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) {
+        self.codegen_stmt_debuginfos(bx, &statement.debuginfos);
         self.set_debug_loc(bx, statement.source_info);
         match statement.kind {
             mir::StatementKind::Assign(box (ref place, ref rvalue)) => {
@@ -101,4 +105,74 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             | mir::StatementKind::Nop => {}
         }
     }
+
+    pub(crate) fn codegen_stmt_debuginfo(&mut self, bx: &mut Bx, debuginfo: &StmtDebugInfo<'tcx>) {
+        match debuginfo {
+            StmtDebugInfo::AssignRef(dest, place) => {
+                let local_ref = match self.locals[place.local] {
+                    LocalRef::Place(place_ref) | LocalRef::UnsizedPlace(place_ref) => {
+                        Some(place_ref)
+                    }
+                    LocalRef::Operand(operand_ref) => match operand_ref.val {
+                        OperandValue::Immediate(v) => {
+                            Some(PlaceRef::new_sized(v, operand_ref.layout))
+                        }
+                        OperandValue::Ref(_)
+                        | OperandValue::Pair(_, _)
+                        | OperandValue::ZeroSized => None,
+                    },
+                    LocalRef::PendingOperand => None,
+                }
+                .filter(|place_ref| {
+                    // Drop unsupported projections.
+                    // FIXME: Add a test case.
+                    place.projection.iter().all(|p| p.can_use_in_debuginfo()) &&
+                        // Only pointers can calculate addresses.
+                        bx.type_kind(bx.val_ty(place_ref.val.llval)) == TypeKind::Pointer
+                });
+                let assign_ref = match (local_ref, place.is_indirect_first_projection()) {
+                    (Some(local_ref), false) => {
+                        Some((local_ref.val, local_ref.layout, place.projection.as_slice()))
+                    }
+                    (Some(local_ref), true) => {
+                        let projected_ty = local_ref
+                            .layout
+                            .ty
+                            .builtin_deref(true)
+                            .unwrap_or_else(|| bug!("deref of non-pointer {:?}", local_ref));
+                        let layout = bx.cx().layout_of(projected_ty);
+                        Some((local_ref.val, layout, &place.projection[1..]))
+                    }
+                    _ => None,
+                };
+                let (val, layout, projection) = assign_ref.unwrap_or_else(|| {
+                    // If the address cannot be computed, use poison to indicate that the value has been optimized out.
+                    let ty = self.monomorphize(self.mir.local_decls[*dest].ty);
+                    let layout = bx.cx().layout_of(ty);
+                    let to_backend_ty = bx.cx().immediate_backend_type(layout);
+                    let place_ref =
+                        PlaceRef::new_sized(bx.cx().const_poison(to_backend_ty), layout);
+                    (place_ref.val, layout, [].as_slice())
+                });
+                self.debug_new_value_to_local(bx, *dest, val, layout, projection);
+            }
+            StmtDebugInfo::InvalidAssign(local) => {
+                let ty = self.monomorphize(self.mir.local_decls[*local].ty);
+                let layout = bx.cx().layout_of(ty);
+                let to_backend_ty = bx.cx().immediate_backend_type(layout);
+                let place_ref = PlaceRef::new_sized(bx.cx().const_poison(to_backend_ty), layout);
+                self.debug_new_value_to_local(bx, *local, place_ref.val, layout, &[]);
+            }
+        }
+    }
+
+    pub(crate) fn codegen_stmt_debuginfos(
+        &mut self,
+        bx: &mut Bx,
+        debuginfos: &[StmtDebugInfo<'tcx>],
+    ) {
+        for debuginfo in debuginfos {
+            self.codegen_stmt_debuginfo(bx, debuginfo);
+        }
+    }
 }

compiler/rustc_codegen_ssa/src/traits/debuginfo.rs

@@ -79,6 +79,18 @@ pub trait DebugInfoBuilderMethods: BackendTypes {
         // if this is a fragment of a composite `DIVariable`.
         fragment: Option<Range<Size>>,
     );
+    fn dbg_var_value(
+        &mut self,
+        dbg_var: Self::DIVariable,
+        dbg_loc: Self::DILocation,
+        value: Self::Value,
+        direct_offset: Size,
+        // NB: each offset implies a deref (i.e. they're steps in a pointer chain).
+        indirect_offsets: &[Size],
+        // Byte range in the `dbg_var` covered by this fragment,
+        // if this is a fragment of a composite `DIVariable`.
+        fragment: Option<Range<Size>>,
+    );
     fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation);
     fn clear_dbg_loc(&mut self);
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self);

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -58,6 +58,7 @@ using namespace llvm::object;
 // This opcode is an LLVM detail that could hypothetically change (?), so
 // verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM.
 static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000);
+static_assert(dwarf::DW_OP_stack_value == 0x9f);
 
 // LLVMAtomicOrdering is already an enum - don't create another
 // one.
@@ -1241,6 +1242,18 @@ LLVMRustDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef V,
       DebugLoc(cast<MDNode>(unwrap(DL))), unwrap(InsertAtEnd));
 }
 
+extern "C" void
+LLVMRustDIBuilderInsertDbgValueAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef V,
+                                     LLVMMetadataRef VarInfo, uint64_t *AddrOps,
+                                     unsigned AddrOpsCount, LLVMMetadataRef DL,
+                                     LLVMBasicBlockRef InsertAtEnd) {
+  unwrap(Builder)->insertDbgValueIntrinsic(
+      unwrap(V), unwrap<DILocalVariable>(VarInfo),
+      unwrap(Builder)->createExpression(
+          llvm::ArrayRef<uint64_t>(AddrOps, AddrOpsCount)),
+      DebugLoc(cast<MDNode>(unwrap(DL))), unwrap(InsertAtEnd));
+}
+
 extern "C" LLVMMetadataRef
 LLVMRustDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder, const char *Name,
                                   size_t NameLen, const uint64_t Value[2],

compiler/rustc_middle/src/mir/mod.rs

@@ -1342,6 +1342,10 @@ pub struct BasicBlockData<'tcx> {
     /// List of statements in this block.
     pub statements: Vec<Statement<'tcx>>,
 
+    /// All debuginfos happen before the statement.
+    /// Put debuginfos here when the last statement is eliminated.
+    pub after_last_stmt_debuginfos: StmtDebugInfos<'tcx>,
+
     /// Terminator for this block.
     ///
     /// N.B., this should generally ONLY be `None` during construction.
@@ -1369,7 +1373,12 @@ impl<'tcx> BasicBlockData<'tcx> {
         terminator: Option<Terminator<'tcx>>,
         is_cleanup: bool,
     ) -> BasicBlockData<'tcx> {
-        BasicBlockData { statements, terminator, is_cleanup }
+        BasicBlockData {
+            statements,
+            after_last_stmt_debuginfos: StmtDebugInfos::default(),
+            terminator,
+            is_cleanup,
+        }
     }
 
     /// Accessor for terminator.
@@ -1404,6 +1413,36 @@ impl<'tcx> BasicBlockData<'tcx> {
             self.terminator().successors()
         }
     }
+
+    pub fn retain_statements<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&Statement<'tcx>) -> bool,
+    {
+        // Place debuginfos into the next retained statement,
+        // this `debuginfos` variable is used to cache debuginfos between two retained statements.
+        let mut debuginfos = StmtDebugInfos::default();
+        self.statements.retain_mut(|stmt| {
+            let retain = f(stmt);
+            if retain {
+                stmt.debuginfos.prepend(&mut debuginfos);
+            } else {
+                debuginfos.append(&mut stmt.debuginfos);
+            }
+            retain
+        });
+        self.after_last_stmt_debuginfos.prepend(&mut debuginfos);
+    }
+
+    pub fn strip_nops(&mut self) {
+        self.retain_statements(|stmt| !matches!(stmt.kind, StatementKind::Nop))
+    }
+
+    pub fn drop_debuginfo(&mut self) {
+        self.after_last_stmt_debuginfos.drop_debuginfo();
+        for stmt in self.statements.iter_mut() {
+            stmt.debuginfos.drop_debuginfo();
+        }
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -1708,10 +1747,10 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 128);
+    static_assert_size!(BasicBlockData<'_>, 152);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
-    static_assert_size!(Statement<'_>, 32);
+    static_assert_size!(Statement<'_>, 56);
     static_assert_size!(Terminator<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end