[TOY] Run MIR opts before coroutine state transform

compiler/rustc_middle/src/query/mod.rs

@@ -2651,11 +2651,11 @@ rustc_queries! {
     }
 
     query generics_require_sized_self(def_id: DefId) -> bool {
-        desc { "check whether the item has a `where Self: Sized` bound" }
+        desc { |tcx| "check whether `{}` has a `where Self: Sized` bound", tcx.def_path_str(def_id) }
     }
 
     query cross_crate_inlinable(def_id: DefId) -> bool {
-        desc { "whether the item should be made inlinable across crates" }
+        desc { |tcx| "whether `{}` should be made inlinable across crates", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
 

compiler/rustc_mir_transform/src/coroutine/drop.rs

@@ -126,6 +126,7 @@ fn build_pin_fut<'tcx>(
 //     Ready() => ready_block
 //     Pending => yield_block
 //}
+#[tracing::instrument(level = "trace", skip(tcx, body), ret)]
 fn build_poll_switch<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
@@ -179,6 +180,7 @@ fn build_poll_switch<'tcx>(
 }
 
 // Gather blocks, reachable through 'drop' targets of Yield and Drop terminators (chained)
+#[tracing::instrument(level = "trace", skip(body), ret)]
 fn gather_dropline_blocks<'tcx>(body: &mut Body<'tcx>) -> DenseBitSet<BasicBlock> {
     let mut dropline: DenseBitSet<BasicBlock> = DenseBitSet::new_empty(body.basic_blocks.len());
     for (bb, data) in traversal::reverse_postorder(body) {
@@ -249,6 +251,7 @@ pub(super) fn has_expandable_async_drops<'tcx>(
 }
 
 /// Expand Drop terminator for async drops into mainline poll-switch and dropline poll-switch
+#[tracing::instrument(level = "trace", skip(tcx, body), ret)]
 pub(super) fn expand_async_drops<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
@@ -259,6 +262,7 @@ pub(super) fn expand_async_drops<'tcx>(
     let dropline = gather_dropline_blocks(body);
     // Clean drop and async_fut fields if potentially async drop is not expanded (stays sync)
     let remove_asyncness = |block: &mut BasicBlockData<'tcx>| {
+        tracing::trace!("remove_asyncness");
         if let TerminatorKind::Drop {
             place: _,
             target: _,
@@ -273,6 +277,7 @@ pub(super) fn expand_async_drops<'tcx>(
         }
     };
     for bb in START_BLOCK..body.basic_blocks.next_index() {
+        tracing::trace!(?bb);
         // Drops in unwind path (cleanup blocks) are not expanded to async drops, only sync drops in unwind path
         if body[bb].is_cleanup {
             remove_asyncness(&mut body[bb]);
@@ -461,6 +466,7 @@ pub(super) fn expand_async_drops<'tcx>(
     }
 }
 
+#[tracing::instrument(level = "trace", skip(tcx, body))]
 pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     use crate::elaborate_drop::{Unwind, elaborate_drop};
     use crate::patch::MirPatch;
@@ -519,6 +525,7 @@ pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body
     elaborator.patch.apply(body);
 }
 
+#[tracing::instrument(level = "trace", skip(tcx, body), ret)]
 pub(super) fn insert_clean_drop<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
@@ -550,6 +557,7 @@ pub(super) fn insert_clean_drop<'tcx>(
         .push(BasicBlockData::new(Some(Terminator { source_info, kind: term }), false))
 }
 
+#[tracing::instrument(level = "trace", skip(tcx, transform, body))]
 pub(super) fn create_coroutine_drop_shim<'tcx>(
     tcx: TyCtxt<'tcx>,
     transform: &TransformVisitor<'tcx>,
@@ -619,6 +627,7 @@ pub(super) fn create_coroutine_drop_shim<'tcx>(
 }
 
 // Create async drop shim function to drop coroutine itself
+#[tracing::instrument(level = "trace", skip(tcx, transform, body))]
 pub(super) fn create_coroutine_drop_shim_async<'tcx>(
     tcx: TyCtxt<'tcx>,
     transform: &TransformVisitor<'tcx>,

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -92,6 +92,11 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         InliningThreshold::Never => return false,
     };
 
+    // Avoid inlining coroutines, since that does not make any sense.
+    if tcx.is_coroutine(def_id.to_def_id()) {
+        return false;
+    }
+
     let mir = tcx.optimized_mir(def_id);
     let mut checker =
         CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -46,6 +46,12 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
             return;
         }
 
+        // Avoid computing layout inside coroutines, since their `optimized_mir` is used for layout
+        // computation, which can create a cycle.
+        if body.coroutine.is_some() {
+            return;
+        }
+
         // We want to have a somewhat linear runtime w.r.t. the number of statements/terminators.
         // Let's call this number `n`. Dataflow analysis has `O(h*n)` transfer function
         // applications, where `h` is the height of the lattice. Because the height of our lattice
@@ -241,9 +247,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
             TerminatorKind::Drop { place, .. } => {
                 state.flood_with(place.as_ref(), &self.map, FlatSet::<Scalar>::BOTTOM);
             }
-            TerminatorKind::Yield { .. } => {
-                // They would have an effect, but are not allowed in this phase.
-                bug!("encountered disallowed terminator");
+            TerminatorKind::Yield { resume_arg, .. } => {
+                state.flood_with(resume_arg.as_ref(), &self.map, FlatSet::<Scalar>::BOTTOM);
             }
             TerminatorKind::SwitchInt { discr, targets } => {
                 return self.handle_switch_int(discr, targets, state);

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -639,6 +639,10 @@ impl WriteInfo {
                     self.add_operand(&arg.node);
                 }
             }
+            TerminatorKind::Yield { resume_arg, value, .. } => {
+                self.add_place(*resume_arg);
+                self.add_operand(value);
+            }
             TerminatorKind::InlineAsm { operands, .. } => {
                 for asm_operand in operands {
                     match asm_operand {
@@ -669,14 +673,12 @@ impl WriteInfo {
             | TerminatorKind::UnwindResume
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
+            | TerminatorKind::CoroutineDrop
             | TerminatorKind::Unreachable { .. } => (),
             TerminatorKind::Drop { .. } => {
                 // `Drop`s create a `&mut` and so are not considered
             }
-            TerminatorKind::Yield { .. }
-            | TerminatorKind::CoroutineDrop
-            | TerminatorKind::FalseEdge { .. }
-            | TerminatorKind::FalseUnwind { .. } => {
+            TerminatorKind::FalseEdge { .. } | TerminatorKind::FalseUnwind { .. } => {
                 bug!("{:?} not found in this MIR phase", terminator)
             }
         }

compiler/rustc_mir_transform/src/gvn.rs

@@ -1767,14 +1767,18 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
     }
 
     fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) {
-        if let Terminator { kind: TerminatorKind::Call { destination, .. }, .. } = terminator {
-            if let Some(local) = destination.as_local()
-                && self.ssa.is_ssa(local)
-            {
-                let ty = self.local_decls[local].ty;
-                let opaque = self.new_opaque(ty);
-                self.assign(local, opaque);
-            }
+        let destination = match terminator.kind {
+            TerminatorKind::Call { destination, .. } => Some(destination),
+            TerminatorKind::Yield { resume_arg, .. } => Some(resume_arg),
+            _ => None,
+        };
+        if let Some(destination) = destination
+            && let Some(local) = destination.as_local()
+            && self.ssa.is_ssa(local)
+        {
+            let ty = self.local_decls[local].ty;
+            let opaque = self.new_opaque(ty);
+            self.assign(local, opaque);
         }
         // Function calls and ASM may invalidate (nested) derefs. We must handle them carefully.
         // Currently, only preserving derefs for trivial terminators like SwitchInt and Goto.

compiler/rustc_mir_transform/src/inline.rs

@@ -349,14 +349,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
         }
     }
 
-    fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool {
-        // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
-        // which can create a cycle, even when no attempt is made to inline the function in the other
-        // direction.
-        if body.coroutine.is_some() {
-            return false;
-        }
-
+    fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool {
         true
     }
 
@@ -709,6 +702,11 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
         return Err("self-recursion");
     }
 
+    // Avoid inlining coroutines, since that does not make any sense.
+    if inliner.tcx().is_coroutine(callee_def_id) {
+        return Err("coroutine");
+    }
+
     match callee.def {
         InstanceKind::Item(_) => {
             // If there is no MIR available (either because it was not in metadata or

compiler/rustc_mir_transform/src/inline/cycle.rs

@@ -94,6 +94,15 @@ fn process<'tcx>(
             continue;
         }
 
+        // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
+        // which can create a cycle, even when no attempt is made to inline the function in the other
+        // direction.
+        if tcx.is_coroutine(callee.def_id()) {
+            reaches_root = true;
+            seen.insert(callee, true);
+            continue;
+        }
+
         if tcx.is_constructor(callee.def_id()) {
             trace!("constructors always have MIR");
             // Constructor functions cannot cause a query cycle.

compiler/rustc_mir_transform/src/jump_threading.rs

@@ -612,9 +612,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             | TerminatorKind::Unreachable
             | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"),
             // Disallowed during optimizations.
-            TerminatorKind::FalseEdge { .. }
-            | TerminatorKind::FalseUnwind { .. }
-            | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"),
+            TerminatorKind::FalseEdge { .. } | TerminatorKind::FalseUnwind { .. } => {
+                bug!("{term:?} invalid")
+            }
             // Cannot reason about inline asm.
             TerminatorKind::InlineAsm { .. } => return,
             // `SwitchInt` is handled specially.
@@ -623,6 +623,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             TerminatorKind::Goto { .. } => None,
             // Flood the overwritten place, and progress through.
             TerminatorKind::Drop { place: destination, .. }
+            | TerminatorKind::Yield { resume_arg: destination, .. }
             | TerminatorKind::Call { destination, .. } => Some(destination),
             // Ignore, as this can be a no-op at codegen time.
             TerminatorKind::Assert { .. } => None,

compiler/rustc_mir_transform/src/lib.rs

@@ -616,7 +616,6 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         // Otherwise it should run fairly late, but before optimizations begin.
         &add_retag::AddRetag,
         &elaborate_box_derefs::ElaborateBoxDerefs,
-        &coroutine::StateTransform,
         &Lint(known_panics_lint::KnownPanicsLint),
     ];
     pm::run_passes_no_validate(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::Initial)));
@@ -722,6 +721,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
             &simplify::SimplifyLocals::Final,
             &multiple_return_terminators::MultipleReturnTerminators,
             &large_enums::EnumSizeOpt { discrepancy: 128 },
+            &coroutine::StateTransform,
             // Some cleanup necessary at least for LLVM and potentially other codegen backends.
             &add_call_guards::CriticalCallEdges,
             // Cleanup for human readability, off by default.

compiler/rustc_mir_transform/src/shim.rs

@@ -149,7 +149,6 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
                     tcx,
                     &mut body,
                     &[
-                        &mentioned_items::MentionedItems,
                         &abort_unwinding_calls::AbortUnwindingCalls,
                         &add_call_guards::CriticalCallEdges,
                     ],

compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs

@@ -216,6 +216,7 @@ fn build_adrop_for_coroutine_shim<'tcx>(
     body.source.instance = instance;
     body.phase = MirPhase::Runtime(RuntimePhase::Initial);
     body.var_debug_info.clear();
+    body.mentioned_items = None;
     let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span));
     let args = tcx.mk_args(&[proxy_ref.into()]);
     let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args);

compiler/rustc_mir_transform/src/ssa.rs

@@ -228,6 +228,11 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor<'_, 'tcx> {
         match ctxt {
             PlaceContext::MutatingUse(MutatingUseContext::Projection)
             | PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) => bug!(),
+            // Count drops as moves.
+            PlaceContext::MutatingUse(MutatingUseContext::Drop) => {
+                self.check_dominates(local, loc);
+                self.direct_uses[local] += 1;
+            }
             // Anything can happen with raw pointers, so remove them.
             PlaceContext::NonMutatingUse(NonMutatingUseContext::RawBorrow)
             | PlaceContext::MutatingUse(_) => {

compiler/rustc_mir_transform/src/validate.rs

@@ -77,7 +77,7 @@
 
         // Also run the TypeChecker.
         for (location, msg) in validate_types(tcx, typing_env, body, body) {
             cfg_checker.fail(location, msg);
         }
 
         if let MirPhase::Runtime(_) = body.phase
@@ -450,7 +450,7 @@
                 if self.body.coroutine.is_none() {
                     self.fail(location, "`Yield` cannot appear outside coroutine bodies");
                 }
-                if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) {
+                if self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) {
                     self.fail(location, "`Yield` should have been replaced by coroutine lowering");
                 }
                 self.check_edge(location, *resume, EdgeKind::Normal);
@@ -488,7 +488,7 @@
                 if self.body.coroutine.is_none() {
                     self.fail(location, "`CoroutineDrop` cannot appear outside coroutine bodies");
                 }
-                if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) {
+                if self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) {
                     self.fail(
                         location,
                         "`CoroutineDrop` should have been replaced by coroutine lowering",