Rollup merge of rust-lang#159625 - camsteffen:opsem-inhabited-refactor, r=RalfJung

Refactor is_opsem_inhabited

Refactor of rust-lang#156977

Introduce a struct to generally simplify the recursion.

r? WaffleLapkin
cc @RalfJung

Author: jhpratt

compiler/rustc_middle/src/ty/inhabitedness/mod.rs

@@ -51,7 +51,7 @@ use rustc_type_ir::TyKind::*;
 use tracing::instrument;
 
 use crate::query::Providers;
-use crate::ty::{self, DefId, Ty, TyCtxt, TypeVisitableExt, VariantDef, Visibility};
+use crate::ty::{self, DefId, Ty, TyCtxt, TypeVisitableExt, TypingEnv, VariantDef, Visibility};
 
 pub mod inhabited_predicate;
 
@@ -221,10 +221,7 @@ impl<'tcx> Ty<'tcx> {
     /// Beyond that, the value returned by this function is not a stable guarantee.
     pub fn is_opsem_inhabited(self, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool {
         // Handle simple cases directly, use the query with its cache for the rest.
-        is_opsem_inhabited_recursor(self, tcx, &mut (), /* stop_at_ref */ false, &|ty, _, _| {
-            // ADT handler: stop recursing, invoke the query.
-            tcx.is_opsem_inhabited_raw(typing_env.as_query_input(ty))
-        })
+        OpsemInhabitedCtx { tcx, typing_env, seen: None, stop_at_ref: false }.is_inhabited_ty(self)
     }
 }
 
@@ -249,109 +246,129 @@ fn inhabited_predicate_type<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> InhabitedP
     }
 }
 
-/// Recurse over a type to determine whether it is inhabited on the opsem level.
+/// Context for computing whether a type is inhabited on the opsem level.
 /// See `is_opsem_inhabited` above for the spec of what we compute.
-///
-/// When we encounter an ADT, we call `adt_handler`, giving it as its last argument a closure that
-/// it can invoke to continue the recursion. This lets us share the logic for "simple" cases
-/// (i.e., everything except for ADTs) between `Ty::is_opsem_inhabited` and the query.
-///
-/// `seen` is used to detect infinite recursion: the set contains all ADTs that we encountered
-/// on our path to the current type.
-/// If `stop_at_ref` is true, we stop recursing at the next reference we encounter.
-fn is_opsem_inhabited_recursor<'tcx, SEEN>(
-    ty: Ty<'tcx>,
+struct OpsemInhabitedCtx<'tcx> {
     tcx: TyCtxt<'tcx>,
-    seen: &mut SEEN,
+    typing_env: TypingEnv<'tcx>,
+    /// IDs of ADTs that have been encountered in the current stack.
+    /// It's `None` unless we are inside the `is_opsem_inhabited_raw` query,
+    /// which is only invoked for more complex types.
+    seen: Option<FxHashSet<DefId>>,
+    /// If an ADT is encountered recursively within itself, then `stop_at_ref`
+    /// is set to `true`, and then any nested references are considered inhabited.
     stop_at_ref: bool,
-    adt_handler: &impl Fn(
-        Ty<'tcx>,
-        &mut SEEN,
-        &dyn Fn(Ty<'tcx>, &mut SEEN, /* stop_at_ref */ bool) -> bool,
-    ) -> bool,
-) -> bool {
-    match *ty.kind() {
-        // Trivially (un)inhabited types
-        ty::Int(_)
-        | ty::Uint(_)
-        | ty::Float(_)
-        | ty::Bool
-        | ty::Char
-        | ty::Str
-        | ty::Foreign(..)
-        | ty::RawPtr(..)
-        | ty::FnPtr(..)
-        | ty::FnDef(..) => true,
-        ty::Dynamic(..) => true, // We can't reason about traits, assume they are inhabited
-        ty::Slice(..) => true,   // Slices can always be empty
-        ty::Never => false,
+}
 
-        // Types where we recurse
-        ty::Ref(_, pointee, _) => {
-            if stop_at_ref {
-                // Bailing out here is safe as the layout code always considers references
-                // inhabited, so the implication ("layout uninhabited => opsem uninhabited")
-                // is upheld.
-                return true;
+impl<'tcx> OpsemInhabitedCtx<'tcx> {
+    /// See `is_opsem_inhabited` above for the spec of what we compute.
+    fn is_inhabited_ty(&mut self, ty: Ty<'tcx>) -> bool {
+        let tcx = self.tcx;
+        match *ty.kind() {
+            // Trivially (un)inhabited types
+            ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Bool
+            | ty::Char
+            | ty::Str
+            | ty::Foreign(..)
+            | ty::RawPtr(..)
+            | ty::FnPtr(..)
+            | ty::FnDef(..) => true,
+            ty::Dynamic(..) => true, // We can't reason about traits, assume they are inhabited
+            ty::Slice(..) => true,   // Slices can always be empty
+            ty::Never => false,
+
+            // Types where we recurse
+            ty::Ref(_, pointee, _) => {
+                if self.stop_at_ref {
+                    // Bailing out here is safe as the layout code always considers references
+                    // inhabited, so the implication ("layout uninhabited => opsem uninhabited")
+                    // is upheld.
+                    return true;
+                }
+                self.is_inhabited_ty(pointee)
+            }
+            ty::Tuple(tys) => tys.iter().all(|ty| self.is_inhabited_ty(ty)),
+            ty::Array(elem, len) => {
+                len.try_to_target_usize(tcx).unwrap() == 0 || self.is_inhabited_ty(elem)
+            }
+            ty::Pat(inner, _pat) => self.is_inhabited_ty(inner),
+            ty::Closure(_def, args) => {
+                let args = args.as_closure();
+                args.upvar_tys().iter().all(|ty| self.is_inhabited_ty(ty))
+            }
+            ty::Coroutine(_def, args) => {
+                let args = args.as_coroutine();
+                args.upvar_tys().iter().all(|ty| self.is_inhabited_ty(ty))
+            }
+            ty::CoroutineClosure(_def, args) => {
+                let args = args.as_coroutine_closure();
+                args.upvar_tys().iter().all(|ty| self.is_inhabited_ty(ty))
+            }
+            ty::UnsafeBinder(base) => {
+                let base = tcx.instantiate_bound_regions_with_erased((*base).into());
+                self.is_inhabited_ty(base)
+            }
+            ty::Adt(..) => self.is_inhabited_adt_ty(ty),
+
+            ty::Error(_error_guaranteed) => {
+                // We have a token proving there was an error, so we can return a dummy value.
+                true
+            }
+
+            ty::Infer(..)
+            | ty::Placeholder(..)
+            | ty::Bound(..)
+            | ty::Param(..)
+            | ty::Alias(..)
+            | ty::CoroutineWitness(..) => {
+                bug!("non-normalized type in `is_opsem_uninhabited`: `{ty}`")
             }
-            is_opsem_inhabited_recursor(pointee, tcx, seen, stop_at_ref, adt_handler)
-        }
-        ty::Tuple(tys) => tys
-            .iter()
-            .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler)),
-        ty::Array(elem, len) => {
-            len.try_to_target_usize(tcx).unwrap() == 0
-                || is_opsem_inhabited_recursor(elem, tcx, seen, stop_at_ref, adt_handler)
-        }
-        ty::Pat(inner, _pat) => {
-            is_opsem_inhabited_recursor(inner, tcx, seen, stop_at_ref, adt_handler)
-        }
-        ty::Closure(_def, args) => {
-            let args = args.as_closure();
-            args.upvar_tys()
-                .iter()
-                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
-        }
-        ty::Coroutine(_def, args) => {
-            let args = args.as_coroutine();
-            args.upvar_tys()
-                .iter()
-                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
-        }
-        ty::CoroutineClosure(_def, args) => {
-            let args = args.as_coroutine_closure();
-            args.upvar_tys()
-                .iter()
-                .all(|ty| is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler))
         }
-        ty::UnsafeBinder(base) => {
-            let base = tcx.instantiate_bound_regions_with_erased((*base).into());
-            is_opsem_inhabited_recursor(base, tcx, seen, stop_at_ref, adt_handler)
+    }
+
+    fn is_inhabited_adt_ty(&mut self, ty: Ty<'tcx>) -> bool {
+        let ty::Adt(adt_def, adt_args) = *ty.kind() else {
+            unreachable! {}
+        };
+        let Self { tcx, typing_env, .. } = *self;
+
+        if adt_def.is_union() {
+            // Unions are always inhabited.
+            return true;
         }
-        ty::Adt(..) => {
-            // ADTs need a special handler to avoid infinite recursion. That handler is meant to
-            // call back into the recursor. Ideally it'd just call `is_opsem_inhabited_recursor` but
-            // then it would have to pass itself as the adt_handler argument which is not possible
-            // in Rust... so we provide the handler with a callback that it can use to continue the
-            // recursion with the same `adt_handler`.
-            adt_handler(ty, seen, &|ty, seen, stop_at_ref| {
-                is_opsem_inhabited_recursor(ty, tcx, seen, stop_at_ref, adt_handler)
+
+        let Some(seen) = self.seen.as_mut() else {
+            // stop recursing, invoke the query.
+            return tcx.is_opsem_inhabited_raw(typing_env.as_query_input(ty));
+        };
+
+        let new_adt = seen.insert(adt_def.did());
+        // If we have seen this ADT before, stop at the next reference to avoid infinite
+        // recursion. We can't stop here since we have to ensure that "layout uninhabited"
+        // implies "opsem uninhabited". References are always layout-inhabited so the
+        // implication is vacuously true.
+        let stop_at_ref_prev = self.stop_at_ref;
+        self.stop_at_ref |= !new_adt;
+
+        // We are inhabited if in some variant all fields are inhabited.
+        let inhabited = adt_def.variants().iter().any(|variant| {
+            variant.fields.iter().all(|field| {
+                let ty = field.ty(tcx, adt_args);
+                let ty = tcx.normalize_erasing_regions(typing_env, ty);
+                self.is_inhabited_ty(ty)
             })
-        }
+        });
 
-        ty::Error(_error_guaranteed) => {
-            // We have a token proving there was an error, so we can return a dummy value.
-            true
+        self.stop_at_ref = stop_at_ref_prev;
+        // Remove the type again so that we allow it to appear on other branches.
+        if new_adt {
+            self.seen.as_mut().unwrap().remove(&adt_def.did());
         }
 
-        ty::Infer(..)
-        | ty::Placeholder(..)
-        | ty::Bound(..)
-        | ty::Param(..)
-        | ty::Alias(..)
-        | ty::CoroutineWitness(..) => {
-            bug!("non-normalized type in `is_opsem_uninhabited`: `{ty}`")
-        }
+        inhabited
     }
 }
 
@@ -366,42 +383,6 @@ fn is_opsem_inhabited_raw<'tcx>(
         "the query should only be invoked by `Ty::is_opsem_inhabited`"
     );
 
-    is_opsem_inhabited_recursor(
-        ty,
-        tcx,
-        &mut FxHashSet::<DefId>::default(),
-        /* stop_at_ref */ false,
-        &|ty, seen, rec| {
-            let ty::Adt(adt_def, adt_args) = *ty.kind() else {
-                unreachable! {}
-            };
-            if adt_def.is_union() {
-                // Unions are always inhabited.
-                return true;
-            }
-
-            let new_adt = seen.insert(adt_def.did());
-            // If we have seen this ADT before, stop at the next reference to avoid infinite
-            // recursion. We can't stop here since we have to ensure that "layout uninhabited"
-            // implies "opsem uninhabited". References are always layout-inhabited so the
-            // implication is vacuously true.
-            let stop_at_ref = !new_adt;
-
-            // We are inhabited if in some variant all fields are inhabited.
-            let inhabited = adt_def.variants().iter().any(|variant| {
-                variant.fields.iter().all(|field| {
-                    let ty = field.ty(tcx, adt_args);
-                    let ty = tcx.normalize_erasing_regions(typing_env, ty);
-                    rec(ty, seen, stop_at_ref)
-                })
-            });
-
-            // Remove the type again so that we allow it to appear on other branches.
-            if new_adt {
-                seen.remove(&adt_def.did());
-            }
-
-            inhabited
-        },
-    )
+    OpsemInhabitedCtx { tcx, typing_env, seen: Some(FxHashSet::default()), stop_at_ref: false }
+        .is_inhabited_adt_ty(ty)
 }