diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index b704d8f0a7692..df7767fe99432 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -1,4 +1,4 @@ -use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet}; use rustc_index::bit_set::DenseBitSet; use rustc_index::interval::IntervalSet; use rustc_infer::infer::canonical::QueryRegionConstraints; @@ -7,10 +7,10 @@ use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, HasLocalDecls, Loc use rustc_middle::traits::query::DropckOutlivesResult; use rustc_middle::ty::relate::Relate; use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt}; -use rustc_mir_dataflow::impls::MaybeInitializedPlaces; -use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex}; +use rustc_mir_dataflow::impls::{FilteringMovePathIndexMapper, MaybeInitializedPlaces2}; +use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex, SparseMovePathIndex}; use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; -use rustc_mir_dataflow::{Analysis, ResultsCursor}; +use rustc_mir_dataflow::{Analysis, ResultsCursor, on_all_children_bits}; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::traits::ObligationCtxt; @@ -45,6 +45,25 @@ pub(super) fn trace<'tcx>( boring_locals: Vec, ) { let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, typeck.body); + + let mut dense_mpis = FxHashMap::default(); + // let mut dense_mpis = smallvec::SmallVec::new(); + for &local in relevant_live_locals.iter() { + let Some(mpi) = move_data.rev_lookup.find_local(local) else { unreachable!() }; + + // We only compute initializedness in drop-liveness on locals with drop points. + if local_use_map.drops(local).next().is_none() { + continue; + } + + on_all_children_bits(move_data, mpi, |child| { + // dense_mpis.push(child); + let dense_idx = dense_mpis.len(); + let sparse_idx = SparseMovePathIndex::from_u32(dense_idx.try_into().unwrap()); + dense_mpis.insert(child, sparse_idx); + }); + } + let cx = LivenessContext { typeck, flow_inits: None, @@ -52,6 +71,7 @@ pub(super) fn trace<'tcx>( local_use_map, move_data, drop_data: FxIndexMap::default(), + dense_mpis, }; let mut results = LivenessResults::new(cx); @@ -81,11 +101,16 @@ struct LivenessContext<'a, 'typeck, 'tcx> { /// Results of dataflow tracking which variables (and paths) have been /// initialized. Computed lazily when needed by drop-liveness. - flow_inits: Option>>, + flow_inits: Option< + ResultsCursor<'a, 'tcx, MaybeInitializedPlaces2<'a, 'tcx, FilteringMovePathIndexMapper>>, + >, /// Index indicating where each variable is assigned, used, or /// dropped. local_use_map: &'a LocalUseMap, + + // dense_mpis: smallvec::SmallVec<[MovePathIndex; 1]>, + dense_mpis: FxHashMap, } struct DropData<'tcx> { @@ -468,7 +493,10 @@ impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> { /// /// This happens as part of the drop-liveness computation: it's the only place checking for /// maybe-initializedness of `MovePathIndex`es. - fn flow_inits(&mut self) -> &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>> { + fn flow_inits( + &mut self, + ) -> &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces2<'a, 'tcx, FilteringMovePathIndexMapper>> + { self.flow_inits.get_or_insert_with(|| { let tcx = self.typeck.tcx(); let body = self.typeck.body; @@ -484,9 +512,86 @@ impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> { // a much, much smaller domain: in our benchmarks, when it's not zero (the most likely // case), there are a few dozens compared to e.g. thousands or tens of thousands of // locals and move paths. - let flow_inits = MaybeInitializedPlaces::new(tcx, body, self.move_data) + // eprintln!( + // "computing flow_inits: {} MPIs total, {} relevant MPIs", + // self.move_data.move_paths.len(), + // self.dense_mpis.len() + // ); + + // FIXME: use the sparse vec + bitset metadata trick instead of a map! + + // let map_timer = std::time::Instant::now(); + // let mut sparse_map = FxIndexMap::default(); + // for (idx, &dense_idx) in self.dense_mpis.iter().enumerate() { + // let sparse_idx = SparseMovePathIndex::from_usize(idx); + // sparse_map.insert(dense_idx, sparse_idx); + // } + // // for local in body.args_iter() { + // // let Some(mpi) = self.move_data.rev_lookup.find_local(local) else { unreachable!() }; + // // self.dense_mpis.push(mpi); + // // map.insert(mpi, self.dense_mpis.len()); + // // } + + // let map_len = sparse_map.len(); + // let mapper = FilteringMovePathIndexMapper { sparse_map }; + // let map_elapsed = map_timer.elapsed(); + + // // let mapper = rustc_mir_dataflow::impls::NoOpMapper; + + // let sparse_bitset_timer = std::time::Instant::now(); + + // // let mut sparse_bitset = Sparse32::new(self.dense_mpis.len()); + // // // FIXME: do this in asc order to keep idxes stable and not shuffle the vec inside + // // for &dense_idx in self.dense_mpis.iter() { + // // sparse_bitset.insert(dense_idx); + // // } + + // let sparse_bitset = Sparse32::new(&self.dense_mpis); + + // // let sparse_bitset_elapsed = sparse_bitset_timer.elapsed(); + // // let sparse_bitset_len = sparse_bitset.dense.len(); + + // // let sparse_bitset_timer = std::time::Instant::now(); + + // let mut sparse_bitset = Sparse::new( + // 1 + self.dense_mpis.iter().max().unwrap().as_usize(), + // self.dense_mpis.len(), + // ); + // // FIXME: do this in asc order to keep idxes stable and not shuffle the vec inside + // for &dense_idx in self.dense_mpis.iter() { + // sparse_bitset.insert(dense_idx); + // } + // // also: move this into the sparse ctor, so that prefixes can be computed there after inserting stuff + + // sparse_bitset.compute_prefixes(); + + // // let sparse_bitset_elapsed = sparse_bitset_timer.elapsed(); + // // let sparse_bitset_len = sparse_bitset.sparse.len(); + + // let mapper = sparse_bitset; + + // let timer = std::time::Instant::now(); + let x = std::mem::take(&mut self.dense_mpis); + let flow_inits = MaybeInitializedPlaces2::new(tcx, body, self.move_data) + .filter_move_paths(x) + // .with_mapper(mapper) .iterate_to_fixpoint(tcx, body, Some("borrowck")) .into_results_cursor(body); + // let elapsed = timer.elapsed(); + + // use std::sync::OnceLock; + // static PROFILE: OnceLock = OnceLock::new(); + // if *PROFILE.get_or_init(|| std::env::var("LETSGO1").is_ok()) { + // eprintln!( + // "flow_inits took {:?} ns, map of {} took: {} ns, sparse bitset of {} took {} ns, {:?}", + // elapsed.as_nanos(), + // map_len, + // map_elapsed.as_nanos(), + // sparse_bitset_len, + // sparse_bitset_elapsed.as_nanos(), + // body.span, + // ); + // } flow_inits }) } @@ -502,13 +607,24 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { /// the cursor to the desired location. fn initialized_at_curr_loc(&mut self, mpi: MovePathIndex) -> bool { let flow_inits = self.flow_inits(); + let analysis = flow_inits.analysis(); + let idx = analysis + .map_index(mpi) + .unwrap_or_else(|| unreachable!("dataflow is somehow missing MPI {mpi:?}")); let state = flow_inits.get(); - if state.contains(mpi) { + if state.contains(idx) { return true; } - let move_paths = &flow_inits.analysis().move_data().move_paths; - move_paths[mpi].find_descendant(move_paths, |mpi| state.contains(mpi)).is_some() + let move_paths = &analysis.move_data().move_paths; + move_paths[mpi] + .find_descendant(move_paths, |mpi| { + let idx = analysis + .map_index(mpi) + .unwrap_or_else(|| unreachable!("dataflow is somehow missing MPI {mpi:?}")); + state.contains(idx) + }) + .is_some() } /// Returns `true` if the local variable (or some part of it) is initialized in diff --git a/compiler/rustc_data_structures/src/fx.rs b/compiler/rustc_data_structures/src/fx.rs index f0db9623b674e..c1a5c8ebc7645 100644 --- a/compiler/rustc_data_structures/src/fx.rs +++ b/compiler/rustc_data_structures/src/fx.rs @@ -1,6 +1,6 @@ use std::hash::BuildHasherDefault; -pub use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; +pub use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet, FxHasher}; pub type StdEntry<'a, K, V> = std::collections::hash_map::Entry<'a, K, V>; diff --git a/compiler/rustc_mir_dataflow/src/framework/fmt.rs b/compiler/rustc_mir_dataflow/src/framework/fmt.rs index 38599cd094933..3976b27b5a302 100644 --- a/compiler/rustc_mir_dataflow/src/framework/fmt.rs +++ b/compiler/rustc_mir_dataflow/src/framework/fmt.rs @@ -250,3 +250,17 @@ where write!(f, "{}", ctxt.move_data().move_paths[*self]) } } + +impl DebugWithContext for crate::move_paths::SparseMovePathIndex {} +impl DebugWithContext for u32 {} + +// impl<'tcx, C> DebugWithContext for crate::move_paths::SparseMovePathIndex +// where +// C: crate::move_paths::HasMoveData<'tcx> + +// C: crate::impls::MovePathIndexMapper +// { +// fn fmt_with(&self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// let +// write!(f, "{}", ctxt.move_data().move_paths[*self]) +// } +// } diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs index 5937d68f389e4..44ea1f86a8e09 100644 --- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs +++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs @@ -21,14 +21,14 @@ use crate::{ // Used by both `MaybeInitializedPlaces` and `MaybeUninitializedPlaces`. pub struct MaybePlacesSwitchIntData<'tcx> { - enum_place: mir::Place<'tcx>, + pub(super) enum_place: mir::Place<'tcx>, discriminants: Vec<(VariantIdx, Discr<'tcx>)>, index: usize, } impl<'tcx> MaybePlacesSwitchIntData<'tcx> { /// Creates a `SmallVec` mapping each target in `targets` to its `VariantIdx`. - fn variants(&mut self, targets: &mir::SwitchTargets) -> SmallVec<[VariantIdx; 4]> { + pub(super) fn variants(&mut self, targets: &mir::SwitchTargets) -> SmallVec<[VariantIdx; 4]> { self.index = 0; targets.all_values().iter().map(|value| self.next_discr(value.get())).collect() } @@ -36,7 +36,7 @@ impl<'tcx> MaybePlacesSwitchIntData<'tcx> { // The discriminant order in the `SwitchInt` targets should match the order yielded by // `AdtDef::discriminants`. We rely on this to match each discriminant in the targets to its // corresponding variant in linear time. - fn next_discr(&mut self, value: u128) -> VariantIdx { + pub(super) fn next_discr(&mut self, value: u128) -> VariantIdx { // An out-of-bounds abort will occur if the discriminant ordering isn't as described above. loop { let (variant, discr) = self.discriminants[self.index]; @@ -49,7 +49,7 @@ impl<'tcx> MaybePlacesSwitchIntData<'tcx> { } impl<'tcx> MaybePlacesSwitchIntData<'tcx> { - fn new( + pub(super) fn new( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, block: mir::BasicBlock, diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized2.rs b/compiler/rustc_mir_dataflow/src/impls/initialized2.rs new file mode 100644 index 0000000000000..28b07a7015393 --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/initialized2.rs @@ -0,0 +1,401 @@ +use rustc_data_structures::fx::FxHashMap; +use rustc_index::Idx; +use rustc_index::bit_set::MixedBitSet; +use rustc_middle::mir::{ + self, Body, CallReturnPlaces, Location, SwitchTargetValue, TerminatorEdges, +}; +use rustc_middle::ty::TyCtxt; + +use crate::drop_flag_effects::{DropFlagState, InactiveVariants}; +use crate::impls::initialized::MaybePlacesSwitchIntData; +use crate::move_paths::{HasMoveData, LookupResult, MoveData, MovePathIndex, SparseMovePathIndex}; +use crate::{ + Analysis, GenKill, MaybeReachable, drop_flag_effects, drop_flag_effects_for_function_entry, + drop_flag_effects_for_location, on_all_children_bits, on_lookup_result_bits, +}; + +/// `MaybeInitializedPlaces` tracks all places that might be +/// initialized upon reaching a particular point in the control flow +/// for a function. +/// +/// For example, in code like the following, we have corresponding +/// dataflow information shown in the right-hand comments. +/// +/// ```rust +/// struct S; +/// #[rustfmt::skip] +/// fn foo(pred: bool) { // maybe-init: +/// // {} +/// let a = S; let mut b = S; let c; let d; // {a, b} +/// +/// if pred { +/// drop(a); // { b} +/// b = S; // { b} +/// +/// } else { +/// drop(b); // {a} +/// d = S; // {a, d} +/// +/// } // {a, b, d} +/// +/// c = S; // {a, b, c, d} +/// } +/// ``` +/// +/// To determine whether a place is *definitely* initialized at a +/// particular control-flow point, one can take the set-complement +/// of the data from `MaybeUninitializedPlaces` at the corresponding +/// control-flow point. +/// +/// Similarly, at a given `drop` statement, the set-intersection +/// between this data and `MaybeUninitializedPlaces` yields the set of +/// places that would require a dynamic drop-flag at that statement. +/// +/// When needed for efficiency purposes, a [`MovePathIndexMapper`] can +/// be used to only track a subset of places +/// ([`FilteringMovePathIndexMapper`]). The default, most common, case +/// is to compute maybe-initializedness for all places (via the +/// [`DefaultMovePathIndexMapper`] index mapper). +pub struct MaybeInitializedPlaces2<'a, 'tcx, M = DefaultMovePathIndexMapper> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + move_data: &'a MoveData<'tcx>, + exclude_inactive_in_otherwise: bool, + skip_unreachable_unwind: bool, + index_mapper: M, +} + +/// Maps between source `MovePathIndex`es and target `Idx`. The two common cases are +/// - filtering uninteresting move paths, described below. +/// - an identity mapping, using `MovePathIndex`es as-is. +/// +/// The dataflow engine, and MIR visitor infrastructure will still refer to elements +/// using `MovePathIndex`, as will code visiting the dataflow results, and only the analysis will +/// internally use the target indices. +/// +/// When filtering move paths, the source `MovePathIndex`es are considered the dense domain, and the +/// subset of "interesting" move paths is the sparse domain. The mapper accepts dense indices from the +/// datalflow engine and result-visiting code, and returns the associated sparse index. Note that +/// this mapping is optional: the point is that there are more uninteresting dense values than +/// interesting sparse values, and we want to use the smaller more efficien domain in the dataflow +/// analysis. +pub trait MovePathIndexMapper { + type TargetIndex: Idx; + + /// Returns the size of the mapped domain: + /// - either the original, dense, domain size: when not filtering any move paths, + /// - or, the sparse domain size: the subset of interesting move paths we want to track. + fn mapped_domain_size<'tcx>(&self, analysis: &impl HasMoveData<'tcx>) -> usize; + fn map_index(&self, source_idx: MovePathIndex) -> Option; +} + +/// This `MovePathIndex` mapper is the common case: +/// - it doesn't filter any paths, there's only the dense domain, and this mapping is an identity. +/// - we always use all `MovePathIndex`es outside and inside the datafkiw analysis. +pub struct DefaultMovePathIndexMapper; +impl MovePathIndexMapper for DefaultMovePathIndexMapper { + type TargetIndex = MovePathIndex; + + #[inline(always)] + fn mapped_domain_size<'tcx>(&self, analysis: &impl HasMoveData<'tcx>) -> usize { + analysis.move_data().move_paths.len() + } + + #[inline(always)] + fn map_index(&self, source_idx: MovePathIndex) -> Option { + Some(source_idx) + } +} + +/// This `MovePathIndex` filters move paths, and maps the source dense indices to the sparse indices +/// of interest. +pub struct FilteringMovePathIndexMapper { + /// Map of dense move paths to their associated sparse index. + sparse_map: FxHashMap, +} + +impl MovePathIndexMapper for FilteringMovePathIndexMapper { + type TargetIndex = SparseMovePathIndex; + + #[inline(always)] + fn mapped_domain_size<'tcx>(&self, _analysis: &impl HasMoveData<'tcx>) -> usize { + self.sparse_map.len() + } + + #[inline(always)] + fn map_index(&self, source_idx: MovePathIndex) -> Option { + self.sparse_map.get(&source_idx).copied() + } +} + +impl<'a, 'tcx> MaybeInitializedPlaces2<'a, 'tcx, DefaultMovePathIndexMapper> { + pub fn filter_move_paths( + self, + // interesting_move_paths: &[MovePathIndex], + sparse_map: FxHashMap, + ) -> MaybeInitializedPlaces2<'a, 'tcx, FilteringMovePathIndexMapper> { + // use rustc_data_structures::fx::FxBuildHasher; + // let mut sparse_map = + // FxHashMap::with_capacity_and_hasher(interesting_move_paths.len(), FxBuildHasher); + // for (idx, &dense_idx) in interesting_move_paths.iter().enumerate() { + // let sparse_idx = SparseMovePathIndex::from_u32(idx.try_into().unwrap()); + // sparse_map.insert(dense_idx, sparse_idx); + // } + let index_mapper = FilteringMovePathIndexMapper { sparse_map }; + MaybeInitializedPlaces2 { + tcx: self.tcx, + body: self.body, + move_data: self.move_data, + exclude_inactive_in_otherwise: self.exclude_inactive_in_otherwise, + skip_unreachable_unwind: self.skip_unreachable_unwind, + index_mapper, + } + } + + pub fn with_mapper( + self, + index_mapper: N, + ) -> MaybeInitializedPlaces2<'a, 'tcx, N> { + MaybeInitializedPlaces2 { + tcx: self.tcx, + body: self.body, + move_data: self.move_data, + exclude_inactive_in_otherwise: self.exclude_inactive_in_otherwise, + skip_unreachable_unwind: self.skip_unreachable_unwind, + index_mapper, + } + } +} + +impl<'a, 'tcx> MaybeInitializedPlaces2<'a, 'tcx, DefaultMovePathIndexMapper> { + pub fn new( + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + move_data: &'a MoveData<'tcx>, + ) -> MaybeInitializedPlaces2<'a, 'tcx, DefaultMovePathIndexMapper> { + MaybeInitializedPlaces2 { + tcx, + body, + move_data, + exclude_inactive_in_otherwise: false, + skip_unreachable_unwind: false, + index_mapper: DefaultMovePathIndexMapper, + } + } +} + +impl<'a, 'tcx, M: MovePathIndexMapper> MaybeInitializedPlaces2<'a, 'tcx, M> { + /// Ensures definitely inactive variants are excluded from the set of initialized places for + /// blocks reached through an `otherwise` edge. + pub fn exclude_inactive_in_otherwise(mut self) -> Self { + self.exclude_inactive_in_otherwise = true; + self + } + + pub fn skipping_unreachable_unwind(mut self) -> Self { + self.skip_unreachable_unwind = true; + self + } + + pub fn is_unwind_dead( + &self, + place: mir::Place<'tcx>, + state: &>::Domain, + ) -> bool { + if let LookupResult::Exact(path) = self.move_data().rev_lookup.find(place.as_ref()) { + let mut maybe_live = false; + // FIXME: use traversal that can early return instead of iterating through all children, + // (similar to `MovePath::find_descendant`). + on_all_children_bits(&self.move_data, path, |child| { + if let Some(idx) = self.index_mapper.map_index(child) { + maybe_live |= state.contains(idx); + } + }); + !maybe_live + } else { + false + } + } + + // FIXME: put this only in filtered mapper? + /// Map the source `MovePathIndex` to the target index tracked in the analysis: + /// - either the `MovePathIndex` itself, + /// - or its image in the sparse domain: its "interesting move path" index. + #[inline(always)] + pub fn map_index(&self, source_idx: MovePathIndex) -> Option { + self.index_mapper.map_index(source_idx) + } +} + +impl<'a, 'tcx, M> HasMoveData<'tcx> for MaybeInitializedPlaces2<'a, 'tcx, M> { + fn move_data(&self) -> &MoveData<'tcx> { + self.move_data + } +} + +impl<'a, 'tcx, M: MovePathIndexMapper> MaybeInitializedPlaces2<'a, 'tcx, M> { + fn update_bits( + state: &mut >::Domain, + path: M::TargetIndex, + dfstate: DropFlagState, + ) { + match dfstate { + DropFlagState::Absent => state.kill(path), + DropFlagState::Present => state.gen_(path), + } + } +} + +impl<'tcx, M: MovePathIndexMapper> Analysis<'tcx> for MaybeInitializedPlaces2<'_, 'tcx, M> { + /// There can be many more `MovePathIndex` than there are locals in a MIR body. + /// We use a mixed bitset to avoid paying too high a memory footprint. + type Domain = MaybeReachable>; + + type SwitchIntData = MaybePlacesSwitchIntData<'tcx>; + + const NAME: &'static str = "maybe_init"; + + fn bottom_value(&self, _: &mir::Body<'tcx>) -> Self::Domain { + // bottom = uninitialized + MaybeReachable::Unreachable + } + + fn initialize_start_block(&self, _: &mir::Body<'tcx>, state: &mut Self::Domain) { + // The mapper controls the domain mapping and thus the number of move paths we track here. + let domain_size = self.index_mapper.mapped_domain_size(self); + *state = MaybeReachable::Reachable(MixedBitSet::new_empty(domain_size)); + + drop_flag_effects_for_function_entry(self.body, self.move_data, |path, s| { + assert!(s == DropFlagState::Present); + if let Some(idx) = self.index_mapper.map_index(path) { + state.gen_(idx); + } + }); + } + + fn apply_primary_statement_effect( + &mut self, + state: &mut Self::Domain, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + drop_flag_effects_for_location(self.body, self.move_data, location, |path, s| { + if let Some(idx) = self.index_mapper.map_index(path) { + Self::update_bits(state, idx, s); + } + }); + + // Mark all places as "maybe init" if they are mutably borrowed. See #90752. + if self.tcx.sess.opts.unstable_opts.precise_enum_drop_elaboration + && let Some((_, rvalue)) = statement.kind.as_assign() + && let mir::Rvalue::Ref(_, mir::BorrowKind::Mut { .. }, place) + // FIXME: Does `&raw const foo` allow mutation? See #90413. + | mir::Rvalue::RawPtr(_, place) = rvalue + && let LookupResult::Exact(mpi) = self.move_data().rev_lookup.find(place.as_ref()) + { + // We only check this MPI's children if it itself is interesting. + if self.index_mapper.map_index(mpi).is_none() { + return; + } + + on_all_children_bits(self.move_data(), mpi, |child| { + if let Some(idx) = self.index_mapper.map_index(child) { + state.gen_(idx); + } + }) + } + } + + fn apply_primary_terminator_effect<'mir>( + &mut self, + state: &mut Self::Domain, + terminator: &'mir mir::Terminator<'tcx>, + location: Location, + ) -> TerminatorEdges<'mir, 'tcx> { + // Note: `edges` must be computed first because `drop_flag_effects_for_location` can change + // the result of `is_unwind_dead`. + let mut edges = terminator.edges(); + if self.skip_unreachable_unwind + && let mir::TerminatorKind::Drop { + target, + unwind, + place, + replace: _, + drop: _, + async_fut: _, + } = terminator.kind + && matches!(unwind, mir::UnwindAction::Cleanup(_)) + && self.is_unwind_dead(place, state) + { + edges = TerminatorEdges::Single(target); + } + drop_flag_effects_for_location(self.body, self.move_data, location, |path, s| { + if let Some(idx) = self.index_mapper.map_index(path) { + Self::update_bits(state, idx, s); + } + }); + edges + } + + fn apply_call_return_effect( + &mut self, + state: &mut Self::Domain, + _block: mir::BasicBlock, + return_places: CallReturnPlaces<'_, 'tcx>, + ) { + return_places.for_each(|place| { + // when a call returns successfully, that means we need to set + // the bits for that dest_place to 1 (initialized). + on_lookup_result_bits( + self.move_data(), + self.move_data().rev_lookup.find(place.as_ref()), + |child| { + if let Some(idx) = self.index_mapper.map_index(child) { + state.gen_(idx); + } + }, + ); + }); + } + + fn get_switch_int_data( + &mut self, + block: mir::BasicBlock, + discr: &mir::Operand<'tcx>, + ) -> Option { + if !self.tcx.sess.opts.unstable_opts.precise_enum_drop_elaboration { + return None; + } + + MaybePlacesSwitchIntData::new(self.tcx, self.body, block, discr) + } + + fn apply_switch_int_edge_effect( + &mut self, + data: &mut Self::SwitchIntData, + state: &mut Self::Domain, + value: SwitchTargetValue, + targets: &mir::SwitchTargets, + ) { + let inactive_variants = match value { + SwitchTargetValue::Normal(value) => InactiveVariants::Active(data.next_discr(value)), + SwitchTargetValue::Otherwise if self.exclude_inactive_in_otherwise => { + InactiveVariants::Inactives(data.variants(targets)) + } + _ => return, + }; + + // Kill all move paths that correspond to variants we know to be inactive along this + // particular outgoing edge of a `SwitchInt`. + drop_flag_effects::on_all_inactive_variants( + self.move_data, + data.enum_place, + &inactive_variants, + |path| { + if let Some(idx) = self.index_mapper.map_index(path) { + state.kill(idx) + } + }, + ); + } +} diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 3f29b819a6d18..a0959b10b8883 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,5 +1,6 @@ mod borrowed_locals; mod initialized; +mod initialized2; mod liveness; mod storage_liveness; @@ -8,6 +9,10 @@ pub use self::initialized::{ EverInitializedPlaces, EverInitializedPlacesDomain, MaybeInitializedPlaces, MaybeUninitializedPlaces, MaybeUninitializedPlacesDomain, }; +pub use self::initialized2::{ + DefaultMovePathIndexMapper, FilteringMovePathIndexMapper, MaybeInitializedPlaces2, + MovePathIndexMapper, +}; pub use self::liveness::{ MaybeLiveLocals, MaybeTransitiveLiveLocals, TransferFunction as LivenessTransferFunction, }; diff --git a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs index 466416d63f53e..eefcdf277a61f 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs @@ -34,6 +34,14 @@ impl polonius_engine::Atom for MovePathIndex { } } +rustc_index::newtype_index! { + /// A subset of `MovePathIndex`es, to track the interesting move paths. Used in the + /// `MaybeInitializedPlaces` analysis. + #[orderable] + #[debug_format = "smp{}"] + pub struct SparseMovePathIndex {} +} + rustc_index::newtype_index! { #[orderable] #[debug_format = "mo{}"] diff --git a/compiler/rustc_mir_dataflow/src/rustc_peek.rs b/compiler/rustc_mir_dataflow/src/rustc_peek.rs index a899ec1fa8846..1eaad805c13f3 100644 --- a/compiler/rustc_mir_dataflow/src/rustc_peek.rs +++ b/compiler/rustc_mir_dataflow/src/rustc_peek.rs @@ -1,4 +1,3 @@ -use rustc_ast::MetaItem; use rustc_middle::mir::{self, Body, Local, Location}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::DefId; @@ -14,17 +13,18 @@ use crate::impls::{MaybeInitializedPlaces, MaybeLiveLocals, MaybeUninitializedPl use crate::move_paths::{HasMoveData, LookupResult, MoveData, MovePathIndex}; use crate::{Analysis, JoinSemiLattice, ResultsCursor}; -fn has_rustc_mir_with(tcx: TyCtxt<'_>, def_id: DefId, name: Symbol) -> Option { +fn has_rustc_mir_with(tcx: TyCtxt<'_>, def_id: DefId, name: Symbol) -> bool { for attr in tcx.get_attrs(def_id, sym::rustc_mir) { let items = attr.meta_item_list(); for item in items.iter().flat_map(|l| l.iter()) { - match item.meta_item() { - Some(mi) if mi.has_name(name) => return Some(mi.clone()), - _ => continue, + if let Some(mi) = item.meta_item() + && mi.has_name(name) + { + return true; } } } - None + false } pub fn sanity_check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { @@ -38,27 +38,27 @@ pub fn sanity_check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let move_data = MoveData::gather_moves(body, tcx, |_| true); - if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_maybe_init).is_some() { + if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_maybe_init) { let flow_inits = MaybeInitializedPlaces::new(tcx, body, &move_data) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); sanity_check_via_rustc_peek(tcx, flow_inits); } - if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_maybe_uninit).is_some() { + if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_maybe_uninit) { let flow_uninits = MaybeUninitializedPlaces::new(tcx, body, &move_data) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); sanity_check_via_rustc_peek(tcx, flow_uninits); } - if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_liveness).is_some() { + if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_liveness) { let flow_liveness = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, None).into_results_cursor(body); sanity_check_via_rustc_peek(tcx, flow_liveness); } - if has_rustc_mir_with(tcx, def_id, sym::stop_after_dataflow).is_some() { + if has_rustc_mir_with(tcx, def_id, sym::stop_after_dataflow) { tcx.dcx().emit_fatal(StopAfterDataFlowEndedCompilation); } }