From 8468d443a10f568b602bc92da595cf01f94fdc9d Mon Sep 17 00:00:00 2001 From: jackh726 Date: Tue, 8 Sep 2026 01:31:40 +0000 Subject: [PATCH 1/5] Cache block seeking results for initialized_at_terminator and _exit --- .../src/type_check/liveness/trace.rs | 68 ++++++++++++++----- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index dc8e8e077be95..aab47fdf3f049 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -1,5 +1,6 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; -use rustc_index::bit_set::DenseBitSet; +use rustc_index::IndexVec; +use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use rustc_index::interval::IntervalSet; use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_infer::traits::TraitErrors; @@ -10,7 +11,7 @@ use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt}; use rustc_mir_dataflow::impls::MaybeInitializedPlaces; use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex}; use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; -use rustc_mir_dataflow::{Analysis, ResultsCursor}; +use rustc_mir_dataflow::{Analysis, MaybeReachable, ResultsCursor}; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::traits::ObligationCtxt; @@ -55,6 +56,8 @@ pub(super) fn trace<'tcx>( location_map, local_use_map, move_data, + term_states: IndexVec::new(), + exit_states: IndexVec::new(), drop_data: FxIndexMap::default(), }; @@ -90,6 +93,10 @@ struct LivenessContext<'a, 'typeck, 'tcx> { /// Index indicating where each variable is assigned, used, or /// dropped. local_use_map: &'a LocalUseMap, + + // Caches for the results of `initialized_at_terminator` and `initialized_at_exit`. + term_states: IndexVec>>>, + exit_states: IndexVec>>>, } struct DropData<'tcx> { @@ -302,9 +309,11 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { let location = self.cx.location_map.to_location(drop_point); debug_assert_eq!(self.cx.body().terminator_loc(location.block), location,); - if self.cx.initialized_at_terminator(location.block, mpi) - && self.drop_live_at.insert(drop_point) - { + if self.cx.initialized_at_terminator(location.block, mpi) { + let inserted = self.drop_live_at.insert(drop_point); + // Right now, we should visit a drop_point twice. + // If we do, this trigger a debug assert so we need we can optimize. + debug_assert!(inserted, "drop point should not have been visited yet"); self.drop_locations.push(location); self.stack.push(drop_point); } @@ -497,12 +506,19 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { self.typeck.body } - /// Returns `true` if the local variable (or some part of it) is initialized at the current - /// cursor position. Callers should call one of the `seek` methods immediately before to point - /// the cursor to the desired location. - fn initialized_at_curr_loc(&mut self, mpi: MovePathIndex) -> bool { - let flow_inits = self.flow_inits(); - let state = flow_inits.get(); + /// Returns `true` if the local variable (or some part of it) is initialized at the + /// location as set by `seek`. Results are cached in `states`. + fn initialized_at( + states: &mut IndexVec>>>, + flow_inits: &mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>, + block: BasicBlock, + mpi: MovePathIndex, + seek: impl FnOnce(&mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>), + ) -> bool { + let state = states.get_or_insert_with(block, || { + seek(flow_inits); + flow_inits.get().clone() + }); if state.contains(mpi) { return true; } @@ -516,9 +532,18 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { /// DROP of some local variable will have an effect -- note that /// drops, as they may unwind, are always terminators. fn initialized_at_terminator(&mut self, block: BasicBlock, mpi: MovePathIndex) -> bool { - let terminator_location = self.body().terminator_loc(block); - self.flow_inits().seek_before_primary_effect(terminator_location); - self.initialized_at_curr_loc(mpi) + // Ensure self.flow_inits is initialized + let _ = self.flow_inits(); + Self::initialized_at( + &mut self.term_states, + self.flow_inits.as_mut().unwrap(), + block, + mpi, + |flow_inits: &mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>| { + let terminator_location = self.typeck.body.terminator_loc(block); + flow_inits.seek_before_primary_effect(terminator_location); + }, + ) } /// Returns `true` if the path `mpi` (or some part of it) is initialized at @@ -527,9 +552,18 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { /// **Warning:** Does not account for the result of `Call` /// instructions. fn initialized_at_exit(&mut self, block: BasicBlock, mpi: MovePathIndex) -> bool { - let terminator_location = self.body().terminator_loc(block); - self.flow_inits().seek_after_primary_effect(terminator_location); - self.initialized_at_curr_loc(mpi) + // Ensure self.flow_inits is initialized + let _ = self.flow_inits(); + Self::initialized_at( + &mut self.exit_states, + self.flow_inits.as_mut().unwrap(), + block, + mpi, + |flow_inits: &mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>| { + let terminator_location = self.typeck.body.terminator_loc(block); + flow_inits.seek_after_primary_effect(terminator_location); + }, + ) } /// Stores the result that all regions in `value` are live for the From 4f240eab7a67d3662f851453561665fce3b54564 Mon Sep 17 00:00:00 2001 From: jackh726 Date: Fri, 28 Aug 2026 14:39:30 +0000 Subject: [PATCH 2/5] Refactor LivenessResults into LivenessCalculation, without typeck --- .../src/type_check/liveness/trace.rs | 410 ++++++++++-------- 1 file changed, 222 insertions(+), 188 deletions(-) diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index aab47fdf3f049..7b8a2bf09c66d 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -20,11 +20,12 @@ use rustc_trait_selection::traits::query::dropck_outlives; use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOpOutput}; use tracing::debug; -use crate::BorrowckInferCtxt; -use crate::polonius::{self, record_live_region_variance}; -use crate::region_infer::values; +use crate::polonius::{LiveRegionVariances, record_live_region_variance}; +use crate::region_infer::values::{self, LivenessValues}; use crate::type_check::liveness::local_use_map::LocalUseMap; use crate::type_check::{NormalizeLocation, TypeChecker}; +use crate::universal_regions::UniversalRegions; +use crate::{BorrowckInferCtxt, polonius}; /// This is the heart of the liveness computation. For each variable X /// that requires a liveness computation, it walks over all the uses @@ -49,19 +50,16 @@ pub(super) fn trace<'tcx>( ) { let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness_trace"); - let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, typeck.body); - let cx = LivenessContext { - typeck, - flow_inits: None, + let local_use_map = LocalUseMap::build(&relevant_live_locals, location_map, typeck.body); + let calc = LivenessCalculation::new( + typeck.infcx, + typeck.body, location_map, - local_use_map, move_data, - term_states: IndexVec::new(), - exit_states: IndexVec::new(), - drop_data: FxIndexMap::default(), - }; + &local_use_map, + ); - let mut results = LivenessResults::new(cx); + let mut results = LivenessResults::new(typeck, calc); results.add_extra_drop_facts(relevant_live_locals); @@ -70,12 +68,10 @@ pub(super) fn trace<'tcx>( results.dropck_boring_locals(boring_locals); } -/// Contextual state for the type-liveness coroutine. -struct LivenessContext<'a, 'typeck, 'tcx> { - /// Current type-checker, giving us our inference context etc. - /// - /// This also stores the body we're currently analyzing. - typeck: &'a mut TypeChecker<'typeck, 'tcx>, +pub(crate) struct LivenessCalculation<'a, 'tcx> { + pub(crate) infcx: &'a BorrowckInferCtxt<'tcx>, + + pub(crate) body: &'a Body<'tcx>, /// Defines the `PointIndex` mapping location_map: &'a DenseLocationMap, @@ -83,9 +79,6 @@ struct LivenessContext<'a, 'typeck, 'tcx> { /// Mapping to/from the various indices used for initialization tracking. move_data: &'a MoveData<'tcx>, - /// Cache for the results of `dropck_outlives` query. - drop_data: FxIndexMap, DropData<'tcx>>, - /// Results of dataflow tracking which variables (and paths) have been /// initialized. Computed lazily when needed by drop-liveness. flow_inits: Option>>, @@ -97,27 +90,18 @@ struct LivenessContext<'a, 'typeck, 'tcx> { // Caches for the results of `initialized_at_terminator` and `initialized_at_exit`. term_states: IndexVec>>>, exit_states: IndexVec>>>, -} - -struct DropData<'tcx> { - dropck_result: DropckOutlivesResult<'tcx>, - region_constraint_data: Option<&'tcx QueryRegionConstraints<'tcx>>, -} - -struct LivenessResults<'a, 'typeck, 'tcx> { - cx: LivenessContext<'a, 'typeck, 'tcx>, /// Set of points that define the current local. defs: DenseBitSet, /// Points where the current variable is "use live" -- meaning /// that there is a future "full use" that may use its value. - use_live_at: IntervalSet, + pub(crate) use_live_at: IntervalSet, /// Points where the current variable is "drop live" -- meaning /// that there is no future "full use" that may use its value, but /// there is a future drop. - drop_live_at: DenseBitSet, + pub(crate) drop_live_at: DenseBitSet, /// Locations where drops may occur. drop_locations: Vec, @@ -126,43 +110,69 @@ struct LivenessResults<'a, 'typeck, 'tcx> { stack: Vec, } +struct DropData<'tcx> { + dropck_result: DropckOutlivesResult<'tcx>, + region_constraint_data: Option<&'tcx QueryRegionConstraints<'tcx>>, +} + +struct LivenessResults<'a, 'typeck, 'tcx> { + /// Current type-checker, giving us our inference context etc. + /// + /// This also stores the body we're currently analyzing. + typeck: &'a mut TypeChecker<'typeck, 'tcx>, + + /// Cache for the results of `dropck_outlives` query. + drop_data: FxIndexMap, DropData<'tcx>>, + + calc: LivenessCalculation<'a, 'tcx>, +} + impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { - fn new(cx: LivenessContext<'a, 'typeck, 'tcx>) -> Self { - let num_points = cx.location_map.num_points(); - LivenessResults { - cx, - defs: DenseBitSet::new_empty(num_points), - use_live_at: IntervalSet::new(num_points), - drop_live_at: DenseBitSet::new_empty(num_points), - drop_locations: vec![], - stack: vec![], - } + fn new( + typeck: &'a mut TypeChecker<'typeck, 'tcx>, + calc: LivenessCalculation<'a, 'tcx>, + ) -> Self { + LivenessResults { typeck, drop_data: FxIndexMap::default(), calc } } fn compute_for_all_locals(&mut self, relevant_live_locals: &[Local]) { for &local in relevant_live_locals { - self.reset_local_state(); - self.add_defs_for(local); - self.compute_use_live_points_for(local); - self.compute_drop_live_points_for(local); - - let local_ty = self.cx.body().local_decls[local].ty; - - if !self.use_live_at.is_empty() { - self.cx.add_use_live_facts_for(local_ty, &self.use_live_at); + self.calc.compute(local); + + let local_ty = self.calc.body.local_decls[local].ty; + + if !self.calc.use_live_at.is_empty() { + make_all_regions_live( + self.typeck.infcx, + self.typeck.universal_regions, + &mut self.typeck.constraints.liveness_constraints, + self.typeck.polonius_context.as_mut().map(|p| &mut p.live_region_variances), + local_ty, + &self.calc.use_live_at, + ); } - if !self.drop_live_at.is_empty() { + if !self.calc.drop_live_at.is_empty() { // `drop_live_at` is using a DenseBitSet, but `add_drop_live_facts_for` expects // an IntervalSet. We thus convert between those two here. let mut set: IntervalSet = - IntervalSet::new(self.drop_live_at.domain_size()); - for item in self.drop_live_at.iter() { + IntervalSet::new(self.calc.drop_live_at.domain_size()); + for item in self.calc.drop_live_at.iter() { // We iterate the `drop_live_at` set from smallest to largest values, so // we can use append to add things to the interval set at the end. set.append(item); } - self.cx.add_drop_live_facts_for(local, local_ty, &self.drop_locations, &set); + let local_span = self.calc.body.local_decls[local].source_info.span; + Self::add_drop_live_facts_for( + self.typeck, + &mut self.drop_data, + &self.calc.location_map, + local, + local_ty, + local_span, + &self.calc.drop_locations, + &set, + ); } } } @@ -175,9 +185,9 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { /// and can therefore safely be dropped. fn dropck_boring_locals(&mut self, boring_locals: &[Local]) { for &local in boring_locals { - let local_ty = self.cx.body().local_decls[local].ty; - let local_span = self.cx.body().local_decls[local].source_info.span; - dropck_local(&self.cx.typeck.infcx, &mut self.cx.drop_data, local_ty, local_span); + let local_ty = self.calc.body.local_decls[local].ty; + let local_span = self.calc.body.local_decls[local].source_info.span; + dropck_local(&self.typeck.infcx, &mut self.drop_data, local_ty, local_span); } } @@ -195,7 +205,7 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { // and probably maybe plausibly does not need to go back in. // It may be necessary to just pick out the parts of // `add_drop_live_facts_for()` that make sense. - let Some(facts) = self.cx.typeck.polonius_facts.as_ref() else { return }; + let Some(facts) = self.typeck.polonius_facts.as_ref() else { return }; let facts_to_add: Vec<_> = { let relevant_live_locals: FxIndexSet<_> = relevant_live_locals.iter().copied().collect(); @@ -204,23 +214,127 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { .var_dropped_at .iter() .filter_map(|&(local, location_index)| { - let local_ty = self.cx.body().local_decls[local].ty; + let local_ty = self.calc.body.local_decls[local].ty; if relevant_live_locals.contains(&local) || !local_ty.has_free_regions() { return None; } - let location = self.cx.typeck.location_table.to_location(location_index); + let location = self.typeck.location_table.to_location(location_index); Some((local, local_ty, location)) }) .collect() }; - let live_at = IntervalSet::new(self.cx.location_map.num_points()); + let live_at = IntervalSet::new(self.calc.location_map.num_points()); for (local, local_ty, location) in facts_to_add { - self.cx.add_drop_live_facts_for(local, local_ty, &[location], &live_at); + let local_span = self.calc.body.local_decls[local].source_info.span; + Self::add_drop_live_facts_for( + self.typeck, + &mut self.drop_data, + &self.calc.location_map, + local, + local_ty, + local_span, + &[location], + &live_at, + ); } } + /// Some variable with type `live_ty` is "drop live" at `location` + /// -- i.e., it may be dropped later. This means that *some* of + /// the regions in its type must be live at `location`. The + /// precise set will depend on the dropck constraints, and in + /// particular this takes `#[may_dangle]` into account. + fn add_drop_live_facts_for( + typeck: &mut TypeChecker<'typeck, 'tcx>, + drop_data: &mut FxIndexMap, DropData<'tcx>>, + location_map: &DenseLocationMap, + dropped_local: Local, + dropped_ty: Ty<'tcx>, + dropped_span: Span, + drop_locations: &[Location], + live_at: &IntervalSet, + ) { + debug!( + "add_drop_live_constraint(\ + dropped_local={:?}, \ + dropped_ty={:?}, \ + drop_locations={:?}, \ + live_at={:?})", + dropped_local, + dropped_ty, + drop_locations, + values::pretty_print_points(location_map, live_at.iter()), + ); + + let drop_data = dropck_local(&typeck.infcx, drop_data, dropped_ty, dropped_span); + + if let Some(data) = &drop_data.region_constraint_data { + for &drop_location in drop_locations { + typeck.push_region_constraints( + drop_location.to_locations(), + ConstraintCategory::Boring, + data, + ); + } + } + + // All things in the `outlives` array may be touched by + // the destructor and must be live at this point. + for &kind in &drop_data.dropck_result.kinds { + make_all_regions_live( + typeck.infcx, + typeck.universal_regions, + &mut typeck.constraints.liveness_constraints, + typeck.polonius_context.as_mut().map(|p| &mut p.live_region_variances), + kind, + live_at, + ); + polonius::legacy::emit_drop_facts( + typeck.tcx(), + dropped_local, + &kind, + typeck.universal_regions, + typeck.polonius_facts, + ); + } + } +} + +impl<'a, 'tcx> LivenessCalculation<'a, 'tcx> { + pub(crate) fn new( + infcx: &'a BorrowckInferCtxt<'tcx>, + body: &'a Body<'tcx>, + location_map: &'a DenseLocationMap, + move_data: &'a MoveData<'tcx>, + local_use_map: &'a LocalUseMap, + ) -> Self { + let num_points = location_map.num_points(); + LivenessCalculation { + infcx, + body, + location_map, + move_data, + flow_inits: None, + local_use_map, + term_states: IndexVec::new(), + exit_states: IndexVec::new(), + defs: DenseBitSet::new_empty(num_points), + use_live_at: IntervalSet::new(num_points), + drop_live_at: DenseBitSet::new_empty(num_points), + drop_locations: vec![], + stack: vec![], + } + } + + pub(crate) fn compute(&mut self, local: Local) { + self.reset_local_state(); + self.add_defs_for(local); + self.compute_use_live_points_for(local); + self.compute_drop_live_points_for(local); + } + /// Clear the value of fields that are "per local variable". fn reset_local_state(&mut self) { self.defs.clear(); @@ -232,7 +346,7 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { /// Adds the definitions of `local` into `self.defs`. fn add_defs_for(&mut self, local: Local) { - for def in self.cx.local_use_map.defs(local) { + for def in self.local_use_map.defs(local) { debug!("- defined at {:?}", def); self.defs.insert(def); } @@ -247,14 +361,14 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { fn compute_use_live_points_for(&mut self, local: Local) { debug!("compute_use_live_points_for(local={:?})", local); - self.stack.extend(self.cx.local_use_map.uses(local)); + self.stack.extend(self.local_use_map.uses(local)); while let Some(p) = self.stack.pop() { // We are live in this block from the closest to us of: // // * Inclusively, the block start // * Exclusively, the previous definition (if it's in this block) // * Exclusively, the previous live_at setting (an optimization) - let block_start = self.cx.location_map.to_block_start(p); + let block_start = self.location_map.to_block_start(p); let previous_defs = self.defs.last_set_in(block_start..=p); let previous_live_at = self.use_live_at.last_set_in(block_start..=p); @@ -278,12 +392,12 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { // terminators of predecessor basic blocks. Push those onto the // stack so that the next iteration(s) will process them. - let block = self.cx.location_map.to_location(block_start).block; + let block = self.location_map.to_location(block_start).block; self.stack.extend( - self.cx.body().basic_blocks.predecessors()[block] + self.body.basic_blocks.predecessors()[block] .iter() - .map(|&pred_bb| self.cx.body().terminator_loc(pred_bb)) - .map(|pred_loc| self.cx.location_map.point_from_location(pred_loc)), + .map(|&pred_bb| self.body.terminator_loc(pred_bb)) + .map(|pred_loc| self.location_map.point_from_location(pred_loc)), ); } } @@ -301,15 +415,15 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { fn compute_drop_live_points_for(&mut self, local: Local) { debug!("compute_drop_live_points_for(local={:?})", local); - let Some(mpi) = self.cx.move_data.rev_lookup.find_local(local) else { return }; + let Some(mpi) = self.move_data.rev_lookup.find_local(local) else { return }; debug!("compute_drop_live_points_for: mpi = {:?}", mpi); // Find the drops where `local` is initialized. - for drop_point in self.cx.local_use_map.drops(local) { - let location = self.cx.location_map.to_location(drop_point); - debug_assert_eq!(self.cx.body().terminator_loc(location.block), location,); + for drop_point in self.local_use_map.drops(local) { + let location = self.location_map.to_location(drop_point); + debug_assert_eq!(self.body.terminator_loc(location.block), location,); - if self.cx.initialized_at_terminator(location.block, mpi) { + if self.initialized_at_terminator(location.block, mpi) { let inserted = self.drop_live_at.insert(drop_point); // Right now, we should visit a drop_point twice. // If we do, this trigger a debug assert so we need we can optimize. @@ -343,8 +457,8 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { fn compute_drop_live_points_for_block(&mut self, mpi: MovePathIndex, term_point: PointIndex) { debug!( "compute_drop_live_points_for_block(mpi={:?}, term_point={:?})", - self.cx.move_data.move_paths[mpi].place, - self.cx.location_map.to_location(term_point), + self.move_data.move_paths[mpi].place, + self.location_map.to_location(term_point), ); // We are only invoked with terminators where `mpi` is @@ -354,14 +468,14 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { // Otherwise, scan backwards through the statements in the // block. One of them may be either a definition or use // live point. - let term_location = self.cx.location_map.to_location(term_point); - debug_assert_eq!(self.cx.body().terminator_loc(term_location.block), term_location,); + let term_location = self.location_map.to_location(term_point); + debug_assert_eq!(self.body.terminator_loc(term_location.block), term_location,); let block = term_location.block; - let entry_point = self.cx.location_map.entry_point(term_location.block); + let entry_point = self.location_map.entry_point(term_location.block); for p in (entry_point..term_point).rev() { debug!( "compute_drop_live_points_for_block: p = {:?}", - self.cx.location_map.to_location(p) + self.location_map.to_location(p) ); if self.defs.contains(p) { @@ -380,7 +494,7 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { } } - let body = self.cx.typeck.body; + let body = self.body; for &pred_block in body.basic_blocks.predecessors()[block].iter() { debug!("compute_drop_live_points_for_block: pred_block = {:?}", pred_block,); @@ -402,13 +516,13 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { // terminator. *But*, in that case, the terminator is also // a *definition* of the variable, in which case we want // to stop the search anyhow. (But see Note 1 below.) - if !self.cx.initialized_at_exit(pred_block, mpi) { + if !self.initialized_at_exit(pred_block, mpi) { debug!("compute_drop_live_points_for_block: not initialized"); continue; } - let pred_term_loc = self.cx.body().terminator_loc(pred_block); - let pred_term_point = self.cx.location_map.point_from_location(pred_term_loc); + let pred_term_loc = self.body.terminator_loc(pred_block); + let pred_term_point = self.location_map.point_from_location(pred_term_loc); // If the terminator of this predecessor either *assigns* // our value or is a "normal use", then stop. @@ -464,9 +578,7 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { // for the call (`TMP = call()...`) and then a // `Drop(X)` followed by `X = TMP` to swap that with `X`. } -} -impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> { /// Computes the `MaybeInitializedPlaces` dataflow analysis if it hasn't been done already. /// /// In practice, the results of this dataflow analysis are rarely needed but can be expensive to @@ -478,8 +590,8 @@ impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> { /// maybe-initializedness of `MovePathIndex`es. fn flow_inits(&mut self) -> &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>> { self.flow_inits.get_or_insert_with(|| { - let tcx = self.typeck.tcx(); - let body = self.typeck.body; + let tcx = self.infcx.tcx; + let body = self.body; // FIXME: reduce the `MaybeInitializedPlaces` domain to the useful `MovePath`s. // // This dataflow analysis computes maybe-initializedness of all move paths, which @@ -499,12 +611,6 @@ impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> { flow_inits }) } -} - -impl<'tcx> LivenessContext<'_, '_, 'tcx> { - fn body(&self) -> &Body<'tcx> { - self.typeck.body - } /// Returns `true` if the local variable (or some part of it) is initialized at the /// location as set by `seek`. Results are cached in `states`. @@ -540,7 +646,7 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { block, mpi, |flow_inits: &mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>| { - let terminator_location = self.typeck.body.terminator_loc(block); + let terminator_location = self.body.terminator_loc(block); flow_inits.seek_before_primary_effect(terminator_location); }, ) @@ -560,102 +666,30 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { block, mpi, |flow_inits: &mut ResultsCursor<'_, 'tcx, MaybeInitializedPlaces<'_, 'tcx>>| { - let terminator_location = self.typeck.body.terminator_loc(block); + let terminator_location = self.body.terminator_loc(block); flow_inits.seek_after_primary_effect(terminator_location); }, ) } +} - /// Stores the result that all regions in `value` are live for the - /// points `live_at`. - fn add_use_live_facts_for(&mut self, value: Ty<'tcx>, live_at: &IntervalSet) { - debug!("add_use_live_facts_for(value={:?})", value); - Self::make_all_regions_live(self.location_map, self.typeck, value, live_at); - } - - /// Some variable with type `live_ty` is "drop live" at `location` - /// -- i.e., it may be dropped later. This means that *some* of - /// the regions in its type must be live at `location`. The - /// precise set will depend on the dropck constraints, and in - /// particular this takes `#[may_dangle]` into account. - fn add_drop_live_facts_for( - &mut self, - dropped_local: Local, - dropped_ty: Ty<'tcx>, - drop_locations: &[Location], - live_at: &IntervalSet, - ) { - debug!( - "add_drop_live_constraint(\ - dropped_local={:?}, \ - dropped_ty={:?}, \ - drop_locations={:?}, \ - live_at={:?})", - dropped_local, - dropped_ty, - drop_locations, - values::pretty_print_points(self.location_map, live_at.iter()), - ); - - let dropped_span = self.body().local_decls[dropped_local].source_info.span; - let drop_data = - dropck_local(&self.typeck.infcx, &mut self.drop_data, dropped_ty, dropped_span); - - if let Some(data) = &drop_data.region_constraint_data { - for &drop_location in drop_locations { - self.typeck.push_region_constraints( - drop_location.to_locations(), - ConstraintCategory::Boring, - data, - ); - } - } - - // All things in the `outlives` array may be touched by - // the destructor and must be live at this point. - for &kind in &drop_data.dropck_result.kinds { - Self::make_all_regions_live(self.location_map, self.typeck, kind, live_at); - polonius::legacy::emit_drop_facts( - self.typeck.tcx(), - dropped_local, - &kind, - self.typeck.universal_regions, - self.typeck.polonius_facts, - ); - } - } - - fn make_all_regions_live( - location_map: &DenseLocationMap, - typeck: &mut TypeChecker<'_, 'tcx>, - value: impl TypeVisitable> + Relate>, - live_at: &IntervalSet, - ) { - debug!("make_all_regions_live(value={:?})", value); - debug!( - "make_all_regions_live: live_at={}", - values::pretty_print_points(location_map, live_at.iter()), - ); - - value.visit_with(&mut FreeRegionsVisitor { - tcx: typeck.tcx(), - param_env: typeck.infcx.param_env, - op: |r| { - let live_region_vid = typeck.universal_regions.to_region_vid(r); - - typeck.constraints.liveness_constraints.add_points(live_region_vid, live_at); - }, - }); - - // When using `-Zpolonius=next`, we record the variance of each live region. - if let Some(polonius_context) = typeck.polonius_context.as_mut() { - record_live_region_variance( - typeck.infcx.tcx, - &mut polonius_context.live_region_variances, - typeck.universal_regions, - value, - ); - } +fn make_all_regions_live<'tcx>( + infcx: &BorrowckInferCtxt<'tcx>, + universal_regions: &UniversalRegions<'tcx>, + liveness: &mut LivenessValues, + variances: Option<&mut LiveRegionVariances>, + value: impl TypeVisitable> + Relate>, + live_at: &IntervalSet, +) { + debug!("make_all_regions_live(value={value:?})"); + value.visit_with(&mut FreeRegionsVisitor { + tcx: infcx.tcx, + param_env: infcx.param_env, + op: |r| liveness.add_points(universal_regions.to_region_vid(r), live_at), + }); + + if let Some(variances) = variances { + record_live_region_variance(infcx.tcx, variances, universal_regions, value); } } From 036972efc74c407563242696db7d022651601710 Mon Sep 17 00:00:00 2001 From: jackh726 Date: Mon, 14 Sep 2026 23:25:48 +0000 Subject: [PATCH 3/5] Pass liveness/variances through a separate LivenessSource trait --- .../src/polonius/constraints.rs | 163 ++++++++++-------- compiler/rustc_borrowck/src/polonius/dump.rs | 11 +- compiler/rustc_borrowck/src/polonius/mod.rs | 23 +-- .../rustc_borrowck/src/region_infer/values.rs | 4 + 4 files changed, 111 insertions(+), 90 deletions(-) diff --git a/compiler/rustc_borrowck/src/polonius/constraints.rs b/compiler/rustc_borrowck/src/polonius/constraints.rs index ce32b6ee99012..46e86d1ec6f7d 100644 --- a/compiler/rustc_borrowck/src/polonius/constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/constraints.rs @@ -1,8 +1,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_index::interval::SparseIntervalMatrix; use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::RegionVid; -use rustc_mir_dataflow::points::PointIndex; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; use crate::BorrowSet; use crate::constraints::OutlivesConstraint; @@ -49,11 +48,65 @@ pub(super) struct LocalizedConstraintGraph { logical_edges: FxHashMap>, } +/// For a given region, the relevant liveness and variance information. +pub(super) struct RegionLiveness<'a> { + region: RegionVid, + direction: ConstraintDirection, + liveness: &'a LivenessValues, +} + +impl<'a> RegionLiveness<'a> { + pub(super) fn new( + region: RegionVid, + live_region_variances: &LiveRegionVariances, + liveness: &'a LivenessValues, + ) -> Self { + // Note: there currently are cases related to promoted and const generics, where we don't yet + // have variance information (possibly about temporary regions created when typeck sanitizes the + // promoteds). Until that is done, we conservatively fallback to maximizing reachability by + // adding a bidirectional edge here. This will not limit traversal whatsoever, and thus + // propagate liveness when needed. + // + // FIXME: add the missing variance information and remove this fallback bidirectional edge. + let direction = live_region_variances + .get(region) + .copied() + .flatten() + .unwrap_or(ConstraintDirection::Bidirectional); + Self { region, direction, liveness } + } + + fn is_live_at(&self, point: PointIndex) -> bool { + self.liveness.points().contains(self.region, point) + } +} + +/// The source of liveness information for a given region. +pub(super) trait LivenessSource { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_>; + fn location_map(&self) -> &DenseLocationMap; +} + +/// A `LivenessSource` for already-existing liveness and variance data. +pub(super) struct CachedLivenessSource<'a> { + pub(super) live_region_variances: &'a LiveRegionVariances, + pub(super) liveness: &'a LivenessValues, +} + +impl<'a> LivenessSource for CachedLivenessSource<'a> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { + RegionLiveness::new(region, self.live_region_variances, self.liveness) + } + fn location_map(&self) -> &DenseLocationMap { + self.liveness.location_map() + } +} + /// The visitor interface when traversing a `LocalizedConstraintGraph`. pub(super) trait LocalizedConstraintGraphVisitor { /// Callback called when traversing a given `loan` encounters a localized `node` it hasn't /// visited before. - fn on_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode) {} + fn on_live_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode) {} /// Callback called when discovering a new `successor` node for the `current_node`. fn on_successor_discovered(&mut self, _current_node: LocalizedNode, _successor: LocalizedNode) { @@ -63,7 +116,7 @@ pub(super) trait LocalizedConstraintGraphVisitor { impl LocalizedConstraintGraph { /// Traverses the constraints and returns the indexed graph of edges per node. pub(super) fn new<'tcx>( - liveness: &LivenessValues, + location_map: &DenseLocationMap, outlives_constraints: impl Iterator>, ) -> Self { let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); @@ -81,7 +134,7 @@ impl LocalizedConstraintGraph { Locations::Single(location) => { let node = LocalizedNode { region: outlives_constraint.sup, - point: liveness.point_from_location(location), + point: location_map.point_from_location(location), }; edges.entry(node).or_default().insert(outlives_constraint.sub); } @@ -96,14 +149,11 @@ impl LocalizedConstraintGraph { pub(super) fn traverse<'tcx>( &self, body: &Body<'tcx>, - liveness: &LivenessValues, - live_region_variances: &LiveRegionVariances, universal_regions: &UniversalRegions<'tcx>, borrow_set: &BorrowSet<'tcx>, + liveness_source: &mut impl LivenessSource, visitor: &mut impl LocalizedConstraintGraphVisitor, ) { - let live_regions = liveness.points(); - let mut visited = FxHashSet::default(); let mut stack = Vec::new(); @@ -115,7 +165,7 @@ impl LocalizedConstraintGraph { let start_node = LocalizedNode { region: loan.region, - point: liveness.point_from_location(loan.reserve_location), + point: liveness_source.location_map().point_from_location(loan.reserve_location), }; stack.push(start_node); @@ -124,9 +174,12 @@ impl LocalizedConstraintGraph { continue; } + let liveness = liveness_source.liveness_for_region(node.region); // We've reached a node we haven't visited before. - let location = liveness.location_from_point(node.point); - visitor.on_node_traversed(loan_idx, node); + let location = liveness.liveness.location_map().to_location(node.point); + if liveness.is_live_at(node.point) { + visitor.on_live_node_traversed(loan_idx, node); + } // When we find a _new_ successor, we'd like to // - visit it eventually, @@ -161,13 +214,9 @@ impl LocalizedConstraintGraph { // Intra-block edges, straight line constraints from each point to its successor // within the same block. let next_point = node.point + 1; - if let Some(succ) = compute_forward_successor( - node.region, - next_point, - live_regions, - live_region_variances, - is_universal_region, - ) { + if let Some(succ) = + compute_forward_successor(&liveness, next_point, is_universal_region) + { successor_found(succ); } } else { @@ -175,14 +224,10 @@ impl LocalizedConstraintGraph { // entry point. for successor_block in body[location.block].terminator().successors() { let next_location = Location { block: successor_block, statement_index: 0 }; - let next_point = liveness.point_from_location(next_location); - if let Some(succ) = compute_forward_successor( - node.region, - next_point, - live_regions, - live_region_variances, - is_universal_region, - ) { + let next_point = liveness.liveness.point_from_location(next_location); + if let Some(succ) = + compute_forward_successor(&liveness, next_point, is_universal_region) + { successor_found(succ); } } @@ -194,13 +239,9 @@ impl LocalizedConstraintGraph { if location.statement_index > 0 { // Backward edges to the predecessor point in the same block. let previous_point = PointIndex::from(node.point.as_usize() - 1); - if let Some(succ) = compute_backward_successor( - node.region, - node.point, - previous_point, - live_regions, - live_region_variances, - ) { + if let Some(succ) = + compute_backward_successor(&liveness, node.point, previous_point) + { successor_found(succ); } } else { @@ -212,14 +253,11 @@ impl LocalizedConstraintGraph { block: pred_block, statement_index: body[pred_block].statements.len(), }; - let previous_point = liveness.point_from_location(previous_location); - if let Some(succ) = compute_backward_successor( - node.region, - node.point, - previous_point, - live_regions, - live_region_variances, - ) { + let previous_point = + liveness.liveness.point_from_location(previous_location); + if let Some(succ) = + compute_backward_successor(&liveness, node.point, previous_point) + { successor_found(succ); } } @@ -239,12 +277,12 @@ impl LocalizedConstraintGraph { /// Returns the successor for the current region/point node when propagating a loan through forward /// edges, if applicable, according to liveness and variance. fn compute_forward_successor( - region: RegionVid, + liveness: &RegionLiveness<'_>, next_point: PointIndex, - live_regions: &SparseIntervalMatrix, - live_region_variances: &LiveRegionVariances, is_universal_region: bool, ) -> Option { + let region = liveness.region; + // 1. Universal regions are semantically live at all points. if is_universal_region { let succ = LocalizedNode { region, point: next_point }; @@ -252,27 +290,14 @@ fn compute_forward_successor( } // 2. Otherwise, gather the edges due to explicit region liveness, when applicable. - if !live_regions.contains(region, next_point) { + if !liveness.is_live_at(next_point) { return None; } // Here, `region` could be live at the current point, and is live at the next point: add a // constraint between them, according to variance. - // Note: there currently are cases related to promoted and const generics, where we don't yet - // have variance information (possibly about temporary regions created when typeck sanitizes the - // promoteds). Until that is done, we conservatively fallback to maximizing reachability by - // adding a bidirectional edge here. This will not limit traversal whatsoever, and thus - // propagate liveness when needed. - // - // FIXME: add the missing variance information and remove this fallback bidirectional edge. - let direction = live_region_variances - .get(region) - .copied() - .flatten() - .unwrap_or(ConstraintDirection::Bidirectional); - - match direction { + match liveness.direction { ConstraintDirection::Backward => { // Contravariant cases: loans flow in the inverse direction, but we're only interested // in forward successors and there are none here. @@ -291,27 +316,19 @@ fn compute_forward_successor( /// Returns the successor for the current region/point node when propagating a loan through backward /// edges, if applicable, according to liveness and variance. fn compute_backward_successor( - region: RegionVid, + liveness: &RegionLiveness<'_>, current_point: PointIndex, previous_point: PointIndex, - live_regions: &SparseIntervalMatrix, - live_region_variances: &LiveRegionVariances, ) -> Option { + let region = liveness.region; + // Liveness flows into the regions live at the next point. So, in a backwards view, we'll link // the region from the current point, if it's live there, to the previous point. - if !live_regions.contains(region, current_point) { + if !liveness.is_live_at(current_point) { return None; } - // FIXME: add the missing variance information and remove this fallback bidirectional edge. See - // the same comment in `compute_forward_successor`. - let direction = live_region_variances - .get(region) - .copied() - .flatten() - .unwrap_or(ConstraintDirection::Bidirectional); - - match direction { + match liveness.direction { ConstraintDirection::Forward => { // Covariant cases: loans flow in the regular direction, but we're only interested in // backward successors and there are none here. diff --git a/compiler/rustc_borrowck/src/polonius/dump.rs b/compiler/rustc_borrowck/src/polonius/dump.rs index 5285f724b02ec..1466a40af0e75 100644 --- a/compiler/rustc_borrowck/src/polonius/dump.rs +++ b/compiler/rustc_borrowck/src/polonius/dump.rs @@ -10,7 +10,9 @@ use rustc_session::config::MirIncludeSpans; use crate::borrow_set::BorrowSet; use crate::constraints::OutlivesConstraint; -use crate::polonius::{LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext}; +use crate::polonius::{ + CachedLivenessSource, LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext, +}; use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext}; @@ -36,14 +38,17 @@ pub(crate) fn dump_polonius_mir<'tcx>( // If we have a polonius graph to dump along the rest of the MIR and NLL info, we extract its // constraints here. + let mut liveness_source = CachedLivenessSource { + live_region_variances: &polonius_context.live_region_variances, + liveness: regioncx.liveness_constraints(), + }; let mut collector = LocalizedOutlivesConstraintCollector { constraints: Vec::new() }; if let Some(graph) = &polonius_context.graph { graph.traverse( body, - regioncx.liveness_constraints(), - &polonius_context.live_region_variances, regioncx.universal_regions(), borrow_set, + &mut liveness_source, &mut collector, ); } diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 0735aa6120c37..30aed55e809b2 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -139,18 +139,16 @@ impl PoloniusContext { // From the outlives constraints, liveness, and variances, we can compute reachability // on the lazy localized constraint graph to trace the liveness of loans, for the next // step in the chain (the NLL loan scope and active loans computations). - let graph = LocalizedConstraintGraph::new(liveness, outlives_constraints); + let graph = + LocalizedConstraintGraph::new(liveness.location_map(), outlives_constraints); let mut live_loans = LiveLoans::new(num_points, borrow_set.len()); - let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans }; - graph.traverse( - body, + let mut liveness_source = CachedLivenessSource { + live_region_variances: &self.live_region_variances, liveness, - &self.live_region_variances, - universal_regions, - borrow_set, - &mut visitor, - ); + }; + let mut visitor = LoanLivenessVisitor { live_loans: &mut live_loans }; + graph.traverse(body, universal_regions, borrow_set, &mut liveness_source, &mut visitor); liveness.record_live_loans(live_loans); // The graph can be traversed again during MIR dumping, so we store it here. @@ -161,12 +159,11 @@ impl PoloniusContext { /// Visitor to record loan liveness when traversing the localized constraint graph. struct LoanLivenessVisitor<'a> { - liveness: &'a LivenessValues, live_loans: &'a mut LiveLoans, } impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> { - fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) { + fn on_live_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) { // Record the loan as being live on entry to this point if it reaches a live region // there. // @@ -208,8 +205,6 @@ impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> { // // FIXME: analyze potential unsoundness, possibly in concert with a borrowck // implementation in a-mir-formality, fuzzing, or manually crafting counter-examples. - if self.liveness.is_live_at_point(node.region, node.point) { - self.live_loans.insert(node.point, loan); - } + self.live_loans.insert(node.point, loan); } } diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index 841e5713751cd..458167d88d0bd 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -200,6 +200,10 @@ impl LivenessValues { self.location_map.to_location(point) } + pub(crate) fn location_map(&self) -> &Rc { + &self.location_map + } + /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active /// loans dataflow computations. pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) { From 36cf1f421028ce993f923bfdb81f7ed3c969a560 Mon Sep 17 00:00:00 2001 From: jackh726 Date: Mon, 14 Sep 2026 23:25:48 +0000 Subject: [PATCH 4/5] Basics needed for deferred liveness --- compiler/rustc_borrowck/src/lib.rs | 4 +- compiler/rustc_borrowck/src/nll.rs | 10 +- .../src/polonius/constraints.rs | 15 --- compiler/rustc_borrowck/src/polonius/dump.rs | 22 ++++- .../rustc_borrowck/src/polonius/liveness.rs | 66 +++++++++++++ compiler/rustc_borrowck/src/polonius/mod.rs | 99 +++++++++++++++++-- .../src/type_check/liveness/mod.rs | 7 +- .../src/type_check/liveness/trace.rs | 11 ++- compiler/rustc_borrowck/src/type_check/mod.rs | 4 +- 9 files changed, 197 insertions(+), 41 deletions(-) create mode 100644 compiler/rustc_borrowck/src/polonius/liveness.rs diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index b10570db1cdd2..83976de986af5 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -311,7 +311,7 @@ struct CollectRegionConstraintsResult<'tcx> { deferred_closure_requirements: DeferredClosureRequirements<'tcx>, deferred_opaque_type_errors: Vec>, polonius_facts: Option>, - polonius_context: Option, + polonius_context: Option>, } /// Start borrow checking by collecting the region constraints for @@ -799,7 +799,7 @@ pub(crate) struct MirBorrowckCtxt<'a, 'diag, 'tcx> { /// Results of Polonius analysis. polonius_output: Option<&'a PoloniusOutput>, /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics. - polonius_context: Option<&'a PoloniusContext>, + polonius_context: Option<&'a PoloniusContext<'tcx>>, } // Check that: diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 1a328f62fc73e..fed7fdd97ac4d 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -46,7 +46,7 @@ pub(crate) struct NllOutput<'tcx> { /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics, e.g. /// localized typeck and liveness constraints. - pub polonius_context: Option, + pub polonius_context: Option>, } /// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal @@ -121,7 +121,7 @@ pub(crate) fn compute_regions<'tcx>( universal_region_relations: Frozen>, constraints: MirTypeckRegionConstraints<'tcx>, mut polonius_facts: Option>, - mut polonius_context: Option, + mut polonius_context: Option>, ) -> NllOutput<'tcx> { let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output()) || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); @@ -144,20 +144,20 @@ pub(crate) fn compute_regions<'tcx>( &lowered_constraints, ); - let num_points = location_map.num_points(); - // If requested for `-Zpolonius=next`, compute loan liveness information. // This is done prior to `RegionInferenceContext::new`, because we may add // additional liveness constraints. if let Some(polonius_context) = polonius_context.as_mut() { let _timer = infcx.tcx.prof.generic_activity("borrowck_polonius_loan_liveness"); polonius_context.compute_loan_liveness( + infcx, &mut lowered_constraints.liveness_constraints, lowered_constraints.outlives_constraints.outlives().iter().copied(), &universal_region_relations.universal_regions, body, + move_data, + &location_map, borrow_set, - num_points, ); } diff --git a/compiler/rustc_borrowck/src/polonius/constraints.rs b/compiler/rustc_borrowck/src/polonius/constraints.rs index 46e86d1ec6f7d..8cff2ac25aa32 100644 --- a/compiler/rustc_borrowck/src/polonius/constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/constraints.rs @@ -87,21 +87,6 @@ pub(super) trait LivenessSource { fn location_map(&self) -> &DenseLocationMap; } -/// A `LivenessSource` for already-existing liveness and variance data. -pub(super) struct CachedLivenessSource<'a> { - pub(super) live_region_variances: &'a LiveRegionVariances, - pub(super) liveness: &'a LivenessValues, -} - -impl<'a> LivenessSource for CachedLivenessSource<'a> { - fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { - RegionLiveness::new(region, self.live_region_variances, self.liveness) - } - fn location_map(&self) -> &DenseLocationMap { - self.liveness.location_map() - } -} - /// The visitor interface when traversing a `LocalizedConstraintGraph`. pub(super) trait LocalizedConstraintGraphVisitor { /// Callback called when traversing a given `loan` encounters a localized `node` it hasn't diff --git a/compiler/rustc_borrowck/src/polonius/dump.rs b/compiler/rustc_borrowck/src/polonius/dump.rs index 1466a40af0e75..b3a6884bb190a 100644 --- a/compiler/rustc_borrowck/src/polonius/dump.rs +++ b/compiler/rustc_borrowck/src/polonius/dump.rs @@ -5,18 +5,34 @@ use rustc_index::IndexVec; use rustc_middle::mir::pretty::{MirDumper, PassWhere, PrettyPrintMirOptions}; use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::{RegionVid, TyCtxt}; -use rustc_mir_dataflow::points::PointIndex; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; use rustc_session::config::MirIncludeSpans; use crate::borrow_set::BorrowSet; use crate::constraints::OutlivesConstraint; use crate::polonius::{ - CachedLivenessSource, LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext, + LiveRegionVariances, LivenessSource, LocalizedConstraintGraphVisitor, LocalizedNode, + PoloniusContext, RegionLiveness, }; use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext}; +/// A `LivenessSource` for already-existing liveness and variance data. +struct CachedLivenessSource<'a> { + live_region_variances: &'a LiveRegionVariances, + liveness: &'a LivenessValues, +} + +impl<'a> LivenessSource for CachedLivenessSource<'a> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { + RegionLiveness::new(region, self.live_region_variances, self.liveness) + } + fn location_map(&self) -> &DenseLocationMap { + self.liveness.location_map() + } +} + /// `-Zdump-mir=polonius` dumps MIR annotated with NLL and polonius specific information. pub(crate) fn dump_polonius_mir<'tcx>( infcx: &BorrowckInferCtxt<'tcx>, @@ -24,7 +40,7 @@ pub(crate) fn dump_polonius_mir<'tcx>( regioncx: &RegionInferenceContext<'tcx>, closure_region_requirements: &Option>, borrow_set: &BorrowSet<'tcx>, - polonius_context: Option<&PoloniusContext>, + polonius_context: Option<&PoloniusContext<'tcx>>, ) { let tcx = infcx.tcx; if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { diff --git a/compiler/rustc_borrowck/src/polonius/liveness.rs b/compiler/rustc_borrowck/src/polonius/liveness.rs new file mode 100644 index 0000000000000..54073b851f96e --- /dev/null +++ b/compiler/rustc_borrowck/src/polonius/liveness.rs @@ -0,0 +1,66 @@ +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::mir::Local; +use rustc_middle::ty::{GenericArg, RegionVid, Ty}; + +use crate::BorrowckInferCtxt; +use crate::universal_regions::UniversalRegions; + +#[derive(Default)] +pub(crate) struct DeferredLocals<'tcx> { + /// For each region, the local whose liveness is deferred. + /// + /// Importantly, because of MIR renumbering, this will always be a 1:1 relationship. + by_region: FxHashMap, + + /// For each deferred local, gets the regions contained within that local at use and drop. + drop_args_by_local: FxHashMap>>, +} + +impl<'tcx> DeferredLocals<'tcx> { + pub(crate) fn defer_local( + &mut self, + infcx: &BorrowckInferCtxt<'tcx>, + universal_regions: &UniversalRegions<'tcx>, + local: Local, + local_ty: Ty<'tcx>, + dropck_kinds: &[GenericArg<'tcx>], + ) { + let tcx = infcx.tcx; + + // We already have drop data for this local, because we need to register + // region constraints eagerly. So, we'll store this so we don't need to + // recompute. + self.drop_args_by_local.insert(local, dropck_kinds.to_vec()); + + // Then, we want to map all the regions contained within this local to + // the local itself. Later, when asked for liveness of a given region, + // we can trace liveness for the local containing it. + let by_region = &mut self.by_region; + tcx.for_each_free_region(&local_ty, |region| { + // See note in [`VarianceExtractor::record_variance`]. + if region.is_bound() || region.is_erased() { + return; + } + let vid = universal_regions.to_region_vid(region); + // Because of MIR renumbering, we should always have a 1:1 mapping + // between a region and a local. + let previous = by_region.insert(vid, local); + debug_assert!( + previous.is_none(), + "{vid:?} is in the type of both {previous:?} and {local:?}, but \ + MIR renumbering should ensure that this is impossible.", + ); + }); + } + + /// For a given region, return the local whose liveness is deferred, and + /// the regions within that local at use and drop. + pub(crate) fn use_deferred_local( + &mut self, + region: RegionVid, + ) -> Option<(Local, Vec>)> { + let local = self.by_region.remove(®ion)?; + let drop_args = self.drop_args_by_local.remove(&local)?; + Some((local, drop_args)) + } +} diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 30aed55e809b2..6bbb345d5adfd 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -36,23 +36,28 @@ mod constraints; mod dump; pub(crate) mod legacy; +mod liveness; mod liveness_constraints; use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{Body, Local}; -use rustc_middle::ty::RegionVid; -use rustc_mir_dataflow::points::PointIndex; +use rustc_middle::ty::{RegionVid, TypeVisitable}; +use rustc_mir_dataflow::move_paths::MoveData; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; +use rustc_trait_selection::traits::outlives_for_liveness::FreeRegionsVisitor; pub(self) use self::constraints::*; pub(crate) use self::dump::dump_polonius_mir; pub(crate) use self::liveness_constraints::record_live_region_variance; -use crate::BorrowSet; use crate::constraints::OutlivesConstraint; use crate::dataflow::BorrowIndex; +pub(crate) use crate::polonius::liveness::DeferredLocals; use crate::region_infer::values::LivenessValues; +use crate::type_check::liveness::{LivenessCalculation, LocalUseMap}; use crate::universal_regions::UniversalRegions; +use crate::{BorrowSet, BorrowckInferCtxt}; pub(crate) type LiveRegionVariances = IndexVec>; @@ -84,7 +89,7 @@ impl LiveLoans { /// polonius localized constraints, during NLL region inference as well as MIR dumping, /// - data needed by the borrowck error computation and diagnostics. #[derive(Default)] -pub(crate) struct PoloniusContext { +pub(crate) struct PoloniusContext<'tcx> { /// The graph from which we extract the localized outlives constraints. graph: Option, @@ -97,6 +102,10 @@ pub(crate) struct PoloniusContext { /// currently has more boring locals than NLLs so we record the latter to use in errors and /// diagnostics, to focus on the locals we consider relevant and match NLL diagnostics. pub(crate) boring_nll_locals: FxHashSet, + + pub(crate) deferred_locals_for_liveness: DeferredLocals<'tcx>, + + pub(crate) local_use_map: Option, } /// The direction a constraint can flow into. Used to create liveness constraints according to @@ -113,7 +122,7 @@ pub(crate) enum ConstraintDirection { Bidirectional, } -impl PoloniusContext { +impl<'tcx> PoloniusContext<'tcx> { /// Computes live loans using the set of loans model for `-Zpolonius=next`. /// /// First, creates a constraint graph combining regions and CFG points, by: @@ -124,14 +133,16 @@ impl PoloniusContext { /// loan scope and active loans computations. /// /// The constraint data will be used to compute errors and diagnostics. - pub(crate) fn compute_loan_liveness<'tcx>( + pub(crate) fn compute_loan_liveness( &mut self, + infcx: &BorrowckInferCtxt<'tcx>, liveness: &mut LivenessValues, outlives_constraints: impl Iterator>, universal_regions: &UniversalRegions<'tcx>, body: &Body<'tcx>, + move_data: &MoveData<'tcx>, + location_map: &DenseLocationMap, borrow_set: &BorrowSet<'tcx>, - num_points: usize, ) { // We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to // trace throughout localized constraints. @@ -142,10 +153,21 @@ impl PoloniusContext { let graph = LocalizedConstraintGraph::new(liveness.location_map(), outlives_constraints); - let mut live_loans = LiveLoans::new(num_points, borrow_set.len()); - let mut liveness_source = CachedLivenessSource { - live_region_variances: &self.live_region_variances, + let local_use_map = self + .local_use_map + .as_ref() + .expect("local use map should be computed before loan liveness"); + let deferred_locals_for_liveness = + std::mem::take(&mut self.deferred_locals_for_liveness); + let mut live_loans = LiveLoans::new(location_map.num_points(), borrow_set.len()); + let calc = + LivenessCalculation::new(infcx, body, location_map, move_data, &local_use_map); + let mut liveness_source = DeferredLivenessSource { liveness, + live_region_variances: &mut self.live_region_variances, + universal_regions, + deferred_locals_for_liveness, + calc, }; let mut visitor = LoanLivenessVisitor { live_loans: &mut live_loans }; graph.traverse(body, universal_regions, borrow_set, &mut liveness_source, &mut visitor); @@ -157,6 +179,63 @@ impl PoloniusContext { } } +struct DeferredLivenessSource<'a, 'tcx> { + liveness: &'a mut LivenessValues, + live_region_variances: &'a mut LiveRegionVariances, + universal_regions: &'a UniversalRegions<'tcx>, + deferred_locals_for_liveness: DeferredLocals<'tcx>, + calc: LivenessCalculation<'a, 'tcx>, +} + +impl LivenessSource for DeferredLivenessSource<'_, '_> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { + if let Some((local, drop_args)) = + self.deferred_locals_for_liveness.use_deferred_local(region) + { + self.calc.compute(local); + + if !self.calc.use_live_at.is_empty() || !self.calc.drop_live_at.is_empty() { + record_live_region_variance( + self.calc.infcx.tcx, + &mut self.live_region_variances, + self.universal_regions, + self.calc.body.local_decls[local].ty, + ); + } + if !self.calc.use_live_at.is_empty() { + let local_ty = self.calc.body.local_decls[local].ty; + + local_ty.visit_with(&mut FreeRegionsVisitor { + tcx: self.calc.infcx.tcx, + param_env: self.calc.infcx.param_env, + op: |live_region| { + let region = self.universal_regions.to_region_vid(live_region); + self.liveness.add_points(region, &self.calc.use_live_at); + }, + }); + } + if !self.calc.drop_live_at.is_empty() { + for drop_arg in drop_args { + drop_arg.visit_with(&mut FreeRegionsVisitor { + tcx: self.calc.infcx.tcx, + param_env: self.calc.infcx.param_env, + op: |live_region| { + let region = self.universal_regions.to_region_vid(live_region); + self.liveness.add_points(region, &self.calc.drop_live_at); + }, + }); + } + } + } + + RegionLiveness::new(region, self.live_region_variances, self.liveness) + } + + fn location_map(&self) -> &rustc_mir_dataflow::points::DenseLocationMap { + self.calc.location_map + } +} + /// Visitor to record loan liveness when traversing the localized constraint graph. struct LoanLivenessVisitor<'a> { live_loans: &'a mut LiveLoans, diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs index dfab2fd071773..15cfdfb43c33f 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs @@ -18,6 +18,9 @@ use crate::universal_regions::UniversalRegions; mod local_use_map; mod trace; +pub(crate) use local_use_map::LocalUseMap; +pub(crate) use trace::LivenessCalculation; + /// Combines liveness analysis with initialization analysis to /// determine which variables are live at which points, both due to /// ordinary uses and drops. Returns a set of (ty, location) pairs @@ -152,7 +155,7 @@ fn record_regular_live_regions<'tcx>( tcx: TyCtxt<'tcx>, liveness_constraints: &mut LivenessValues, universal_regions: &UniversalRegions<'tcx>, - polonius_context: &mut Option, + polonius_context: &mut Option>, body: &Body<'tcx>, ) { let mut visitor = @@ -167,7 +170,7 @@ struct LiveVariablesVisitor<'a, 'tcx> { tcx: TyCtxt<'tcx>, liveness_constraints: &'a mut LivenessValues, universal_regions: &'a UniversalRegions<'tcx>, - polonius_context: &'a mut Option, + polonius_context: &'a mut Option>, } impl<'a, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'a, 'tcx> { diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 7b8a2bf09c66d..0e426c216c122 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -20,7 +20,7 @@ use rustc_trait_selection::traits::query::dropck_outlives; use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOpOutput}; use tracing::debug; -use crate::polonius::{LiveRegionVariances, record_live_region_variance}; +use crate::polonius::{DeferredLocals, LiveRegionVariances, record_live_region_variance}; use crate::region_infer::values::{self, LivenessValues}; use crate::type_check::liveness::local_use_map::LocalUseMap; use crate::type_check::{NormalizeLocation, TypeChecker}; @@ -61,11 +61,18 @@ pub(super) fn trace<'tcx>( let mut results = LivenessResults::new(typeck, calc); + let deferred_locals = DeferredLocals::default(); + results.add_extra_drop_facts(relevant_live_locals); results.compute_for_all_locals(relevant_live_locals); results.dropck_boring_locals(boring_locals); + + if let Some(polonius_context) = &mut typeck.polonius_context { + polonius_context.deferred_locals_for_liveness = deferred_locals; + polonius_context.local_use_map = Some(local_use_map); + } } pub(crate) struct LivenessCalculation<'a, 'tcx> { @@ -74,7 +81,7 @@ pub(crate) struct LivenessCalculation<'a, 'tcx> { pub(crate) body: &'a Body<'tcx>, /// Defines the `PointIndex` mapping - location_map: &'a DenseLocationMap, + pub(crate) location_map: &'a DenseLocationMap, /// Mapping to/from the various indices used for initialization tracking. move_data: &'a MoveData<'tcx>, diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 65cced536603a..31fe121ca6b86 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -248,7 +248,7 @@ struct TypeChecker<'a, 'tcx> { constraints: &'a mut MirTypeckRegionConstraints<'tcx>, deferred_closure_requirements: &'a mut DeferredClosureRequirements<'tcx>, /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints. - polonius_context: Option, + polonius_context: Option>, } /// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions @@ -259,7 +259,7 @@ pub(crate) struct MirTypeckResults<'tcx> { pub(crate) region_bound_pairs: Frozen>, pub(crate) known_type_outlives_obligations: Frozen>>, pub(crate) deferred_closure_requirements: DeferredClosureRequirements<'tcx>, - pub(crate) polonius_context: Option, + pub(crate) polonius_context: Option>, } /// A collection of region constraints that must be satisfied for the From 0f752d245da21188869b348202503cc946d91937 Mon Sep 17 00:00:00 2001 From: jackh726 Date: Fri, 28 Aug 2026 05:21:45 +0000 Subject: [PATCH 5/5] Defer nll-boring/polonius-relevant locals --- compiler/rustc_borrowck/src/polonius/mod.rs | 2 +- .../src/type_check/liveness/mod.rs | 64 +++++-- .../src/type_check/liveness/trace.rs | 181 +++++++++++++----- 3 files changed, 179 insertions(+), 68 deletions(-) diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 6bbb345d5adfd..0aa24d62c79ff 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -221,7 +221,7 @@ impl LivenessSource for DeferredLivenessSource<'_, '_> { param_env: self.calc.infcx.param_env, op: |live_region| { let region = self.universal_regions.to_region_vid(live_region); - self.liveness.add_points(region, &self.calc.drop_live_at); + self.liveness.add_points(region, &self.calc.drop_live_at()); }, }); } diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs index 15cfdfb43c33f..f5c087e510db3 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs @@ -42,35 +42,57 @@ pub(super) fn generate<'tcx>( typeck.constraints.liveness_constraints.add_all_points(region); } - let mut free_regions = regions_that_outlive_free_regions( + let free_regions = regions_that_outlive_free_regions( typeck.infcx.num_region_vars(), &typeck.universal_regions, &typeck.constraints.outlives_constraints, ); - // NLLs can avoid computing some liveness data here because its constraints are - // location-insensitive, but that doesn't work in polonius: locals whose type contains a region - // that outlives a free region are not necessarily live everywhere in a flow-sensitive setting, - // unlike NLLs. - // We do record these regions in the polonius context, since they're used to differentiate - // relevant and boring locals, which is a key distinction used later in diagnostics. - // This additional liveness information is ultimately used for *loan* liveness, - // so we don't need to compute it when there are no loans. - // FIXME: this NLL optimization idea, to reduce work to relevant locals only, still makes sense - // for polonius, and should be investigated to improve liveness performance. - if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() - && typeck.borrow_set.len() > 0 - { - let (_, boring_locals) = - compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); - typeck.polonius_context.as_mut().unwrap().boring_nll_locals = - boring_locals.into_iter().collect(); - free_regions = typeck.universal_regions.universal_regions_iter().collect(); - } let (relevant_live_locals, boring_locals) = compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); - trace::trace(typeck, location_map, move_data, &relevant_live_locals, &boring_locals); + // Under Polonius Alpha, a larger set of locals are considered relevant: specifically, + // locals containing regions *outliving* universal regions are relevant and only + // locals containing solely universal regions are considered boring. + // + // However, we don't actually need liveness information for *all* these locals, + // only when actually computing loans. So, we can defer computing the liveness + // until we try to compute the loan, which is gated on `LocalizedConstraintGraph` + // traversal. + // + // Potentially in theory, we could defer computing liveness for *all* locals, + // but that's a much bigger refactor (many things rely on liveness of + // NLL-relevant locals). So, we only defer NLL-boring/Polonius-relevant locals + // for now. + let deferred_locals = 'deferred: { + // If we aren't going to be using the additional liveness information, + // don't even bother computing the larger relevant set. + // Similarly, since this liveness information is ultimately used for *loan* + // liveness, we don't need to compute it when there are no loans. + if typeck.polonius_context.is_none() || typeck.borrow_set.len() == 0 { + break 'deferred vec![]; + } + + let free_regions: FxHashSet = + typeck.universal_regions.universal_regions_iter().collect(); + let (polonius_relevant, _) = + compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); + + let boring: FxHashSet = boring_locals.iter().copied().collect(); + let deferred = + polonius_relevant.into_iter().filter(|local| boring.contains(local)).collect(); + typeck.polonius_context.as_mut().unwrap().boring_nll_locals = boring; + deferred + }; + + trace::trace( + typeck, + location_map, + move_data, + &relevant_live_locals, + &boring_locals, + &deferred_locals, + ); // Mark regions that should be live where they appear within rvalues or within a call: like // args, regions, and types. diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 0e426c216c122..1372345d8aa4e 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -47,10 +47,14 @@ pub(super) fn trace<'tcx>( move_data: &MoveData<'tcx>, relevant_live_locals: &[Local], boring_locals: &[Local], + deferred: &[Local], ) { let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness_trace"); - let local_use_map = LocalUseMap::build(&relevant_live_locals, location_map, typeck.body); + // The use map must also cover the deferred locals: their liveness is computed later, from + // this same map, when the loan liveness traversal first reaches one of their regions. + let use_map_locals: Vec = relevant_live_locals.iter().chain(deferred).copied().collect(); + let local_use_map = LocalUseMap::build(&use_map_locals, location_map, typeck.body); let calc = LivenessCalculation::new( typeck.infcx, typeck.body, @@ -61,13 +65,14 @@ pub(super) fn trace<'tcx>( let mut results = LivenessResults::new(typeck, calc); - let deferred_locals = DeferredLocals::default(); + let deferred: FxIndexSet = deferred.iter().copied().collect(); + let mut deferred_locals = DeferredLocals::default(); - results.add_extra_drop_facts(relevant_live_locals); + results.add_extra_drop_facts(relevant_live_locals, &deferred); results.compute_for_all_locals(relevant_live_locals); - results.dropck_boring_locals(boring_locals); + results.dropck_boring_locals(boring_locals, &deferred, &mut deferred_locals); if let Some(polonius_context) = &mut typeck.polonius_context { polonius_context.deferred_locals_for_liveness = deferred_locals; @@ -144,43 +149,38 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { fn compute_for_all_locals(&mut self, relevant_live_locals: &[Local]) { for &local in relevant_live_locals { - self.calc.compute(local); - - let local_ty = self.calc.body.local_decls[local].ty; - - if !self.calc.use_live_at.is_empty() { - make_all_regions_live( - self.typeck.infcx, - self.typeck.universal_regions, - &mut self.typeck.constraints.liveness_constraints, - self.typeck.polonius_context.as_mut().map(|p| &mut p.live_region_variances), - local_ty, - &self.calc.use_live_at, - ); - } + self.compute_for_local(local); + } + } - if !self.calc.drop_live_at.is_empty() { - // `drop_live_at` is using a DenseBitSet, but `add_drop_live_facts_for` expects - // an IntervalSet. We thus convert between those two here. - let mut set: IntervalSet = - IntervalSet::new(self.calc.drop_live_at.domain_size()); - for item in self.calc.drop_live_at.iter() { - // We iterate the `drop_live_at` set from smallest to largest values, so - // we can use append to add things to the interval set at the end. - set.append(item); - } - let local_span = self.calc.body.local_decls[local].source_info.span; - Self::add_drop_live_facts_for( - self.typeck, - &mut self.drop_data, - &self.calc.location_map, - local, - local_ty, - local_span, - &self.calc.drop_locations, - &set, - ); - } + fn compute_for_local(&mut self, local: Local) { + self.calc.compute(local); + + let local_ty = self.calc.body.local_decls[local].ty; + + if !self.calc.use_live_at.is_empty() { + make_all_regions_live( + self.typeck.infcx, + self.typeck.universal_regions, + &mut self.typeck.constraints.liveness_constraints, + self.typeck.polonius_context.as_mut().map(|p| &mut p.live_region_variances), + local_ty, + &self.calc.use_live_at, + ); + } + + if !self.calc.drop_live_at.is_empty() { + let local_span = self.calc.body.local_decls[local].source_info.span; + Self::add_drop_live_facts_for( + self.typeck, + &mut self.drop_data, + &self.calc.location_map, + local, + local_ty, + local_span, + &self.calc.drop_locations, + &self.calc.drop_live_at(), + ); } } @@ -190,19 +190,94 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { /// These are all the locals which do not potentially reference a region local /// to this body. Locals which only reference free regions are always drop-live /// and can therefore safely be dropped. - fn dropck_boring_locals(&mut self, boring_locals: &[Local]) { + fn dropck_boring_locals( + &mut self, + boring_locals: &[Local], + deferred: &FxIndexSet, + deferred_locals: &mut DeferredLocals<'tcx>, + ) { for &local in boring_locals { - let local_ty = self.calc.body.local_decls[local].ty; - let local_span = self.calc.body.local_decls[local].source_info.span; - dropck_local(&self.typeck.infcx, &mut self.drop_data, local_ty, local_span); + self.dropck_boring_local(local, deferred, deferred_locals); + } + } + + fn dropck_boring_local( + &mut self, + local: Local, + deferred: &FxIndexSet, + deferred_locals: &mut DeferredLocals<'tcx>, + ) { + let typeck = &mut *self.typeck; + let local_ty = self.calc.body.local_decls[local].ty; + let local_span = self.calc.body.local_decls[local].source_info.span; + + // If we had treated this as "relevant", we would have run `compute_for_local`. This + // in turn would have skipped calculating dropck *at all* for locals without drop-liveness. + // Calculating drop-liveness is expensive, but we can skip it when we know that there + // are *no* drops (which is relatively cheap). + if deferred.contains(&local) && self.calc.local_use_map.drops(local).next().is_none() { + deferred_locals.defer_local( + typeck.infcx, + typeck.universal_regions, + local, + local_ty, + &[], + ); + return; + } + + // We need to compute dropck for *all* boring locals because we report overflows. + // + // FIXME: there is an argument to be made that we don't need to do this for boring locals + // without drop-liveness, because we skip it for *relevant* locals without drop-liveness. + // But, this is preexisting even on NLL, so leaving it for now. + let drop_data = dropck_local(&typeck.infcx, &mut self.drop_data, local_ty, local_span); + + // We are done with *truly* boring locals. + if !deferred.contains(&local) { + return; + } + + // If this local is deferred and has drop region constraints, we need to register + // them, but *only if the local is drop-live*. + // It doesn't really make sense to only check drop-liveness but defer use-liveness, + // so we just treat this as eager. + if drop_data.region_constraint_data.is_some() { + self.compute_for_local(local); + return; } + + // The only other thing we need to do *eagerly* for deferred locals is to register + // legacy drop facts (because these facts are on `typeck`). + for &kind in &drop_data.dropck_result.kinds { + polonius::legacy::emit_drop_facts( + typeck.tcx(), + local, + &kind, + typeck.universal_regions, + typeck.polonius_facts, + ); + } + + // Finally, we mark that this local is deferred, including the drop kinds. + deferred_locals.defer_local( + typeck.infcx, + typeck.universal_regions, + local, + local_ty, + &drop_data.dropck_result.kinds, + ); } /// Add extra drop facts needed for Polonius. /// /// Add facts for all locals with free regions, since regions may outlive /// the function body only at certain nodes in the CFG. - fn add_extra_drop_facts(&mut self, relevant_live_locals: &[Local]) { + fn add_extra_drop_facts( + &mut self, + relevant_live_locals: &[Local], + deferred: &FxIndexSet, + ) { // This collect is more necessary than immediately apparent // because these facts go into `add_drop_live_facts_for()`, // which also writes to `polonius_facts`, and so this is genuinely @@ -222,7 +297,10 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { .iter() .filter_map(|&(local, location_index)| { let local_ty = self.calc.body.local_decls[local].ty; - if relevant_live_locals.contains(&local) || !local_ty.has_free_regions() { + if relevant_live_locals.contains(&local) + || deferred.contains(&local) + || !local_ty.has_free_regions() + { return None; } @@ -335,6 +413,17 @@ impl<'a, 'tcx> LivenessCalculation<'a, 'tcx> { } } + /// Drop-live points are stored as a DenseBitSet; this converts them into an IntervalSet. + pub(crate) fn drop_live_at(&self) -> IntervalSet { + let mut set: IntervalSet = IntervalSet::new(self.drop_live_at.domain_size()); + for item in self.drop_live_at.iter() { + // We iterate the `drop_live_at` set from smallest to largest values, so + // we can use append to add things to the interval set at the end. + set.append(item); + } + set + } + pub(crate) fn compute(&mut self, local: Local) { self.reset_local_state(); self.add_defs_for(local);