diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index 6a0d70790cfef..626b7e1084192 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -5,6 +5,7 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; use rustc_index::Idx; use rustc_index::bit_set::SparseBitMatrix; use rustc_index::interval::{IntervalSet, SparseIntervalMatrix}; +use rustc_middle::bug; use rustc_middle::mir::{BasicBlock, Location}; use rustc_middle::ty::{self, RegionVid}; use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; @@ -35,6 +36,17 @@ pub(crate) enum RegionElement<'tcx> { PlaceholderRegion(ty::PlaceholderRegion<'tcx>), } +/// Either a mapping of which points a region is live at (for regular bodies), +/// or which regions are live in the body somewhere (for promoteds, which do +/// not care about where they are live, only that they are). +#[derive(Clone)] // FIXME(#146079) +enum LiveRegions { + /// region `'r` is live at locations `L`. + AtPoints(SparseIntervalMatrix), + /// Region `'r` is live in function body. + InBody(FxHashSet), +} + /// Records the CFG locations where each region is live. When we initially compute liveness, we use /// an interval matrix storing liveness ranges for each region-vid. #[derive(Clone)] // FIXME(#146079) @@ -42,15 +54,8 @@ pub(crate) struct LivenessValues { /// The map from locations to points. location_map: Rc, - /// Which regions are live. This is exclusive with the fine-grained tracking in `points`, and - /// currently only used for validating promoteds (which don't care about more precise tracking). - live_regions: Option>, - - /// For each region: the points where it is live. - /// - /// This is not initialized for promoteds, because we don't care *where* within a promoted a - /// region is live, only that it is. - points: Option>, + /// Where a region is live. + live_regions: LiveRegions, /// When using `-Zpolonius=next`, the set of loans that are live at a given point in the CFG. live_loans: Option, @@ -60,8 +65,9 @@ impl LivenessValues { /// Create an empty map of regions to locations where they're live. pub(crate) fn with_specific_points(location_map: Rc) -> Self { LivenessValues { - live_regions: None, - points: Some(SparseIntervalMatrix::new(location_map.num_points())), + live_regions: LiveRegions::AtPoints(SparseIntervalMatrix::new( + location_map.num_points(), + )), location_map, live_loans: None, } @@ -73,8 +79,7 @@ impl LivenessValues { /// which regions are live. pub(crate) fn without_specific_points(location_map: Rc) -> Self { LivenessValues { - live_regions: Some(Default::default()), - points: None, + live_regions: LiveRegions::InBody(Default::default()), location_map, live_loans: None, } @@ -83,14 +88,16 @@ impl LivenessValues { /// Returns the liveness matrix of points where each region is live. Panics if the liveness /// values have been created without any per-point data (that is, for promoteds). pub(crate) fn points(&self) -> &SparseIntervalMatrix { - self.points - .as_ref() - .expect("this `LivenessValues` wasn't created using `with_specific_points`") + if let LiveRegions::AtPoints(points) = &self.live_regions { + points + } else { + bug!("this `LivenessValues` wasn't created using `with_specific_points`") + } } /// Iterate through each region that has a value in this set. pub(crate) fn regions(&self) -> impl Iterator { - self.points.as_ref().expect("use with_specific_points").rows() + self.points().rows() } /// Iterate through each region that has a value in this set. @@ -98,36 +105,53 @@ impl LivenessValues { #[rustc_lint_query_instability] #[allow(rustc::potential_query_instability)] pub(crate) fn live_regions_unordered(&self) -> impl Iterator { - self.live_regions.as_ref().unwrap().iter().copied() + if let LiveRegions::InBody(live_regions) = &self.live_regions { + live_regions.iter().copied() + } else { + bug!("this `LivenessValues` wasn't created using `without_specific_points`") + } } /// Records `region` as being live at the given `location`. pub(crate) fn add_location(&mut self, region: RegionVid, location: Location) { let point = self.location_map.point_from_location(location); debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location); - if let Some(points) = &mut self.points { - points.insert(region, point); - } else if self.location_map.point_in_range(point) { - self.live_regions.as_mut().unwrap().insert(region); - } + match &mut self.live_regions { + LiveRegions::AtPoints(points) => { + points.insert(region, point); + } + + LiveRegions::InBody(live_regions) if self.location_map.point_in_range(point) => { + live_regions.insert(region); + } + + LiveRegions::InBody(_) => (), + }; } /// Records `region` as being live at all the given `points`. pub(crate) fn add_points(&mut self, region: RegionVid, points: &IntervalSet) { debug!("LivenessValues::add_points(region={:?}, points={:?})", region, points); - if let Some(this) = &mut self.points { - this.union_row(region, points); - } else if points.iter().any(|point| self.location_map.point_in_range(point)) { - self.live_regions.as_mut().unwrap().insert(region); - } + match &mut self.live_regions { + LiveRegions::AtPoints(these_points) => { + these_points.union_row(region, points); + } + LiveRegions::InBody(live_regions) + if points.iter().any(|point| self.location_map.point_in_range(point)) => + { + live_regions.insert(region); + } + LiveRegions::InBody(_) => (), + }; } /// Records `region` as being live at all the control-flow points. pub(crate) fn add_all_points(&mut self, region: RegionVid) { - if let Some(points) = &mut self.points { - points.insert_all_into_row(region); - } else { - self.live_regions.as_mut().unwrap().insert(region); + match &mut self.live_regions { + LiveRegions::AtPoints(points) => points.insert_all_into_row(region), + LiveRegions::InBody(live_regions) => { + live_regions.insert(region); + } } } @@ -142,23 +166,12 @@ impl LivenessValues { /// [`point`][rustc_mir_dataflow::points::PointIndex]. #[inline] pub(crate) fn is_live_at_point(&self, region: RegionVid, point: PointIndex) -> bool { - if let Some(points) = &self.points { - points.row(region).is_some_and(|r| r.contains(point)) - } else { - unreachable!( - "Should be using LivenessValues::with_specific_points to ask whether live at a location" - ) - } + self.points().row(region).is_some_and(|r| r.contains(point)) } /// Returns an iterator of all the points where `region` is live. fn live_points(&self, region: RegionVid) -> impl Iterator { - let Some(points) = &self.points else { - unreachable!( - "Should be using LivenessValues::with_specific_points to ask whether live at a location" - ) - }; - points + self.points() .row(region) .into_iter() .flat_map(|set| set.iter()) @@ -328,10 +341,7 @@ impl<'tcx, N: Idx> RegionValues<'tcx, N> { /// elements for the region `from` from `values` and add them to /// the region `to` in `self`. pub(crate) fn merge_liveness(&mut self, to: N, from: RegionVid, values: &LivenessValues) { - let Some(value_points) = &values.points else { - panic!("LivenessValues must track specific points for use in merge_liveness"); - }; - if let Some(set) = value_points.row(from) { + if let Some(set) = values.points().row(from) { self.points.union_row(to, set); } }