diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index 841e5713751cd..59e8e0fda8408 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -354,7 +354,7 @@ impl<'tcx, N: Idx> RegionValues<'tcx, N> { /// Returns just the universal regions that are contained in a given region's value. pub(crate) fn universal_regions_outlived_by(&self, r: N) -> impl Iterator { - self.free_regions.row(r).map(|set| set.iter()).into_flat_iter() + self.free_regions.row(r).map(|set| set.into_iter()).into_flat_iter() } /// Returns all the elements contained in a given region's value. @@ -364,7 +364,7 @@ impl<'tcx, N: Idx> RegionValues<'tcx, N> { ) -> impl Iterator> { self.placeholders .row(r) - .map(|set| set.iter()) + .map(|set| set.into_iter()) .into_flat_iter() .map(move |p| self.placeholder_indices.lookup_placeholder(p)) } diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index aa3c759a6adbd..ba4980cb04be3 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -1,13 +1,15 @@ use std::marker::PhantomData; -use std::ops::{Bound, Range, RangeBounds}; +use std::ops::{Bound, Deref, DerefMut, Range, RangeBounds}; use std::rc::Rc; +use std::slice::GetDisjointMutError::{IndexOutOfBounds, OverlappingIndices}; use std::{fmt, iter, slice}; use Chunk::*; #[cfg(feature = "nightly")] use rustc_macros::{Decodable_NoContext, Encodable_NoContext}; +use smallvec::SmallVec; -use crate::{Idx, IndexVec}; +use crate::Idx; #[cfg(test)] mod tests; @@ -76,19 +78,94 @@ fn inclusive_start_end( /// #[cfg_attr(feature = "nightly", derive(Decodable_NoContext, Encodable_NoContext))] #[derive(Eq, PartialEq, Hash)] -pub struct DenseBitSet { +pub struct DenseBitSet> { domain_size: usize, - words: Vec, + words: S, marker: PhantomData, } -impl DenseBitSet { +impl DenseBitSet { /// Gets the domain size. pub fn domain_size(&self) -> usize { self.domain_size } } +pub trait DenseBitSetStorage: AsRef<[Word]> + Deref {} +pub trait DenseBitSetStorageMut: + DenseBitSetStorage + AsMut<[Word]> + DerefMut +{ +} + +impl DenseBitSetStorage for Vec {} +impl DenseBitSetStorageMut for Vec {} +impl<'a> DenseBitSetStorage for &'a [Word] {} +impl<'a> DenseBitSetStorage for &'a mut [Word] {} +impl<'a> DenseBitSetStorageMut for &'a mut [Word] {} + +impl DenseBitSetStorage for SmallVec<[Word; N]> {} +impl DenseBitSetStorageMut for SmallVec<[Word; N]> {} + +// workaround because arrays don't implement DerefMut +#[repr(transparent)] +pub struct ArrayStorage { + inner: [Word; N], +} +impl AsRef<[Word]> for ArrayStorage { + #[inline] + fn as_ref(&self) -> &[Word] { + &self.inner + } +} + +impl AsMut<[Word]> for ArrayStorage { + #[inline] + fn as_mut(&mut self) -> &mut [Word] { + &mut self.inner + } +} + +impl Deref for ArrayStorage { + type Target = [Word]; + #[inline] + fn deref(&self) -> &[Word] { + self.inner.as_ref() + } +} + +impl DerefMut for ArrayStorage { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner.as_mut() + } +} + +impl DenseBitSetStorage for ArrayStorage {} +impl DenseBitSetStorageMut for ArrayStorage {} + +impl<'a, T: Idx> DenseBitSet { + fn from_slice(domain_size: usize, storage: &'a [Word]) -> DenseBitSet { + // todo assert storage has space? + DenseBitSet { domain_size, words: storage, marker: PhantomData } + } + + /// Iterates over the indices of set bits in a sorted order. + #[inline] + pub fn into_iter(self) -> BitIter<'a, T> { + BitIter::new(self.words) + } +} + +impl<'a, T: Idx> DenseBitSet { + fn from_slice_mut( + domain_size: usize, + storage: &'a mut [Word], + ) -> DenseBitSet { + // todo assert storage has space? + DenseBitSet { domain_size, words: storage, marker: PhantomData } + } +} + impl DenseBitSet { /// Creates a new, empty bitset with a given `domain_size`. #[inline] @@ -106,18 +183,9 @@ impl DenseBitSet { result.clear_excess_bits(); result } +} - /// Clear all elements. - #[inline] - pub fn clear(&mut self) { - self.words.fill(0); - } - - /// Clear excess bits in the final word. - fn clear_excess_bits(&mut self) { - clear_excess_bits_in_final_word(self.domain_size, &mut self.words); - } - +impl DenseBitSet { /// Count the number of set bits in the set. pub fn count(&self) -> usize { count_ones(&self.words) @@ -133,9 +201,9 @@ impl DenseBitSet { /// Is `self` is a (non-strict) superset of `other`? #[inline] - pub fn superset(&self, other: &DenseBitSet) -> bool { + pub fn superset(&self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - self.words.iter().zip(&other.words).all(|(a, b)| (a & b) == *b) + self.words.iter().zip(&*other.words).all(|(a, b)| (a & b) == *b) } /// Is the set empty? @@ -144,56 +212,6 @@ impl DenseBitSet { self.words.iter().all(|a| *a == 0) } - /// Insert `elem`. Returns whether the set has changed. - #[inline] - pub fn insert(&mut self, elem: T) -> bool { - assert!( - elem.index() < self.domain_size, - "inserting element at index {} but domain size is {}", - elem.index(), - self.domain_size, - ); - let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; - let word = *word_ref; - let new_word = word | mask; - *word_ref = new_word; - new_word != word - } - - #[inline] - pub fn insert_range(&mut self, elems: impl RangeBounds) { - let Some((start, end)) = inclusive_start_end(elems, self.domain_size) else { - return; - }; - - let (start_word_index, start_mask) = word_index_and_mask(start); - let (end_word_index, end_mask) = word_index_and_mask(end); - - // Set all words in between start and end (exclusively of both). - for word_index in (start_word_index + 1)..end_word_index { - self.words[word_index] = !0; - } - - if start_word_index != end_word_index { - // Start and end are in different words, so we handle each in turn. - // - // We set all leading bits. This includes the start_mask bit. - self.words[start_word_index] |= !(start_mask - 1); - // And all trailing bits (i.e. from 0..=end) in the end word, - // including the end. - self.words[end_word_index] |= end_mask | (end_mask - 1); - } else { - self.words[start_word_index] |= end_mask | (end_mask - start_mask); - } - } - - /// Sets all bits to true. - pub fn insert_all(&mut self) { - self.words.fill(!0); - self.clear_excess_bits(); - } - /// Checks whether any bit in the given range is a 1. #[inline] pub fn contains_any(&self, elems: impl RangeBounds) -> bool { @@ -220,18 +238,6 @@ impl DenseBitSet { } } - /// Returns `true` if the set has changed. - #[inline] - pub fn remove(&mut self, elem: T) -> bool { - assert!(elem.index() < self.domain_size); - let (word_index, mask) = word_index_and_mask(elem); - let word_ref = &mut self.words[word_index]; - let word = *word_ref; - let new_word = word & !mask; - *word_ref = new_word; - new_word != word - } - /// Iterates over the indices of set bits in a sorted order. #[inline] pub fn iter(&self) -> BitIter<'_, T> { @@ -282,9 +288,84 @@ impl DenseBitSet { None } +} + +impl DenseBitSet { + /// Clear all elements. + #[inline] + pub fn clear(&mut self) { + self.words.fill(0); + } + + /// Clear excess bits in the final word. + fn clear_excess_bits(&mut self) { + clear_excess_bits_in_final_word(self.domain_size, &mut self.words); + } + + /// Insert `elem`. Returns whether the set has changed. + #[inline] + pub fn insert(&mut self, elem: T) -> bool { + assert!( + elem.index() < self.domain_size, + "inserting element at index {} but domain size is {}", + elem.index(), + self.domain_size, + ); + let (word_index, mask) = word_index_and_mask(elem); + let word_ref = &mut self.words[word_index]; + let word = *word_ref; + let new_word = word | mask; + *word_ref = new_word; + new_word != word + } + + #[inline] + pub fn insert_range(&mut self, elems: impl RangeBounds) { + let Some((start, end)) = inclusive_start_end(elems, self.domain_size) else { + return; + }; + + let (start_word_index, start_mask) = word_index_and_mask(start); + let (end_word_index, end_mask) = word_index_and_mask(end); + + // Set all words in between start and end (exclusively of both). + for word_index in (start_word_index + 1)..end_word_index { + self.words[word_index] = !0; + } + + if start_word_index != end_word_index { + // Start and end are in different words, so we handle each in turn. + // + // We set all leading bits. This includes the start_mask bit. + self.words[start_word_index] |= !(start_mask - 1); + // And all trailing bits (i.e. from 0..=end) in the end word, + // including the end. + self.words[end_word_index] |= end_mask | (end_mask - 1); + } else { + self.words[start_word_index] |= end_mask | (end_mask - start_mask); + } + } + + /// Sets all bits to true. + pub fn insert_all(&mut self) { + self.words.fill(!0); + self.clear_excess_bits(); + } + + /// Returns `true` if the set has changed. + #[inline] + pub fn remove(&mut self, elem: T) -> bool { + assert!(elem.index() < self.domain_size); + let (word_index, mask) = word_index_and_mask(elem); + let word_ref = &mut self.words[word_index]; + let word = *word_ref; + let new_word = word & !mask; + *word_ref = new_word; + new_word != word + } /// Sets `self = self | !other`. - pub fn union_not(&mut self, other: &DenseBitSet) { + pub fn union_not(&mut self, other: &DenseBitSet) { assert_eq!(self.domain_size, other.domain_size); // FIXME(Zalathar): If we were to forcibly _set_ all excess bits before @@ -299,21 +380,21 @@ impl DenseBitSet { } /// Returns true if `self` was modified. - pub fn union(&mut self, other: &DenseBitSet) -> bool { + pub fn union(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); update_words(&mut self.words, &other.words, |a, b| a | b) } /// Returns true if `self` was modified. - pub fn subtract(&mut self, other: &DenseBitSet) -> bool { + pub fn subtract(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); update_words(&mut self.words, &other.words, |a, b| a & !b) } /// Returns true if `self` was modified. - pub fn intersect(&mut self, other: &DenseBitSet) -> bool { + pub fn intersect(&mut self, other: &DenseBitSet) -> bool { assert_eq!(self.domain_size, other.domain_size); - update_words(&mut self.words, &other.words, |a, b| a & b) + update_words(&mut *self.words, &other.words, |a, b| a & b) } } @@ -323,7 +404,7 @@ impl From> for DenseBitSet { } } -impl Clone for DenseBitSet { +impl Clone for DenseBitSet { fn clone(&self) -> Self { DenseBitSet { domain_size: self.domain_size, @@ -338,13 +419,13 @@ impl Clone for DenseBitSet { } } -impl fmt::Debug for DenseBitSet { +impl fmt::Debug for DenseBitSet { fn fmt(&self, w: &mut fmt::Formatter<'_>) -> fmt::Result { w.debug_list().entries(self.iter()).finish() } } -impl ToString for DenseBitSet { +impl ToString for DenseBitSet { fn to_string(&self) -> String { let mut result = String::new(); let mut sep = '['; @@ -353,7 +434,7 @@ impl ToString for DenseBitSet { // i tracks how many bits we have printed so far. let mut i = 0; - for word in &self.words { + for word in &*self.words { let mut word = *word; for _ in 0..WORD_BYTES { // for each byte in `word`: @@ -503,6 +584,18 @@ enum Chunk { #[cfg(target_pointer_width = "64")] crate::static_assert_size!(Chunk, 16); +#[cfg(target_pointer_width = "64")] +crate::static_assert_size!(MixedBitSet, 32); + +#[cfg(target_pointer_width = "64")] +crate::static_assert_size!(DenseBitSet, 32); + +#[cfg(target_pointer_width = "64")] +crate::static_assert_size!(DenseBitSet, 16); + +#[cfg(target_pointer_width = "64")] +crate::static_assert_size!(DenseBitSet, 24); + impl ChunkedBitSet { pub fn domain_size(&self) -> usize { self.domain_size @@ -1512,19 +1605,60 @@ where C: Idx, { num_columns: usize, - rows: IndexVec>>, + // rows: IndexVec>>, + + // flat array of rows (as opposed to a vec of vecs) + // every row uses num_words based on num_columns + data: Vec, + + marker: PhantomData<(R, C)>, } impl SparseBitMatrix { /// Creates a new empty sparse bit matrix with no rows or columns. pub fn new(num_columns: usize) -> Self { - Self { num_columns, rows: IndexVec::new() } + Self { num_columns, data: Vec::new(), marker: PhantomData } } - fn ensure_row(&mut self, row: R) -> &mut DenseBitSet { + fn ensure_row(&mut self, row: R) -> DenseBitSet { // Instantiate any missing rows up to and including row `row` with an empty `DenseBitSet`. // Then replace row `row` with a full `DenseBitSet` if necessary. - self.rows.get_or_insert_with(row, || DenseBitSet::new_empty(self.num_columns)) + let row_index = row.index(); + let desired_num_rows = row_index + 1; + let num_words_per_row = num_words(self.num_columns); + let desired_len = desired_num_rows * num_words_per_row; + + if self.data.len() < desired_len { + self.data.resize(desired_len, 0); + } + + let start_index = row_index * num_words_per_row; + let end_index = (row_index + 1) * num_words_per_row; + let row_storage = &mut self.data[start_index..end_index]; + + DenseBitSet::from_slice_mut(self.num_columns, row_storage) + } + + pub fn row(&self, row: R) -> Option> { + let row_index = row.index(); + let num_words_per_row = num_words(self.num_columns); + + let start_index = row_index * num_words_per_row; + let end_index = (row_index + 1) * num_words_per_row; + let row_storage = self.data.get(start_index..end_index)?; + + Some(DenseBitSet::from_slice(self.num_columns, row_storage)) + } + + fn row_mut(&mut self, row: R) -> Option> { + let row_index = row.index(); + let num_words_per_row = num_words(self.num_columns); + + let start_index = row_index * num_words_per_row; + let end_index = (row_index + 1) * num_words_per_row; + let row_storage = self.data.get_mut(start_index..end_index)?; + + Some(DenseBitSet::from_slice_mut(self.num_columns, row_storage)) } /// Sets the cell at `(row, column)` to true. Put another way, insert @@ -1541,8 +1675,8 @@ impl SparseBitMatrix { /// /// Returns `true` if this changed the matrix. pub fn remove(&mut self, row: R, column: C) -> bool { - match self.rows.get_mut(row) { - Some(Some(row)) => row.remove(column), + match self.row_mut(row) { + Some(mut row) => row.remove(column), _ => false, } } @@ -1550,7 +1684,7 @@ impl SparseBitMatrix { /// Sets all columns at `row` to false. Has no effect if `row` does /// not exist. pub fn clear(&mut self, row: R) { - if let Some(Some(row)) = self.rows.get_mut(row) { + if let Some(mut row) = self.row_mut(row) { row.clear(); } } @@ -1571,16 +1705,32 @@ impl SparseBitMatrix { /// `write` can reach everything that `read` can (and /// potentially more). pub fn union_rows(&mut self, read: R, write: R) -> bool { - if read == write || self.row(read).is_none() { + if read == write || self.num_columns == 0 || self.row(read).is_none() { return false; } self.ensure_row(write); - if let (Some(read_row), Some(write_row)) = self.rows.pick2_mut(read, write) { - write_row.union(read_row) - } else { - unreachable!() - } + + let num_word_per_row = num_words(self.num_columns); + + let read_start = read.index() * num_word_per_row; + let write_start = write.index() * num_word_per_row; + let (ai, bi) = ( + read_start..read_start + num_word_per_row, + write_start..write_start + num_word_per_row, + ); + + let (read_row, mut write_row) = match self.data.get_disjoint_mut([ai.clone(), bi.clone()]) { + Ok([a, b]) => ( + DenseBitSet::::from_slice_mut(self.num_columns, a), + DenseBitSet::::from_slice_mut(self.num_columns, b), + ), + Err(OverlappingIndices) => panic!("Indices {ai:?} and {bi:?} are not disjoint!"), + Err(IndexOutOfBounds) => { + panic!("Some indices among ({ai:?}, {bi:?}) are out of bounds") + } + }; + write_row.union(&read_row) } /// Insert all bits in the given row. @@ -1589,18 +1739,24 @@ impl SparseBitMatrix { } pub fn rows(&self) -> impl Iterator { - self.rows.indices() + let num_words_per_row = num_words(self.num_columns); + + let len = if num_words_per_row == 0 { 0 } else { self.data.len() / num_words_per_row }; + + // fixme copypasta from SliceIndex::indices() + let _ = R::new(len); + (0..len).map(|n| R::new(n)) } /// Iterates through all the columns set to true in a given row of /// the matrix. pub fn iter(&self, row: R) -> impl Iterator { - self.row(row).into_iter().flat_map(|r| r.iter()) + self.row(row).into_iter().flat_map(|r| r.into_iter()) } - pub fn row(&self, row: R) -> Option<&DenseBitSet> { - self.rows.get(row)?.as_ref() - } + // pub fn row(&self, row: R) -> Option<&DenseBitSet> { + // self.rows.get(row)?.as_ref() + // } } #[inline]