diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs index 4e3aa41942845..6e4a0783094f5 100644 --- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs +++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs @@ -12,7 +12,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::find_attr; use rustc_middle::bug; -use rustc_middle::ty::fast_reject::{SimplifiedType, TreatParams, simplify_type}; +use rustc_middle::ty::fast_reject::{TreatParams, simplify_type}; use rustc_middle::ty::{self, CrateInherentImpls, Ty, TyCtxt}; use rustc_span::ErrorGuaranteed; @@ -40,13 +40,6 @@ pub(crate) fn crate_inherent_impls_validity_check( tcx.crate_inherent_impls(()).1 } -pub(crate) fn crate_incoherent_impls(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] { - let (crate_map, _) = tcx.crate_inherent_impls(()); - tcx.arena.alloc_from_iter( - crate_map.incoherent_impls.get(&simp).unwrap_or(&Vec::new()).iter().map(|d| d.to_def_id()), - ) -} - /// On-demand query: yields a vector of the inherent impls for a specific type. pub(crate) fn inherent_impls(tcx: TyCtxt<'_>, ty_def_id: LocalDefId) -> &[DefId] { let (crate_map, _) = tcx.crate_inherent_impls(()); diff --git a/compiler/rustc_hir_analysis/src/coherence/mod.rs b/compiler/rustc_hir_analysis/src/coherence/mod.rs index 894cd584ae41d..d932225961be3 100644 --- a/compiler/rustc_hir_analysis/src/coherence/mod.rs +++ b/compiler/rustc_hir_analysis/src/coherence/mod.rs @@ -142,8 +142,7 @@ fn enforce_empty_impls_for_marker_traits( pub(crate) fn provide(providers: &mut Providers) { use self::builtin::coerce_unsized_info; use self::inherent_impls::{ - crate_incoherent_impls, crate_inherent_impls, crate_inherent_impls_validity_check, - inherent_impls, + crate_inherent_impls, crate_inherent_impls_validity_check, inherent_impls, }; use self::inherent_impls_overlap::crate_inherent_impls_overlap_check; use self::orphan::orphan_check_impl; @@ -151,7 +150,6 @@ pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { coherent_trait, crate_inherent_impls, - crate_incoherent_impls, inherent_impls, crate_inherent_impls_validity_check, crate_inherent_impls_overlap_check, diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index 8879c175da2f9..b8f3a481c276d 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -651,7 +651,6 @@ impl CStore { }; let crate_metadata = CrateMetadata::new( - tcx, metadata, crate_root, raw_proc_macros, diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 3a7cffb5989fc..b40c941fbff76 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -9,7 +9,6 @@ use std::{io, mem}; pub(super) use cstore_impl::provide; use rustc_ast as ast; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::owned_slice::OwnedSlice; use rustc_data_structures::sync::Lock; use rustc_data_structures::unhash::UnhashMap; @@ -97,15 +96,6 @@ pub(crate) struct CrateMetadata { // --- Some data pre-decoded from the metadata blob, usually for performance --- /// Data about the top-level items in a crate, as well as various crate-level metadata. root: CrateRoot, - /// Trait impl data. - /// FIXME: Used only from queries and can use query cache, - /// so pre-decoding can probably be avoided. - trait_impls: FxIndexMap<(u32, DefIndex), LazyArray<(DefIndex, Option)>>, - /// Inherent impls which do not follow the normal coherence rules. - /// - /// These can be introduced using either `#![rustc_coherence_is_core]` - /// or `#[rustc_allow_incoherent_impl]`. - incoherent_impls: FxIndexMap>, /// Proc macro function pointers for this crate, if it's a proc macro crate. raw_proc_macros: Option<&'static [ProcMacroClient]>, /// Source maps for code from the crate. @@ -1489,43 +1479,76 @@ impl CrateMetadata { /// Decodes all trait impls in the crate (for rustdoc). fn get_trait_impls(&self, tcx: TyCtxt<'_>) -> impl Iterator { - self.trait_impls.values().flat_map(move |impls| { - impls.decode((self, tcx)).map(move |(impl_index, _)| self.local_def_id(impl_index)) + self.root.impls.decode((self, tcx)).flat_map(move |trait_impls| { + trait_impls + .impls + .decode((self, tcx)) + .map(move |(impl_index, _)| self.local_def_id(impl_index)) }) } - fn get_incoherent_impls<'tcx>(&self, tcx: TyCtxt<'tcx>, simp: SimplifiedType) -> &'tcx [DefId] { - if let Some(impls) = self.incoherent_impls.get(&simp) { - tcx.arena.alloc_from_iter(impls.decode((self, tcx)).map(|idx| self.local_def_id(idx))) - } else { - &[] + /// Decodes this crate's trait impl index, invoking `f` with each trait + /// (translated into the current session) and the position and length of + /// the trait's impl list, without decoding the impl list itself. + pub(crate) fn for_each_trait_impls_ref( + &self, + tcx: TyCtxt<'_>, + mut f: impl FnMut(DefId, usize, usize), + ) { + for trait_impls in self.root.impls.decode((self, tcx)) { + let (raw_cnum, index) = trait_impls.trait_id; + let raw_cnum = CrateNum::from_u32(raw_cnum); + let krate = if raw_cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[raw_cnum] }; + let trait_def_id = DefId { krate, index }; + f(trait_def_id, trait_impls.impls.position.get(), trait_impls.impls.num_elems); } } - fn get_implementations_of_trait<'tcx>( + /// Decodes the impl list at the given position, as previously reported by + /// `for_each_trait_impls_ref`. + pub(crate) fn decode_trait_impls_at( &self, - tcx: TyCtxt<'tcx>, - trait_def_id: DefId, - ) -> &'tcx [(DefId, Option)] { - if self.trait_impls.is_empty() { - return &[]; - } + tcx: TyCtxt<'_>, + position: usize, + num_impls: usize, + ) -> impl Iterator)> { + let impls = LazyArray::<(DefIndex, Option)>::from_position_and_num_elems( + NonZero::new(position).unwrap(), + num_impls, + ); + impls.decode((self, tcx)).map(move |(index, simp)| (self.local_def_id(index), simp)) + } - // Do a reverse lookup beforehand to avoid touching the crate_num - // hash map in the loop below. - let key = match self.reverse_translate_def_id(trait_def_id) { - Some(def_id) => (def_id.krate.as_u32(), def_id.index), - None => return &[], - }; + /// Decodes this crate's trait impl index, invoking `f` with each trait + /// (translated into the current session) and its impls, in encoding order. + pub(crate) fn for_each_trait_impls( + &self, + tcx: TyCtxt<'_>, + mut f: impl FnMut(DefId, DefId, Option), + ) { + for trait_impls in self.root.impls.decode((self, tcx)) { + let (raw_cnum, index) = trait_impls.trait_id; + let raw_cnum = CrateNum::from_u32(raw_cnum); + let krate = if raw_cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[raw_cnum] }; + let trait_def_id = DefId { krate, index }; + for (impl_index, simplified_self_ty) in trait_impls.impls.decode((self, tcx)) { + f(trait_def_id, self.local_def_id(impl_index), simplified_self_ty); + } + } + } - if let Some(impls) = self.trait_impls.get(&key) { - tcx.arena.alloc_from_iter( - impls - .decode((self, tcx)) - .map(|(idx, simplified_self_ty)| (self.local_def_id(idx), simplified_self_ty)), - ) - } else { - &[] + /// Decodes this crate's incoherent impl index, invoking `f` with each self + /// type and its impls, in encoding order. + pub(crate) fn for_each_incoherent_impls( + &self, + tcx: TyCtxt<'_>, + mut f: impl FnMut(SimplifiedType, DefId), + ) { + for incoherent_impls in self.root.incoherent_impls.decode((self, tcx)) { + let simp = incoherent_impls.self_ty.decode((self, tcx)); + for impl_index in incoherent_impls.impls.decode((self, tcx)) { + f(simp, self.local_def_id(impl_index)); + } } } @@ -1941,7 +1964,6 @@ impl CrateMetadata { impl CrateMetadata { pub(crate) fn new( - tcx: TyCtxt<'_>, blob: MetadataBlob, root: CrateRoot, raw_proc_macros: Option<&'static [ProcMacroClient]>, @@ -1952,11 +1974,6 @@ impl CrateMetadata { private_dep: bool, host_hash: Option, ) -> CrateMetadata { - let trait_impls = root - .impls - .decode(&blob) - .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls)) - .collect(); let alloc_decoding_state = AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect()); @@ -1964,11 +1981,9 @@ impl CrateMetadata { // that does not copy any data. It just does some data verification. let def_path_hash_map = root.def_path_hash_map.decode(&blob); - let mut cdata = CrateMetadata { + CrateMetadata { blob, root, - trait_impls, - incoherent_impls: Default::default(), raw_proc_macros, source_map_import_info: Lock::new(Vec::new()), def_path_hash_map, @@ -1984,18 +1999,7 @@ impl CrateMetadata { extern_crate: None, hygiene_context: Default::default(), def_key_cache: Default::default(), - }; - - cdata.incoherent_impls = cdata - .root - .incoherent_impls - .decode((&cdata, tcx)) - .map(|incoherent_impls| { - (incoherent_impls.self_ty.decode((&cdata, tcx)), incoherent_impls.impls) - }) - .collect(); - - cdata + } } pub(crate) fn dependencies(&self) -> impl Iterator { @@ -2122,16 +2126,4 @@ impl CrateMetadata { fn local_def_id(&self, index: DefIndex) -> DefId { DefId { krate: self.cnum, index } } - - // Translate a DefId from the current compilation environment to a DefId - // for an external crate. - fn reverse_translate_def_id(&self, did: DefId) -> Option { - for (local, &global) in self.cnum_map.iter_enumerated() { - if global == did.krate { - return Some(DefId { krate: local, index: did.index }); - } - } - - None - } } diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index a9d524711a141..f5a506be5afb1 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -15,6 +15,9 @@ use rustc_middle::middle::stability::DeprecationEntry; use rustc_middle::queries::ExternProviders; use rustc_middle::query::LocalCrate; use rustc_middle::ty::fast_reject::SimplifiedType; +use rustc_middle::ty::trait_def::{ + ForeignImplsRef, ForeignIncoherentImpls, ForeignTraitImplsIndex, +}; use rustc_middle::ty::{self, TyCtxt, TypeVisitable}; use rustc_middle::util::Providers; use rustc_serialize::Decoder; @@ -386,8 +389,6 @@ provide! { tcx, def_id, other, cdata, traits => { tcx.arena.alloc_from_iter(cdata.get_traits(tcx)) } trait_impls_in_crate => { tcx.arena.alloc_from_iter(cdata.get_trait_impls(tcx)) } - implementations_of_trait => { cdata.get_implementations_of_trait(tcx, other) } - crate_incoherent_impls => { cdata.get_incoherent_impls(tcx, other) } crate_dep_kind => { cdata.dep_kind } module_children => { @@ -456,6 +457,59 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) { foreign_modules: foreign_modules::collect, externally_implementable_items: eii::collect, + foreign_trait_impls_index: |tcx, ()| { + // Force `crates` before borrowing the cstore: computing it for the + // first time freezes the cstore, which would deadlock against the + // borrow below. + let crates = tcx.crates(()); + let cstore = CStore::from_tcx(tcx); + let mut index = ForeignTraitImplsIndex::default(); + for &cnum in crates.iter() { + // Register a dependency on the crate metadata, + // like external query providers do. + if tcx.dep_graph.is_fully_enabled() { + tcx.ensure_ok().crate_hash(cnum); + } + cstore.get_crate_data(cnum).for_each_trait_impls_ref( + tcx, + |trait_def_id, position, num_impls| { + index.impls.entry(trait_def_id).or_default().push(ForeignImplsRef { + cnum, + position, + num_impls, + }); + }, + ); + } + index + }, + foreign_implementations_of_trait: |tcx, trait_id| { + let Some(refs) = tcx.foreign_trait_impls_index(()).impls.get(&trait_id) else { + return &[]; + }; + let cstore = CStore::from_tcx(tcx); + tcx.arena.alloc_from_iter(refs.iter().flat_map(|r| { + cstore.get_crate_data(r.cnum).decode_trait_impls_at(tcx, r.position, r.num_impls) + })) + }, + foreign_incoherent_impls: |tcx, ()| { + // Force `crates` before borrowing the cstore, as above. + let crates = tcx.crates(()); + let cstore = CStore::from_tcx(tcx); + let mut impls = ForeignIncoherentImpls::default(); + for &cnum in crates.iter() { + // Register a dependency on the crate metadata, + // like external query providers do. + if tcx.dep_graph.is_fully_enabled() { + tcx.ensure_ok().crate_hash(cnum); + } + cstore.get_crate_data(cnum).for_each_incoherent_impls(tcx, |simp, impl_def_id| { + impls.impls.entry(simp).or_default().push(impl_def_id); + }); + } + impls + }, + // Returns a map from a sufficiently visible external item (i.e., an // external item that is visible from at least one local module) to a // sufficiently visible parent (considering modules that re-export the @@ -664,6 +718,18 @@ impl CStore { /// Only public-facing way to traverse all the definitions in a non-local crate. /// Critically useful for this third-party project: . /// See for context. + /// Visits all trait impls of the given crate, without registering any + /// dependency on the crate metadata. Used from resolver diagnostics, where + /// the crate list must not be frozen yet. + pub fn for_each_trait_impl_in_crate_untracked( + &self, + tcx: TyCtxt<'_>, + cnum: CrateNum, + f: impl FnMut(DefId, DefId, Option), + ) { + self.get_crate_data(cnum).for_each_trait_impls(tcx, f) + } + pub fn num_def_ids_untracked(&self, cnum: CrateNum) -> usize { self.get_crate_data(cnum).num_def_ids() } diff --git a/compiler/rustc_middle/src/queries.rs b/compiler/rustc_middle/src/queries.rs index ca1cd2f45975f..82b7dd636cdc4 100644 --- a/compiler/rustc_middle/src/queries.rs +++ b/compiler/rustc_middle/src/queries.rs @@ -2058,20 +2058,31 @@ rustc_queries! { separate_provide_extern } - /// Given a crate and a trait, look up all impls of that trait in the crate. + /// Collects an index of all trait impls from external crates, keyed by + /// trait, without decoding the impl lists themselves. + /// + /// Do not call this directly, but instead use the + /// `foreign_implementations_of_trait` query. + query foreign_trait_impls_index(_: ()) -> &'tcx ty::trait_def::ForeignTraitImplsIndex { + arena_cache + desc { "indexing trait impls from all dependency crates" } + } + + /// Given a trait, look up all impls of that trait in all external crates. /// Return `(impl_id, self_ty)`. - query implementations_of_trait(_: (CrateNum, DefId)) -> &'tcx [(DefId, Option)] { - desc { "looking up implementations of a trait in a crate" } - separate_provide_extern + /// + /// Do not call this directly, but instead use the `trait_impls_of` query. + query foreign_implementations_of_trait(trait_id: DefId) -> &'tcx [(DefId, Option)] { + desc { "looking up foreign implementations of a trait" } } - /// Collects all incoherent impls for the given crate and type. + /// Collects all incoherent inherent impls from external crates, keyed by self type. /// /// Do not call this directly, but instead use the `incoherent_impls` query. /// This query is only used to get the data necessary for that query. - query crate_incoherent_impls(key: (CrateNum, SimplifiedType)) -> &'tcx [DefId] { - desc { "collecting all impls for a type in a crate" } - separate_provide_extern + query foreign_incoherent_impls(_: ()) -> &'tcx ty::trait_def::ForeignIncoherentImpls { + arena_cache + desc { "gathering incoherent impls from all dependency crates" } } /// Get the corresponding native library from the `native_libraries` query diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs index 5309e35b1073c..5e9a376291577 100644 --- a/compiler/rustc_middle/src/ty/trait_def.rs +++ b/compiler/rustc_middle/src/ty/trait_def.rs @@ -1,9 +1,7 @@ -use std::iter; - use rustc_data_structures::fx::FxIndexMap; use rustc_errors::ErrorGuaranteed; use rustc_hir::def::DefKind; -use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_hir::def_id::{CrateNum, DefId}; use rustc_hir::{self as hir, find_attr}; use rustc_macros::{Decodable, Encodable, StableHash}; use rustc_span::Span; @@ -107,6 +105,35 @@ pub struct TraitImpls { non_blanket_impls: FxIndexMap>, } +/// Reference to the impl list a single crate encodes for one trait, as an +/// offset into that crate's metadata blob. Only `rustc_metadata` knows how to +/// decode it. +#[derive(Debug, Copy, Clone, StableHash)] +pub struct ForeignImplsRef { + pub cnum: CrateNum, + pub position: usize, + pub num_impls: usize, +} + +/// Index of all trait impls from external crates, keyed by trait. +/// +/// Entries are recorded in crate loading order, and in metadata encoding order +/// within a crate, so decoding a trait's impl lists produces impls in the same +/// order as querying each crate separately would. The impl lists themselves +/// stay lazy: only `foreign_implementations_of_trait` decodes them, per trait. +#[derive(Default, Debug, StableHash)] +pub struct ForeignTraitImplsIndex { + pub impls: FxIndexMap>, +} + +/// All incoherent inherent impls from external crates, keyed by self type. +/// +/// Like `ForeignTraitImplsIndex`, entries are in crate loading order. +#[derive(Default, Debug, StableHash)] +pub struct ForeignIncoherentImpls { + pub impls: FxIndexMap>, +} + impl TraitImpls { pub fn is_empty(&self) -> bool { self.blanket_impls.is_empty() && self.non_blanket_impls.is_empty() @@ -308,19 +335,11 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait // Traits defined in the current crate can't have impls in upstream // crates, so we don't bother querying the cstore. if !trait_id.is_local() { - for &cnum in tcx.crates(()).iter() { - for &(impl_def_id, simplified_self_ty) in - tcx.implementations_of_trait((cnum, trait_id)).iter() - { - if let Some(simplified_self_ty) = simplified_self_ty { - impls - .non_blanket_impls - .entry(simplified_self_ty) - .or_default() - .push(impl_def_id); - } else { - impls.blanket_impls.push(impl_def_id); - } + for &(impl_def_id, simplified_self_ty) in tcx.foreign_implementations_of_trait(trait_id) { + if let Some(simplified_self_ty) = simplified_self_ty { + impls.non_blanket_impls.entry(simplified_self_ty).or_default().push(impl_def_id); + } else { + impls.blanket_impls.push(impl_def_id); } } } @@ -351,10 +370,14 @@ pub(super) fn incoherent_impls_provider(tcx: TyCtxt<'_>, simp: SimplifiedType) - } let mut impls = Vec::new(); - for cnum in iter::once(LOCAL_CRATE).chain(tcx.crates(()).iter().copied()) { - for &impl_def_id in tcx.crate_incoherent_impls((cnum, simp)) { - impls.push(impl_def_id) - } + // Local impls come first, then foreign impls in crate loading order, + // matching the previous per-crate iteration order. + let (crate_map, _) = tcx.crate_inherent_impls(()); + if let Some(local) = crate_map.incoherent_impls.get(&simp) { + impls.extend(local.iter().map(|def_id| def_id.to_def_id())); + } + if let Some(foreign) = tcx.foreign_incoherent_impls(()).impls.get(&simp) { + impls.extend_from_slice(foreign); } debug!(?impls); diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index b126272583692..29382b93f9a6e 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -12,7 +12,6 @@ use rustc_ast::{ }; use rustc_ast_pretty::pprust::{path_to_string, where_bound_predicate_to_string}; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet}; -use rustc_data_structures::unord::UnordItems; use rustc_errors::codes::*; use rustc_errors::{ Applicability, Diag, Diagnostic, ErrorGuaranteed, MultiSpan, SuggestionStyle, pluralize, @@ -2718,23 +2717,25 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> { let Some(default_trait) = default_trait else { return; }; - if self - .r - .extern_crate_map - .items() - // FIXME: This doesn't include impls like `impl Default for String`. - .flat_map(|(_, crate_)| { - UnordItems::new( - self.r.tcx.implementations_of_trait((*crate_, default_trait)).into_iter(), - ) - }) - .filter_map(|(_, simplified_self_ty)| *simplified_self_ty) - .filter_map(|simplified_self_ty| match simplified_self_ty { - SimplifiedType::Adt(did) => Some(did), - _ => None, - }) - .any(|did| did == def_id) - { + // This deliberately avoids queries that would snapshot and freeze the + // loaded-crate list, as the resolver may still load crates after this. + // FIXME: This doesn't include impls like `impl Default for String`, + // because only direct extern crates are considered. + let cstore = self.r.cstore(); + let has_default_impl = self.r.extern_crate_map.items().any(|(_, crate_)| { + let mut found = false; + cstore.for_each_trait_impl_in_crate_untracked( + self.r.tcx, + *crate_, + |trait_def_id, _impl_def_id, simplified_self_ty| { + found |= trait_def_id == default_trait + && matches!(simplified_self_ty, Some(SimplifiedType::Adt(did)) if did == def_id); + }, + ); + found + }); + drop(cstore); + if has_default_impl { err.multipart_suggestion( "consider using the `Default` trait", vec![