Replace per-crate trait impl queries with merged foreign impl indexes

compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs

@@ -12,7 +12,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::find_attr;
 use rustc_middle::bug;
-use rustc_middle::ty::fast_reject::{SimplifiedType, TreatParams, simplify_type};
+use rustc_middle::ty::fast_reject::{TreatParams, simplify_type};
 use rustc_middle::ty::{self, CrateInherentImpls, Ty, TyCtxt};
 use rustc_span::ErrorGuaranteed;
 
@@ -40,13 +40,6 @@ pub(crate) fn crate_inherent_impls_validity_check(
     tcx.crate_inherent_impls(()).1
 }
 
-pub(crate) fn crate_incoherent_impls(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] {
-    let (crate_map, _) = tcx.crate_inherent_impls(());
-    tcx.arena.alloc_from_iter(
-        crate_map.incoherent_impls.get(&simp).unwrap_or(&Vec::new()).iter().map(|d| d.to_def_id()),
-    )
-}
-
 /// On-demand query: yields a vector of the inherent impls for a specific type.
 pub(crate) fn inherent_impls(tcx: TyCtxt<'_>, ty_def_id: LocalDefId) -> &[DefId] {
     let (crate_map, _) = tcx.crate_inherent_impls(());

compiler/rustc_hir_analysis/src/coherence/mod.rs

@@ -142,16 +142,14 @@ fn enforce_empty_impls_for_marker_traits(
 pub(crate) fn provide(providers: &mut Providers) {
     use self::builtin::coerce_unsized_info;
     use self::inherent_impls::{
-        crate_incoherent_impls, crate_inherent_impls, crate_inherent_impls_validity_check,
-        inherent_impls,
+        crate_inherent_impls, crate_inherent_impls_validity_check, inherent_impls,
     };
     use self::inherent_impls_overlap::crate_inherent_impls_overlap_check;
     use self::orphan::orphan_check_impl;
 
     *providers = Providers {
         coherent_trait,
         crate_inherent_impls,
-        crate_incoherent_impls,
         inherent_impls,
         crate_inherent_impls_validity_check,
         crate_inherent_impls_overlap_check,

compiler/rustc_metadata/src/creader.rs

@@ -651,7 +651,6 @@ impl CStore {
         };
 
         let crate_metadata = CrateMetadata::new(
-            tcx,
             metadata,
             crate_root,
             raw_proc_macros,

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -9,7 +9,6 @@ use std::{io, mem};
 pub(super) use cstore_impl::provide;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::owned_slice::OwnedSlice;
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::unhash::UnhashMap;
@@ -97,15 +96,6 @@ pub(crate) struct CrateMetadata {
     // --- Some data pre-decoded from the metadata blob, usually for performance ---
     /// Data about the top-level items in a crate, as well as various crate-level metadata.
     root: CrateRoot,
-    /// Trait impl data.
-    /// FIXME: Used only from queries and can use query cache,
-    /// so pre-decoding can probably be avoided.
-    trait_impls: FxIndexMap<(u32, DefIndex), LazyArray<(DefIndex, Option<SimplifiedType>)>>,
-    /// Inherent impls which do not follow the normal coherence rules.
-    ///
-    /// These can be introduced using either `#![rustc_coherence_is_core]`
-    /// or `#[rustc_allow_incoherent_impl]`.
-    incoherent_impls: FxIndexMap<SimplifiedType, LazyArray<DefIndex>>,
     /// Proc macro function pointers for this crate, if it's a proc macro crate.
     raw_proc_macros: Option<&'static [ProcMacroClient]>,
     /// Source maps for code from the crate.
@@ -1489,43 +1479,76 @@ impl CrateMetadata {
 
     /// Decodes all trait impls in the crate (for rustdoc).
     fn get_trait_impls(&self, tcx: TyCtxt<'_>) -> impl Iterator<Item = DefId> {
-        self.trait_impls.values().flat_map(move |impls| {
-            impls.decode((self, tcx)).map(move |(impl_index, _)| self.local_def_id(impl_index))
+        self.root.impls.decode((self, tcx)).flat_map(move |trait_impls| {
+            trait_impls
+                .impls
+                .decode((self, tcx))
+                .map(move |(impl_index, _)| self.local_def_id(impl_index))
         })
     }
 
-    fn get_incoherent_impls<'tcx>(&self, tcx: TyCtxt<'tcx>, simp: SimplifiedType) -> &'tcx [DefId] {
-        if let Some(impls) = self.incoherent_impls.get(&simp) {
-            tcx.arena.alloc_from_iter(impls.decode((self, tcx)).map(|idx| self.local_def_id(idx)))
-        } else {
-            &[]
+    /// Decodes this crate's trait impl index, invoking `f` with each trait
+    /// (translated into the current session) and the position and length of
+    /// the trait's impl list, without decoding the impl list itself.
+    pub(crate) fn for_each_trait_impls_ref(
+        &self,
+        tcx: TyCtxt<'_>,
+        mut f: impl FnMut(DefId, usize, usize),
+    ) {
+        for trait_impls in self.root.impls.decode((self, tcx)) {
+            let (raw_cnum, index) = trait_impls.trait_id;
+            let raw_cnum = CrateNum::from_u32(raw_cnum);
+            let krate = if raw_cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[raw_cnum] };
+            let trait_def_id = DefId { krate, index };
+            f(trait_def_id, trait_impls.impls.position.get(), trait_impls.impls.num_elems);
         }
     }
 
-    fn get_implementations_of_trait<'tcx>(
+    /// Decodes the impl list at the given position, as previously reported by
+    /// `for_each_trait_impls_ref`.
+    pub(crate) fn decode_trait_impls_at(
         &self,
-        tcx: TyCtxt<'tcx>,
-        trait_def_id: DefId,
-    ) -> &'tcx [(DefId, Option<SimplifiedType>)] {
-        if self.trait_impls.is_empty() {
-            return &[];
-        }
+        tcx: TyCtxt<'_>,
+        position: usize,
+        num_impls: usize,
+    ) -> impl Iterator<Item = (DefId, Option<SimplifiedType>)> {
+        let impls = LazyArray::<(DefIndex, Option<SimplifiedType>)>::from_position_and_num_elems(
+            NonZero::new(position).unwrap(),
+            num_impls,
+        );
+        impls.decode((self, tcx)).map(move |(index, simp)| (self.local_def_id(index), simp))
+    }
 
-        // Do a reverse lookup beforehand to avoid touching the crate_num
-        // hash map in the loop below.
-        let key = match self.reverse_translate_def_id(trait_def_id) {
-            Some(def_id) => (def_id.krate.as_u32(), def_id.index),
-            None => return &[],
-        };
+    /// Decodes this crate's trait impl index, invoking `f` with each trait
+    /// (translated into the current session) and its impls, in encoding order.
+    pub(crate) fn for_each_trait_impls(
+        &self,
+        tcx: TyCtxt<'_>,
+        mut f: impl FnMut(DefId, DefId, Option<SimplifiedType>),
+    ) {
+        for trait_impls in self.root.impls.decode((self, tcx)) {
+            let (raw_cnum, index) = trait_impls.trait_id;
+            let raw_cnum = CrateNum::from_u32(raw_cnum);
+            let krate = if raw_cnum == LOCAL_CRATE { self.cnum } else { self.cnum_map[raw_cnum] };
+            let trait_def_id = DefId { krate, index };
+            for (impl_index, simplified_self_ty) in trait_impls.impls.decode((self, tcx)) {
+                f(trait_def_id, self.local_def_id(impl_index), simplified_self_ty);
+            }
+        }
+    }
 
-        if let Some(impls) = self.trait_impls.get(&key) {
-            tcx.arena.alloc_from_iter(
-                impls
-                    .decode((self, tcx))
-                    .map(|(idx, simplified_self_ty)| (self.local_def_id(idx), simplified_self_ty)),
-            )
-        } else {
-            &[]
+    /// Decodes this crate's incoherent impl index, invoking `f` with each self
+    /// type and its impls, in encoding order.
+    pub(crate) fn for_each_incoherent_impls(
+        &self,
+        tcx: TyCtxt<'_>,
+        mut f: impl FnMut(SimplifiedType, DefId),
+    ) {
+        for incoherent_impls in self.root.incoherent_impls.decode((self, tcx)) {
+            let simp = incoherent_impls.self_ty.decode((self, tcx));
+            for impl_index in incoherent_impls.impls.decode((self, tcx)) {
+                f(simp, self.local_def_id(impl_index));
+            }
         }
     }
 
@@ -1941,7 +1964,6 @@ impl CrateMetadata {
 
 impl CrateMetadata {
     pub(crate) fn new(
-        tcx: TyCtxt<'_>,
         blob: MetadataBlob,
         root: CrateRoot,
         raw_proc_macros: Option<&'static [ProcMacroClient]>,
@@ -1952,23 +1974,16 @@ impl CrateMetadata {
         private_dep: bool,
         host_hash: Option<Svh>,
     ) -> CrateMetadata {
-        let trait_impls = root
-            .impls
-            .decode(&blob)
-            .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
-            .collect();
         let alloc_decoding_state =
             AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect());
 
         // Pre-decode the DefPathHash->DefIndex table. This is a cheap operation
         // that does not copy any data. It just does some data verification.
         let def_path_hash_map = root.def_path_hash_map.decode(&blob);
 
-        let mut cdata = CrateMetadata {
+        CrateMetadata {
             blob,
             root,
-            trait_impls,
-            incoherent_impls: Default::default(),
             raw_proc_macros,
             source_map_import_info: Lock::new(Vec::new()),
             def_path_hash_map,
@@ -1984,18 +1999,7 @@ impl CrateMetadata {
             extern_crate: None,
             hygiene_context: Default::default(),
             def_key_cache: Default::default(),
-        };
-
-        cdata.incoherent_impls = cdata
-            .root
-            .incoherent_impls
-            .decode((&cdata, tcx))
-            .map(|incoherent_impls| {
-                (incoherent_impls.self_ty.decode((&cdata, tcx)), incoherent_impls.impls)
-            })
-            .collect();
-
-        cdata
+        }
     }
 
     pub(crate) fn dependencies(&self) -> impl Iterator<Item = CrateNum> {
@@ -2122,16 +2126,4 @@ impl CrateMetadata {
     fn local_def_id(&self, index: DefIndex) -> DefId {
         DefId { krate: self.cnum, index }
     }
-
-    // Translate a DefId from the current compilation environment to a DefId
-    // for an external crate.
-    fn reverse_translate_def_id(&self, did: DefId) -> Option<DefId> {
-        for (local, &global) in self.cnum_map.iter_enumerated() {
-            if global == did.krate {
-                return Some(DefId { krate: local, index: did.index });
-            }
-        }
-
-        None
-    }
 }

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -15,6 +15,9 @@ use rustc_middle::middle::stability::DeprecationEntry;
 use rustc_middle::queries::ExternProviders;
 use rustc_middle::query::LocalCrate;
 use rustc_middle::ty::fast_reject::SimplifiedType;
+use rustc_middle::ty::trait_def::{
+    ForeignImplsRef, ForeignIncoherentImpls, ForeignTraitImplsIndex,
+};
 use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
 use rustc_middle::util::Providers;
 use rustc_serialize::Decoder;
@@ -386,8 +389,6 @@ provide! { tcx, def_id, other, cdata,
 
     traits => { tcx.arena.alloc_from_iter(cdata.get_traits(tcx)) }
     trait_impls_in_crate => { tcx.arena.alloc_from_iter(cdata.get_trait_impls(tcx)) }
-    implementations_of_trait => { cdata.get_implementations_of_trait(tcx, other) }
-    crate_incoherent_impls => { cdata.get_incoherent_impls(tcx, other) }
 
     crate_dep_kind => { cdata.dep_kind }
     module_children => {
@@ -456,6 +457,59 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
         foreign_modules: foreign_modules::collect,
         externally_implementable_items: eii::collect,
 
+        foreign_trait_impls_index: |tcx, ()| {
+            // Force `crates` before borrowing the cstore: computing it for the
+            // first time freezes the cstore, which would deadlock against the
+            // borrow below.
+            let crates = tcx.crates(());
+            let cstore = CStore::from_tcx(tcx);
+            let mut index = ForeignTraitImplsIndex::default();
+            for &cnum in crates.iter() {
+                // Register a dependency on the crate metadata,
+                // like external query providers do.
+                if tcx.dep_graph.is_fully_enabled() {
+                    tcx.ensure_ok().crate_hash(cnum);
+                }
+                cstore.get_crate_data(cnum).for_each_trait_impls_ref(
+                    tcx,
+                    |trait_def_id, position, num_impls| {
+                        index.impls.entry(trait_def_id).or_default().push(ForeignImplsRef {
+                            cnum,
+                            position,
+                            num_impls,
+                        });
+                    },
+                );
+            }
+            index
+        },
+        foreign_implementations_of_trait: |tcx, trait_id| {
+            let Some(refs) = tcx.foreign_trait_impls_index(()).impls.get(&trait_id) else {
+                return &[];
+            };
+            let cstore = CStore::from_tcx(tcx);
+            tcx.arena.alloc_from_iter(refs.iter().flat_map(|r| {
+                cstore.get_crate_data(r.cnum).decode_trait_impls_at(tcx, r.position, r.num_impls)
+            }))
+        },
+        foreign_incoherent_impls: |tcx, ()| {
+            // Force `crates` before borrowing the cstore, as above.
+            let crates = tcx.crates(());
+            let cstore = CStore::from_tcx(tcx);
+            let mut impls = ForeignIncoherentImpls::default();
+            for &cnum in crates.iter() {
+                // Register a dependency on the crate metadata,
+                // like external query providers do.
+                if tcx.dep_graph.is_fully_enabled() {
+                    tcx.ensure_ok().crate_hash(cnum);
+                }
+                cstore.get_crate_data(cnum).for_each_incoherent_impls(tcx, |simp, impl_def_id| {
+                    impls.impls.entry(simp).or_default().push(impl_def_id);
+                });
+            }
+            impls
+        },
+
         // Returns a map from a sufficiently visible external item (i.e., an
         // external item that is visible from at least one local module) to a
         // sufficiently visible parent (considering modules that re-export the
@@ -664,6 +718,18 @@ impl CStore {
     /// Only public-facing way to traverse all the definitions in a non-local crate.
     /// Critically useful for this third-party project: <https://github.com/hacspec/hacspec>.
     /// See <https://github.com/rust-lang/rust/pull/85889> for context.
+    /// Visits all trait impls of the given crate, without registering any
+    /// dependency on the crate metadata. Used from resolver diagnostics, where
+    /// the crate list must not be frozen yet.
+    pub fn for_each_trait_impl_in_crate_untracked(
+        &self,
+        tcx: TyCtxt<'_>,
+        cnum: CrateNum,
+        f: impl FnMut(DefId, DefId, Option<SimplifiedType>),
+    ) {
+        self.get_crate_data(cnum).for_each_trait_impls(tcx, f)
+    }
+
     pub fn num_def_ids_untracked(&self, cnum: CrateNum) -> usize {
         self.get_crate_data(cnum).num_def_ids()
     }

compiler/rustc_middle/src/queries.rs

@@ -2058,20 +2058,31 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Given a crate and a trait, look up all impls of that trait in the crate.
+    /// Collects an index of all trait impls from external crates, keyed by
+    /// trait, without decoding the impl lists themselves.
+    ///
+    /// Do not call this directly, but instead use the
+    /// `foreign_implementations_of_trait` query.
+    query foreign_trait_impls_index(_: ()) -> &'tcx ty::trait_def::ForeignTraitImplsIndex {
+        arena_cache
+        desc { "indexing trait impls from all dependency crates" }
+    }
+
+    /// Given a trait, look up all impls of that trait in all external crates.
     /// Return `(impl_id, self_ty)`.
-    query implementations_of_trait(_: (CrateNum, DefId)) -> &'tcx [(DefId, Option<SimplifiedType>)] {
-        desc { "looking up implementations of a trait in a crate" }
-        separate_provide_extern
+    ///
+    /// Do not call this directly, but instead use the `trait_impls_of` query.
+    query foreign_implementations_of_trait(trait_id: DefId) -> &'tcx [(DefId, Option<SimplifiedType>)] {
+        desc { "looking up foreign implementations of a trait" }
     }
 
-    /// Collects all incoherent impls for the given crate and type.
+    /// Collects all incoherent inherent impls from external crates, keyed by self type.
     ///
     /// Do not call this directly, but instead use the `incoherent_impls` query.
     /// This query is only used to get the data necessary for that query.
-    query crate_incoherent_impls(key: (CrateNum, SimplifiedType)) -> &'tcx [DefId] {
-        desc { "collecting all impls for a type in a crate" }
-        separate_provide_extern
+    query foreign_incoherent_impls(_: ()) -> &'tcx ty::trait_def::ForeignIncoherentImpls {
+        arena_cache
+        desc { "gathering incoherent impls from all dependency crates" }
     }
 
     /// Get the corresponding native library from the `native_libraries` query