From 085f7abb56474d77409fdf97dfe54682bf170dd6 Mon Sep 17 00:00:00 2001 From: xmakro Date: Tue, 7 Jul 2026 09:12:48 -0700 Subject: [PATCH] Carry green query values forward from the previous cache file Saving the incremental query cache used to require every value it contains to be in memory: values of green nodes that were never used during the session were loaded from the previous cache file by the promotion pass, only to be re-encoded, byte for byte equivalent, into the new file. On a warm rebuild this decode and re-encode round trip for the entire cache dominates the cost of saving. Instead, copy the previous cache file's data region into the new file verbatim and reference the values of green nodes at their old positions. Copying the region unchanged keeps every internal absolute reference valid: type and predicate shorthands, symbol offsets and allocation data all stay at their offsets. The lookup tables around the region are adjusted instead: source file indices preserve all previous assignments and new files append after them, the allocation index keeps the previous entries and offsets this session's, expansions are keyed by their stable hash and merge, and syntax context tables become generational, one per carried region, selected by the position a context id is decoded from, since the raw ids are local to the session that wrote them. After eight carried generations the save falls back to a full re-encode, which compacts away dead data left behind by red nodes and stale tables. Values are now tagged with their node's key fingerprint instead of its session-local index, so the load side sanity check keeps working for values of any age. The promotion pass is deleted: green values no longer need to be in memory at all, and its remaining purpose, referencing them in the next session, is now a walk over the color map that costs a map insert per green node. Values that are in memory anyway, because their query ran this session, are still encoded freshly, and the load path prefers the fresh entry. On Windows the previous file cannot stay mapped while it is replaced, so the carry is disabled there for now and green values that were not loaded during the session are recomputed by the next one, matching what already happened to never-loaded values before this change. The header format version is bumped so caches written by earlier compilers, whose footer layout differs, are discarded cleanly. --- .../src/persist/file_format.rs | 2 +- .../rustc_incremental/src/persist/save.rs | 31 ++- .../rustc_middle/src/dep_graph/dep_node.rs | 13 +- compiler/rustc_middle/src/dep_graph/graph.rs | 34 +-- .../rustc_middle/src/query/on_disk_cache.rs | 253 +++++++++++++++--- .../rustc_query_impl/src/dep_kind_vtables.rs | 17 +- compiler/rustc_query_impl/src/plumbing.rs | 70 +---- 7 files changed, 260 insertions(+), 160 deletions(-) diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs index 853a5c9ba7ab0..0e4102e54b77f 100644 --- a/compiler/rustc_incremental/src/persist/file_format.rs +++ b/compiler/rustc_incremental/src/persist/file_format.rs @@ -26,7 +26,7 @@ use crate::diagnostics; const FILE_MAGIC: &[u8] = b"RSIC"; /// Change this if the header format changes. -const HEADER_FORMAT_VERSION: u16 = 0; +const HEADER_FORMAT_VERSION: u16 = 1; pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) { stream.emit_raw_bytes(FILE_MAGIC); diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs index 12f674fe2a859..d3cb38b4f4168 100644 --- a/compiler/rustc_incremental/src/persist/save.rs +++ b/compiler/rustc_incremental/src/persist/save.rs @@ -62,25 +62,24 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) { // even if there was no previous session. let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap(); - // For every green dep node that has a disk-cached value from the - // previous session, make sure the value is loaded into the memory - // cache, so that it will be serialized as part of this session. - // - // This reads data from the previous session, so it needs to happen - // before dropping the mmap. - // - // FIXME(Zalathar): This step is intended to be cheap, but still does - // quite a lot of work, especially in builds with few or no changes. - // Can we be smarter about how we identify values that need promotion? - // Can we promote values without decoding them into the memory cache? - tcx.dep_graph.exec_cache_promotions(tcx); - - // Drop the memory map so that we can remove the file and write to it. - on_disk_cache.close_serialized_data_mmap(); + // The values of green nodes are carried forward from the previous + // cache file byte for byte, so its contents must stay readable + // while the new file is written. On unix, removing the old file + // does not invalidate the mapping, so it is kept alive until + // serialization is done. On Windows the file cannot be removed + // while mapped, so the mapping is dropped and the values of green + // nodes are lost to the next session, like they already were for + // nodes that were never loaded during this one. + let carried_data = if cfg!(windows) { + on_disk_cache.close_serialized_data_mmap(); + None + } else { + on_disk_cache.take_serialized_data_mmap() + }; file_format::save_in(sess, query_cache_path, "query cache", |encoder| { tcx.sess.time("incr_comp_serialize_result_cache", || { - on_disk_cache::OnDiskCache::serialize(tcx, encoder) + on_disk_cache::OnDiskCache::serialize(tcx, encoder, carried_data) }) }); }); diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs index 6abec9a4ff465..46f74ae966d1d 100644 --- a/compiler/rustc_middle/src/dep_graph/dep_node.rs +++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs @@ -57,7 +57,7 @@ use rustc_hir::definitions::DefPathHash; use rustc_macros::{Decodable, Encodable, StableHash}; use rustc_span::Symbol; -use super::{DepNodeIndex, KeyFingerprintStyle, SerializedDepNodeIndex}; +use super::{KeyFingerprintStyle, SerializedDepNodeIndex}; use crate::dep_graph::DepNodeKey; use crate::mono::MonoItem; use crate::ty::{TyCtxt, tls}; @@ -216,17 +216,6 @@ pub struct DepKindVTable<'tcx> { fn(tcx: TyCtxt<'tcx>, dep_node: DepNode, prev_index: SerializedDepNodeIndex) -> bool, >, - /// Load the on-disk cached value of a query into memory. The node is known - /// to be green, with `prev_index` its index in the previous session's dep - /// graph and `dep_node_index` its index in the current session's dep graph. - pub promote_from_disk_fn: Option< - fn( - tcx: TyCtxt<'tcx>, - dep_node: DepNode, - prev_index: SerializedDepNodeIndex, - dep_node_index: DepNodeIndex, - ), - >, } /// A "work product" corresponds to a `.o` (or other) file that we diff --git a/compiler/rustc_middle/src/dep_graph/graph.rs b/compiler/rustc_middle/src/dep_graph/graph.rs index cba7c14533484..d7c6765b90c5e 100644 --- a/compiler/rustc_middle/src/dep_graph/graph.rs +++ b/compiler/rustc_middle/src/dep_graph/graph.rs @@ -701,6 +701,13 @@ impl DepGraphData { } #[inline] + /// The key fingerprint of a node of the previous session. Used as a + /// session-stable tag when loading cached query values, since the node's + /// index changes from session to session while its key does not. + pub fn prev_key_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> PackedFingerprint { + self.previous.index_to_node(prev_index).key_fingerprint + } + pub fn prev_value_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint { self.previous.value_fingerprint_for_index(prev_index) } @@ -1067,25 +1074,18 @@ impl DepGraph { /// /// This method will only load queries that will end up in the disk cache. /// Other queries will not be executed. - pub fn exec_cache_promotions<'tcx>(&self, tcx: TyCtxt<'tcx>) { - let _prof_timer = tcx.prof.generic_activity("incr_comp_query_cache_promotion"); - + /// Invokes `f` for every node of the previous session that was marked + /// green during this session, together with its current-session index. + /// Used when saving the query cache, to reference the still-valid values + /// of green nodes at their positions in the previous cache file. + pub fn for_each_green_prev_index( + &self, + f: &mut dyn FnMut(SerializedDepNodeIndex, DepNodeIndex), + ) { let data = self.data.as_ref().unwrap(); for prev_index in data.colors.values.indices() { - match data.colors.get(prev_index) { - DepNodeColor::Green(dep_node_index) => { - let dep_node = data.previous.index_to_node(prev_index); - if let Some(promote_fn) = - tcx.dep_kind_vtable(dep_node.kind).promote_from_disk_fn - { - promote_fn(tcx, *dep_node, prev_index, dep_node_index) - }; - } - DepNodeColor::Unknown | DepNodeColor::Red => { - // We can skip red nodes because a node can only be marked - // as red if the query result was recomputed and thus is - // already in memory. - } + if let DepNodeColor::Green(dep_node_index) = data.colors.get(prev_index) { + f(prev_index, dep_node_index); } } } diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index 1dd510da06886..044c8548bd448 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use std::{fmt, mem}; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::fingerprint::PackedFingerprint; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::sync::{HashMapExt, Lock, RwLock}; use rustc_data_structures::unhash::UnhashMap; @@ -53,6 +54,13 @@ pub struct OnDiskCache { // The complete cache data in serialized form. serialized_data: RwLock>, + // The byte range of the previous session's data region (everything before + // the footer). When possible, this region is carried forward verbatim into + // the next cache file, so that the values of green nodes never need to be + // decoded and re-encoded. + start_pos: usize, + footer_pos: usize, + file_index_to_stable_id: FxHashMap, // Caches that are populated lazily during decoding. @@ -68,12 +76,22 @@ pub struct OnDiskCache { alloc_decoding_state: AllocDecodingState, - // A map from syntax context ids to the position of their associated + /// The previous session's raw allocation index, kept for seeding the next + /// session's index when the data region is carried forward. + prev_interpret_alloc_index: Vec, + + // Maps from syntax context ids to the position of their associated // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext` // to represent the fact that we are storing *encoded* ids. When we decode // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`, // which will almost certainly be different than the serialized id. - syntax_contexts: FxHashMap, + // + // The ids are local to the session that encoded them, and the data region + // of a cache file can contain regions carried forward from several + // earlier sessions: one table per region, tagged with the position one + // past the region's end, ordered oldest first. The table for a given + // encoded id is selected by the position the id was decoded from. + syntax_context_tables: Vec<(u64, FxHashMap)>, // A map from the `DefPathHash` of an `ExpnId` to the position // of their associated `ExpnData`. Ideally, we would store a `DefId`, // but we need to decode this before we've constructed a `TyCtxt` (which @@ -84,8 +102,9 @@ pub struct OnDiskCache { // we could look up the `ExpnData` from the metadata of foreign crates, // but it seemed easier to have `OnDiskCache` be independent of the `CStore`. expn_data: UnhashMap, - // Additional information used when decoding hygiene data. - hygiene_context: HygieneDecodeContext, + // Additional information used when decoding hygiene data, one per + // syntax context table (the decoded-id caches are id-space specific). + hygiene_contexts: Vec, // Maps `ExpnHash`es to their raw value from the *previous* // compilation session. This is used as an initial 'guess' when // we try to map an `ExpnHash` to its value in the current @@ -103,8 +122,8 @@ struct Footer { // Most uses only need values up to u32::MAX, but benchmarking indicates that we can use a u64 // without measurable overhead. This permits larger const allocations without ICEing. interpret_alloc_index: Vec, - // See `OnDiskCache.syntax_contexts` - syntax_contexts: FxHashMap, + // See `OnDiskCache.syntax_context_tables` + syntax_context_tables: Vec<(u64, FxHashMap)>, // See `OnDiskCache.expn_data` expn_data: UnhashMap, foreign_expn_data: UnhashMap, @@ -128,7 +147,7 @@ impl AbsoluteBytePos { } } -#[derive(Encodable, Decodable, Clone, Debug)] +#[derive(Encodable, Decodable, Clone, Debug, PartialEq, Eq, Hash)] struct EncodedSourceFileId { stable_source_file_id: StableSourceFileId, stable_crate_id: StableCrateId, @@ -164,32 +183,40 @@ impl OnDiskCache { let footer: Footer = decoder.with_position(footer_pos, |decoder| decode_tagged(decoder, TAG_FILE_FOOTER)); + let hygiene_contexts = + footer.syntax_context_tables.iter().map(|_| Default::default()).collect(); Ok(Self { serialized_data: RwLock::new(Some(data)), + start_pos, + footer_pos, file_index_to_stable_id: footer.file_index_to_stable_id, file_index_to_file: Default::default(), query_values_index: footer.query_values_index.into_iter().collect(), side_effects_index: footer.side_effects_index.into_iter().collect(), + prev_interpret_alloc_index: footer.interpret_alloc_index.clone(), alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index), - syntax_contexts: footer.syntax_contexts, + syntax_context_tables: footer.syntax_context_tables, expn_data: footer.expn_data, foreign_expn_data: footer.foreign_expn_data, - hygiene_context: Default::default(), + hygiene_contexts, }) } pub fn new_empty() -> Self { Self { serialized_data: RwLock::new(None), + start_pos: 0, + footer_pos: 0, file_index_to_stable_id: Default::default(), file_index_to_file: Default::default(), query_values_index: Default::default(), side_effects_index: Default::default(), + prev_interpret_alloc_index: Vec::new(), alloc_decoding_state: AllocDecodingState::new(Vec::new()), - syntax_contexts: FxHashMap::default(), + syntax_context_tables: Vec::new(), expn_data: UnhashMap::default(), foreign_expn_data: UnhashMap::default(), - hygiene_context: Default::default(), + hygiene_contexts: Vec::new(), } } @@ -199,38 +226,118 @@ impl OnDiskCache { *self.serialized_data.write() = None; } + /// Take ownership of the serialized backing `Mmap`, so its data region can + /// be carried forward into the next cache file while the old file itself + /// is unlinked and replaced. + pub fn take_serialized_data_mmap(&self) -> Option { + self.serialized_data.write().take() + } + /// Serialize the current-session data that will be loaded by [`OnDiskCache`] /// in a subsequent incremental compilation session. - pub fn serialize(tcx: TyCtxt<'_>, encoder: FileEncoder<'static>) -> FileEncodeResult { + /// + /// When `carried_data` holds the previous session's cache contents, its + /// data region is copied into the new file verbatim, and the values of + /// green dep nodes are referenced at their old positions instead of being + /// decoded into memory and re-encoded. All position-dependent references + /// inside the region (type and symbol shorthands, allocation data) stay + /// valid because the region keeps its exact offsets. + pub fn serialize( + tcx: TyCtxt<'_>, + mut encoder: FileEncoder<'static>, + carried_data: Option, + ) -> FileEncodeResult { // Serializing the `DepGraph` should not modify it. tcx.dep_graph.with_ignore(|| { + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap(); + + // Bound the number of carried generations: every generation keeps + // its own syntax context table alive and dead data from red nodes + // accumulates, so occasionally fall back to a full re-encode, + // which compacts the cache again. + const MAX_CARRIED_GENERATIONS: usize = 8; + + let carried: Option<&[u8]> = carried_data.as_deref().filter(|data| { + on_disk_cache.footer_pos > on_disk_cache.start_pos + // The copied region keeps its offsets only if the new + // file's header has the same length as the old one. + && on_disk_cache.start_pos == encoder.position() + && on_disk_cache.syntax_context_tables.len() < MAX_CARRIED_GENERATIONS + && data.len() >= on_disk_cache.footer_pos + }).map(|data| &data[on_disk_cache.start_pos..on_disk_cache.footer_pos]); + + // Copy the previous data region before anything else is encoded. + if let Some(bytes) = carried { + encoder.emit_raw_bytes(bytes); + } + // Allocate `SourceFileIndex`es. let (file_to_file_index, file_index_to_stable_id) = { let files = tcx.sess.source_map().files(); let mut file_to_file_index = FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); - let mut file_index_to_stable_id = - FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); - for (index, file) in files.iter().enumerate() { - let index = SourceFileIndex(index as u32); - let file_ptr: *const SourceFile = &raw const **file; - file_to_file_index.insert(file_ptr, index); - let source_file_id = EncodedSourceFileId::new(tcx, file); - file_index_to_stable_id.insert(index, source_file_id); - } + if carried.is_some() { + // Spans in the carried region reference the previous + // sessions' file indices: preserve every old assignment, + // including ones whose file is gone (their indices must + // not be reused), and append new files after them. + let mut file_index_to_stable_id = on_disk_cache.file_index_to_stable_id.clone(); + // The maps are only used for lookups and max computation here, + // so the iteration order does not affect the output. + #[allow(rustc::potential_query_instability)] + let mut stable_id_to_index: FxHashMap = + file_index_to_stable_id.iter().map(|(&i, id)| (id.clone(), i)).collect(); + #[allow(rustc::potential_query_instability)] + let next_index_init = + file_index_to_stable_id.keys().map(|i| i.0).max().map_or(0, |m| m + 1); + let mut next_index = next_index_init; + + for file in files.iter() { + let source_file_id = EncodedSourceFileId::new(tcx, file); + let index = + *stable_id_to_index.entry(source_file_id.clone()).or_insert_with(|| { + let index = SourceFileIndex(next_index); + next_index += 1; + index + }); + let file_ptr: *const SourceFile = &raw const **file; + file_to_file_index.insert(file_ptr, index); + file_index_to_stable_id.insert(index, source_file_id); + } + (file_to_file_index, file_index_to_stable_id) + } else { + let mut file_index_to_stable_id = + FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); + + for (index, file) in files.iter().enumerate() { + let index = SourceFileIndex(index as u32); + let file_ptr: *const SourceFile = &raw const **file; + file_to_file_index.insert(file_ptr, index); + let source_file_id = EncodedSourceFileId::new(tcx, file); + file_index_to_stable_id.insert(index, source_file_id); + } - (file_to_file_index, file_index_to_stable_id) + (file_to_file_index, file_index_to_stable_id) + } }; let hygiene_encode_context = HygieneEncodeContext::default(); + // Allocation indices embedded in the carried region reference the + // previous sessions' allocation table: keep its entries (their + // data lives in the copied region at unchanged positions) and + // make this session's encoder assign indices after them. + let alloc_index_offset = + if carried.is_some() { on_disk_cache.prev_interpret_alloc_index.len() } else { 0 }; + let mut encoder = CacheEncoder { tcx, encoder, type_shorthands: Default::default(), predicate_shorthands: Default::default(), interpret_allocs: Default::default(), + alloc_index_offset: alloc_index_offset.try_into().unwrap(), caching_source_map_view: CachingSourceMapView::new(tcx.sess.source_map()), file_to_file_index, hygiene_context: &hygiene_encode_context, @@ -239,6 +346,22 @@ impl OnDiskCache { side_effects_index: Default::default(), }; + // Reference the values of green nodes at their positions in the + // carried region. Values that are in memory anyway (for example + // because the query re-executed) are also encoded freshly below; + // fresh entries are appended after these carried entries, and the + // load path lets later entries win. + if carried.is_some() { + tcx.dep_graph.for_each_green_prev_index(&mut |prev_index, current_index| { + if let Some(&pos) = on_disk_cache.query_values_index.get(&prev_index) { + encoder.query_values_index.push(( + SerializedDepNodeIndex::from_curr_for_serialization(current_index), + pos, + )); + } + }); + } + // Encode query return values. tcx.sess.time("encode_query_values", || { tcx.encode_query_values(&mut encoder); @@ -250,7 +373,13 @@ impl OnDiskCache { } let interpret_alloc_index = { - let mut interpret_alloc_index = Vec::new(); + // Carried values reference the previous sessions' allocation + // entries by index: keep them, and append this session's. + let mut interpret_alloc_index = if carried.is_some() { + on_disk_cache.prev_interpret_alloc_index.clone() + } else { + Vec::new() + }; let mut n = 0; loop { let new_n = encoder.interpret_allocs.len(); @@ -272,8 +401,19 @@ impl OnDiskCache { }; let mut syntax_contexts = FxHashMap::default(); - let mut expn_data = UnhashMap::default(); - let mut foreign_expn_data = UnhashMap::default(); + // Expansions are keyed by their session-independent hash, so + // carried entries (whose data lives in the copied region) share + // one table with this session's; fresh entries overwrite. + let mut expn_data = if carried.is_some() { + on_disk_cache.expn_data.clone() + } else { + UnhashMap::default() + }; + let mut foreign_expn_data = if carried.is_some() { + on_disk_cache.foreign_expn_data.clone() + } else { + UnhashMap::default() + }; // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current // session. @@ -300,6 +440,17 @@ impl OnDiskCache { let footer_pos = encoder.position() as u64; let query_values_index = mem::take(&mut encoder.query_values_index); let side_effects_index = mem::take(&mut encoder.side_effects_index); + + // The carried generations keep their syntax context tables (their + // encoded ids live in their own id spaces); this session's table + // covers the region up to the footer. + let mut syntax_context_tables = if carried.is_some() { + on_disk_cache.syntax_context_tables.clone() + } else { + Vec::new() + }; + syntax_context_tables.push((footer_pos, syntax_contexts)); + encoder.encode_tagged( TAG_FILE_FOOTER, &Footer { @@ -307,7 +458,7 @@ impl OnDiskCache { query_values_index, side_effects_index, interpret_alloc_index, - syntax_contexts, + syntax_context_tables, expn_data, foreign_expn_data, }, @@ -351,7 +502,16 @@ impl OnDiskCache { where T: for<'a> Decodable>, { - self.load_indexed(tcx, dep_node_index, &self.query_values_index) + let pos = self.query_values_index.get(&dep_node_index).cloned()?; + // See `encode_query_value` for why values are tagged with their + // node's key fingerprint instead of its index. + let key_fingerprint = tcx + .dep_graph + .data() + .expect("always present in incremental mode") + .prev_key_fingerprint_of(dep_node_index); + let value = self.with_decoder(tcx, pos, |decoder| decode_tagged(decoder, key_fingerprint)); + Some(value) } fn load_indexed<'tcx, T>( @@ -385,10 +545,10 @@ impl OnDiskCache { file_index_to_file: &self.file_index_to_file, file_index_to_stable_id: &self.file_index_to_stable_id, alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(), - syntax_contexts: &self.syntax_contexts, + syntax_context_tables: &self.syntax_context_tables, expn_data: &self.expn_data, foreign_expn_data: &self.foreign_expn_data, - hygiene_context: &self.hygiene_context, + hygiene_contexts: &self.hygiene_contexts, }; f(&mut decoder) } @@ -405,10 +565,10 @@ pub struct CacheDecoder<'a, 'tcx> { file_index_to_file: &'a Lock>>, file_index_to_stable_id: &'a FxHashMap, alloc_decoding_session: AllocDecodingSession<'a>, - syntax_contexts: &'a FxHashMap, + syntax_context_tables: &'a [(u64, FxHashMap)], expn_data: &'a UnhashMap, foreign_expn_data: &'a UnhashMap, - hygiene_context: &'a HygieneDecodeContext, + hygiene_contexts: &'a [HygieneDecodeContext], } impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { @@ -548,8 +708,16 @@ impl<'a, 'tcx> Decodable> for Vec { impl<'a, 'tcx> SpanDecoder for CacheDecoder<'a, 'tcx> { fn decode_syntax_context(&mut self) -> SyntaxContext { - let syntax_contexts = self.syntax_contexts; - rustc_span::hygiene::decode_syntax_context(self, self.hygiene_context, |this, id| { + // Encoded syntax context ids are local to the session that wrote + // them, and the data region can contain regions carried forward from + // several earlier sessions. Select the table (and its id remapping + // cache) belonging to the region this id is being decoded from. + let position = self.opaque.position() as u64; + let table_index = + self.syntax_context_tables.partition_point(|&(region_end, _)| region_end <= position); + let (_, syntax_contexts) = &self.syntax_context_tables[table_index]; + let hygiene_context = &self.hygiene_contexts[table_index]; + rustc_span::hygiene::decode_syntax_context(self, hygiene_context, |this, id| { // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing. // We look up the position of the associated `SyntaxData` and decode it. let pos = syntax_contexts.get(&id).unwrap(); @@ -787,6 +955,9 @@ pub struct CacheEncoder<'a, 'tcx> { type_shorthands: FxHashMap, usize>, predicate_shorthands: FxHashMap, usize>, interpret_allocs: FxIndexSet, + /// Number of allocation entries carried over from previous sessions; + /// indices assigned by this encoder start after them. + alloc_index_offset: u32, caching_source_map_view: CachingSourceMapView<'tcx>, file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>, hygiene_context: &'a HygieneEncodeContext, @@ -825,11 +996,21 @@ impl<'a, 'tcx> CacheEncoder<'a, 'tcx> { ((end_pos - start_pos) as u64).encode(self); } - pub fn encode_query_value>(&mut self, index: DepNodeIndex, value: &V) { + pub fn encode_query_value>( + &mut self, + index: DepNodeIndex, + key_fingerprint: PackedFingerprint, + value: &V, + ) { let index = SerializedDepNodeIndex::from_curr_for_serialization(index); self.query_values_index.push((index, AbsoluteBytePos::new(self.position()))); - self.encode_tagged(index, value); + // Values are tagged with the key fingerprint of their node rather + // than its index: a value carried forward across several sessions + // keeps its original bytes while the node's index changes every + // session, and the key fingerprint is the session-stable identity + // the load path can still verify. + self.encode_tagged(key_fingerprint, value); } fn encode_side_effect(&mut self, index: DepNodeIndex, side_effect: &QuerySideEffect) { @@ -970,7 +1151,7 @@ impl<'a, 'tcx> TyEncoder<'tcx> for CacheEncoder<'a, 'tcx> { fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) { let (index, _) = self.interpret_allocs.insert_full(*alloc_id); - index.encode(self); + (self.alloc_index_offset as usize + index).encode(self); } } diff --git a/compiler/rustc_query_impl/src/dep_kind_vtables.rs b/compiler/rustc_query_impl/src/dep_kind_vtables.rs index 5adcf6c7bb576..deebce4135438 100644 --- a/compiler/rustc_query_impl/src/dep_kind_vtables.rs +++ b/compiler/rustc_query_impl/src/dep_kind_vtables.rs @@ -4,7 +4,6 @@ use rustc_middle::dep_graph::{DepKindVTable, DepNodeKey, KeyFingerprintStyle}; use rustc_middle::query::QueryCache; use crate::GetQueryVTable; -use crate::plumbing::promote_from_disk_inner; /// [`DepKindVTable`] constructors for special dep kinds that aren't queries. #[expect(non_snake_case, reason = "use non-snake case to avoid collision with query names")] @@ -19,7 +18,6 @@ mod non_query { force_from_dep_node_fn: Some(|_, dep_node, _| { bug!("force_from_dep_node: encountered {dep_node:?}") }), - promote_from_disk_fn: None, } } @@ -31,7 +29,6 @@ mod non_query { force_from_dep_node_fn: Some(|_, dep_node, _| { bug!("force_from_dep_node: encountered {dep_node:?}") }), - promote_from_disk_fn: None, } } @@ -43,7 +40,6 @@ mod non_query { tcx.dep_graph.force_side_effect(tcx, prev_index); true }), - promote_from_disk_fn: None, } } @@ -52,7 +48,6 @@ mod non_query { is_eval_always: false, key_fingerprint_style: KeyFingerprintStyle::Opaque, force_from_dep_node_fn: Some(|_, _, _| bug!("cannot force an anon node")), - promote_from_disk_fn: None, } } @@ -61,7 +56,6 @@ mod non_query { is_eval_always: false, key_fingerprint_style: KeyFingerprintStyle::Unit, force_from_dep_node_fn: None, - promote_from_disk_fn: None, } } @@ -70,7 +64,6 @@ mod non_query { is_eval_always: false, key_fingerprint_style: KeyFingerprintStyle::Opaque, force_from_dep_node_fn: None, - promote_from_disk_fn: None, } } @@ -79,7 +72,6 @@ mod non_query { is_eval_always: false, key_fingerprint_style: KeyFingerprintStyle::Opaque, force_from_dep_node_fn: None, - promote_from_disk_fn: None, } } @@ -88,7 +80,6 @@ mod non_query { is_eval_always: false, key_fingerprint_style: KeyFingerprintStyle::Unit, force_from_dep_node_fn: None, - promote_from_disk_fn: None, } } } @@ -96,7 +87,7 @@ mod non_query { /// Shared implementation of the [`DepKindVTable`] constructor for queries. /// Called from macro-generated code for each query. pub(crate) fn make_dep_kind_vtable_for_query<'tcx, Q>( - is_cache_on_disk: bool, + _is_cache_on_disk: bool, is_eval_always: bool, is_no_force: bool, ) -> DepKindVTable<'tcx> @@ -117,12 +108,6 @@ where crate::execution::force_query_dep_node(tcx, query, dep_node) }, ), - promote_from_disk_fn: (can_recover && is_cache_on_disk).then_some( - |tcx, dep_node, prev_index, dep_node_index| { - let query = Q::query_vtable(tcx); - promote_from_disk_inner(tcx, query, dep_node, prev_index, dep_node_index) - }, - ), } } diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 8de442309d7b2..14225ee681c01 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -1,23 +1,21 @@ use std::num::NonZero; use rustc_data_structures::Limit; -use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::unord::UnordMap; use rustc_middle::bug; #[expect(unused_imports, reason = "used by doc comments")] use rustc_middle::dep_graph::DepKindVTable; -use rustc_middle::dep_graph::{DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex}; +use rustc_middle::dep_graph::{DepNode, SerializedDepNodeIndex}; use rustc_middle::query::erase::{Erasable, Erased}; use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder}; use rustc_middle::query::{QueryCache, QueryJobId, QueryVTable, erase}; use rustc_middle::ty::TyCtxt; use rustc_middle::ty::tls::{self, ImplicitCtxt}; -use rustc_middle::verify_ich::incremental_verify_ich; use rustc_serialize::{Decodable, Encodable}; use rustc_span::def_id::LOCAL_CRATE; use crate::diagnostics::{QueryOverflow, QueryOverflowNote}; -use crate::execution::{all_inactive, should_verify_loaded_value}; +use crate::execution::all_inactive; use crate::job::find_dep_kind_root; use crate::query_impl::for_each_query_vtable; use crate::{CollectActiveJobsKind, collect_active_query_jobs}; @@ -95,7 +93,12 @@ fn encode_query_values_inner<'a, 'tcx, C, V>( assert!(all_inactive(&query.state)); query.cache.for_each(&mut |key, value, dep_node| { if query.will_cache_on_disk_for_key(*key) { - encoder.encode_query_value::(dep_node, &erase::restore_val::(*value)); + let key_fingerprint = DepNode::construct(tcx, query.dep_kind, key).key_fingerprint; + encoder.encode_query_value::( + dep_node, + key_fingerprint, + &erase::restore_val::(*value), + ); } }); } @@ -136,63 +139,6 @@ fn verify_query_key_hashes_inner<'tcx, C: QueryCache>( }); } -/// Inner implementation of [`DepKindVTable::promote_from_disk_fn`] for queries. -pub(crate) fn promote_from_disk_inner<'tcx, C: QueryCache>( - tcx: TyCtxt<'tcx>, - query: &'tcx QueryVTable<'tcx, C>, - dep_node: DepNode, - prev_index: SerializedDepNodeIndex, - dep_node_index: DepNodeIndex, -) { - debug_assert!(tcx.dep_graph.is_green(&dep_node)); - - let key = C::Key::try_recover_key(tcx, &dep_node).unwrap_or_else(|| { - panic!( - "Failed to recover key for {dep_node:?} with key fingerprint {}", - dep_node.key_fingerprint - ) - }); - - // If the recovered key isn't eligible for cache-on-disk, then there's no - // value on disk to promote. - if !query.will_cache_on_disk_for_key(key) { - return; - } - - // If the value is already in memory, then promotion isn't needed. - if query.cache.lookup(&key).is_some() { - return; - } - - // Load the disk-cached value into memory. - let dep_graph_data = - tcx.dep_graph.data().expect("should always be present in incremental mode"); - - let prof_timer = tcx.prof.incr_cache_loading(); - let value = ensure_sufficient_stack(|| (query.try_load_from_disk_fn)(tcx, prev_index)); - prof_timer.finish_with_query_invocation_id(dep_node_index.into()); - - let Some(value) = value else { - // The key is cache-on-disk and its node is green, so a value must be on disk. - bug!("failed to load disk-cached value for green node {dep_node:?}"); - }; - - // Verify the fingerprints of the same subset of loaded values as - // `load_from_disk_or_invoke_provider_green` does. - if should_verify_loaded_value(tcx, dep_graph_data, dep_node.key_fingerprint) { - incremental_verify_ich( - tcx, - dep_graph_data, - &value, - prev_index, - query.hash_value_fn, - query.format_value, - ); - } - - query.cache.complete(key, value, dep_node_index); -} - pub(crate) fn try_load_from_disk<'tcx, V>( tcx: TyCtxt<'tcx>, prev_index: SerializedDepNodeIndex,