diff --git a/crates/deslop-core/src/content/rename.rs b/crates/deslop-core/src/content/rename.rs index c8debd30..4ac4e87b 100644 --- a/crates/deslop-core/src/content/rename.rs +++ b/crates/deslop-core/src/content/rename.rs @@ -24,6 +24,11 @@ use super::{ vacuous_share, }; +use literal_echo::{affirming_literal_count, literal_echoes, LiteralEchoes}; + +/// Literal echoes of a rename, and the byte transform that proves one. +mod literal_echo; + /// Minimum occurrences of a substituted identifier pair before it counts /// as rename evidence ([TECH-PMATCH-BAKER]). In Baker's prev-encoding a /// parameter symbol's first occurrence matches anything and constrains @@ -47,14 +52,10 @@ const RENAME_CORROBORATION_MIN_OCCURRENCES: usize = 2; /// routing floor. const RENAME_EVIDENCE_HALF_MASS: f64 = 4.0; -/// Literal echoes of a rename ([REPAIR-RENAME-LITERAL-ECHO]). -mod echo; - /// The contradiction-free rename test ([FUSED-CONTENT-GATE-RENAME]). mod consistent; pub(super) use consistent::pair_rename_is_consistent; -use echo::{affirming_literal_count, literal_echoes}; /// Type-2 rename evidence between two members ([TECH-PMATCH-BAKER]): one /// pooled coverage over the pair's constrained identifier positions and @@ -67,9 +68,11 @@ use echo::{affirming_literal_count, literal_echoes}; /// rename family is the #197 sibling shape, and its literal axis must /// vouch on its own before a rename alone admits the pair. /// The pool opens only where the literal population affirms at all: -/// aligned literals with zero preservation and zero echoes are the +/// constrained literals with zero preservation and zero echoes are the /// #134 stride family — every substantive byte disagrees and nothing -/// outside the substitution vouches, so the axis is `0.0`. +/// outside the substitution vouches, so the axis is `0.0`. Which +/// literals are constrained is [`LiteralEvidence::measure`]'s call +/// ([FUSED-CONTENT-GATE-PARAMETER]). /// /// Baker's prev-encoding is the discriminator the deleted literal-anchor /// cliff could not provide: a substituted identifier pair seen once is @@ -99,40 +102,143 @@ pub(super) fn pair_rename_consistency( if !frontiers_aligned(canonical, member) { return 0.0; } - let echoes = literal_echoes(canonical, member, sources); + let positions = literal_positions(canonical, member); + let echoes = literal_echoes(canonical, member, sources, &positions); let mapping = rename_mapping( &population(&canonical.keys, &member.keys, Population::Identifier), &echoes.per_substitution, ); - let literal_total = population(&canonical.keys, &member.keys, Population::Literal).len(); - let affirming_literals = affirming_literal_count(canonical, member, &echoes); - if affirming_literals == 0 && literal_total > 0 { + let literals = LiteralEvidence::measure(&positions, &echoes, &mapping); + if literals.affirming == 0 && literals.constrained > 0 { return 0.0; } - // A window carved from inside a function that carries no literal at - // all offers the substitution nothing to contradict — the literal - // that would is on the line the window left out — so a substitution - // corroborated only by its own repetition cannot anchor it. Its - // anchors are the positions the rename did not supply: identity - // identifiers ([FUSED-CONTENT-GATE-INTERIOR]). A whole authored - // function or module with no literal is judged as before. - let identifier_anchors = if scope.interior && literal_total == 0 { - mapping.identity - } else { - mapping.explained - }; - let anchors = affirming_literals.saturating_add(identifier_anchors); - let coverage = if scope.same_file { - vacuous_share(affirming_literals, literal_total) - .min(vacuous_share(mapping.explained, mapping.constrained)) - } else { - let explained = mapping.explained.saturating_add(affirming_literals); - let constrained = mapping.constrained.saturating_add(literal_total); - vacuous_share(explained, constrained) - }; + let anchors = literals + .affirming + .saturating_add(mapping.anchors(scope, literals.aligned)); + let coverage = literals.coverage(&mapping, scope); coverage * evidence_weight(coverage, anchors) } +/// The pair's aligned literal positions, split into what the coverage +/// must explain and what it does explain ([FUSED-CONTENT-GATE], +/// [FUSED-CONTENT-GATE-PARAMETER]). +struct LiteralEvidence { + /// Every aligned literal position, whatever it says. + aligned: usize, + /// Positions the coverage must explain — see + /// [`LiteralEvidence::measure`]. + constrained: usize, + /// Positions that affirm the copy: preserved bytes or an echo of a + /// bijection-explained substitution. + affirming: usize, +} + +impl LiteralEvidence { + /// Measures one pair's literal positions. + /// + /// A preserved literal and a literal echo affirm the copy at the + /// position itself. A drifted literal that echoes nothing + /// contradicts the *rename* the identifier bijection claims, so it + /// is constrained and unexplained — the `#134` stride family renames + /// consistently end to end and diverges at one aligned literal, and + /// that one position is the whole difference between it and a + /// reportable Type-2 clone. + /// + /// [FUSED-CONTENT-GATE-PARAMETER] Where the bijection claims no + /// rename — no substituted identifier position is corroborated — + /// there is no claim for a drifted literal to contradict, and + /// [TECH-PMATCH-BAKER]'s prev-encoding applies to the literal + /// alphabet exactly as it does to the identifier one: a substitution + /// seen *once* is an unconstrained wildcard. Two declarations whose + /// every identifier position is byte-identical and whose literals + /// each substitute once are one parameterised declaration, and those + /// literals are its parameters — `csharp-merge-manyholes` keeps + /// every identifier and every call and substitutes at all twelve + /// literal positions, which is what `[AUTOFIX-MERGE-GATE]` + /// independently calls a clone too parameterised to merge + /// mechanically. + /// + /// A *repeated* substitution is not a wildcard. It is the sibling + /// family's own subject carried through its body — the star-shadow + /// fixture's `ApplyAlpha` says `"alpha"` three times against + /// `"dup"` — and it stays constrained, so a sibling that shares a + /// shape and no byte cannot join the copy it sits beside. An + /// inconsistent substitution stays constrained too: it contradicts + /// the parameterisation as surely as it would a rename. + fn measure( + positions: &[LiteralPosition], + echoes: &LiteralEchoes, + mapping: &RenameMapping, + ) -> Self { + let affirming = affirming_literal_count(positions, echoes); + let pairs = literal_pairs(positions); + let bijection = ModalBijection::over(&substituted_pairs(&pairs)); + let occurrences = pair_counts(pairs.iter().copied()); + let constrained = if mapping.renames() { + positions.len() + } else { + positions + .iter() + .filter(|(index, keys)| { + keys.0 == keys.1 + || echoes.positions.contains(index) + || !bijection.explains(keys) + || occurrences.get(keys).copied().unwrap_or_default() + >= RENAME_CORROBORATION_MIN_OCCURRENCES + }) + .count() + }; + Self { + aligned: positions.len(), + constrained, + affirming, + } + } + + /// The pooled coverage over this pair's constrained positions. + /// + /// A cross-file pair pools the literal and identifier populations + /// into one share. A same-file pair keeps the stricter min of the + /// two, matching the promote floor's conservatism: a same-file + /// rename family is the `#197` sibling shape, and its literal axis + /// must vouch on its own. + fn coverage(&self, mapping: &RenameMapping, scope: PairScope) -> f64 { + if scope.same_file { + return vacuous_share(self.affirming, self.constrained) + .min(vacuous_share(mapping.explained, mapping.constrained)); + } + vacuous_share( + mapping.explained.saturating_add(self.affirming), + mapping.constrained.saturating_add(self.constrained), + ) + } +} + +/// One aligned literal position: its frontier index and the two content +/// keys at it. The frontier is walked once per pair and every literal +/// measure reads the result — the affirming count, the echo candidates, +/// and [`LiteralEvidence`]. +pub(super) type LiteralPosition = (usize, (u64, u64)); + +/// Aligned positions where both members carry a literal. +fn literal_positions(canonical: &MemberContent, member: &MemberContent) -> Vec { + canonical + .keys + .iter() + .zip(member.keys.iter()) + .enumerate() + .filter(|(_, (left, right))| { + left.population == Population::Literal && right.population == Population::Literal + }) + .map(|(index, (left, right))| (index, (left.key, right.key))) + .collect() +} + +/// The key pairs of [`literal_positions`], for the literal bijection. +fn literal_pairs(positions: &[LiteralPosition]) -> Vec<(u64, u64)> { + positions.iter().map(|(_, keys)| *keys).collect() +} + /// Rename-mapping evidence over one pair's aligned identifier positions /// ([TECH-PMATCH-BAKER]), produced by [`rename_mapping`]. struct RenameMapping { @@ -167,6 +273,32 @@ struct RenameMapping { corroborated: usize, } +impl RenameMapping { + /// Whether the pair claims a rename at all: some substituted + /// identifier position is explained, so a bijection is asserting + /// that this copy was renamed rather than merely reused. + fn renames(&self) -> bool { + self.explained > self.identity + } + + /// The identifier positions that anchor the proof. + /// + /// A window carved from inside a function that carries no literal at + /// all offers the substitution nothing to contradict — the literal + /// that would is on the line the window left out — so a substitution + /// corroborated only by its own repetition cannot anchor it. Its + /// anchors are the positions the rename did not supply: identity + /// identifiers ([FUSED-CONTENT-GATE-INTERIOR]). A whole authored + /// function or module with no literal is judged as before. + fn anchors(&self, scope: PairScope, aligned_literals: usize) -> usize { + if scope.interior && aligned_literals == 0 { + self.identity + } else { + self.explained + } + } +} + /// Measures [`RenameMapping`] for one pair's identifier positions, /// classifying each position exactly as [TECH-PMATCH-BAKER]'s /// prev-encoding constrains it: identity and corroborated substitutions @@ -227,7 +359,7 @@ fn rename_mapping( /// The aligned positions whose raw bytes differ — [TECH-PMATCH-BAKER]'s /// parameter alphabet, the population [`rename_mapping`] derives its /// bijection over. -fn substituted_pairs(identifiers: &[(u64, u64)]) -> Vec<(u64, u64)> { +pub(super) fn substituted_pairs(identifiers: &[(u64, u64)]) -> Vec<(u64, u64)> { identifiers .iter() .filter(|(left, right)| left != right) diff --git a/crates/deslop-core/src/content/rename/consistent.rs b/crates/deslop-core/src/content/rename/consistent.rs index ba84fc06..b1b03343 100644 --- a/crates/deslop-core/src/content/rename/consistent.rs +++ b/crates/deslop-core/src/content/rename/consistent.rs @@ -10,7 +10,7 @@ use crate::state::FileId; use super::{ super::frontier::{frontiers_aligned, leaf_bytes, population, MemberContent, Population}, - literal_echoes, rename_mapping, + literal_echoes, literal_positions, rename_mapping, }; /// Whether the pair is one code written twice under a consistent @@ -59,7 +59,8 @@ fn corroboration( member: &MemberContent, sources: &HashMap, S>, ) -> BTreeMap<(u64, u64), usize> { - let mut counts = literal_echoes(canonical, member, sources).per_substitution; + let positions = literal_positions(canonical, member); + let mut counts = literal_echoes(canonical, member, sources, &positions).per_substitution; for (keys, siblings) in transformation_siblings(canonical, member, sources) { let slot = counts.entry(keys).or_insert(0_usize); *slot = slot.saturating_add(siblings); diff --git a/crates/deslop-core/src/content/rename/echo.rs b/crates/deslop-core/src/content/rename/literal_echo.rs similarity index 84% rename from crates/deslop-core/src/content/rename/echo.rs rename to crates/deslop-core/src/content/rename/literal_echo.rs index 4ccc422c..b964f9c7 100644 --- a/crates/deslop-core/src/content/rename/echo.rs +++ b/crates/deslop-core/src/content/rename/literal_echo.rs @@ -1,16 +1,23 @@ -//! Literal echoes of a rename ([REPAIR-RENAME-LITERAL-ECHO], gh #409): -//! the aligned literal positions whose bytes transform into the -//! partner's bytes by exactly one bijection-explained identifier -//! substitution, and the affirming-literal count built on them. +//! Literal echoes of a rename ([REPAIR-RENAME-LITERAL-ECHO], gh #409). +//! +//! A literal renamed *alongside the symbol it names* is part of the +//! rename, not evidence against it: `"OrderService"` renamed to +//! `"UserService"` with the `OrderService` symbol is the rename done +//! *thoroughly*, and counting it as a differing literal inverted the +//! score — the half-finished rename outscored the complete one +//! (`crates/deslop/tests/rename_literal_monotonicity.rs`). Such an +//! **echo** is recognised by content, never by coincidence: the +//! literal's bytes must transform into the partner's bytes exactly by +//! the same substitution the identifier bijection explains, and the echo +//! then corroborates that substitution the way a repeated identifier +//! occurrence would. use std::{collections::BTreeMap, collections::HashMap, hash::BuildHasher}; use crate::state::FileId; -use super::{ - super::frontier::{leaf_bytes, population, MemberContent, Population}, - substituted_pairs, ModalBijection, -}; +use super::super::frontier::{leaf_bytes, population, MemberContent, Population}; +use super::{substituted_pairs, LiteralPosition, ModalBijection}; /// Literal echoes of the bijection's identifier substitutions (#409), as a /// per-substitution count: an aligned literal position whose bytes @@ -24,12 +31,13 @@ pub(super) fn literal_echoes( canonical: &MemberContent, member: &MemberContent, sources: &HashMap, S>, + positions: &[LiteralPosition], ) -> LiteralEchoes { let identifiers = population(&canonical.keys, &member.keys, Population::Identifier); let bijection = ModalBijection::over(&substituted_pairs(&identifiers)); let substitutions = explained_substitution_bytes(canonical, member, &bijection, sources); let mut echoes = LiteralEchoes::default(); - for index in substituted_literal_positions(canonical, member) { + for index in substituted_literal_positions(positions) { let bytes = leaf_bytes(canonical, index, sources).zip(leaf_bytes(member, index, sources)); let Some((left, right)) = bytes else { continue; @@ -55,19 +63,12 @@ pub(super) fn literal_echoes( /// preserved literal; the drifted fragment beside it is a drifted one, /// and weakens the proof in proportion like any other. pub(super) fn affirming_literal_count( - canonical: &MemberContent, - member: &MemberContent, + positions: &[LiteralPosition], echoes: &LiteralEchoes, ) -> usize { - canonical - .keys + positions .iter() - .zip(member.keys.iter()) - .enumerate() - .filter(|(_, (left, right))| { - left.population == Population::Literal && right.population == Population::Literal - }) - .filter(|(index, (left, right))| left.key == right.key || echoes.positions.contains(index)) + .filter(|(index, (left, right))| left == right || echoes.positions.contains(index)) .count() } @@ -82,21 +83,13 @@ pub(super) struct LiteralEchoes { pub(super) positions: std::collections::BTreeSet, } -/// Frontier indices of aligned positions where both members carry a -/// literal and the raw bytes differ — the candidates an echo can -/// explain. -fn substituted_literal_positions(canonical: &MemberContent, member: &MemberContent) -> Vec { - canonical - .keys +/// Frontier indices of the aligned literal positions whose raw bytes +/// differ — the candidates an echo can explain. +fn substituted_literal_positions(positions: &[LiteralPosition]) -> Vec { + positions .iter() - .zip(member.keys.iter()) - .enumerate() - .filter(|(_, (left, right))| { - left.population == Population::Literal - && right.population == Population::Literal - && left.key != right.key - }) - .map(|(index, _)| index) + .filter(|(_, (left, right))| left != right) + .map(|(index, _)| *index) .collect() } diff --git a/crates/deslop-core/src/overlap/rescue.rs b/crates/deslop-core/src/overlap/rescue.rs index 09c746f8..a7059aa1 100644 --- a/crates/deslop-core/src/overlap/rescue.rs +++ b/crates/deslop-core/src/overlap/rescue.rs @@ -27,9 +27,9 @@ use crate::{ ast::NormalizedNode, cluster::scope::DeclarationScopes, content::pair_content_agreement, - fingerprint::Fingerprint, + fingerprint::{ranges_overlap, Fingerprint}, pair::{ - alignment_required, crosses_files, CandidatePair, ExactFunctionAnchors, + alignment_required, crosses_files, CandidatePair, ExactClones, RESCUE_MIN_CONTENT_AGREEMENT, SHARED_SUBTREE_MIN_NODE_COUNT, SHARED_SUBTREE_MIN_OVERLAP, }, state::FileId, @@ -48,7 +48,13 @@ pub(super) struct RescueContext<'a, S, L: BuildHasher> { languages: &'a HashMap, /// The exact whole-function clones a container may not merely wrap /// ([FUSED-SHARED-SUBTREE-ECHO]). - anchors: ExactFunctionAnchors, + anchors: ExactClones, + /// The exact clones inside each file, for the same-file scope rule + /// ([FUSED-SHARED-SUBTREE-SAME-FILE]). + interiors: ExactClones, + /// Authored declarations per file, for the same-file scope rule + /// ([FUSED-SHARED-SUBTREE-SAME-FILE]). + scopes: DeclarationScopes<'a, L>, } impl<'a, S: BuildHasher, L: BuildHasher> RescueContext<'a, S, L> { @@ -61,12 +67,65 @@ impl<'a, S: BuildHasher, L: BuildHasher> RescueContext<'a, S, L> { languages: &'a HashMap, ) -> Self { let scopes = DeclarationScopes::new(trees, languages); + let anchors = ExactClones::whole_functions_across_files(pairs, fingerprints, &scopes); Self { tree_index: trees.iter().map(|tree| (tree.file_id, tree)).collect(), sources, languages, - anchors: ExactFunctionAnchors::index(pairs, fingerprints, &scopes), + anchors, + interiors: ExactClones::within_one_file(pairs, fingerprints), + scopes, + } + } + + /// [FUSED-SHARED-SUBTREE-SAME-FILE] Whether the rescue may measure + /// this pair at all. + /// + /// Across files every eligible pair is measured. Inside one file the + /// route is open only to two endpoints that are each a whole + /// authored declaration — modifier through closing brace — and are + /// disjoint. Two methods that drifted apart inside one file are the + /// same duplication as two that drifted apart across files; a window + /// cut over part of one, a nested view of another, and a table row + /// are none of them a declaration. Admitting those unioned a file's + /// subtrees into a single component that the same-file collapse then + /// reduced to one logical location, and the file's real duplication + /// disappeared rather than being reported + /// (`issue_119_role_gate_exercised`). + fn measures(&self, left: &Fingerprint, right: &Fingerprint) -> bool { + crosses_files(left, right) + || (!ranges_overlap(left, right) + && self.scopes.aligned_function(left).is_some() + && self.scopes.aligned_function(right).is_some() + && self.shares_a_copied_interior(left, right)) + } + + /// [FUSED-SHARED-SUBTREE-SAME-FILE] Whether the two declarations + /// still hold a copied interior: a Merkle-equal clone inside both of + /// them, substantive enough to clear the floor every rescued + /// endpoint clears ([`SHARED_SUBTREE_MIN_NODE_COUNT`]). + /// + /// This is the discriminator between a copy that drifted and a + /// family that never was one. `csharp-merge-drift`'s two methods + /// still share four whole statements outright, 32 nodes of authored + /// code the edit never touched, while the `dart-issue-197` settings + /// accessors share a skeleton and no statement at all: their overlap + /// is 0.81 to 0.88, indistinguishable from the drifted pair's 0.84, + /// and their raw-content agreement reaches 0.56 against its 0.55. + /// Shape and agreement cannot separate them. Copied code can. + fn shares_a_copied_interior(&self, left: &Fingerprint, right: &Fingerprint) -> bool { + self.interiors.enclosed_nodes(left, right) >= SHARED_SUBTREE_MIN_NODE_COUNT + } + + /// The exact clones this pair could merely be echoing: across files + /// the whole-function runs, inside one file the file's own exact + /// clones ([FUSED-SHARED-SUBTREE-ECHO]). + fn echo_anchors(&self, left: &Fingerprint, right: &Fingerprint) -> &ExactClones { + if crosses_files(left, right) { + &self.anchors + } else { + &self.interiors } } } @@ -192,10 +251,10 @@ fn measure_one( return; }; tally.eligible(); - if !crosses_files(left, right) { + if !context.measures(left, right) { return; } - tally.cross_file(); + tally.in_scope(crosses_files(left, right)); pair.shared_subtree_overlap = measurer.rescue_overlap(left, right); record_rescue_verdict(pair, left, right, context, measurer, tally); } @@ -250,7 +309,7 @@ fn is_container_echo( right: &Fingerprint, context: &RescueContext<'_, S, L>, ) -> bool { - let Some(claimed) = context.anchors.claimed_nodes(left, right) else { + let Some(claimed) = context.echo_anchors(left, right).claimed_nodes(left, right) else { return false; }; let larger = left.node_count.max(right.node_count); @@ -263,206 +322,7 @@ fn usize_to_f64(nodes: usize) -> f64 { u32::try_from(nodes).map_or(f64::MAX, f64::from) } +/// [PERF-FLUTTER-TODO-RESCUE] Sharded and serial rescue must agree. #[cfg(test)] -mod shard_equivalence_tests { - //! [PERF-FLUTTER-TODO-RESCUE] The sharded rescue must produce the - //! byte-identical pair outcomes and counters as the serial path: - //! every measurement is a pure function of the corpus, so sharding - //! may change which thread computes a value but never the value - //! (`docs/release-audit.md`, "parallel rescue"). - - use std::path::PathBuf; - - use super::{ - apply_shared_subtree_rescue, measure_chunk, RescueContext, RescueTally, MIN_SHARD_WORK, - }; - use crate::{ - ast::NormalizedNode, - fingerprint::Fingerprint, - lang::LanguageParser, - pair::{ - CandidatePair, PairScore, FUSED_THRESHOLD, LSH_ONLY_MIN_JACCARD, - LSH_ONLY_MIN_NODE_COUNT, SHARED_SUBTREE_MIN_JACCARD, - }, - state::{FileId, FileRegistry}, - }; - - /// One serial shard over `chunk`: the reference a single worker - /// computes, assembled from the very `measure_chunk` the workers - /// run so the reference can never drift from the live path. - fn run_shard( - chunk: &mut [CandidatePair], - fingerprints: &[Fingerprint], - trees: &[NormalizedNode], - sources: &std::collections::HashMap, S>, - languages: &std::collections::HashMap, - ) -> (RescueTally, crate::overlap::MeasureStats) { - let mut measurer = crate::overlap::OverlapMeasurer::new(trees); - let mut tally = RescueTally::new(); - let context = RescueContext::new(chunk, fingerprints, trees, sources, languages); - measure_chunk(chunk, fingerprints, &context, &mut measurer, &mut tally); - let stats = measurer.stats(); - (tally, stats) - } - - /// Parses `source` as Rust and fingerprints its root. - fn parse(source: &str, file_id: FileId) -> Result<(NormalizedNode, Fingerprint), String> { - let tree = crate::lang::rust_lang::RustParser - .parse_and_normalize(source.as_bytes(), file_id) - .map_err(|error| format!("the Rust fixture must parse: {error}"))?; - let whole = Fingerprint { - hash: [0_u8; 32], - file_id, - byte_range: tree.byte_range, - node_count: count_nodes(&tree), - }; - Ok((tree, whole)) - } - - /// Total nodes in a subtree, including the root. - fn count_nodes(node: &NormalizedNode) -> usize { - node.children - .iter() - .map(count_nodes) - .fold(1, usize::saturating_add) - } - - /// A wide function past every gate: well over the LSH-only floor - /// and large enough for real overlap measurement. - fn wide_function(statements: usize) -> String { - let body = (0..statements).fold(String::new(), |mut body, index| { - use std::fmt::Write as _; - let _written = writeln!(body, " total = total + {index};"); - body - }); - format!("fn alpha(seed: u32) -> u32 {{\n let mut total = seed;\n{body} total\n}}\n") - } - - /// A rescue-eligible cross-file pair over two whole-file endpoints. - fn eligible_pair(nodes: usize) -> CandidatePair { - CandidatePair { - left: 0, - right: 1, - endpoint_node_counts: (nodes, nodes), - lsh_only_node_floor: LSH_ONLY_MIN_NODE_COUNT, - lsh_only_min_jaccard: LSH_ONLY_MIN_JACCARD, - fused_min_score: FUSED_THRESHOLD, - shared_subtree_overlap: 0.0, - score: PairScore { - structural: 0.0, - token_jaccard: SHARED_SUBTREE_MIN_JACCARD, - embedding_cos: 0.0, - }, - } - } - - /// Twice [`MIN_SHARD_WORK`] pairs must measure identically whether - /// the population runs through the (thread-pooling) entry point or - /// one `run_shard` over the whole list — the serial reference a - /// worker computes. Shard boundaries are pinned separately: two - /// disjoint `run_shard` calls over the halves must reproduce the - /// whole-list values, and their merged tallies must account for - /// every pair. A blind rescue (zero measured) fails, it does not - /// pass vacuously. - #[test] - fn sharded_rescue_matches_serial_outcomes() -> Result<(), String> { - let pair_count = MIN_SHARD_WORK.get().saturating_mul(2); - let mut registry = FileRegistry::new(); - let left_id = registry.register(PathBuf::from("left.rs")); - let right_id = registry.register(PathBuf::from("right.rs")); - let left_source = wide_function(120); - let right_source = wide_function(121); - let left = parse(&left_source, left_id)?; - let right = parse(&right_source, right_id)?; - let nodes = left.1.node_count; - let fingerprints = [left.1.clone(), right.1.clone()]; - let trees = [left.0, right.0]; - let sources = std::collections::HashMap::from([ - (left_id, left_source.into_bytes()), - (right_id, right_source.into_bytes()), - ]); - let languages = std::collections::HashMap::from([(left_id, "rust"), (right_id, "rust")]); - let fixture = || { - (0..pair_count) - .map(|_| eligible_pair(nodes)) - .collect::>() - }; - - // The threaded entry point — whichever core count routes it. - let mut sharded = fixture(); - apply_shared_subtree_rescue(&mut sharded, &fingerprints, &trees, &sources, &languages); - - // The serial reference: one measurer, one tally, every pair. - let mut serial = fixture(); - let (serial_tally, serial_stats) = - run_shard(&mut serial, &fingerprints, &trees, &sources, &languages); - - for (index, (shard_pair, serial_pair)) in sharded.iter().zip(&serial).enumerate() { - assert!( - (shard_pair.shared_subtree_overlap - serial_pair.shared_subtree_overlap).abs() - < f64::EPSILON, - "pair {index}: sharded overlap {} must equal serial {}", - shard_pair.shared_subtree_overlap, - serial_pair.shared_subtree_overlap - ); - assert!( - shard_pair.shared_subtree_overlap > 0.0, - "pair {index}: the fixture is a real near-duplicate — a rescue that measures \ - nothing is blind, and overlap was {}", - shard_pair.shared_subtree_overlap - ); - } - - // Shard boundaries change nothing: halves measured as separate - // shards reproduce the whole-list values exactly. - let mut halved = fixture(); - let midpoint = pair_count / 2; - let (head, tail) = halved.split_at_mut(midpoint); - let (head_tally, head_stats) = run_shard(head, &fingerprints, &trees, &sources, &languages); - let (tail_tally, tail_stats) = run_shard(tail, &fingerprints, &trees, &sources, &languages); - for (index, (half_pair, serial_pair)) in halved.iter().zip(&serial).enumerate() { - assert!( - (half_pair.shared_subtree_overlap - serial_pair.shared_subtree_overlap).abs() - < f64::EPSILON, - "pair {index}: shard-split overlap {} must equal whole-list {}", - half_pair.shared_subtree_overlap, - serial_pair.shared_subtree_overlap - ); - } - - // The merged shard counters account for every pair, exactly as - // the module contract promises: absorb halves, compare to the - // whole-list tally, and check the stats fold. - let mut merged = head_tally; - merged.absorb(&tail_tally); - assert_eq!( - merged.scanned, serial_tally.scanned, - "merged shard tallies must count every scanned pair" - ); - assert_eq!( - merged.eligible, serial_tally.eligible, - "merged shard tallies must count every eligible pair" - ); - assert_eq!( - merged.cross_file, serial_tally.cross_file, - "merged shard tallies must count every cross-file pair" - ); - assert_eq!( - merged.measured, serial_tally.measured, - "merged shard tallies must count every measured pair" - ); - let u64_count = u64::try_from(pair_count).unwrap_or(u64::MAX); - assert_eq!( - merged.measured, u64_count, - "every fixture pair is eligible and cross-file: all {pair_count} must be measured, \ - got {}", - merged.measured - ); - let folded_stats = head_stats.add(tail_stats); - assert_eq!( - folded_stats.alignments, serial_stats.alignments, - "merged measurement stats must fold to the whole-list stats" - ); - Ok(()) - } -} +#[path = "rescue/shard_equivalence.rs"] +mod shard_equivalence_tests; diff --git a/crates/deslop-core/src/overlap/rescue/shard_equivalence.rs b/crates/deslop-core/src/overlap/rescue/shard_equivalence.rs new file mode 100644 index 00000000..d7c8d907 --- /dev/null +++ b/crates/deslop-core/src/overlap/rescue/shard_equivalence.rs @@ -0,0 +1,201 @@ +//! [PERF-FLUTTER-TODO-RESCUE] The sharded rescue must produce the +//! byte-identical pair outcomes and counters as the serial path: +//! every measurement is a pure function of the corpus, so sharding +//! may change which thread computes a value but never the value +//! (`docs/release-audit.md`, "parallel rescue"). + +use std::path::PathBuf; + +use super::super::{ + rescue::{apply_shared_subtree_rescue, measure_chunk, RescueContext, MIN_SHARD_WORK}, + tally::RescueTally, +}; +use crate::{ + ast::NormalizedNode, + fingerprint::Fingerprint, + lang::LanguageParser, + pair::{ + CandidatePair, PairScore, FUSED_THRESHOLD, LSH_ONLY_MIN_JACCARD, LSH_ONLY_MIN_NODE_COUNT, + SHARED_SUBTREE_MIN_JACCARD, + }, + state::{FileId, FileRegistry}, +}; + +/// One serial shard over `chunk`: the reference a single worker +/// computes, assembled from the very `measure_chunk` the workers +/// run so the reference can never drift from the live path. +fn run_shard( + chunk: &mut [CandidatePair], + fingerprints: &[Fingerprint], + trees: &[NormalizedNode], + sources: &std::collections::HashMap, S>, + languages: &std::collections::HashMap, +) -> (RescueTally, crate::overlap::MeasureStats) { + let mut measurer = crate::overlap::OverlapMeasurer::new(trees); + let mut tally = RescueTally::new(); + let context = RescueContext::new(chunk, fingerprints, trees, sources, languages); + measure_chunk(chunk, fingerprints, &context, &mut measurer, &mut tally); + let stats = measurer.stats(); + (tally, stats) +} + +/// Parses `source` as Rust and fingerprints its root. +fn parse(source: &str, file_id: FileId) -> Result<(NormalizedNode, Fingerprint), String> { + let tree = crate::lang::rust_lang::RustParser + .parse_and_normalize(source.as_bytes(), file_id) + .map_err(|error| format!("the Rust fixture must parse: {error}"))?; + let whole = Fingerprint { + hash: [0_u8; 32], + file_id, + byte_range: tree.byte_range, + node_count: count_nodes(&tree), + }; + Ok((tree, whole)) +} + +/// Total nodes in a subtree, including the root. +fn count_nodes(node: &NormalizedNode) -> usize { + node.children + .iter() + .map(count_nodes) + .fold(1, usize::saturating_add) +} + +/// A wide function past every gate: well over the LSH-only floor +/// and large enough for real overlap measurement. +fn wide_function(statements: usize) -> String { + let body = (0..statements).fold(String::new(), |mut body, index| { + use std::fmt::Write as _; + let _written = writeln!(body, " total = total + {index};"); + body + }); + format!("fn alpha(seed: u32) -> u32 {{\n let mut total = seed;\n{body} total\n}}\n") +} + +/// A rescue-eligible cross-file pair over two whole-file endpoints. +fn eligible_pair(nodes: usize) -> CandidatePair { + CandidatePair { + left: 0, + right: 1, + endpoint_node_counts: (nodes, nodes), + lsh_only_node_floor: LSH_ONLY_MIN_NODE_COUNT, + lsh_only_min_jaccard: LSH_ONLY_MIN_JACCARD, + fused_min_score: FUSED_THRESHOLD, + shared_subtree_overlap: 0.0, + score: PairScore { + structural: 0.0, + token_jaccard: SHARED_SUBTREE_MIN_JACCARD, + embedding_cos: 0.0, + }, + } +} + +/// Twice [`MIN_SHARD_WORK`] pairs must measure identically whether +/// the population runs through the (thread-pooling) entry point or +/// one `run_shard` over the whole list — the serial reference a +/// worker computes. Shard boundaries are pinned separately: two +/// disjoint `run_shard` calls over the halves must reproduce the +/// whole-list values, and their merged tallies must account for +/// every pair. A blind rescue (zero measured) fails, it does not +/// pass vacuously. +#[test] +fn sharded_rescue_matches_serial_outcomes() -> Result<(), String> { + let pair_count = MIN_SHARD_WORK.get().saturating_mul(2); + let mut registry = FileRegistry::new(); + let left_id = registry.register(PathBuf::from("left.rs")); + let right_id = registry.register(PathBuf::from("right.rs")); + let left_source = wide_function(120); + let right_source = wide_function(121); + let left = parse(&left_source, left_id)?; + let right = parse(&right_source, right_id)?; + let nodes = left.1.node_count; + let fingerprints = [left.1.clone(), right.1.clone()]; + let trees = [left.0, right.0]; + let sources = std::collections::HashMap::from([ + (left_id, left_source.into_bytes()), + (right_id, right_source.into_bytes()), + ]); + let languages = std::collections::HashMap::from([(left_id, "rust"), (right_id, "rust")]); + let fixture = || { + (0..pair_count) + .map(|_| eligible_pair(nodes)) + .collect::>() + }; + + // The threaded entry point — whichever core count routes it. + let mut sharded = fixture(); + apply_shared_subtree_rescue(&mut sharded, &fingerprints, &trees, &sources, &languages); + + // The serial reference: one measurer, one tally, every pair. + let mut serial = fixture(); + let (serial_tally, serial_stats) = + run_shard(&mut serial, &fingerprints, &trees, &sources, &languages); + + for (index, (shard_pair, serial_pair)) in sharded.iter().zip(&serial).enumerate() { + assert!( + (shard_pair.shared_subtree_overlap - serial_pair.shared_subtree_overlap).abs() + < f64::EPSILON, + "pair {index}: sharded overlap {} must equal serial {}", + shard_pair.shared_subtree_overlap, + serial_pair.shared_subtree_overlap + ); + assert!( + shard_pair.shared_subtree_overlap > 0.0, + "pair {index}: the fixture is a real near-duplicate — a rescue that measures \ + nothing is blind, and overlap was {}", + shard_pair.shared_subtree_overlap + ); + } + + // Shard boundaries change nothing: halves measured as separate + // shards reproduce the whole-list values exactly. + let mut halved = fixture(); + let midpoint = pair_count / 2; + let (head, tail) = halved.split_at_mut(midpoint); + let (head_tally, head_stats) = run_shard(head, &fingerprints, &trees, &sources, &languages); + let (tail_tally, tail_stats) = run_shard(tail, &fingerprints, &trees, &sources, &languages); + for (index, (half_pair, serial_pair)) in halved.iter().zip(&serial).enumerate() { + assert!( + (half_pair.shared_subtree_overlap - serial_pair.shared_subtree_overlap).abs() + < f64::EPSILON, + "pair {index}: shard-split overlap {} must equal whole-list {}", + half_pair.shared_subtree_overlap, + serial_pair.shared_subtree_overlap + ); + } + + // The merged shard counters account for every pair, exactly as + // the module contract promises: absorb halves, compare to the + // whole-list tally, and check the stats fold. + let mut merged = head_tally; + merged.absorb(&tail_tally); + assert_eq!( + merged.scanned, serial_tally.scanned, + "merged shard tallies must count every scanned pair" + ); + assert_eq!( + merged.eligible, serial_tally.eligible, + "merged shard tallies must count every eligible pair" + ); + assert_eq!( + merged.cross_file, serial_tally.cross_file, + "merged shard tallies must count every cross-file pair" + ); + assert_eq!( + merged.measured, serial_tally.measured, + "merged shard tallies must count every measured pair" + ); + let u64_count = u64::try_from(pair_count).unwrap_or(u64::MAX); + assert_eq!( + merged.measured, u64_count, + "every fixture pair is eligible and cross-file: all {pair_count} must be measured, \ + got {}", + merged.measured + ); + let folded_stats = head_stats.add(tail_stats); + assert_eq!( + folded_stats.alignments, serial_stats.alignments, + "merged measurement stats must fold to the whole-list stats" + ); + Ok(()) +} diff --git a/crates/deslop-core/src/overlap/tally.rs b/crates/deslop-core/src/overlap/tally.rs index ee9bb251..086b4b92 100644 --- a/crates/deslop-core/src/overlap/tally.rs +++ b/crates/deslop-core/src/overlap/tally.rs @@ -40,9 +40,16 @@ pub(super) struct RescueTally { /// threshold despite token corroboration, or carried by the token axis /// alone ([`crate::pair::alignment_required`]). pub(super) eligible: u64, - /// Eligible pairs whose endpoints live in different files — the - /// population handed to the measurer. + /// Eligible pairs whose endpoints live in different files — part of + /// the population handed to the measurer. pub(super) cross_file: u64, + /// Eligible pairs whose two endpoints are whole authored + /// declarations inside one file — the rest of that population + /// ([FUSED-SHARED-SUBTREE-SAME-FILE]). Counted apart from + /// `cross_file` because the same-file route is the narrow one: a + /// same-file count that climbs with corpus size is the scope rule + /// leaking, and conflating the two would hide it. + pub(super) same_file: u64, /// Cross-file pairs the measurer answered, from any route. pub(super) measured: u64, /// Measured pairs whose overlap cleared @@ -78,6 +85,7 @@ impl RescueTally { scanned: 0, eligible: 0, cross_file: 0, + same_file: 0, measured: 0, rescued: 0, content_gate_rejected: 0, @@ -96,9 +104,13 @@ impl RescueTally { bump(&mut self.eligible); } - /// Records one pair past the cross-file gate. - pub(super) fn cross_file(&mut self) { - bump(&mut self.cross_file); + /// Records one pair past the scope gate, on the side it came in. + pub(super) fn in_scope(&mut self, crosses_files: bool) { + if crosses_files { + bump(&mut self.cross_file); + } else { + bump(&mut self.same_file); + } } /// Records one measured pair and whether it cleared the admission @@ -130,6 +142,7 @@ impl RescueTally { self.scanned = self.scanned.saturating_add(other.scanned); self.eligible = self.eligible.saturating_add(other.eligible); self.cross_file = self.cross_file.saturating_add(other.cross_file); + self.same_file = self.same_file.saturating_add(other.same_file); self.measured = self.measured.saturating_add(other.measured); self.rescued = self.rescued.saturating_add(other.rescued); self.content_gate_rejected = self @@ -156,6 +169,7 @@ impl RescueTally { scanned = self.scanned, eligible = self.eligible, cross_file = self.cross_file, + same_file = self.same_file, measured = self.measured, rescued_pairs = self.rescued, content_gate_rejected = self.content_gate_rejected, diff --git a/crates/deslop-core/src/pair.rs b/crates/deslop-core/src/pair.rs index 989e0100..dabf1713 100644 --- a/crates/deslop-core/src/pair.rs +++ b/crates/deslop-core/src/pair.rs @@ -22,7 +22,7 @@ pub use candidates::{candidate_pairs, candidate_pairs_for_language_policy, LshPa mod content_gate; mod echo; pub(crate) use content_gate::apply_pair_content_gate; -pub(crate) use echo::ExactFunctionAnchors; +pub(crate) use echo::ExactClones; /// Transitive-closure clustering over surviving pairs. mod closure; @@ -468,16 +468,17 @@ fn shared_subtree_can_reach_floor((smaller, larger): (usize, usize)) -> bool { /// True when the pair's endpoints live in different files. /// -/// The rescue is deliberately cross-file only. Every clone this route -/// exists to recover is a copy *between* files ([FUSED-SHARED-SUBTREE], -/// gh #408), and admitting same-file pairs on shape overlap is the -/// #197 in-file sibling-family shape, which the report already spends a -/// dedicated proof suppressing. It is also what keeps a single-file -/// corpus intact: same-file rescues union that file's subtrees into one -/// transitive component, and the same-file overlap collapse then -/// reduces it to a single logical location, which is dropped below -/// `MIN_REPORTABLE_MEMBERS` — so the file's real duplication -/// disappeared entirely rather than being reported +/// The rescue measures every eligible cross-file pair; inside one file +/// it measures only the narrow population +/// [FUSED-SHARED-SUBTREE-SAME-FILE] describes, so this predicate is the +/// scope split rather than the scope itself +/// (`RescueContext::measures`). Admitting same-file pairs on shape +/// overlap alone is the `#197` in-file sibling-family shape, which the +/// report already spends a dedicated proof suppressing, and it is also +/// how a single-file corpus loses its findings: unconstrained same-file +/// rescues union that file's subtrees into one transitive component, +/// and the same-file overlap collapse then reduces it to a single +/// logical location, which is dropped below `MIN_REPORTABLE_MEMBERS` /// (`issue_119_role_gate_exercised`). pub(crate) fn crosses_files(left: &Fingerprint, right: &Fingerprint) -> bool { left.file_id != right.file_id diff --git a/crates/deslop-core/src/pair/candidates.rs b/crates/deslop-core/src/pair/candidates.rs index 10be8624..78a320aa 100644 --- a/crates/deslop-core/src/pair/candidates.rs +++ b/crates/deslop-core/src/pair/candidates.rs @@ -247,15 +247,6 @@ fn same_language_indexes( } } -/// True when the pair's endpoints live in different files — the rescue -/// route's scope ([FUSED-SHARED-SUBTREE]). -fn pair_crosses_files(pair: &CandidatePair, fingerprints: &[Fingerprint]) -> bool { - match (fingerprints.get(pair.left), fingerprints.get(pair.right)) { - (Some(left), Some(right)) => left.file_id != right.file_id, - _ => false, - } -} - /// Keeps non-structural candidates from connecting nested same-file ranges. fn candidate_ranges_are_valid(pair: &CandidatePair, fingerprints: &[Fingerprint]) -> bool { if pair.score.structural > 0.0 { diff --git a/crates/deslop-core/src/pair/candidates/builder.rs b/crates/deslop-core/src/pair/candidates/builder.rs index 1e640401..4e1b3298 100644 --- a/crates/deslop-core/src/pair/candidates/builder.rs +++ b/crates/deslop-core/src/pair/candidates/builder.rs @@ -12,13 +12,17 @@ use super::super::{ FUSED_THRESHOLD, LSH_ONLY_MIN_JACCARD, LSH_ONLY_MIN_NODE_COUNT, }; use super::{ - candidate_ranges_are_valid, endpoint_node_counts, jaccard_for, order, pair_crosses_files, - same_language_indexes, + candidate_ranges_are_valid, endpoint_node_counts, jaccard_for, order, same_language_indexes, }; use crate::{ embedding::EmbeddingPair, fingerprint::Fingerprint, lsh::SignatureLookup, state::FileId, }; +/// Bucket members below which the canonical star already carries every +/// within-file pair, so [`PairBuilder::pair_within_files`] has nothing +/// to add: with two members the star *is* the only pair there is. +const BUCKET_MIN_MEMBERS_FOR_WITHIN_FILE_PAIRS: usize = 3; + /// The ordered pair key packed as one `u64`: high half the lower /// index, low half the higher ([PERF-FLUTTER-TODO-MEMORY]). fn packed_key(key: (usize, usize)) -> u64 { @@ -69,6 +73,10 @@ pub(super) struct PairBuilder<'corpus, S: BuildHasher> { /// retained pair averages a dozen emissions on a corpus-scale run /// — pushing every one again is gigabytes of dead entries). kept_keys: std::collections::HashSet, + /// Scratch `(file, member)` buffer reused by + /// [`Self::pair_within_files`], so grouping one bucket by file costs + /// no allocation of its own ([PERF-FLUTTER-TODO-MEMORY]). + by_file: Vec<(FileId, usize)>, } impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { @@ -87,6 +95,7 @@ impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { kept: Vec::new(), evidence: HashMap::new(), kept_keys: std::collections::HashSet::new(), + by_file: Vec::new(), } } @@ -140,7 +149,9 @@ impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { /// every member with the bucket's first member, and every member /// that shares the first member's file with the bucket's first /// member in another file, so no member of a bucket that spans files - /// is judged on a within-file pair alone. + /// is judged on a within-file pair alone. Members that share a file + /// are then paired with each other by [`Self::pair_within_files`], + /// which is what the star owes a bucket no other file reaches. fn pair_bucket(&mut self, run: &[usize]) { let Some(&canonical) = run.first() else { return; @@ -157,6 +168,49 @@ impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { self.add_evidence(other, foreign, 1.0, 0.0); } } + self.pair_within_files(run); + } + + /// Pairs every two members of `run` that share a file + /// ([FUSED-CANDIDATE-BUCKET-STAR]). + /// + /// The star is only sound when the pair each member is judged on can + /// pass, and inside one file there is no lower-floor scope to borrow + /// that soundness from: the within-file content floor decides every + /// pair, and which member sorts first is an accident of write order. + /// A bucket holding one member that *differs* ahead of a + /// byte-identical copy therefore judged the copy only against the + /// member that differs, and one unrelated sibling deleted an exact + /// duplicate from the report. Recall may not depend on what else + /// happens to share the shape, and no member of the bucket can tell + /// in advance which partner its content will vouch for, so within a + /// file the bucket is completely paired. + /// + /// Members are grouped through the builder's reused scratch buffer + /// rather than a per-bucket map: the historical + /// `HashMap>` here is what + /// [PERF-FLUTTER-TODO-MEMORY] removed from the pass above. + fn pair_within_files(&mut self, run: &[usize]) { + if run.len() < BUCKET_MIN_MEMBERS_FOR_WITHIN_FILE_PAIRS { + return; + } + let mut grouped = std::mem::take(&mut self.by_file); + grouped.clear(); + grouped.extend(run.iter().filter_map(|index| { + self.fingerprints + .get(*index) + .map(|entry| (entry.file_id, *index)) + })); + grouped.sort_unstable(); + for (offset, &(file_id, member)) in grouped.iter().enumerate() { + for &(other_file, other) in grouped.get(offset.saturating_add(1)..).unwrap_or(&[]) { + if other_file != file_id { + break; + } + self.add_evidence(member, other, 1.0, 0.0); + } + } + self.by_file = grouped; } /// Merges the embedding ANN pairs, recording each measured cosine @@ -175,6 +229,7 @@ impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { /// pairs in deterministic key order. pub(super) fn finish(mut self) -> Vec { drop(std::mem::take(&mut self.kept_keys)); + drop(std::mem::take(&mut self.by_file)); self.kept .sort_unstable_by_key(|pair| (pair.left, pair.right)); self.kept.shrink_to_fit(); @@ -279,11 +334,18 @@ impl<'corpus, S: BuildHasher> PairBuilder<'corpus, S> { /// admit — those are the pairs the closure would keep, so the /// retained set induces the same clusters the ungated construction /// produced, at a fraction of the resident memory. + /// + /// The rescue reaches inside one file as well as across files + /// ([FUSED-SHARED-SUBTREE-SAME-FILE]), and only the rescue pass + /// holds the normalised trees its scope rule reads, so the retained + /// set here is bounded instead by what every rescue candidate must + /// be anyway: `candidate_ranges_are_valid` has already refused an + /// anchor-free same-file pair whose endpoints overlap, which is the + /// nested-window population a file would otherwise contribute. fn gate(&self, pair: &CandidatePair) -> bool { if !candidate_ranges_are_valid(pair, self.fingerprints) { return false; } - construction_survives(pair) - || (rescue_eligible(pair) && pair_crosses_files(pair, self.fingerprints)) + construction_survives(pair) || rescue_eligible(pair) } } diff --git a/crates/deslop-core/src/pair/content_gate.rs b/crates/deslop-core/src/pair/content_gate.rs index dbe3b0b5..449eaab4 100644 --- a/crates/deslop-core/src/pair/content_gate.rs +++ b/crates/deslop-core/src/pair/content_gate.rs @@ -13,8 +13,8 @@ use crate::{ }; use super::{ - token_carried, CandidatePair, ExactFunctionAnchors, EMBEDDING_SUPPORT_FLOOR, - LSH_ONLY_MIN_JACCARD, SHARED_SUBTREE_MIN_NODE_COUNT, SHARED_SUBTREE_MIN_OVERLAP, + token_carried, CandidatePair, ExactClones, EMBEDDING_SUPPORT_FLOOR, LSH_ONLY_MIN_JACCARD, + SHARED_SUBTREE_MIN_NODE_COUNT, SHARED_SUBTREE_MIN_OVERLAP, }; /// Structural overlap at which normalised shape saturates the content guard. @@ -42,7 +42,7 @@ pub(crate) fn apply_pair_content_gate( let tree_index: HashMap = trees.iter().map(|tree| (tree.file_id, tree)).collect(); let scopes = DeclarationScopes::new(trees, languages); - let anchors = ExactFunctionAnchors::index(pairs, fingerprints, &scopes); + let anchors = ExactClones::whole_functions_across_files(pairs, fingerprints, &scopes); pairs.retain(|pair| { pair_passes_content_gate( pair, @@ -65,7 +65,7 @@ struct GateContext<'a, L: BuildHasher> { tree_index: &'a HashMap, /// The exact whole-function clones a token-only pair may not merely /// wrap ([FUSED-SHARED-SUBTREE-ECHO]). - anchors: &'a ExactFunctionAnchors, + anchors: &'a ExactClones, /// Authored declarations per file, for the interior-window rule /// ([FUSED-CONTENT-GATE-INTERIOR]). scopes: &'a DeclarationScopes<'a, L>, diff --git a/crates/deslop-core/src/pair/echo.rs b/crates/deslop-core/src/pair/echo.rs index 855dcb77..3d379d96 100644 --- a/crates/deslop-core/src/pair/echo.rs +++ b/crates/deslop-core/src/pair/echo.rs @@ -1,16 +1,26 @@ -//! [FUSED-SHARED-SUBTREE-ECHO] Exact whole-function clones a rescue -//! pair may not merely wrap. +//! Exact clones a rescue pair sits around, or inside +//! ([FUSED-SHARED-SUBTREE-ECHO], [FUSED-SHARED-SUBTREE-SAME-FILE]). //! //! A shared-subtree rescue exists to admit a near-miss the anchor axis -//! cannot see. It is not a second way to publish a clone the anchor -//! axis already proved: a class shell or module preamble that encloses -//! a Merkle-equal authored function in both files measures high overlap -//! *because of that function*, and admitting the container hands -//! subsumption a wider, byte-divergent view that then eats the exact -//! one ([PIPELINE-CLUSTER-SUBSUME] prefers enclosure). The index below -//! records every candidate pair that is Merkle-equal, cross-file, and a -//! run of whole authored functions on both sides, so the rescue can ask -//! how much of a container's shared mass is already claimed. +//! cannot see, and the exact clones the anchor axis *did* prove answer +//! two questions about it. +//! +//! **Across files, what has already been proved?** A class shell or +//! module preamble that encloses a Merkle-equal authored function in +//! both files measures high overlap *because of that function*, and +//! admitting the container hands subsumption a wider, byte-divergent +//! view that then eats the exact one ([PIPELINE-CLUSTER-SUBSUME] prefers +//! enclosure). [`ExactClones::whole_functions_across_files`] records +//! every candidate pair that is Merkle-equal, cross-file, and a run of +//! whole authored functions on both sides, so the rescue can ask how +//! much of a container's shared mass is already claimed. +//! +//! **Inside one file, is this a copy at all?** Two methods that drifted +//! apart in one file share whole statements outright — the same +//! statement, Merkle-equal, in both — while a family of sibling +//! accessors that merely share a skeleton shares none. +//! [`ExactClones::within_one_file`] records the Merkle-equal pairs of one +//! file so the same-file rescue can measure that interior. use std::collections::HashMap; @@ -20,49 +30,44 @@ use crate::{ use super::CandidatePair; -/// One exact whole-function clone: the two ranges it occupies, keyed by -/// the ordered file pair, and the nodes it claims. +/// One exact clone: the two ranges it occupies in canonical order, and +/// the nodes it claims. #[derive(Clone, Copy)] -struct ExactFunctionPair { - /// Range in the lower-numbered file. +struct ExactClone { + /// Range on the lower side of [`ordered`]. first: ByteRange, - /// Range in the higher-numbered file. + /// Range on the higher side of [`ordered`]. second: ByteRange, /// Nodes of the clone — both endpoints agree, being Merkle-equal. nodes: usize, } -/// Every exact whole-function clone among the candidate pairs, indexed -/// by ordered file pair. -pub(crate) struct ExactFunctionAnchors { - /// Exact whole-function pairs by ordered file pair. - by_files: HashMap<(FileId, FileId), Vec>, +/// The Merkle-equal candidate pairs of a corpus, indexed by ordered file +/// pair so a rescue candidate's own file pair is one map hit. +pub(crate) struct ExactClones { + /// Exact pairs by ordered file pair. + by_files: HashMap<(FileId, FileId), Vec>, } -impl ExactFunctionAnchors { - /// Indexes the Merkle-equal, cross-file, function-aligned pairs of - /// `pairs`. - pub(crate) fn index( +impl ExactClones { + /// Indexes every Merkle-equal pair of `pairs` that `admits` accepts. + fn index( pairs: &[CandidatePair], fingerprints: &[Fingerprint], - scopes: &DeclarationScopes<'_, L>, + admits: impl Fn(&Fingerprint, &Fingerprint) -> bool, ) -> Self { - let mut by_files: HashMap<(FileId, FileId), Vec> = HashMap::new(); + let mut by_files: HashMap<(FileId, FileId), Vec> = HashMap::new(); for pair in pairs { let (Some(left), Some(right)) = (fingerprints.get(pair.left), fingerprints.get(pair.right)) else { continue; }; - if left.file_id == right.file_id - || left.hash != right.hash - || !scopes.aligned_function_run(left) - || !scopes.aligned_function_run(right) - { + if left.hash != right.hash || !admits(left, right) { continue; } let (key, first, second) = ordered(left, right); - by_files.entry(key).or_default().push(ExactFunctionPair { + by_files.entry(key).or_default().push(ExactClone { first, second, nodes: left.node_count, @@ -71,6 +76,28 @@ impl ExactFunctionAnchors { Self { by_files } } + /// The cross-file, function-aligned exact clones a container may + /// merely be echoing ([FUSED-SHARED-SUBTREE-ECHO]). + pub(crate) fn whole_functions_across_files( + pairs: &[CandidatePair], + fingerprints: &[Fingerprint], + scopes: &DeclarationScopes<'_, L>, + ) -> Self { + Self::index(pairs, fingerprints, |left, right| { + left.file_id != right.file_id + && scopes.aligned_function_run(left) + && scopes.aligned_function_run(right) + }) + } + + /// The exact clones of one file — the shared interior a same-file + /// rescue is measured against ([FUSED-SHARED-SUBTREE-SAME-FILE]). + pub(crate) fn within_one_file(pairs: &[CandidatePair], fingerprints: &[Fingerprint]) -> Self { + Self::index(pairs, fingerprints, |left, right| { + left.file_id == right.file_id + }) + } + /// Whether an unanchored token-only pair merely wraps an exact /// whole-function clone: both endpoints enclose one, and the larger /// endpoint holds fewer than `floor` nodes beyond it. With no @@ -90,24 +117,49 @@ impl ExactFunctionAnchors { }) } - /// The most nodes any exact whole-function clone claims of the - /// pair's shared mass, or `None` when the pair neither wraps nor - /// sits inside one. A pair is never its own anchor: only a clone - /// that differs from the pair on at least one side is something the - /// pair could merely echo. + /// The most nodes any exact clone claims of the pair's shared mass, + /// or `None` when the pair neither wraps nor sits inside one. A pair + /// is never its own anchor: only a clone that differs from the pair + /// on at least one side is something the pair could merely echo. pub(crate) fn claimed_nodes(&self, left: &Fingerprint, right: &Fingerprint) -> Option { let (key, first, second) = ordered(left, right); let sizes = ( left.node_count.min(right.node_count), left.node_count.max(right.node_count), ); - self.by_files - .get(&key)? - .iter() - .filter(|exact| first != exact.first || second != exact.second) + self.others(key, first, second)? .filter_map(|exact| claimed_by(exact, first, second, sizes)) .max() } + + /// Nodes of the largest exact clone the pair encloses, one endpoint + /// each — the copied interior two drifted declarations still share + /// ([FUSED-SHARED-SUBTREE-SAME-FILE]). `0` when they share none. + pub(crate) fn enclosed_nodes(&self, left: &Fingerprint, right: &Fingerprint) -> usize { + let (key, first, second) = ordered(left, right); + self.others(key, first, second) + .into_iter() + .flatten() + .filter(|exact| first.covers(exact.first) && second.covers(exact.second)) + .map(|exact| exact.nodes) + .max() + .unwrap_or(0) + } + + /// The file pair's exact clones other than the pair itself. + fn others( + &self, + key: (FileId, FileId), + first: ByteRange, + second: ByteRange, + ) -> Option> { + Some( + self.by_files + .get(&key)? + .iter() + .filter(move |exact| first != exact.first || second != exact.second), + ) + } } /// The nodes one exact clone claims of a pair's shared mass. A container @@ -121,7 +173,7 @@ impl ExactFunctionAnchors { /// claimed and a container is never rescued on the strength of a copy /// it merely wraps. fn claimed_by( - exact: &ExactFunctionPair, + exact: &ExactClone, first: ByteRange, second: ByteRange, (smaller, larger): (usize, usize), @@ -141,10 +193,14 @@ fn claimed_by( } } -/// The pair's file key and ranges in file order, so a container pair and -/// the exact pair it wraps line up whichever way each was enumerated. +/// The pair's file key and ranges in canonical order, so a container +/// pair and the exact pair it wraps line up whichever way each was +/// enumerated. Across files the file id orders them; inside one file the +/// candidate's index order says nothing about position, so the byte +/// offset does. fn ordered(left: &Fingerprint, right: &Fingerprint) -> ((FileId, FileId), ByteRange, ByteRange) { - if left.file_id <= right.file_id { + let leads = (left.file_id, left.byte_range.start) <= (right.file_id, right.byte_range.start); + if leads { ( (left.file_id, right.file_id), left.byte_range, diff --git a/crates/deslop-test-support/src/lib.rs b/crates/deslop-test-support/src/lib.rs index 19eeb9fa..2e3334ed 100644 --- a/crates/deslop-test-support/src/lib.rs +++ b/crates/deslop-test-support/src/lib.rs @@ -198,3 +198,54 @@ pub fn write_dart_data_table_fixture(src: &Path) -> Result<()> { )?; Ok(()) } + +/// The body both copies of the star-shadow fixture's duplicated method +/// share, byte for byte. One constant, so the two copies cannot drift +/// into a near-miss and the assertions can name the copied bytes. +pub const CSHARP_COPIED_BODY: &str = " policy.SetCeiling(\"dup\", 250);\n \ + policy.EnableAlerts(\"dup\");\n policy.Audit(\"dup\", 250);\n \ + policy.Commit();\n"; + +/// The sibling's body: the same four statements over different literals, +/// so it shares the copied pair's normalised shape and none of its bytes. +const CSHARP_SIBLING_BODY: &str = " policy.SetCeiling(\"alpha\", 100);\n \ + policy.EnableAlerts(\"alpha\");\n policy.Audit(\"alpha\", 100);\n \ + policy.Commit();\n"; + +/// Wraps `body` in a `Rates.cs` method declaration called `name`. +fn csharp_rate_method(name: &str, body: &str) -> String { + format!(" public void {name}(RatePolicy policy)\n {{\n{body} }}\n") +} + +/// Writes the same-file star-shadow fixture ([FUSED-CANDIDATE-BUCKET-STAR] +/// / [FUSED-SHARED-SUBTREE-SAME-FILE]): one `Rates.cs` holding +/// `ApplyDelta` and `ApplyEpsilon` over [`CSHARP_COPIED_BODY`], optionally +/// preceded by `ApplyAlpha` — the same shape carrying different literals. +/// +/// The sibling changes no byte of the copied pair. It shares the pair's +/// structural hash, so it joins the pair's bucket and, sorting first, +/// becomes the only member every other member is paired against. The +/// copied pair is then never a candidate at all, and one unrelated +/// sibling hides a copy-paste duplicate. +/// +/// # Errors +/// +/// Returns an error when the fixture directory or file cannot be written. +pub fn write_csharp_star_shadow_fixture(src: &Path, with_sibling: bool) -> Result<()> { + std::fs::create_dir_all(src)?; + let sibling = if with_sibling { + format!( + "{}\n", + csharp_rate_method("ApplyAlpha", CSHARP_SIBLING_BODY) + ) + } else { + String::new() + }; + let copied = csharp_rate_method("ApplyDelta", CSHARP_COPIED_BODY); + let pasted = csharp_rate_method("ApplyEpsilon", CSHARP_COPIED_BODY); + std::fs::write( + src.join("Rates.cs"), + format!("public class Rates\n{{\n{sibling}{copied}\n{pasted}}}\n"), + )?; + Ok(()) +} diff --git a/crates/deslop/tests/common/mod.rs b/crates/deslop/tests/common/mod.rs index 31e50573..50b2c220 100644 --- a/crates/deslop/tests/common/mod.rs +++ b/crates/deslop/tests/common/mod.rs @@ -108,6 +108,7 @@ pub(crate) mod verbatim_subgroup; use std::{ collections::{BTreeMap, BTreeSet}, fs, + ops::RangeInclusive, path::{Path, PathBuf}, }; @@ -537,3 +538,39 @@ pub(crate) fn visible_cluster_lines(report: &Value) -> Vec { /// explicitly with `use crate::common::go_scope::*;`, for the same reason /// as `signals`. pub(crate) mod go_scope; + +/// Asserts the cluster's occurrences are exactly `spans` in `file`, in line +/// order. Anything wider has lumped in code that is not duplicated; +/// anything narrower has published a fragment of the authored declaration +/// instead of the declaration ([PIPELINE-CLUSTER-EXACT-SCOPE]). +pub(crate) fn assert_occurrence_extents( + cluster: &Value, + file: &str, + spans: &[RangeInclusive], +) -> Result<()> { + let mut extents: Vec<(String, u64, u64)> = occurrences(cluster) + .iter() + .map(|occurrence| { + Ok(( + occurrence_path(occurrence)?.to_owned(), + field(occurrence, "start_line") + .as_u64() + .ok_or_else(|| anyhow::anyhow!("start_line missing: {occurrence:#}"))?, + field(occurrence, "end_line") + .as_u64() + .ok_or_else(|| anyhow::anyhow!("end_line missing: {occurrence:#}"))?, + )) + }) + .collect::>()?; + extents.sort(); + let expected: Vec<(String, u64, u64)> = spans + .iter() + .map(|lines| (file.to_owned(), *lines.start(), *lines.end())) + .collect(); + assert_eq!( + extents, expected, + "each occurrence is the authored declaration, never its container \ + and never a fragment of it: {cluster:#}" + ); + Ok(()) +} diff --git a/crates/deslop/tests/common/verdict.rs b/crates/deslop/tests/common/verdict.rs index 6203ad99..027f627a 100644 --- a/crates/deslop/tests/common/verdict.rs +++ b/crates/deslop/tests/common/verdict.rs @@ -10,14 +10,16 @@ //! `duplicated_loc`, and a percentage inflated by a shape match would //! have passed every one of them. -use std::path::Path; +use std::{collections::BTreeSet, ops::RangeInclusive, path::Path}; +use anyhow::anyhow; use serde_json::Value; use super::{ - cluster_size, clusters, clusters_hidden, expect_cluster_spanning, field, metric_field, - occurrence_files, occurrence_texts, per_file_metrics, - signals::assert_no_pair_surface_on_cluster, signals::has_verbatim_pair, Result, + assert_occurrence_extents, cluster_size, clusters, clusters_hidden, expect_cluster_spanning, + field, line_count, metric_field, occurrence_files, occurrence_texts, per_file_metrics, signals, + signals::assert_no_pair_surface_on_cluster, signals::has_verbatim_pair, + visible_duplicated_lines, Result, }; /// `metrics.duplicated_loc`, defaulting to `0` so a missing metric @@ -174,6 +176,23 @@ pub(crate) fn expect_cross_file_duplicate( occurrence_texts(scan_root, cluster) } +/// Asserts every string in `evidence` reached the reported occurrence +/// text. `why` says what the evidence is, so a failure names the missing +/// proof rather than the needle. +/// +/// The evidence a control names is the whole point of publishing the +/// cluster — the member that holds the copy, the literal that makes one +/// call dead, the helper that parameterising would absorb — so the loop +/// that checks for it lives here rather than being restated per binary. +pub(crate) fn assert_reported(texts: &[String], evidence: &[&str], why: &str) { + for needle in evidence { + assert!( + texts.iter().any(|text| text.contains(needle)), + "{why}; {needle} must be reported: {texts:#?}" + ); + } +} + /// Asserts every name appears somewhere in the cluster's reported text, /// returning the texts for any further per-test assertions. pub(crate) fn assert_cluster_mentions( @@ -191,6 +210,9 @@ pub(crate) fn assert_cluster_mentions( Ok(texts) } +/// A ratio rendered as a percentage. +const PERCENT_SCALE: f64 = 100.0; + /// Lossless `u64 → f64` for LOC counts and cluster sizes (all far below /// `2^32`), mirroring the renderer's own clamp-then-widen order. Every /// metric re-derivation in the golden suites goes through it so no @@ -198,3 +220,100 @@ pub(crate) fn assert_cluster_mentions( pub(crate) fn loc_as_f64(value: u64) -> Result { Ok(f64::from(u32::try_from(value)?)) } + +/// [METRICS-REPO] Asserts the report's percentage is exactly the report's +/// own line counts, at the repo level and for every file it lists. The +/// headline figure is the reader's to check: a percentage that does not +/// divide the lines beside it is a transparency defect whatever the +/// clusters say. +pub(crate) fn assert_percent_matches_lines(report: &Value) -> Result<()> { + let rows = std::iter::once(("", field(report, "metrics"))).chain( + per_file_metrics(report) + .iter() + .map(|row| (field(row, "path").as_str().unwrap_or("?"), row)), + ); + for (label, row) in rows { + let analysed = field(row, "analysed_loc").as_u64().unwrap_or(0); + let duplicated = field(row, "duplicated_loc").as_u64().unwrap_or(0); + let percent = field(row, "duplication_percent").as_f64().unwrap_or(-1.0); + // Through `loc_as_f64`, like every other metric re-derivation + // here: a raw `as f64` would round a count silently, which is + // the one thing an assertion about an exact division may not do. + let expected = if analysed == 0 { + 0.0 + } else { + loc_as_f64(duplicated)? / loc_as_f64(analysed)? * PERCENT_SCALE + }; + assert!( + (percent - expected).abs() < 0.0001, + "{label}: duplication_percent must be duplicated_loc / analysed_loc \ + — {duplicated}/{analysed} is {expected}, the report says {percent}: \ + {report:#}" + ); + } + Ok(()) +} + +/// The whole published contract for a report whose one finding is a +/// same-file pair: exactly one visible cluster with nothing hidden, two +/// occurrences at `spans` in `file`, the wire mass formula, no pair-only +/// surface, rank one, `distinct` distinct occurrence texts, and metrics +/// that both count the pair's lines and divide into the reported +/// percentage. `why` states what a suppression would prove. +/// +/// Returns the cluster's occurrence texts, so each control still pins the +/// evidence that varies across its own pair. +pub(crate) fn expect_only_finding_is_the_pair( + scan_root: &Path, + report: &Value, + file: &str, + spans: &[RangeInclusive], + distinct: usize, + why: &str, +) -> Result> { + assert!( + metric_field(report, "analysed_loc").as_u64().unwrap_or(0) > 0, + "{why} the pair's file must be parsed (analysed_loc > 0) — a scan \ + that never opened it proves nothing: {report:#}" + ); + let cluster = expect_visible_only(report, 1, why) + .first() + .ok_or_else(|| anyhow!("one visible cluster asserted above"))?; + assert_single_file_cluster(cluster, 2, file); + assert_occurrence_extents(cluster, file, spans)?; + // [RANK-MASS-SUM] / [PIPELINE-CLUSTER-CLOSURE] mass is the wire + // formula over the visible membership, nothing is hidden behind + // `report_hide`, and no pair-only evidence reaches a cluster surface. + signals::assert_structural_only_contract(cluster, why); + signals::assert_no_pair_surface_on_cluster(cluster, why); + assert_eq!( + signals::rank_of(report, cluster)?, + 0, + "{why} the file's one finding is its worst offender: {report:#}" + ); + assert_eq!( + field(cluster, "rank_band").as_str(), + Some("worst"), + "{why} the only finding sits in the worst band: {cluster:#}" + ); + assert_eq!( + signals::distinct_texts(scan_root, cluster)?.len(), + distinct, + "{why} the occurrences' byte truth decides whether this is a \ + verbatim copy or a near-miss, and it must match the fixture: {cluster:#}" + ); + // [METRICS-REPO] the pair's own lines, counted once and divided honestly. + let expected: BTreeSet = spans.iter().cloned().flatten().collect(); + assert_eq!( + visible_duplicated_lines(report).get(file), + Some(&expected), + "{why} only the pair's own lines are duplicated: {report:#}" + ); + assert_eq!( + duplicated_loc(report), + line_count(&expected), + "{why} the metric counts the pair's lines: {report:#}" + ); + assert_percent_matches_lines(report)?; + occurrence_texts(scan_root, cluster) +} diff --git a/crates/deslop/tests/dart_forwarding_fail_open.rs b/crates/deslop/tests/dart_forwarding_fail_open.rs index a9748457..2f62e7d2 100644 --- a/crates/deslop/tests/dart_forwarding_fail_open.rs +++ b/crates/deslop/tests/dart_forwarding_fail_open.rs @@ -105,6 +105,13 @@ struct Scenario { duplication_percent: f64, /// Small windows the lower floor admits and the noise bank convicts. hidden_at_lower_floor: u64, + /// Occurrence counts per visible cluster at [`LOWER_FLOOR`], when a + /// finer floor sees a duplication the shared one cannot fingerprint. + /// Empty where both floors report the same clusters. + finer_sizes: &'static [u64], + /// Every occurrence line at [`LOWER_FLOOR`], in report order, paired + /// with `finer_sizes`. + finer_lines: &'static [(u64, u64)], members: &'static [&'static str], literals: &'static [&'static str], why: &'static str, @@ -120,6 +127,8 @@ const COMPUTING_PAIR: Scenario = Scenario { analysed_loc: 21, duplication_percent: 28.571_428_571_428_57, hidden_at_lower_floor: 0, + finer_sizes: &[], + finer_lines: &[], members: &["scaledDomestic", "scaledExport"], literals: &["* rate + 7", "* factor + 7"], why: "two one-statement Dart methods that multiply and add are liftable \ @@ -138,6 +147,8 @@ const BUSINESS_PAIR: Scenario = Scenario { analysed_loc: 48, duplication_percent: 16.666_666_666_666_664, hidden_at_lower_floor: 2, + finer_sizes: &[], + finer_lines: &[], members: &[ "standardTotal", "premiumTotal", @@ -162,6 +173,8 @@ const AFTER_DELEGATION: Scenario = Scenario { analysed_loc: 47, duplication_percent: 17.021_276_595_744_68, hidden_at_lower_floor: 1, + finer_sizes: &[], + finer_lines: &[], members: &[ "standardTotal", "premiumTotal", @@ -186,6 +199,8 @@ const BEFORE_DELEGATION: Scenario = Scenario { analysed_loc: 48, duplication_percent: 16.666_666_666_666_664, hidden_at_lower_floor: 0, + finer_sizes: &[], + finer_lines: &[], members: &["quarterlyFee", "annualCharge", "normalise", "client.submit"], literals: &["\"standard\"", "\"premium\"", "100", "250"], why: "the class computes on its own inputs through `normalise` and only \ @@ -204,6 +219,25 @@ const DUPLICATE_ROUTE_WRAPPER_FAMILY: Scenario = Scenario { analysed_loc: 45, duplication_percent: 33.333_333_333_333_33, hidden_at_lower_floor: 0, + // [FUSED-CANDIDATE-BUCKET-STAR] At eight nodes the wrappers' bodies + // are fingerprinted in their own right, and `resetDelta`'s and + // `resetEpsilon`'s are byte-identical — the dead route this fixture + // exists to catch, which its header says is "visible only in the + // proven bodies" because the declarations differ by name. The family + // view cannot show it; this one names it. Both publish: the narrow + // view covers a strict sub-region of the wide one, and + // [PIPELINE-CLUSTER-SUBSUME] treats one-sided containment as two + // findings rather than a re-description. + finer_sizes: &[DUPLICATE_ROUTE_FAMILY, PAIR], + finer_lines: &[ + (22, 24), + (26, 28), + (30, 32), + (34, 36), + (38, 40), + (35, 35), + (39, 39), + ], members: &[ "resetAlpha", "resetBeta", @@ -275,6 +309,42 @@ fn assert_family(scenario: &Scenario, report: &Value, hidden: u64) { ); } +/// The lower floor's family, which may be finer than the shared one. +/// +/// A floor fine enough to fingerprint a wrapper's body can prove a +/// duplication the declaration view cannot express, so a scenario may +/// name what that floor adds. The metrics do not move with it: the finer +/// occurrences lie inside lines the wider view already counted. +fn assert_lower_floor(scenario: &Scenario, report: &Value, hidden: u64) { + if scenario.finer_sizes.is_empty() { + assert_family(scenario, report, hidden); + return; + } + let why = scenario.why; + let sizes: Vec = clusters(report).iter().map(cluster_size).collect(); + assert_eq!( + sizes, scenario.finer_sizes, + "{why} a finer floor proves the interior copy the declarations hide, and publishes it beside them: {report:#}" + ); + for cluster in clusters(report) { + assert_eq!( + occurrence_files(cluster), + vec![scenario.file.to_owned(); usize::try_from(cluster_size(cluster)).unwrap_or(0)], + "{why} every occurrence is in the one file: {cluster:#}" + ); + } + assert_eq!( + reported_lines(report), + scenario.finer_lines, + "{why} every occurrence must be reported at its authored extent: {report:#}" + ); + assert_eq!( + clusters_hidden(report), + hidden, + "{why} hidden count: {report:#}" + ); +} + /// [METRICS-REPO] The file's figures, as the engine computed them. fn assert_metrics(scenario: &Scenario, report: &Value) -> Result<()> { let why = scenario.why; @@ -316,18 +386,6 @@ fn assert_evidence(scenario: &Scenario, report: &Value) -> Result> { Ok(texts) } -/// Asserts every string in `evidence` reached the reported occurrence -/// text. `why` says what the evidence is, so a failure names the missing -/// proof rather than the needle. -fn assert_reported(texts: &[String], evidence: &[&str], why: &str) { - for needle in evidence { - assert!( - texts.iter().any(|text| text.contains(needle)), - "{why}; {needle} must be reported: {texts:#?}" - ); - } -} - /// [CLI-TEXT] The text renderer prints the same headline figures and /// every occurrence as `file:start:end`. fn assert_text_report(scenario: &Scenario, text: &str) { @@ -400,7 +458,7 @@ fn run_control(scenario: &Scenario) -> Result> { assert_metrics(scenario, &higher)?; let (lower, _) = scan(scenario, LOWER_FLOOR, &[], 0)?; - assert_family(scenario, &lower, scenario.hidden_at_lower_floor); + assert_lower_floor(scenario, &lower, scenario.hidden_at_lower_floor); assert_metrics(scenario, &lower)?; assert_breached_fail_over(scenario)?; diff --git a/crates/deslop/tests/same_file_rescue.rs b/crates/deslop/tests/same_file_rescue.rs index ce1e402b..2881946f 100644 --- a/crates/deslop/tests/same_file_rescue.rs +++ b/crates/deslop/tests/same_file_rescue.rs @@ -6,15 +6,25 @@ //! that admitted them would hand [PIPELINE-CLUSTER-SUBSUME] a wider, //! byte-divergent view that encloses the exact one and replaces it. The //! finding is the method, at its own lines, in both classes. +//! +//! [FUSED-CANDIDATE-BUCKET-STAR] A same-file bucket pairs every member +//! with the member that sorts first and with nothing else. When the +//! first member is the one that *differs*, the byte-identical pair +//! behind it is never a candidate, so one unrelated sibling deletes an +//! exact duplicate from the report. Recall must not depend on what else +//! happens to share the shape: the copied pair is the same finding with +//! or without the sibling standing in front of it. use std::{collections::BTreeSet, ops::RangeInclusive}; use anyhow::Result; +use deslop_test_support::{write_csharp_star_shadow_fixture, CSHARP_COPIED_BODY}; + use crate::common::signals::{ - assert_no_pair_surface_on_cluster, assert_structural_only_contract, distinct_texts, - has_verbatim_pair, + assert_no_pair_surface_on_cluster, assert_structural_only_contract, has_verbatim_pair, }; +use crate::common::verdict::{assert_reported, expect_only_finding_is_the_pair}; use crate::common::*; /// One file, two classes, one method copied byte for byte between them. @@ -30,6 +40,14 @@ const BETA_RECONCILE_LINES: RangeInclusive = 31..=46; const SIBLING_CLASS_MIN_NODES: u32 = 20; /// Files the fixture holds; a one-file scan must still analyse it. const SIBLING_CLASS_FILE_COUNT: u64 = 1; +/// The two copies are byte-identical, so they slice to one text. +const SIBLING_CLASS_DISTINCT_TEXTS: usize = 1; +/// What a suppression of this pair would prove. +const SIBLING_CLASS_WHY: &str = + "one method copied byte for byte between two sibling classes in one file \ + is the finding, at its own lines in both classes. Publishing the classes \ + instead would widen a byte-divergent view over the exact one; publishing \ + nothing would mean the same-file echo rule refused the copy itself."; #[test] fn sibling_classes_wrapping_one_exact_method_publish_the_method() -> Result<()> { @@ -42,75 +60,204 @@ fn sibling_classes_wrapping_one_exact_method_publish_the_method() -> Result<()> ); // [FUSED-SHARED-SUBTREE-ECHO] The class pair shares nothing beyond // the method it wraps, so it is refused and cannot widen the finding. - let published = clusters(&report); - assert_eq!( - published.len(), - 1, - "the copied method is the only duplication at this floor: {report:#}" - ); - let cluster = published - .first() - .ok_or_else(|| anyhow::anyhow!("one cluster asserted above"))?; - assert_method_pair_extents(cluster)?; + let cluster = expect_only_finding_is_the_pair( + &scan_root, + &report, + SIBLING_CLASS_FILE, + &[ALPHA_RECONCILE_LINES, BETA_RECONCILE_LINES], + SIBLING_CLASS_DISTINCT_TEXTS, + SIBLING_CLASS_WHY, + ) + .map(|_| ()) + .and_then(|()| { + clusters(&report) + .first() + .ok_or_else(|| anyhow::anyhow!("one cluster asserted above")) + })?; // [PIPELINE-CLUSTER-EXACT-SCOPE] Both occurrences are the authored - // method: byte-identical, verbatim, mass-honest, clean-surfaced. - assert_eq!( - distinct_texts(&scan_root, cluster)?.len(), - 1, - "the two methods slice to one text: {cluster:#}" - ); + // method, and a byte-for-byte copy is a verbatim pair. assert!( has_verbatim_pair(&scan_root, cluster)?, - "a byte-for-byte copy is a verbatim pair: {cluster:#}" + "{SIBLING_CLASS_WHY} a byte-for-byte copy is a verbatim pair: {cluster:#}" ); assert_structural_only_contract(cluster, SIBLING_CLASS_FIXTURE); assert_no_pair_surface_on_cluster(cluster, SIBLING_CLASS_FIXTURE); - assert_published_lines_are_the_two_methods(&report); Ok(()) } -/// Each occurrence sits in the one file at its method's own lines; the -/// class shells, fields and accessors around them are not duplicated. -fn assert_method_pair_extents(cluster: &serde_json::Value) -> Result<()> { - let mut extents: Vec<(String, u64, u64)> = occurrences(cluster) - .iter() - .map(|occurrence| { - Ok(( - occurrence_path(occurrence)?.to_owned(), - field(occurrence, "start_line") - .as_u64() - .ok_or_else(|| anyhow::anyhow!("start_line missing: {occurrence:#}"))?, - field(occurrence, "end_line") - .as_u64() - .ok_or_else(|| anyhow::anyhow!("end_line missing: {occurrence:#}"))?, - )) - }) - .collect::>()?; - extents.sort(); - let expected: Vec<(String, u64, u64)> = [ALPHA_RECONCILE_LINES, BETA_RECONCILE_LINES] - .iter() - .map(|lines| (SIBLING_CLASS_FILE.to_owned(), *lines.start(), *lines.end())) - .collect(); +/// The one file the star-shadow fixture holds. +const STAR_SHADOW_FILE: &str = "Rates.cs"; +/// A floor above the individual statements, so the copied method is the +/// only duplication the scan can reach. +const STAR_SHADOW_MIN_NODES: u32 = 12; +/// `ApplyDelta` and `ApplyEpsilon` with `ApplyAlpha` written ahead of +/// them — the sibling that shares their shape and none of their bytes. +const SHADOWED_COPY_LINES: RangeInclusive = 11..=17; +/// `ApplyEpsilon` in the same variant. +const SHADOWED_PASTE_LINES: RangeInclusive = 19..=25; +/// `ApplyDelta` with no sibling ahead of it. +const ALONE_COPY_LINES: RangeInclusive = 3..=9; +/// `ApplyEpsilon` in that variant. +const ALONE_PASTE_LINES: RangeInclusive = 11..=17; +/// `ApplyAlpha`, which is nobody's duplicate and must reach no cluster. +const SIBLING_LINES: RangeInclusive = 3..=9; +/// The sibling's own name, which may not appear in the finding either. +const SIBLING_METHOD_NAME: &str = "ApplyAlpha"; +/// The declaration names differ, so the two occurrences are two texts. +const STAR_SHADOW_DISTINCT_TEXTS: usize = 2; +/// Files each variant holds; a one-file scan must still analyse it. +const STAR_SHADOW_FILE_COUNT: u64 = 1; +/// The two declaration names over the one copied body. Both must be +/// reported: the copy-paste is visible only in the bodies, because the +/// names the reader would compare are exactly what differs. +const COPIED_METHOD_NAMES: [&str; 2] = ["ApplyDelta", "ApplyEpsilon"]; + +/// [FUSED-CANDIDATE-BUCKET-STAR] An exact same-file duplicate stays +/// reported when a differing sibling of the same shape is written ahead +/// of it. The two scans below hold the same copied method; the only +/// difference is the sibling, which duplicates nothing. +#[test] +fn a_shape_sibling_may_not_hide_an_exact_same_file_copy() -> Result<()> { + let alone = assert_copied_pair_published(false, &[ALONE_COPY_LINES, ALONE_PASTE_LINES])?; + let shadowed = + assert_copied_pair_published(true, &[SHADOWED_COPY_LINES, SHADOWED_PASTE_LINES])?; assert_eq!( - extents, expected, - "both occurrences are the `Reconcile` method, never its class: {cluster:#}" + alone, shadowed, + "the sibling changes no byte of the copied method, so it must not \ + change the reported occurrences either" ); Ok(()) } -/// [METRICS-REPO] The duplicated lines are exactly the two method -/// bodies, so the headline count is honest about what was found. -fn assert_published_lines_are_the_two_methods(report: &serde_json::Value) { - let expected: BTreeSet = ALPHA_RECONCILE_LINES.chain(BETA_RECONCILE_LINES).collect(); - let published = visible_duplicated_lines(report); - assert_eq!( - published.get(SIBLING_CLASS_FILE), - Some(&expected), - "only the two methods are duplicated lines: {report:#}" - ); +/// Scans one variant of the star-shadow fixture and asserts the copied +/// pair is the report: one cluster, both methods at `spans`, the copied +/// body in both occurrence texts, and no line outside the pair counted as +/// duplicated. Returns the reported texts sorted, so the caller can hold +/// both variants to the same finding. +fn assert_copied_pair_published( + with_sibling: bool, + spans: &[RangeInclusive], +) -> Result> { + let workspace = tempfile::tempdir()?; + let scan_root = workspace.path().join("src"); + write_csharp_star_shadow_fixture(&scan_root, with_sibling)?; + let report = run_report(&scan_root, STAR_SHADOW_MIN_NODES)?; assert_eq!( - visible_duplicated_loc(report), - line_count(&expected), - "the duplicated line count is the two methods: {report:#}" + field(&report, "files_analysed").as_u64(), + Some(STAR_SHADOW_FILE_COUNT), + "the one file must be analysed (with_sibling={with_sibling}): {report:#}" ); + let why = star_shadow_why(with_sibling); + let mut texts = expect_only_finding_is_the_pair( + &scan_root, + &report, + STAR_SHADOW_FILE, + spans, + STAR_SHADOW_DISTINCT_TEXTS, + &why, + )?; + texts.sort(); + for text in &texts { + assert!( + text.contains(CSHARP_COPIED_BODY), + "{why} each occurrence must report the copied body itself, not a \ + fragment of it: {text}" + ); + } + assert_reported(&texts, &COPIED_METHOD_NAMES, &why); + if with_sibling { + let duplicated: BTreeSet = spans.iter().cloned().flatten().collect(); + assert!( + SIBLING_LINES + .clone() + .all(|line| !duplicated.contains(&line)), + "{why} the sibling duplicates nothing and none of its lines may \ + be counted: {report:#}" + ); + for text in &texts { + assert!( + !text.contains(SIBLING_METHOD_NAME), + "{why} the sibling may not be pulled into the finding: {text}" + ); + } + } + Ok(texts) +} + +/// What a suppression of this scan would prove. The two variants fail for +/// different reasons, so each names its own. +fn star_shadow_why(with_sibling: bool) -> String { + let shared = "two byte-identical method bodies in one file are a copy-paste \ + duplicate and the report's whole finding"; + if with_sibling { + format!( + "{shared}. Hiding them when a differing sibling of the same shape is \ + written above them proves recall depends on write order: the bucket \ + pairs every member with the member that sorts first, and that member \ + is the one that differs." + ) + } else { + format!( + "{shared}. Hiding them with nothing else in the file proves the detector went blind." + ) + } +} + +/// Two methods in one file that differ in nothing but their literals. +const MANY_HOLES_FIXTURE: &str = "csharp-merge-manyholes"; +/// The only file the fixture holds. +const MANY_HOLES_FILE: &str = "Sprawl.cs"; +/// `Sprawl.ApplyStandard` — six `Set` calls and a `Commit`. +const MANY_HOLES_STANDARD_LINES: RangeInclusive = 3..=12; +/// `Sprawl.ApplyPremium`, the copy, at twelve different literals. +const MANY_HOLES_PREMIUM_LINES: RangeInclusive = 14..=23; +/// The same floor the rest of the same-file band is measured at. +const MANY_HOLES_MIN_NODES: u32 = 12; +/// Both member names; the pair is the finding, so both must be reported. +const MANY_HOLES_METHOD_NAMES: [&str; 2] = ["ApplyStandard", "ApplyPremium"]; +/// Twelve literals differ, so the two declarations are never one text. +const MANY_HOLES_DISTINCT_TEXTS: usize = 2; +/// What a suppression of this pair would prove. +const MANY_HOLES_WHY: &str = + "two methods that share every identifier and every call, and substitute \ + consistently at all twelve literal positions, are one parameterised \ + method. Hiding them proves the same-file promote floor judged a \ + literal-only copy on the one axis its own edit demolishes."; + +/// [FUSED-CONTENT-GATE] A same-file pair is routed on +/// `support = max(agreement, rename_consistency)` against +/// `content_gate.promote_floor`. `ApplyStandard` and `ApplyPremium` share +/// every identifier position and every call, and substitute consistently +/// at all twelve literal positions; their measured `agreement` is 0.567 +/// and their `rename_consistency` is 0.0, so the gate refuses them and +/// the file publishes nothing. +/// +/// The two halves of that reading cannot both be right. [TECH-PMATCH-BAKER] +/// makes `rename_consistency` the Type-2 discriminator over *"the +/// identifier positions the bijection must explain plus every aligned +/// literal position"* — and here every identifier position is preserved +/// and every literal substitutes. Reporting `0.0` for that pair leaves +/// `max(agreement, rename_consistency)` reading agreement alone, so a +/// literal-only copy is judged on the one axis its own edit demolishes. +/// +/// [AUTOFIX-MERGE-GATE] independently calls this pair a duplication: the +/// merge gate refuses it for *twelve distinct substitutions exceeding the +/// budget*, which is a statement about a clone too parameterised to merge +/// mechanically, not about two unrelated methods. `too_many_holes_refuse` +/// reaches that verdict through a synthetic plan, so it holds even while +/// the detector publishes nothing to plan over. +#[test] +fn a_literal_only_copy_inside_one_file_is_a_finding() -> Result<()> { + let scan_root = fixture(MANY_HOLES_FIXTURE); + let report = run_report(&scan_root, MANY_HOLES_MIN_NODES)?; + let texts = expect_only_finding_is_the_pair( + &scan_root, + &report, + MANY_HOLES_FILE, + &[MANY_HOLES_STANDARD_LINES, MANY_HOLES_PREMIUM_LINES], + MANY_HOLES_DISTINCT_TEXTS, + MANY_HOLES_WHY, + )?; + assert_reported(&texts, &MANY_HOLES_METHOD_NAMES, MANY_HOLES_WHY); + Ok(()) } diff --git a/crates/deslop/tests/skip_policy_contract.rs b/crates/deslop/tests/skip_policy_contract.rs index b12bdd2f..18edad39 100644 --- a/crates/deslop/tests/skip_policy_contract.rs +++ b/crates/deslop/tests/skip_policy_contract.rs @@ -69,7 +69,7 @@ const TEST_TARGET_KIND: &str = "test"; /// /// Those counts are prose, and prose drifts. [`SKIPS_PER_ISSUE`] is what /// stops it drifting silently. -const CURATED_SKIPS: [(&str, &str, u32); 18] = [ +const CURATED_SKIPS: [(&str, &str, u32); 17] = [ ( "crates/deslop-lsp/tests/lsp_embedding_determinism.rs", "lsp_embedding_refresh_is_bounded_and_reproducible", @@ -147,11 +147,6 @@ const CURATED_SKIPS: [(&str, &str, u32); 18] = [ "perf_sample_bounded_scan", 422, ), - ( - "crates/deslop/tests/type3_enclosing_method.rs", - "csharp_same_file_type3_reports_both_methods_in_one_cluster", - 492, - ), ]; /// How many curated skips each tracking issue owns. @@ -161,7 +156,7 @@ const CURATED_SKIPS: [(&str, &str, u32); 18] = [ /// #432–#435 entries when the registry held nine across #432–#434 and none /// for #435. That is a wrong answer to the question a reader is actually /// asking: which plan still owns this block of silence, and how much of it. -const SKIPS_PER_ISSUE: [(u32, usize); 5] = [(369, 2), (422, 13), (489, 1), (491, 1), (492, 1)]; +const SKIPS_PER_ISSUE: [(u32, usize); 4] = [(369, 2), (422, 13), (489, 1), (491, 1)]; /// How many skips each issue owns, counted from the registry itself. fn skips_by_issue() -> BTreeMap { diff --git a/crates/deslop/tests/type3_enclosing_method.rs b/crates/deslop/tests/type3_enclosing_method.rs index 5cd5dcbc..14cde7aa 100644 --- a/crates/deslop/tests/type3_enclosing_method.rs +++ b/crates/deslop/tests/type3_enclosing_method.rs @@ -242,18 +242,16 @@ fn csharp_type3_reports_the_enclosing_method_pair() -> Result<()> { ) } -// [FUSED-SHARED-SUBTREE] A shared-subtree rescue is evidence about the -// two authored methods, and the file boundary records where the copy was -// pasted rather than whether it is a copy. The rescue is cross-file only, -// so `DriftLimits.cs` publishes the statement fragments its two methods -// share and never the methods. +// [FUSED-SHARED-SUBTREE-SAME-FILE] A shared-subtree rescue is evidence +// about the two authored methods, and the file boundary records where the +// copy was pasted rather than whether it is a copy — the spec says so in +// as many words. `ApplyStandard` and `ApplyPremium` measure overlap 0.82. +// While the rescue was cross-file only, `DriftLimits.cs` published +// `:6-8`/`:18-20` and `:9-12`/`:25-28` — two statement fragments that +// name neither method — and never the pair, so a reader was told about +// pieces of a duplication and never about the duplication. The pair is +// the finding, and the fragments are absorbed into it. #[test] -#[ignore = "[SKIP-UNFINISHED] GH #492 [FUSED-SHARED-SUBTREE] docs/plans/same-file-rescue-plan.md — \ - shared-subtree rescue is cross-file only, so two methods that drifted apart \ - inside one file publish as fragments. Admitting every disjoint same-file pair \ - ranks shape families above real clones (the settings-getter and helper-call-site \ - fixtures), so the route needs a discriminator this release does not have. \ - Assertions are intact — run with `-- --ignored`."] fn csharp_same_file_type3_reports_both_methods_in_one_cluster() -> Result<()> { const FIXTURE: &str = "csharp-merge-drift"; const FILE: &str = "DriftLimits.cs"; diff --git a/docs/plans/same-file-rescue-plan.md b/docs/plans/same-file-rescue-plan.md index 18820d83..6e28ccff 100644 --- a/docs/plans/same-file-rescue-plan.md +++ b/docs/plans/same-file-rescue-plan.md @@ -2,7 +2,7 @@ Tracking issue: gh #492 (two drifted methods never cluster). gh #496 (two methods differing only in literals were refused below a same-file promote floor) is settled below and no longer needs the rescue. -Two methods that drifted apart inside one file are the same duplication as two that drifted apart across files. The file boundary records where the copy was pasted, not whether it is a copy. Today the shared-subtree rescue of [FUSED-SHARED-SUBTREE](../specs/fused.md) is cross-file only, so a same-file near-miss publishes the statement fragments its two methods share and never the methods. +Two methods that drifted apart inside one file are the same duplication as two that drifted apart across files. The file boundary records where the copy was pasted, not whether it is a copy. The shared-subtree rescue of [FUSED-SHARED-SUBTREE](../specs/fused.md) was cross-file only, so a same-file near-miss published the statement fragments its two methods share and never the methods. It reaches them now, and gh #492 is closed alongside gh #496. ## What the gap costs @@ -10,13 +10,13 @@ The band has two halves. Below the 0.70 support floor, [FUSED-CONTENT-GATE] refu `csharp-merge-drift` holds `ApplyStandard` and `ApplyPremium` in `DriftLimits.cs`. They share a five-call skeleton; the premium copy grew an escalation guard and its own literals. The pair measures shared-subtree overlap 0.82. Nothing publishes it: 0.32.0 reported four fragment clusters covering two-line windows and single statements, and the current release reports the exact tail alone. A reader is told about pieces of a duplication and never about the duplication. -Pinned by `type3_enclosing_method::csharp_same_file_type3_reports_both_methods_in_one_cluster`, which asserts one cluster over lines 3-13 and 15-29 with every fragment absorbed. It is `#[ignore]`d, with its assertions intact, until the route below exists. +Pinned by `type3_enclosing_method::csharp_same_file_type3_reports_both_methods_in_one_cluster`, which asserts one cluster over lines 3-13 and 15-29 with every fragment absorbed. Its `#[ignore]` is gone and its assertions are unchanged. The other half needed no rescue at all, only the floor, and it is settled. `dart-forwarding-business-pair` holds `standardTotal` and `premiumTotal`: structurally identical, differing in one string literal and one integer, measuring agreement 0.727 and rename consistency 0.0. The 0.85 same-file admission floor that refused it was PR #485's relocation of a render-time bucket grade to admission; 0.32.0 published the pair as `structural_only` and left the sibling-family question to [RANK-STRUCTURAL-ONLY-FORWARDING], which reads where each call goes. That is the discriminator: the REST settings family and a two-literal copy of one method measure in the same 0.70–0.85 band, so no admission floor separates them, and the forwarding proof does. Every pair now pays `content_gate.support_floor` in every scope; only an unanchored LSH-only pair pays `promote_floor`. `dart_forwarding_fail_open.rs` asserts the positive contract its documentation always stated (gh #496, gh #497), `declaration_family_plurality` again publishes the nonbijective pair its fixture calls liftable, and `content_gate_admits.rs` pins the admission at the pipeline seam. `rename_consistency` was right to report 0.0 for the pair: nothing in it is renamed, and the rename axis measures renames. A literal-only copy is judged on agreement, which is the lever that now does the work. -`csharp-merge-manyholes` (agreement 0.50–0.57) still falls below the support floor and stays with gh #492 below. +`csharp-merge-manyholes` (agreement 0.50–0.57) still falls below the support floor, and agreement is not the axis that should judge it. Every identifier and every call is preserved and only the twelve literals move, which [FUSED-CONTENT-GATE-PARAMETER] now reads as the parameterisation it is: where the bijection claims no rename, a literal substituted once is Baker's unconstrained wildcard rather than a contradiction. `Sprawl.cs:3-12` / `:14-23` publishes. ## Why admitting every same-file pair is wrong @@ -28,7 +28,13 @@ Admitting each otherwise-valid same-file candidate to rescue measurement was tri Requiring both endpoints to be whole authored functions removes the table and window cases but not these: a class of sibling accessors is a set of whole authored functions. Requiring a Merkle-equal fragment inside both endpoints does not remove them either, because the call sites share byte-identical argument runs. -## What the route needs +## What landed + +**The rescue, inside one file** ([FUSED-SHARED-SUBTREE-SAME-FILE]). A same-file pair reaches shared-subtree measurement when both endpoints are whole authored declarations, they do not overlap, they still enclose a Merkle-equal clone of at least `admission.shared_subtree_min_node_count` nodes, and the shared mass *beyond* that clone clears the same floor. The second condition is the discriminator this plan was looking for, and it is neither of the thresholds below: the settings accessors' overlap (0.81–0.88) brackets the drifted pair's 0.84 and their agreement (up to 0.56) brackets its 0.55, so neither shape nor content sorts them — but the drifted pair keeps four whole statements the edit never touched and the family keeps none. The third condition is the echo rule turned inward, and it is why `csharp-merge-readafter` still publishes the contiguous run its two methods share rather than the methods wrapping it. + +**The bucket star, inside one file** ([FUSED-CANDIDATE-BUCKET-STAR]). A structural-hash bucket paired every member with the member that sorted first. Inside one file that member decides everything, and when it is the one that *differs*, the byte-identical pair behind it was never a candidate at all — one unrelated sibling deleted an exact duplicate from the report. Members that share a file are now paired with each other, all of them. The cost is quadratic in same-file bucket size and is tracked as gh #506. + +## Discriminators considered and not taken A discriminator that separates a copied method from a shape family, computed from evidence the pipeline already produces, not a threshold fitted between two fixtures. Candidates worth measuring: @@ -38,7 +44,7 @@ A discriminator that separates a copied method from a shape family, computed fro ## Acceptance — how gh #492 and its skip end -- `csharp_same_file_type3_reports_both_methods_in_one_cluster` passes with its assertions unchanged and its `#[ignore]` removed. -- `dart_forwarding_fail_open`'s controls keep asserting what the module documentation states, and `csharp-merge-manyholes` gains an occurrence and range pin. -- `dart_issue_197_single_file_structural_only`, `python_issue_103_helper_call_sites`, the three `issue_190` modes, `cli::bucket_groups` and both `refactor_merge_refusals` same-file pins stay green. -- The paired 0.32.0 fixture scan loses no finding. +- [x] `csharp_same_file_type3_reports_both_methods_in_one_cluster` passes with its assertions unchanged and its `#[ignore]` removed. +- [x] `dart_forwarding_fail_open`'s controls keep asserting what the module documentation states, and `csharp-merge-manyholes` gains an occurrence and range pin. +- [x] `dart_issue_197_single_file_structural_only`, `python_issue_103_helper_call_sites`, the three `issue_190` modes, `cli::bucket_groups` and both `refactor_merge_refusals` same-file pins stay green. +- [x] The paired 0.32.0 fixture scan loses no finding. diff --git a/docs/specs/fused.md b/docs/specs/fused.md index 76b15c09..64ad65e1 100644 --- a/docs/specs/fused.md +++ b/docs/specs/fused.md @@ -39,7 +39,7 @@ The shape and semantic axes are correlated views of the same two occurrences, so 4. **Admission is decided pair by pair.** A pair must pass the size-coherence and applicable LSH-only guards, clear its pair-specific shape/semantic threshold or the cross-file rescue, and pass every applicable pair-content guard. Group-level similarity judgement and averaging are forbidden. Clusters are the transitive closure of admitted pairs before the separate convicted-noise rule in [CLONE-NOISE-VERBATIM-SUBGROUP]. 5. Rank clusters by duplicated mass alone ([pipeline.md §RANK-MASS-SUM](pipeline.md#rank-mass-sum)). No pair evidence participates in mass or order. -#### [FUSED-CANDIDATE-BUCKET-STAR] A structural-hash bucket pairs every member across files +#### [FUSED-CANDIDATE-BUCKET-STAR] A structural-hash bucket pairs every member across files, and every pair of members inside one file Members of one structural-hash bucket are Merkle-equal, so a pair between any two of them scores `S = 1.0` and only the pair-content gate decides it. The candidate generator therefore does not emit every pair of a bucket: each member is paired with the bucket's first member, and closure carries the rest. @@ -47,6 +47,8 @@ That star is only sound when the pair each member is judged on can pass. The con Without this, the second and third copies in the first file are compared only with the first copy — a within-file pair the floor refuses — and drop out of the family even though byte-for-byte the same comparison admits the copies in every other file. +Inside one file there is no other scope to borrow that soundness from, and which member sorts first is an accident of write order. So **members of a bucket that share a file are paired with each other, all of them.** A bucket holding one member that *differs* ahead of a byte-identical copy otherwise judged the copy only against the member that differs, and one unrelated sibling deleted an exact duplicate from the report — recall may not depend on what else happens to share the shape. No member can tell in advance which partner its content will vouch for, so within a file the bucket is completely paired. Pinned by `same_file_rescue::a_shape_sibling_may_not_hide_an_exact_same_file_copy`. + This way, a Type-1 pair scores 1.0 on exact structure, a Type-2 pair normally scores 1.0 on normalised structure and is corroborated by raw-content or consistent-rename evidence, a Type-3 pair may score high on token or graded structural evidence, and a Type-4 pair relies primarily on embeddings. Cosine, Jaccard, alignment, and content support are not interchangeable probabilities; their configured thresholds are corpus-derived operating points. ### [FUSED-SCOPE] `fused` is a pair quantity @@ -92,11 +94,15 @@ The anchor is an authored **function** or a run of them, never a sub-block: a Ty Implemented in `pair/echo.rs`, applied in `overlap/rescue.rs` and `pair/content_gate.rs`; pinned by `issue_389_subsumption_modifier_straddle`, the `incremental-multilang` golden across all six languages, `fsharp_issue_339_sibling_window_rename` (an F# module wrapping an exact two-binding window), and `js_ts_extensions::javascript_family_clusters_across_js_mjs_and_cjs_extensions` (three whole files that may not widen past the declaration they share). -### [FUSED-SHARED-SUBTREE-SAME-FILE] A near-miss inside one file is not rescued, and that is a known gap +### [FUSED-SHARED-SUBTREE-SAME-FILE] A near-miss inside one file is rescued when the two declarations still share copied code + +Two methods that drifted apart inside one file are the same duplication as two that drifted apart across files: the file boundary records where the copy was pasted, not whether it is a copy. The rescue reaches them, on three conditions that hold together. -Two methods that drifted apart inside one file are the same duplication as two that drifted apart across files: the file boundary records where the copy was pasted, not whether it is a copy. The rescue does not reach them. `csharp-merge-drift` holds two such methods at measured overlap 0.82 and publishes only the statement fragments they share. +1. **Both endpoints are whole authored declarations**, modifier through closing brace, and they do not overlap. A window cut across statements, a nested view, and a table row are none of them a declaration. Admitting those unioned a file's subtrees into one component that the same-file collapse then reduced to a single location, and the file's real duplication disappeared rather than being reported (`issue_119_role_gate_exercised`). +2. **They still enclose a Merkle-equal clone** of at least `admission.shared_subtree_min_node_count` nodes — authored code the edit never touched. This is what separates a copy that drifted from a family that never was one. `csharp-merge-drift`'s two methods keep four whole statements, 32 nodes; the `dart-issue-197` settings accessors keep no statement at all, yet their overlap (0.81–0.88) brackets the drifted pair's 0.84 and their raw-content agreement reaches 0.56 against its 0.55. Shape and agreement cannot tell them apart. Copied code can. +3. **The shared mass beyond that clone clears the same floor** ([FUSED-SHARED-SUBTREE-ECHO], applied inside a file against the file's own exact clones). Otherwise the clone already says everything the wider view would, and publishing the declarations only hands subsumption a byte-divergent container that eats the exact finding — `csharp-merge-readafter`'s two methods share one contiguous run and nothing else, and the run is the finding (`cross_cluster_collapse::widest_same_declaration_view_is_the_published_finding`). -Admitting every otherwise-valid same-file candidate was tried and reverted: within one file a class of sibling accessors, a table of rows and a set of already-extracted call sites all clear the structural floors, so the settings-getter family, the helper call sites and the `issue_190` data tables published or outranked real clones. Requiring both endpoints to be whole authored functions, or a Merkle-equal fragment inside both, refuses the tables and windows but not the sibling families. The route needs a discriminator that separates a copied method from a shape family; `docs/plans/same-file-rescue-plan.md` carries the candidates and the acceptance conditions, and the pin that ends the gap is `type3_enclosing_method::csharp_same_file_type3_reports_both_methods_in_one_cluster`, `#[ignore]`d with its assertions intact under gh #492. +Admitting *every* otherwise-valid same-file candidate was tried and reverted: within one file a class of sibling accessors, a table of rows and a set of already-extracted call sites all clear the structural floors, so the settings-getter family, the helper call sites and the `issue_190` data tables published or outranked real clones. Conditions 2 and 3 are what refuse them on evidence the pipeline already produces. Pinned by `type3_enclosing_method::csharp_same_file_type3_reports_both_methods_in_one_cluster`, `dart_issue_197_single_file_structural_only`, and `cross_cluster_collapse`. ### [FUSED-SHARED-SUBTREE-MEMO] Overlap is memoised by ordered Merkle hash pair @@ -125,7 +131,7 @@ An explicit pair comparison identifies both endpoints and may render that pair's 1. For each candidate pair, walk both occurrences' normalised subtrees and hash the **raw source bytes** of every collapsed leaf, keeping the leaf's population (identifier vs literal position). 2. Measure two independent populations for those exact two occurrences, both in `[0, 1]`: - `agreement` — fraction of all collapsed positions whose raw bytes match, identifiers and literals pooled. Byte-identical members score 1.0; lightly-edited copies stay high; framework-mandated scaffolding (every name differs) and data tables (every literal differs) fall low. A disagreement between behaviour-bearing operators is a hard content contradiction and makes `agreement = 0.0`: `+` and `-` compute different answers, so the surrounding matching positions cannot outvote the operation that changed. - - `rename_consistency` — the Type-2 discriminator, [TECH-PMATCH-BAKER] quantified: one pooled coverage over every constrained position of the pair — the identifier positions the bijection must explain plus every aligned literal position — scaled by the smooth anchor factor `anchors / (anchors + content_gate.rename_evidence_half_anchors)`. Preserved literals, literal echoes of a bijection-explained substitution ([REPAIR-RENAME-LITERAL-ECHO]), identity identifiers, and substitutions corroborated by repetition are explained; a drifted literal that echoes nothing and an inconsistent substitution are constrained positions the evidence cannot explain, so each weakens the proof in proportion to the evidence around it instead of vetoing it — one changed threshold inside an otherwise fully-anchored rename is a near-miss edit, not proof the copy is coincidence. A consistent substitution seen once is unconstrained and belongs to neither numerator nor denominator. Missing positional alignment or a behaviour-bearing operator disagreement makes the value zero. Certification removes the coincidence discount only when coverage is perfect and the configured anchor support is met. Every input and output belongs to this pair only. The frontier is positional: each collapsed literal position is judged on its own bytes, the fragments of an interpolated string included, so a preserved fragment is a preserved literal and the drifted fragment beside it is a drifted one. No group of positions is judged as a unit. + - `rename_consistency` — the Type-2 discriminator, [TECH-PMATCH-BAKER] quantified: one pooled coverage over every constrained position of the pair — the identifier positions the bijection must explain plus every aligned literal position — scaled by the smooth anchor factor `anchors / (anchors + content_gate.rename_evidence_half_anchors)`. Preserved literals, literal echoes of a bijection-explained substitution ([REPAIR-RENAME-LITERAL-ECHO]), identity identifiers, and substitutions corroborated by repetition are explained; a drifted literal that echoes nothing and an inconsistent substitution are constrained positions the evidence cannot explain, so each weakens the proof in proportion to the evidence around it instead of vetoing it — one changed threshold inside an otherwise fully-anchored rename is a near-miss edit, not proof the copy is coincidence. A consistent substitution seen once is unconstrained and belongs to neither numerator nor denominator. That prev-encoding reaches the literal alphabet too, but only where the pair claims no rename ([FUSED-CONTENT-GATE-PARAMETER]). Missing positional alignment or a behaviour-bearing operator disagreement makes the value zero. Certification removes the coincidence discount only when coverage is perfect and the configured anchor support is met. Every input and output belongs to this pair only. The frontier is positional: each collapsed literal position is judged on its own bytes, the fragments of an interpolated string included, so a preserved fragment is a preserved literal and the drifted fragment beside it is a drifted one. No group of positions is judged as a unit. 3. **Pair routing uses `support = max(agreement, rename_consistency)`** (either population may vouch; never their mean), and a pair whose identifier bijection is contradiction-free is admitted on that ground alone ([FUSED-CONTENT-GATE-RENAME]). Every pair uses `content_gate.support_floor` (0.70), across files and within one file alike: two methods copied inside one file measure no differently from two copied across files, and whether a same-file pair is a sibling family is answered after closure by [RANK-STRUCTURAL-ONLY-FORWARDING], which reads where each call goes — an admission floor cannot, because the REST settings family and a two-literal copy of one method both measure in the same 0.70–0.85 band. An unanchored LSH-only pair pays `content_gate.promote_floor` (0.85) instead: with no structural anchor, no embedding support, and no shared-subtree alignment, the token echo is the pair's whole case, and it must be corroborated more strongly before it may weld two views into one closure — at cross-file support strength this route admits whole-file-against-interior-window pairs and manufactures mixed-extent clusters (#339). A pair is unanchored only once its alignment has been *measured* and failed the shared-subtree overlap floor. A token-carried pair — no structural anchor, no embedding support, Jaccard at the LSH-only floor, the fused floor cleared on that echo alone — has its alignment measured by the rescue pass exactly as a rescue candidate does, and the content guard applies to it either way — at the support floor when the alignment clears the overlap floor, at the promote floor when it does not — because an unmeasured overlap reads as no alignment: it sent a near-identical run of two renamed Go functions to the promote floor, where a swapped literal scores 0.81, while the lopsided pair of one of those functions against the other file's whole run was rescued on measured overlap and admitted with no content check at all — and the cluster published one function against two (`cluster_extent_alignment`). The content guard applies when normalized shape or token evidence saturates, and also to an unanchored LSH-only pair that clears its own Jaccard floor without reaching shared-subtree rescue; no independent semantic route applies in either case. A gate-eligible pair below its content floor is not admitted. `E` is never relabelled as content support: a qualifying embedding route makes the shape-echo guard inapplicable rather than making `content_ok` true. This happens before transitive closure; no content score is stamped onto the resulting cluster. 4. **Token-signal correction.** A pair whose endpoints share one Merkle hash has equal normalised k-gram sets by construction; a lower `token_jaccard` is a fallback-signature artifact and is corrected to 1.0 for that pair only. @@ -135,6 +141,14 @@ The correction is scoped by that digest equality, tested directly on the members `token_jaccard` itself stays rename-invariant (normalised k-grams); the gate adds evidence rather than redefining an existing signal. +#### [FUSED-CONTENT-GATE-PARAMETER] Where nothing was renamed, a consistent literal substitution is a parameter + +A drifted literal that echoes nothing contradicts the *rename* the identifier bijection claims — that is what separates the `#134` stride family, renamed consistently end to end and diverging at one aligned literal, from a reportable Type-2 clone. + +Where the bijection claims no rename at all — no substituted identifier position is corroborated by repetition — there is no claim for a drifted literal to contradict, and [TECH-PMATCH-BAKER]'s prev-encoding applies to the literal alphabet exactly as it does to the identifier one: a substitution seen **once** is an unconstrained wildcard and leaves the coverage population. Two declarations whose every identifier position is byte-identical and whose literals each substitute once are one parameterised declaration, and those literals are its parameters. `csharp-merge-manyholes` keeps every identifier and every call and substitutes at all twelve literal positions, which is exactly what [AUTOFIX-MERGE-GATE] independently calls a clone too parameterised to merge mechanically; judging it on `agreement` alone judged a literal-only copy on the one axis its own edit demolishes. + +A **repeated** substitution is not a wildcard: it is a sibling family's own subject carried through its body, so it stays constrained and unexplained — the star-shadow fixture's `ApplyAlpha` says `"alpha"` three times against `"dup"`, and a sibling sharing a shape and no byte may not join the copy it sits beside. An inconsistent substitution stays constrained too: it contradicts the parameterisation as surely as it would a rename. Pinned by `same_file_rescue::a_literal_only_copy_inside_one_file_is_a_finding`, `same_file_rescue::a_shape_sibling_may_not_hide_an_exact_same_file_copy`, and `issue_134_structural_only_not_nearly_identical`. + **The token echo is shape evidence too.** The LSH pass hashes k-grams of the same normalised kinds as the structural pass, so high `token_jaccard` can echo shape rather than independently corroborating authored content. The content guard therefore applies to the pair whenever its normalised evidence saturates, or an unanchored LSH-only pair clears its Jaccard floor without qualifying for rescue, and no independent semantic route already vouches for it. **Shape mismatch changes the pair-content measurement, not its ownership.** When positional alignment is unavailable, agreement uses the key-set Jaccard of the two occurrences' content keys. A genuine Type-3 near-miss can retain high pair content despite one inserted statement, while renamed scaffolding shares few raw keys. Post-closure noise suppression may partition a connected component under [CLONE-NOISE-VERBATIM-SUBGROUP], but it may not invent, select, aggregate, or publish pair evidence for the cluster. @@ -213,7 +227,7 @@ $$ P_{\mathrm{candidate}}(s) = 1 - \left(1-s^r\right)^b $$ -A pair is admitted when its pre-rescue score clears its threshold or the shared-subtree rescue fires, subject to the size-coherence and LSH-only guards. The rescue is cross-file only and requires its own raw-content agreement; those conditions are load-bearing parts of the implementation, not optional prose. +A pair is admitted when its pre-rescue score clears its threshold or the shared-subtree rescue fires, subject to the size-coherence and LSH-only guards. The rescue requires its own raw-content agreement, and inside one file it additionally requires the copied interior of [FUSED-SHARED-SUBTREE-SAME-FILE]; those conditions are load-bearing parts of the implementation, not optional prose. $$ \begin{aligned}