From cfc869fae089319ca354933007e578e8a72af184 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 08:24:06 -0700 Subject: [PATCH 01/22] fix(py): give SARIF findings a deterministic order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(py): emit SARIF findings in the CLI's walk order `push_child_spaces` pushed a space's children onto the walk stack in source order while `collect_offenders` pops from the end, so every sibling set was visited in reverse, at every level. `bca.to_sarif` and `bca check -O sarif` therefore produced the same finding set in different sequences, against the byte-equivalence the module doc, the `_native.pyi` docstring and the book page all claim. Record the stack length before pushing and reverse only the tail this call appended, the way the CLI's `evaluate_with_policy` pushes `spaces.iter().rev()`. Reversing the appended tail rather than the input keeps the walk working for any Python iterable — `spaces` need not be a sequence — and allocates nothing extra. Emission order is now stated as part of the contract in all three places that claim parity, and pinned by two tests: a positional, unsorted comparison of the two documents on a fixture with three nested sibling pairs, and a hand-built-dict test covering the shapes the child walk tolerates but `analyze` never produces (a skipped non-dict child, a missing `spaces` key, an empty `spaces`). The existing parity helpers keep their sorts — they serve callers whose claim is set membership — but `_sarif_rows` is now a wrapper over a new unsorted extractor so the SARIF location traversal keeps one spelling. Fixes #1402 fix(py): order one space's SARIF findings by metric name Review of the sibling-order fix turned up the second half of the same divergence. The CLI builds its threshold entries by iterating a `BTreeMap`, so a space breaching several metrics reports them alphabetically by canonical name; the binding iterated the `thresholds` `PyDict` and so reported them in the caller's insertion order. `{"nargs": 1, "cyclomatic": 1}` came out `nargs` first against the CLI's `cyclomatic` first, and the same call spelled the other way round changed the document. Sort the resolved thresholds by name, which reproduces the CLI's order and drops the dependency on how the caller spelled the dict. Also qualify the parity claim where it was overstated: order *between* files is the caller's, since `to_sarif` follows the iterable it is handed while `bca check` follows its own resolved walk list. `_cli_check_sarif` now accepts a sequence of `--threshold` specs so the multi-metric shape is reachable from the harness at all. Fixes #1402 test(py): make the SARIF ordering tests able to fail An audit pass found three ways the new ordering tests could decay into passing against the unfixed binding, and one dead assertion. `test_to_sarif_orders_one_spaces_metrics_alphabetically` rested on its limits dict being spelled reverse-alphabetically, which nothing asserted — spelled the other way round it passes with the sort deleted. It also could not tell "alphabetical by metric name" from "emitted in `METRIC_FIELDS` declaration order", since `cyclomatic`/`nargs` sits the same way round under both; a binding iterating `METRIC_FIELDS` passed. It now loops over two pairs, adding `abc`/`cognitive`, which separates the two rules, and asserts the non-alphabetical spelling instead of describing it in a docstring. `test_to_sarif_child_order_survives_skipped_and_childless_spaces` named two tolerated shapes that are invisible in its asserted rows, so trimming the non-dict children or giving the childless space an empty `spaces` list left it green with nothing left to skip. Both are now asserted present. The `assert py_rows == cli_rows` in `test_to_sarif_emits_results_in_cli_walk_order` could not fail: the two lines above it already compare both operands against the same literal. Fixes #1402 Squashed from 9dc5ec69, e7f8357a, 40356732. --- big-code-analysis-book/src/python/sarif.md | 9 + .../python/big_code_analysis/_native.pyi | 8 + big-code-analysis-py/src/sarif.rs | 48 ++++ big-code-analysis-py/tests/test_sarif.py | 256 ++++++++++++++++-- 4 files changed, 301 insertions(+), 20 deletions(-) diff --git a/big-code-analysis-book/src/python/sarif.md b/big-code-analysis-book/src/python/sarif.md index 021db8f57..2290ded1b 100644 --- a/big-code-analysis-book/src/python/sarif.md +++ b/big-code-analysis-book/src/python/sarif.md @@ -10,6 +10,15 @@ writer that backs `bca check --report-format sarif`, so the schema URL, tool driver name / version, and rule descriptions match the CLI byte-for-byte. +Findings match in order as well as in content. Within a file, both +surfaces walk the space tree depth-first in source order — a space, then +its children left to right — and report a space's several breaches +alphabetically by metric name. For the same file and thresholds the two +`results` arrays therefore line up entry for entry, and a diff between +them is a real divergence rather than a walk-order artifact. Order +*between* files is the caller's: `to_sarif` follows the iterable you pass +it, while `bca check` follows the paths it resolved. + Examples on this page import the package as `bca` (`import big_code_analysis as bca`). A bare `bca` in a shell command is the CLI binary. diff --git a/big-code-analysis-py/python/big_code_analysis/_native.pyi b/big-code-analysis-py/python/big_code_analysis/_native.pyi index c5d8780b2..90ba5e80c 100644 --- a/big-code-analysis-py/python/big_code_analysis/_native.pyi +++ b/big-code-analysis-py/python/big_code_analysis/_native.pyi @@ -1005,6 +1005,14 @@ def to_sarif( emits) and the ``None``-name parse-failure case both collapse to ````, matching the CLI's ``space_segment``. + Findings are emitted in the CLI's order as well as its shape + (#1402): depth-first through the space tree in source order, a + space before its children and siblings left to right, and within + one space, alphabetically by metric name — so ``results`` for a + given file is comparable positionally, not just as a set. Order + *between* files is whatever the iterable passed as ``result`` + yields, where ``bca check`` follows its own resolved walk list. + Raises ------ TypeError diff --git a/big-code-analysis-py/src/sarif.rs b/big-code-analysis-py/src/sarif.rs index 771c10427..dfd078f3a 100644 --- a/big-code-analysis-py/src/sarif.rs +++ b/big-code-analysis-py/src/sarif.rs @@ -62,6 +62,32 @@ //! metric at the own-value field and the binding emits at every space — //! no leaf-only special-casing remains. //! +//! # Emission order +//! +//! Order is part of the parity contract, not just the finding set: for +//! one input file the two front-ends emit the same results in the same +//! sequence, so a consumer may diff the documents positionally. Two +//! axes decide that sequence, and #1402 fixed both: +//! +//! * **Across spaces** — both walk the space tree depth-first in source +//! order, a space then its children left to right, which for a LIFO +//! stack means pushing each sibling set reversed (see +//! [`push_child_spaces`]). This binding pushed them in source order +//! and so reported every sibling set backwards. +//! * **Within one space** — the CLI's `ThresholdSet` iterates a +//! `BTreeMap`, so a space breaching several metrics reports them +//! alphabetically by canonical name; iterating the `thresholds` +//! `PyDict` yielded the caller's insertion order instead. +//! [`resolve_thresholds`] now sorts to match. +//! +//! Both bugs left the finding *set* correct and only the sequence +//! wrong, which is why the sorted parity helpers in +//! `tests/test_sarif.py` never saw either. +//! +//! Order *across files* is the caller's: [`collect_offenders_from_iter`] +//! follows the iterable it is handed, where `bca check` follows its own +//! resolved walk list. +//! //! `nargs` is the fifth and arrived by the opposite route: its serialized //! shape did not change, its *gate* did. #1196 moved the CLI extractor //! from `total()` to the callable's own parameter list, leaving this @@ -337,6 +363,16 @@ fn resolve_thresholds(thresholds: Option<&Bound<'_, PyDict>>) -> PyResult` + // (`ThresholdSet::build_tiered`), so a space breaching several metrics + // reports them alphabetically by canonical name. Iterating a `PyDict` + // yields the caller's insertion order instead, which made a two-metric + // `to_sarif` disagree with `bca check` on the order of one space's + // findings — and made the binding's own output depend on how the + // caller happened to spell the dict. Sorting here reproduces the + // CLI's order and drops that dependency (#1402). Names are unique, + // so the sort needs no tiebreak. + out.sort_unstable_by_key(|t| t.name); Ok(out) } @@ -555,12 +591,23 @@ fn record_threshold_breaches( /// `child_prefix` as the parent qualified prefix. Non-dict children are /// skipped (mirroring the original inline tolerance). Factored out of /// [`collect_offenders`]. +/// +/// Children land on the stack in **reverse** source order so the caller's +/// `pop()` visits them in source order, matching the CLI's +/// `evaluate_with_policy` (which pushes `spaces.iter().rev()` for the same +/// reason) — emission order is part of the SARIF parity contract (#1402). +/// Reversing the tail this call appended, rather than the input, keeps the +/// walk working for any Python iterable — `spaces` need not be a sequence — +/// and allocates nothing extra. fn push_child_spaces<'py>( space: &Bound<'py, PyDict>, child_prefix: &str, stack: &mut Vec<(Bound<'py, PyDict>, String)>, ) -> PyResult<()> { let py = space.py(); + // Captured before any push and nothing pops here, so the tail slice + // below is always in range. + let children_start = stack.len(); if let Some(spaces) = space.get_item(intern!(py, "spaces"))? && let Ok(seq) = spaces.try_iter() { @@ -571,6 +618,7 @@ fn push_child_spaces<'py>( } } } + stack[children_start..].reverse(); Ok(()) } diff --git a/big-code-analysis-py/tests/test_sarif.py b/big-code-analysis-py/tests/test_sarif.py index 2d8a0a7ce..67a5fd3fc 100644 --- a/big-code-analysis-py/tests/test_sarif.py +++ b/big-code-analysis-py/tests/test_sarif.py @@ -21,6 +21,7 @@ import json import subprocess +from collections.abc import Sequence from pathlib import Path from typing import Any, cast @@ -41,8 +42,14 @@ # the actual deduplication the conftest hoist was meant to deliver. -def _cli_check_sarif(bca_path: str, path: Path, *, threshold: str) -> dict[str, Any]: - """Run ``bca check --threshold X -O sarif --paths ``. +def _cli_check_sarif( + bca_path: str, path: Path, *, threshold: str | Sequence[str] +) -> dict[str, Any]: + """Run ``bca check --threshold X … -O sarif --paths ``. + + ``threshold`` takes one ``"metric=limit"`` string or a sequence of + them; each becomes its own ``--threshold`` flag, the way the CLI + accepts several limits in one run. The CLI writes a one-line offender summary to stderr and the SARIF document to stdout; we want the JSON, so parse stdout. @@ -51,6 +58,7 @@ def _cli_check_sarif(bca_path: str, path: Path, *, threshold: str) -> dict[str, regression" from "tool crashed") — `check=False` keeps subprocess from raising on it. """ + specs = [threshold] if isinstance(threshold, str) else list(threshold) argv = [ bca_path, "check", @@ -61,8 +69,7 @@ def _cli_check_sarif(bca_path: str, path: Path, *, threshold: str) -> dict[str, # tests assert exact finding sets, so a future manifest edit # would surface as a binding divergence that is not one. "--no-config", - "--threshold", - threshold, + *(arg for spec in specs for arg in ("--threshold", spec)), "-O", "sarif", "--paths", @@ -683,29 +690,34 @@ def _qualified_names(doc: dict[str, Any]) -> set[str]: ) -def _sarif_rows(results: list[dict[str, Any]]) -> list[tuple[int, str, str]]: +def _sarif_rows_in_emission_order(results: list[dict[str, Any]]) -> list[tuple[int, str, str]]: """Reduce SARIF results to ``(startLine, fullyQualifiedName, message)`` - triples in a deterministic order, so a test can compare them against a - hand-written sequence instead of two structurally-equal containers. - - The sort is a normalisation of the *observed* value, which is normally - the wrong side to normalise — but emission order genuinely differs - between the two front-ends and is not what these tests are about. The - binding walks the space tree with an explicit LIFO stack, so it emits - sibling spaces in reverse relative to ``bca check``: on the fixture - below it reports ``outer``, ````, ```` where the CLI - reports ``outer``, ````, ````. ``_assert_sarif_results_match`` - sorts for the same reason. Ordering therefore has to be pinned - somewhere else if it is ever made part of the parity contract. + triples **without reordering them**, so a test can assert what the + document emitted and in which sequence. The first two fields come from ``_sarif_sort_key`` rather than being read out again here, so the SARIF location traversal has one spelling in this file and a shape change cannot leave the row extractor raising ``KeyError`` while the sort key still resolves. """ - return [ - (*_sarif_sort_key(r), r["message"]["text"]) for r in sorted(results, key=_sarif_sort_key) - ] + return [(*_sarif_sort_key(r), r["message"]["text"]) for r in results] + + +def _sarif_rows(results: list[dict[str, Any]]) -> list[tuple[int, str, str]]: + """``_sarif_rows_in_emission_order`` over results sorted by + ``_sarif_sort_key``, so a test whose subject is *which* spaces breach + can compare against a hand-written sequence without also restating the + walk order. + + The sort is a normalisation of the *observed* value, which is normally + the wrong side to normalise. It is kept deliberately for the callers + whose claim is set membership: it keeps a future walk-order change + from failing every parity test at once, the way a metric change would. + Emission order is a real part of the contract (#1402) and is pinned by + ``test_to_sarif_emits_results_in_cli_walk_order``, which compares + positionally through ``_sarif_rows_in_emission_order``. + """ + return _sarif_rows_in_emission_order(sorted(results, key=_sarif_sort_key)) def test_to_sarif_matches_cli_check_for_nargs_own_parameter_list( @@ -784,6 +796,157 @@ def test_to_sarif_matches_cli_check_for_nargs_own_parameter_list( assert cli_rows == want, f"CLI at nargs={limit}: {cli_rows!r}" +def _sibling_set_sizes(space: FuncSpaceDict) -> list[int]: + """Every sibling-set size in ``space``'s subtree, its own child list + first. Lets an ordering test state the shape it needs, so a trimmed + fixture fails on the missing sibling pairs rather than on a row + mismatch the reader has to diagnose.""" + children = space["spaces"] + return [len(children), *(n for child in children for n in _sibling_set_sizes(child))] + + +def test_to_sarif_emits_results_in_cli_walk_order(bca_binary: str, tmp_path: Path) -> None: + """Emission order is part of the parity contract, not only the finding + set (#1402). + + Both front-ends walk the space tree depth-first in source order — a + space, then its children left to right — so the two ``results`` arrays + are comparable **positionally**. The binding walks with an explicit + LIFO stack and, before the fix, pushed each sibling set in source + order, so every sibling set popped reversed at every level: on this + fixture it emitted ``second``, ``first``, ````, + ``::``, ``::``, ````. + The set was right and only the sequence was wrong, which is why + ``_sarif_rows`` and ``_assert_sarif_results_match`` — both of which + sort — could not see it. + + The fixture nests three sibling sets of two so a single-level fix, or + a fix that reversed the whole result list instead of each sibling set, + still fails. + """ + src = tmp_path / "nested_closures.rs" + src.write_text( + "fn first(a: i32, b: i32) -> i32 {\n" + " let inner_one = |x: i32, y: i32| x + y;\n" + " let inner_two = |p: i32, q: i32| {\n" + " let deep_a = |m: i32, n: i32| m * n;\n" + " let deep_b = |s: i32, t: i32| s - t;\n" + " deep_a(p, q) + deep_b(p, q)\n" + " };\n" + " inner_one(a, b) + inner_two(a, b)\n" + "}\n" + "\n" + "fn second(c: i32, d: i32) -> i32 {\n" + " c * d\n" + "}\n" + ) + + analyzed = bca.analyze(src) + assert analyzed is not None, "fixture must not be skipped" + # Fixture adequacy: at least three sibling sets must hold two children + # each, or a reversal has nothing to reverse and this test cannot fail. + multi = [n for n in _sibling_set_sizes(analyzed) if n >= 2] + assert len(multi) >= 3, f"fixture must keep its nested sibling pairs; sizes {multi!r}" + + py_results = _parse(bca.to_sarif(analyzed, thresholds={"nargs": 1}))["runs"][0]["results"] + cli_results = _cli_check_sarif(bca_binary, src, threshold="nargs=1")["runs"][0]["results"] + + message = "nargs 2 exceeds limit 1" + expected = [ + (1, "first", message), + (2, "first::", message), + (3, "first::", message), + (4, "first::::", message), + (5, "first::::", message), + (11, "second", message), + ] + py_rows = _sarif_rows_in_emission_order(py_results) + cli_rows = _sarif_rows_in_emission_order(cli_results) + # Pin the CLI first: it is the reference this contract is defined + # against, so a walk-order change there is a finding of its own rather + # than something the binding should silently follow. + assert cli_rows == expected, f"CLI reference walk order moved: {cli_rows!r}" + assert py_rows == expected, f"binding must emit in the CLI's order: {py_rows!r}" + + +def test_to_sarif_orders_one_spaces_metrics_alphabetically(bca_binary: str, tmp_path: Path) -> None: + """The second ordering axis (#1402): within a single space, several + breaches come out alphabetically by metric name on both sides. + + The CLI builds its threshold entries from a ``BTreeMap``, so it + reports ``cyclomatic`` before ``nargs``. The binding iterated the + ``thresholds`` dict, which yields Python insertion order — so + ``{"nargs": 1, "cyclomatic": 1}`` came out ``nargs`` first and the + same call spelled the other way round came out ``cyclomatic`` first. + The finding set was identical either way, so only a positional + comparison sees it. + + The fixture is one function breaching both metrics, so the space walk + fixed above cannot supply the ordering: every row here shares a line + and a symbol, and only the metric name separates them. + + Two properties of the limits dict carry this test, and both are + asserted rather than left to the reader: + + * Each dict is spelled **reverse-alphabetically**; spelled the other + way round it would pass against the unsorted code, so the loop + pins the spelling and derives the opposite one instead of writing + it out. + * ``abc``/``cognitive`` separates "sorted by name" — the CLI's + ``BTreeMap`` order — from "emitted in ``METRIC_FIELDS`` declaration + order", which lists ``cognitive`` first. ``cyclomatic``/``nargs`` + sits the same way round under both rules and so cannot: a binding + that iterated ``METRIC_FIELDS`` instead of sorting passed the + single-pair version of this test. + """ + src = tmp_path / "two_metrics.rs" + src.write_text("fn outer(a: i32, b: i32) -> i32 {\n if a > b { 1 } else { 2 }\n}\n") + + analyzed = bca.analyze(src) + assert analyzed is not None, "fixture must not be skipped" + + cases: tuple[tuple[dict[str, float], list[tuple[int, str, str]]], ...] = ( + ( + {"nargs": 1, "cyclomatic": 1}, + [ + (1, "outer", "cyclomatic 2 exceeds limit 1"), + (1, "outer", "nargs 2 exceeds limit 1"), + ], + ), + ( + {"cognitive": 0, "abc": 0}, + [ + (1, "outer", "abc 2 exceeds limit 0"), + (1, "outer", "cognitive 2 exceeds limit 0"), + ], + ), + ) + for limits, expected in cases: + assert list(limits) != sorted(limits), ( + f"{limits!r} must be spelled non-alphabetically, or an unsorted binding passes" + ) + specs = tuple(f"{name}={limit}" for name, limit in limits.items()) + py_rows = _sarif_rows_in_emission_order( + _parse(bca.to_sarif(analyzed, thresholds=limits))["runs"][0]["results"] + ) + cli_rows = _sarif_rows_in_emission_order( + _cli_check_sarif(bca_binary, src, threshold=specs)["runs"][0]["results"] + ) + assert cli_rows == expected, f"CLI reference metric order moved: {cli_rows!r}" + assert py_rows == expected, f"binding must match the CLI's metric order: {py_rows!r}" + + # Spelling the same limits the other way round must not change the + # document — the pre-fix binding's output tracked the caller's dict. + reversed_spelling = _sarif_rows_in_emission_order( + _parse(bca.to_sarif(analyzed, thresholds=dict(reversed(limits.items()))))["runs"][0][ + "results" + ] + ) + assert reversed_spelling == expected, ( + f"output must not depend on thresholds dict order: {reversed_spelling!r}" + ) + + def test_to_sarif_anonymous_space_collapses_to_anon_line() -> None: """A space whose name is the literal ```` (every grammar's closure/lambda sentinel) collapses to ````, @@ -934,6 +1097,59 @@ def test_to_sarif_treats_unit_kind_case_insensitively() -> None: ) +def test_to_sarif_child_order_survives_skipped_and_childless_spaces() -> None: + """The source-order emission contract (#1402) holds across the shapes + the child walk tolerates rather than rejects. + + ``analyze`` never produces these, so they are only reachable from a + hand-built dict: children the walk skips (a non-dict entry), a child + carrying no ``spaces`` key at all, and a child whose ``spaces`` is + empty. The fix reverses the stack tail the child loop appended, so a + skipped entry must not leave a gap that scrambles the surviving + siblings, and a childless space must not reverse its parent's tail a + second time. + """ + childless: dict[str, Any] = { + "name": "beta", + "kind": "function", + "start_line": 20, + "end_line": 25, + # No "spaces" key at all — the `get_item` miss branch. + "metrics": {"cyclomatic": {"value": 5.0, "sum": 5.0}}, + } + root: dict[str, Any] = { + "name": "synthetic.py", + "kind": "unit", + "start_line": 1, + "end_line": 40, + "spaces": [ + _fake_function_dict(name="alpha", start_line=10, end_line=15), + "not a space", + childless, + 42, + _fake_function_dict(name="gamma", start_line=30, end_line=35), + ], + "metrics": {"cyclomatic": {"value": 1.0, "sum": 11.0}}, + } + + # Fixture adequacy: both tolerated shapes must still be present. Neither + # is visible in the rows below — trimming the non-dict entries, or giving + # ``beta`` an empty ``spaces`` list, leaves this test green with nothing + # left to skip and no ``get_item`` miss to take. + assert "spaces" not in childless, "childless space must take the get_item miss branch" + skipped = [child for child in root["spaces"] if not isinstance(child, dict)] + assert len(skipped) == 2, f"fixture must keep its skipped non-dict children; got {skipped!r}" + + parsed = _parse(bca.to_sarif(cast("FuncSpaceDict", root), thresholds={"cyclomatic": 1})) + rows = _sarif_rows_in_emission_order(parsed["runs"][0]["results"]) + message = "cyclomatic 5 exceeds limit 1" + assert rows == [ + (10, "alpha", message), + (20, "beta", message), + (30, "gamma", message), + ], f"surviving children must stay in source order: {rows!r}" + + def test_to_sarif_rejects_mappingproxytype_with_clear_error() -> None: """``types.MappingProxyType`` is a Mapping but not a dict, so the inner ``cast_into::()`` fails. Without the explicit From 666b176230fe7813eb7a13a290a4a4f2e265065a Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 08:24:06 -0700 Subject: [PATCH 02/22] fix(vcs/cache): invalidate the history cache on mailmap change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(vcs/cache): invalidate the history cache on mailmap change Author identities are canonicalised through the repository `.mailmap` at walk time and stored in the event log as digests, but neither the entry key (`head_sha`) nor `cache::fingerprint` observed the mailmap. An edit therefore served stale author statistics — and the incremental splice re-persisted the pre-edit digests under each new head, so the divergence survived HEAD moving until `--clear-cache`. Fold a digest of the repository's effective mailmap into `cache::fingerprint`. That covers all three cache paths at once (the pure hit, `load_compatible`'s ancestor selection for the splice, and the persisted entry) and needs no `CACHE_SCHEMA_VERSION` bump: every pre-fix entry simply fingerprints differently and costs one cold walk. `repo::mailmap_digest` hashes gix's *merged snapshot* rather than the raw source bytes. Re-deriving the four conditional sources `open_mailmap` consults would be a coverage claim nothing checks, and a source missed — or added by a future gix release, the dependency being caret-ranged — silently reproduces this bug for that source. Also document the sibling blind spot the issue flags: the fingerprint hashes an option's *value*, never the behaviour it selects, so a change to how `BotFilter` matches its pattern belongs to `CACHE_SCHEMA_VERSION` instead (as #1265 already did). Fixes #1262 test(vcs/cache): pin the mailmap digest's design claims Review and test-audit findings on the #1262 fix. The regression tests covered the bug; these cover the reasoning the fix rests on. - `mailmap.file` invalidation. The digest hashes gix's merged snapshot so a source cannot be missed, but every test wrote the working-tree `.mailmap` — the one source a naive byte digest also covers. Verified: re-implementing the digest as `fs::read` of the work-tree file left the whole suite green before this test, and fails it now. - A comment-only `.mailmap` edit still hits. This is the sole behavioural difference from a byte digest, and was asserted nowhere. - The splice test's "the persisted entry replays correctly" step could not tell a hit from a second cold walk — writing a mismatched fingerprint in `persist` left it green. It now empties the entries and proves the replay was served. - Cross-process fingerprint stability had no guard at all: the CLI's two-process test compared two runs' stdout, which matches whether or not the second hits. Mixing the pid into `fingerprint` now fails it. Pre-existing, but the mailmap term is a new way to break it. Also record the residual window the review found (#1409): the digest and the walk open the mailmap separately, so an edit landing between them mis-stamps an entry. Self-healing unless the mailmap is reverted before the next run; closing it means threading one snapshot through the walk, which is a larger change than this fix. Squashed from 702b5aca, 206cb834. --- big-code-analysis-cli/tests/vcs/vcs_rank.rs | 47 ++++ src/vcs/cache.rs | 50 ++++- src/vcs/cache_tests.rs | 44 +++- src/vcs/git/cached.rs | 11 +- src/vcs/git/repo.rs | 55 +++++ tests/vcs/vcs_cache.rs | 236 ++++++++++++++++++++ 6 files changed, 428 insertions(+), 15 deletions(-) diff --git a/big-code-analysis-cli/tests/vcs/vcs_rank.rs b/big-code-analysis-cli/tests/vcs/vcs_rank.rs index 67fae7c9a..6520f45fe 100644 --- a/big-code-analysis-cli/tests/vcs/vcs_rank.rs +++ b/big-code-analysis-cli/tests/vcs/vcs_rank.rs @@ -210,6 +210,53 @@ fn vcs_cache_dir_persists_and_replays_identically() { first, second, "a cache hit is byte-identical to the first run" ); + + // Equality alone cannot tell a hit from a second cold walk — both + // produce the first run's bytes. Emptying the persisted event log + // separates them: a *served* entry now replays zero commits, while a + // walk would reproduce `first`. This is the only guard in the suite on + // the fingerprint being stable **across processes**, which every + // persisted entry depends on and which an in-process test cannot see + // (the mailmap digest #1262 folded in is hashed from gix-owned values, + // so it is a new way to break that old property). + empty_cache_entry_events(cache.path()); + let third = run(); + assert_ne!( + first, third, + "the emptied entry was not served, so the second process computed a \ + different fingerprint and cold-walked instead of hitting" + ); +} + +/// Rewrite every persisted `*.json` cache entry under `root` with an empty +/// `events` array, leaving it valid JSON so it still loads as a hit +/// candidate. Mirrors the library suite's helper of the same shape. +fn empty_cache_entry_events(root: &Path) { + let mut stack = vec![root.to_path_buf()]; + let mut emptied = 0_usize; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|ext| ext == "json") { + let bytes = std::fs::read(&path).expect("read entry"); + let mut value: serde_json::Value = + serde_json::from_slice(&bytes).expect("entry is valid JSON"); + value["events"] = serde_json::Value::Array(Vec::new()); + std::fs::write(&path, serde_json::to_vec(&value).expect("reserialize")) + .expect("rewrite entry"); + emptied += 1; + } + } + } + assert!( + emptied > 0, + "no cache entry was emptied, so the run that follows proves nothing" + ); } #[test] diff --git a/src/vcs/cache.rs b/src/vcs/cache.rs index ca199330d..f281cd661 100644 --- a/src/vcs/cache.rs +++ b/src/vcs/cache.rs @@ -45,11 +45,14 @@ //! //! A cached entry is honoured only when its [`CACHE_SCHEMA_VERSION`], //! [`VCS_SCHEMA_VERSION`], [`RISK_SCORE_VERSION`], the `fingerprint` of -//! the walk-affecting options, and the shallow state under which it was -//! walked all match the current run; otherwise it is ignored and the -//! history recomputed. Window changes alter the fingerprint, so they -//! force a fresh walk, as the issue specifies. A shallow clone that is -//! later deepened (`git fetch --unshallow`) leaves `HEAD` unmoved, so the +//! the walk-affecting options *and the repository mailmap*, and the +//! shallow state under which it was walked all match the current run; +//! otherwise it is ignored and the history recomputed. Window changes +//! alter the fingerprint, so they force a fresh walk, as the issue +//! specifies. A `.mailmap` edit does too: it changes the author identities +//! resolved into the event log while leaving `HEAD` — and every option — +//! untouched (issue #1262). A shallow clone that is later deepened +//! (`git fetch --unshallow`) leaves `HEAD` unmoved, so the //! entry key is unchanged; the shallow-state match in //! `HistoryCache::is_compatible` is what forces a re-walk to replace the //! truncated counts (issue #810). A corrupt or unreadable entry is @@ -189,7 +192,9 @@ pub(crate) struct HistoryCache { pub vcs_schema_version: u32, /// Composite-formula version the events were produced under. pub risk_score_version: u32, - /// [`fingerprint`] of the walk-affecting options. + /// [`fingerprint`] of the walk-affecting options and the repository + /// mailmap. The field name predates the mailmap term (#1262) and is + /// kept because renaming it would change the serialized shape. pub options_fingerprint: u64, /// The `HEAD` (or `--ref`) object id the walk reached, hex-encoded. pub head_sha: String, @@ -228,10 +233,32 @@ impl HistoryCache { } } -/// A stable 64-bit fingerprint of every option that changes *which -/// commits the walk visits or how they are recorded* — the window -/// lengths, traversal mode, merge/rename/bot toggles, the bot pattern, -/// and the `--as-of` reference time. +/// A stable 64-bit fingerprint of everything that changes *which commits +/// the walk visits or how they are recorded* — the window lengths, +/// traversal mode, merge/rename/bot toggles, the bot pattern, and the +/// `--as-of` reference time from [`Options`], plus `mailmap_digest`. +/// +/// # Walk input that lives outside `Options` +/// +/// `mailmap_digest` (issue #1262) covers the repository `.mailmap`, which +/// is walk input the way an option is: author identities are canonicalised +/// through it *at walk time* and stored in the event log as digests, so a +/// mailmap edit changes what a replay would produce while moving neither +/// `HEAD` (the entry key) nor any field of `Options`. Left out, an edited +/// mailmap served stale author counts indefinitely — the incremental +/// splice re-persisted the pre-edit digests under each new head, so the +/// divergence outlived `HEAD` moving. The backend's `repo::mailmap_digest` +/// documents what the value covers and why it digests gix's merged +/// snapshot rather than the raw source bytes. +/// +/// Note what this term does *not* generalise to. An input is fingerprinted +/// only when it is hashed here, and what is hashed is the option's +/// *value*, never the behaviour it selects: `bot_pattern` is the pattern +/// string, so a change to how [`BotFilter`](super::identity::BotFilter) +/// *matches* that string leaves every fingerprint untouched. Such a change +/// belongs to [`CACHE_SCHEMA_VERSION`], which versions the meaning of a +/// recorded event — see its docs for the case-sensitivity bump (#1265) +/// that set the precedent. /// /// Finalization-only knobs (`--risk-formula`, `--emit-author-details`, /// `--author-hash-key` (#956), `--include-deleted`, the bus-factor options) @@ -255,8 +282,9 @@ impl HistoryCache { /// inputs, so the fingerprint need only be self-consistent within one /// format version. #[must_use] -pub(crate) fn fingerprint(options: &Options) -> u64 { +pub(crate) fn fingerprint(options: &Options, mailmap_digest: u64) -> u64 { let mut hasher = DefaultHasher::new(); + mailmap_digest.hash(&mut hasher); options.long_window_secs.hash(&mut hasher); options.recent_window_secs.hash(&mut hasher); options.full_history.hash(&mut hasher); diff --git a/src/vcs/cache_tests.rs b/src/vcs/cache_tests.rs index 8827ac627..0cc7e7a7e 100644 --- a/src/vcs/cache_tests.rs +++ b/src/vcs/cache_tests.rs @@ -2,6 +2,13 @@ use super::*; use crate::vcs::options::{Options, RiskFormula}; use std::path::PathBuf; +/// A stand-in mailmap digest, deliberately not `0`, so an assertion here +/// cannot pass because two default values coincide +/// (`.claude/rules/testing.md`). No real digest is computed in this +/// module — `repo::mailmap_digest` needs a repository, so it is covered by +/// the `vcs_cache` integration tests. +const SAMPLE_MAILMAP_DIGEST: u64 = 0xfeed_face_dead_beef; + fn sample_event(oid: &str, time: i64) -> CommitEvent { CommitEvent { oid: oid.to_owned(), @@ -59,10 +66,41 @@ fn fingerprint_ignores_finalization_only_knobs() { }, ]; for case in cases { - assert_eq!(fingerprint(&base), fingerprint(&case)); + assert_eq!( + fingerprint(&base, SAMPLE_MAILMAP_DIGEST), + fingerprint(&case, SAMPLE_MAILMAP_DIGEST) + ); } } +#[test] +fn fingerprint_changes_with_the_mailmap_digest() { + // The `.mailmap` is walk input that lives outside `Options`: author + // identities are canonicalised through it at walk time and recorded in + // the event log as digests, while an edit moves neither `HEAD` (the + // entry key) nor any option. Without this term a stale event log is + // replayed — and re-persisted by the incremental splice — under a + // mailmap it was not walked with (issue #1262). + let options = Options::default(); + let base = fingerprint(&options, SAMPLE_MAILMAP_DIGEST); + assert_ne!( + base, + fingerprint(&options, SAMPLE_MAILMAP_DIGEST ^ 1), + "a mailmap change must change the fingerprint" + ); + // Both operands differ from `0` and from each other, so the inequality + // above cannot hold for an incidental reason. The equality below is + // determinism *within one process* only — the cross-process stability + // a persisted entry actually depends on cannot be observed from here, + // and is guarded by `vcs_cache_dir_persists_and_replays_identically` + // in the CLI suite, which primes and reads in two separate processes. + assert_eq!( + base, + fingerprint(&options, SAMPLE_MAILMAP_DIGEST), + "identical inputs fingerprint identically" + ); +} + #[test] fn fingerprint_changes_with_walk_affecting_knobs() { let base = Options::default(); @@ -102,8 +140,8 @@ fn fingerprint_changes_with_walk_affecting_knobs() { ]; for case in cases { assert_ne!( - fingerprint(&base), - fingerprint(&case), + fingerprint(&base, SAMPLE_MAILMAP_DIGEST), + fingerprint(&case, SAMPLE_MAILMAP_DIGEST), "a walk-affecting option must change the fingerprint" ); } diff --git a/src/vcs/git/cached.rs b/src/vcs/git/cached.rs index 4afbfb424..1a2724b0f 100644 --- a/src/vcs/git/cached.rs +++ b/src/vcs/git/cached.rs @@ -15,6 +15,15 @@ //! Every path ends in the same [`replay`](crate::vcs::replay), so a cache //! hit is bit-identical to a fresh walk, and re-windowing tracks the //! current `now` rather than freezing at cache-write time. +//! +//! That contract holds only while every input the walk *records* is +//! covered by the entry key or the fingerprint. The repository `.mailmap` +//! is such an input and is neither: author identities are canonicalised +//! through it at walk time and stored as digests, while an edit to it +//! moves no `HEAD` (issue #1262). [`repo::mailmap_digest`] therefore +//! enters [`cache::fingerprint`] here, which covers all three cache paths +//! at once — the pure hit, `load_compatible`'s ancestor selection for the +//! incremental splice, and the entry [`persist`] writes back. use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; @@ -101,7 +110,7 @@ pub(crate) fn build_cached( return Ok(assemble(seed, &events, options, now, workdir, shallow)); }; - let fingerprint = cache::fingerprint(options); + let fingerprint = cache::fingerprint(options, repo::mailmap_digest(&repo)); let long_boundary = window_boundary(now, options.long_window_secs); // Pure hit: an exact, compatible entry that reaches back far enough. diff --git a/src/vcs/git/repo.rs b/src/vcs/git/repo.rs index 9bc53e76b..15bf3f110 100644 --- a/src/vcs/git/repo.rs +++ b/src/vcs/git/repo.rs @@ -1,6 +1,8 @@ //! Repository discovery and target-tree file enumeration. use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::ops::ControlFlow; use std::path::{Path, PathBuf}; @@ -41,6 +43,59 @@ pub(crate) fn open(root: &Path) -> Result { }) } +/// A stable 64-bit digest of the repository's **effective mailmap** — the +/// merged snapshot [`gix::Repository::open_mailmap`] produces, which is +/// exactly what the walk resolves author identities against +/// ([`history::collect_events`](super::history::collect_events)). +/// +/// Author identities are canonicalised at *walk* time and stored in the +/// cache as digests, so a mailmap edit changes the recorded events without +/// moving `HEAD` or any field of [`Options`](crate::vcs::Options). Feeding +/// this value into [`cache::fingerprint`](crate::vcs::cache::fingerprint) +/// is what stops a stale event log being replayed — or, worse, spliced +/// into and re-persisted — under a mailmap it was not walked with +/// (issue #1262). +/// +/// # Why the parsed snapshot rather than the source bytes +/// +/// `open_mailmap` merges up to four sources whose selection is +/// conditional: the working-tree `.mailmap` (uncommitted edits included), +/// `HEAD:.mailmap` for a bare repo *when `mailmap.blob` is unset*, the +/// `mailmap.blob` object, and the `mailmap.file` path. Re-deriving that +/// list here would be a coverage claim nothing checks — a source missed +/// (or one added by a future gix release, the dependency being +/// caret-ranged) silently reproduces this bug for that source. Digesting +/// the merged snapshot instead delegates the source list to gix, so it +/// cannot drift. +/// +/// `Snapshot::iter` is documented as ordered by `(old_email, old_name)`, +/// so the digest is deterministic across processes, and `DefaultHasher` is +/// created with fixed keys for the same cross-process stability +/// [`cache::fingerprint`](crate::vcs::cache::fingerprint) relies on. That +/// documented order is why this hashes `entries()` rather than the +/// `Snapshot` itself, which also derives `Hash`: the derive would rest on +/// gix's private field layout, where `iter`'s ordering is part of its +/// contract. The `Vec` is a mailmap's worth of borrowed slices. +/// +/// Digesting the parsed entries also means a comment- or whitespace-only +/// `.mailmap` edit does not cost a needless cold walk: the resolution it +/// feeds is unchanged, so replaying is correct. +/// +/// # Residual window +/// +/// The walk re-opens the mailmap for itself, so an edit landing between +/// this call and that one stamps the entry with the pre-edit fingerprint +/// over post-edit events. The next run under the edited mailmap +/// fingerprints differently and heals it; only reverting the mailmap +/// before any such run leaves a wrong hit reachable. Closing the window +/// means threading one snapshot through the walk — see issue #1409. +#[must_use] +pub(crate) fn mailmap_digest(repo: &gix::Repository) -> u64 { + let mut hasher = DefaultHasher::new(); + repo.open_mailmap().entries().hash(&mut hasher); + hasher.finish() +} + /// Discover the working-tree root of the repository containing `path`. /// /// Returns the canonicalised work-tree directory (the same value diff --git a/tests/vcs/vcs_cache.rs b/tests/vcs/vcs_cache.rs index c5bbc7197..77a897b2c 100644 --- a/tests/vcs/vcs_cache.rs +++ b/tests/vcs/vcs_cache.rs @@ -133,6 +133,242 @@ fn build_repo() -> Repo { repo } +/// A repo whose single file is edited by two distinct identities, so a +/// `.mailmap` that folds one into the other is observable as +/// `authors_long` 2 → 1 (and in `ownership_top_share`, the bus factor, and +/// the distinct-author term of `risk_score`). +fn build_two_author_repo() -> Repo { + let repo = Repo::init(); + repo.write("a.rs", "fn a() {}\n"); + repo.commit( + "Ada", + "ada@example.com", + FIXED_NOW - 30 * DAY, + "feat: add a", + ); + repo.write("a.rs", "fn a() { work(); }\n"); + repo.commit( + "Grace", + "grace@example.com", + FIXED_NOW - 10 * DAY, + "fix: a crash", + ); + repo +} + +/// The `.mailmap` line that folds Grace's identity into Ada's. +const FOLD_GRACE_INTO_ADA: &str = "Ada Grace \n"; + +/// One file's distinct-author count over the long window. +fn authors_long(index: &vcs::HistoryIndex, path: &str) -> u32 { + index + .iter() + .find(|(candidate, _)| candidate.to_string_lossy() == path) + .map(|(_, stats)| stats.authors_long) + .expect("file is ranked") +} + +#[test] +fn a_working_tree_mailmap_edit_is_not_served_from_a_stale_entry() { + // Issue #1262, first repro: identities are canonicalised through the + // mailmap at *walk* time and stored as digests, so an edit changes what + // a replay should produce while moving neither `HEAD` (the entry key) + // nor any field of `Options`. gix reads the working-tree `.mailmap` + // uncommitted, so nothing about the commit graph changes at all. + let repo = build_two_author_repo(); + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cfg = config(cache_dir.path(), true, false); + + let primed = build_history_index_cached(repo.path(), &opts(), &cfg).expect("prime"); + assert_eq!( + authors_long(&primed, "a.rs"), + 2, + "the fixture must have two distinct identities before the mailmap, \ + or folding them is not observable" + ); + + repo.mailmap(FOLD_GRACE_INTO_ADA); + + let cached = build_history_index_cached(repo.path(), &opts(), &cfg).expect("cached"); + let fresh = build_history_index(repo.path(), &opts()).expect("fresh"); + assert_eq!( + authors_long(&fresh, "a.rs"), + 1, + "the mailmap must fold the two identities, or a stale hit and a \ + fresh walk agree for the wrong reason" + ); + assert_eq!( + snapshot(&fresh), + snapshot(&cached), + "a mailmap edit must invalidate the entry rather than replay it" + ); +} + +#[test] +fn a_mailmap_edit_is_not_baked_into_the_incremental_splice() { + // Issue #1262, second repro — the worse one. The splice adopts a cached + // tail's events wholesale and re-persists them under the new head, so a + // stale entry's pre-mailmap digests would survive `HEAD` moving and be + // reproduced by every later pure hit, indefinitely. + let repo = build_two_author_repo(); + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cfg = config(cache_dir.path(), true, false); + + let primed = build_history_index_cached(repo.path(), &opts(), &cfg).expect("prime"); + assert_eq!( + authors_long(&primed, "a.rs"), + 2, + "the primed entry records two distinct identities" + ); + + // Commit the mailmap and advance `HEAD`, so the run below takes the + // incremental path rather than the exact-entry hit. Committing is how + // `HEAD` moves, not a second mailmap *source* under test: the file + // stays in the work tree, and `open_mailmap` reads the work-tree copy + // for any non-bare repo. The variable here is the cache path. (The + // bare-repo `HEAD:.mailmap` source has no coverage; it is reachable + // only from a repo with no work tree, which no fixture here builds.) + repo.mailmap(FOLD_GRACE_INTO_ADA); + repo.commit( + "Ada", + "ada@example.com", + FIXED_NOW - 5 * DAY, + "chore: add mailmap", + ); + repo.write("a.rs", "fn a() { work(); more(); }\n"); + repo.commit( + "Grace", + "grace@example.com", + FIXED_NOW - DAY, + "fix: a crash again", + ); + + let spliced = build_history_index_cached(repo.path(), &opts(), &cfg).expect("incremental"); + let fresh = build_history_index(repo.path(), &opts()).expect("fresh"); + assert_eq!( + authors_long(&fresh, "a.rs"), + 1, + "the mailmap must fold the two identities across all four commits" + ); + assert_eq!( + snapshot(&fresh), + snapshot(&spliced), + "the splice must not adopt the cached tail's pre-mailmap digests" + ); + + // The entry the splice persisted must replay correctly too: staleness + // baked in here is what outlives `HEAD` moving. + let hit = build_history_index_cached(repo.path(), &opts(), &cfg).expect("pure hit"); + assert_eq!( + snapshot(&fresh), + snapshot(&hit), + "the persisted entry replays the post-mailmap identities" + ); + + // …but that equality holds just as well if the run above *missed* and + // cold-walked, so on its own it is "a fresh walk equals a fresh walk". + // Emptying the entries and re-running separates the two: a served hit + // now replays zero commits, a miss reproduces `fresh`. + empty_entry_events(cache_dir.path()); + let served = snapshot( + &build_history_index_cached(repo.path(), &opts(), &cfg).expect("emptied pure hit"), + ); + assert!(!served.is_empty(), "the index seeds a.rs regardless of hit"); + assert!( + served.values().all(|stats| stats.commits_long == 0), + "the persisted entry was not served, so the assertion above was \ + comparing two cold walks" + ); + + // Cache-file hygiene rather than a #1262 claim: the pre-mailmap entry + // fingerprints differently, so it is neither a splice base nor + // superseded (a splice removes the ancestor it consumed) and lingers + // until `--clear-cache`. Update this if fingerprint-orphaned entries + // ever start being pruned — it pins current behaviour, not a contract. + assert_eq!( + count_entries(cache_dir.path()), + 2, + "the pre-mailmap entry is bypassed, not spliced onto" + ); +} + +#[test] +fn a_mailmap_file_config_edit_is_not_served_from_a_stale_entry() { + // The digest hashes gix's *merged* mailmap snapshot rather than + // re-deriving the source list, precisely so a source cannot be missed + // (issue #1262). `mailmap.file` exercises that delegation on a source + // the working-tree tests never reach: it is a path outside the + // repository, so nothing about the tree or the commit graph changes. + let repo = build_two_author_repo(); + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cfg = config(cache_dir.path(), true, false); + + // The mailmap lives outside the work tree, so writing it cannot be + // mistaken for the working-tree `.mailmap` source. + let mailmap_home = tempfile::tempdir().expect("tempdir"); + let mailmap_path = mailmap_home.path().join("identities.mailmap"); + let mailmap_arg = mailmap_path.to_str().expect("temp path is valid UTF-8"); + repo.git(&["config", "mailmap.file", mailmap_arg]); + + let primed = build_history_index_cached(repo.path(), &opts(), &cfg).expect("prime"); + assert_eq!( + authors_long(&primed, "a.rs"), + 2, + "`mailmap.file` points at a file that does not exist yet, so the \ + two identities are still distinct" + ); + + std::fs::write(&mailmap_path, FOLD_GRACE_INTO_ADA).expect("write mailmap.file"); + + let cached = build_history_index_cached(repo.path(), &opts(), &cfg).expect("cached"); + let fresh = build_history_index(repo.path(), &opts()).expect("fresh"); + assert_eq!( + authors_long(&fresh, "a.rs"), + 1, + "gix reads `mailmap.file`, or this test proves nothing about it" + ); + assert_eq!( + snapshot(&fresh), + snapshot(&cached), + "a `mailmap.file` edit must invalidate the entry too" + ); +} + +#[test] +fn a_semantically_empty_mailmap_edit_still_hits_the_cache() { + // The digest covers gix's *parsed* snapshot, not the source bytes, so + // an edit that changes no mapping must not cost a cold walk — the one + // behavioural difference between this and a byte digest, and the + // claim `repo::mailmap_digest` documents. + let repo = build_two_author_repo(); + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cfg = config(cache_dir.path(), true, false); + + repo.mailmap(FOLD_GRACE_INTO_ADA); + build_history_index_cached(repo.path(), &opts(), &cfg).expect("prime"); + assert_eq!(count_entries(cache_dir.path()), 1, "one entry primed"); + + // Emptying the primed entry's events makes a *served* hit observably + // wrong (an empty index) while an invalidation recomputes the right + // answer — so this distinguishes "hit" from "miss", which comparing + // against a fresh walk alone cannot (#951's technique). + empty_entry_events(cache_dir.path()); + + // A comment and blank line: different bytes, identical mappings. + repo.mailmap(&format!("# canonical identities\n\n{FOLD_GRACE_INTO_ADA}")); + + let cached = build_history_index_cached(repo.path(), &opts(), &cfg).expect("cached"); + let served = snapshot(&cached); + // `all` over an empty map is vacuously true, and the seeded index is + // the only reason it is not empty — assert the row exists first. + assert!(!served.is_empty(), "the index seeds a.rs regardless of hit"); + assert!( + served.values().all(|stats| stats.commits_long == 0), + "the emptied entry was still served, so the comment-only edit did \ + not invalidate it" + ); +} + #[test] fn cache_hit_is_bit_identical_to_a_fresh_walk() { let repo = build_repo(); From 24a6cae39fd47ee790ecf6466c44ed4d02fbbf72 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 15:40:56 -0700 Subject: [PATCH 03/22] fix(abc/kotlin): count primary-constructor superclass call Kotlin ABC counted a secondary constructor's `: super(x)` delegation (`constructor_delegation_call`, #1279) but not the primary-constructor form `class Sub : Base(1, 2)`, which the grammar spells as a `constructor_invocation` under a `delegation_specifier`. Both invoke the superclass constructor at run time, so both are branches. The arm is gated on that parent. tree-sitter-kotlin-ng gives `constructor_invocation` exactly three parents, and two of them are annotations -- `@Suppress("x")` is `annotation > constructor_invocation` and `@file:Suppress("x")` is `file_annotation > constructor_invocation` -- so an ungated arm would bill every argument-carrying annotation in a Kotlin file as a branch. Gating positively on `delegation_specifier` rather than denying the two annotation kinds also keeps any future annotation-shaped parent at zero. `kotlin_super_type_argument_is_not_a_condition` used `class B : A()` as scaffolding and anchored on `branches_sum() == 1`; its comment already recorded that the `A()` header contributed none and named this issue. The anchor moves to 2 and the comment with it. Siblings checked, no change needed: Kotlin `nom`, `wmc` and cyclomatic reference neither constructor production, correctly -- a superclass call is not a method declaration nor a decision point. `is_call` stays `CallExpression` only, matching Java's `MethodInvocation` and C#'s invocation kinds. Fixes #1384 --- src/metrics/abc.rs | 94 ++++++++++++++++++++++++++++++++++++--- src/metrics/abc/kotlin.rs | 21 +++++++++ 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 5b8c83c3f..429659891 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -1717,6 +1717,90 @@ mod tests { ); } + #[test] + fn kotlin_primary_constructor_superclass_call_is_a_branch() { + // Kotlin's *primary*-constructor superclass call is a + // `constructor_invocation` under a `delegation_specifier` — a third + // production, distinct from both `CallExpression` and the + // `ConstructorDelegationCall` #1279 added, so it scored zero while + // the secondary form beside it scored one (#1384). An object + // expression's superclass call uses the same production. + // + // `class Plain : Marker` is the negative case: a delegation + // specifier with no argument list is a plain `user_type`, so it + // must stay at zero. It contributes to no ABC axis, so nothing + // anchors it — deleting the line keeps this test green. What it + // buys is discrimination: with it present, broadening the arm to + // bare `DelegationSpecifier` fails here; without it, only + // `kotlin_constructor_delegation_is_a_branch` catches that. + // + // expected: 2 branches — `Base(1, 2)` and `Base(3)`. Nothing else + // in the fixture is a call, so deleting either construction from + // the fixture moves the total. + check_metrics::( + "class Sub : Base(1, 2) { } + class Plain : Marker { } + fun make(): Any = object : Base(3) { }", + "foo.kt", + |metric| { + assert_eq!(metric.abc.branches_sum(), 2); + }, + ); + } + + #[test] + fn kotlin_annotation_arguments_are_not_a_branch() { + // The parent gate on that arm is load-bearing, not decoration: + // tree-sitter-kotlin-ng spells an annotation's argument list with + // the *same* `constructor_invocation` production — `@Mark("a")` + // parses as `annotation > constructor_invocation` and + // `@file:Suppress(…)` as `file_annotation > constructor_invocation`. + // Ungated, every argument-carrying annotation in a Kotlin file + // would bill a branch, and annotations are everywhere (#1384). + // + // The fixture pairs three annotation spellings — file-level, + // class-level and use-site-targeted — with one real superclass + // call, so the total discriminates in both directions: 1 with the + // gate, 4 without it, 0 without the arm at all. + // + // A correctly-excluded annotation contributes to no ABC axis, so + // `branches_sum()` alone cannot notice the annotations being + // trimmed out of the fixture — measured: dropping any of the three + // leaves this test green, and with all three gone, removing the + // production gate fails nothing. The node census below is the + // anchor: it pins that the fixture really does hand the arm four + // `constructor_invocation` nodes for it to score 1 out of, and + // that the `file_annotation` parent — the spelling a denylist of + // `Annotation` alone would have missed — is among them. + // + // expected: 1 branch — `Base(1)` only. + let src = r#"@file:Suppress("unused") + @Mark("a") + class Ann : Base(1) { + @get:Mark("b") + val v: Int = 0 + }"#; + let parser = KotlinParser::new( + src.as_bytes().to_vec(), + &std::path::PathBuf::from("foo.kt"), + None, + ); + assert_eq!( + parser + .root() + .preorder() + .filter(|n| n.kind_id() == Kotlin::ConstructorInvocation as u16) + .count(), + 4, + "fixture must keep three annotations plus the superclass call" + ); + assert!(ast_has_kind_id(&parser, Kotlin::FileAnnotation as u16)); + + check_metrics::(src, "foo.kt", |metric| { + assert_eq!(metric.abc.branches_sum(), 1); + }); + } + #[test] fn groovy_constructor_delegation_is_a_branch() { // Groovy already counted this shape before #1279; the assertion @@ -4556,11 +4640,11 @@ function f(int $a, int $b): int { // Non-vacuity guard: 1 is also what a body whose `if` // survived but whose super call did not would score, so // pin the call itself. Measured: dropping - // `super.g(a, b)` takes `branches_sum()` to 0 — the - // `A()` primary-constructor delegation in the class - // header contributes none, which is #1384 and not this - // test's subject. - assert_eq!(metric.abc.branches_sum(), 1); + // `super.g(a, b)` takes `branches_sum()` from 2 to 1 — + // the remaining branch is the `A()` primary-constructor + // delegation in the class header, which #1384 taught this + // metric to count. + assert_eq!(metric.abc.branches_sum(), 2); }, ); } diff --git a/src/metrics/abc/kotlin.rs b/src/metrics/abc/kotlin.rs index f84a78e9f..eeed3af63 100644 --- a/src/metrics/abc/kotlin.rs +++ b/src/metrics/abc/kotlin.rs @@ -197,6 +197,27 @@ impl Abc for KotlinCode { CallExpression | ConstructorDelegationCall => { stats.branches += 1.; } + // `ConstructorInvocation` is the *primary*-constructor + // superclass call — `class Sub : Base(1, 2)`, and the same + // production inside `object : Base(1) { }`. #1279 added the + // secondary form above and left this one at zero (#1384). + // + // The parent gate is not optional. tree-sitter-kotlin-ng gives + // the production exactly three parents (node-types.json), and + // two of them are annotations: `@Suppress("x")` parses as + // `annotation > constructor_invocation` and `@file:Suppress("x")` + // as `file_annotation > constructor_invocation`, so an ungated + // arm bills every argument-carrying annotation as a branch. + // Gating *positively* on the delegation specifier — rather than + // denying the two annotation kinds — keeps any future + // annotation-shaped parent at zero too. A supertype with no + // argument list (`class Sub : Marker`) is a plain `user_type` + // and never reaches here. + ConstructorInvocation + if ancestors.parent_has_kind(node, DelegationSpecifier as u16) => + { + stats.branches += 1.; + } // Conditions: comparison operators, identity equality, // ternary-elvis (`?:`), `as?` safe-cast, and the arms of // control-flow constructs (`else`, `catch`, `when` entries). From ce643dd0f6a6967c9df4fa417490d96ef49a2239 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 08:24:06 -0700 Subject: [PATCH 04/22] chore(self-scan): refresh the threshold baseline chore(self-scan): refresh baseline after wave 1 Two entries moved, both from fixes merged in this wave: - KotlinCode::compute cyclomatic 16 -> 18, from the gated ConstructorInvocation arm (#1384). - build_cached halstead.effort 73179.83 -> 74812.54, from the mailmap digest call (#1262). Refreshed with the headroom variant so the soft tier does not re-fire on untouched files. chore(self-scan): refresh baseline after wave 3 Four entries grew from the #1381 seam; no new offenders (241 -> 241). - alterator.rs loc.ploc 545 -> 561 - Alterator::get_ast_node nargs 6 -> 7 (the ancestor chain; branching at the call site instead measured +53% halstead on the dump walk) - ast.rs build halstead.effort 72581 -> 87299 - parser.rs Parser::filters halstead.effort 58860 -> 85508 Squashed from aa5632ec, 8a512433. --- .bca-baseline.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 6f0accfdf..5de8afd4a 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -15,7 +15,7 @@ headroom = 0.95 path = "big-code-analysis-ast/src/alterator.rs" qualified = "" metric = "loc.ploc" -value = 545.0 +value = 561.0 [[entry]] path = "big-code-analysis-ast/src/alterator.rs" @@ -27,7 +27,7 @@ value = 5.0 path = "big-code-analysis-ast/src/alterator.rs" qualified = "Alterator::get_ast_node" metric = "nargs" -value = 6.0 +value = 7.0 [[entry]] path = "big-code-analysis-ast/src/alterator.rs" @@ -189,7 +189,7 @@ value = 6.0 path = "big-code-analysis-ast/src/ast.rs" qualified = "build" metric = "halstead.effort" -value = 72581.41181251501 +value = 87299.1671815905 [[entry]] path = "big-code-analysis-ast/src/c_macro.rs" @@ -273,7 +273,7 @@ value = 33.0 path = "big-code-analysis-ast/src/parser.rs" qualified = "Parser::filters" metric = "halstead.effort" -value = 58860.61554860244 +value = 85508.52631578948 [[entry]] path = "big-code-analysis-ast/src/preproc.rs" @@ -915,7 +915,7 @@ value = 15.0 path = "src/metrics/abc/kotlin.rs" qualified = "KotlinCode::compute" metric = "cyclomatic" -value = 16.0 +value = 18.0 [[entry]] path = "src/metrics/abc/objc.rs" @@ -1257,7 +1257,7 @@ value = 6.0 path = "src/vcs/git/cached.rs" qualified = "build_cached" metric = "halstead.effort" -value = 73179.83359417088 +value = 74812.5424082547 [[entry]] path = "src/vcs/git/cached.rs" From ccf3b2cbbd4f082b64707b7192a35dfbfb6a48f4 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 16:31:15 -0700 Subject: [PATCH 05/22] fix(abc): count numeric operands in Ruby, Elixir and Perl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The terminal-bool operand sets that ABC's Fitzpatrick Rule 9 walker consults named `integer` alone for Ruby and Elixir and no numeric kind at all for Perl, so a bare numeric operand scored no condition: Ruby `a && 1` 2, `a && 1.0` / `1r` / `2i` / `1ri` 1 Elixir `a && 1` 2, `a && 1.0` 1, `a && ?a` 1 Perl `$a && $b` 2, `$a && 1` 1, `1.0` / `1.5e10` / `0xff` 1, `if (1)` 0 All three are truthy-valued languages, and Python has counted both `Integer` and `Float` since #772, so the three disagreed with the control for no recorded reason. The count drops rather than errors because the walker recurses one level past an unlisted kind and the recursion then fails the parent list-kind gate. The unit to sweep is the grammar's numeric *supertype*, not the alias list. None of these kinds is aliased, so an alias sweep comes back clean and proves nothing — the misses were sibling rules under a shared hidden choice. Perl's `_numeric_literals` has five members and Elixir's `char` is an integer codepoint, both easy to miss by checking only for numeric suffixes. Ruby's `rational` / `complex` wrap the numeral (`1ri` is `complex(rational(integer))`) and the walker cannot descend into a wrapper, so the wrapper is what must be listed; `Integer` stays alongside for the bare `1`. The sweep also measured Lua, Tcl, iRules and the JS family (one `Number` kind covering every spelling, no gap) and found PHP and Groovy carrying the identical defect. Both have integration-corpus files, so their fix moves snapshots and is deferred to #1410 with a FIXME anchor at each set. Fixes #1379 --- big-code-analysis-ast/src/macros/kind_sets.rs | 106 ++++++- src/metrics/abc.rs | 266 ++++++++++++++++++ 2 files changed, 368 insertions(+), 4 deletions(-) diff --git a/big-code-analysis-ast/src/macros/kind_sets.rs b/big-code-analysis-ast/src/macros/kind_sets.rs index 95e343a08..b21c7ab16 100644 --- a/big-code-analysis-ast/src/macros/kind_sets.rs +++ b/big-code-analysis-ast/src/macros/kind_sets.rs @@ -141,6 +141,11 @@ macro_rules! java_bool_terminal_kinds { // `CastExpression`, `ParenthesizedTypeCast`, `InstanceofExpression`); // the dekobon Groovy grammar has no `await` or `array_access` // analogues, so those collapse out of the C# set. +// +// FIXME(#1410): Groovy truth makes every non-zero number truthy, so this +// set is missing the numeric literal kinds — `a && 1` scores one +// condition where `a && b` scores two. Deferred out of #1379 because the +// integration corpora carry Groovy files and the fix moves snapshots. #[macro_export] #[doc(hidden)] macro_rules! groovy_bool_terminal_kinds { @@ -237,6 +242,10 @@ macro_rules! cpp_bool_terminal_kinds { }; } +// FIXME(#1410): PHP treats every non-zero number as truthy, so this set +// is missing the numeric literal kinds — `$a && 1` scores one condition +// where `$a && $b` scores two. Deferred out of #1379 because the +// integration corpora carry PHP files and the fix moves snapshots. #[macro_export] #[doc(hidden)] macro_rules! php_bool_terminal_kinds { @@ -299,6 +308,39 @@ macro_rules! python_bool_terminal_kinds { }; } +// Terminal-bool operand kinds for Perl's ABC unary-conditional walker +// (Fitzpatrick Rule 9; issue #557): the bare boolean operands of a +// `binary_expression` short-circuit chain and of an `if` / `while` / +// `unless` / `until` / ternary / C-style-`for` condition slot. +// +// Perl is truthy-valued — every scalar but `0`, `"0"`, `""` and `undef` +// is true — so a numeric literal is a unary condition here for the same +// reason `Number` is in Lua and JavaScript (#772) and `Integer` / +// `Float` are in Python. Naming none of them scored `$a && 1` one +// condition against Python's two for `a and 1`, and `if (1)` zero +// against Python's one for `if 1:` (#1379). +// +// **The unit to check is the supertype, not the alias list.** +// tree-sitter-perl 1.1.2 has no numeric-suffix aliases at all, so an +// alias sweep (grammar-dispatch §1) comes back clean and proves nothing: +// the numerals are five *sibling rules* under the hidden +// `_numeric_literals` choice (`Perl::NumericLiterals`) — `integer`, +// `floating_point`, `scientific_notation`, `hexadecimal`, `octal`, ids +// 128-132. #1379 first landed with only the first two, leaving `$a && +// 0xff` and `$a && 1.5e10` scoring 1 against `$a && $b`'s 2. Read the +// supertype's arm list before calling such a set complete. +// +// `octal` is currently unreachable and listed defensively: the lexer +// resolves `017` to `integer` (verified by `bca dump`), and `0o17` is +// not Perl syntax — it parses as a bareword call. A future grammar that +// starts emitting it should count it, so there is nothing to guard +// against by omission (grammar-dispatch §2). +// +// The statically-typed sets (C#, Java, Kotlin, Rust, Go, C, C++) +// deliberately name no numeric kind: a bare number in a boolean slot is +// a compile error there, so there is nothing to count. PHP and Groovy +// are the two remaining truthy-valued languages that still omit one — +// tracked in #1410, not deliberate. #[macro_export] #[doc(hidden)] macro_rules! perl_bool_terminal_kinds { @@ -307,6 +349,11 @@ macro_rules! perl_bool_terminal_kinds { | $crate::Perl::Boolean | $crate::Perl::True | $crate::Perl::False + | $crate::Perl::Integer + | $crate::Perl::FloatingPoint + | $crate::Perl::ScientificNotation + | $crate::Perl::Hexadecimal + | $crate::Perl::Octal | $crate::Perl::ScalarVariable | $crate::Perl::ArrayVariable | $crate::Perl::HashVariable @@ -323,6 +370,11 @@ macro_rules! perl_bool_terminal_kinds { }; } +// Lua's `number` is one kind for the integer and the float spelling +// alike, so `a and 1` and `a and 1.0` both score through `Number` and +// the language has no counterpart of the #1379 Ruby / Elixir / Perl gap. +// The same holds for Tcl, iRules and the four JS-family sets, each +// measured rather than read off the grammar. #[macro_export] #[doc(hidden)] macro_rules! lua_bool_terminal_kinds { @@ -520,8 +572,35 @@ macro_rules! kotlin_bool_terminal_kinds { // (`Call`..`Call4` — lesson #2; a bare predicate method `ready?` is a // `call`), the literals `true` / `false` / `nil`, the variable sigils // (`@ivar`, `@@cvar`, `$gvar`), `constant`, `element_reference` -// (`items[0]`), and `integer`. Comparison operands (`x > 0`) are nested +// (`items[0]`), and the four numeric literal kinds `integer` / `float` / +// `rational` / `complex`. Comparison operands (`x > 0`) are nested // `binary` nodes, so they are absent here and contribute nothing. +// +// Ruby is truthy-valued — every number including `0` and `0.0` is +// truthy — so a bare numeric operand is a Fitzpatrick unary condition +// exactly as it is in Python and Lua (#772). Listing `integer` alone +// scored `a && 1.0` / `a && 1r` / `a && 2i` one condition where +// `a && 1` scores two (#1379). +// +// `rational` and `complex` are WRAPPERS over the numeral (`1r` is +// `rational(integer)`, `2i` is `complex(integer)`, `1ri` is +// `complex(rational(integer))` — verified by `bca dump`), and the +// **wrapper** is what has to be listed: `ruby_inspect_container` breaks +// out of its descent for any node that is neither +// `parenthesized_statements` nor a `!` / `not` unary, so the walker +// cannot reach the inner numeral at all. Listing `Integer` alone scores +// all three suffixed literals zero, which is what #1379 measured. +// +// The mirror-image hazard — grammar-dispatch §5's container/contained +// double-count — is absent here for the same reason, and `Integer` +// staying in the set alongside them is not redundancy: it is what scores +// a bare `1`. Do not "simplify" by removing either half. (#1359 reached +// the same keep-the-wrapper answer for Halstead operand identity, where +// the walk *does* visit every node and the double-count is real.) +// +// None of the four kinds has a numeric-suffix alias in tree-sitter-ruby +// 0.23.1; `_int_or_float` (`Ruby::IntOrFloat`) is a hidden supertype the +// parser never emits (grammar-dispatch §2). #[macro_export] #[doc(hidden)] macro_rules! ruby_bool_terminal_kinds { @@ -540,6 +619,9 @@ macro_rules! ruby_bool_terminal_kinds { | $crate::Ruby::Constant | $crate::Ruby::ElementReference | $crate::Ruby::Integer + | $crate::Ruby::Float + | $crate::Ruby::Rational + | $crate::Ruby::Complex }; } @@ -550,9 +632,23 @@ macro_rules! ruby_bool_terminal_kinds { // boolean operands surface as: `identifier`, `call` (both `ready?()` and // the no-paren dot access `cfg.enabled` parse as `call`), `dot` // (`Mod.fun` reference), the `boolean` literal wrapper (`true` / `false` -// parse as `boolean`, verified by AST dump), `nil`, `atom`, `integer`, -// and `access_call` (`xs[i]`). Comparison operands are nested -// `binary_operator` nodes and so contribute nothing. +// parse as `boolean`, verified by AST dump), `nil`, `atom`, the three +// numeric literal kinds `integer` / `float` / `char`, and `access_call` +// (`xs[i]`). Comparison operands are nested `binary_operator` nodes and +// so contribute nothing. +// +// Elixir's `&&` / `||` are truthy operators (everything but `false` and +// `nil` is truthy), so a bare numeric operand counts as a Fitzpatrick +// unary condition. `integer` alone scored `a && 1.0` one condition where +// `a && 1` scores two (#1379). +// +// The grammar's numeric family is `integer` / `float` / `char`, none of +// them aliased, and there are no rational or complex kinds. `char` is +// here because `?a` **is** an integer in Elixir — it evaluates to the +// codepoint 97 — so it is a numeric literal wearing a sigil, not a +// string; `x && ?a` scored 1 against `x && b`'s 2 until it was listed. +// Radix prefixes (`0x`, `0o`, `0b`) fold into `integer`, verified by +// measurement. #[macro_export] #[doc(hidden)] macro_rules! elixir_bool_terminal_kinds { @@ -570,6 +666,8 @@ macro_rules! elixir_bool_terminal_kinds { | $crate::Elixir::Nil | $crate::Elixir::Atom | $crate::Elixir::Integer + | $crate::Elixir::Float + | $crate::Elixir::Char | $crate::Elixir::AccessCall }; } diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 429659891..e5510ca96 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -11492,3 +11492,269 @@ mod keyword_negation_parity { ); } } + +/// A numeric literal used as a bare `&&` / `||` operand is a Fitzpatrick +/// Rule 9 unary condition in every truthy-valued language (#1379). +/// +/// Ruby's terminal-bool set named `integer` and none of `float` / +/// `rational` / `complex`, Elixir's named `integer` and not `float`, and +/// Perl's named no numeric kind at all — so `a && 1.0` scored one +/// condition where `a && 1` scored two, and in Perl even `$a && 1` +/// scored one. Python had both kinds since #772 and is the control the +/// other three were brought level with. +/// +/// The headline claim of each case is a *comparison*: a numeric operand +/// must score exactly what an identifier operand scores in the same slot. +/// That is what discriminates the defect — with the kind missing from the +/// set the numeric form drops while the identifier form does not — and it +/// gives a failure message naming both spellings. +/// `every_operand_scores_its_recorded_values` then pins the absolute +/// numbers, so a regression moving *both* sides equally still fails, and +/// carries the cross-metric anchor `.claude/rules/grammar-dispatch.md` §8 +/// asks for. +/// +/// Two slots per language, because the sets feed two independent walker +/// paths (grammar-dispatch §11) and Perl's defect showed in both: the +/// operands of a `&&` chain, and the predicate of an `if`. A fixture of +/// only the first leaves `perl_count_condition` and `ruby_count_condition` +/// untested. +/// +/// The recorded cyclomatic figure is the same 3 for both slots — one file +/// space, one function space, one decision — while conditions differ (2 +/// for the chain, 1 for the predicate, since ABC scores an `if` through +/// its predicate rather than the keyword). So §8's `conditions == +/// cyclomatic - 1` identity is chain-specific arithmetic, not a law; what +/// generalises is that cyclomatic must not move when only the operand +/// spelling does, which is what makes a `conditions` move unambiguously +/// ABC's. +/// +/// `for_each_case` guards the two ways this table could decay into +/// asserting nothing: an emptied `numerics` slice, and a template that +/// loses its `{}` slot (which would make every comparison `x == x`). +/// Both were reachable in the first draft and neither failed a test. +#[cfg(test)] +#[cfg(any( + feature = "ruby", + feature = "elixir", + feature = "perl", + feature = "python", + feature = "lua", + feature = "javascript" +))] +mod numeric_bool_operands { + use crate::test_support::metrics_verbatim; + use crate::{LANG, MetricsOptions}; + + /// One fixture shape: a source template with a `{}` operand slot, and + /// the `abc.conditions_sum` / `cyclomatic_sum` every spelling of that + /// operand must produce. + type Slot = (&'static str, u64, u64); + + /// A language's two slots, its identifier baseline operand, the + /// numeric operands that must score the same, and how many of those + /// there should be. + /// + /// The count is not bookkeeping. `for_each_case` counts *languages*, + /// so trimming a row's operand list back to `&["1"]` — which is + /// exactly the pre-#1379 fixture — left the whole module green when + /// measured. Pinning the length makes that a deliberate two-line + /// edit instead of a silent one. + type Case = ([Slot; 2], &'static str, &'static [&'static str], usize); + + fn conditions(lang: LANG, source: &str) -> u64 { + metrics_verbatim(lang, source.as_bytes(), MetricsOptions::default()) + .abc + .conditions_sum() + } + + fn cyclomatic_sum(lang: LANG, source: &str) -> u64 { + metrics_verbatim(lang, source.as_bytes(), MetricsOptions::default()) + .cyclomatic + .cyclomatic_sum() + } + + /// `([chain_slot, condition_slot], identifier, numerics)` per language. + /// + /// `{}` is the right-hand operand of a two-operand short-circuit chain + /// in the first slot and the whole `if` predicate in the second, so + /// every fixture differs from its own baseline in exactly one token. + /// + /// Each row lists **one spelling per numeric kind the grammar emits**, + /// which is the unit that matters: #1379's misses were sibling rules + /// under a shared supertype, not aliases of one rule. + /// + /// - Ruby: `integer` / `float` / `rational` / `complex`. `1r` parses + /// to `rational(integer)`, `2i` to `complex(integer)` and `1ri` to + /// `complex(rational(integer))` — verified by `bca dump` — and the + /// walker cannot descend into any wrapper, so listing the wrapper + /// is the only way each scores at all. Radix prefixes and `_` + /// separators fold into `integer`. + /// - Perl: all five members of the hidden `_numeric_literals` choice. + /// `0xff` and `1.5e10` are here because the first cut of #1379 + /// named only `integer` and `floating_point` and left them scoring + /// 1 against an identifier's 2. `017` is *not* a sixth row — it + /// lexes as `integer`, and the `octal` kind is unreachable. + /// - Elixir: `integer` / `float` / `char`. `?a` is the codepoint 97, + /// a numeric literal wearing a sigil. + /// + /// Python, Lua and JavaScript were already correct and ride along as + /// controls — they are what the first three were measured against, + /// and a future edit that breaks them fails here too. Each folds + /// every radix into one `Number` / `integer` kind, checked by + /// measurement. + fn cases(lang: LANG) -> Option { + Some(match lang { + LANG::Ruby => ( + [ + ("def f(a)\n a && {}\nend\n", 2, 3), + ("def f\n if {}\n 1\n end\nend\n", 1, 3), + ], + "b", + &["1", "1.0", "1r", "2i", "1ri"], + 5, + ), + LANG::Elixir => ( + [ + ("def f(a) do\n a && {}\nend\n", 2, 3), + ("def f() do\n if {} do\n 1\n end\nend\n", 1, 3), + ], + "b", + &["1", "1.0", "?a"], + 3, + ), + LANG::Perl => ( + [ + ("sub f {\n my $x = $a && {};\n}\n", 2, 3), + ("sub f {\n if ({}) { 1; }\n}\n", 1, 3), + ], + "$b", + &["1", "1.0", "1.5e10", "0xff"], + 4, + ), + LANG::Python => ( + [ + ("def f(a):\n return a and {}\n", 2, 3), + ("def f():\n if {}:\n return 1\n", 1, 3), + ], + "b", + &["1", "1.0", "0xff", "1j"], + 4, + ), + LANG::Lua => ( + [ + ("function f(a)\n return a and {}\nend\n", 2, 3), + ("function f()\n if {} then return 1 end\nend\n", 1, 3), + ], + "b", + &["1", "1.0", "0xff"], + 3, + ), + LANG::Javascript => ( + [ + ("function f(a) {\n return a && {};\n}\n", 2, 3), + ("function f() {\n if ({}) { return 1; }\n}\n", 1, 3), + ], + "b", + &["1", "1.0", "0xff"], + 3, + ), + _ => return None, + }) + } + + /// Runs `check` once per enabled language that has a case, having + /// first established that the case can still assert something. + /// + /// Three guards, all of which had to be added after the first draft + /// shipped without them and a measured perturbation of each left the + /// whole module green: + /// + /// - **`checked > 0`** is the non-vacuity half of the rule in + /// `.claude/rules/testing.md`. The `#[cfg(any(feature = …))]` on + /// this module makes the tests *absent* when no truthy-valued + /// language is compiled in; this catches the residual case where + /// the runtime `is_enabled()` check stops agreeing with the feature + /// it compiled under. + /// - **the `numerics` list keeps its recorded length** — `checked` + /// counts *languages*, so trimming a row's numeric list back to + /// `&["1"]` (the pre-#1379 fixture) or emptying it left every test + /// passing when measured, with that language's coverage deleted. + /// - **every template keeps its `{}`** — without the slot, + /// `str::replace` is a no-op, baseline and candidate are computed + /// from the same string, and the comparison degenerates to + /// `x == x`. + /// + /// Sharing one driver means no test can lose any of the three. + fn for_each_case(check: impl Fn(LANG, Case)) { + let mut checked = 0; + for lang in LANG::into_enum_iter() { + if !lang.is_enabled() { + continue; + } + let Some(case @ (slots, _, numerics, expected_kinds)) = cases(lang) else { + continue; + }; + assert_eq!( + numerics.len(), + expected_kinds, + "{lang:?}: the numeric-operand list no longer covers one spelling \ + per grammar numeric kind" + ); + for (template, _, _) in slots { + assert!( + template.contains("{}"), + "{lang:?}: template lost its `{{}}` operand slot: {template}" + ); + } + check(lang, case); + checked += 1; + } + assert!( + checked > 0, + "no truthy-valued language enabled; this test asserted nothing" + ); + } + + #[test] + fn a_numeric_operand_scores_like_an_identifier_operand() { + for_each_case(|lang, (slots, identifier, numerics, _)| { + for (template, _, _) in slots { + let baseline = conditions(lang, &template.replace("{}", identifier)); + for numeric in numerics { + let source = template.replace("{}", numeric); + let scored = conditions(lang, &source); + assert_eq!( + scored, baseline, + "{lang:?}: `{numeric}` scored {scored} unary conditions against \ + `{identifier}`'s {baseline}\n source: {source}" + ); + } + } + }); + } + + /// The absolute anchor under the comparison above: every operand + /// spelling must produce the slot's recorded `conditions`, and must + /// leave `cyclomatic` alone. The second half is what rules out a + /// regression that moved both metrics together. + #[test] + fn every_operand_scores_its_recorded_values() { + for_each_case(|lang, (slots, identifier, numerics, _)| { + for (template, expected_conditions, expected_cyclomatic) in slots { + for operand in std::iter::once(identifier).chain(numerics.iter().copied()) { + let source = template.replace("{}", operand); + assert_eq!( + conditions(lang, &source), + expected_conditions, + "{lang:?}: `{operand}` conditions\n source: {source}" + ); + assert_eq!( + cyclomatic_sum(lang, &source), + expected_cyclomatic, + "{lang:?}: `{operand}` cyclomatic_sum\n source: {source}" + ); + } + } + }); + } +} From 36de474c364519f4b392c6c0b36119a9a8b3d370 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 16:36:45 -0700 Subject: [PATCH 06/22] fix(metrics/loc): credit PHP heredoc and nowdoc rows to ploc #778 routed PHP's quoted literals through add_multiline_string_ploc and excluded the heredoc, recording that its body "already reaches PLOC through its inner statement nodes". Half true: tree-sitter-php emits a body child only for a row that has text, so a row empty inside the literal held no node and blank = sloc - ploc - cloc claimed it. Nowdoc was worse than the report measured. On the issue's six-row fixture heredoc gave ploc 5, blank 1 while nowdoc gave ploc 4, blank 2, because its body is not one nowdoc_string per row: the grammar emits one for the first line and a single multi-row node for the rest, whose interior rows the catch-all's start-row insertion all lost. Route the heredoc / nowdoc wrapper rather than heredoc_body / nowdoc_body: a body of one empty row emits no body node at all, so the wrapper is the node present for every spelling. Review found the backtick shell_command_expression carrying the nowdoc shape exactly, so it is routed too - the arm now agrees with PhpCode::is_string on every kind that grammar can span rows with. Fixes #1396 --- src/metrics/loc.rs | 171 +++++++++++++++++++++++++++++++++++++++++ src/metrics/loc/php.rs | 42 ++++++++-- 2 files changed, 207 insertions(+), 6 deletions(-) diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index f195e7f22..2848d5f3a 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -8402,6 +8402,177 @@ $y = 10 + match ($x) { 1 => 2, default => 0 };", }); } + /// Analyses `source` byte-for-byte as PHP, for the two #1396 tests + /// `check_metrics` cannot carry: one ends at EOF, which that shim + /// normalises away, and the other loops over labelled cases, which + /// its bare `fn` callback cannot close over. + #[cfg(feature = "php")] + fn php_loc(source: &[u8]) -> Stats { + metrics_verbatim( + crate::LANG::Php, + source, + crate::MetricsOptions::default().with_only(&[crate::Metric::Loc]), + ) + .loc + } + + /// #1396: a heredoc row that is empty *inside* the literal read as + /// blank. tree-sitter-php emits one `string_content` child per body + /// row that **has text**, so a row empty inside the literal held no + /// node at all and `blank = sloc - ploc - cloc` claimed it. #778 + /// routed PHP's quoted literals through `add_multiline_string_ploc` + /// but excluded the heredoc, on the premise that its inner nodes + /// already covered every row. + /// + /// `sloc` is the fixture anchor (`.claude/rules/testing.md`): + /// deleting the empty row drops it to 5 and fails this test, rather + /// than leaving it quietly asserting `blank 0` about a literal with + /// no interior gap left to misclassify. + #[cfg(feature = "php")] + #[test] + fn php_heredoc_empty_interior_row_is_code_not_blank() { + // expected: six rows, every one code — `("( + " { add_cloc_lines(stats, start, end); } - // A PHP double-quoted (`encapsed_string`) or single-quoted - // (`string`) literal can span several rows; credit every spanned - // row to PLOC to match Python's #415 decision (#778). Heredoc / - // nowdoc bodies already reach PLOC through their inner statement - // nodes, so they are not routed here. - EncapsedString | String => { + // Every PHP literal that can span rows, so its interior rows + // reach PLOC instead of being claimed by + // `blank = sloc - ploc - cloc` (#778, #1396) — the decision #415 + // took for Python and #1260 took for the last four languages. + // + // #778 routed the quoted forms and excluded the heredoc, on the + // premise that its body "already reaches PLOC through its inner + // statement nodes". Half true, and it cost a phantom blank row + // per spelling: tree-sitter-php emits a body child only for a row + // that *has* text. Heredoc drops just the row empty inside the + // literal; nowdoc is worse, emitting one `nowdoc_string` for the + // first line and a single multi-row one for the rest, whose + // interior rows the catch-all's start-row insertion all lost + // whether or not any of them was empty. + // + // The wrapper is routed rather than `HeredocBody` / `NowdocBody` + // because a body of one empty row emits no body node at all — + // `heredoc` is the node present for every spelling + // (`.claude/rules/grammar-dispatch.md` section 6). Its span runs + // from `<<<` to the closing marker, so its interior is the body + // rows plus that marker's row, which is code either way. + // + // `ShellCommandExpression` (`` `…` ``) is the fifth form and had + // the nowdoc shape exactly: one multi-row `string_content` child, + // so every interior row was lost. It is routed here for the same + // reason, which makes this arm agree with + // `PhpCode::is_string` (`big-code-analysis-ast/src/checker/php.rs`) + // on every kind that grammar can span rows with — section 7's + // parity cross-walk. + // + // Aliases (section 1): none of the five routed kinds has a + // numeric-suffix variant. `String2` is the anonymous `string` + // *type* keyword and `String3` is the hidden `_string` supertype + // the parser never emits (section 2) — neither is a literal, so + // neither belongs here. + EncapsedString | String | Heredoc | Nowdoc | ShellCommandExpression => { add_multiline_string_ploc(node, ancestors, stats, start); } // Statement kinds that contribute one logical line each. From 7dd805466b9f6eed605b8b960a0b29ca869a6d76 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 16:38:10 -0700 Subject: [PATCH 07/22] fix(npm/ruby): count initialize as a private method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruby privatises `initialize`, `initialize_copy`, `initialize_dup`, `initialize_clone` and `respond_to_missing?` at definition, so essentially every Ruby class with a constructor reported `npm` one higher than `instance_methods(false)`. The rule is applied in `RubyClassBody::declare`, where the name is already read, rather than at the tally. #1255's retroactive refile pass runs afterwards over the recorded declarations, so an explicit `public :initialize` republishes the method with no extra code. Four edges of the rule were measured against ruby 3.0.2 rather than inferred, and each is pinned by a test: - it outranks the body-wide flag, so an explicit `public` marker above `def initialize` still yields a private method; - it loses to a keyword naming the declaration directly, so `public def initialize` is public; - it is *not* overridden by a keyword naming the other method family: `public_class_method def initialize` leaves the instance method private; - it is instance-only, so `def self.initialize` and a `def initialize` inside `class << self` both stay public. The last needs the enclosing body's kind, since that declaration is an ordinary `Method` node. `nm` is unchanged throughout — only the public/private split moves. `RubyVisibilityCall::governs` names the "does this keyword decide the visibility of that declaration" test that `ruby_wrapped_is_public` already applied, so `Npm` can ask it without restating the rule. Fixes #1400 --- src/metrics/npa/shared.rs | 12 +- src/metrics/npm.rs | 249 ++++++++++++++++++ src/metrics/npm/ruby.rs | 65 ++++- ...keyword_does_not_republish_initialize.snap | 15 ++ ...vate_auto_private_name_is_not_flipped.snap | 15 ++ ...automatically_private_name_is_demoted.snap | 15 ++ ...n_a_singleton_class_body_stays_public.snap | 15 ++ ...uby_initialize_is_not_a_public_method.snap | 15 ++ ...blic_keyword_wrapping_initialize_wins.snap | 15 ++ ..._marker_does_not_republish_initialize.snap | 15 ++ ..._public_symbol_republishes_initialize.snap | 15 ++ ...uby_singleton_initialize_stays_public.snap | 15 ++ 12 files changed, 449 insertions(+), 12 deletions(-) create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_a_class_method_keyword_does_not_republish_initialize.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_an_already_private_auto_private_name_is_not_flipped.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_every_automatically_private_name_is_demoted.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_in_a_singleton_class_body_stays_public.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_is_not_a_public_method.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_keyword_wrapping_initialize_wins.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_marker_does_not_republish_initialize.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_symbol_republishes_initialize.snap create mode 100644 src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_singleton_initialize_stays_public.snap diff --git a/src/metrics/npa/shared.rs b/src/metrics/npa/shared.rs index e4c634ed0..b64369041 100644 --- a/src/metrics/npa/shared.rs +++ b/src/metrics/npa/shared.rs @@ -207,6 +207,16 @@ pub(crate) struct RubyVisibilityCall { pub(crate) targets_singleton: bool, } +impl RubyVisibilityCall { + // Whether this keyword names the given method family, and so decides + // the visibility of a declaration in its argument list. `private def + // self.x` does not govern the singleton it wraps, and + // `private_class_method def x` does not govern the instance method. + pub(crate) fn governs(self, singleton: bool) -> bool { + self.targets_singleton == singleton + } +} + // What a visibility-keyword `Call` in a Ruby class body does. Shared by // `Npm` and `Npa` so the two walkers cannot drift on the same Ruby rule // (grammar-dispatch rule 7). @@ -265,7 +275,7 @@ pub(crate) fn ruby_wrapped_is_public( singleton: bool, body_flag: RubyVisibility, ) -> bool { - if keyword.targets_singleton == singleton { + if keyword.governs(singleton) { keyword.visibility == RubyVisibility::Public } else { ruby_declaration_is_public(singleton, body_flag) diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index 095cbce45..c07d10343 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -2934,6 +2934,255 @@ class C { ); } + // --- Ruby's automatic-private methods (#1400) ------------------------- + // + // Ruby privatises `initialize`, `initialize_copy`, `initialize_dup`, + // `initialize_clone` and `respond_to_missing?` the moment they are + // defined. Every expectation below was measured against ruby 3.0.2 + // with `instance_methods(false)` / `singleton_methods(false)` rather + // than reasoned about, because three of the rule's edges are not + // guessable from the spelling: it beats the body-wide flag, it loses + // to a keyword naming the declaration directly, and it does not + // reach a singleton. + // + // Each fixture pairs the auto-private method with an ordinary public + // one, so `npm` discriminates between "the rule fired" (1) and "the + // whole tally broke" (0) — a class holding only `initialize` reports + // 0 either way. `nm` is asserted alongside in every case, because the + // rule moves the public/private split and must never drop a method. + // + // That `nm` is also the fixture-decay anchor, but only against + // *deletion*: trim the auto-private `def` out and the count falls. + // A **rename** is caught only by the fixtures whose expected answer + // is "the rule fired" — `initialize` renamed to `setup` moves `npm` + // there. In the four whose expected answer is "the rule does not + // apply" (`…wins`, `…singleton_initialize…`, + // `…in_a_singleton_class_body…`, `…public_symbol_republishes…`) the + // name contributes to no axis once exempt, so a rename is silent and + // no anchor is available — measured, not assumed. That matters most + // for `ruby_public_keyword_wrapping_initialize_wins`, the sole guard + // on the `named_by_keyword ||` disjunct: rename its method and that + // branch goes uncovered with nothing going red. + + #[test] + fn ruby_initialize_is_not_a_public_method() { + // The issue's own fixture. Ruby reports `[:value]` for + // `Init.instance_methods(false)`; `initialize` is reachable only + // through `Init.new`. + // + // expected: nm = 2 (initialize, value), npm = 1 (value). + check_metrics::( + "class Init\n def initialize(x)\n @x = x\n end\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 1); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_every_automatically_private_name_is_demoted() { + // All five names in one body, so no member of + // `RUBY_AUTO_PRIVATE_METHODS` can be dropped without moving a + // number here. `respond_to_missing?` is the one whose spelling + // is at risk: the grammar's `name` field carries the trailing + // `?` as part of the `identifier` token, so a comparison that + // lost it would leave that method public and report npm = 2. + // + // expected: nm = 6 (the five plus `value`), npm = 1 (`value`). + check_metrics::( + "class A\n def initialize(x)\n @x = x\n end\n def initialize_copy(o)\n 1\n end\n def initialize_dup(o)\n 2\n end\n def initialize_clone(o)\n 3\n end\n def respond_to_missing?(n, p)\n 4\n end\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 6); + assert_eq!(metric.npm.class_npm_sum(), 1); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_public_symbol_republishes_initialize() { + // `public :initialize` is legal and does exactly what it says + // (measured: `B.instance_methods(false)` is `[:initialize]`). + // The rule is applied where the name is declared, so #1255's + // retroactive refile pass — which runs afterwards, over the + // methods already recorded — restores it with no extra code. + // + // expected: nm = 2, npm = 2. Dropping the `public :initialize` + // line takes npm to 1, which is what makes this a test of the + // refile ordering rather than of the tally. + check_metrics::( + "class B\n def initialize(x)\n @x = x\n end\n public :initialize\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 2); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_public_keyword_wrapping_initialize_wins() { + // `public def initialize` is public in Ruby, so a keyword that + // names the declaration directly outranks the automatic rule. + // This is the fixture that fails if the rule is applied + // unconditionally in `declare` instead of only to declarations + // no keyword governs. + // + // expected: nm = 2, npm = 2. + check_metrics::( + "class H\n public def initialize(x)\n @x = x\n end\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 2); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_a_class_method_keyword_does_not_republish_initialize() { + // The one shape where a visibility keyword wraps an + // auto-private `def` and still does not decide it: + // `public_class_method` names the singleton family, so it + // governs nothing about the *instance* `initialize` in its + // argument list, which falls back to its own default and is + // demoted. Measured: `PCM.instance_methods(false)` is + // `[:value]` and `initialize` is private, despite the keyword + // reading `public`. + // + // Checking this against Ruby shows one thing that looks like a + // disagreement and is not: `PCM.singleton_methods(false)` is + // `[:initialize]`, because the keyword republished the + // *inherited* `Class#initialize`. That is not a declaration in + // the file, and this walk counts declarations, so `nm` stays 2. + // + // This is the fixture that makes `RubyVisibilityCall::governs` + // load-bearing. Every other test here pairs an auto-private + // name with a keyword that *does* govern it, so weakening the + // check to `keyword.is_some()` passes all of them and reports 2 + // here. + // + // expected: nm = 2, npm = 1 (`value`). + check_metrics::( + "class PCM\n public_class_method def initialize(x)\n @x = x\n end\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 1); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_public_marker_does_not_republish_initialize() { + // The body-wide flag is *not* a keyword naming the declaration, + // and Ruby agrees: with an explicit `public` marker above it, + // `G.private_instance_methods(false)` is still `[:initialize]`. + // Separating this from the wrapping form above is the whole + // reason `declare` distinguishes the two. + // + // No perturbation of the current code isolates this test — every + // one that reaches the marker also republishes the plain + // `initialize` above, so it always fails alongside + // `ruby_initialize_is_not_a_public_method`. It is kept because + // the rule it pins is the one a reader is most likely to + // "correct" in the wrong direction, and because the fixture + // states Ruby's answer where prose would only assert it. + // + // expected: nm = 2, npm = 1 (`value`). + check_metrics::( + "class G\n public\n def initialize(x)\n @x = x\n end\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 1); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_an_already_private_auto_private_name_is_not_flipped() { + // Both spellings of "already private" in one body: `initialize` + // demoted by a wrapping keyword, `initialize_copy` by the + // body-wide flag. Measured: `D.private_instance_methods(false)` + // is `[:initialize, :initialize_copy]`, `instance_methods(false)` + // is `[:value]`. + // + // The second half is the one that carries weight. The rule + // *forces* private rather than toggling, and those two agree + // everywhere except here — on a name that is already private for + // an unrelated reason. Spelling the combination as an `^` passes + // every other fixture in this block and republishes + // `initialize_copy`, so this test is its only guard. + // + // The trailing `public` marker is what keeps `value` public + // across the flag flip, so npm can distinguish 1 from 0. + // + // expected: nm = 3, npm = 1 (`value`). + check_metrics::( + "class D\n private def initialize(x)\n @x = x\n end\n private\n def initialize_copy(o)\n 1\n end\n public\n def value\n @x\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 3); + assert_eq!(metric.npm.class_npm_sum(), 1); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_singleton_initialize_stays_public() { + // `def self.initialize` defines a method on the class object, + // which the automatic rule does not reach — measured: + // `C.singleton_methods(false)` is `[:initialize, :plain]`. The + // `!singleton` half of the gate is what keeps it there. + // + // expected: nm = 2, npm = 2. + check_metrics::( + "class C\n def self.initialize\n 1\n end\n def self.plain\n 2\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 2); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + + #[test] + fn ruby_initialize_in_a_singleton_class_body_stays_public() { + // The other spelling of the same exemption, and the one the + // node kind cannot express: inside `class << self` the + // declaration is a plain `method` node, indistinguishable from + // an instance method, so only the enclosing `SingletonClass` + // says the rule does not apply. Measured: + // `J.singleton_methods(false)` is `[:initialize, :other]`. + // + // Note this cuts the opposite way to + // `ruby_private_in_a_singleton_class_body_demotes`, where the + // body-wide flag *does* reach these declarations. Both are + // Ruby's behaviour; neither generalises to the other. + // + // expected: nm = 2, npm = 2. + check_metrics::( + "class J\n class << self\n def initialize\n 1\n end\n def other\n 2\n end\n end\nend\n", + "foo.rb", + |metric| { + assert_eq!(metric.npm.class_nm_sum(), 2); + assert_eq!(metric.npm.class_npm_sum(), 2); + insta::assert_json_snapshot!(metric.npm); + }, + ); + } + #[test] fn ruby_visibility_call_on_another_object_is_ignored() { // A receiver other than `self` puts the call on a different diff --git a/src/metrics/npm/ruby.rs b/src/metrics/npm/ruby.rs index 082dd9fcf..32bbeec19 100644 --- a/src/metrics/npm/ruby.rs +++ b/src/metrics/npm/ruby.rs @@ -13,6 +13,27 @@ use super::npa::{ }; use super::*; +// The five instance methods Ruby makes private the moment they are +// defined: `Class#new` calls `initialize`, and the other four are +// dispatched by the runtime rather than by a caller, so `obj.initialize` +// raises `NoMethodError` (#1400). +// +// Measured on ruby 3.0.2 rather than assumed. Three properties of the +// rule that the spelling alone does not give away: +// - it beats the body-wide flag, so an explicit `public` marker above +// `def initialize` still yields a private method; +// - it loses to a keyword that names the declaration directly, so +// `public def initialize` is public; +// - it is instance-only, so both `def self.initialize` and a +// `def initialize` inside `class << self` stay public. +const RUBY_AUTO_PRIVATE_METHODS: [&str; 5] = [ + "initialize", + "initialize_copy", + "initialize_dup", + "initialize_clone", + "respond_to_missing?", +]; + // One method declared by a Ruby class body. Visibility cannot be settled // arm-by-arm: `private :foo` demotes a method declared *earlier* in the // same body, so the tally is taken once the whole body has been read @@ -32,14 +53,19 @@ struct RubyMethodDecl<'a> { // travel together rather than through a parameter list. struct RubyClassBody<'a> { code: &'a [u8], + // `class << x` rather than `class X`. Ruby's automatic-private rule + // does not reach a singleton class, so the body needs to know which + // of the two it is walking. + in_singleton_class: bool, visibility: RubyVisibility, methods: Vec>, } impl<'a> RubyClassBody<'a> { - fn new(code: &'a [u8]) -> Self { + fn new(code: &'a [u8], in_singleton_class: bool) -> Self { Self { code, + in_singleton_class, // Ruby class bodies open in default-public state, whatever // the previous body's trailing visibility was. visibility: RubyVisibility::Public, @@ -47,13 +73,31 @@ impl<'a> RubyClassBody<'a> { } } - // Records a `method` / `singleton_method` node. - fn declare(&mut self, method: &Node<'a>, singleton: bool, public: bool) { + // Whether `RUBY_AUTO_PRIVATE_METHODS` covers a declaration of this + // name. Two independent ways to be exempt, and the walk sees them at + // different levels: `def self.x` carries its own singleton flag, + // while a `def x` inside `class << x` is an ordinary `Method` node + // that only the enclosing body knows about. + fn is_auto_private(&self, name: Option<&str>, singleton: bool) -> bool { + !singleton + && !self.in_singleton_class + && name.is_some_and(|n| RUBY_AUTO_PRIVATE_METHODS.contains(&n)) + } + + // Records a `method` / `singleton_method` node, taking its + // visibility from `keyword` when a visibility call wraps it and from + // the body-wide flag otherwise. + fn declare(&mut self, method: &Node<'a>, singleton: bool, keyword: Option) { let name = ruby_method_name(method, self.code); + let named_by_keyword = keyword.is_some_and(|kw| kw.governs(singleton)); + let public = keyword.map_or_else( + || ruby_declaration_is_public(singleton, self.visibility), + |kw| ruby_wrapped_is_public(kw, singleton, self.visibility), + ); self.methods.push(RubyMethodDecl { name, singleton, - public, + public: public && (named_by_keyword || !self.is_auto_private(name, singleton)), }); } @@ -113,8 +157,7 @@ impl<'a> RubyClassBody<'a> { continue; } }; - let declared_public = ruby_wrapped_is_public(keyword, singleton, self.visibility); - self.declare(&arg, singleton, declared_public); + self.declare(&arg, singleton, Some(keyword)); } } @@ -129,9 +172,7 @@ impl<'a> RubyClassBody<'a> { let kind = child.kind_id().into(); match kind { Method | SingletonMethod => { - let singleton = matches!(kind, SingletonMethod); - let public = ruby_declaration_is_public(singleton, self.visibility); - self.declare(child, singleton, public); + self.declare(child, matches!(kind, SingletonMethod), None); } Call | Call2 | Call3 | Call4 => match ruby_visibility_effect(child, self.code) { Some(RubyVisibilityEffect::Flag(flag)) => self.visibility = flag, @@ -158,7 +199,9 @@ impl<'a> RubyClassBody<'a> { // `private_class_method` demotes one; // - the argument forms do not touch the flag, but do govern what they // name: `private def x` declares a private `x`, and `private :foo` -// re-files a method declared earlier in the same body. +// re-files a method declared earlier in the same body; +// - the five `RUBY_AUTO_PRIVATE_METHODS` names are private from the +// moment they are defined, over the top of the flag (#1400). // // `Module` bodies are not classes (the getter routes them to // `SpaceKind::Namespace`); they do not contribute to `Npm` so a @@ -182,7 +225,7 @@ impl Npm for RubyCode { return; } - let mut body = RubyClassBody::new(code); + let mut body = RubyClassBody::new(code, matches!(parent_kind, SingletonClass)); for child in node.children() { body.visit(&child); } diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_a_class_method_keyword_does_not_republish_initialize.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_a_class_method_keyword_does_not_republish_initialize.snap new file mode 100644 index 000000000..18abc2dbe --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_a_class_method_keyword_does_not_republish_initialize.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 1, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 0.5, + "interface_coa": 0.0, + "total": 1, + "total_methods": 2, + "coa": 0.5 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_an_already_private_auto_private_name_is_not_flipped.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_an_already_private_auto_private_name_is_not_flipped.snap new file mode 100644 index 000000000..c7dee7256 --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_an_already_private_auto_private_name_is_not_flipped.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 1, + "interface_npm_sum": 0, + "class_methods": 3, + "interface_methods": 0, + "class_coa": 0.3333333333333333, + "interface_coa": 0.0, + "total": 1, + "total_methods": 3, + "coa": 0.3333333333333333 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_every_automatically_private_name_is_demoted.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_every_automatically_private_name_is_demoted.snap new file mode 100644 index 000000000..5fc898b41 --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_every_automatically_private_name_is_demoted.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 1, + "interface_npm_sum": 0, + "class_methods": 6, + "interface_methods": 0, + "class_coa": 0.16666666666666666, + "interface_coa": 0.0, + "total": 1, + "total_methods": 6, + "coa": 0.16666666666666666 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_in_a_singleton_class_body_stays_public.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_in_a_singleton_class_body_stays_public.snap new file mode 100644 index 000000000..0db3ecd0e --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_in_a_singleton_class_body_stays_public.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 2, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 1.0, + "interface_coa": 0.0, + "total": 2, + "total_methods": 2, + "coa": 1.0 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_is_not_a_public_method.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_is_not_a_public_method.snap new file mode 100644 index 000000000..18abc2dbe --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_initialize_is_not_a_public_method.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 1, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 0.5, + "interface_coa": 0.0, + "total": 1, + "total_methods": 2, + "coa": 0.5 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_keyword_wrapping_initialize_wins.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_keyword_wrapping_initialize_wins.snap new file mode 100644 index 000000000..0db3ecd0e --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_keyword_wrapping_initialize_wins.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 2, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 1.0, + "interface_coa": 0.0, + "total": 2, + "total_methods": 2, + "coa": 1.0 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_marker_does_not_republish_initialize.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_marker_does_not_republish_initialize.snap new file mode 100644 index 000000000..18abc2dbe --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_marker_does_not_republish_initialize.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 1, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 0.5, + "interface_coa": 0.0, + "total": 1, + "total_methods": 2, + "coa": 0.5 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_symbol_republishes_initialize.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_symbol_republishes_initialize.snap new file mode 100644 index 000000000..0db3ecd0e --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_public_symbol_republishes_initialize.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 2, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 1.0, + "interface_coa": 0.0, + "total": 2, + "total_methods": 2, + "coa": 1.0 +} diff --git a/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_singleton_initialize_stays_public.snap b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_singleton_initialize_stays_public.snap new file mode 100644 index 000000000..0db3ecd0e --- /dev/null +++ b/src/metrics/snapshots/big_code_analysis__metrics__npm__tests__ruby_singleton_initialize_stays_public.snap @@ -0,0 +1,15 @@ +--- +source: src/metrics/npm.rs +expression: metric.npm +--- +{ + "class_npm_sum": 2, + "interface_npm_sum": 0, + "class_methods": 2, + "interface_methods": 0, + "class_coa": 1.0, + "interface_coa": 0.0, + "total": 2, + "total_methods": 2, + "coa": 1.0 +} From 64c91965f4ef751f3c2a12c112ce2524f98b509d Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 17:42:32 -0700 Subject: [PATCH 08/22] fix(checker): tell a Tcl script body from a braced literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `braced_word` is both the literal of `lappend x {a b}` and the script of a `proc` body or an iRules `when` handler, and `Checker::is_string` is a kind table that cannot separate them. So `bca find --type string` reported every script body in the file as a string literal, and `Alterator::alterate` flattened one into a single leaf, dropping the whole body from the AST dump and the REST `/ast` endpoint. #1318 built the predicate that settles this but could not apply it: `is_string` takes neither `code` nor `ancestors`. Add `Checker::is_string_with_code`, a `_with_code` sibling with a forwarding default, and an `Alterator::keeps_children` veto for the dump. Both resolve the role through one shared `Getter::is_braced_script_word` over one kinds table per dialect, hoisted into `lang_helpers` where the three classifiers that ask it can share it (grammar-dispatch §7). Both questions are about an enclosing command, so both need an ancestor chain, and `Node::parent` is `O(depth)` — asking it per node made the walks quadratic in nesting depth. `find`, `count` and the dump walk now thread the chain they already had the information to build, which is what `parser.rs` had recorded as the fix since #1162. On 8 KB of nested braces, measured against an unknown chain: `count --type string` 7.4 ms against 434 ms, `Ast::dump` 10 ms against 418 ms, and both growth curves linear rather than quadratic. `Filter::any` and `Alterator::get_ast_node` take the chain as a new argument. Neither is covered by the stability contract — STABILITY.md places the whole `big-code-analysis-ast` crate outside it — and the root crate re-exports neither. Fixes #1381 --- big-code-analysis-ast/src/alterator.rs | 58 +++++- big-code-analysis-ast/src/ast.rs | 19 ++ big-code-analysis-ast/src/checker.rs | 173 ++++++++++++++++++ big-code-analysis-ast/src/checker/irules.rs | 15 ++ big-code-analysis-ast/src/checker/tcl.rs | 18 ++ big-code-analysis-ast/src/count.rs | 16 +- big-code-analysis-ast/src/find.rs | 24 ++- big-code-analysis-ast/src/getter.rs | 31 ++++ big-code-analysis-ast/src/getter/irules.rs | 15 +- big-code-analysis-ast/src/getter/tcl.rs | 20 +- big-code-analysis-ast/src/lang_helpers.rs | 3 + .../src/lang_helpers/irules.rs | 18 ++ big-code-analysis-ast/src/lang_helpers/tcl.rs | 30 ++- big-code-analysis-ast/src/parser.rs | 108 +++++++---- tests/api/ast_seam_test.rs | 130 +++++++++++++ tests/grammars/alterator_string_flattening.rs | 23 ++- 16 files changed, 612 insertions(+), 89 deletions(-) create mode 100644 big-code-analysis-ast/src/lang_helpers/irules.rs diff --git a/big-code-analysis-ast/src/alterator.rs b/big-code-analysis-ast/src/alterator.rs index 8903d179d..b8c27de3a 100644 --- a/big-code-analysis-ast/src/alterator.rs +++ b/big-code-analysis-ast/src/alterator.rs @@ -10,6 +10,8 @@ //! language implements to reshape a node before it is rendered as an //! [`AstNode`]. +use crate::lang_helpers::irules::BRACED_WORD_KINDS as IRULES_BRACED_WORD_KINDS; +use crate::lang_helpers::tcl::BRACED_WORD_KINDS as TCL_BRACED_WORD_KINDS; use crate::*; /// A trait to create a richer `AST` node for a programming language, mainly @@ -83,24 +85,54 @@ where AstNode::with_field_name(node.kind(), text, span, field_name, children) } + /// Whether `node` must keep its children even though + /// [`Self::alterate`] would flatten it into a verbatim leaf. + /// + /// The flattening arms are keyed on node *kind*, which is the right + /// question for a language whose string literals have a kind of + /// their own. The Tcl family is the exception: `braced_word` is both + /// the literal of `lappend x {a b}` and the *script* of a `proc` + /// body or an iRules `when` handler, so flattening by kind dropped + /// every script body from the dump — the whole body collapsed into + /// one leaf holding its text (#1381). + /// + /// It is a veto rather than an extra `alterate` arm so that + /// `alterate` keeps its byte-and-kind signature: only this question + /// needs the ancestor chain, and threading it through all twenty-odd + /// `alterate` impls to serve two of them would make every language + /// pay for one grammar's ambiguity. `ancestors` is the chain the + /// dump walk descended through, for [`Node::parent`]'s `O(depth)` + /// reason (#1084) — off an unknown chain this question took + /// `Ast::dump` on 8 KB of nested Tcl braces from 7 ms to 391 ms. + /// + #[inline] + #[must_use] + fn keeps_children<'a>(_node: &Node<'a>, _code: &[u8], _ancestors: Ancestors<'a, '_>) -> bool { + false + } + /// Gets a new `AST` node if and only if the code is not a comment, /// otherwise [`None`] is returned. /// /// Parameter order mirrors [`Self::alterate`] and [`Self::get_default`] /// (the flags-before-data convention `span, comment, field_name, /// children`) so positional confusion between adjacent boolean - /// toggles is harder to introduce on the next edit. + /// toggles is harder to introduce on the next edit. `ancestors` is + /// last, matching every other chain-taking predicate in the crate. #[must_use] - fn get_ast_node( - node: &Node, + fn get_ast_node<'a>( + node: &Node<'a>, code: &[u8], span: bool, comment: bool, field_name: Option<&'static str>, children: Vec, + ancestors: Ancestors<'a, '_>, ) -> Option { if comment && Self::is_comment(node) { None + } else if Self::keeps_children(node, code, ancestors) { + Some(Self::get_default(node, code, span, field_name, children)) } else { Some(Self::alterate(node, code, span, field_name, children)) } @@ -591,6 +623,13 @@ impl Alterator for TclCode { ) -> AstNode { match Tcl::from(node.kind_id()) { // Preserve string literals verbatim to avoid whitespace trimming. + // `BracedWord` is listed for the braced *value* of + // `lappend x {a b}` — Tcl evaluates nothing between its + // braces, so the value is its whole span. The same kind is + // also every script body, which must keep its children; + // `keeps_children` below vetoes this arm for those, because + // only the enclosing command separates the two roles and + // that question needs the ancestor chain (#1381). Tcl::QuotedWord | Tcl::BracedWord | Tcl::BracedWordSimple => { let (text, span) = Self::get_text_span(node, code, span, true); AstNode::with_field_name(node.kind(), text, span, field_name, Vec::new()) @@ -598,6 +637,10 @@ impl Alterator for TclCode { _ => Self::get_default(node, code, span, field_name, children), } } + + fn keeps_children<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + ::is_braced_script_word(node, code, ancestors, &TCL_BRACED_WORD_KINDS) + } } impl Alterator for IrulesCode { @@ -610,6 +653,11 @@ impl Alterator for IrulesCode { ) -> AstNode { match Irules::from(node.kind_id()) { // Preserve string literals verbatim to avoid whitespace trimming. + // The twin of the Tcl arm above, `keeps_children` veto and + // all. It matters more here: a `when` handler's body is the + // whole of a typical iRules file, so flattening every one of + // them left the dump with a single leaf per handler and no + // structure at all (#1381). Irules::QuotedWord | Irules::BracedWord | Irules::BracedWordSimple => { let (text, span) = Self::get_text_span(node, code, span, true); AstNode::with_field_name(node.kind(), text, span, field_name, Vec::new()) @@ -617,6 +665,10 @@ impl Alterator for IrulesCode { _ => Self::get_default(node, code, span, field_name, children), } } + + fn keeps_children<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + ::is_braced_script_word(node, code, ancestors, &IRULES_BRACED_WORD_KINDS) + } } impl Alterator for PhpCode { diff --git a/big-code-analysis-ast/src/ast.rs b/big-code-analysis-ast/src/ast.rs index 0e1621ce0..a5864063a 100644 --- a/big-code-analysis-ast/src/ast.rs +++ b/big-code-analysis-ast/src/ast.rs @@ -262,6 +262,22 @@ fn build(parser: &T, span: bool, comment: bool) -> Option> = vec![root]; loop { let frame = stack @@ -288,10 +304,12 @@ fn build(parser: &T, span: bool, comment: bool) -> Option(parser: &T, span: bool, comment: bool) -> Option parent.children.push(ast), diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 6797bd0b1..1eaa07084 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -425,11 +425,47 @@ pub trait Checker { } /// Whether `node` is a string literal, under every aliased kind the /// grammar emits for one. + /// + /// This is the *kind table* half of the answer. Where a grammar + /// spells a string literal and something else with one kind, the + /// table cannot separate them and the walk must ask + /// [`is_string_with_code`](Self::is_string_with_code) instead. #[inline] #[must_use] fn is_string(_: &Node) -> bool { false } + /// Source-aware variant of [`is_string`](Self::is_string), and the + /// spelling every walk calls. + /// + /// The default forwards to the byte-less predicate, so a language + /// whose string literals are a closed set of kinds needs no + /// override. The Tcl family is the case that does: `braced_word` is + /// both the literal of `lappend x {a b}` and the *script* of a + /// `proc` body or an iRules `when` handler, and only the enclosing + /// command's leading word tells the two apart (#1318, #1381). So + /// `bca find --type string` reported every script body in the file + /// as a string literal. + /// + /// **A call site reaching for the byte-less spelling reads as + /// correct against every language without an override, and is wrong + /// only on the one that has one** — + /// `.claude/rules/grammar-dispatch.md` §7. There is one call site, + /// the `"string"` arm of [`ParserTrait::filters`], and it takes this + /// spelling; `tcl_script_body_is_a_string_only_to_the_byteless_spelling` + /// pins that the two really do disagree, so a future call site that + /// picks the wrong one is not silently right. + /// + /// [`ParserTrait::filters`]: crate::traits::ParserTrait::filters + #[inline] + #[must_use] + fn is_string_with_code<'a>( + node: &Node<'a>, + _code: &[u8], + _ancestors: Ancestors<'a, '_>, + ) -> bool { + Self::is_string(node) + } /// Whether `node` is the `if` that continues an `else if` chain, /// rather than a freshly nested branch. /// @@ -2555,4 +2591,141 @@ mod tests { not add a fourth, which is the answer the parent lookup decides" ); } + + /// The verdicts the byte-less and the source-aware `is_string` give + /// the `braced_word`-kind nodes of `code`, as their source texts, + /// plus how many string-kind nodes of *any other* kind were seen. + /// + /// Returns `(kept_literal, withdrawn, other_strings)`. Texts rather + /// than counts because a count is polarity-blind: `(1, 1)` holds + /// just as well when the predicate calls the `proc` body a literal + /// and `{a b}` a script, which is the claim inverted. `other_strings` + /// anchors the control the third fixture line supplies — without it + /// the every-other-kind assertion below can be deleted from the + /// fixture and the kind-gate perturbation stops failing anything. + /// + /// Also asserts the two spellings agree off a known chain and off a + /// `Node::parent` climb: the walk now threads a chain, and the + /// answer must not depend on which it gets. + fn braced_word_string_verdicts( + label: &str, + code: &[u8], + script_kind: u16, + ) -> (Vec, Vec, usize) { + let (mut kept, mut withdrawn, mut other) = (Vec::new(), Vec::new(), 0); + for_each_node_with_chain::(code, |node, chain| { + let known = L::is_string_with_code(node, code, Ancestors::known(chain)); + assert_eq!( + known, + L::is_string_with_code(node, code, Ancestors::unknown()), + "{label}: is_string_with_code disagrees between a known chain and a \ + parent climb on {} at row {}", + node.kind(), + node.start_row() + ); + if node.kind_id() != script_kind { + // Every other kind must answer identically, or the + // override has widened past the one ambiguous kind. + assert_eq!( + known, + L::is_string(node), + "{label}: the override moved a {} node at row {}", + node.kind(), + node.start_row() + ); + other += usize::from(known); + return; + } + assert!( + L::is_string(node), + "{label}: the kind table must still list the script kind, or this \ + test is measuring its absence rather than the override" + ); + let text = String::from_utf8_lossy(&code[node.start_byte()..node.end_byte()]); + if known { + kept.push(text.into_owned()); + } else { + withdrawn.push(text.into_owned()); + } + }); + (kept, withdrawn, other) + } + + /// A Tcl-family script body is a string literal to + /// [`Checker::is_string`] and not to + /// [`Checker::is_string_with_code`] (#1381). + /// + /// This is the test the end-to-end `bca find` ones cannot be: they + /// observe the *result*, which is equally consistent with the + /// override never firing and the byte-less default happening to be + /// right. Asserting the disagreement directly is what makes a future + /// call site that reaches for the byte-less spelling a wrong answer + /// rather than an indistinguishable one + /// (`.claude/rules/grammar-dispatch.md` §7). + /// + /// Each fixture holds a braced value, a script body reached through + /// a *modelled* construct, a script body reached through the + /// **command-name** list (`eval`), and a non-braced string literal — + /// four rows that no constant and no inverted polarity satisfies. + /// The two script routes are structurally independent + /// (grammar-dispatch §11): `is_value_braced_word` answers "script" + /// either because the word fills a modelled slot or because the + /// enclosing command is in `SCRIPT_TAKING_COMMANDS`, and a fixture + /// exercising only the first leaves the second dead. + #[test] + #[cfg(any(feature = "tcl", feature = "irules"))] + fn tcl_script_body_is_a_string_only_to_the_byteless_spelling() { + let mut ran = 0; + #[cfg(feature = "tcl")] + { + ran += 1; + // `{x}` is the parameter list, which the grammar spells + // `arguments` rather than a braced word, so it is neither. + // `set s "q"` is the control for the kind gate: a + // `quoted_word` under a *modelled* command is a node the + // role predicate answers "not a value" for, so an override + // that dropped its `kind_id` test would unstring it — which + // the `other` count below is what notices. + let (kept, withdrawn, other) = braced_word_string_verdicts::( + "tcl", + b"proc p {x} { puts $x }\nlappend l {a b}\neval {puts hi}\nset s \"q\"\n", + Tcl::BracedWord as u16, + ); + assert_eq!(kept, ["{a b}"], "tcl: only the braced value is a literal"); + assert_eq!( + withdrawn, + ["{ puts $x }", "{puts hi}"], + "tcl: the proc body (a modelled slot) and the `eval` argument \ + (a named script-taking command) are both scripts" + ); + assert_eq!(other, 1, "tcl: the `set` quoted word stays a string"); + } + #[cfg(feature = "irules")] + { + ran += 1; + // The `"hi"` inside the handler body is this dialect's + // `other` row, so the control sits inside the construct + // under test rather than beside it. + let (kept, withdrawn, other) = braced_word_string_verdicts::( + "irules", + b"when HTTP_REQUEST { log local0. \"hi\" }\nlappend l {x y}\neval {puts hi}\n", + Irules::BracedWord as u16, + ); + assert_eq!( + kept, + ["{x y}"], + "irules: only the braced value is a literal" + ); + assert_eq!( + withdrawn, + ["{ log local0. \"hi\" }", "{puts hi}"], + "irules: the `when` handler body and the `eval` argument are scripts" + ); + assert_eq!(other, 1, "irules: the quoted word stays a string"); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); + } } diff --git a/big-code-analysis-ast/src/checker/irules.rs b/big-code-analysis-ast/src/checker/irules.rs index 62ff0cec1..b853281cb 100644 --- a/big-code-analysis-ast/src/checker/irules.rs +++ b/big-code-analysis-ast/src/checker/irules.rs @@ -2,6 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::irules::BRACED_WORD_KINDS; impl Checker for IrulesCode { fn is_comment(node: &Node) -> bool { @@ -51,8 +52,22 @@ impl Checker for IrulesCode { false } + // `BracedWord` is the literal of `lappend b {x y}` as well as every + // script body, for the reason the Tcl twin records; the kind table + // cannot separate the two roles and `is_string_with_code` does. impl_simple_is_string!(Irules, QuotedWord, BracedWord, BracedWordSimple); + // The twin of `TclCode::is_string_with_code` (#1381). This grammar + // models more script positions than Tcl's — `when`, `for`, `switch` + // and the `dict` loops each have a node of their own — so the + // handler bodies this rescues are recognised structurally rather + // than by command name, and a `when` body is a script whatever it + // is called. + fn is_string_with_code<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + Self::is_string(node) + && !::is_braced_script_word(node, code, ancestors, &BRACED_WORD_KINDS) + } + // iRules grammar has a dedicated `elseif` named node (id 145), not a // nested `if`. `Elseif2` (id 92) is the `elseif` keyword token, not // the clause, so it is intentionally excluded here (lesson #34 diff --git a/big-code-analysis-ast/src/checker/tcl.rs b/big-code-analysis-ast/src/checker/tcl.rs index 7c67e7ae3..0a065903e 100644 --- a/big-code-analysis-ast/src/checker/tcl.rs +++ b/big-code-analysis-ast/src/checker/tcl.rs @@ -2,6 +2,7 @@ #![allow(clippy::wildcard_imports, clippy::enum_glob_use)] use super::*; +use crate::lang_helpers::tcl::BRACED_WORD_KINDS; impl Checker for TclCode { fn is_comment(node: &Node) -> bool { @@ -30,8 +31,25 @@ impl Checker for TclCode { false } + // `BracedWord` is listed because it *is* the literal of + // `lappend x {a b}` — the grammar reserves `BracedWordSimple` for the + // handful of commands it models (`set`, `foreach`, `regexp`, …) and + // spells every other command's braced argument `BracedWord`. The same + // kind is also the `proc` / `if` body, which this table cannot + // exclude; `is_string_with_code` below does, and is what the walk + // calls. impl_simple_is_string!(Tcl, QuotedWord, BracedWord, BracedWordSimple); + // The half of the rule that needs the source bytes (#1381), the twin + // of `TclCode::get_op_type_with_code`. `is_value_braced_word` lives + // on `Getter`, and `TclCode` implements both traits, so the two + // classifiers answer from one predicate over one kinds table rather + // than from two copies (grammar-dispatch §7). + fn is_string_with_code<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { + Self::is_string(node) + && !::is_braced_script_word(node, code, ancestors, &BRACED_WORD_KINDS) + } + // Tcl grammar has a dedicated `elseif` named node, not a nested `if`. impl_is_else_if_clause!(Tcl, Elseif); } diff --git a/big-code-analysis-ast/src/count.rs b/big-code-analysis-ast/src/count.rs index 914057542..c68184593 100644 --- a/big-code-analysis-ast/src/count.rs +++ b/big-code-analysis-ast/src/count.rs @@ -23,6 +23,7 @@ use num_format::{Locale, ToFormattedString}; use std::fmt; use std::sync::{Arc, Mutex}; +use crate::node::Ancestors; use crate::traits::ParserTrait; /// Counts the types of nodes specified in the input slice and the @@ -35,18 +36,25 @@ pub fn count(parser: &T, filters: &[String]) -> (usize, usize) { let mut stack = Vec::new(); let mut good = 0; let mut total = 0; + // See `find` for why the chain is threaded rather than climbed. The + // truncate/push discipline is order-independent — a node's ancestors + // are always the chain prefix at its own depth — so it holds for + // this walk's unordered child push too. + let mut chain = Vec::new(); - stack.push(node); + stack.push((node, 0_usize)); - while let Some(node) = stack.pop() { + while let Some((node, depth)) = stack.pop() { total += 1; - if filters.any(&node) { + chain.truncate(depth); + if filters.any(&node, Ancestors::checked(&chain, &node)) { good += 1; } + chain.push(node); // No reversal: this walk only tallies, so visit order is // immaterial and imposing one would imply a guarantee no caller // relies on. Matches the previous push-in-source-order form. - stack.extend(node.children_with(&mut cursor)); + stack.extend(node.children_with(&mut cursor).map(|c| (c, depth + 1))); } (good, total) } diff --git a/big-code-analysis-ast/src/find.rs b/big-code-analysis-ast/src/find.rs index 4c4b7b359..73474c44d 100644 --- a/big-code-analysis-ast/src/find.rs +++ b/big-code-analysis-ast/src/find.rs @@ -8,7 +8,7 @@ //! Node finding by kind or category. -use crate::node::Node; +use crate::node::{Ancestors, Node}; use crate::error::MetricsError; use crate::traits::ParserTrait; @@ -41,18 +41,32 @@ pub fn find<'a, T: ParserTrait>( let mut cursor = node.cursor(); let mut stack = Vec::new(); let mut good = Vec::new(); + // The ancestry of whichever node is about to be popped. A predicate + // that asks about an enclosing construct — `"function"` for the + // JS family and Elixir, `"string"` for the Tcl family — would + // otherwise climb with `Node::parent`, which restarts at the root + // and so costs `O(depth)` *per lookup*: `bca find --type string` + // over 8 KB of nested Tcl braces took 823 ms that way and 5 ms with + // the chain (#1381, the shape of #1052 / #1122). + // + // Maintained exactly as `spaces::compute::metrics_inner` does — + // truncate to the node's depth before the visit, push after it — + // so `Ancestors::checked` has the same meaning here as there. + let mut chain: Vec> = Vec::new(); - stack.push(node); + stack.push((node, 0_usize)); - while let Some(node) = stack.pop() { - if filters.any(&node) { + while let Some((node, depth)) = stack.pop() { + chain.truncate(depth); + if filters.any(&node, Ancestors::checked(&chain, &node)) { good.push(node); } + chain.push(node); // Source order in, tail reversed in place, so the LIFO `stack` // yields the leftmost child first — matches were already // returned in source order and must stay that way. let first_child = stack.len(); - stack.extend(node.children_with(&mut cursor)); + stack.extend(node.children_with(&mut cursor).map(|c| (c, depth + 1))); stack[first_child..].reverse(); } Ok(good) diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index f2443e713..d43b50ab2 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -738,6 +738,37 @@ pub trait Getter { && Self::is_switch_arm_body(word, &command)) } + /// Whether `node` is a Tcl-family braced *script* — a `proc` or `if` + /// body, an iRules `when` handler — rather than a braced literal + /// (#1381). + /// + /// The positive form of [`is_value_braced_word`], narrowed to the + /// script kind so it answers `false` for every node of every other + /// kind and for the two literal spellings. That makes it the + /// question the *non-Halstead* classifiers ask: + /// `Checker::is_string_with_code` must not call a `proc` body a + /// string literal, and `Alterator::alterate` must not flatten one + /// into a leaf, dropping the body from the AST dump. Both once + /// listed `braced_word` beside `quoted_word` and + /// `braced_word_simple`, which is right for the literal role and + /// wrong for the script role the same kind also serves. + /// + /// Stated here rather than in each of the four call sites so the + /// three classifiers cannot drift apart on the same bytes + /// (grammar-dispatch §7) — the drift `braced_word_op_type` opened + /// when it revised the operator half alone. + /// + /// [`is_value_braced_word`]: Self::is_value_braced_word + #[must_use] + fn is_braced_script_word<'a>( + node: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + kinds: &BracedWordKinds, + ) -> bool { + node.kind_id() == kinds.script && !Self::is_value_braced_word(node, code, ancestors, kinds) + } + /// Whether `word`, an argument of a `switch` arm command /// ([`is_switch_arm`]), sits in a *body* position rather than a /// *pattern* position. diff --git a/big-code-analysis-ast/src/getter/irules.rs b/big-code-analysis-ast/src/getter/irules.rs index b341a1805..317705785 100644 --- a/big-code-analysis-ast/src/getter/irules.rs +++ b/big-code-analysis-ast/src/getter/irules.rs @@ -3,20 +3,7 @@ use super::*; -/// The braced-word kinds `Getter::is_subsumed_braced_word` and -/// `Getter::braced_word_op_type` are instantiated with (#1354, #1318) — -/// the twin of the Tcl table, at this grammar's own id block. Every -/// field means what the Tcl one documents; only the ids differ. -const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { - value: Irules::BracedWordSimple as u16, - script: Irules::BracedWord as u16, - comment: Irules::Comment as u16, - command: Irules::Command as u16, - word_list: Irules::WordList as u16, - simple_word: Irules::SimpleWord as u16, - argument: Irules::Argument as u16, - open_brace: Irules::LBRACE as u16, -}; +use crate::lang_helpers::irules::BRACED_WORD_KINDS; impl Getter for IrulesCode { fn get_space_kind(node: &Node) -> SpaceKind { diff --git a/big-code-analysis-ast/src/getter/tcl.rs b/big-code-analysis-ast/src/getter/tcl.rs index 921df714c..5fb9cbaa0 100644 --- a/big-code-analysis-ast/src/getter/tcl.rs +++ b/big-code-analysis-ast/src/getter/tcl.rs @@ -3,25 +3,7 @@ use super::*; -/// The braced-word kinds `Getter::is_subsumed_braced_word` and -/// `Getter::braced_word_op_type` are instantiated with (#1354, #1318): -/// the literal *value* form the guard keys on, the *script* form it -/// gates on holding a command, the comment kind that gate must not -/// mistake for one, and the four kinds #1318's role recognition walks — -/// the generic `command`, its `word_list` argument list, the -/// `simple_word` a resolvable command name is spelled with, and the -/// `argument` whose braced child is a parameter default rather than a -/// script. -const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { - value: Tcl::BracedWordSimple as u16, - script: Tcl::BracedWord as u16, - comment: Tcl::Comment as u16, - command: Tcl::Command as u16, - word_list: Tcl::WordList as u16, - simple_word: Tcl::SimpleWord as u16, - argument: Tcl::Argument as u16, - open_brace: Tcl::LBRACE as u16, -}; +use crate::lang_helpers::tcl::BRACED_WORD_KINDS; impl Getter for TclCode { fn get_space_kind(node: &Node) -> SpaceKind { diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index 2c42f98c0..e16019936 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -14,5 +14,8 @@ //! on the metric layer, the inversion #1376 exists to remove. pub mod elixir; +// Crate-private: this dialect's only helper is a kind table the three +// classifiers in this crate share, and nothing outside names it. +pub(crate) mod irules; pub mod python; pub mod tcl; diff --git a/big-code-analysis-ast/src/lang_helpers/irules.rs b/big-code-analysis-ast/src/lang_helpers/irules.rs new file mode 100644 index 000000000..f4403cf41 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/irules.rs @@ -0,0 +1,18 @@ +//! iRules: the grammar's braced-word kind ids. + +use crate::Irules; +use crate::getter::BracedWordKinds; + +/// The twin of [`crate::lang_helpers::tcl::BRACED_WORD_KINDS`], at this +/// grammar's own id block. Every field means what the Tcl table +/// documents; only the ids differ. +pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { + value: Irules::BracedWordSimple as u16, + script: Irules::BracedWord as u16, + comment: Irules::Comment as u16, + command: Irules::Command as u16, + word_list: Irules::WordList as u16, + simple_word: Irules::SimpleWord as u16, + argument: Irules::Argument as u16, + open_brace: Irules::LBRACE as u16, +}; diff --git a/big-code-analysis-ast/src/lang_helpers/tcl.rs b/big-code-analysis-ast/src/lang_helpers/tcl.rs index b6125a17e..07693a816 100644 --- a/big-code-analysis-ast/src/lang_helpers/tcl.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl.rs @@ -1,8 +1,36 @@ -//! Tcl: the leading word of a `command` node. +//! Tcl: the leading word of a `command` node, and the grammar's +//! braced-word kind ids. use crate::Tcl; +use crate::getter::BracedWordKinds; use crate::node::Node; +/// The braced-word kinds `Getter::is_subsumed_braced_word`, +/// `Getter::braced_word_op_type` and `Getter::is_braced_script_word` are +/// instantiated with (#1354, #1318): the literal *value* form the guard +/// keys on, the *script* form it gates on holding a command, the comment +/// kind that gate must not mistake for one, and the four kinds #1318's +/// role recognition walks — the generic `command`, its `word_list` +/// argument list, the `simple_word` a resolvable command name is spelled +/// with, and the `argument` whose braced child is a parameter default +/// rather than a script. +/// +/// It lives here rather than beside the `Getter` impl because three +/// classifiers now read it — `Getter::get_op_type_with_code`, +/// `Checker::is_string_with_code` and `Alterator::alterate` — and a +/// second copy is exactly the drift `lang_helpers` exists to prevent +/// (#1381). +pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { + value: Tcl::BracedWordSimple as u16, + script: Tcl::BracedWord as u16, + comment: Tcl::Comment as u16, + command: Tcl::Command as u16, + word_list: Tcl::WordList as u16, + simple_word: Tcl::SimpleWord as u16, + argument: Tcl::Argument as u16, + open_brace: Tcl::LBRACE as u16, +}; + /// Reads the leading word of a Tcl `command` node when it is a plain /// `simple_word` (`switch`, `for`, `puts`, …). Returns `None` for any other /// node kind, for commands whose leading word is computed (`$cmd`, `[cmd]` diff --git a/big-code-analysis-ast/src/parser.rs b/big-code-analysis-ast/src/parser.rs index 8e7e144ab..816985d4c 100644 --- a/big-code-analysis-ast/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -38,10 +38,19 @@ pub struct Parser { } /// A single node-matching predicate. The `'a` bound lets a predicate -/// borrow the parser's source buffer, which the `"function"` filter -/// needs to answer for a language whose function declarations are -/// identified by their text rather than their kind (#1162). -type FilterFn<'a> = dyn Fn(&Node) -> bool + 'a; +/// borrow the parser's source buffer, which the `"function"` and +/// `"string"` filters need to answer for a language whose functions or +/// string literals are identified by their text rather than their kind +/// (#1162, #1381). +/// +/// The `Ancestors` argument is the chain the caller descended through. +/// Both of those predicates ask about an enclosing construct, and +/// resolving one from the node alone costs [`Node::parent`]'s +/// `O(depth)` *per lookup* — which over a whole walk is the quadratic +/// #1052 and #1122 warn about. The higher-ranked bound ties the chain's +/// tree lifetime to the node's, so a predicate cannot be handed the +/// ancestry of a different tree. +type FilterFn<'a> = dyn for<'t, 'c> Fn(&Node<'t>, Ancestors<'t, 'c>) -> bool + 'a; /// Collection of node-matching predicates used by the AST-walking /// metric and dump routines to decide whether to visit a node. @@ -50,11 +59,19 @@ pub struct Filter<'a> { } impl Filter<'_> { - /// Returns `true` if *any* of the configured predicates matches `node`. + /// Returns `true` if *any* of the configured predicates matches + /// `node`, reached through `ancestors`. + /// + /// A walker that maintains an ancestor chain should pass it. The + /// `"function"` and `"string"` predicates ask about an enclosing + /// construct, and off [`Ancestors::unknown`] each such lookup is + /// [`Node::parent`]'s `O(depth)` — which made `count --type string` + /// quadratic in nesting depth until `find` and `count` threaded a + /// real chain (#1381). #[must_use] - pub fn any(&self, node: &Node) -> bool { + pub fn any<'t>(&self, node: &Node<'t>, ancestors: Ancestors<'t, '_>) -> bool { for f in &self.filters { - if f(node) { + if f(node, ancestors) { return true; } } @@ -116,41 +133,52 @@ impl ParserTrait for P } fn filters(&self, requested: &[String]) -> Filter<'_> { - // Borrowed by the `"function"` arm below, which is why `Filter` - // carries a lifetime. + // Borrowed by the `"function"` and `"string"` arms below, which + // is why `Filter` carries a lifetime. let code = self.code(); let mut res: Vec>> = Vec::new(); for f in requested { let f = f.as_str(); match f { - "all" => res.push(Box::new(|_: &Node| -> bool { true })), - // `is_call` / `is_comment` / `is_error` / `is_string` - // take `&Node` and nothing else, so no language *can* - // make them text-dependent. The #1162 gap is confined by - // construction to the three `Checker` predicates that - // accept `code`, and `"function"` is the only filter - // that reaches one. - "call" => res.push(Box::new(T::is_call)), - "comment" => res.push(Box::new(T::is_comment)), - "error" => res.push(Box::new(T::is_error)), - "string" => res.push(Box::new(T::is_string)), - // `Ancestors::unknown()`: a `--filter` predicate is applied - // to nodes the dump walk reaches without a chain. The - // JS-family `is_func` and Elixir's `is_func_with_code` - // consult one, and both answer the same either way — only - // the cost differs (#1088, #1162). + "all" => res.push(Box::new(|_: &Node, _| -> bool { true })), + // `is_call` / `is_comment` / `is_error` take `&Node` and + // nothing else, so no language *can* make them + // text-dependent. The #1162 gap is confined by + // construction to the `Checker` predicates that accept + // `code`, and `"function"` and `"string"` are the two + // filters that reach one. + "call" => res.push(Box::new(|node: &Node, _| T::is_call(node))), + "comment" => res.push(Box::new(|node: &Node, _| T::is_comment(node))), + "error" => res.push(Box::new(|node: &Node, _| T::is_error(node))), + // `is_string_with_code`, not `is_string`: a Tcl-family + // `braced_word` is a string literal in a value position + // and a `proc` / `when` body everywhere else, and only + // the bytes of the enclosing command's leading word + // separate the two (#1381). This arm is the only caller + // of either spelling in the workspace, so the byte-less + // one now serves purely as the per-language kind table + // that each override narrows. // - // That cost is `O(depth^2)` per *candidate* node, not - // `O(depth)`: an unknown chain climbs by `Node::parent`, - // which is itself `O(depth)` per step. It stays off the - // general walk because each predicate rejects on - // `kind_id` first — Elixir climbs only for a `Call` - // already spelling `def`/`defp`/`defmacro`. Giving - // `find`/`count` the `(node, depth)` stack `act_on_node` - // already carries would supply a known chain and drop - // this to `O(depth)`. - "function" => res.push(Box::new(move |node: &Node| { - T::is_func_with_code(node, code, Ancestors::unknown()) + // This is also the arm that makes the chain load-bearing + // rather than merely cheaper. `braced_word` is *every* + // Tcl block and list literal, so the candidate set is + // dense, and the Tcl override asks about the enclosing + // command — roughly ten ancestor lookups per candidate. + // Off an unknown chain each is `Node::parent`'s + // `O(depth)`, which took `bca find --type string` on 8 KB + // of nested braces from 5 ms to 823 ms before the chain + // was threaded here. + "string" => res.push(Box::new(move |node: &Node, ancestors| { + T::is_string_with_code(node, code, ancestors) + })), + // The JS-family `is_func` and Elixir's `is_func_with_code` + // consult the chain to tell a named function from a + // closure and a `def` `Call` from any other (#1088, + // #1162). Both answer the same off an unknown chain — + // only the cost differs, and `find` / `count` now supply + // a real one. + "function" => res.push(Box::new(move |node: &Node, ancestors| { + T::is_func_with_code(node, code, ancestors) })), _ => { if let Ok(n) = f.parse::() { @@ -165,7 +193,9 @@ impl ParserTrait for P // the end/ERROR sentinel. Documented as unstable in // big-code-analysis-book/src/commands/nodes.md; the // string (`kind()`) path below is the supported one. - res.push(Box::new(move |node: &Node| -> bool { node.kind_id() == n })); + res.push(Box::new(move |node: &Node, _| -> bool { + node.kind_id() == n + })); } else { // Exact match on `node.kind()` — the CLI documents // `find ` / `count ` as searching @@ -173,13 +203,13 @@ impl ParserTrait for P // big-code-analysis-book/src/commands/nodes.md and // issue #293). let f = f.to_owned(); - res.push(Box::new(move |node: &Node| -> bool { node.kind() == f })); + res.push(Box::new(move |node: &Node, _| -> bool { node.kind() == f })); } } } } if res.is_empty() { - res.push(Box::new(|_: &Node| -> bool { true })); + res.push(Box::new(|_: &Node, _| -> bool { true })); } Filter { filters: res } diff --git a/tests/api/ast_seam_test.rs b/tests/api/ast_seam_test.rs index f7416f12c..48450c7c1 100644 --- a/tests/api/ast_seam_test.rs +++ b/tests/api/ast_seam_test.rs @@ -38,6 +38,8 @@ use big_code_analysis::SpaceKind; feature = "rust", feature = "python", feature = "cpp", + feature = "tcl", + feature = "irules", not(feature = "javascript") ))] use big_code_analysis::{LANG, Source}; @@ -733,3 +735,131 @@ fn preprocess_harvest_feeds_the_macro_masking_pass() { Ast::parse(Source::from_bytes(LANG::Cpp, source.to_vec())).expect("cpp feature enabled"); assert_eq!(untouched.source(), source); } + +/// The texts `Ast::find` reports for `--type string`, in source order. +/// +/// Returning the texts rather than a count is what lets the callers below +/// assert *which* nodes were reported. A count alone cannot tell "the +/// script body dropped out" from "the literal dropped out and something +/// else appeared", and the two failures want opposite fixes. +/// A `proc` body (the script) holding a quoted word, plus a braced word +/// (the two literals). Shared by both tests below so the `find` list and +/// the `count` total describe the same bytes. +/// +/// The quoted word sits **inside** the body deliberately. With it +/// outside, deleting the whole `proc` line left the expected list +/// unchanged and the fixture could lose its entire subject with no test +/// failing (`.claude/rules/testing.md`, "Perturb the fixture as well as +/// the production line"). Inside, the `proc` is load-bearing for the +/// expected sequence. +#[cfg(feature = "tcl")] +const TCL_SCRIPT_AND_LITERALS: &str = "proc p {x} { puts \"q\" }\nlappend l {a b}\n"; + +/// The iRules twin, which already had the quoted word inside the body. +#[cfg(feature = "irules")] +const IRULES_SCRIPT_AND_LITERALS: &str = + "when HTTP_REQUEST { log local0. \"hi\" }\nlappend l {x y}\n"; + +#[cfg(any(feature = "tcl", feature = "irules"))] +fn strings_found(lang: LANG, code: &str) -> Vec { + let ast = Ast::parse(Source::new(lang, code.as_bytes())).expect("language feature enabled"); + let source = ast.source(); + ast.find(&["string".to_owned()]) + .expect("find is infallible") + .iter() + .map(|node| { + std::str::from_utf8(&source[node.start_byte()..node.end_byte()]) + .expect("fixture is ASCII") + .to_owned() + }) + .collect() +} + +/// `bca find --type string` must not report a Tcl-family script body, +/// and must still report a braced literal (#1381). +/// +/// `braced_word` is both, and `Checker::is_string`'s kind table cannot +/// separate them, so before the fix every `proc` body and every iRules +/// `when` handler came back as a string literal. The `"string"` filter +/// now asks `is_string_with_code`, which resolves the role from the +/// enclosing command's leading word. +/// +/// Each fixture holds a script body *and* a literal, and the assertion is +/// on the exact reported list rather than on the body's absence: an +/// absence assertion also passes when the whole filter stops matching, +/// which is the over-correction this rule invites +/// (`.claude/rules/testing.md`). +#[cfg(any(feature = "tcl", feature = "irules"))] +#[test] +fn find_string_reports_tcl_family_literals_and_not_script_bodies() { + let mut ran = 0; + #[cfg(feature = "tcl")] + { + ran += 1; + // `{ puts "q" }` is the proc body and must not appear; `"q"` + // (inside it) and `{a b}` are the literals. The quoted word + // anchors the list on a second kind, so a rule that suppressed + // *every* braced word would still fail here rather than quietly + // reduce the test to one assertion — and, sitting inside the + // body, it also proves dropping the body did not drop its + // contents. + assert_eq!( + strings_found(LANG::Tcl, TCL_SCRIPT_AND_LITERALS), + vec!["\"q\"", "{a b}"], + ); + } + #[cfg(feature = "irules")] + { + ran += 1; + // The iRules twin: the `when` handler body is the script, the + // `"hi"` inside it and `{x y}` are the literals. The quoted word + // sits *inside* the body, so it also pins that dropping the body + // did not drop its contents with it. + assert_eq!( + strings_found(LANG::Irules, IRULES_SCRIPT_AND_LITERALS), + vec!["\"hi\"", "{x y}"], + ); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); +} + +/// `bca count --type string` reads the same `Filter`, so it must report +/// the number of nodes `find` returns (#1381). +/// +/// The two walks are separate functions over one predicate list, and only +/// one of them is exercised above. +#[cfg(any(feature = "tcl", feature = "irules"))] +#[test] +fn count_string_agrees_with_find_on_tcl_family_bodies() { + let mut ran = 0; + for (lang, code) in [ + #[cfg(feature = "tcl")] + (LANG::Tcl, TCL_SCRIPT_AND_LITERALS), + #[cfg(feature = "irules")] + (LANG::Irules, IRULES_SCRIPT_AND_LITERALS), + ] { + ran += 1; + let found = strings_found(lang, code).len(); + assert_eq!(found, 2, "{lang:?}: fixture must report both literals"); + let (matching, total) = Ast::parse(Source::new(lang, code.as_bytes())) + .expect("language feature enabled") + .count(&["string".to_owned()]); + assert_eq!(matching, found, "{lang:?}: count and find disagree"); + // The script body is among the nodes `total` counts and + // `matching` does not, so this is the file-level shape of the + // same claim rather than the `total > matching` truism (which + // holds for any fixture with more than two nodes). + assert!( + total > 20, + "{lang:?}: {total} nodes is too few for the fixture to still \ + contain a script body" + ); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); +} diff --git a/tests/grammars/alterator_string_flattening.rs b/tests/grammars/alterator_string_flattening.rs index 66d168e23..77c4fc681 100644 --- a/tests/grammars/alterator_string_flattening.rs +++ b/tests/grammars/alterator_string_flattening.rs @@ -73,10 +73,25 @@ flatten_cases! { lua_flattens_string_literal: LANG::Lua, "local s = \"hi\"", "f.lua", "\"hi\""; tcl_flattens_quoted_word: LANG::Tcl, "set s \"hi\"", "f.tcl", "\"hi\""; // In valid iRules a quoted word only appears inside an event handler's - // `{ … }` body, which `alterate` flattens as a single `braced_word` leaf - // (the same match arm that handles `quoted_word`), so the verbatim text - // to look for is the whole brace block. - irules_flattens_braced_word: LANG::Irules, "when HTTP_REQUEST { set s \"hi\" }", "f.irul", "{ set s \"hi\" }"; + // `{ … }` body. Until #1381 `alterate` flattened that body as a single + // `braced_word` leaf, so the only verbatim text in the whole dump was + // the brace block and the `quoted_word` arm was unreachable from any + // valid input. The handler body is a script now, so the arm this file + // is about is finally what this row tests — the same claim its Tcl + // twin above makes. + irules_flattens_quoted_word: LANG::Irules, "when HTTP_REQUEST { set s \"hi\" }", "f.irule", "\"hi\""; + // The Tcl half of the same claim. `tcl_flattens_quoted_word` above + // reaches the literal at statement level, where the body guard has + // nothing to do — verified by perturbation: deleting the Tcl guard + // failed no test until this row existed, while the iRules twin failed + // on its own because iRules has no statement level to test from. + tcl_flattens_quoted_word_inside_a_proc_body: LANG::Tcl, "proc p {} { puts \"hi\" }", "f.tcl", "\"hi\""; + // The braced *value* half of the same rule: `lappend`'s argument is a + // literal, so it keeps the flattening a script body gives up. Without + // it the #1381 guard would read as "braced words are never flattened", + // which is the opposite over-correction. + tcl_flattens_braced_value: LANG::Tcl, "lappend x {a b}\n", "f.tcl", "{a b}"; + irules_flattens_braced_value: LANG::Irules, "lappend b {x y}\n", "f.irule", "{x y}"; ruby_flattens_string_literal: LANG::Ruby, "s = \"hi\"\n", "f.rb", "\"hi\""; elixir_flattens_string_literal: LANG::Elixir, "s = \"hi\"\n", "f.ex", "\"hi\""; } From f64a1fe66868b1d5b9c96976ff2d8d56f991caa8 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 19:14:06 -0700 Subject: [PATCH 09/22] fix(getter): bill self- and super-references as operands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Java, C# and Kotlin swept `this` / `super` / `base` into their operator arm under a `// Operator: … keywords` heading, while the eleven other languages that classify a self-reference call it an operand. A member access is ` `, so billing the receiver as an operator made `this.x` a binary operator with one operand where `p.x` is one operator with two, and scored the same source differently on each side of a translation between two of these languages. Two uses of the same token kinds are not references and keep the operator classification, each behind a parent gate: C#'s `indexer_declaration` names the member with the `this` keyword, and Java's `? super String` wildcard bound mirrors `? extends String`, whose `extends` this same match already bills as an operator. C#'s extension-method receiver needs no gate — it is a childless `modifier` node, not the `this` kind at all, and remains unclassified as it was. Java's own declarator use goes the other way: the explicit receiver parameter `void m(T T.this)` is a parameter *name*, and parameter names are operands here. Each site now cross-references the other so the two opposite calls read as one decision. Kotlin's labelled spellings were not a reclassification at all. `ThisAT` / `SuperAT` were in neither arm before, so `this@Outer` and `super@Inner` contributed nothing — the silent-drop shape of #1361 rather than an operator-to-operand move — and their `n2` / `N2` rise with no offsetting `n1` / `N1` fall. The leaves are the keepers rather than the `this_expression` / `super_expression` wrappers, because a `constructor_delegation_call` emits a bare leaf with no wrapper at all. PHP's `Zelf` / `Parent` are untouched: `self::` / `parent::` are class references in scope-resolution position, a different construct from `$this`, which is a `variable_name` and already an operand. Fixes #1380 --- .rustfmt-bail-baseline.txt | 4 +- big-code-analysis-ast/src/getter/csharp.rs | 32 +- big-code-analysis-ast/src/getter/java.rs | 29 +- big-code-analysis-ast/src/getter/kotlin.rs | 24 +- src/metrics/halstead.rs | 465 +++++++++++++++++- tests/parity/main.rs | 21 + tests/parity/self_reference_operand_parity.rs | 220 +++++++++ 7 files changed, 779 insertions(+), 16 deletions(-) create mode 100644 tests/parity/self_reference_operand_parity.rs diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index c05424a45..efd96e14b 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -119,12 +119,12 @@ big-code-analysis-ast/src/getter.rs 3 big-code-analysis-ast/src/getter/bash.rs 3 big-code-analysis-ast/src/getter/c.rs 3 big-code-analysis-ast/src/getter/cpp.rs 5 -big-code-analysis-ast/src/getter/csharp.rs 5 +big-code-analysis-ast/src/getter/csharp.rs 8 big-code-analysis-ast/src/getter/elixir.rs 3 big-code-analysis-ast/src/getter/go.rs 1 big-code-analysis-ast/src/getter/groovy.rs 5 big-code-analysis-ast/src/getter/irules.rs 6 -big-code-analysis-ast/src/getter/java.rs 2 +big-code-analysis-ast/src/getter/java.rs 5 big-code-analysis-ast/src/getter/kotlin.rs 4 big-code-analysis-ast/src/getter/lua.rs 3 big-code-analysis-ast/src/getter/mozcpp.rs 5 diff --git a/big-code-analysis-ast/src/getter/csharp.rs b/big-code-analysis-ast/src/getter/csharp.rs index b13520227..ed6748b77 100644 --- a/big-code-analysis-ast/src/getter/csharp.rs +++ b/big-code-analysis-ast/src/getter/csharp.rs @@ -60,7 +60,7 @@ impl Getter for CsharpCode { | Volatile | Async | Required | File | New | Fixed | Implicit | Explicit // Expression-keyword operators | Await | Is | As | Typeof | Sizeof | Checked | Unchecked | Ref | Out | In - | Params | This | Base | Lock | Stackalloc | Where | With | When | Operator + | Params | Lock | Stackalloc | Where | With | When | Operator | Scoped | Not | And | Or // Property/event accessor keywords | Get | Set | Init | Add | Remove @@ -95,6 +95,27 @@ impl Getter for CsharpCode { Some(BooleanLiteral) => TokenRole::Unknown, _ => TokenRole::Operand, }, + // `this` is a self-reference everywhere (`this.x`, + // `this[i]`, `: this(1)`, `f(this)`) except directly under + // an `indexer_declaration`, where the keyword *names* the + // member being declared rather than denoting a value — + // `public int this[int i] { … }`. That position keeps its + // operator classification alongside the `operator` keyword + // of an overload declaration, which this match already + // bills as an operator (#1380). The extension-method + // receiver (`static void M(this Foo f)`) is a childless + // `modifier` node, kind 249, not this kind at all, so it is + // unreached here and stays unclassified as before. + // + // Java's receiver parameter (`void m(J J.this)`) is the + // other declarator use of the keyword in this workspace and + // `java.rs` calls it an operand — a parameter *name* is an + // operand in every language here, where a member named by a + // keyword belongs with `operator +`. + This => match ancestors.parent(node).map(|p| p.kind_id().into()) { + Some(IndexerDeclaration) => TokenRole::Operator, + _ => TokenRole::Operand, + }, // Operands: identifiers and literals. `NullLiteral` is a // childless leaf, so it needs no such guard. // @@ -108,7 +129,14 @@ impl Getter for CsharpCode { // (grammar-dispatch section 5). Probed with `bca ops`: // dropping them leaves `System`+`Text`+`.`, // `List`+`<`+`int`+`>`, and `global`+`::`+`Foo` intact. - Identifier + // + // `Base` joined this list in #1380 for the same structural + // reason `This` did: it is the receiver of `base.M()` / + // `base[i]` / `: base(x)`. The `base_list` container that + // spells inheritance (`class D : B`) holds the base type's + // identifier and no `base` keyword, so nothing bills this + // text twice. + Identifier | Base | IntegerLiteral | RealLiteral | BooleanLiteral | NullLiteral | CharacterLiteral | StringLiteral | VerbatimStringLiteral | RawStringLiteral => TokenRole::Operand, diff --git a/big-code-analysis-ast/src/getter/java.rs b/big-code-analysis-ast/src/getter/java.rs index efdf1966f..20e351671 100644 --- a/big-code-analysis-ast/src/getter/java.rs +++ b/big-code-analysis-ast/src/getter/java.rs @@ -78,7 +78,7 @@ impl Getter for JavaCode { } } - fn get_op_type<'a>(node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole { + fn get_op_type<'a>(node: &Node<'a>, ancestors: Ancestors<'a, '_>) -> TokenRole { use Java::*; // Some guides that informed grammar choice for Halstead // keywords, operators, literals: https://docs.oracle.com/javase/specs/jls/se18/html/jls-3.html#jls-3.12 @@ -89,7 +89,7 @@ impl Getter for JavaCode { | While | Continue | Break | Do | Finally // Operator: keywords | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final - | Implements | Transient | Synchronized | Super | This | VoidType + | Implements | Transient | Synchronized | VoidType // Operator: brackets and comma and terminators (separators) | SEMI | COMMA | COLONCOLON | DOT | DASHGT | LBRACE | LBRACK | LPAREN // Operator: operators @@ -103,11 +103,32 @@ impl Getter for JavaCode { => { TokenRole::Operator }, - // Operands: variables, constants, literals + // `super` is the receiver of `super.f()` / `super(x)` / + // `super::f` everywhere except a wildcard type bound, where + // `? super String` denotes no value and is the mirror image + // of `? extends String` — whose `extends` this same match + // bills as an operator. Both keywords are direct children of + // the same `wildcard` node, so the parent alone separates + // the bound from the reference (#1380). + Super => match ancestors.parent(node).map(|p| p.kind_id().into()) { + Some(Wildcard) => TokenRole::Operator, + _ => TokenRole::Operand, + }, + // Operands: variables, constants, literals. `This` joined + // them in #1380: a self-reference names the receiver a `.` + // or `::` acts on, so billing it as an operator made + // `this.x` a binary operator with one operand while `p.x` + // has one operator and two. The explicit receiver parameter + // (`void m(J J.this)`) reaches this arm too, and is an + // operand for the same reason a parameter name is — which is + // the opposite call from C#'s indexer declarator in + // `csharp.rs`, deliberately: a parameter name is an operand + // everywhere here, while a *member* named by a keyword sits + // with `operator +`. Identifier | NullLiteral | ClassLiteral | True | False | StringLiteral | CharacterLiteral | HexIntegerLiteral | OctalIntegerLiteral | BinaryIntegerLiteral | DecimalIntegerLiteral | HexFloatingPointLiteral - | DecimalFloatingPointLiteral => { + | DecimalFloatingPointLiteral | This => { TokenRole::Operand }, _ => { diff --git a/big-code-analysis-ast/src/getter/kotlin.rs b/big-code-analysis-ast/src/getter/kotlin.rs index abeefc156..7c2bad617 100644 --- a/big-code-analysis-ast/src/getter/kotlin.rs +++ b/big-code-analysis-ast/src/getter/kotlin.rs @@ -219,7 +219,7 @@ impl Getter for KotlinCode { | ReturnAT // Operator: other keywords | Class | Fun | Object | Val | Var | In | Is | As | AsQMARK | BANGis | BANGin - | This | Super | Constructor + | Constructor // Operator: brackets, separators, terminators | SEMI | COMMA | COLONCOLON | DOT | LBRACE | LBRACK | LPAREN // Operator: assignment and arithmetic @@ -232,10 +232,24 @@ impl Getter for KotlinCode { | AMPAMP | PIPEPIPE | BANG | BANGBANG | QMARK | QMARKCOLON | QMARKDOT | DOTDOT | DOTDOTLT | DASHGT | COLON => TokenRole::Operator, - // Operands: identifiers and literals - Identifier | NumberLiteral | FloatLiteral | CharacterLiteral | Label => { - TokenRole::Operand - } + // Operands: identifiers and literals, plus the self- and + // super-reference leaves (#1380). Four leaf kinds, not two: + // the label-qualified spellings `this@Outer` / `super@Inner` + // are `this@` / `super@` tokens, distinct kind_ids that + // carry no `this` / `super` leaf of their own — the same + // shape as `return@`, already listed among the operators + // above. + // + // The leaves are the keepers, not the `this_expression` / + // `super_expression` wrappers (grammar-dispatch section 6): + // a `constructor_delegation_call` (`constructor() : this(0)`, + // `: super(x)`) emits a bare leaf with no wrapper at all, so + // billing the wrapper would score constructor delegation + // zero. Classifying both would double-count, and the + // wrapper's span swallows the label identifier that is + // already billed on its own (section 5). + Identifier | NumberLiteral | FloatLiteral | CharacterLiteral | Label | This + | Super | ThisAT | SuperAT => TokenRole::Operand, // Regression #191: a Kotlin string template wraps an // `Interpolation` child (the long `"${expr}"` form) whose // inner expressions are walked and counted separately, so diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 0c5b9f37c..70f1e352a 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -502,6 +502,15 @@ mod tests { assert!(!matches!(via_alias, TokenRole::Operator)); } + /// Runs the `--ops` walk over `source` and returns the root space's + /// merged vocabulary, with every nested space still reachable + /// through [`crate::ops::Ops::spaces`]. + fn ops_of(source: &str, file: &str) -> crate::ops::Ops { + let path = PathBuf::from(file); + let parser = T::new(source.as_bytes().to_vec(), &path, None); + crate::ops::ops_inner(&parser, None).expect("ops walk succeeds") + } + // Pins the lesson-4 invariant `n2 == len(dedupe(ops.operands))` by // running `operands_and_operators` (the text-keyed `--ops` store) // on the same source and comparing its deduplicated operand count @@ -520,9 +529,7 @@ mod tests { expected_n2: usize, mut expected_operands: Vec<&str>, ) { - let path = PathBuf::from(file); - let parser = T::new(source.as_bytes().to_vec(), &path, None); - let ops = crate::ops::ops_inner(&parser, None).expect("ops walk succeeds"); + let ops = ops_of::(source, file); let unique: HashSet<&str> = ops.operands.iter().map(String::as_str).collect(); assert_eq!( @@ -538,6 +545,155 @@ mod tests { assert_eq!(got, expected_operands, "operand vocabulary for {file}"); } + /// Asserts each keyword in `keywords` reaches the operand vocabulary + /// and reaches the operator vocabulary nowhere. + /// + /// Both halves are load-bearing for #1380. A keyword listed in two + /// arms is billed twice — once into `n1`/`N1` and once into + /// `n2`/`N2` — and an assertion that only looked for its arrival + /// among the operands would pass on that. The operator side is what + /// pins the removal. + #[track_caller] + fn assert_keywords_are_operands_only( + source: &str, + file: &str, + keywords: &[&str], + ) { + let ops = ops_of::(source, file); + + for keyword in keywords { + assert!( + ops.operands.iter().any(|o| o == keyword), + "{file}: `{keyword}` must be an operand; operands were {:?}", + ops.operands + ); + assert!( + !ops.operators.iter().any(|o| o == keyword), + "{file}: `{keyword}` must not be an operator; operators were {:?}", + ops.operators + ); + } + } + + /// How one space's vocabularies classify a single keyword. + #[derive(Debug, PartialEq, Eq)] + enum Role { + OperatorOnly, + OperandOnly, + Both, + } + + /// Records `keyword`'s role in `ops` and in every space below it. + /// + /// A space that bills the keyword in neither vocabulary contributes + /// no row, so the result is the set of places the keyword actually + /// reaches — which is what distinguishes a parent gate from its own + /// inverse, and what the merged root vocabulary cannot show. + fn collect_keyword_roles(ops: &crate::ops::Ops, keyword: &str, out: &mut Vec) { + let operator = ops.operators.iter().any(|o| o == keyword); + let operand = ops.operands.iter().any(|o| o == keyword); + match (operator, operand) { + (true, true) => out.push(Role::Both), + (true, false) => out.push(Role::OperatorOnly), + (false, true) => out.push(Role::OperandOnly), + (false, false) => {} + } + for space in &ops.spaces { + collect_keyword_roles(space, keyword, out); + } + } + + /// Pins the three grammar facts every #1380 operand arm rests on, + /// over a fixture that samples one self- or super-reference per + /// syntactic position the language allows. + /// + /// Each `keywords` row is `(spelling, kind_id, occurrences)`. + /// + /// * **Grammar-dispatch section 1.** Every node the grammar spells + /// `this` / `super` / `base` / `this@` / `super@` carries the one + /// `kind_id` its arm lists. An alias arises in a *different* + /// syntactic position, which no fixture can enumerate, so this is + /// the weaker half of the alias evidence — the strong half is that + /// these generated enums carry numeric-suffix aliases in quantity + /// and none spells a `This2`. What this loop catches is a grammar + /// bump introducing one under an existing fixture. + /// * **Grammar-dispatch section 5.** Each is a childless leaf and no + /// node *containing* it is classified, so the reference cannot be + /// billed twice. Every ancestor is checked, not just the parent: + /// Kotlin puts `this_expression` *and* `navigation_expression` + /// between the leaf and the nearest classified node, and the claim + /// the arms make is about containment, not parentage. + /// * **Grammar-dispatch section 11.** The arm is unconditional. A + /// parent-scoped guard added later — the shape the two deliberate + /// carve-outs use, and so the shape a reviewer might reasonably + /// add — would satisfy the single-position count tests and still + /// drop `this` in a method reference or a constructor delegation. + /// + /// The fixtures deliberately exclude the two gated positions (C#'s + /// indexer declarator, Java's wildcard bound); those are operators, + /// and their own tests pin them. + #[track_caller] + fn assert_self_reference_leaves( + source: &str, + keywords: &[(&str, u16, usize)], + label: &str, + ) { + let code = source.as_bytes(); + let mut seen = vec![0_usize; keywords.len()]; + + for_each_node_with_chain::(code, |node, chain| { + let Some(index) = keywords.iter().position(|(kw, _, _)| *kw == node.kind()) else { + return; + }; + let (keyword, kind_id, _) = keywords[index]; + seen[index] += 1; + + assert_eq!( + node.kind_id(), + kind_id, + "{label}: a `{keyword}` carries kind_id {} rather than the {kind_id} its arm \ + lists — an alias the arm cannot see", + node.kind_id() + ); + assert_eq!( + node.child_count(), + 0, + "{label}: `{keyword}` grew children, which the operand arm would now \ + double-count against" + ); + for (depth, ancestor) in chain.iter().enumerate() { + assert!( + matches!( + L::get_op_type_with_code(ancestor, code, Ancestors::known(&chain[..depth])), + TokenRole::Unknown + ), + "{label}: `{}` contains a `{keyword}` and is itself classified, so the \ + reference now counts twice", + ancestor.kind() + ); + } + // `_with_code` is the spelling `compute_halstead` calls. The + // default forwards to the byte-less form, so asking the wrong + // one reads as correct right up until a language grows an + // override (grammar-dispatch section 7). + assert!( + matches!( + L::get_op_type_with_code(node, code, Ancestors::known(chain)), + TokenRole::Operand + ), + "{label}: a `{keyword}` under `{}` is not an operand", + chain.last().map_or("", Node::kind) + ); + }); + + for ((keyword, _, expected), got) in keywords.iter().zip(&seen) { + assert_eq!( + got, expected, + "{label}: fixture holds {got} `{keyword}` rather than {expected}" + ); + } + } + /// Asserts the root space's `[n1, N1, n2, N2]`, naming `label` when /// it does not hold. /// @@ -2274,6 +2430,96 @@ mod tests { ); } + // #1380: `this` and `super` were swept into the operator arm under a + // `// Operator: … keywords` heading — classification by lexical + // class rather than by Halstead role. A member access is + // ` `, so billing the receiver as an operator + // scored `this.x` as a binary operator with one operand while `p.x` + // is one operator and two operands. Both are operands now, matching + // the eleven other languages that classify a self-reference. + #[test] + fn java_self_and_super_references_are_operands() { + let source = "class T {\n int f() { return this.x + super.y; }\n}"; + // Operators (n1=7, N1=9): {} x2, int, (), return, . x2, +, ; + // Operands (n2=6, N2=6): T, f, this, x, super, y + // Before the fix this read (9, 11, 4, 4) — `this` and `super` + // billed into the operator side of both counts. + assert_halstead_counts::( + source, + "foo.java", + [7, 9, 6, 6], + "java self/super references", + ); + assert_keywords_are_operands_only::(source, "foo.java", &["this", "super"]); + } + + // The `super` of a wildcard type bound denotes no value, so it keeps + // the operator classification `? extends T`'s `extends` already has + // — the two keywords are siblings under the same `wildcard` node and + // nothing but the parent separates them from a real self-reference + // (#1380). The explicit receiver parameter in the same fixture is + // the independent path through the operand arm (grammar-dispatch + // section 11): only it can put `this` in the operand vocabulary + // here, and only the wildcard can put `super` in the operator one. + #[test] + fn java_wildcard_super_bound_stays_an_operator() { + let source = "import java.util.List;\n\ + class T {\n \ + void m(T T.this, List a, List b) { }\n\ + }"; + let ops = ops_of::(source, "foo.java"); + + assert!( + ops.operators.iter().any(|o| o == "super"), + "`? super String` must keep `super` an operator; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == "super"), + "a wildcard bound must not bill `super` as an operand; operands were {:?}", + ops.operands + ); + assert!( + ops.operators.iter().any(|o| o == "extends"), + "the fixture must still contain the `? extends String` \ + bound this arm mirrors; operators were {:?}", + ops.operators + ); + assert_keywords_are_operands_only::(source, "foo.java", &["this"]); + } + + /// One `this` and one `super` per container the pinned + /// `tree-sitter-java` can put them in: a delegating constructor + /// call, an explicit superclass constructor call, a field access, a + /// method-invocation receiver, a method reference, a call argument, + /// and the two qualified forms an inner class allows. + const JAVA_SELF_POSITIONS: &str = "class Pos extends P { + int x; + Pos() { this(1); } + Pos(int v) { super(v); this.x = v; } + int a() { return this.x; } + int b() { return super.h(); } + java.util.function.IntSupplier c() { return this::a; } + void d() { g(this); } + void g(Object o) { } + class In { + int e() { return Pos.this.x; } + int f() { return Pos.super.h(); } + } + }"; + + #[test] + fn java_self_and_super_leaves_are_unaliased_and_unconditional() { + assert_self_reference_leaves::( + JAVA_SELF_POSITIONS, + &[ + ("this", Java::This as u16, 6), + ("super", Java::Super as u16, 3), + ], + "java", + ); + } + #[test] fn groovy_operators_and_operands() { check_metrics::( @@ -2979,6 +3225,106 @@ mod tests { ); } + // C# half of #1380 — see `java_self_and_super_references_are_operands` + // for the structural argument. `base` moves with `this`: both are + // receivers of a member access. + #[test] + fn csharp_self_and_base_references_are_operands() { + let source = "class T {\n int F() { return this.x + base.y; }\n}"; + // Operators (n1=8, N1=10): class, {} x2, int, (), return, . x2, +, ; + // Operands (n2=6, N2=6): T, F, this, x, base, y + // Before the fix this read (10, 12, 4, 4). + assert_halstead_counts::( + source, + "foo.cs", + [8, 10, 6, 6], + "csharp self/base references", + ); + assert_keywords_are_operands_only::(source, "foo.cs", &["this", "base"]); + } + + // An `indexer_declaration` spells the member's *name* with the same + // `this` token kind the receiver uses (both are kind 91), so a + // blanket move would have billed a declarator keyword as a value. + // The declaration keeps the operator classification that the + // `operator` keyword of an overload declaration already has (#1380). + // + // The fixture carries both uses, which is what makes the gate + // observable: `this` has to reach *both* vocabularies from one file. + // Asserting only the operand side would pass with the gate deleted, + // and only the operator side would pass with the whole #1380 change + // reverted (grammar-dispatch section 11). + #[test] + fn csharp_indexer_declaration_keyword_is_not_a_self_reference() { + let source = "class C {\n int[] _a;\n \ + public int this[int i] { get { return this._a[i]; } }\n}"; + + // Asserted per space, not through the root. The root merges + // every space's vocabulary, and the fixture has one declarator + // `this` and one receiver `this`, so at the root `this` is a + // member of *both* vocabularies whether the gate is right or + // exactly backwards — and every Halstead count is bit-identical + // under the swap, one occurrence on each side either way. The + // space tree is where the two uses stay apart: the declarator + // sits in the class's own vocabulary and the receiver in the + // accessor's. + let mut roles = Vec::new(); + collect_keyword_roles( + &ops_of::(source, "foo.cs"), + "this", + &mut roles, + ); + + // `Both` can only arise from two `this` nodes in one space that + // the getter classifies differently — it is the gate's whole + // effect, and it disappears the moment `this` gets one role. + // `OperandOnly` is the accessor, the innermost space and the one + // holding the receiver alone; it is what flips when the gate is + // inverted rather than dropped. Correct gives + // `[Both, Both, OperandOnly]`; inverting gives + // `[Both, Both, OperatorOnly]`; dropping the gate gives three + // `OperandOnly`; reverting #1380 gives three `OperatorOnly`. + assert!( + roles.contains(&Role::Both), + "some space must bill `this` as an operator *and* an operand, which only \ + the declarator and the receiver disagreeing can produce; roles were {roles:?}", + ); + assert!( + roles.contains(&Role::OperandOnly), + "the accessor space holds only the `this._a` receiver, so it must bill \ + `this` as an operand and nothing else; roles were {roles:?}", + ); + } + + /// One `this` and one `base` per container the pinned + /// `tree-sitter-c-sharp` can put them in: a delegating constructor + /// initializer, a base-constructor initializer, a member access, an + /// element access, and a call argument. No `indexer_declaration` — + /// that position is the gated one and is an operator. + const CSHARP_SELF_POSITIONS: &str = "class Pos : B { + int[] _a; + public Pos() : this(1) { } + public Pos(int x) : base(x) { } + int A() { return this._a[0]; } + int C() { return base.H(); } + int E() { return base[0]; } + int F() { return this[0]; } + void G() { M(this); } + void M(object o) { } + }"; + + #[test] + fn csharp_self_and_base_leaves_are_unaliased_and_unconditional() { + assert_self_reference_leaves::( + CSHARP_SELF_POSITIONS, + &[ + ("this", Csharp::This as u16, 4), + ("base", Csharp::Base as u16, 3), + ], + "csharp", + ); + } + #[test] fn go_operators_and_operands() { check_metrics::( @@ -4126,6 +4472,119 @@ end", ); } + // Kotlin half of #1380 — see + // `java_self_and_super_references_are_operands` for the structural + // argument. + #[test] + fn kotlin_self_and_super_references_are_operands() { + let source = "class T {\n fun f() = this.x + super.y\n}"; + // Operators (n1=7, N1=8): class, {}, fun, (), =, . x2, + + // Operands (n2=6, N2=6): T, f, this, x, super, y + // Before the fix this read (9, 10, 4, 4). + assert_halstead_counts::( + source, + "foo.kt", + [7, 8, 6, 6], + "kotlin self/super references", + ); + assert_keywords_are_operands_only::(source, "foo.kt", &["this", "super"]); + } + + // The label-qualified spellings are their own leaf kinds — `this@` + // (106) and `super@` (107) — carrying no `this` / `super` leaf, so + // listing only the two bare kinds would score every `this@Outer` + // reference zero (grammar-dispatch section 1). The labels `Outer` + // and `Inner` stay separate operands: the `this_expression` wrapper + // whose span would have swallowed them is deliberately unclassified + // (section 5). + #[test] + fn kotlin_labelled_self_and_super_references_are_operands() { + let source = "class Outer {\n inner class Inner : A() {\n \ + fun f() = this@Outer.x + super@Inner.y\n }\n}"; + assert_keywords_are_operands_only::(source, "foo.kt", &["this@", "super@"]); + assert_ops_operands::( + source, + "foo.kt", + 8, + vec!["A", "Inner", "Outer", "f", "super@", "this@", "x", "y"], + ); + // The metrics store is a second, independent walk — it keys + // operands by `get_operand_id` where `ops_inner` keys by text — + // so `assert_ops_operands` on its own pins the ops store against + // a literal rather than against the metric (lesson 4). + // + // `N2` is what makes the "the labels stay separate operands" + // claim above testable. Both labels duplicate an enclosing class + // name, so dropping their contribution entirely leaves `n2` at 8 + // with the same eight strings; only `N2` falls, 10 to 8. + check_metrics::(source, "foo.kt", |metric| { + assert_eq!(metric.halstead.unique_operands(), 8); + assert_eq!(metric.halstead.total_operands(), 10); + }); + } + + // The keeper question of grammar-dispatch section 6: a + // `constructor_delegation_call` emits a bare `this` / `super` leaf + // with no `this_expression` / `super_expression` wrapper around it, + // so classifying the wrapper instead would score constructor + // delegation zero. This is the only spelling that can tell the two + // choices apart — every other `this` carries both nodes — so it is + // the independent path grammar-dispatch section 11 asks for. + #[test] + fn kotlin_constructor_delegation_self_reference_is_an_operand() { + let source = "class C(val n: Int) {\n constructor() : this(0)\n}"; + assert_keywords_are_operands_only::(source, "foo.kt", &["this"]); + assert_ops_operands::( + source, + "foo.kt", + 5, + vec!["0", "C", "Int", "n", "this"], + ); + check_metrics::(source, "foo.kt", |metric| { + assert_eq!(metric.halstead.unique_operands(), 5); + assert_eq!(metric.halstead.total_operands(), 5); + }); + } + + /// One `this` / `super` per container the pinned + /// `tree-sitter-kotlin-ng` can put them in: a bare expression, a + /// call argument, a navigation receiver, a type-argument-qualified + /// `super

`, both label-qualified forms, and the + /// `constructor_delegation_call` that carries no wrapper. + const KOTLIN_SELF_POSITIONS: &str = "class P { + fun h() = 1 + } + class Outer : P() { + val x = 1 + fun c() = super.h() + fun d() = this + fun e(o: Any): Any = e(this) + inner class Inner : P() { + fun a() = this@Outer.x + fun b() = this.hashCode() + fun f() = super@Inner.h() + fun g() = super

.h() + } + } + class Del(val n: Int) { + constructor() : this(0) + } +"; + + #[test] + fn kotlin_self_and_super_leaves_are_unaliased_and_unconditional() { + assert_self_reference_leaves::( + KOTLIN_SELF_POSITIONS, + &[ + ("this", Kotlin::This as u16, 4), + ("super", Kotlin::Super as u16, 2), + ("this@", Kotlin::ThisAT as u16, 1), + ("super@", Kotlin::SuperAT as u16, 1), + ], + "kotlin", + ); + } + #[test] fn python_fstring_no_double_count() { // Regression: issue #191. A Python f-string (`f"Hi {name}!"`) diff --git a/tests/parity/main.rs b/tests/parity/main.rs index d83379302..ac2899813 100644 --- a/tests/parity/main.rs +++ b/tests/parity/main.rs @@ -12,4 +12,25 @@ mod functions_metrics_parity; mod halstead_set_target_parity; mod nargs_cross_language_parity; mod ops_metrics_space_parity; +// Gated on the union of the languages whose fixture rows are `Some`, so +// a build enabling only self-reference-free languages drops the module +// rather than failing its non-vacuity guard (`.claude/rules/testing.md`). +#[cfg(any( + feature = "cpp", + feature = "csharp", + feature = "groovy", + feature = "java", + feature = "javascript", + feature = "kotlin", + feature = "lua", + feature = "mozcpp", + feature = "mozjs", + feature = "objc", + feature = "php", + feature = "python", + feature = "ruby", + feature = "rust", + feature = "typescript", +))] +mod self_reference_operand_parity; mod space_span_containment; diff --git a/tests/parity/self_reference_operand_parity.rs b/tests/parity/self_reference_operand_parity.rs new file mode 100644 index 000000000..6760eb81e --- /dev/null +++ b/tests/parity/self_reference_operand_parity.rs @@ -0,0 +1,220 @@ +//! Cross-language parity test for the Halstead classification of a +//! **self-reference** (`this` / `self` / `$this`) and a +//! **super-reference** (`super` / `base`). +//! +//! The workspace used to split three ways on this. Java, C# and Kotlin +//! swept the keyword into their operator arm under a +//! `// Operator: … keywords` heading — classification by lexical class +//! — while every other language that classifies one at all called it an +//! operand, so the same source translated between two languages scored +//! different `n1` / `N1` / `n2` / `N2` and every value derived from +//! them. #1380 settled it on **operand**: a member access is +//! ` `, and billing the receiver as an operator +//! made `this.x` a binary operator with one operand while `p.x` is one +//! operator with two. +//! +//! Nothing but this test holds the three back together. Each language's +//! `get_op_type` is an independent `match`, and a keyword added to the +//! wrong arm of one of them produces no compile error, no clippy +//! warning and no failure in that language's own suite — which is how +//! the split survived until #1361 tripped over it. +//! +//! The fixture table is an exhaustive `match` on [`LANG`], so adding a +//! language variant fails to compile until its row is supplied. A +//! language with no self-reference says so with `None` rather than +//! falling through a wildcard. +//! +//! The two *declarator* uses of the same keyword — C#'s +//! `public int this[int i]` and Java's `? super String` wildcard bound — +//! are deliberately absent from these fixtures: they are operators, so +//! a fixture containing one would put the keyword in both vocabularies +//! and fail the assertion below. They are pinned instead by +//! `csharp_indexer_declaration_keyword_is_not_a_self_reference` and +//! `java_wildcard_super_bound_stays_an_operator` in +//! `src/metrics/halstead.rs`. + +use big_code_analysis::{Ast, LANG, Source}; + +/// Returns `(source, extension, keywords)` for a language that spells a +/// self- or super-reference, or `None` for one that does not. +/// +/// Every keyword listed must be the *value* form — a receiver standing +/// where a variable would stand. The source spelling is what reaches +/// the vocabulary, so PHP's row names `$this` and C#'s names `base`. +/// +/// Five rows reach the operand vocabulary through the generic +/// identifier arm rather than through a decision anyone made: Groovy, +/// Objective-C, Python and Lua spell the keyword as a plain +/// `identifier` (kind 1), and PHP's `$this` is a `variable_name`. Each +/// says so at its row. They still earn their place — each is a guard +/// against a grammar bump promoting the keyword to a kind of its own +/// and the language falling out of the majority unnoticed. +fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static str])> { + // Exhaustive per-language dispatch table: one arm per LANG variant + // is the point of this function, so a new language cannot be added + // without deciding whether it has a self-reference. The repo's own + // `.bcaignore` excludes `./tests/**`, so this marker is for the + // per-edit `bca check` hook rather than for the self-scan gate. + // bca: suppress(cyclomatic) + let row: (&str, &str, &[&str]) = match lang { + LANG::Javascript | LANG::Mozjs => ( + "class A extends B {\n f() { return this.x; }\n g() { return super.h(); }\n}\n", + "js", + &["this", "super"], + ), + LANG::Typescript => ( + "class A extends B {\n f(): number { return this.x; }\n \ + g(): number { return super.h(); }\n}\n", + "ts", + &["this", "super"], + ), + LANG::Tsx => ( + "class A extends B {\n f(): number { return this.x; }\n \ + g(): number { return super.h(); }\n}\n", + "tsx", + &["this", "super"], + ), + LANG::Java => ( + "class A extends B {\n int f() { return this.x; }\n \ + int g() { return super.h(); }\n}\n", + "java", + &["this", "super"], + ), + LANG::Kotlin => ( + "class A : B() {\n fun f() = this.x\n fun g() = super.h()\n}\n", + "kt", + &["this", "super"], + ), + // C# spells the super-reference `base`. + LANG::Csharp => ( + "class A : B {\n int F() { return this.x; }\n int G() { return base.H(); }\n}\n", + "cs", + &["this", "base"], + ), + // Grammar accident, and the interesting one: `getter/groovy.rs` + // *does* list `Super` among its operators, but the grammar emits + // a plain `identifier` for both `this` and `super` in receiver + // position — verified by dump for `super(1)`, `super.h()`, + // `A.super.h()` and `super::h` — so that arm is dead at the + // current pin and Groovy is an operand language in fact. This + // row is what notices if a bump ever wakes the arm up. + LANG::Groovy => ( + "class A extends B {\n def f() { return this.x }\n \ + def g() { return super.h() }\n}\n", + "groovy", + &["this", "super"], + ), + LANG::Ruby => ( + "class A < B\n def f; self.x; end\n def g; super; end\nend\n", + "rb", + &["self", "super"], + ), + // Rust's `self` is a receiver; it has no super-reference + // (`super::` is a module path, not an object). + LANG::Rust => ( + "struct A { x: i32 }\nimpl A { fn f(&self) -> i32 { self.x } }\n", + "rs", + &["self"], + ), + // C++ `this` is a pointer, so the fixture dereferences with + // `->`. #1361 added the arm that classifies it at all. + LANG::Cpp | LANG::Mozcpp => ( + "struct S { int x; int f() { return this->x; } };\n", + "cpp", + &["this"], + ), + // Grammar accident: `self` is a plain `identifier`, so no arm is + // involved. + LANG::Objc => ( + "@implementation S\n- (int)f { return [self g]; }\n- (int)g { return 1; }\n@end\n", + "m", + &["self"], + ), + // PHP's `$this` is a `variable_name`, not a keyword, so it is an + // operand by grammar rather than by an arm — the fixture pins + // that it stays one. PHP's `Zelf` / `Parent` arms cover the + // unrelated `self::` / `parent::` class references, which are + // scope-resolution operators and deliberately untouched (#1380). + LANG::Php => ( + "x; } }\n", + "php", + &["$this"], + ), + // Grammar accidents, as Groovy and Objective-C are: `self` is a + // plain `identifier` in both grammars. + LANG::Python => ( + "class A:\n def f(self):\n return self.x\n", + "py", + &["self"], + ), + LANG::Lua => ( + "local S = {}\nfunction S:m1() return self.x end\n", + "lua", + &["self"], + ), + // No self-reference to classify. Go names the receiver in the + // method signature; C, Bash, Tcl, iRules and Perl have no + // object receiver at all (Perl's `$self` is an ordinary lexical, + // indistinguishable from any other variable); Elixir spells the + // enclosing module `__MODULE__`, a compile-time macro rather + // than a receiver; and the two helper grammars parse fragments. + LANG::Go + | LANG::C + | LANG::Bash + | LANG::Tcl + | LANG::Irules + | LANG::Perl + | LANG::Elixir + | LANG::Ccomment + | LANG::Preproc => return None, + }; + Some(row) +} + +#[test] +fn every_language_bills_a_self_reference_as_an_operand() { + let mut checked = 0; + + for lang in LANG::into_enum_iter() { + if !lang.is_enabled() { + continue; + } + let Some((source, ext, keywords)) = fixture(lang) else { + continue; + }; + checked += 1; + + let name = format!("parity.{ext}"); + let ops = Ast::parse(Source::new(lang, source.as_bytes()).with_name(Some(name))) + .unwrap_or_else(|e| panic!("{lang:?}: parse failed: {e}")) + .ops() + .unwrap_or_else(|e| panic!("{lang:?}: ops failed: {e}")); + + for keyword in keywords { + assert!( + ops.operands.iter().any(|o| o == keyword), + "{lang:?}: `{keyword}` must be a Halstead operand; operands were {:?}", + ops.operands, + ); + // The other half of the parity claim. A keyword listed in + // both arms is billed twice — once into `n1`/`N1` and once + // into `n2`/`N2` — and would satisfy the assertion above. + assert!( + !ops.operators.iter().any(|o| o == keyword), + "{lang:?}: `{keyword}` must not be a Halstead operator; operators were {:?}", + ops.operators, + ); + } + } + + // Every language is feature-gated, so a build enabling only + // languages with no self-reference leaves a zero-iteration loop and + // a test that reports green while asserting nothing. The `cfg` above + // keeps this from firing spuriously: it names exactly the features + // whose rows are `Some`. + assert!( + checked > 0, + "at least one language feature with a self-reference must be \ + enabled for this test to mean anything", + ); +} From 4c9511c2696f6f0ab50bf39f3aff8952e3a67e99 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 19:15:12 -0700 Subject: [PATCH 10/22] fix(metrics/loc): clamp a space's line sets to its own row span A grammar hands back spans it cannot honour: an unterminated Bash heredoc gets a zero-width `heredoc_end` at (3, 1) of a two-row file, an unterminated Elixir, Lua or Groovy string gets its closing delimiter the same way. Such a token is childless, so it reaches the leaf branch of its language's catch-all arm, which inserts a raw start row into PLOC -- and there that start row is itself the phantom. `Node::end_line` already encodes "a node whose end column is 0 does not occupy the row it ends on"; nothing encoded the same rule for a node that begins past the span. The result is `ploc > sloc`, a contract violation rather than a rounding artifact: `Stats::blank` saturates at 0, so the clamp there hides it while a consumer computing `ploc / sloc` gets a ratio above 1. A sweep of 322 truncated fixtures across all twenty-three languages found ten reproducing it -- Bash, C, C++, Mozcpp, Objective-C, Elixir, Groovy, Lua, Perl and Ruby -- over four unrelated recovery shapes, plus a matching `cloc > sloc` on an unterminated Perl POD block. Ten per-arm skips would have been ten edits that still could not cover a grammar whose recovery shape nobody sampled, so the rule lives once, in `Stats::clamp_line_sets_to_span`, keyed on the span every language already reports and run per space at finalization. Per space rather than per unit: an unterminated Elixir `do` block violated the contract on its own `defmodule` space as well as on the file. A `debug_assert!` on the same path pins the invariant for every space of every walk -- 821 workspace tests reach it -- rather than only for the fixtures the regression tests name. It is deliberately stated against the row span and not against `sloc()`: `Sloc::exclude_span` counts a pruned span whole, including a row a retained sibling shares, so `sloc()` can still fall below `ploc` under `--exclude-tests`. That is a separate defect with a separate cause, filed as #1417. Two things the review turned up that the fix's first draft got wrong, both now recorded where they will be read: - "no parsed space records a row outside its span, measured over the corpora" was a load-bearing claim in `line_set.rs`, and it was false. `DeepSpeech/parse_valgrind_suppressions.sh` leaves a MISSING `}` at (58, 1) of a 57-row file; its `ploc` drops 36 -> 35 here. No snapshot covers it -- `snapshots/` holds only the C-family files -- so a green integration run is not evidence this change has no effect on real trees. - A clean parse is not evidence of an in-span tree. Ruby's `x = <<~DOC\na\n` parses with no error node at all and still yields the phantom row, so the regression table anchors each fixture on a node past the last row rather than on `has_error()`. `make bench-scaling`: all 27 probes within bound, the five `loc/*` at exponents 0.64-1.10 against 1.50. `make chain-audit` clean. Fixes #1398 --- src/metrics/loc.rs | 360 ++++++++++++++++++++++++++++++++++++ src/metrics/loc/line_set.rs | 163 +++++++++++++++- src/spaces/compute.rs | 39 +++- 3 files changed, 552 insertions(+), 10 deletions(-) diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index 2848d5f3a..ba27aeac0 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -757,6 +757,119 @@ impl Stats { self.sloc.start = start; self.sloc.end_line = end_line; } + + /// Drops every recorded PLOC / CLOC row that lies outside this + /// space's own `sloc` row span. + /// + /// That bounds `ploc` and `cloc` by the span, which is what the + /// `debug_assert!` below states. It is *not* the same as pinning + /// the public `ploc() <= sloc()`: `sloc()` subtracts + /// `exclude_tests`-pruned rows, and can still come out below `ploc` + /// — see the assertion's comment and #1417. + /// + /// A grammar's error recovery synthesises **zero-width tokens one + /// row past the last row the file has**: an unterminated Bash + /// heredoc gets a `heredoc_end` at `(3, 1)..(3, 1)` in a two-row + /// file, an unterminated Elixir / Lua / Groovy string gets its + /// closing quote the same way. Those tokens are childless, so they + /// reach the leaf branch of a language's catch-all arm, which + /// inserts a raw `start` row — and here that `start` *is* the + /// phantom row. [`Node::end_line`] already encodes "a node whose + /// end column is 0 does not occupy the row it ends on"; nothing + /// encoded the same rule for a node that *begins* past the span. + /// + /// `ploc > sloc` is a contract violation rather than a rounding + /// artifact: [`Stats::blank`] saturates at 0, so the clamp there + /// hides it while a consumer computing `ploc / sloc` gets a ratio + /// above 1. Ten of the twenty-odd languages reproduced it on + /// truncated input before #1398 — a per-arm skip would have been + /// ten edits that still could not cover a grammar whose recovery + /// shape nobody sampled, so the rule lives once, here, keyed on the + /// span every language already reports. + /// + /// **Input the grammar parses is untouched — which is not the same + /// as input that looks fine.** Over the `pdf.js`, `DeepSpeech` and + /// `serde` corpora exactly one of 1,876 files moves, and it is a + /// real bug fixed rather than a real row lost: + /// `DeepSpeech/parse_valgrind_suppressions.sh` leaves a MISSING `}` + /// at `(58, 1)` of a 57-row file, so its `ploc` drops 36 → 35 and + /// `blank` rises 6 → 7. No snapshot covers it — `snapshots/` holds + /// only the C-family files — so the integration suite is silent + /// about the one case that proves the fix reaches ordinary trees. + /// Do not read a green snapshot run as evidence of no effect. + /// + /// The clamp's one latent hazard is the mirror image: a unit whose + /// span is *shorter* than the file. That happens today for a file + /// ending in a blank row (`"package m\n\n"` reports `unit 1..1`), + /// and is harmless only because a trailing blank row reaches + /// neither line set. Nothing pins `unit.end_line >= line_count` — + /// `tests/parity/space_span_containment.rs` asserts + /// `unit == (1, line_count)`, but none of its fixtures ends in a + /// blank row — so a grammar that ever credited such a row to PLOC + /// would have it deleted here silently. + pub(crate) fn clamp_line_sets_to_span(&mut self) { + let start = self.sloc.start; + let span = span_rows(start, self.sloc.end_line); + // `end_line` is the 1-based inclusive last row of the span, so + // the last 0-based row it covers is `end_line - 1` — exact + // wherever the span is non-empty, which is what `span == 0` + // selects against. `(1, 0)` is the inverted range + // `LineSet::retain_range` reads as "keep nothing", and it is + // what the empty file's `0..0` needs: retaining `0..=0` there + // would keep a row the file does not have. + // + // That last branch is defensive rather than live. The three + // walks that reach `span == 0` (an empty file, and the two + // whitespace-only root contracts) all arrive with both line + // sets already empty, so `(0, 0)` here is byte-identical over + // every corpus input — measured. `a_zero_span_keeps_no_row` + // below is therefore a direct `Stats` test rather than a + // fixture, per `.claude/rules/testing.md`: it is the only shape + // that can tell the two spellings apart. + let (first, last) = if span == 0 { + (1, 0) + } else { + (start, self.sloc.end_line.saturating_sub(1)) + }; + self.ploc.lines.retain_range(first, last); + self.cloc.only_comment_line_starts.retain_range(first, last); + self.cloc.code_comment_line_starts.retain_range(first, last); + + // The invariant the clamp establishes, asserted on the path + // every walk takes for every space rather than only on the + // fixtures the regression tests name — 821 of the workspace's + // tests reach it. + // + // Against the span and not against `sloc()`, which is weaker on + // purpose. `sloc()` subtracts the rows of `exclude_tests`-pruned + // subtrees, and `Sloc::exclude_span` counts each pruned span + // whole — including a row a retained sibling also occupies, a + // case its #722 comment excludes by assuming rustfmt's layout. + // Hand-written one-liners break that assumption, so + // `fn a() {} #[cfg(test)] mod t { … }` reports `sloc 0, ploc 1` + // under `--exclude-tests` today. That is #1417, a different + // cause from the phantom row above, and asserting `sloc()` here + // would fire on it. Tighten this to `sloc()` once #1417 lands. + // + // Both values are bound first because `ploc()` and `cloc()` + // popcount their word arrays since #1109, and a `debug_assert!` + // evaluates its message arguments separately from its + // condition. O(words) per space, the same order as the + // `compute_minmax` that follows; per node it would be the + // quadratic shape #1122 removed. + #[cfg(debug_assertions)] + { + let (ploc, cloc) = (self.ploc(), self.cloc()); + debug_assert!( + ploc <= span as u64, + "ploc {ploc} exceeds the {span} row span it was clamped to" + ); + debug_assert!( + cloc <= span as u64, + "cloc {cloc} exceeds the {span} row span it was clamped to" + ); + } + } } #[doc(hidden)] @@ -11339,6 +11452,253 @@ class A { assert_eq!(subs, vec![3, 3], "both subs occupy three rows"); } + /// #1398, whose mechanism [`Stats::clamp_line_sets_to_span`] + /// documents: a recovery token starting one row past end-of-input + /// reached a catch-all's leaf branch and became a line of code, so + /// `ploc` came out one above `sloc`. + /// + /// The assertion is exact equality with the *unchanged* `sloc`, in + /// both directions. `ploc <= sloc` alone would also be satisfied by + /// a clamp wide enough to delete real code rows, and every fixture + /// here is all-code, so `ploc == sloc` is the only passing value. + /// + /// Each row additionally asserts the tree still holds a node past + /// the file's last row. Without that anchor every fixture decays + /// into an ordinary all-code file the moment someone edits the + /// source string, and `ploc == sloc` is then true of any + /// well-formed input — measured: replacing all fifteen fixtures + /// with well-formed sources of the same row count leaves this test + /// green. The phantom node *is* the subject, so it is the axis to + /// anchor on (`.claude/rules/testing.md`, "Perturb the fixture as + /// well as the production line"). + /// + /// Writing that anchor as `has_error()` — the obvious choice — is + /// wrong, and finding out why is worth the paragraph: Ruby's + /// `x = <<~DOC\na\n` parses with **no error node at all**, and + /// still gets a zero-width `heredoc_end` at `(3, 1)` of a two-row + /// file. A grammar need not report failure to hand back a span it + /// cannot honour, which is half of why this defect survived. + /// + /// `metrics_verbatim`, not `check_metrics`, for a duller reason + /// than #1051's: a `LANG`-keyed table cannot dispatch the generic + /// `check_metrics::` shim, and that shim's `fn` callback returns + /// nothing. Every fixture here is already a fixed point of the + /// trailing-newline normalisation, so — measured — a + /// `check_metrics` twin of any of these rows fails against unfixed + /// code too. This is *not* the EOF class both harnesses hide. + #[cfg(any( + feature = "bash", + feature = "c", + feature = "cpp", + feature = "elixir", + feature = "groovy", + feature = "lua", + feature = "mozcpp", + feature = "objc", + feature = "perl", + feature = "ruby", + ))] + #[test] + fn a_recovery_token_past_the_span_is_not_a_code_row() { + // Every fixture that reproduced the defect, one row per + // language, as `(language, source, rows the file has, ploc + // before #1398)`. That last column is what makes this a + // regression test rather than a restatement of the fixed + // behaviour: a reviewer can see which row the assertion is + // about, and it is one above `sloc` in every case. + // + // Four unrelated recovery shapes are represented, which is the + // argument for one clamp over ten per-arm skips: a heredoc + // whose terminator never arrives (Bash, Perl, Ruby), an + // unterminated string delimiter (Elixir, Lua, Groovy), an + // unclosed bracket (Ruby's `%w[`), and a dangling line + // continuation (the C family, where the `\` splices a row that + // is not there). + const FIXTURES: &[(crate::LANG, &[u8], u64, u64)] = &[ + #[cfg(feature = "bash")] + (crate::LANG::Bash, b"cat <= rows { + phantom = Some(node.start_row()); + break; + } + stack.extend(node.children()); + } + assert!( + phantom.is_some(), + "{lang:?}: {text:?} no longer produces a node past row {rows}, \ + so it no longer exercises #1398" + ); + + let loc = metrics_verbatim(lang, source, MetricsOptions::default()).loc; + assert_eq!(loc.sloc(), rows, "{lang:?}: {text:?} occupies {rows} rows"); + assert_eq!( + loc.ploc(), + rows, + "{lang:?}: every row of {text:?} is code and no row past it is \ + (was {ploc_before_1398} before #1398)" + ); + // One-sided, unlike the two above: every fixture is all + // code, so this cannot catch a clamp that over-reaches into + // the comment sets — an over-reach can only leave 0 at 0. + // What it does pin is the other direction, a future change + // that files the phantom row as a *comment* instead. The + // CLOC half of the clamp is covered by + // `an_unterminated_pod_block_does_not_comment_a_row_past_eof`, + // which is its only guard. + assert_eq!((loc.cloc(), loc.blank()), (0, 0), "{lang:?}: {text:?}"); + } + } + + /// The `span == 0` arm of [`Stats::clamp_line_sets_to_span`], which + /// no parsed input can reach with a populated line set. + /// + /// Measured: spelling it `(0, 0)` instead of `(1, 0)` — retaining + /// row 0 of a span that covers no row — is byte-identical over + /// every corpus file and fails none of the lib suite, because the + /// three walks that arrive here (the empty file and the two + /// whitespace-only root contracts) all arrive with both line sets + /// already empty. A fixture therefore cannot cover this branch at + /// all; seeding `Stats` directly is the only shape that can + /// (`.claude/rules/testing.md`, "Pair any end-to-end test with a + /// direct unit test on the function whose contract is verified"). + /// + /// The seed is row 0 specifically: it is the one row `(0, 0)` would + /// wrongly keep, so a test seeding any other row would pass under + /// both spellings. + #[test] + fn a_zero_span_keeps_no_row() { + let mut stats = Stats::default(); + // The empty file's span: `0..0`, covering no row at all. + stats.init_unit_span(0, 0); + stats.ploc.lines.insert(0); + stats.cloc.only_comment_line_starts.insert(0); + stats.cloc.code_comment_line_starts.insert(0); + assert_eq!((stats.sloc(), stats.ploc(), stats.cloc()), (0, 1, 1)); + + stats.clamp_line_sets_to_span(); + + assert_eq!( + (stats.ploc(), stats.cloc()), + (0, 0), + "a span of no rows retains no row" + ); + } + + /// The phantom row lands on whichever space was open when the + /// recovery token was visited, so the contract has to hold per + /// space and not only on the file. Elixir's unterminated `do` block + /// is the case that separates the two: before #1398 the `defmodule` + /// space reported `ploc 2` against its own `sloc 1`, and so did the + /// unit, and a unit-only clamp would have left the nested space + /// wrong while the file read as fixed. + /// + /// `space_verbatim`, not `metrics_verbatim`: the claim is about the + /// nested space, whose numbers the root aggregate unions away. + #[cfg(feature = "elixir")] + #[test] + fn the_clamp_reaches_a_nested_space_not_only_the_unit() { + let space = space_verbatim( + crate::LANG::Elixir, + b"defmodule M do\n", + MetricsOptions::default(), + ); + assert_eq!( + (space.metrics.loc.sloc(), space.metrics.loc.ploc()), + (1, 1), + "the unit is one row, all code" + ); + let module = space + .spaces + .first() + .expect("the `defmodule` opens a space even unterminated"); + assert_eq!(module.name.as_deref(), Some("M")); + assert_eq!( + (module.metrics.loc.sloc(), module.metrics.loc.ploc()), + (1, 1), + "the module's own space is one row too — `ploc` was 2 before #1398" + ); + } + + /// The same phantom row reaches CLOC, which has its own contract: + /// `cloc > sloc` pushes MI's comments percentage above 100%, the + /// defect #461 fixed for co-located comments and left open for a + /// comment node ending past end-of-input. An unterminated Perl POD + /// block is the reproduction the #1398 sweep turned up — two rows, + /// `cloc 3`. + /// + /// Pinned alongside the PLOC sweep because the clamp covers all + /// three line sets in one pass; without this row the CLOC half of + /// `Stats::clamp_line_sets_to_span` has no test. + #[cfg(feature = "perl")] + #[test] + fn an_unterminated_pod_block_does_not_comment_a_row_past_eof() { + let loc = + metrics_verbatim(crate::LANG::Perl, b"=pod\nabc\n", MetricsOptions::default()).loc; + // expected: both rows are POD, so both are comment-only and + // neither is code or blank. `cloc` was 3 before #1398. + assert_eq!((loc.sloc(), loc.cloc()), (2, 2)); + assert_eq!((loc.ploc(), loc.blank()), (0, 0)); + } + /// #1135: Tcl and its iRules dialect are the only grammars here that /// surface the row terminator as a token child of the root. `LF`'s /// start row is the row it *terminates*, so the `_` catch-all in diff --git a/src/metrics/loc/line_set.rs b/src/metrics/loc/line_set.rs index 4b11158b3..1b5d5fcef 100644 --- a/src/metrics/loc/line_set.rs +++ b/src/metrics/loc/line_set.rs @@ -22,10 +22,19 @@ //! afterwards. And containment is a property of the *callers*, not of //! this type — `Stats::with_cloc_sloc` already records rows past the //! span's end on purpose, and a bitset that assumed containment would -//! silently drop an out-of-span row rather than fail. Measured over the -//! `pdf.js`, `DeepSpeech` and `serde` corpora, no parsed space records a -//! row outside its span today; a data-derived offset means that stays a -//! fact about the walk rather than a correctness precondition here. +//! silently drop an out-of-span row rather than fail. +//! +//! This header used to add that no parsed space records a row outside +//! its span, measured over the `pdf.js`, `DeepSpeech` and `serde` +//! corpora. That was wrong, and it was the *load-bearing* half of the +//! argument. `DeepSpeech/parse_valgrind_suppressions.sh` does not parse +//! — the grammar leaves a MISSING `}` at `(58, 1)` of a 57-row file — +//! and the row it put in PLOC was one the file does not have (#1398). +//! An anchored bitset would have dropped that row silently instead of +//! letting `Stats::clamp_line_sets_to_span` decide; the data-derived +//! offset is what kept the choice at the caller, which is the reason to +//! prefer it. Out-of-span rows are the norm on malformed input, not an +//! absent case. //! //! # Density //! @@ -143,6 +152,53 @@ impl LineSet { } } + /// Removes every row outside the inclusive range `first..=last`. + /// + /// An inverted range (`last < first`) empties the set, which is how + /// a caller spells "this span covers no row at all" — the empty + /// file's `0..0`. + /// + /// `clear()` there rather than zeroing the words in place, though + /// the two are indistinguishable through every method on this type: + /// `reserve` re-seeds `first_word` from an empty `words`, and reads + /// of an all-zero array answer the same as reads of an absent one. + /// Measured — swapping one for the other fails no test. It is the + /// cheaper of two equals, not a behaviour. + /// + /// One pass over the held words, the same order as + /// [`LineSet::len`], so a caller running it once per space adds a + /// constant factor to work `compute_minmax` already does. Per + /// *node* it would be quadratic in the span, as every method here + /// that is not `insert` would be. + pub(super) fn retain_range(&mut self, first: usize, last: usize) { + if last < first { + self.words.clear(); + return; + } + let (first_word, last_word) = (word_of(first), word_of(last)); + // Bits at or above `first`'s position within its word, and bits + // at or below `last`'s. Both shift amounts are in + // `0..BITS_PER_WORD`, mirroring `insert_range`. + let from_first = u64::MAX << (first % BITS_PER_WORD); + let through_last = u64::MAX >> (BITS_PER_WORD - 1 - last % BITS_PER_WORD); + // Copied out because the loop borrows `self.words` mutably. + let base = self.first_word; + for (index, word) in self.words.iter_mut().enumerate() { + let absolute = base + index; + if !(first_word..=last_word).contains(&absolute) { + *word = 0; + continue; + } + // Both masks apply when the range lies inside one word. + if absolute == first_word { + *word &= from_first; + } + if absolute == last_word { + *word &= through_last; + } + } + } + /// Removes `row`. A no-op when it is absent. #[inline] pub(super) fn remove(&mut self, row: usize) { @@ -578,6 +634,105 @@ mod tests { assert!(!unit.contains(ROWS + 1)); } + /// `retain_range` keeps exactly the rows inside the inclusive + /// bounds, at both ends of a word and across word boundaries. + /// + /// The rows are chosen so a mask built off-by-one at either end + /// changes the answer: `first` and `last` are themselves in the + /// set, and so is a row immediately outside each bound. + #[test] + fn retain_range_keeps_the_bounds_and_drops_everything_else() { + let mut set = set_of(&[ + 0, + 2, + 3, + BITS_PER_WORD - 1, + BITS_PER_WORD, + BITS_PER_WORD + 5, + 2 * BITS_PER_WORD, + 2 * BITS_PER_WORD + 1, + ]); + set.retain_range(3, 2 * BITS_PER_WORD); + + assert_eq!( + rows_of(&set), + vec![ + 3, + BITS_PER_WORD - 1, + BITS_PER_WORD, + BITS_PER_WORD + 5, + 2 * BITS_PER_WORD, + ] + ); + } + + /// A range narrower than one word masks from both ends of the same + /// word, the branch `insert_range` spells `first == last`. + #[test] + fn retain_range_within_one_word_masks_both_ends() { + let mut set = set_of(&[10, 11, 12, 13, 14]); + set.retain_range(11, 13); + assert_eq!(rows_of(&set), vec![11, 12, 13]); + } + + /// An inverted range is how a caller spells "this span covers no + /// row" — the empty file's `0..0` in + /// `loc::Stats::clamp_line_sets_to_span`. It must empty the set + /// rather than retain row 0. + /// + /// Seeded from `[0, 1, 400]` rather than a default set, so the + /// emptiness is something the call produced. The trailing + /// `insert` is a usability check, not a guard: it holds whether + /// the inverted branch clears the array or zeroes it in place + /// (measured — swapping them fails nothing), because `reserve` + /// grows correctly from either state. + #[test] + fn retain_range_inverted_empties_the_set() { + let mut set = set_of(&[0, 1, 400]); + set.retain_range(1, 0); + assert_eq!(set.len(), 0); + assert!(!set.contains(0)); + set.insert(9_000); + assert_eq!(rows_of(&set), vec![9_000]); + } + + /// Retaining a range that already contains every row is a no-op, + /// which is what the clamp does on every well-formed file. + #[test] + fn retain_range_covering_the_whole_set_changes_nothing() { + let rows = &[0, 7, BITS_PER_WORD + 3, 1_000]; + let mut set = set_of(rows); + set.retain_range(0, 1_000); + assert_eq!(rows_of(&set), rows.to_vec()); + } + + /// Whole words on either side of the range are cleared, not just + /// the rows sharing a word with a bound. + /// + /// The two masks and the two word indices are separate mistakes, + /// and only this shape separates them: every bound here sits at a + /// word boundary, so both masks are `u64::MAX` and cannot hide an + /// index that is off by one. Measured — `word_of(first) - 1` fails + /// nothing without this test, and the clamp reaches `first > 63` + /// on any nested space in a file longer than 64 rows. + #[test] + fn retain_range_drops_whole_words_on_either_side() { + const W: usize = BITS_PER_WORD; + let mut set = set_of(&[1, W + 1, 2 * W + 7, 3 * W, 3 * W + 4, 4 * W, 5 * W + 2]); + set.retain_range(3 * W, 4 * W - 1); + assert_eq!(rows_of(&set), vec![3 * W, 3 * W + 4]); + } + + /// A range entirely outside the held words clears the set without + /// indexing past it — the shape that would panic if the loop used + /// absolute word indices as slot indices. + #[test] + fn retain_range_disjoint_from_the_set_clears_it() { + let mut set = set_of(&[5_000, 5_001]); + set.retain_range(0, 10); + assert_eq!(set.len(), 0); + } + /// `LineSet` is a bitset, so its derived-looking `Debug` is /// hand-written to print rows rather than words. Sparse, non-adjacent /// rows spanning more than one word make a word-order or diff --git a/src/spaces/compute.rs b/src/spaces/compute.rs index b002076cd..4c9d41a33 100644 --- a/src/spaces/compute.rs +++ b/src/spaces/compute.rs @@ -206,12 +206,38 @@ fn anchor_unit_sloc_span(state: &mut State, selected: MetricSet) { } } -/// Runs the per-space finalization passes (unit-span anchoring, min/max, -/// sum, Halstead, MI, WMC, averages) on a single [`State`]. Shared by both -/// the single-element and pop arms of [`finalize`] so the call sequence -/// stays identical in both, and reached exactly once per space — every -/// state is finalized either when it is popped or, for the root, in the -/// single-element arm. +/// Discards PLOC / CLOC rows a grammar's error recovery placed outside +/// the space's own row span, so `ploc <= sloc` holds for malformed input +/// as it already does for well-formed input (#1398). +/// +/// Runs per space rather than only on the unit, because the phantom row +/// lands on whichever space was open when the recovery token was +/// visited — an unterminated Elixir `do` block violated the contract on +/// its own `defmodule` space as well as on the file. Child spaces are +/// clamped before they merge upward and a parent's span contains each +/// child's, so a parent's later clamp cannot take back a row a child +/// legitimately contributed. +/// +/// Ordered after [`anchor_unit_sloc_span`] so it clamps against the +/// unit's *final* span. Before that pass the unit's recorded span still +/// starts at the root node's first token, and while no row currently +/// sits between that token and line 1 that either line set records — +/// blank rows are in neither, and a leading comment is in the tree, so +/// the root starts on it — depending on that is the coupling #1247 +/// removed rather than a property to lean on again. +#[inline] +fn clamp_loc_line_sets(state: &mut State, selected: MetricSet) { + if selected.contains(Metric::Loc) { + state.space.metrics.loc.clamp_line_sets_to_span(); + } +} + +/// Runs the per-space finalization passes (unit-span anchoring, line-set +/// clamping, min/max, sum, Halstead, MI, WMC, averages) on a single +/// [`State`]. Shared by both the single-element and pop arms of +/// [`finalize`] so the call sequence stays identical in both, and +/// reached exactly once per space — every state is finalized either +/// when it is popped or, for the root, in the single-element arm. /// /// [`anchor_unit_sloc_span`] runs first because everything after it reads /// the span it fixes: `compute_minmax` folds `sloc` into the unit's @@ -227,6 +253,7 @@ fn anchor_unit_sloc_span(state: &mut State, selected: MetricSet) { /// maps anyway (#1106). fn finalize_state(state: &mut State, selected: MetricSet) { anchor_unit_sloc_span(state, selected); + clamp_loc_line_sets(state, selected); compute_minmax(state, selected); compute_sum(state, selected); compute_halstead_and_mi::(state, selected); From a0a1e925b8848cfa46e04b7456a3fe1d0f647b61 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 20:28:01 -0700 Subject: [PATCH 11/22] fix(abc/csharp): stop double-counting pattern operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A C# `relational_pattern`'s comparison operator scored an ABC condition of its own, on top of the `switch_expression_arm` or `if` condition slot that owns it. A relational arm therefore scored twice what the equivalent constant arm scores, and twice C#'s own cyclomatic decision count. Both spellings are fixed, because the pattern can use any of the four relational tokens and they reach two different arms: `<` / `>` lose their `RelationalPattern` entry in the parent allowlist, and `<=` / `>=` gain a `RelationalPattern` denial they never had. Fixing only the first would have left `x switch { >= 10 => … }` double-counted while `x switch { > 10 => … }` was not. This changes a published metric for any C# file using pattern matching. Over the two C# corpora (27 files, 291 spaces) the aggregate falls from 369 to 348 conditions; the DeepSpeech corpus predates C# patterns and is unchanged, so all of it lands on one snapshot fixture, whose `Bucket` goes 13 -> 6. The gate is on the operator's parent, not on what encloses the pattern, so a relational pattern outside any decision slot (`bool b = x is > 5;`) drops from 1 to 0 as well. That puts it in agreement with the plain type test `x is int` and with cyclomatic, where it previously disagreed with both; the price is that the equivalent `x > 5` still scores 1 in the same slot. Pinned by `csharp_relational_pattern_outside_a_decision_slot_scores_zero`. Two adjacent divergences are measured and deliberately left alone. An operator inside a `when` guard keeps counting, so ABC can still exceed `cyclomatic() - 1` on a guarded arm — but not because either metric models guards: neither does, and a call-shaped guard (`when IsEven(x)`) is at parity, so the count follows the guard's spelling (#1422). And the `<=` / `>=` / `==` / `!=` operator overloads still score a spurious condition, which is residue of #1297 rather than of this fix (#1420), marked with a FIXME at the arm. The book's C# row said the opposite of what the code now does, so it is corrected here rather than left to drift; the `## [Unreleased]` #1297 entry needs the same correction and is left to the batch's changelog commit. Fixes #1383 --- big-code-analysis-book/src/metrics.md | 2 +- src/metrics/abc.rs | 299 +++++++++++++++++++++++--- src/metrics/abc/csharp.rs | 76 ++++--- 3 files changed, 323 insertions(+), 54 deletions(-) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index 62df965d7..db0418af9 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -151,7 +151,7 @@ application would over-count. | Ruby | Bare-predicate `if` / `unless` / `while` / `until` (block and modifier forms) count one condition | Idiomatic Ruby favours bare predicates (`if flag`, `x if flag`); counting the condition slot keeps ABC conditions at or above Ruby's cyclomatic decision count (the alignment enforced across the other languages). A comparison (`if a == b`) or `&&` / `\|\|` chain in the predicate is counted by its own operator / walker arm and is not double-counted. | | Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` therefore contributes nothing: it carries no branch keyword, so it falls outside the rule set rather than through a gap in it. | | Kotlin | `try` counts a condition alongside `catch` | Fitzpatrick counts both keywords, and Java / C# / C++ / Groovy already count both; Kotlin previously counted only the catch block. | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TypeScript, TSX, JavaScript and Mozjs; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally *keeps* the operator of a relational pattern (`x is > 0`), a genuine comparison that lives outside `binary_expression`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TypeScript, TSX, JavaScript and Mozjs; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | #### Worked example diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index e5510ca96..0aedf7c81 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -498,6 +498,71 @@ mod tests { assert_eq!(deepest.metrics.abc.conditions(), decisions); } + // The sibling form of the helper above, for a fixture whose point is + // that several *different* spellings of one construct agree. The + // deepest-space walk cannot serve those: it follows `spaces.last()` + // and so inspects exactly one member, which is the shape #1383's + // over-count hid behind — a merged root total cannot tell a + // relational method scoring 2 from the constant control scoring 2. + fn assert_every_member_scores( + container: &crate::FuncSpace, + members: usize, + expected: u64, + why: &str, + ) { + // The count is a parameter rather than a `!is_empty()` check so + // a caller cannot omit it: without it a fixture that lost every + // member but one still satisfies "every member scores N". + assert_eq!( + container.spaces.len(), + members, + "member count changed — the fixture moved, not the metric" + ); + for member in &container.spaces { + let name = member.name.as_deref().unwrap_or("?"); + assert_eq!(member.metrics.abc.conditions(), expected, "{name}: {why}"); + assert_eq!( + member.metrics.abc.conditions(), + member.metrics.cyclomatic.cyclomatic() - 1, + "{name}: §8 parity with the cyclomatic decision count" + ); + } + } + + // #1383's fix makes a relational pattern's operator score *zero*, so + // its tests have no second axis to anchor on: trimming `> 5` down to + // `5` leaves every assertion satisfied and the construct under test + // gone (`.claude/rules/testing.md`, "Perturb the fixture as well as + // the production line"). Asserting the kind ids are still present in + // the parsed fixture is the anchor that replaces it — editing a + // spelling out of the source now fails here by name instead of + // silently turning the method into a copy of the control. + // Counts rather than presence, because these fixtures carry several + // methods spelling the same construct: a bare `ast_has_kind_id` is + // still satisfied after one method loses its pattern, which is + // exactly the decay that turns that method into a silent duplicate + // of the control. Measured — with presence-only anchoring, rewriting + // `if (x is > 0)` to `if (x > 0)` in one method of five failed + // nothing. + fn assert_csharp_fixture_spells(src: &str, kinds: &[(u16, usize, &str)]) { + let parser = CsharpParser::new( + src.as_bytes().to_vec(), + std::path::Path::new("foo.cs"), + None, + ); + for (kind, want, spelling) in kinds { + let found = parser + .root() + .preorder() + .filter(|n| n.kind_id() == *kind) + .count(); + assert_eq!( + found, *want, + "fixture has {found} of {spelling}, expected {want} — the construct under test was edited" + ); + } + } + /// Regression for #227: a `Stats::default()` that never sees an /// observation must not leak the `f64::MAX` sentinel for /// `assignments_min`, `branches_min`, or `conditions_min`. All @@ -3679,11 +3744,11 @@ mod tests { // // The fixture carries the two overloads plus one `a < b` inside a // `binary_expression` and one `x is > 0` inside a - // `relational_pattern`, which is the second decision parent in the - // allowlist. Every mis-aim lands on its own number: 5 pre-fix, 3 - // once both decision parents are allowed, 2 if `RelationalPattern` - // is dropped, 1 if the gate swallows `BinaryExpression` too (only - // the ternary `?` survives), 0 if the fixture stops parsing. + // `relational_pattern`. Every mis-aim lands on its own number: 5 + // with neither gate, 3 if `RelationalPattern` is readmitted to the + // allowlist (#1383 dropped it), 1 if the gate swallows + // `BinaryExpression` too (only the ternary `?` survives), 0 if the + // fixture stops parsing. #[test] fn csharp_operator_declaration_is_not_a_condition() { check_func_space::( @@ -3698,7 +3763,7 @@ mod tests { "foo.cs", |space| { // Assert the claim per space rather than through the - // file total, which is 3 with the two overloads and 3 + // file total, which is 2 with the two overloads and 2 // without them — an aggregate assertion would pass on a // fixture that had lost the very construct under test. let class = &space.spaces[0]; @@ -3710,36 +3775,214 @@ mod tests { "operator declaration {i} must score no condition" ); } - assert_eq!(class.spaces[2].metrics.abc.conditions(), 3); + // 2, not 3, since #1383: the `a < b` comparison and the + // ternary `?`. `m`'s `x is > 0` no longer adds a third + // — the ternary it sits in is the decision. + assert_eq!(class.spaces[2].metrics.abc.conditions(), 2); }, ); } - // The `RelationalPattern` half of the allowlist above, on its own - // input. Per `.claude/rules/grammar-dispatch.md` §11, the fixture - // above cannot prove that entry alone: its `is > 0` sits beside a - // `binary_expression` comparison, so a gate allowing only - // `BinaryExpression` would still leave a non-zero, plausible total. - // Here the relational operators are the *only* `<` / `>` in the - // file, so the entry is the only thing that can produce the count. + // #1383: a `relational_pattern`'s operator is not a condition of + // its own — the `switch_expression_arm` that owns it already scores + // the decision, exactly as it does for the constant arm `5 => 1`. + // Counting both charged a relational arm twice, and twice C#'s own + // cyclomatic decision count. // - // 4, not 2: a `switch_expression_arm` is counted by its own arm - // above and the pattern's operator by this one, so a relational arm - // scores twice what the constant arm `5 => 1` scores. That - // divergence from C#'s own cyclomatic decision count predates - // #1297 — the old denylist did not name `relational_pattern` - // either — and is filed as #1383 rather than changed here, which is - // why this asserts the value the gate preserves rather than the §8 - // parity value. - #[test] - fn csharp_relational_pattern_still_counts_as_a_condition() { - check_metrics::( - "class A { + // Per `.claude/rules/grammar-dispatch.md` §11 the `operator <` + // fixture above cannot prove this alone: its `is > 0` sits beside a + // `binary_expression` comparison that supplies a plausible total on + // its own. Here the pattern operators are the *only* `<` / `>` / + // `>=` / `<=` in the file, so a readmitted `RelationalPattern` + // parent is the only thing that can lift the count. + // + // Both spellings are covered because they reach two different arms: + // `n`'s `>` / `<` are gated by the `GT | LT` parent allowlist, + // `g`'s `>=` / `<=` by their own `RelationalPattern` denial. Fixing + // one arm and not the other leaves the *other method* at 4, which + // is why this asserts per member and not through a total: + // + // | state | `n` | `g` | `c` | + // |---|---|---|---| + // | both halves gated (shipped) | 2 | 2 | 2 | + // | only `GT \| LT` gated | 2 | 4 | 2 | + // | only `GTEQ \| LTEQ` gated | 4 | 2 | 2 | + // | neither | 4 | 4 | 2 | + // + // `c` is the constant-pattern control and reads 2 in every column: + // it is what the relational methods are supposed to agree with. + #[test] + fn csharp_relational_pattern_does_not_double_count_its_arm() { + let src = "class A { int n(int x) => x switch { > 5 => 1, < 0 => 2, _ => 3 }; + int g(int x) => x switch { >= 5 => 1, <= 0 => 2, _ => 3 }; + int c(int x) => x switch { 5 => 1, 0 => 2, _ => 3 }; + }"; + assert_csharp_fixture_spells( + src, + &[ + (Csharp::RelationalPattern as u16, 4, "relational patterns"), + (Csharp::GT as u16, 1, "`>`"), + (Csharp::LT as u16, 1, "`<`"), + (Csharp::GTEQ as u16, 1, "`>=`"), + (Csharp::LTEQ as u16, 1, "`<=`"), + ], + ); + check_func_space::(src, "foo.cs", |space| { + // Per space, not through the root: the root can only see + // the sum, and a `{4, 2, 0}` regression sums to the same 6 + // as the correct `{2, 2, 2}`. `cyclomatic()` parity is also + // only expressible per space — `cyclomatic_sum()` folds in + // a base of 1 for each one. + assert_every_member_scores(&space.spaces[0], 3, 2, "one condition per non-discard arm"); + }); + } + + // The `is` half of #1383, and the one whose over-count came from a + // different owner: `if (x is > 0)` scored the `IfStatement` + // condition slot *and* the pattern's `>`. `f`'s guard is the + // deliberate control — a `binary_expression` comparison in the same + // slot still scores exactly one, so the fix cannot be read as + // "stop counting `if` conditions". + // + // The two combinator methods pin the compound forms the issue asks + // about. `and` / `or` are keyword tokens C# ABC never listed and C# + // cyclomatic never counts, so the arm stays the single decision + // however many relational operands it carries — which is exactly + // what would regress if a later fix re-derived the gate from the + // operand instead of the parent. + #[test] + fn csharp_is_pattern_and_combinators_score_one_decision() { + let src = "class A { + int m(int x) { if (x is > 0) { return 1; } return 0; } + int f(int x) { if (x > 0) { return 1; } return 0; } + int a(int x) { if (x is > 0 and < 9) { return 1; } return 0; } + int o(int x) { if (x is < 0 or >= 100) { return 1; } return 0; } + int n(int x) { if (x is not > 5) { return 1; } return 0; } + }"; + assert_csharp_fixture_spells( + src, + &[ + (Csharp::RelationalPattern as u16, 6, "relational patterns"), + (Csharp::IsPatternExpression as u16, 4, "`is` pattern tests"), + (Csharp::AndPattern as u16, 1, "`and`"), + (Csharp::OrPattern as u16, 1, "`or`"), + (Csharp::NegatedPattern as u16, 1, "`not`"), + (Csharp::GTEQ as u16, 1, "`>=` (only `o` carries one)"), + ], + ); + check_func_space::(src, "foo.cs", |space| { + assert_every_member_scores(&space.spaces[0], 5, 1, "the `if` is the only decision"); + }); + } + + // #1383's second, quieter effect, and the one its issue does not + // mention: a relational pattern outside any decision slot went from + // 1 to 0 as well, because the gate is on the operator's parent and + // not on what encloses the pattern. + // + // That is the right side of the trade, but it is a trade and the + // numbers should be visible. It puts `x is > 5` in agreement with + // the plain type test `x is int` (`t`, always 0 — note the grammar + // spells that one `is_expression`, not a pattern at all) and with + // cyclomatic, where before the fix it disagreed with both. The + // price is `q`: + // the equivalent binary comparison `x > 5` still scores 1 in the + // same slot, so a relational pattern now reads one lower than the + // comparison it is sugar for. Fitzpatrick counts comparison + // operators wherever they appear, so `q` is the spec-faithful one + // and these four are the deliberate exception — kept because the + // decision-slot case is what the metric is for, and Option 2 in + // #1383 (count the operator, drop the arm) could not justify + // itself. + #[test] + fn csharp_relational_pattern_outside_a_decision_slot_scores_zero() { + let src = "class A { + static bool M(bool b) { return b; } + bool p(int x) { bool b = x is > 5; return b; } + bool q(int x) { bool b = x > 5; return b; } + bool r(int x) { return x is > 5; } + bool s(int x) { return M(x is > 5); } + bool t(object x) { bool b = x is int; return b; } + }"; + assert_csharp_fixture_spells( + src, + &[ + (Csharp::RelationalPattern as u16, 3, "relational patterns"), + (Csharp::IsPatternExpression as u16, 3, "`is` pattern tests"), + (Csharp::IsExpression as u16, 1, "the `is int` control"), + ], + ); + check_func_space::(src, "foo.cs", |space| { + let class = &space.spaces[0]; + assert_eq!(class.spaces.len(), 6, "`M` plus five probes"); + let by_name = |n: &str| { + class + .spaces + .iter() + .find(|s| s.name.as_deref() == Some(n)) + .unwrap_or_else(|| panic!("fixture lost `{n}`")) + .metrics + .abc + .conditions() + }; + // Named rather than indexed: the claim is about which + // spelling scores what, so a reordering of the fixture must + // not silently re-point the assertions. + for probe in ["p", "r", "s"] { + assert_eq!(by_name(probe), 0, "`{probe}`: pattern operator excluded"); + } + assert_eq!(by_name("t"), 0, "type pattern, the agreement target"); + assert_eq!(by_name("q"), 1, "a plain comparison still counts"); + }); + } + + // The boundary of #1383: the pattern's operator stops counting, an + // operator in the arm's `when` guard keeps counting. Both sit under + // the same `switch_expression_arm`, so this is what stops a later + // "patterns don't count" pass from suppressing the guard too. + // + // It is also where §8 does not hold, and the reason is worth + // stating precisely, because the obvious reading is wrong. Neither + // metric models the guard as a branch — C# cyclomatic has no + // `when_clause` arm, and ABC has no guard rule either. ABC's extra + // count is simply the `==` *token*, which happens to sit inside the + // guard. Measured: + // + // | guard | `conditions()` | `cyclomatic() - 1` | + // |---|---|---| + // | `when x % 2 == 0` (this fixture) | 3 | 2 | + // | `when x > 2` | 3 | 2 | + // | `when IsEven(x)` | 2 | 2 | + // + // So the gap is not "ABC models guards better"; it is that a + // call-shaped guard restores parity while an operator-shaped one + // does not. That inconsistency is real, predates #1383, and is + // filed rather than changed here — see #1422. + // + // The `==` is the guard's own operator and the only `==` in the + // file, so trimming the `when` clause out of the fixture drops the + // count to 2 rather than leaving the assertion satisfied by + // something else. + #[test] + fn csharp_switch_arm_guard_operator_still_counts() { + check_func_space::( + "class A { + int w(int x) => x switch { > 0 when x % 2 == 0 => 1, > 0 => 2, _ => 3 }; }", "foo.cs", - |metric| { - assert_eq!(metric.abc.conditions_sum(), 4); + |space| { + let m = &space.spaces[0].spaces[0]; + assert_eq!(m.name.as_deref(), Some("w")); + // 4 if the `when_clause` itself started counting, 2 if + // the guard's `==` were suppressed along with the + // pattern operators — both are live regressions. + assert_eq!( + m.metrics.abc.conditions(), + 3, + "two arms plus the guard's `==`" + ); + assert_eq!(m.metrics.cyclomatic.cyclomatic(), 3); }, ); } diff --git a/src/metrics/abc/csharp.rs b/src/metrics/abc/csharp.rs index 2b5a25e16..892004abf 100644 --- a/src/metrics/abc/csharp.rs +++ b/src/metrics/abc/csharp.rs @@ -107,10 +107,10 @@ fn csharp_count_unary_conditions(list_node: &Node, conditions: &mut f64) { // ABC token-level helpers for C#. Mirror of Java's helper layout with // C#-specific deltas: every aliased kind id is matched via the // `csharp_*_kinds!()` macros (lesson #2); `ObjectCreationExpression` -// joins `InvocationExpression*` as a branch; the `<` / `>` parent -// allowlist names `RelationalPattern` alongside the two -// `BinaryExpression` ids, the only such second entry in the workspace, -// because a C# comparison can sit outside a binary expression; +// joins `InvocationExpression*` as a branch; each of the four tokens a +// `relational_pattern` can spell (`<` `>` `<=` `>=`) excludes that +// parent, because a C# pattern's operator belongs to the arm that owns +// it rather than scoring on its own (#1383); // `ConditionalExpression` replaces Java's `TernaryExpression`; // `for_statement` exposes its condition via the named `condition` // field rather than positional index. @@ -222,7 +222,32 @@ fn csharp_count_token_condition<'a>( // arrow `default ->` forms) is the unconditional fallthrough and // is excluded, mirroring cyclomatic's `Case`-only count and the // expression-arm discard rule below (issues #456, #469). - GTEQ | LTEQ | EQEQ | BANGEQ | Else | Case | Try | Catch => { + EQEQ | BANGEQ | Else | Case | Try | Catch => { + stats.conditions += 1.; + } + // The other half of #1383. A `relational_pattern` spells its + // operator as any of `<` `>` `<=` `>=`, and the last two are + // distinct tokens that never reach the `GT | LT` arm below, so + // fixing only that arm would leave `x switch { >= 10 => … }` + // double-counted while `x switch { > 10 => … }` was not — the + // same bug surviving under a different token. The arm that owns + // the pattern is the decision; see the `GT | LT` comment. + // + // Denylist polarity here rather than the allowlist the `GT` / + // `LT` arm uses, because these two tokens have no type syntax + // to fail closed against: `<=` and `>=` reach this arm from + // `binary_expression`, `relational_pattern` and + // `operator_declaration` only. That third parent is the + // `>=` / `<=` spelling of #1297's operator-overload bug, which + // that fix reached only through the `GT` / `LT` allowlist and + // which `==` / `!=` share. + // + // FIXME(#1420): `operator <=` / `>=` / `==` / `!=` still score + // a spurious condition each, measured. Left alone here so this + // commit changes one behaviour; the fix is to give all four + // tokens the `BinaryExpression` allowlist polarity, which + // subsumes the denial below. + GTEQ | LTEQ if !ancestors.parent_has_kind(node, RelationalPattern as u16) => { stats.conditions += 1.; } // tree-sitter-c-sharp emits a bare `?` from exactly four @@ -287,37 +312,38 @@ fn csharp_count_token_condition<'a>( { stats.conditions += 1.; } - // Counts `<` / `>` only where they are a comparison. A - // `grammar.json` sweep of tree-sitter-c-sharp 0.23.5 finds a - // bare `<` / `>` in exactly six productions, and two of them - // are decisions: `binary_expression` (`a < b`) and - // `relational_pattern` (`x is > 0`, and the `> 5 =>` arm of a - // switch expression). The other four are type syntax — - // `type_argument_list` (`Dictionary`), + // Counts `<` / `>` only where they score a decision of their + // own. A `grammar.json` sweep of tree-sitter-c-sharp 0.23.5 + // finds a bare `<` / `>` in exactly six productions, and only + // `binary_expression` (`a < b`) qualifies. Four are type + // syntax — `type_argument_list` (`Dictionary`), // `type_parameter_list` (`class Foo`), `function_pointer_type` // (`delegate*`) and `operator_declaration` // (`public static bool operator <(V a, V b)`), whose `<` names - // the operator being *defined* rather than applying it. + // the operator being *defined* rather than applying it. The + // sixth is `relational_pattern`, excluded for a different + // reason — see below. // // The previous denylist named three of those four and not // `operator_declaration`, so every comparison-operator overload // scored a condition per declaration (#1297). Allowlist // polarity, matching Java's #1274 fix and unlike the `QMARK` - // arm above: `<` / `>` have two decision parents against four - // type-syntax ones, and a grammar bump that grows a seventh - // production should fail closed + // arm above: `<` / `>` have one decision parent against five + // excluded productions, and a grammar bump that grows a seventh + // should fail closed // (`.claude/rules/grammar-dispatch.md` §1). The `QMARK` arm // takes the opposite polarity for a reason specific to that // token — see its comment. // - // `relational_pattern` is in the allowlist to preserve the - // count, not to add it: the previous denylist did not name it - // either, so the pattern's operator counted then and counts - // now. That count double-charges the enclosing - // `switch_expression_arm` or `if` condition slot, a - // pre-existing divergence from C#'s own cyclomatic decision - // count tracked in #1383 — settling it here would have been an - // unmeasured behaviour change riding along with #1297. + // `relational_pattern` is deliberately *not* in the allowlist + // (#1383). A pattern's comparison operator is not its own + // decision: the enclosing `switch_expression_arm` or `if` + // condition slot already scores one, so counting the operator + // too charged `x switch { > 5 => … }` twice what the constant + // arm `x switch { 5 => … }` scores and twice C#'s own + // cyclomatic decision count. #1297 kept the entry to avoid an + // unmeasured behaviour change riding along with it; this is + // that measured change. // // `BinaryExpression2` is the id the grammar aliases // `preproc_binary_expression` to, and it is listed defensively @@ -331,7 +357,7 @@ fn csharp_count_token_condition<'a>( if ancestors.parent(node).is_some_and(|parent| { matches!( parent.kind_id().into(), - BinaryExpression | BinaryExpression2 | RelationalPattern + BinaryExpression | BinaryExpression2 ) }) => { From 65b0ae777eda0993d46843a8e9014b8ff6b7523a Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 20:54:49 -0700 Subject: [PATCH 12/22] test(corpus): record the refreshed snapshot submodule Points at c03ffa97, which refreshes csharp/control_flow.cs for #1383 and php/strings.php for #1396. Both diffs are metric-value-only. The submodule commit is not yet pushed to its remote; it must be pushed before this branch is, or a fresh clone cannot fetch the recorded SHA. --- tests/repositories/big-code-analysis-output | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/repositories/big-code-analysis-output b/tests/repositories/big-code-analysis-output index 3cd242f27..c03ffa977 160000 --- a/tests/repositories/big-code-analysis-output +++ b/tests/repositories/big-code-analysis-output @@ -1 +1 @@ -Subproject commit 3cd242f27e0e74a94b821639b3db64fab8d2c367 +Subproject commit c03ffa977062d691ecc54cf66eeb21ed1f1a3a12 From 0dbb1b7f3539097954205017a0d9ef13c424bb1f Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 20:56:02 -0700 Subject: [PATCH 13/22] docs(changelog): consolidate entries from the batch fix Ten entries under [Unreleased], each naming the metric drift it introduces where values move. Also corrects the #1297 entry: it recorded that C# "keeps counting a relational pattern's operator", which #1383 reverses. --- CHANGELOG.md | 153 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c844ffc98..f747b29b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,152 @@ for historical reference. ### Fixed +- **A grammar span reaching past end-of-input no longer counts as a line + of code** (#1398), so `loc.ploc` and `loc.cloc` can no longer exceed a + space's own row span. A childless zero-width recovery token placed one + row past the last physical row was inserted into PLOC verbatim, making + the arithmetic identity `blank = sloc − ploc − cloc` read as satisfied + (`blank` saturates at 0) while `ploc / sloc` exceeded 1. A sweep of 322 + truncated fixtures across 23 languages found the shape in **ten** — + Bash, C, C++, Mozcpp, Objective-C, Elixir, Groovy, Lua, Perl and Ruby + — over four unrelated recovery shapes (heredoc, unterminated string + delimiter, unclosed bracket, dangling line continuation), so each + space's line sets are now clamped to its own span at finalization + rather than per arm. A `cloc > sloc` case (Perl POD) of the same + mechanism is fixed with it. Note a clean parse is not evidence of an + in-span tree: Ruby's `x = <<~DOC` with an unterminated body parses + without an error node and still emits the phantom row. **Metric + drift:** `loc.ploc` / `loc.cloc` fall and `loc.blank` rises on files a + grammar cannot fully parse, including at least one ordinary shell + script in the DeepSpeech corpus. + +- **ABC counts a bare numeric literal operand in Ruby, Elixir and Perl** + (#1379). All three are truthy-valued, so a number used as a bare `&&` + / `||` operand is a Fitzpatrick Rule 9 unary condition — but their + terminal-operand sets named `integer` alone (Ruby, Elixir) or no + numeric kind at all (Perl). Ruby scored `a && 1.0`, `a && 1r`, + `a && 2i` and `a && 1ri` one condition against `a && 1`'s two; Perl + scored even `$a && 1` one against `$a && $b`'s two, and `if (1)` zero + against Python's one for `if 1:`. Each set now names every member of + its grammar's numeric family — five sibling rules for Perl + (`integer`, `floating_point`, `scientific_notation`, `hexadecimal`, + `octal`), and `char` for Elixir, since `?a` is the codepoint 97. + Ruby's `rational` / `complex` wrap the numeral and the walker cannot + descend into a wrapper, so the wrapper is classified and `integer` + stays listed alongside it for the bare `1`. Lua, Tcl, iRules and the + JS family were measured and have no gap. PHP and Groovy carry the same + defect and are tracked in #1410. **Metric drift:** `abc.conditions` + and `abc.magnitude` rise for any Ruby, Elixir or Perl file using a + numeric in a boolean-operand or condition slot. + +- **A relational pattern no longer double-counts its operator against + the arm that owns it** in C# ABC (#1383). `x switch { > 5 => …, < 0 + => … }` scored one condition per arm *plus* each arm's `>` / `<`, + giving 4 where the equivalent constant-pattern switch gives 2 and + where `cyclomatic() - 1` is 2; `if (x is > 0)` gave 2 against 1. The + enclosing switch arm and `is` condition slot already pay for the + decision, so the pattern's own operator is now excluded — matching how + a constant pattern is treated. This covers `>=` and `<=` as well as + `>` and `<`: those are distinct token ids reaching a separate arm, so + the allowlist named in #1297 never saw them. **Metric drift:** + `abc.conditions` and `abc.magnitude` fall for every C# file using + relational patterns; `abc` is a gated threshold metric. Kotlin shares + the defect in `when { x > 5 -> }` and is tracked in #1421. + +- **Kotlin ABC counts a primary-constructor superclass call** (#1384). + `class Sub : Base(1, 2)` parses as a `constructor_invocation` under a + `delegation_specifier` — a third production, distinct from both + `call_expression` and the `constructor_delegation_call` #1279 added + for the secondary form — so the spelling most Kotlin actually uses + contributed nothing while `constructor(x) : super(x)` beside it scored + one. It is now one branch, as is an object expression's superclass + call (`object : Base(1) { }`). The arm is gated on its parent: the + grammar reuses `constructor_invocation` for an annotation's argument + list (`@Suppress("x")`, `@file:Suppress("x")`), which is not a + run-time call and stays at zero. A supertype with no argument list + (`class Sub : Marker`) is unaffected. **Metric drift:** Kotlin `abc` + rises by one per class or object expression passing arguments to its + supertype. + +- **A self-reference and a super-reference are Halstead operands in + Java, C# and Kotlin** (#1380), matching the eleven other languages + that classify one. A member access is ` `, so + billing the receiver as an operator scored `this.x` as a binary + operator with one operand where `p.x` is one operator with two. Two + declarator uses of the same token kinds stay operators behind a parent + gate: C#'s `indexer_declaration`, which names the member with `this`, + and Java's `? super T` wildcard bound, the mirror of `? extends T`. + Kotlin's label-qualified `this@` / `super@` were previously classified + as *nothing* and are now operands. PHP's `self::` / `parent::` are + unchanged — those are scope-resolution class references, not instance + references. **Metric drift:** `n1` / `N1` fall by one occurrence per + self/super reference and `n2` / `N2` rise by the same, carrying + `volume`, `difficulty`, `effort`, `time`, `bugs` and all three + maintainability-index variants with them. + +- **PHP heredoc, nowdoc and backtick rows that are empty inside the + literal are counted as `ploc` rather than `blank`** (#1396), matching + every other language with a multi-line literal. #778 recorded PHP as + already correct; it was not, for the one shape that release never + measured. The nowdoc case was worse than the heredoc case and + structurally different — its body is one node for the first line plus + a single multi-row node for the rest, so it lost every interior row + regardless of emptiness. The wrapper is routed rather than the body, + because a heredoc whose body is a single empty row emits no body node + at all. **Metric drift:** `loc.ploc` rises and `loc.blank` falls for + PHP files containing these literals. + +- **Ruby `npm` no longer counts `initialize`, `initialize_copy`, + `initialize_dup`, `initialize_clone` or `respond_to_missing?` as + public methods** (#1400). Ruby privatises all five at definition, so + any class with a constructor reported one public method too many. + `nm` is unchanged — they are still methods; only the public split + moves. An explicit `public :initialize` or `public def initialize` + still counts as public, and a `class << self` singleton `initialize` + stays public, because the rule is instance-only. **Metric drift:** + `npm` falls by one for most Ruby classes. + +- **`bca find --type string` and `bca count --type string` no longer + report a Tcl or iRules script body as a string literal** (#1381) — a + `proc` or `if` body, or an iRules `when` handler. A braced *value* + such as `lappend x {a b}` is still reported. `Checker::is_string` + could not tell the two apart because it received neither the source + bytes nor the ancestor chain; it gains an `is_string_with_code` + sibling that routes both dialects through the role predicate #1318 + built. The AST dump and the REST `/ast` endpoint likewise keep a + script body's children instead of flattening it to a single leaf, + which had been dropping entire `proc` and `when` bodies. + +- **A `.mailmap` edit invalidates the persistent VCS history cache** + (#1262). Author identities are canonicalised through the repository + mailmap at walk time and stored in the cached event log as digests, + but neither the entry key (`head_sha`) nor the options fingerprint + observed the mailmap — so `bca vcs` served stale `authors_long`, + `ownership_top_share`, bus-factor and `risk_score` values after any + mailmap change. Worse, the incremental splice re-persisted the + pre-edit digests under each new head, so the divergence survived + `HEAD` moving and only `--clear-cache` cleared it. A digest of the + repository's effective mailmap now feeds `cache::fingerprint`, + covering the pure hit, the splice's ancestor selection and the + persisted entry at once. No `CACHE_SCHEMA_VERSION` bump is needed: + pre-fix entries simply fingerprint differently and cost one cold + walk. The digest hashes gix's *merged* mailmap snapshot rather than + re-deriving the four conditional sources `open_mailmap` consults, so + a source cannot be missed. + +- **`bca.to_sarif` emits findings in the same order as `bca check -O + sarif`** (#1402), so the two documents can be compared positionally + and not just as sets — which is what the binding's own parity claim + had been promising. Two divergences are fixed: the binding walked the + space tree with a LIFO stack it pushed in source order, so every + sibling set came out reversed at every level; and it iterated the + `thresholds` dict, so a space breaching several metrics reported them + in the caller's insertion order rather than the CLI's + alphabetical-by-metric order — which also made the output depend on + how the dict happened to be spelled. Order *between* files remains + the caller's: `to_sarif` follows the iterable it is handed, where + `bca check` follows its resolved walk list. + - **`bca preproc` documents are byte-identical across runs** (#1304). `PreprocResults.files` and `PreprocFile`'s three `HashSet` fields serialized straight off hash order, so an unchanged tree @@ -142,8 +288,11 @@ for historical reference. declared name, a Kotlin qualified super call (`super.g()`), and Perl's `` / `<$fh>` readlines each scored phantom conditions. Perl was listed as immune by the original survey and was not. C# - additionally keeps counting a relational pattern's operator - (`x is > 0`), a genuine comparison outside `binary_expression`. + initially kept counting a relational pattern's operator + (`x is > 0`) on the grounds that it is a genuine comparison outside + `binary_expression`; #1383 below reverses that, because the switch + arm or `is` condition owning the pattern already pays for the + decision. Elixir was swept the same way in the same release: an operator *named* rather than applied — the capture `& Date: Thu, 10 Sep 2026 21:07:32 -0700 Subject: [PATCH 14/22] test(ast): cover the empty node-filter fallback `Parser::filters` falls back to a match-all predicate when no arm pushed one; a Filter holding no predicates would make Filter::any return false for every node, so a bare `bca find` / `bca count` would silently report zero. Nothing exercised it. Deleting the fallback failed no test in the crate before this one, and fails only this one after. Asserting against the `all` keyword rather than a hand-counted total is what gives the test teeth: both paths push the identical closure, so dropping the fallback sends the empty request to 0 while `all` stays put. --- big-code-analysis-ast/src/parser.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/big-code-analysis-ast/src/parser.rs b/big-code-analysis-ast/src/parser.rs index 816985d4c..615428e43 100644 --- a/big-code-analysis-ast/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -308,4 +308,32 @@ mod tests { let src = "x = 1\n"; assert_eq!(count_kind(src, "definitely_not_a_python_kind"), 0); } + + #[test] + fn get_filters_empty_request_matches_every_node() { + // Requesting nothing means "match everything": `filters` falls + // back to a match-all predicate when no arm pushed one, because + // a `Filter` holding no predicates would make `Filter::any` + // return `false` for every node and `bca find` / `bca count` + // silently report zero on a bare invocation. + // + // Asserting against `"all"` rather than a hand-counted total is + // what makes this able to fail: the two paths push the identical + // closure, so dropping the fallback sends the empty request to 0 + // while `"all"` stays put. A `> 0` assertion could not tell the + // two apart from a source that simply had nodes. + let src = "if x:\n pass\nelse:\n y = foo(1 + 2)\n"; + let parser = parse_python(src); + let everything = count(&parser, &["all".to_string()]).0; + let unfiltered = count(&parser, &[]).0; + + assert!( + everything > 1, + "fixture must hold several nodes or this asserts nothing; got {everything}" + ); + assert_eq!( + unfiltered, everything, + "an empty filter request must match what `all` matches" + ); + } } From 6e5e9ca108bb7f3d98b32ec2d40d40de451e10df Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Sep 2026 21:31:02 -0700 Subject: [PATCH 15/22] fix: correct false claims found by the whole-branch review Six items, none a behaviour change. Each was a statement in a comment, a changelog entry, or a test that did not describe what the code does. kind_sets.rs: the note added with #1379 said C and C++ name no numeric bool-terminal kind because "a bare number in a boolean slot is a compile error there". They are integer-truthy -- `if (1)` and `a && 1` are idiomatic -- and measure 1 condition where Python measures 4 for the same constructs. Worse, `if (true)` scores 1 and `if (1)` scores 0 inside one language. The set is shared by C, C++, Mozcpp and Objective-C; all four are now named on #1410 alongside PHP and Groovy, deferred for snapshot churn rather than because the rule does not apply. CHANGELOG: #1381 credited "the AST dump", but `bca dump` prints the raw tree-sitter tree and never consulted the alterator -- only `Ast::dump` and the REST `/ast` changed. The same entry read as if the Tcl script-body class were closed; recognition is a leading-word heuristic, so `dict for`, `interp eval` and `apply` are still reported. npm.rs: the comment said no fixture-decay anchor was available for the four tests whose expected answer is "the rule does not apply", so a rename of `initialize` would be silent. No *count* can anchor them, but the space tree can -- the Ruby walk names a space per `def`. `ruby_public_keyword_wrapping_initialize_wins` now carries that anchor, since it is the sole guard on the `named_by_keyword` disjunct; verified by renaming the fixture, which now fails only that test. halstead.rs: `csharp_indexer_declaration_keyword_is_not_a_self_reference` derived the exact role sequence in its comment and asserted two `contains` calls, leaving the length unpinned so a space could appear or vanish unnoticed. Asserts the sequence now. cache_tests.rs: dropped a comparison of a pure fixed-seed function to itself, keeping the note about where cross-process stability is actually covered. getter/groovy.rs: records that the `Super` operator arm never fires -- the grammar emits a plain identifier -- so Groovy agrees with #1380 by accident and would flip on a grammar bump. Tracked in #1419. --- CHANGELOG.md | 11 +++++-- big-code-analysis-ast/src/getter/groovy.rs | 13 +++++++++ big-code-analysis-ast/src/macros/kind_sets.rs | 24 +++++++++++---- src/metrics/halstead.rs | 18 ++++++------ src/metrics/npm.rs | 29 ++++++++++++++----- src/vcs/cache_tests.rs | 18 +++++------- 6 files changed, 79 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f747b29b3..483707b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -233,9 +233,14 @@ for historical reference. could not tell the two apart because it received neither the source bytes nor the ancestor chain; it gains an `is_string_with_code` sibling that routes both dialects through the role predicate #1318 - built. The AST dump and the REST `/ast` endpoint likewise keep a - script body's children instead of flattening it to a single leaf, - which had been dropping entire `proc` and `when` bodies. + built. The `Ast::dump` API and the REST `/ast` endpoint likewise keep + a script body's children instead of flattening it to a single leaf, + which had been dropping entire `proc` and `when` bodies. (The `bca + dump` subcommand prints the raw tree-sitter tree and never consulted + the alterator, so it is unaffected.) Recognition is a leading-word + heuristic, so Tcl's subcommand-dispatched script takers — `dict for`, + `interp eval`, `apply` — are still reported; iRules models its + handlers structurally and has no such gap. - **A `.mailmap` edit invalidates the persistent VCS history cache** (#1262). Author identities are canonicalised through the repository diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index 05f603f70..e03e6c65c 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -132,6 +132,19 @@ impl Getter for GroovyCode { // Control-flow + keyword operators (mirrors Java's set, // minus tokens that no longer exist in the dekobon grammar // — `This`, `VoidType`, `Throws2`). + // + // `Super` is listed but never fires: the grammar spells a + // super-reference as a plain `identifier` in every position + // measured (`super(1)`, `super.h()`, `A.super.h()`, + // `super::h`), so Groovy already bills it as an operand and + // agrees with the decision #1380 settled for Java, C# and + // Kotlin — by grammar accident rather than by this arm. The + // arm is therefore a latent disagreement: a bump that starts + // emitting `Groovy::Super` would flip Groovy to operator with + // no test failing here. Tracked in #1419, which owes it + // either a §2 unreachability pin or a move to the operand + // arm. `tests/parity/self_reference_operand_parity.rs` is + // what currently catches the flip, one crate away. If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Super | Def | In | As diff --git a/big-code-analysis-ast/src/macros/kind_sets.rs b/big-code-analysis-ast/src/macros/kind_sets.rs index b21c7ab16..2b06b7659 100644 --- a/big-code-analysis-ast/src/macros/kind_sets.rs +++ b/big-code-analysis-ast/src/macros/kind_sets.rs @@ -336,11 +336,25 @@ macro_rules! python_bool_terminal_kinds { // starts emitting it should count it, so there is nothing to guard // against by omission (grammar-dispatch §2). // -// The statically-typed sets (C#, Java, Kotlin, Rust, Go, C, C++) -// deliberately name no numeric kind: a bare number in a boolean slot is -// a compile error there, so there is nothing to count. PHP and Groovy -// are the two remaining truthy-valued languages that still omit one — -// tracked in #1410, not deliberate. +// The sets for C#, Java, Kotlin, Rust and Go deliberately name no +// numeric kind: a bare number in a boolean slot is a compile error in +// those five, so there is nothing to count. +// +// That rationale does **not** extend to the C family, which an earlier +// revision of this comment wrongly grouped with them. C and C++ are +// integer-truthy — `if (1)`, `while (1)`, `do { … } while (0)` and +// `a && 1` are all legal and idiomatic — so they carry the same gap PHP +// and Groovy do. `cpp_bool_terminal_kinds!` is name-keyed and shared by +// C, C++, Mozcpp and Objective-C, so all four are affected, and the +// omission is visible *within* one language: `if (true)` scores one +// condition and `if (1)` scores none, because `"true"` is in the set +// and `number_literal` is not. +// +// PHP, Groovy and the four C-family languages are all tracked in #1410. +// They are deferred rather than deliberate, and for a scheduling reason +// only: each has integration-corpus files (the DeepSpeech `native_client` +// snapshots are C/C++), so fixing them moves snapshots and wants its own +// measurement pass. #[macro_export] #[doc(hidden)] macro_rules! perl_bool_terminal_kinds { diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 70f1e352a..7435f2931 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -3284,15 +3284,15 @@ mod tests { // `[Both, Both, OperandOnly]`; inverting gives // `[Both, Both, OperatorOnly]`; dropping the gate gives three // `OperandOnly`; reverting #1380 gives three `OperatorOnly`. - assert!( - roles.contains(&Role::Both), - "some space must bill `this` as an operator *and* an operand, which only \ - the declarator and the receiver disagreeing can produce; roles were {roles:?}", - ); - assert!( - roles.contains(&Role::OperandOnly), - "the accessor space holds only the `this._a` receiver, so it must bill \ - `this` as an operand and nothing else; roles were {roles:?}", + // Asserting the whole sequence rather than two `contains` calls: + // the comment above already derives it, and a `contains` pair + // leaves the length unasserted, so a space appearing or vanishing + // goes unnoticed and `[Both, X, OperandOnly]` passes for any `X`. + assert_eq!( + roles, + vec![Role::Both, Role::Both, Role::OperandOnly], + "the declarator and the receiver must disagree in the two outer spaces \ + and the accessor must bill `this` as an operand alone", ); } diff --git a/src/metrics/npm.rs b/src/metrics/npm.rs index c07d10343..7a630968f 100644 --- a/src/metrics/npm.rs +++ b/src/metrics/npm.rs @@ -2958,11 +2958,17 @@ class C { // there. In the four whose expected answer is "the rule does not // apply" (`…wins`, `…singleton_initialize…`, // `…in_a_singleton_class_body…`, `…public_symbol_republishes…`) the - // name contributes to no axis once exempt, so a rename is silent and - // no anchor is available — measured, not assumed. That matters most - // for `ruby_public_keyword_wrapping_initialize_wins`, the sole guard - // on the `named_by_keyword ||` disjunct: rename its method and that - // branch goes uncovered with nothing going red. + // name contributes to no metric axis once exempt, so no *count* can + // anchor it. + // + // The space tree can, and does: the Ruby walk opens a named + // `Function` space per `def`, so `child_space(…, "initialize")` + // panics on a rename. `ruby_public_keyword_wrapping_initialize_wins` + // carries that anchor, because it is the sole guard on the + // `named_by_keyword ||` disjunct and a silent rename there would + // leave that branch uncovered with nothing going red. The other + // three remain deletion-anchored only; adding the same call to them + // is cheap and welcome if you are already in the file. #[test] fn ruby_initialize_is_not_a_public_method() { @@ -3034,12 +3040,21 @@ class C { // no keyword governs. // // expected: nm = 2, npm = 2. - check_metrics::( + // + // The `child_space` call is the fixture-decay anchor. Both counts + // above are name-blind once the keyword exempts the declaration, + // so renaming `initialize` to `setup` leaves them at 2 and 2 and + // this test — the sole guard on the `named_by_keyword ||` + // disjunct — would go green over an uncovered branch. Naming the + // method space makes that rename panic instead. + check_func_space::( "class H\n public def initialize(x)\n @x = x\n end\n def value\n @x\n end\nend\n", "foo.rb", - |metric| { + |func_space| { + let metric = &func_space.metrics; assert_eq!(metric.npm.class_nm_sum(), 2); assert_eq!(metric.npm.class_npm_sum(), 2); + child_space(child_space(&func_space, "H"), "initialize"); insta::assert_json_snapshot!(metric.npm); }, ); diff --git a/src/vcs/cache_tests.rs b/src/vcs/cache_tests.rs index 0cc7e7a7e..b2a427206 100644 --- a/src/vcs/cache_tests.rs +++ b/src/vcs/cache_tests.rs @@ -89,16 +89,14 @@ fn fingerprint_changes_with_the_mailmap_digest() { "a mailmap change must change the fingerprint" ); // Both operands differ from `0` and from each other, so the inequality - // above cannot hold for an incidental reason. The equality below is - // determinism *within one process* only — the cross-process stability - // a persisted entry actually depends on cannot be observed from here, - // and is guarded by `vcs_cache_dir_persists_and_replays_identically` - // in the CLI suite, which primes and reads in two separate processes. - assert_eq!( - base, - fingerprint(&options, SAMPLE_MAILMAP_DIGEST), - "identical inputs fingerprint identically" - ); + // above cannot hold for an incidental reason. + // + // No same-process re-call is asserted here: `fingerprint` is pure over + // a fixed seed, so calling it twice with identical arguments compares a + // value to itself and cannot fail. The stability a persisted entry + // actually depends on is cross-*process*, which this test cannot + // observe; `vcs_cache_dir_persists_and_replays_identically` in the CLI + // suite covers it by priming and reading in two separate processes. } #[test] From 4b8128faf6257a43f3fac76bf49d15c9fc01a04f Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Fri, 11 Sep 2026 12:34:30 -0700 Subject: [PATCH 16/22] fix: address findings from the max code review A max-effort review of the branch. Three findings change behaviour; the rest correct comments, docs and changelog entries that described the code wrongly. Behaviour: - Tcl literals (#1381 follow-up). is_braced_script_word inherited #1318's script-by-default rule, so braced literals -- a braced proc name, namespace subcommand arguments, on/trap patterns and variable lists -- stopped being strings and were expanded in Ast::dump as invented commands. On main they were strings and flat leaves. Those slots are literals again. Halstead still bills their braces, as it has since #1318; moving the rule into the shared predicate changes `bca metrics` for Tcl and wants its own measured change. - SARIF order (#1402 follow-up). `bca check` stable-sorts findings by (path, start_line, metric) after its walk, while to_sarif emitted walk order, so the entry-for-entry parity #1402 promised was false. to_sarif now sorts with the CLI's own comparator. - Linear Tcl switch. is_switch_arm_body located a word with an O(position) sibling scan, and #1381 routed find, count and dump through it, making all three quadratic in the width of a one-line switch. It now uses tree-sitter's O(log n) goto_first_child_for_byte. find and count now call Search::act_on_node instead of a hand copy of its ancestor-chain bookkeeping, so `make chain-audit` covers them. Corrections: - getter/groovy.rs: the note 4c5af945 added said the Super operator arm never fires. It fires for every `? super T` wildcard bound, which is right -- it mirrors Java's gated wildcard decision. Pinned by a new groovy_wildcard_super_bound_stays_an_operator test. - abc/csharp.rs: the #1383 parent gate also zeroes a relational pattern inside a `when` guard or `catch` filter, where no slot pays for it. Docs corrected and a FIXME(#1422) added; the behaviour is left to #1422. - vcs: the cache contract is not complete once the mailmap is fingerprinted -- git's diff config also decides recorded churn and is not fingerprinted -- and gix's open_mailmap swallows read errors, so a transient read does not heal. Docs corrected; the code gaps are tracked separately. - loc.rs: the clamp docs claimed parseable input is untouched; Perl POD to end-of-file and Ruby `<<~` parse cleanly and still move. - CHANGELOG, fuzz docs, benchmarking.md, AGENTS.md, Makefile, Cargo.toml and several test comments carried claims the #1381 chain threading made stale. - A vcs test helper matched paths via to_string_lossy, which AGENTS.md bans for identifiers. --- .rustfmt-bail-baseline.txt | 7 + AGENTS.md | 5 +- CHANGELOG.md | 94 +++++--- Cargo.toml | 4 +- Makefile | 9 +- big-code-analysis-ast/src/checker.rs | 90 ++++++++ big-code-analysis-ast/src/checker/irules.rs | 13 +- big-code-analysis-ast/src/checker/tcl.rs | 13 +- big-code-analysis-ast/src/count.rs | 34 +-- big-code-analysis-ast/src/find.rs | 46 ++-- big-code-analysis-ast/src/getter.rs | 218 +++++++++++++++--- big-code-analysis-ast/src/getter/groovy.rs | 26 ++- big-code-analysis-ast/src/getter/tcl.rs | 29 +-- .../src/lang_helpers/irules.rs | 2 + big-code-analysis-ast/src/lang_helpers/tcl.rs | 20 +- big-code-analysis-ast/src/node.rs | 25 +- big-code-analysis-ast/src/parser.rs | 4 +- big-code-analysis-ast/src/traits.rs | 4 +- big-code-analysis-book/src/commands/vcs.md | 9 +- big-code-analysis-book/src/metrics.md | 2 +- big-code-analysis-book/src/python/sarif.md | 18 +- big-code-analysis-book/src/python/vcs.md | 3 +- .../python/big_code_analysis/_native.pyi | 14 +- big-code-analysis-py/src/sarif.rs | 77 +++---- big-code-analysis-py/tests/test_sarif.py | 91 ++++++-- docs/development/benchmarking.md | 7 +- docs/development/lessons_learned.md | 11 +- fuzz/src/lib.rs | 15 +- fuzz/src/nested.rs | 13 +- src/metrics/abc.rs | 46 +++- src/metrics/halstead.rs | 50 ++++ src/metrics/loc.rs | 66 +++--- src/metrics/loc/php.rs | 14 +- src/vcs/cache.rs | 4 + src/vcs/git/cached.rs | 8 + src/vcs/git/repo.rs | 23 +- tests/api/ast_seam_test.rs | 12 +- tests/grammars/alterator_string_flattening.rs | 9 + tests/parity/functions_metrics_parity.rs | 4 +- tests/parity/self_reference_operand_parity.rs | 30 +-- tests/vcs/vcs_cache.rs | 12 +- 41 files changed, 822 insertions(+), 359 deletions(-) diff --git a/.rustfmt-bail-baseline.txt b/.rustfmt-bail-baseline.txt index efd96e14b..e0a280a13 100644 --- a/.rustfmt-bail-baseline.txt +++ b/.rustfmt-bail-baseline.txt @@ -48,6 +48,13 @@ # big-code-analysis-ast/src/getter/ruby.rs # 3 -> 4, the `BQUOTE` subshell guard # (#1360) in `get_op_type` +# big-code-analysis-ast/src/getter/csharp.rs +# 5 -> 8, and +# big-code-analysis-ast/src/getter/java.rs +# 2 -> 5: the self- and super-reference arms +# (#1380) in `get_op_type` — each a nested +# two-arm `match` on the parent kind, so +# three stuck arms apiece # src/vcs/error.rs `classify_error_variants!` (#1245): its # matcher, plus the 11 `$pat => $sample` # entries at the invocation, which are diff --git a/AGENTS.md b/AGENTS.md index 68aefe753..d9ece3d42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -468,8 +468,9 @@ exact form is `Node::parent`'s `O(depth)` per node and made every debug-build walk quadratic (#1122). The approximation misses a chain that is short by exactly one, so run this around any change to a walk's truncate/push bookkeeping — `src/spaces/compute.rs`, `src/ops.rs`, -`src/suppression.rs`, and in `big-code-analysis-ast`, `comment_rm.rs` and -`Search::act_on_node`. It is +`src/suppression.rs`, and in `big-code-analysis-ast`, `comment_rm.rs`, +`Search::act_on_node` (which `find` and `count` walk through) and the +dump walk's `build` in `ast.rs`. It is not part of `make pre-commit`; the `chain-audit` CI job runs it per PR. See [Benchmarking](docs/development/benchmarking.md#chain-audit). diff --git a/CHANGELOG.md b/CHANGELOG.md index 483707b0e..95359a04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,13 +132,18 @@ for historical reference. — over four unrelated recovery shapes (heredoc, unterminated string delimiter, unclosed bracket, dangling line continuation), so each space's line sets are now clamped to its own span at finalization - rather than per arm. A `cloc > sloc` case (Perl POD) of the same - mechanism is fixed with it. Note a clean parse is not evidence of an - in-span tree: Ruby's `x = <<~DOC` with an unterminated body parses - without an error node and still emits the phantom row. **Metric - drift:** `loc.ploc` / `loc.cloc` fall and `loc.blank` rises on files a + rather than per arm. A Perl POD block that runs to end-of-file without + `=cut` — valid Perl that parses cleanly — reached `cloc > sloc` by a + different route: the node ends at column 0 of the row past the last + one, and the comment arms counted that raw end row. The same clamp + removes it. A clean parse is not evidence of an in-span tree in the + other direction either: Ruby's `x = <<~DOC` with an unterminated body + parses without an error node and still emits the phantom row. + **Metric drift:** `loc.ploc` / `loc.cloc` fall and `loc.blank` rises + on files whose tree reaches past their last row — mostly files a grammar cannot fully parse, including at least one ordinary shell - script in the DeepSpeech corpus. + script in the DeepSpeech corpus, and also Perl files whose last POD + block has no `=cut` (`cloc` −1, `blank` +1). - **ABC counts a bare numeric literal operand in Ruby, Elixir and Perl** (#1379). All three are truthy-valued, so a number used as a bare `&&` @@ -154,10 +159,12 @@ for historical reference. Ruby's `rational` / `complex` wrap the numeral and the walker cannot descend into a wrapper, so the wrapper is classified and `integer` stays listed alongside it for the bare `1`. Lua, Tcl, iRules and the - JS family were measured and have no gap. PHP and Groovy carry the same - defect and are tracked in #1410. **Metric drift:** `abc.conditions` - and `abc.magnitude` rise for any Ruby, Elixir or Perl file using a - numeric in a boolean-operand or condition slot. + JS family were measured and have no gap. PHP, Groovy and the C family + (C, C++, Mozcpp, Objective-C) carry the same defect and are tracked in + #1410. **Metric drift:** `abc.conditions` and `abc.magnitude` rise for + Perl files using any numeric literal, and Ruby or Elixir files using a + non-integer one (`1.0`, `1r`, `2i`, `?a`), as a bare `&&` / `||` + operand or (Perl, Ruby) an `if` predicate. - **A relational pattern no longer double-counts its operator against the arm that owns it** in C# ABC (#1383). `x switch { > 5 => …, < 0 @@ -166,7 +173,12 @@ for historical reference. where `cyclomatic() - 1` is 2; `if (x is > 0)` gave 2 against 1. The enclosing switch arm and `is` condition slot already pay for the decision, so the pattern's own operator is now excluded — matching how - a constant pattern is treated. This covers `>=` and `<=` as well as + a constant pattern is treated. The gate is on the operator's parent, + so a relational pattern no arm or condition slot owns scores 0 too — + `bool b = x is > 5;`, `return x is > 5;`, a lambda or expression body, + a `when` guard or `catch` filter — in step with `x is 5`, `x is int` + and cyclomatic but one below the equivalent `x > 5` (guards: #1422). + This covers `>=` and `<=` as well as `>` and `<`: those are distinct token ids reaching a separate arm, so the allowlist named in #1297 never saw them. **Metric drift:** `abc.conditions` and `abc.magnitude` fall for every C# file using @@ -204,17 +216,21 @@ for historical reference. `volume`, `difficulty`, `effort`, `time`, `bugs` and all three maintainability-index variants with them. -- **PHP heredoc, nowdoc and backtick rows that are empty inside the - literal are counted as `ploc` rather than `blank`** (#1396), matching - every other language with a multi-line literal. #778 recorded PHP as - already correct; it was not, for the one shape that release never - measured. The nowdoc case was worse than the heredoc case and - structurally different — its body is one node for the first line plus - a single multi-row node for the rest, so it lost every interior row - regardless of emptiness. The wrapper is routed rather than the body, - because a heredoc whose body is a single empty row emits no body node - at all. **Metric drift:** `loc.ploc` rises and `loc.blank` falls for - PHP files containing these literals. +- **PHP heredoc, nowdoc and backtick literals credit every row they + span to `ploc`** (#1396), as the languages #778 and #1260 routed + already did; rows empty inside the literal had been counted as + `blank`. #778 recorded PHP as already correct; it was not, for the one + shape that release never measured. Nowdoc and backtick also lost + non-empty rows, each for its own reason: a nowdoc body is one + `nowdoc_string` per line, each starting at the end of the row before, + so its last body row was credited to nothing; a multi-row backtick + command is a single node, so every interior row was. The wrapper is + routed rather than the body, because a heredoc whose body is a single + empty row emits no body node at all. Tcl and iRules braced values + (`set x {a\n\nb}`) and C# interpolated strings still lose such rows. + **Metric drift:** `loc.ploc` rises and `loc.blank` falls for PHP files + containing these literals — by one for every nowdoc, empty rows or + not. - **Ruby `npm` no longer counts `initialize`, `initialize_copy`, `initialize_dup`, `initialize_clone` or `respond_to_missing?` as @@ -228,8 +244,11 @@ for historical reference. - **`bca find --type string` and `bca count --type string` no longer report a Tcl or iRules script body as a string literal** (#1381) — a - `proc` or `if` body, or an iRules `when` handler. A braced *value* - such as `lappend x {a b}` is still reported. `Checker::is_string` + `proc` or `if` body, or an iRules `when` handler. A braced *value* is + still reported: `lappend x {a b}`, a braced `proc` name, the arguments + of a `namespace` subcommand other than `eval` / `inscope` / `code`, and + the pattern and variable list of an `on` / `trap` handler clause the + grammar leaves as a plain command. `Checker::is_string` could not tell the two apart because it received neither the source bytes nor the ancestor chain; it gains an `is_string_with_code` sibling that routes both dialects through the role predicate #1318 @@ -240,7 +259,11 @@ for historical reference. the alterator, so it is unaffected.) Recognition is a leading-word heuristic, so Tcl's subcommand-dispatched script takers — `dict for`, `interp eval`, `apply` — are still reported; iRules models its - handlers structurally and has no such gap. + handlers structurally and has no such gap. The reverse misses remain + too: a braced value that `after cancel` or the separate-argument form + of `switch` takes, or that sits in a multi-line `try … trap` clause + the Tcl grammar leaves inside an error node, is still treated as a + script. - **A `.mailmap` edit invalidates the persistent VCS history cache** (#1262). Author identities are canonicalised through the repository @@ -262,15 +285,16 @@ for historical reference. - **`bca.to_sarif` emits findings in the same order as `bca check -O sarif`** (#1402), so the two documents can be compared positionally and not just as sets — which is what the binding's own parity claim - had been promising. Two divergences are fixed: the binding walked the - space tree with a LIFO stack it pushed in source order, so every - sibling set came out reversed at every level; and it iterated the - `thresholds` dict, so a space breaching several metrics reported them - in the caller's insertion order rather than the CLI's - alphabetical-by-metric order — which also made the output depend on - how the dict happened to be spelled. Order *between* files remains - the caller's: `to_sarif` follows the iterable it is handed, where - `bca check` follows its resolved walk list. + had been promising. The binding now sorts its findings as `bca check` + does after its walk — by path, then start line, then metric name — + with ties kept in depth-first source order. It used to emit them in + walk order, and its walk pushed each sibling set onto a LIFO stack in + source order, so every sibling set came out reversed; a space + breaching several metrics followed the caller's `thresholds` dict + order; and files followed the input iterable rather than the CLI's + path order. The comparison is against `bca check --no-suppress`: + `to_sarif` still applies no in-source suppression markers, baseline + or `[check] exclude` globs. - **`bca preproc` documents are byte-identical across runs** (#1304). `PreprocResults.files` and `PreprocFile`'s three `HashSet` @@ -295,7 +319,7 @@ for historical reference. Perl was listed as immune by the original survey and was not. C# initially kept counting a relational pattern's operator (`x is > 0`) on the grounds that it is a genuine comparison outside - `binary_expression`; #1383 below reverses that, because the switch + `binary_expression`; #1383 above reverses that, because the switch arm or `is` condition owning the pattern already pays for the decision. Elixir was swept the same way in the same release: an operator diff --git a/Cargo.toml b/Cargo.toml index 3c1d0d463..9baa6448a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -121,8 +121,8 @@ tempfile = "^3.0" missing_docs = "warn" # `chain_audit` turns `Ancestors::checked` back into the exact # parent-identity assertion it was before #1122. Off by default because -# that assertion costs `Node::parent`'s `O(depth)` per node on all five -# checked walks; `make chain-audit` and the CI lane of the same name set +# that assertion costs `Node::parent`'s `O(depth)` per node on every +# checked walk; `make chain-audit` and the CI lane of the same name set # it. Declared here so `-D warnings` does not reject the `#[cfg]`. unexpected_cfgs = { level = "warn", check-cfg = ['cfg(chain_audit)'] } diff --git a/Makefile b/Makefile index 26d7c8de5..1ee167ecc 100644 --- a/Makefile +++ b/Makefile @@ -322,10 +322,11 @@ test-doc: # triples the lib suite's wall time. Runs in the `chain-audit` CI lane; # run it locally around any change to a walk's truncate/push bookkeeping. # -# Library-scoped, matching that lane: the five walks that thread a chain -# live in the root crate (`spaces::compute`, `ops`, `suppression`) and -# in `big-code-analysis-ast` (`comment_rm`, `Search::act_on_node`), so -# the CLI / web / integration tiers would re-pay the quadratic cost +# Library-scoped, matching that lane: the walks that thread a chain live +# in the root crate (`spaces::compute`, `ops`, `suppression`) and in +# `big-code-analysis-ast` (`comment_rm`, `Search::act_on_node` — which +# `find` and `count` also walk through — and the `Ast` dump's `build`), +# so the CLI / web / integration tiers would re-pay the quadratic cost # without reaching an assertion the lib tests do not already reach. # # RUSTFLAGS rather than a Cargo feature on purpose: `make test` passes diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index 1eaa07084..fe4e4d23c 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -2728,4 +2728,94 @@ mod tests { "neither tcl nor irules is enabled; this test asserted nothing" ); } + + /// The value slots of the three constructs whose argument lists hold + /// both roles stay string literals, though `is_value_braced_word` + /// classifies each construct whole as script-taking + /// (`Getter::is_braced_literal_slot`). + /// + /// Every fixture line carries a literal *and* a script in the same + /// construct, so each rule is pinned from both sides: a rule that + /// withdrew the whole construct fails on the literal, and one that + /// rescued it whole fails on the script. The two switch lines are the + /// guards' own rows — an arm whose *pattern* is spelled `proc`, `on` + /// or `trap` parses as that construct around arm bodies, which must + /// stay scripts. They need a line each: `proc {…} on {…}` parses as + /// one `procedure`, so only the second line puts an `on` *command* + /// in front of the owner guard. + /// + /// The Tcl `trap` line stands alone because the walk helper refuses a + /// tree with a parse error, and `try {…} trap …` on one line reaches + /// the same generic `trap` command only through error recovery (a + /// missing terminator the grammar inserts after the `try` body). + #[test] + #[cfg(any(feature = "tcl", feature = "irules"))] + fn tcl_family_value_slots_of_script_takers_stay_strings() { + let mut ran = 0; + #[cfg(feature = "tcl")] + { + ran += 1; + let (kept, withdrawn, _) = braced_word_string_verdicts::( + "tcl", + b"proc {my proc} {x} { puts $x }\n\ + namespace export {a b}\n\ + namespace eval ns { puts hi }\n\ + namespace ensemble create -map {add ::a}\n\ + trap {POSIX ENOENT} {msg} { puts $msg }\n\ + switch $k { proc {puts p} on {puts o} }\n\ + switch $k { on {puts o} trap {puts t} }\n", + Tcl::BracedWord as u16, + ); + assert_eq!( + kept, + ["{my proc}", "{a b}", "{add ::a}", "{POSIX ENOENT}", "{msg}"], + "tcl: a proc name, a non-`eval` namespace argument, and a \ + `trap` pattern and variable list are literals" + ); + assert_eq!( + withdrawn, + [ + "{ puts $x }", + "{ puts hi }", + "{ puts $msg }", + "{ proc {puts p} on {puts o} }", + "{puts p}", + "{puts o}", + "{ on {puts o} trap {puts t} }", + "{puts o}", + "{puts t}", + ], + "tcl: the proc body, the `namespace eval` body, the `trap` \ + script, and the switch arm list and its bodies are scripts" + ); + } + #[cfg(feature = "irules")] + { + ran += 1; + let (kept, withdrawn, other) = braced_word_string_verdicts::( + "irules", + b"proc {my proc} {x} { log local0. $x }\n\ + namespace export {a b}\n\ + when HTTP_REQUEST { log local0. \"hi\" }\n\ + on error {m2} {drop}\n", + Irules::BracedWord as u16, + ); + assert_eq!( + kept, + ["{my proc}", "{a b}", "{m2}"], + "irules: a proc name, a `namespace` value and an `on` variable \ + list are literals" + ); + assert_eq!( + withdrawn, + ["{ log local0. $x }", "{ log local0. \"hi\" }", "{drop}"], + "irules: the proc and `when` bodies and the `on` handler are scripts" + ); + assert_eq!(other, 1, "irules: the quoted word stays a string"); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); + } } diff --git a/big-code-analysis-ast/src/checker/irules.rs b/big-code-analysis-ast/src/checker/irules.rs index b853281cb..b3fa11079 100644 --- a/big-code-analysis-ast/src/checker/irules.rs +++ b/big-code-analysis-ast/src/checker/irules.rs @@ -57,12 +57,13 @@ impl Checker for IrulesCode { // cannot separate the two roles and `is_string_with_code` does. impl_simple_is_string!(Irules, QuotedWord, BracedWord, BracedWordSimple); - // The twin of `TclCode::is_string_with_code` (#1381). This grammar - // models more script positions than Tcl's — `when`, `for`, `switch` - // and the `dict` loops each have a node of their own — so the - // handler bodies this rescues are recognised structurally rather - // than by command name, and a `when` body is a script whatever it - // is called. + // The twin of `TclCode::is_string_with_code` (#1381), including the + // two places it parts company with Halstead that the Tcl comment + // records. This grammar models more script positions than Tcl's — + // `when`, `for`, `switch` and the `dict` loops each have a node of + // their own — so the handler bodies this rescues are recognised + // structurally rather than by command name, and a `when` body is a + // script whatever it is called. fn is_string_with_code<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { Self::is_string(node) && !::is_braced_script_word(node, code, ancestors, &BRACED_WORD_KINDS) diff --git a/big-code-analysis-ast/src/checker/tcl.rs b/big-code-analysis-ast/src/checker/tcl.rs index 0a065903e..d9fcf5fab 100644 --- a/big-code-analysis-ast/src/checker/tcl.rs +++ b/big-code-analysis-ast/src/checker/tcl.rs @@ -41,10 +41,15 @@ impl Checker for TclCode { impl_simple_is_string!(Tcl, QuotedWord, BracedWord, BracedWordSimple); // The half of the rule that needs the source bytes (#1381), the twin - // of `TclCode::get_op_type_with_code`. `is_value_braced_word` lives - // on `Getter`, and `TclCode` implements both traits, so the two - // classifiers answer from one predicate over one kinds table rather - // than from two copies (grammar-dispatch §7). + // of `TclCode::get_op_type_with_code`. Both start from + // `is_value_braced_word` on `Getter` over one kinds table rather than + // from two copies (grammar-dispatch §7), and part company in two + // known places. `is_braced_script_word` also withdraws the value slots + // `is_braced_literal_slot` names (`proc {my proc}`, + // `namespace export {…}`), whose `{` Halstead still bills as a block. + // And Halstead's *operand* half keys on whether a body holds a + // command, so an empty or comment-only `proc` body is an operand there + // and not a string here. fn is_string_with_code<'a>(node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) -> bool { Self::is_string(node) && !::is_braced_script_word(node, code, ancestors, &BRACED_WORD_KINDS) diff --git a/big-code-analysis-ast/src/count.rs b/big-code-analysis-ast/src/count.rs index c68184593..5a9510c96 100644 --- a/big-code-analysis-ast/src/count.rs +++ b/big-code-analysis-ast/src/count.rs @@ -23,39 +23,21 @@ use num_format::{Locale, ToFormattedString}; use std::fmt; use std::sync::{Arc, Mutex}; -use crate::node::Ancestors; -use crate::traits::ParserTrait; +use crate::traits::{ParserTrait, Search}; /// Counts the types of nodes specified in the input slice and the /// number of nodes in a code. Crate-internal walk core reached through /// the `big_code_analysis::Ast::count` seam. pub fn count(parser: &T, filters: &[String]) -> (usize, usize) { let filters = parser.filters(filters); - let node = parser.root(); - let mut cursor = node.cursor(); - let mut stack = Vec::new(); - let mut good = 0; - let mut total = 0; - // See `find` for why the chain is threaded rather than climbed. The - // truncate/push discipline is order-independent — a node's ancestors - // are always the chain prefix at its own depth — so it holds for - // this walk's unordered child push too. - let mut chain = Vec::new(); - - stack.push((node, 0_usize)); - - while let Some((node, depth)) = stack.pop() { + let (mut good, mut total) = (0, 0); + // The same shared walk as `find`, for the ancestor chain its + // predicates read; this one only tallies, so the visit order it + // imposes is incidental. + parser.root().act_on_node(&mut |node, ancestors| { total += 1; - chain.truncate(depth); - if filters.any(&node, Ancestors::checked(&chain, &node)) { - good += 1; - } - chain.push(node); - // No reversal: this walk only tallies, so visit order is - // immaterial and imposing one would imply a guarantee no caller - // relies on. Matches the previous push-in-source-order form. - stack.extend(node.children_with(&mut cursor).map(|c| (c, depth + 1))); - } + good += usize::from(filters.any(node, ancestors)); + }); (good, total) } diff --git a/big-code-analysis-ast/src/find.rs b/big-code-analysis-ast/src/find.rs index 73474c44d..75f71abbd 100644 --- a/big-code-analysis-ast/src/find.rs +++ b/big-code-analysis-ast/src/find.rs @@ -8,10 +8,10 @@ //! Node finding by kind or category. -use crate::node::{Ancestors, Node}; +use crate::node::Node; use crate::error::MetricsError; -use crate::traits::ParserTrait; +use crate::traits::{ParserTrait, Search}; /// Finds the types of nodes specified in the input slice. Crate-internal /// walk core reached through the `big_code_analysis::Ast::find` seam. @@ -37,37 +37,19 @@ pub fn find<'a, T: ParserTrait>( filters: &[String], ) -> Result>, MetricsError> { let filters = parser.filters(filters); - let node = parser.root(); - let mut cursor = node.cursor(); - let mut stack = Vec::new(); let mut good = Vec::new(); - // The ancestry of whichever node is about to be popped. A predicate - // that asks about an enclosing construct — `"function"` for the - // JS family and Elixir, `"string"` for the Tcl family — would - // otherwise climb with `Node::parent`, which restarts at the root - // and so costs `O(depth)` *per lookup*: `bca find --type string` - // over 8 KB of nested Tcl braces took 823 ms that way and 5 ms with - // the chain (#1381, the shape of #1052 / #1122). - // - // Maintained exactly as `spaces::compute::metrics_inner` does — - // truncate to the node's depth before the visit, push after it — - // so `Ancestors::checked` has the same meaning here as there. - let mut chain: Vec> = Vec::new(); - - stack.push((node, 0_usize)); - - while let Some((node, depth)) = stack.pop() { - chain.truncate(depth); - if filters.any(&node, Ancestors::checked(&chain, &node)) { - good.push(node); + // `act_on_node` visits in source-order pre-order, so matches come back + // in source order, and it hands each node its ancestor chain. The + // `"function"` (JS family, Elixir) and `"string"` (Tcl family) + // predicates ask about an enclosing construct; off an unknown chain + // each lookup is `Node::parent`'s `O(depth)`, which made this walk + // quadratic in nesting depth (#1381). Sharing the walk rather than + // copying its truncate/push bookkeeping keeps one copy under + // `make chain-audit`. + parser.root().act_on_node(&mut |node, ancestors| { + if filters.any(node, ancestors) { + good.push(*node); } - chain.push(node); - // Source order in, tail reversed in place, so the LIFO `stack` - // yields the leftmost child first — matches were already - // returned in source order and must stay that way. - let first_child = stack.len(); - stack.extend(node.children_with(&mut cursor).map(|c| (c, depth + 1))); - stack[first_child..].reverse(); - } + }); Ok(good) } diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index d43b50ab2..c07006df8 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -328,6 +328,17 @@ pub struct BracedWordKinds { /// parameter (`proc p {a {b {x y}}}`) spells its default as a /// `braced_word`, and a default is never evaluated as code. pub argument: u16, + /// `procedure`, the modelled `proc` construct. Its `name` field is a + /// `braced_word` when the name holds a space (`proc {my proc} …`), + /// and a name is a literal although the construct's other braced + /// slot, the body, is a script + /// ([`Getter::is_braced_literal_slot`]). + pub procedure: u16, + /// `namespace`, the modelled construct — not the keyword token that + /// shares its name. Its `word_list` holds a subcommand followed by + /// that subcommand's arguments, and only three subcommands take a + /// script ([`Getter::is_braced_literal_slot`]). + pub namespace: u16, /// `{`, the brace opener — the *only* node /// [`Getter::braced_word_op_type`] may revise. A braced word's /// operator children are not the opener alone: `_terminator` is a @@ -362,17 +373,20 @@ pub struct BracedWordKinds { /// | `after` | `after ms script` | /// | `eval` | `eval arg ?arg …?` | /// | `for` | `for start test next body` — all four are evaluated | -/// | `on` | `on error {script}`, the iRules `try` clause | +/// | `on` | `on code varList script`, a `try` handler clause | /// | `switch` | `switch ?options? string pattern body ?pattern body …?` | /// | `time` | `time script ?count?` | -/// | `trap` | `trap {script}`, the iRules `try` clause | +/// | `trap` | `trap pattern varList script`, a `try` handler clause | /// | `uplevel` | `uplevel ?level? arg ?arg …?` | /// /// `on` and `trap` are listed because the iRules grammar models /// `on_handler` / `trap_handler` only *under* `try` (pinned by -/// `irules_try_handler_kinds_appear_only_under_try`); written at -/// statement level they parse as generic commands, and neither word -/// takes a value in either dialect. +/// `irules_try_handler_kinds_appear_only_under_try`), and the Tcl +/// grammar models neither a `trap` nor a second `on`; written outside +/// that shape they parse as generic commands. Their last argument is +/// the handler script. The pattern and variable list before it are +/// values, which only [`Getter::is_braced_literal_slot`] tells apart — +/// the `{}` operator this table decides still bills them as blocks. /// /// `lmap varname list body` is deliberately **not** listed even though /// its body is a script: the list is per-command, not per-argument, so @@ -409,6 +423,19 @@ const SCRIPT_TAKING_COMMANDS: [&str; 8] = [ /// ([`Getter::is_switch_arm`]). Spelling it twice would let one drift. const SWITCH_COMMAND: &str = "switch"; +/// The `namespace` subcommands with a script argument: +/// `namespace eval ns arg ?arg …?`, `namespace inscope ns script ?arg …?` +/// and `namespace code script`. Every other subcommand takes values — +/// `export {pattern}`, `path {ns …}`, `ensemble create -map {dict}` — +/// which is what [`Getter::is_braced_literal_slot`] keys on. +const NAMESPACE_SCRIPT_SUBCOMMANDS: [&str; 3] = ["eval", "inscope", "code"]; + +/// The `try` handler clauses a grammar leaves as generic commands: +/// `on code varList script` and `trap pattern varList script`, whose +/// every argument but the last is a value +/// ([`Getter::is_braced_literal_slot`]). +const TRY_HANDLER_COMMANDS: [&str; 2] = ["on", "trap"]; + /// Per-language accessors that *name* and *classify* what a node is: /// the function or space name, the [`SpaceKind`] a node opens, and the /// [`TokenRole`] of a leaf. @@ -744,21 +771,27 @@ pub trait Getter { /// /// The positive form of [`is_value_braced_word`], narrowed to the /// script kind so it answers `false` for every node of every other - /// kind and for the two literal spellings. That makes it the - /// question the *non-Halstead* classifiers ask: + /// kind and for the two literal spellings, and narrowed again by the + /// value slots [`is_braced_literal_slot`] recognises. That makes it + /// the question the *non-Halstead* classifiers ask: /// `Checker::is_string_with_code` must not call a `proc` body a - /// string literal, and `Alterator::alterate` must not flatten one - /// into a leaf, dropping the body from the AST dump. Both once - /// listed `braced_word` beside `quoted_word` and + /// string literal, and `Alterator::keeps_children` must stop the dump + /// flattening one into a leaf, which dropped the body from it. Both + /// once listed `braced_word` beside `quoted_word` and /// `braced_word_simple`, which is right for the literal role and /// wrong for the script role the same kind also serves. /// /// Stated here rather than in each of the four call sites so the - /// three classifiers cannot drift apart on the same bytes - /// (grammar-dispatch §7) — the drift `braced_word_op_type` opened - /// when it revised the operator half alone. + /// string and dump classifiers cannot drift apart on the same bytes + /// (grammar-dispatch §7). Halstead is the deliberate exception: + /// `braced_word_op_type` asks [`is_value_braced_word`] alone, so the + /// braces of the value slots `is_braced_literal_slot` adds still + /// bill a `{}` operator, as they have since #1318. Moving that rule + /// into the shared predicate changes `bca metrics` for Tcl, and is + /// its own measured change rather than a rider on this one. /// /// [`is_value_braced_word`]: Self::is_value_braced_word + /// [`is_braced_literal_slot`]: Self::is_braced_literal_slot #[must_use] fn is_braced_script_word<'a>( node: &Node<'a>, @@ -766,7 +799,121 @@ pub trait Getter { ancestors: Ancestors<'a, '_>, kinds: &BracedWordKinds, ) -> bool { - node.kind_id() == kinds.script && !Self::is_value_braced_word(node, code, ancestors, kinds) + node.kind_id() == kinds.script + && !Self::is_value_braced_word(node, code, ancestors, kinds) + && !Self::is_braced_literal_slot(node, code, ancestors, kinds) + } + + /// Whether `word`, a braced word [`is_value_braced_word`] calls a + /// script, fills a slot whose documented syntax takes a *value*. Three + /// constructs hold both roles in one argument list: + /// + /// | construct | value slots | script slot | + /// | --- | --- | --- | + /// | `proc name args body` | the `name` field | the body | + /// | `namespace sub ?arg …?` | any subcommand's but three | `eval`, `inscope`, `code` | + /// | `on code varList script`, `trap pattern varList script` | all but the last | the last | + /// + /// Each literal here was a string and a flat dump leaf before #1381, + /// and would otherwise have become a script under it: the dump + /// rendered `{my proc}` as a command named `my`, and + /// `namespace export {…}` and `namespace ensemble create -map {…}` + /// both occur in the Tcl 8.6 standard library. + /// + /// Two guards keep the construct-wide answer. A switch arm list parses + /// as commands, so an arm whose *pattern* is spelled `proc`, + /// `namespace`, `on` or `trap` builds one of these shapes around what + /// are really arm bodies — [`is_switch_arm`] recognises it first. And + /// an owner holding a parse error has no argument positions worth + /// trusting. The multi-line `try … trap` clause is out of reach + /// entirely: the Tcl grammar leaves it inside an `ERROR` node, where + /// no role signal survives. + /// + /// [`is_value_braced_word`]: Self::is_value_braced_word + /// [`is_switch_arm`]: Self::is_switch_arm + #[must_use] + fn is_braced_literal_slot<'a>( + word: &Node<'a>, + code: &[u8], + ancestors: Ancestors<'a, '_>, + kinds: &BracedWordKinds, + ) -> bool { + let mut chain = ancestors.iter(word); + let Some((parent, above_parent)) = chain.next() else { + return false; + }; + if parent.kind_id() == kinds.procedure { + let is_name = + matches!(parent.child_by_field_name("name"), Some(name) if name.id() == word.id()); + return is_name && !Self::is_switch_arm(&parent, code, above_parent, kinds); + } + let Some((owner, above_owner)) = chain.next() else { + return false; + }; + if parent.kind_id() != kinds.word_list + || owner.has_error() + || Self::is_switch_arm(&owner, code, above_owner, kinds) + { + return false; + } + if owner.kind_id() == kinds.namespace { + Self::namespace_subcommand_takes_values(&parent, code, kinds) + } else { + Self::is_try_handler_value(word, &owner, code, kinds) + } + } + + /// Whether a `namespace` construct's `word_list` names a subcommand + /// whose arguments are values — anything but + /// `NAMESPACE_SCRIPT_SUBCOMMANDS`. A subcommand that is not a plain + /// word (`namespace $sub …`) is unresolvable and keeps the script + /// answer, as an unresolvable command name does. + #[must_use] + fn namespace_subcommand_takes_values( + word_list: &Node<'_>, + code: &[u8], + kinds: &BracedWordKinds, + ) -> bool { + let Some(subcommand) = word_list.child(0) else { + return false; + }; + if subcommand.kind_id() != kinds.simple_word { + return false; + } + let Some(subcommand) = node_text(code, &subcommand) else { + return false; + }; + !NAMESPACE_SCRIPT_SUBCOMMANDS.contains(&subcommand) + } + + /// Whether `word` is an argument of a generic `on` / `trap` command + /// other than its last, the handler script. The index comes from an + /// `O(log n)` cursor lookup, as in [`is_switch_arm_body`], not a + /// sibling scan. + /// + /// [`is_switch_arm_body`]: Self::is_switch_arm_body + #[must_use] + fn is_try_handler_value( + word: &Node<'_>, + command: &Node<'_>, + code: &[u8], + kinds: &BracedWordKinds, + ) -> bool { + let is_handler = command.kind_id() == kinds.command + && matches!( + Self::command_leading_word(command, code, kinds), + Some(name) if TRY_HANDLER_COMMANDS.contains(&name) + ); + if !is_handler { + return false; + } + let Some(arguments) = command.child_by_field_name("arguments") else { + return false; + }; + let mut cursor = arguments.cursor(); + let index = cursor.goto_first_child_for_byte(word.start_byte()); + cursor.node().id() == word.id() + && matches!(index, Some(index) if index + 1 < arguments.child_count()) } /// Whether `word`, an argument of a `switch` arm command @@ -791,22 +938,30 @@ pub trait Getter { /// half. The parity holds across the words that can interpose — a /// `-` fall-through body and a `default` pattern are both /// `simple_word`s and take a slot each — so an even index is a body - /// and an odd one a pattern. This is the one place the rule counts - /// siblings rather than asking a parent kind; the scan is bounded by - /// the arms an author put on one line, and runs once per braced - /// argument of such a command. + /// and an odd one a pattern. This is the one place the rule needs a + /// sibling *index* rather than a parent kind, and it runs once per + /// braced argument of such a command. + /// + /// The index comes from [`Cursor::goto_first_child_for_byte`], which + /// is `O(log n)` in the argument count. A `children().position(..)` + /// scan answers identically in `O(n)`, and since every braced word + /// of a one-line arm list asks, that made the Halstead walk — and, + /// once #1381 routed them through this rule, `find` / `count + /// --type string` and the `Ast` dump — quadratic in the width of + /// one line: seconds per request at tens of KB (#1381 review). The + /// id check keeps the scan's answer for a word that is not one of + /// the command's arguments at all. /// /// [`is_switch_arm`]: Self::is_switch_arm + /// [`Cursor::goto_first_child_for_byte`]: crate::node::Cursor::goto_first_child_for_byte #[must_use] fn is_switch_arm_body(word: &Node<'_>, command: &Node<'_>) -> bool { - command - .child_by_field_name("arguments") - .and_then(|arguments| { - arguments - .children() - .position(|argument| argument.id() == word.id()) - }) - .is_some_and(|index| index.is_multiple_of(2)) + let Some(arguments) = command.child_by_field_name("arguments") else { + return false; + }; + let mut cursor = arguments.cursor(); + let index = cursor.goto_first_child_for_byte(word.start_byte()); + cursor.node().id() == word.id() && matches!(index, Some(index) if index.is_multiple_of(2)) } /// A command's leading word, when it is a statically resolvable @@ -904,15 +1059,14 @@ pub trait Getter { /// where only the command name would (`lappend`). Closing that /// needs a signal neither grammar gives — filed as #1382. /// - /// Keeping to the operator also keeps the whole thing `O(1)`: only + /// Keeping to the operator also keeps the whole thing cheap: only /// a braced word's own opener can change answer, so the test is one /// kind comparison and one parent lookup, the same scope #1354 and /// #1314 use — with the single exception of a `switch` arm - /// command, where [`is_switch_arm_body`] scans that command's - /// arguments, a run bounded by the arms an author wrote on one - /// line. An ancestor scan would have been `O(depth)` per node and - /// quadratic on a deeply nested `expr`, the shape #1122 warns - /// about. + /// command, where [`is_switch_arm_body`] needs the word's index + /// among that command's arguments, an `O(log n)` cursor lookup. An + /// ancestor scan would have been `O(depth)` per node and quadratic + /// on a deeply nested `expr`, the shape #1122 warns about. /// /// [`is_switch_arm_body`]: Self::is_switch_arm_body /// diff --git a/big-code-analysis-ast/src/getter/groovy.rs b/big-code-analysis-ast/src/getter/groovy.rs index e03e6c65c..ad8ea1416 100644 --- a/big-code-analysis-ast/src/getter/groovy.rs +++ b/big-code-analysis-ast/src/getter/groovy.rs @@ -133,18 +133,20 @@ impl Getter for GroovyCode { // minus tokens that no longer exist in the dekobon grammar // — `This`, `VoidType`, `Throws2`). // - // `Super` is listed but never fires: the grammar spells a - // super-reference as a plain `identifier` in every position - // measured (`super(1)`, `super.h()`, `A.super.h()`, - // `super::h`), so Groovy already bills it as an operand and - // agrees with the decision #1380 settled for Java, C# and - // Kotlin — by grammar accident rather than by this arm. The - // arm is therefore a latent disagreement: a bump that starts - // emitting `Groovy::Super` would flip Groovy to operator with - // no test failing here. Tracked in #1419, which owes it - // either a §2 unreachability pin or a move to the operand - // arm. `tests/parity/self_reference_operand_parity.rs` is - // what currently catches the flip, one crate away. + // `Super` fires for exactly one production: the pinned grammar + // emits the `super` token only as the bound of a `wildcard` + // (`List`), where billing it as an operator mirrors + // `? extends T`'s `extends` and is the answer #1380 gated + // Java to (`java_wildcard_super_bound_stays_an_operator`). A + // super-*reference* — `super(1)`, `super.h()`, `A.super.h()`, + // `super::h`, `super?.h()` — is a plain `identifier`, so it + // is already an operand, and Groovy agrees with Java on both + // halves. The reference half holds by grammar accident: this + // arm has no parent gate, so a bump that routes a reference to + // `Groovy::Super` would bill it as an operator. + // `tests/parity/self_reference_operand_parity.rs` catches that + // flip, one crate away; #1419 tracks giving the arm Java's + // `Wildcard` parent gate, which removes the accident. If | Else | Switch | Case | Try | Catch | Throw | Throws | For | While | Continue | Break | Do | Finally | New | Return | Default | Abstract | Assert | Instanceof | Extends | Final | Implements | Transient | Synchronized | Super | Def | In | As diff --git a/big-code-analysis-ast/src/getter/tcl.rs b/big-code-analysis-ast/src/getter/tcl.rs index 5fb9cbaa0..04a6b89d9 100644 --- a/big-code-analysis-ast/src/getter/tcl.rs +++ b/big-code-analysis-ast/src/getter/tcl.rs @@ -99,20 +99,21 @@ impl Getter for TclCode { // else, so every operand this arm decides is unchanged. // // Cross-walked against the sibling predicates - // (grammar-dispatch §7) and left as it was: - // `Checker::is_string` and `Alterator::alterate` both list - // `BracedWord` beside the two literal forms, so - // `bca find --type string` reports a `proc` body as a string - // literal while this arm gives it no operand. That - // disagreement predates #1354 in shape — an interpolating - // `QuotedWord` is already `Unknown` here and a string - // there. #1318 now has the predicate that would settle it - // (`Getter::is_value_braced_word`), but `Checker::is_string` - // takes neither `code` nor `ancestors`, so applying it there - // is a trait widening across all twenty-odd languages rather - // than a Tcl edit — filed as #1381. The literal half is - // already right: `bca find --type string` reports - // `lappend x {a b}`'s `{a b}`, which is a string. + // (grammar-dispatch §7): `Checker::is_string` and + // `Alterator::alterate` both list `BracedWord` beside the two + // literal forms, and #1381 narrows both for a script body — + // `Checker::is_string_with_code` and + // `Alterator::keeps_children` ask + // `Getter::is_braced_script_word` — so `bca find --type + // string` reports `lappend x {a b}`'s `{a b}` and not a + // `proc` body, agreeing with this arm. Three disagreements + // remain, each recorded where it lives: an interpolating + // `QuotedWord` is `Unknown` here and a string there, which + // predates #1354; an empty or comment-only script body is an + // operand here and not a string there; and the value slots + // `is_braced_literal_slot` recognises (`proc {my proc}`, + // `namespace export {…}`) are strings there while + // `get_op_type_with_code` still bills their `{` as a block. // // `Checker::is_call` needs no such follow-up. It calls // every `Command` a call, including the ones inside a value diff --git a/big-code-analysis-ast/src/lang_helpers/irules.rs b/big-code-analysis-ast/src/lang_helpers/irules.rs index f4403cf41..696c9d2e2 100644 --- a/big-code-analysis-ast/src/lang_helpers/irules.rs +++ b/big-code-analysis-ast/src/lang_helpers/irules.rs @@ -14,5 +14,7 @@ pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { word_list: Irules::WordList as u16, simple_word: Irules::SimpleWord as u16, argument: Irules::Argument as u16, + procedure: Irules::Procedure as u16, + namespace: Irules::Namespace as u16, open_brace: Irules::LBRACE as u16, }; diff --git a/big-code-analysis-ast/src/lang_helpers/tcl.rs b/big-code-analysis-ast/src/lang_helpers/tcl.rs index 07693a816..ea6e21403 100644 --- a/big-code-analysis-ast/src/lang_helpers/tcl.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl.rs @@ -9,17 +9,21 @@ use crate::node::Node; /// `Getter::braced_word_op_type` and `Getter::is_braced_script_word` are /// instantiated with (#1354, #1318): the literal *value* form the guard /// keys on, the *script* form it gates on holding a command, the comment -/// kind that gate must not mistake for one, and the four kinds #1318's -/// role recognition walks — the generic `command`, its `word_list` -/// argument list, the `simple_word` a resolvable command name is spelled -/// with, and the `argument` whose braced child is a parameter default -/// rather than a script. +/// kind that gate must not mistake for one, the four kinds #1318's role +/// recognition walks — the generic `command`, its `word_list` argument +/// list, the `simple_word` a resolvable command name is spelled with, and +/// the `argument` whose braced child is a parameter default rather than a +/// script — and the `procedure` and `namespace` constructs whose value +/// slots the string and dump classifiers also recognise (#1381). /// /// It lives here rather than beside the `Getter` impl because three /// classifiers now read it — `Getter::get_op_type_with_code`, -/// `Checker::is_string_with_code` and `Alterator::alterate` — and a -/// second copy is exactly the drift `lang_helpers` exists to prevent +/// `Checker::is_string_with_code` and `Alterator::keeps_children` — and +/// a second copy is exactly the drift `lang_helpers` exists to prevent /// (#1381). +/// +/// `Namespace`, not `Namespace2`: both are spelled `namespace`, and the +/// suffixed variant is the keyword token inside the construct. pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { value: Tcl::BracedWordSimple as u16, script: Tcl::BracedWord as u16, @@ -28,6 +32,8 @@ pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { word_list: Tcl::WordList as u16, simple_word: Tcl::SimpleWord as u16, argument: Tcl::Argument as u16, + procedure: Tcl::Procedure as u16, + namespace: Tcl::Namespace as u16, open_brace: Tcl::LBRACE as u16, }; diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index 9ba749d21..d2bb296b9 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -598,8 +598,8 @@ impl<'tree, 'chain> Ancestors<'tree, 'chain> { /// /// The invariant is "`chain.last()` **is** `node.parent()`", and /// asking that outright costs [`Node::parent`]'s `O(depth)` — the - /// very lookup #1084 exists to remove. Per node, on all five walks - /// that construct a checked chain, it made every debug-build walk + /// very lookup #1084 exists to remove. Per node, on every walk that + /// constructs a checked chain, it made every debug-build walk /// `O(nodes × depth)` while the shipped walk is `O(nodes)`: a tax on /// every `cargo test`, worst on the deep-nesting regression tests /// that exist to pin the shipped walk's linearity (#1122). It now @@ -811,6 +811,20 @@ impl<'a> Cursor<'a> { self.0.goto_first_child() } + /// Moves to the first child that ends after `byte` and returns its + /// index in [`Node::children`] order, or `None` when no child does. + /// + /// The index is what makes this worth having over a sibling scan: + /// tree-sitter stores a `repeat()` child list under balanced hidden + /// nodes and skips each by its cached visible-child count, so this + /// is `O(log n)` in the sibling count where `children().position(..)` + /// is `O(n)` — the difference between a linear and a quadratic walk + /// when every child of a wide list asks for its own index. + #[inline] + pub fn goto_first_child_for_byte(&mut self, byte: usize) -> Option { + self.0.goto_first_child_for_byte(byte) + } + /// The node the cursor currently sits on. #[inline] #[must_use] @@ -1738,9 +1752,10 @@ mod tests { /// /// The seed is what decides this. [`Ancestors`] reads an empty chain /// as "this node is the root", so seeding empty — which is correct - /// for the one caller that exists today, `bca function`, whose walk - /// starts at the root — would report no parent for the subtree root - /// and shift every answer beneath it. For the JS getters that means + /// for every caller today (`bca function`, `find` and `count`, whose + /// walks all start at the root) — would report no parent for the + /// subtree root and shift every answer beneath it. For the JS getters + /// that means /// losing the `variable_declarator` a `function_expression` takes /// its name from, so the space would silently be named /// ``. diff --git a/big-code-analysis-ast/src/parser.rs b/big-code-analysis-ast/src/parser.rs index 615428e43..cd2638e4b 100644 --- a/big-code-analysis-ast/src/parser.rs +++ b/big-code-analysis-ast/src/parser.rs @@ -52,8 +52,8 @@ pub struct Parser { /// ancestry of a different tree. type FilterFn<'a> = dyn for<'t, 'c> Fn(&Node<'t>, Ancestors<'t, 'c>) -> bool + 'a; -/// Collection of node-matching predicates used by the AST-walking -/// metric and dump routines to decide whether to visit a node. +/// Collection of node-matching predicates the `find` and `count` walks +/// apply to each node they visit. pub struct Filter<'a> { filters: Vec>>, } diff --git a/big-code-analysis-ast/src/traits.rs b/big-code-analysis-ast/src/traits.rs index 582682eaf..ac8d13b7b 100644 --- a/big-code-analysis-ast/src/traits.rs +++ b/big-code-analysis-ast/src/traits.rs @@ -67,8 +67,8 @@ pub trait ParserTrait { /// The bytes the tree was parsed from (after macro expansion, for /// the C family). fn code(&self) -> &[u8]; - /// The returned [`Filter`] borrows `self` — the `"function"` - /// predicate reads the source bytes (#1162). + /// The returned [`Filter`] borrows `self` — the `"function"` and + /// `"string"` predicates read the source bytes (#1162, #1381). fn filters(&self, requested: &[String]) -> Filter<'_>; } diff --git a/big-code-analysis-book/src/commands/vcs.md b/big-code-analysis-book/src/commands/vcs.md index 6566793cc..f64b7516c 100644 --- a/big-code-analysis-book/src/commands/vcs.md +++ b/big-code-analysis-book/src/commands/vcs.md @@ -234,10 +234,13 @@ by the resolved `HEAD` SHA and the repository's identity: The cache is a pure optimization: a hit is **bit-identical** to a fresh walk, and the time windows are recomputed against the *current* moment on -every run, so a cached result is never stale. An entry is ignored — and +every run rather than frozen when the entry was written. An entry is ignored — and the history recomputed — whenever the schema, the score-formula version, -or the *walk-affecting* options differ; in particular **changing a window -forces a fresh walk**. (Finalization-only knobs such as `--risk-formula`, +the *walk-affecting* options, or the repository's effective `.mailmap` +differ; in particular **changing a window forces a fresh walk**. One walk +input is not covered: the git diff configuration (`diff.algorithm`, diff +drivers) decides the recorded churn, so after changing it run once with +`--clear-cache`. (Finalization-only knobs such as `--risk-formula`, `--emit-author-details`, `--author-hash-key`, and `--include-deleted` are applied on replay, so they reuse the same cached walk — a cached walk even re-finalizes under a *different* author-hash key without re-walking.) diff --git a/big-code-analysis-book/src/metrics.md b/big-code-analysis-book/src/metrics.md index db0418af9..f1c88a92c 100644 --- a/big-code-analysis-book/src/metrics.md +++ b/big-code-analysis-book/src/metrics.md @@ -151,7 +151,7 @@ application would over-count. | Ruby | Bare-predicate `if` / `unless` / `while` / `until` (block and modifier forms) count one condition | Idiomatic Ruby favours bare predicates (`if flag`, `x if flag`); counting the condition slot keeps ABC conditions at or above Ruby's cyclomatic decision count (the alignment enforced across the other languages). A comparison (`if a == b`) or `&&` / `\|\|` chain in the predicate is counted by its own operator / walker arm and is not double-counted. | | Bash | `if` / `elif` / `while` and each non-wildcard `case` arm count one condition | A Bash predicate is a *command*, so the branch keyword itself — not an embedded boolean expression — is the condition signal. Each matches a Bash cyclomatic decision; the bare `*)` case arm (the analogue of `default:`) is excluded, mirroring the cyclomatic standard count. The arithmetic ternary `$(( a ? b : c ))` therefore contributes nothing: it carries no branch keyword, so it falls outside the rule set rather than through a gap in it. | | Kotlin | `try` counts a condition alongside `catch` | Fitzpatrick counts both keywords, and Java / C# / C++ / Groovy already count both; Kotlin previously counted only the catch block. | -| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TypeScript, TSX, JavaScript and Mozjs; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). | +| C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript, TSX, JavaScript, Mozjs, Lua, Perl, Ruby, Bash, Elixir | A `<` or `>` that is not a comparison is not a condition | Every one of these grammars spells at least one non-comparison construct with the same bare `<` / `>` token a comparison uses, so the comparison rule is gated on the token's parent. What that excludes, per family: template and generic brackets in C++, Objective-C, Mozcpp, Rust, Go, Java, Groovy, C#, Kotlin, TypeScript and TSX (#1274); JSX tag delimiters in TypeScript, TSX, JavaScript and Mozjs; Lua 5.4 variable attributes (`local x = 1`); a C# comparison-operator overload's declared name (`operator <`); Kotlin's qualified super call (`super.g()`) (#1297); Perl's filehandle and lexical-handle readlines (``, `<$fh>` — but not ``, which the grammar lexes as one token); Ruby's superclass clause (`class Foo < Bar`) and comparison-operator method names (`def <(other)`), and Bash I/O redirection (`cmd > out`) (#1280); Elixir's sigil delimiters (`~s`) (#1256). C# additionally excludes the operator of a relational pattern (`x is > 0`, and the `> 5 =>` arm of a switch expression): the arm or `if` condition slot that owns the pattern already scores the decision, so counting the operator too charged a relational arm twice what the constant arm `5 => 1` scores (#1383). Its `>=` / `<=` spelling is excluded by the same rule through a separate token. The exclusion holds wherever the pattern sits, so a pattern no arm or condition slot owns (`bool b = x is > 0;`, `return x is > 0;`, a `when` guard) scores 0, one below the equivalent `x > 0`. PHP, Python, Tcl and iRules emit a bare `<` / `>` from no non-comparison production; C carries the same gate as its C-family siblings although, having no templates, it has nothing to exclude. The gate is a claim about the grammar's productions, not about every parse: where a grammar resolves a generic *call* into nested `binary_expression` nodes, as tree-sitter-kotlin-ng does for `id(a)`, no polarity can exclude it (#1394). | | Java, Groovy, C#, TypeScript, TSX | A `?` used as type syntax is not a ternary | In each of these grammars the ternary `?` and the type-syntax `?` are the *same* anonymous token, so the ternary rule above is gated on the token's parent. Java and Groovy exclude the wildcard bound `List` (#1274); C# excludes the nullable type `int? x` and the constraint `where T : class?`; TypeScript and TSX exclude optional parameters, properties, methods, class fields and tuple elements, and conditional types (`T extends U ? X : Y`, which the type checker resolves and erases before runtime, so it is no more a branch than the `<` / `>` already excluded) (#1275). Safe navigation is untouched: C#'s `a?.b` shares the same token and still counts, while the other languages spell theirs as a distinct one. | #### Worked example diff --git a/big-code-analysis-book/src/python/sarif.md b/big-code-analysis-book/src/python/sarif.md index 2290ded1b..e70d15f1d 100644 --- a/big-code-analysis-book/src/python/sarif.md +++ b/big-code-analysis-book/src/python/sarif.md @@ -10,14 +10,16 @@ writer that backs `bca check --report-format sarif`, so the schema URL, tool driver name / version, and rule descriptions match the CLI byte-for-byte. -Findings match in order as well as in content. Within a file, both -surfaces walk the space tree depth-first in source order — a space, then -its children left to right — and report a space's several breaches -alphabetically by metric name. For the same file and thresholds the two -`results` arrays therefore line up entry for entry, and a diff between -them is a real divergence rather than a walk-order artifact. Order -*between* files is the caller's: `to_sarif` follows the iterable you pass -it, while `bca check` follows the paths it resolved. +Findings match in order as well as in content. Both surfaces sort their +findings by path, then start line, then metric name — the order +`bca check` applies after its walk — and findings tying on all three +keep the depth-first source order of the space tree. For the same files +and thresholds the two `results` arrays therefore line up entry for +entry against `bca check --no-suppress`. `to_sarif` compares raw metric +values, so it applies none of the in-source +[suppression markers](../commands/suppression.md) `bca check` honours by +default (each marked space keeps its `suppressed` key, for a caller that +wants to filter), no baseline, and no `[check] exclude` globs. Examples on this page import the package as `bca` (`import big_code_analysis as bca`). A bare `bca` in a shell command is diff --git a/big-code-analysis-book/src/python/vcs.md b/big-code-analysis-book/src/python/vcs.md index 2c24fc36a..9cc80850f 100644 --- a/big-code-analysis-book/src/python/vcs.md +++ b/big-code-analysis-book/src/python/vcs.md @@ -203,7 +203,8 @@ same opt-in described under `vcs.rank` keeps a [persistent cache](../commands/vcs.md#caching) of each history walk, on by default. A cache hit is bit-identical to a fresh walk, and the time windows are recomputed against the current moment on -every run, so a cached result is never stale. +every run. The [caching section](../commands/vcs.md#caching) lists what +invalidates an entry, and the one walk input that does not. ```python from big_code_analysis import vcs diff --git a/big-code-analysis-py/python/big_code_analysis/_native.pyi b/big-code-analysis-py/python/big_code_analysis/_native.pyi index 90ba5e80c..f16af325e 100644 --- a/big-code-analysis-py/python/big_code_analysis/_native.pyi +++ b/big-code-analysis-py/python/big_code_analysis/_native.pyi @@ -1006,12 +1006,14 @@ def to_sarif( ````, matching the CLI's ``space_segment``. Findings are emitted in the CLI's order as well as its shape - (#1402): depth-first through the space tree in source order, a - space before its children and siblings left to right, and within - one space, alphabetically by metric name — so ``results`` for a - given file is comparable positionally, not just as a set. Order - *between* files is whatever the iterable passed as ``result`` - yields, where ``bca check`` follows its own resolved walk list. + (#1402): sorted by path, then start line, then metric name — the + sort ``bca check`` applies after its walk — with findings that tie + on all three in depth-first source order. ``results`` is therefore + comparable positionally, not just as a set, against + ``bca check --no-suppress``: ``to_sarif`` compares raw metric + values, so it applies no in-source suppression markers (a marked + space keeps its ``suppressed`` key), no baseline and no + ``[check] exclude`` globs. Raises ------ diff --git a/big-code-analysis-py/src/sarif.rs b/big-code-analysis-py/src/sarif.rs index dfd078f3a..b52d623db 100644 --- a/big-code-analysis-py/src/sarif.rs +++ b/big-code-analysis-py/src/sarif.rs @@ -62,32 +62,6 @@ //! metric at the own-value field and the binding emits at every space — //! no leaf-only special-casing remains. //! -//! # Emission order -//! -//! Order is part of the parity contract, not just the finding set: for -//! one input file the two front-ends emit the same results in the same -//! sequence, so a consumer may diff the documents positionally. Two -//! axes decide that sequence, and #1402 fixed both: -//! -//! * **Across spaces** — both walk the space tree depth-first in source -//! order, a space then its children left to right, which for a LIFO -//! stack means pushing each sibling set reversed (see -//! [`push_child_spaces`]). This binding pushed them in source order -//! and so reported every sibling set backwards. -//! * **Within one space** — the CLI's `ThresholdSet` iterates a -//! `BTreeMap`, so a space breaching several metrics reports them -//! alphabetically by canonical name; iterating the `thresholds` -//! `PyDict` yielded the caller's insertion order instead. -//! [`resolve_thresholds`] now sorts to match. -//! -//! Both bugs left the finding *set* correct and only the sequence -//! wrong, which is why the sorted parity helpers in -//! `tests/test_sarif.py` never saw either. -//! -//! Order *across files* is the caller's: [`collect_offenders_from_iter`] -//! follows the iterable it is handed, where `bca check` follows its own -//! resolved walk list. -//! //! `nargs` is the fifth and arrived by the opposite route: its serialized //! shape did not change, its *gate* did. #1196 moved the CLI extractor //! from `total()` to the callable's own parameter list, leaving this @@ -105,6 +79,28 @@ //! its own limits), so this binding adopts the same posture. An empty //! `thresholds` produces a well-formed SARIF run with `results: []` //! and `rules: []`, matching the CLI's empty case. +//! +//! # Emission order +//! +//! Order is part of the parity contract, not only the finding set, so a +//! consumer may diff the two documents positionally (#1402). The binding +//! sorts its findings with the comparator `bca check` applies after its +//! walk — path, then start line, then metric name — in +//! [`collect_offenders_for_input`]. The sort is stable, so findings tying +//! on all three keys (one metric breached by two spaces that start on the +//! same line) keep the depth-first, source-order walk both front-ends +//! share, which is the one job left to [`push_child_spaces`]'s reversal. +//! +//! Mirroring the walk alone is not enough, because the CLI sorts *after* +//! walking. A function starting on line 1 that breaches `cyclomatic` +//! precedes the file unit's `loc.sloc` breach on that line, and several +//! files come out in path order whatever order they were handed in. +//! +//! The reference is `bca check --no-suppress`, not a bare `bca check`: +//! this binding compares raw metric values, so it applies none of the +//! in-source suppression markers `bca check` honours by default (a marked +//! space keeps its `suppressed` key for a caller that wants to filter), +//! no baseline, and no `[check] exclude` globs. use std::path::{Path, PathBuf}; @@ -363,16 +359,6 @@ fn resolve_thresholds(thresholds: Option<&Bound<'_, PyDict>>) -> PyResult` - // (`ThresholdSet::build_tiered`), so a space breaching several metrics - // reports them alphabetically by canonical name. Iterating a `PyDict` - // yields the caller's insertion order instead, which made a two-metric - // `to_sarif` disagree with `bca check` on the order of one space's - // findings — and made the binding's own output depend on how the - // caller happened to spell the dict. Sorting here reproduces the - // CLI's order and drops that dependency (#1402). Names are unique, - // so the sort needs no tiebreak. - out.sort_unstable_by_key(|t| t.name); Ok(out) } @@ -595,10 +581,12 @@ fn record_threshold_breaches( /// Children land on the stack in **reverse** source order so the caller's /// `pop()` visits them in source order, matching the CLI's /// `evaluate_with_policy` (which pushes `spaces.iter().rev()` for the same -/// reason) — emission order is part of the SARIF parity contract (#1402). -/// Reversing the tail this call appended, rather than the input, keeps the -/// walk working for any Python iterable — `spaces` need not be a sequence — -/// and allocates nothing extra. +/// reason). The final sort in [`collect_offenders_for_input`] decides the +/// emission order; this walk order survives it only as the tiebreak between +/// findings that share a path, start line and metric (#1402). Reversing the +/// tail this call appended, rather than the input, keeps the walk working +/// for any Python iterable — `spaces` need not be a sequence — and +/// allocates nothing extra. fn push_child_spaces<'py>( space: &Bound<'py, PyDict>, child_prefix: &str, @@ -710,6 +698,15 @@ fn collect_offenders_for_input( let thresholds = resolve_thresholds(thresholds)?; let mut offenders: Vec = Vec::new(); dispatch_by_input_kind(result, &thresholds, &mut offenders)?; + // `bca check`'s own comparator, applied after the walk exactly as + // `run_check_walk` applies it — see "Emission order" in the module doc. + // Stable, so findings tying on all three keys keep the walk's order. + offenders.sort_by(|a, b| { + a.path + .cmp(&b.path) + .then(a.start_line.cmp(&b.start_line)) + .then(a.metric.cmp(&b.metric)) + }); Ok(offenders) } diff --git a/big-code-analysis-py/tests/test_sarif.py b/big-code-analysis-py/tests/test_sarif.py index 67a5fd3fc..9b8ee47be 100644 --- a/big-code-analysis-py/tests/test_sarif.py +++ b/big-code-analysis-py/tests/test_sarif.py @@ -809,20 +809,21 @@ def test_to_sarif_emits_results_in_cli_walk_order(bca_binary: str, tmp_path: Pat """Emission order is part of the parity contract, not only the finding set (#1402). - Both front-ends walk the space tree depth-first in source order — a - space, then its children left to right — so the two ``results`` arrays - are comparable **positionally**. The binding walks with an explicit - LIFO stack and, before the fix, pushed each sibling set in source - order, so every sibling set popped reversed at every level: on this - fixture it emitted ``second``, ``first``, ````, + Both front-ends emit their findings sorted by path, start line and + metric, ties in depth-first source order, so the two ``results`` arrays + are comparable **positionally**. Before the fix the binding emitted in + walk order, and its LIFO stack took each sibling set in source order, + so every sibling set popped reversed at every level: on this fixture + it emitted ``second``, ``first``, ````, ``::``, ``::``, ````. The set was right and only the sequence was wrong, which is why ``_sarif_rows`` and ``_assert_sarif_results_match`` — both of which sort — could not see it. - The fixture nests three sibling sets of two so a single-level fix, or - a fix that reversed the whole result list instead of each sibling set, - still fails. + Every space here starts on its own line, so the line key alone now + decides this order across three nested sibling sets. The walk's own + reversal is what a tie exposes, and + ``test_to_sarif_same_line_findings_follow_the_cli_sort`` pins that. """ src = tmp_path / "nested_closures.rs" src.write_text( @@ -844,7 +845,7 @@ def test_to_sarif_emits_results_in_cli_walk_order(bca_binary: str, tmp_path: Pat analyzed = bca.analyze(src) assert analyzed is not None, "fixture must not be skipped" # Fixture adequacy: at least three sibling sets must hold two children - # each, or a reversal has nothing to reverse and this test cannot fail. + # each, or the nested ordering this pins has nothing left to order. multi = [n for n in _sibling_set_sizes(analyzed) if n >= 2] assert len(multi) >= 3, f"fixture must keep its nested sibling pairs; sizes {multi!r}" @@ -873,8 +874,8 @@ def test_to_sarif_orders_one_spaces_metrics_alphabetically(bca_binary: str, tmp_ """The second ordering axis (#1402): within a single space, several breaches come out alphabetically by metric name on both sides. - The CLI builds its threshold entries from a ``BTreeMap``, so it - reports ``cyclomatic`` before ``nargs``. The binding iterated the + The CLI sorts its findings by metric name after path and start line, + so it reports ``cyclomatic`` before ``nargs``. The binding iterated the ``thresholds`` dict, which yields Python insertion order — so ``{"nargs": 1, "cyclomatic": 1}`` came out ``nargs`` first and the same call spelled the other way round came out ``cyclomatic`` first. @@ -947,6 +948,57 @@ def test_to_sarif_orders_one_spaces_metrics_alphabetically(bca_binary: str, tmp_ ) +def test_to_sarif_same_line_findings_follow_the_cli_sort(bca_binary: str, tmp_path: Path) -> None: + """``bca check`` sorts its findings by path, start line and metric + *after* walking, so mirroring its walk alone does not reproduce its + order (#1402). + + Two findings on one line are where the two rules part company. In the + first fixture the file unit (always line 1) breaches ``loc.sloc`` and a + function starting on line 1 breaches ``cyclomatic``: the walk visits + the unit first, the CLI's sort puts ``cyclomatic`` first. The first cut + of #1402 mirrored only the walk and emitted the two reversed. + + The second fixture is the tie the sort cannot break — two functions on + one line breaching the same metric — so there only the walk decides. + It is the row that fails if the binding stops reversing each sibling + set it pushes, which no distinct-line fixture can see any more. + """ + cases: tuple[tuple[str, dict[str, float], list[tuple[int, str, str]]], ...] = ( + ( + "fn outer(a: i32, b: i32) -> i32 {\n if a > b { 1 } else { 2 }\n}\n", + {"loc.sloc": 1, "cyclomatic": 1}, + [ + (1, "outer", "cyclomatic 2 exceeds limit 1"), + (1, "", "loc.sloc 3 exceeds limit 1"), + ], + ), + ( + "fn a(x: bool) -> i32 { if x { 1 } else { 2 } } " + "fn b(y: bool) -> i32 { if y { 1 } else { 2 } }\n", + {"cyclomatic": 1}, + [ + (1, "a", "cyclomatic 2 exceeds limit 1"), + (1, "b", "cyclomatic 2 exceeds limit 1"), + ], + ), + ) + for index, (source, limits, expected) in enumerate(cases): + src = tmp_path / f"same_line_{index}.rs" + src.write_text(source) + analyzed = bca.analyze(src) + assert analyzed is not None, "fixture must not be skipped" + specs = tuple(f"{name}={limit}" for name, limit in limits.items()) + cli_rows = _sarif_rows_in_emission_order( + _cli_check_sarif(bca_binary, src, threshold=specs)["runs"][0]["results"] + ) + py_rows = _sarif_rows_in_emission_order( + _parse(bca.to_sarif(analyzed, thresholds=limits))["runs"][0]["results"] + ) + assert cli_rows == expected, f"CLI reference order moved: {cli_rows!r}" + assert py_rows == expected, f"binding must match the CLI's order: {py_rows!r}" + + def test_to_sarif_anonymous_space_collapses_to_anon_line() -> None: """A space whose name is the literal ```` (every grammar's closure/lambda sentinel) collapses to ````, @@ -1108,11 +1160,18 @@ def test_to_sarif_child_order_survives_skipped_and_childless_spaces() -> None: skipped entry must not leave a gap that scrambles the surviving siblings, and a childless space must not reverse its parent's tail a second time. + + All three children start on the same line. The binding sorts its + findings by path, start line and metric as ``bca check`` does, so on + distinct lines the sort alone would restore source order and this test + could no longer see the walk; sharing the line and the metric leaves + the walk order as the only tiebreak, which is the job the reversal + still has. """ childless: dict[str, Any] = { "name": "beta", "kind": "function", - "start_line": 20, + "start_line": 10, "end_line": 25, # No "spaces" key at all — the `get_item` miss branch. "metrics": {"cyclomatic": {"value": 5.0, "sum": 5.0}}, @@ -1127,7 +1186,7 @@ def test_to_sarif_child_order_survives_skipped_and_childless_spaces() -> None: "not a space", childless, 42, - _fake_function_dict(name="gamma", start_line=30, end_line=35), + _fake_function_dict(name="gamma", start_line=10, end_line=35), ], "metrics": {"cyclomatic": {"value": 1.0, "sum": 11.0}}, } @@ -1145,8 +1204,8 @@ def test_to_sarif_child_order_survives_skipped_and_childless_spaces() -> None: message = "cyclomatic 5 exceeds limit 1" assert rows == [ (10, "alpha", message), - (20, "beta", message), - (30, "gamma", message), + (10, "beta", message), + (10, "gamma", message), ], f"surviving children must stay in source order: {rows!r}" diff --git a/docs/development/benchmarking.md b/docs/development/benchmarking.md index 6a717de33..03cf562c7 100644 --- a/docs/development/benchmarking.md +++ b/docs/development/benchmarking.md @@ -213,9 +213,10 @@ above is still the reason to reach for a field lookup first. The `Ancestors::unknown()` call sites that remain are deliberate rather than deferred: the two synthetic-`Unit`-root pushes hand it a node that -*is* the root, `parser.rs`'s `--filter function` predicate is applied -outside any walk, and the `Npm` arms that test a node's children cannot -extend a borrowed slice by one element without allocating. +*is* the root, and the `Npm` arms that test a node's children cannot +extend a borrowed slice by one element without allocating. The +`parser.rs` filter predicates were the third until #1381 routed `find` +and `count` through `Search::act_on_node`'s chain. `nom/nested-attributed-fn` and `nom/wide-attributed-fn` are the first probes that walk under a non-default `MetricsOptions`. Both hot paths diff --git a/docs/development/lessons_learned.md b/docs/development/lessons_learned.md index c0618f3dd..c5d8ab779 100644 --- a/docs/development/lessons_learned.md +++ b/docs/development/lessons_learned.md @@ -3516,10 +3516,13 @@ then each had admitted a role nobody listed — JSX tag delimiters, a `super_expression`, an `operator_declaration` — while JavaScript (and its Mozjs fork), Lua and Perl had no gate at all, and Perl was the row the issue had declared immune. The same fix is also the caveat on the -polarity: C# needed a *second* allowlist entry, because `x is > 0` is -a comparison that lives outside `binary_expression`, and a one-entry -allowlist would have under-counted it silently — the closed form's -failure, which lesson 19 describes and no snapshot shows. Neither +polarity: C# at first kept a *second* allowlist entry, because +`x is > 0` is a comparison outside `binary_expression`, and a one-entry +allowlist would have dropped it silently — the closed form's failure, +which lesson 19 describes and no snapshot shows. #1383 later removed the +entry on measurement, because the arm or condition slot owning the +pattern already scores the decision: the same zero, decided rather than +missed. Neither polarity is free; the allowlist is preferred because its set is the smaller one to enumerate and its miss reads as a zero rather than a phantom. diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index b0d9283d1..e0c9d6a82 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -53,21 +53,20 @@ pub mod nested; /// Node-kind filters handed to [`Ast::count`] and [`Ast::find`]. /// -/// `"function"` earns its place twice over: it is the only filter that -/// reaches a `Checker` predicate taking `code`, and it applies that -/// predicate with an unknown ancestor chain, which climbs by -/// `Node::parent` at `O(depth^2)` per candidate node. That makes it the -/// one filter a deeply-nested input can turn into a complexity problem. +/// `"function"` and `"string"` earn their places twice over: they are the +/// two filters that reach a `Checker` predicate taking `code`, and both +/// consult the ancestor chain `find` and `count` thread (#1381) — the +/// Tcl-family `"string"` on every braced word — so they are the filters +/// a deeply-nested or wide input exercises hardest. /// /// **The order is load-bearing, and `"all"` must stay last.** /// `Filter::any` returns on its first matching predicate, and `"all"` is /// `|_| true`, so listing it first makes every other entry unreachable — -/// `is_call`, `is_comment`, `is_error`, `is_string` and +/// `is_call`, `is_comment`, `is_error`, `is_string_with_code` and /// `is_func_with_code` are then never called on any node, in any target. /// It was first here until a review caught it, which had quietly reduced /// `count` and `find` to a bare walk and left the `"function"` predicate -/// above — the whole reason the nesting generator is sized the way it -/// is — dead. Kept rather than dropped because it still makes every node +/// above dead. Kept rather than dropped because it still makes every node /// match once the real predicates have each had their say, so `find` /// builds a maximal result vector. /// diff --git a/fuzz/src/nested.rs b/fuzz/src/nested.rs index 98dd9213e..a6693360b 100644 --- a/fuzz/src/nested.rs +++ b/fuzz/src/nested.rs @@ -36,13 +36,12 @@ use big_code_analysis::LANG; /// contribute two or three, so this clears both. /// /// It is also an upper bound on run time, which is why it is not simply -/// set enormous. The `"function"` filter applies its predicate with an -/// unknown ancestor chain, climbing by `Node::parent` at `O(depth^2)` -/// per candidate node; with a candidate at every level that is cubic in -/// this constant. At 512 the worst case stays comfortably inside the -/// `-timeout=10` the fuzz runs use, so a timeout report means a real -/// complexity regression rather than the generator outgrowing the -/// budget. +/// set enormous. The `"function"` and `"string"` filters consult the +/// ancestor chain, and `find` / `count` now thread it (#1381) rather +/// than climbing by `Node::parent`, so depth no longer multiplies their +/// per-node cost. `docs/development/fuzzing.md` records what the deepest +/// seed costs under `AddressSanitizer`, and why `FUZZ_TIMEOUT` is sized to +/// that rather than used as a complexity gate. pub const MAX_NESTING_DEPTH: usize = 512; /// Languages the generator knows how to nest. diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 0aedf7c81..421568f3e 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -3743,8 +3743,9 @@ mod tests { // comparison-operator overload scored a condition. // // The fixture carries the two overloads plus one `a < b` inside a - // `binary_expression` and one `x is > 0` inside a - // `relational_pattern`. Every mis-aim lands on its own number: 5 + // `binary_expression` and one `x is > 0 ? 2 : 3`, which the grammar + // parses as a `relational_pattern` whose operand is the ternary (see + // the assertion). Every mis-aim lands on its own number: 5 // with neither gate, 3 if `RelationalPattern` is readmitted to the // allowlist (#1383 dropped it), 1 if the gate swallows // `BinaryExpression` too (only the ternary `?` survives), 0 if the @@ -3776,8 +3777,13 @@ mod tests { ); } // 2, not 3, since #1383: the `a < b` comparison and the - // ternary `?`. `m`'s `x is > 0` no longer adds a third - // — the ternary it sits in is the decision. + // ternary `?`. tree-sitter-c-sharp 0.23.5 parses + // `x is > 0 ? 2 : 3` as `x is > (0 ? 2 : 3)` — the ternary + // is the pattern's operand, so the pattern sits in no + // decision slot and its `>` scores nothing, while the + // ternary's condition is the literal `0`. C# itself binds + // it `(x is > 0) ? 2 : 3`; a grammar that agrees makes + // this 3, because that condition slot scores the `is` test. assert_eq!(class.spaces[2].metrics.abc.conditions(), 2); }, ); @@ -3937,10 +3943,20 @@ mod tests { }); } - // The boundary of #1383: the pattern's operator stops counting, an - // operator in the arm's `when` guard keeps counting. Both sit under - // the same `switch_expression_arm`, so this is what stops a later - // "patterns don't count" pass from suppressing the guard too. + // The boundary of #1383: the pattern's operator stops counting, a + // *binary* operator in the arm's `when` guard keeps counting (`w`). + // Both sit under the same `switch_expression_arm`, so this is what + // stops a later "patterns don't count" pass from suppressing the + // guard too. + // + // A relational *pattern* in the guard (`g`) is the other side of that + // line: it scores nothing, though no slot pays for it, because the + // gate is on the operator's parent and a guard is a decision slot + // neither metric models. It is the outside-slot trade + // `csharp_relational_pattern_outside_a_decision_slot_scores_zero` + // records, and it leaves `when n is > 5` one below `when n > 5`; + // modelling the guard as a slot would score the `is` test once and + // close the gap (#1422). // // It is also where §8 does not hold, and the reason is worth // stating precisely, because the obvious reading is wrong. Neither @@ -3969,6 +3985,7 @@ mod tests { check_func_space::( "class A { int w(int x) => x switch { > 0 when x % 2 == 0 => 1, > 0 => 2, _ => 3 }; + int g(int x) => x switch { int n when n is > 5 => 1, _ => 0 }; }", "foo.cs", |space| { @@ -3983,6 +4000,19 @@ mod tests { "two arms plus the guard's `==`" ); assert_eq!(m.metrics.cyclomatic.cyclomatic(), 3); + + let g = &space.spaces[0].spaces[1]; + assert_eq!(g.name.as_deref(), Some("g")); + // FIXME(#1422): the one non-discard arm alone. The guard's + // pattern `>` scores nothing where `when n > 5` would add + // one; a guard modelled as a condition slot would score the + // `is` test instead, taking this to 2. + assert_eq!( + g.metrics.abc.conditions(), + 1, + "one arm; the guard's relational pattern scores nothing" + ); + assert_eq!(g.metrics.cyclomatic.cyclomatic(), 2); }, ); } diff --git a/src/metrics/halstead.rs b/src/metrics/halstead.rs index 7435f2931..71dba4990 100644 --- a/src/metrics/halstead.rs +++ b/src/metrics/halstead.rs @@ -2520,6 +2520,56 @@ mod tests { ); } + // Groovy's `Super` operator arm, pinned from both sides. The pinned + // grammar emits the `super` token only as a `wildcard` bound, where it + // is an operator exactly as in Java above; every super-*reference* is + // a plain `identifier`, and so an operand. The node census is the + // positive pin #1419 wants rather than an unreachability one — an + // unreachability pin fails on any wildcard — so a grammar bump that + // routes a reference to this kind fails here by name instead of + // silently billing it as an operator through the ungated arm. + #[test] + fn groovy_wildcard_super_bound_stays_an_operator() { + let bounds = + "class T {\n void m(List a, List b) { }\n}"; + let ops = ops_of::(bounds, "foo.groovy"); + assert!( + ops.operators.iter().any(|o| o == "super"), + "`? super String` must keep `super` an operator; operators were {:?}", + ops.operators + ); + assert!( + !ops.operands.iter().any(|o| o == "super"), + "a wildcard bound must not bill `super` as an operand; operands were {:?}", + ops.operands + ); + assert!( + ops.operators.iter().any(|o| o == "extends"), + "the fixture must still contain the `? extends String` bound this arm \ + mirrors; operators were {:?}", + ops.operators + ); + + let mixed = "class T extends B {\n void m(List a) { super.h() }\n}"; + let mut tokens = 0; + for_each_node_with_chain::(mixed.as_bytes(), |node, chain| { + if node.kind_id() == Groovy::Super as u16 { + tokens += 1; + assert_eq!( + chain.last().map(Node::kind_id), + Some(Groovy::Wildcard as u16), + "groovy: a `super` token outside a wildcard bound reaches the \ + ungated operator arm (#1419)" + ); + } + }); + assert_eq!( + tokens, 1, + "the fixture's one wildcard bound must still spell `super`, and its \ + `super.h()` reference must not" + ); + } + #[test] fn groovy_operators_and_operands() { check_metrics::( diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index ba27aeac0..f22be7524 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -776,7 +776,11 @@ impl Stats { /// inserts a raw `start` row — and here that `start` *is* the /// phantom row. [`Node::end_line`] already encodes "a node whose /// end column is 0 does not occupy the row it ends on"; nothing - /// encoded the same rule for a node that *begins* past the span. + /// encoded the same rule for a node that *begins* past the span, nor + /// applied it to the raw end row `init` hands the comment arms — so a + /// comment ending at column 0 of the row past the last one (Perl POD + /// running to end-of-file without `=cut`) counted that row as a + /// comment. The clamp removes both. /// /// `ploc > sloc` is a contract violation rather than a rounding /// artifact: [`Stats::blank`] saturates at 0, so the clamp there @@ -787,8 +791,10 @@ impl Stats { /// shape nobody sampled, so the rule lives once, here, keyed on the /// span every language already reports. /// - /// **Input the grammar parses is untouched — which is not the same - /// as input that looks fine.** Over the `pdf.js`, `DeepSpeech` and + /// **Input whose tree stays inside the file's rows is untouched, and + /// a clean parse does not guarantee that**: Perl POD at end of file + /// and Ruby's unterminated `<<~` heredoc both parse without an error + /// node and still move. Over the `pdf.js`, `DeepSpeech` and /// `serde` corpora exactly one of 1,876 files moves, and it is a /// real bug fixed rather than a real row lost: /// `DeepSpeech/parse_valgrind_suppressions.sh` leaves a MISSING `}` @@ -851,24 +857,21 @@ impl Stats { // cause from the phantom row above, and asserting `sloc()` here // would fire on it. Tighten this to `sloc()` once #1417 lands. // - // Both values are bound first because `ploc()` and `cloc()` - // popcount their word arrays since #1109, and a `debug_assert!` - // evaluates its message arguments separately from its - // condition. O(words) per space, the same order as the - // `compute_minmax` that follows; per node it would be the - // quadratic shape #1122 removed. - #[cfg(debug_assertions)] - { - let (ploc, cloc) = (self.ploc(), self.cloc()); - debug_assert!( - ploc <= span as u64, - "ploc {ploc} exceeds the {span} row span it was clamped to" - ); - debug_assert!( - cloc <= span as u64, - "cloc {cloc} exceeds the {span} row span it was clamped to" - ); - } + // `ploc()` and `cloc()` popcount their word arrays (#1109): + // O(words) per space, the same order as the `compute_minmax` that + // follows, where per node it would be the quadratic shape #1122 + // removed. The message repeats the call, but an assertion's + // message arguments are evaluated only on the failing branch. + debug_assert!( + self.ploc() <= span as u64, + "ploc {} exceeds the {span} row span it was clamped to", + self.ploc() + ); + debug_assert!( + self.cloc() <= span as u64, + "cloc {} exceeds the {span} row span it was clamped to", + self.cloc() + ); } } @@ -8515,10 +8518,11 @@ $y = 10 + match ($x) { 1 => 2, default => 0 };", }); } - /// Analyses `source` byte-for-byte as PHP, for the two #1396 tests - /// `check_metrics` cannot carry: one ends at EOF, which that shim - /// normalises away, and the other loops over labelled cases, which - /// its bare `fn` callback cannot close over. + /// Analyses `source` byte-for-byte as PHP, for the #1396 tests + /// `check_metrics` cannot carry — one ends at EOF, which that shim + /// normalises away, and one loops over labelled cases, which its bare + /// `fn` callback cannot close over — and for the bounds test beside + /// them. #[cfg(feature = "php")] fn php_loc(source: &[u8]) -> Stats { metrics_verbatim( @@ -8558,11 +8562,11 @@ $y = 10 + match ($x) { 1 => 2, default => 0 };", /// The nowdoc half of #1396, which the issue did not measure and /// which was worse: it reported `ploc 4, blank 2` on this fixture /// against the heredoc's `ploc 5, blank 1`. The grammar shape - /// differs — a nowdoc body is *not* one `nowdoc_string` per row. - /// tree-sitter-php 0.24.2 emits one for the first line and a single - /// multi-row `nowdoc_string` for everything after it, so the - /// catch-all's start-row insertion lost every interior row of that - /// second node rather than just the empty one. + /// differs: tree-sitter-php 0.24.2 emits one `nowdoc_string` per body + /// line, but each after the first starts at the *end of the row + /// before it*, so the catch-all's start-row insertion credited every + /// body row to its predecessor and the last one (`b`) to nothing, on + /// top of the empty row. /// /// Same `sloc` fixture anchor as the heredoc test above. #[cfg(feature = "php")] @@ -8611,7 +8615,7 @@ $y = 10 + match ($x) { 1 => 2, default => 0 };", #[test] fn php_heredoc_spellings_credit_every_row_to_ploc() { // (label, source, expected sloc) — expected ploc is that same - // sloc and expected blank is 0 for all four, which is the + // sloc and expected blank is 0 for all five, which is the // property under test. let cases = [ ("all-empty heredoc body", " Result { /// /// # Residual window /// -/// The walk re-opens the mailmap for itself, so an edit landing between -/// this call and that one stamps the entry with the pre-edit fingerprint -/// over post-edit events. The next run under the edited mailmap -/// fingerprints differently and heals it; only reverting the mailmap -/// before any such run leaves a wrong hit reachable. Closing the window -/// means threading one snapshot through the walk — see issue #1409. +/// The walk re-opens the mailmap for itself, and nothing makes the two +/// reads agree. An edit can land between them, and so can a transient +/// that changes nothing on disk: `open_mailmap` turns a failed or +/// mid-rewrite read into an empty or partial snapshot rather than an +/// error — a truncate- or rename-then-write save of unchanged content, +/// or a swallowed I/O error. The entry is then stamped with this call's +/// view over events resolved under the walk's. +/// +/// Which read saw the content that lasts decides whether it heals. When +/// the walk did, the next run at the same `HEAD` fingerprints +/// differently and overwrites the entry. When this call did — every +/// transient, and an edit later reverted — later runs reproduce the +/// stamp: the pure hit serves the entry, and the incremental splice +/// carries its events forward until they leave the long window, a +/// mailmap or option change moves the fingerprint, or `--clear-cache` +/// runs. Closing the window means threading one snapshot through the +/// walk — see issue #1409. #[must_use] pub(crate) fn mailmap_digest(repo: &gix::Repository) -> u64 { let mut hasher = DefaultHasher::new(); diff --git a/tests/api/ast_seam_test.rs b/tests/api/ast_seam_test.rs index 48450c7c1..a5a43c371 100644 --- a/tests/api/ast_seam_test.rs +++ b/tests/api/ast_seam_test.rs @@ -736,12 +736,6 @@ fn preprocess_harvest_feeds_the_macro_masking_pass() { assert_eq!(untouched.source(), source); } -/// The texts `Ast::find` reports for `--type string`, in source order. -/// -/// Returning the texts rather than a count is what lets the callers below -/// assert *which* nodes were reported. A count alone cannot tell "the -/// script body dropped out" from "the literal dropped out and something -/// else appeared", and the two failures want opposite fixes. /// A `proc` body (the script) holding a quoted word, plus a braced word /// (the two literals). Shared by both tests below so the `find` list and /// the `count` total describe the same bytes. @@ -760,6 +754,12 @@ const TCL_SCRIPT_AND_LITERALS: &str = "proc p {x} { puts \"q\" }\nlappend l {a b const IRULES_SCRIPT_AND_LITERALS: &str = "when HTTP_REQUEST { log local0. \"hi\" }\nlappend l {x y}\n"; +/// The texts `Ast::find` reports for `--type string`, in source order. +/// +/// Returning the texts rather than a count is what lets the callers below +/// assert *which* nodes were reported. A count alone cannot tell "the +/// script body dropped out" from "the literal dropped out and something +/// else appeared", and the two failures want opposite fixes. #[cfg(any(feature = "tcl", feature = "irules"))] fn strings_found(lang: LANG, code: &str) -> Vec { let ast = Ast::parse(Source::new(lang, code.as_bytes())).expect("language feature enabled"); diff --git a/tests/grammars/alterator_string_flattening.rs b/tests/grammars/alterator_string_flattening.rs index 77c4fc681..3beb10a15 100644 --- a/tests/grammars/alterator_string_flattening.rs +++ b/tests/grammars/alterator_string_flattening.rs @@ -92,6 +92,15 @@ flatten_cases! { // which is the opposite over-correction. tcl_flattens_braced_value: LANG::Tcl, "lappend x {a b}\n", "f.tcl", "{a b}"; irules_flattens_braced_value: LANG::Irules, "lappend b {x y}\n", "f.irule", "{x y}"; + // The value slots of a construct `is_value_braced_word` classifies + // whole as script-taking. Without `is_braced_literal_slot` the dump + // rendered `{my proc}` as a command named `my` and `{a b}` as a + // command named `a` — the literal's text survived only inside a + // subtree the source does not contain. + tcl_flattens_braced_proc_name: LANG::Tcl, "proc {my proc} {} {}\n", "f.tcl", "{my proc}"; + tcl_flattens_namespace_argument: LANG::Tcl, "namespace export {a b}\n", "f.tcl", "{a b}"; + irules_flattens_braced_proc_name: LANG::Irules, "proc {my proc} {} {}\n", "f.irule", "{my proc}"; + irules_flattens_namespace_argument: LANG::Irules, "namespace export {a b}\n", "f.irule", "{a b}"; ruby_flattens_string_literal: LANG::Ruby, "s = \"hi\"\n", "f.rb", "\"hi\""; elixir_flattens_string_literal: LANG::Elixir, "s = \"hi\"\n", "f.ex", "\"hi\""; } diff --git a/tests/parity/functions_metrics_parity.rs b/tests/parity/functions_metrics_parity.rs index 56aa699a0..bede07559 100644 --- a/tests/parity/functions_metrics_parity.rs +++ b/tests/parity/functions_metrics_parity.rs @@ -152,8 +152,8 @@ fn every_named_function_space_is_reported_by_functions_and_find() { // `bca find --type function` reaches a third copy of the // decision — the `"function"` arm of `parser::filters`, which - // applies the predicate with `Ancestors::unknown()` rather than - // the walk's known chain. Compared by start line rather than by + // `find` applies off its own ancestor chain rather than the + // metrics walk's. Compared by start line rather than by // name because `find` yields raw nodes: two functions sharing a // start line would be a grammar impossibility, and the sorted // multiset still catches a count mismatch. diff --git a/tests/parity/self_reference_operand_parity.rs b/tests/parity/self_reference_operand_parity.rs index 6760eb81e..4140a1864 100644 --- a/tests/parity/self_reference_operand_parity.rs +++ b/tests/parity/self_reference_operand_parity.rs @@ -24,13 +24,14 @@ //! language with no self-reference says so with `None` rather than //! falling through a wildcard. //! -//! The two *declarator* uses of the same keyword — C#'s -//! `public int this[int i]` and Java's `? super String` wildcard bound — -//! are deliberately absent from these fixtures: they are operators, so -//! a fixture containing one would put the keyword in both vocabularies -//! and fail the assertion below. They are pinned instead by -//! `csharp_indexer_declaration_keyword_is_not_a_self_reference` and -//! `java_wildcard_super_bound_stays_an_operator` in +//! The *declarator* uses of the same keywords — C#'s +//! `public int this[int i]`, and the `? super String` wildcard bound in +//! Java and Groovy — are deliberately absent from these fixtures: they +//! are operators, so a fixture containing one would put the keyword in +//! both vocabularies and fail the assertion below. They are pinned +//! instead by `csharp_indexer_declaration_keyword_is_not_a_self_reference`, +//! `java_wildcard_super_bound_stays_an_operator` and +//! `groovy_wildcard_super_bound_stays_an_operator` in //! `src/metrics/halstead.rs`. use big_code_analysis::{Ast, LANG, Source}; @@ -92,12 +93,15 @@ fn fixture(lang: LANG) -> Option<(&'static str, &'static str, &'static [&'static &["this", "base"], ), // Grammar accident, and the interesting one: `getter/groovy.rs` - // *does* list `Super` among its operators, but the grammar emits - // a plain `identifier` for both `this` and `super` in receiver - // position — verified by dump for `super(1)`, `super.h()`, - // `A.super.h()` and `super::h` — so that arm is dead at the - // current pin and Groovy is an operand language in fact. This - // row is what notices if a bump ever wakes the arm up. + // lists `Super` among its operators with no parent gate, but the + // pinned grammar emits `Groovy::Super` only as a `wildcard` bound + // (`? super T`, a declarator use kept an operator as in Java and + // left out of this fixture). In receiver position `this` and + // `super` are a plain `identifier` — verified by dump for + // `super(1)`, `super.h()`, `A.super.h()` and `super::h` — so the + // arm never sees a reference, and Groovy is an operand language + // in fact. This row is what notices if a bump ever routes a + // reference to that kind (#1419). LANG::Groovy => ( "class A extends B {\n def f() { return this.x }\n \ def g() { return super.h() }\n}\n", diff --git a/tests/vcs/vcs_cache.rs b/tests/vcs/vcs_cache.rs index 77a897b2c..fd882cb8b 100644 --- a/tests/vcs/vcs_cache.rs +++ b/tests/vcs/vcs_cache.rs @@ -162,10 +162,14 @@ const FOLD_GRACE_INTO_ADA: &str = "Ada Grace u32 { index - .iter() - .find(|(candidate, _)| candidate.to_string_lossy() == path) - .map(|(_, stats)| stats.authors_long) - .expect("file is ranked") + .get(Path::new(path)) + .unwrap_or_else(|| { + panic!( + "expected {path} to be ranked; index has {} files", + index.len() + ) + }) + .authors_long } #[test] From 79f8acb95dfed675d9de38fb473458b14e43f2a6 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Fri, 11 Sep 2026 22:50:02 -0700 Subject: [PATCH 17/22] test(parity): pin the receiver self, not the declaration Two review findings on PR #1424. The self-reference parity test asserted membership in `Ops::operands`, which is a *deduplicated* vocabulary. Python's `def f(self)` parameter and Rust's `&self` spell the keyword in the declaration, so those two rows passed on the declaration alone: deleting `return self.x` left both green, and the grammar drift the rows exist to catch -- a receiver promoted to its own unclassified kind -- would not have failed them. N2 is the axis that can see the second occurrence. Each masked row now carries a receiver-stripped variant and asserts the receiver contributes exactly one operand occurrence. Verified by deleting the receiver from each fixture in turn: Python fails with N2 4 and 4, Rust with 6 and 6. Also documents a qualification the SARIF parity contract omitted. The CLI folds repeated path seeds together in `SeedSet::seen`, while `analyze_batch` returns one result per input and `collect_offenders_from_iter` renders every one, so `to_sarif(analyze_batch([a, a]))` emits each finding twice where `bca check -p a.py -p a.py` emits it once. Deduplicating in the binding would be wrong, because two distinct results may legitimately share a name, so the contract now states that entry-for-entry parity assumes a unique file set. Recorded in all three places that claim the parity: the `sarif.rs` module doc, the `to_sarif` stub, and the book. --- big-code-analysis-book/src/python/sarif.md | 10 ++++ .../python/big_code_analysis/_native.pyi | 10 ++++ big-code-analysis-py/src/sarif.rs | 19 ++++++ tests/parity/self_reference_operand_parity.rs | 60 ++++++++++++++++++- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/big-code-analysis-book/src/python/sarif.md b/big-code-analysis-book/src/python/sarif.md index e70d15f1d..a9c5f46b2 100644 --- a/big-code-analysis-book/src/python/sarif.md +++ b/big-code-analysis-book/src/python/sarif.md @@ -21,6 +21,16 @@ values, so it applies none of the in-source default (each marked space keeps its `suppressed` key, for a caller that wants to filter), no baseline, and no `[check] exclude` globs. +"The same files" means a **unique file set**. The CLI folds repeated path +seeds together, so `bca check -p a.py -p a.py` analyses `a.py` once and +emits one finding per breach. `analyze_batch` instead returns one result +per input, and `to_sarif` renders every result it is handed, so +`to_sarif(analyze_batch([a, a]), ...)` emits each finding twice. +Deduplicating in the binding would be wrong — two distinct results may +legitimately share a name, since `analyze_source` takes the caller's — +so hand `to_sarif` a unique file set when comparing the two documents +positionally. + Examples on this page import the package as `bca` (`import big_code_analysis as bca`). A bare `bca` in a shell command is the CLI binary. diff --git a/big-code-analysis-py/python/big_code_analysis/_native.pyi b/big-code-analysis-py/python/big_code_analysis/_native.pyi index f16af325e..5838e5621 100644 --- a/big-code-analysis-py/python/big_code_analysis/_native.pyi +++ b/big-code-analysis-py/python/big_code_analysis/_native.pyi @@ -1015,6 +1015,16 @@ def to_sarif( space keeps its ``suppressed`` key), no baseline and no ``[check] exclude`` globs. + That positional comparison also assumes a **unique file set**. The + CLI folds repeated path seeds together, so + ``bca check -p a.py -p a.py`` analyses ``a.py`` once and emits one + finding per breach, while ``analyze_batch`` returns one result per + input and ``to_sarif`` renders every result it is handed — so + ``to_sarif(analyze_batch([a, a]), ...)`` emits each finding twice. + Deduplicating here would be wrong, because two distinct results may + legitimately share a name (:func:`analyze_source` takes the caller's + name), so pass a unique file set when diffing the two documents. + Raises ------ TypeError diff --git a/big-code-analysis-py/src/sarif.rs b/big-code-analysis-py/src/sarif.rs index b52d623db..8b709b5da 100644 --- a/big-code-analysis-py/src/sarif.rs +++ b/big-code-analysis-py/src/sarif.rs @@ -101,6 +101,25 @@ //! in-source suppression markers `bca check` honours by default (a marked //! space keeps its `suppressed` key for a caller that wants to filter), //! no baseline, and no `[check] exclude` globs. +//! +//! Positional parity also assumes a **unique file set**. The CLI folds +//! repeated path seeds together in `SeedSet::seen` +//! (`big-code-analysis-cli/src/walk.rs`), so +//! `bca check -p a.py -p a.py` analyses `a.py` once and emits one +//! finding per breach. This binding does not: `analyze_batch` +//! deliberately returns one result per input, and +//! [`collect_offenders_from_iter`] appends every result it is handed, so +//! `to_sarif(analyze_batch([a, a]), …)` emits each finding twice. +//! +//! That asymmetry is deliberate rather than a gap to close here. +//! Deduplicating by `name` would be wrong: two results may legitimately +//! carry the same name — [`analyze_source`] takes the caller's name, and +//! nothing stops two snippets sharing one — so the binding cannot tell a +//! repeated path from two distinct analyses of the same name. Hand +//! `to_sarif` a unique file set when comparing documents entry for +//! entry. +//! +//! [`analyze_source`]: crate::analysis use std::path::{Path, PathBuf}; diff --git a/tests/parity/self_reference_operand_parity.rs b/tests/parity/self_reference_operand_parity.rs index 4140a1864..36ed2ee2c 100644 --- a/tests/parity/self_reference_operand_parity.rs +++ b/tests/parity/self_reference_operand_parity.rs @@ -34,7 +34,49 @@ //! `groovy_wildcard_super_bound_stays_an_operator` in //! `src/metrics/halstead.rs`. -use big_code_analysis::{Ast, LANG, Source}; +use big_code_analysis::{Ast, LANG, MetricsOptions, Source, analyze}; + +/// The same fixture as [`fixture`], with the *receiver* occurrence of +/// the keyword deleted and nothing else changed — or `None` for a row +/// whose fixture already spells the keyword only as a receiver. +/// +/// Two rows need this because their language spells the keyword in the +/// *declaration* as well: Python's `def f(self)` and Rust's `&self`. +/// The vocabulary in [`Ops::operands`] is deduplicated, so in those two +/// the declaration alone satisfies "`self` is an operand" and the +/// receiver assertion is vacuous — deleting `return self.x` leaves the +/// row green. Measured: Python's `n2` is 4 either way, Rust's `self` +/// survives in the vocabulary from `&self`. +/// +/// `N2` is the axis that can see the difference, so the test asserts +/// the receiver contributes exactly one operand *occurrence*. If a +/// grammar bump gave the receiver a dedicated kind that no arm +/// classifies, `N2` would not drop and the delta assertion fails — +/// which is the drift these rows exist to catch. +/// +/// [`Ops::operands`]: big_code_analysis::Ops::operands +fn receiver_stripped(lang: LANG) -> Option<&'static str> { + match lang { + // `return self.x` -> `return x`. Parses clean; N2 5 -> 4. + LANG::Python => Some("class A:\n def f(self):\n return x\n"), + // `self.x` -> `x`. Parses clean (metrics never type-check); + // N2 7 -> 6. + LANG::Rust => Some("struct A { x: i32 }\nimpl A { fn f(&self) -> i32 { x } }\n"), + _ => None, + } +} + +/// Total operand occurrences (`N2`) for `source` under `lang`. +fn total_operands(lang: LANG, source: &str, name: &str) -> u64 { + analyze( + Source::new(lang, source.as_bytes()).with_name(Some(name.to_owned())), + MetricsOptions::default(), + ) + .unwrap_or_else(|e| panic!("{lang:?}: analyze failed: {e}")) + .metrics + .halstead + .total_operands() +} /// Returns `(source, extension, keywords)` for a language that spells a /// self- or super-reference, or `None` for one that does not. @@ -189,7 +231,7 @@ fn every_language_bills_a_self_reference_as_an_operand() { checked += 1; let name = format!("parity.{ext}"); - let ops = Ast::parse(Source::new(lang, source.as_bytes()).with_name(Some(name))) + let ops = Ast::parse(Source::new(lang, source.as_bytes()).with_name(Some(name.clone()))) .unwrap_or_else(|e| panic!("{lang:?}: parse failed: {e}")) .ops() .unwrap_or_else(|e| panic!("{lang:?}: ops failed: {e}")); @@ -209,6 +251,20 @@ fn every_language_bills_a_self_reference_as_an_operand() { ops.operators, ); } + + // Where the declaration spells the keyword too, the assertions + // above pass on the declaration alone and say nothing about the + // receiver. Pin the receiver by its occurrence count instead. + if let Some(stripped) = receiver_stripped(lang) { + let with = total_operands(lang, source, &name); + let without = total_operands(lang, stripped, &name); + assert_eq!( + with, + without + 1, + "{lang:?}: the receiver must contribute exactly one operand occurrence; \ + N2 was {with} with it and {without} without", + ); + } } // Every language is feature-gated, so a build enabling only From 9dadfcc97f6d7559a9238c4fe9151a48d6fb53d0 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 08:24:06 -0700 Subject: [PATCH 18/22] fix(ast): key Tcl braced-word roles on argument position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(ast): key Tcl braced-word roles on argument position `is_value_braced_word` decides "script or value" by command name alone, which is all Halstead's `{}` operator needs. Six of the eight names it recognises have a mixed signature, so reusing that answer for `Checker::is_string_with_code` and `Alterator::keeps_children` made a value a script: `time {set x 1} {5}` lost the count, `after {100}` the delay, `switch {foo} {…}` the subject, `uplevel {1} {…}` the level and `namespace eval {foo bar} {…}` the namespace name. Each was a string and a flat dump leaf before the braced-word split. `is_braced_literal_slot` now reads a per-command slot table instead of answering `on`/`trap` and `namespace` ad hoc. Each entry pairs a Tcl 8.6 signature with which of its arguments the interpreter evaluates, so the whole policy is one table: `after` all but the first, `time` argument 0, `on`/`trap` the last, `uplevel` all but a leading level specifier, `switch` the arm bodies or the braced arm list, `namespace eval` all but the name, `namespace inscope` the second, `eval` / `for` / `namespace code` everything. `switch` also gets its subject scan: the first argument that is neither a leading option nor a `-matchvar` / `-indexvar` operand, with `--` ending the options. The scan stops at the first non-option word and a braced word never is one, so its length is the leading-option count rather than the argument count — the index itself still comes from the `O(log n)` cursor lookup, so a wide one-line arm list stays linear. The rule moves to `lang_helpers::tcl_family`, beside the two dialects' kind tables, for the reason that module already gives: three classifiers consult it and must agree on the same bytes. Halstead is unchanged by design. `braced_word_op_type` still asks `is_value_braced_word` alone, so a value slot's braces keep billing a `{}` operator; moving that boundary changes `bca metrics` for Tcl and wants its own measured change. test(ast): cover the Tcl argument-slot fallbacks Eleven of the twelve early returns the slot rule added were uncovered. Four turn out to be reachable from ordinary input and now have fixtures; six are unreachable from a walk but are the contract of a `pub` trait method or a `pub(crate)` helper, and now have direct tests asserting the documented answer; one pair collapsed into a single exit. Reachable, and asserted through the existing exact-vector verdict harness in both dialects where both can reach them: - `namespace $sub {…}` — a subcommand that is not a plain word resolves no layout at all. - `switch -matchvar {m} -indexvar {i}` — both options consume the word after them, so the subject scan runs off the end. Two pairs, because with one the invented subject 0 gives the same answers. - `switch bar {p} {puts c} {q} {puts d}` — a bare-word subject, the one spelling the braced and `$v` fixtures miss. Two arms, for the same reason. Unreachable from a walk, tested against the method's own contract: - the two ancestor-chain exits of `is_braced_literal_slot`, asserted over every node at depth 0 and 1 of a real tree — a value slot is an argument, so nothing that shallow can be one. - `is_switch_arm_body` on an argument-less command, paired with its positive answer on a real arm list. - the two placement guards in `fills_script_slot`, using a word nested inside a sibling slot rather than one from another command: a word outside the list's byte range makes the cursor answer `None` and never reaches the id check. `reads_as_uplevel_level` now takes one exit for text that is not a readable `{…}` instead of two. Unreadable bytes and unbraced text are both "not a level" and no caller can tell them apart, so the split bought nothing. The remaining uncovered line is the third-owner arm of `is_braced_literal_slot`. Both pinned grammars give a `word_list` exactly two owners, verified against their `node-types.json`, so no input reaches it; a new assertion walks both dialects and fails if a third owner ever appears, which is the event the arm exists for. Squashed from 15bb9460, ae1b8eda. --- big-code-analysis-ast/src/checker.rs | 209 +++++++ big-code-analysis-ast/src/getter.rs | 403 ++++++++------ big-code-analysis-ast/src/lang_helpers.rs | 4 + .../src/lang_helpers/tcl_family.rs | 513 ++++++++++++++++++ tests/grammars/alterator_string_flattening.rs | 18 + 5 files changed, 990 insertions(+), 157 deletions(-) create mode 100644 big-code-analysis-ast/src/lang_helpers/tcl_family.rs diff --git a/big-code-analysis-ast/src/checker.rs b/big-code-analysis-ast/src/checker.rs index fe4e4d23c..4f2f5af53 100644 --- a/big-code-analysis-ast/src/checker.rs +++ b/big-code-analysis-ast/src/checker.rs @@ -2818,4 +2818,213 @@ mod tests { "neither tcl nor irules is enabled; this test asserted nothing" ); } + + /// A command whose signature mixes the two roles keeps its *value* + /// arguments string literals, one rule per row of + /// `lang_helpers::tcl_family::SCRIPT_TAKING_COMMANDS` (#1381 review). + /// + /// `is_value_braced_word` answers by command name alone, so before + /// the slot table every braced argument of a script-taking command + /// was a script: `time {set x 1} {5}` lost the count, `after {100}` + /// the delay, `switch {foo} {…}` the subject, and + /// `namespace eval {foo bar} {…}` the namespace name. + /// + /// Every line carries a value *and* a script, so each rule is pinned + /// from both sides: a rule that withdrew the whole construct fails on + /// the value, and one that rescued it whole fails on the script. + /// Flipping which index a rule calls the value moves a row from one + /// list to the other, so neither list can be satisfied by a constant + /// or by an inverted polarity. + /// + /// The two `uplevel` lines are the level gate's own pair, the last + /// Tcl line is the `is_switch_arm` guard's — an arm whose *pattern* + /// is spelled `after` builds an `after` command around what are + /// really arm bodies, whose first would otherwise become the delay — + /// and the `-matchvar` line pins the subject scan's option arity and + /// its `--` terminator. iRules has no `switch` rows because its + /// grammar models the construct and rejects a braced subject + /// outright (the word lands under an `ERROR`, where the walk helper + /// refuses to go). + #[test] + #[cfg(any(feature = "tcl", feature = "irules"))] + fn tcl_family_mixed_signature_value_arguments_stay_strings() { + let mut ran = 0; + #[cfg(feature = "tcl")] + { + ran += 1; + let (kept, withdrawn, _) = braced_word_string_verdicts::( + "tcl", + b"after {100} {puts a}\n\ + time {puts b} {5}\n\ + uplevel {1} {puts c}\n\ + uplevel {puts d}\n\ + switch {sub} {p {puts e}}\n\ + switch -exact -- {sub2} {q {puts f}}\n\ + switch -- {sub4} {t} {puts m}\n\ + switch -nocase -matchvar {mv} -regexp {sub3} {r {puts g}}\n\ + switch $v {^s} {puts h}\n\ + namespace eval {my ns} {puts i}\n\ + namespace inscope {my ns2} {puts j}\n\ + namespace code {puts k}\n\ + switch $w { after {puts x} time {puts y} }\n", + Tcl::BracedWord as u16, + ); + assert_eq!( + kept, + [ + "{100}", "{5}", "{1}", "{sub}", "{sub2}", "{sub4}", "{t}", "{mv}", "{sub3}", + "{^s}", "{my ns}", "{my ns2}", + ], + "tcl: an `after` delay, a `time` count, an `uplevel` level, a \ + `switch` subject reached past three option spellings, a \ + `-matchvar` operand, two flat-form patterns and the two \ + `namespace` names are literals" + ); + assert_eq!( + withdrawn, + [ + "{puts a}", + "{puts b}", + "{puts c}", + "{puts d}", + "{p {puts e}}", + "{puts e}", + "{q {puts f}}", + "{puts f}", + "{puts m}", + "{r {puts g}}", + "{puts g}", + "{puts h}", + "{puts i}", + "{puts j}", + "{puts k}", + "{ after {puts x} time {puts y} }", + "{puts x}", + "{puts y}", + ], + "tcl: every arm list, arm body and evaluated argument is a \ + script — including the sole argument of the second \ + `uplevel`, which does not read as a level, and the two \ + bodies of an arm whose pattern is spelled `after`" + ); + } + #[cfg(feature = "irules")] + { + ran += 1; + let (kept, withdrawn, _) = braced_word_string_verdicts::( + "irules", + b"after {100} {log a}\n\ + time {log b} {5}\n\ + uplevel {1} {log c}\n\ + uplevel {log d}\n\ + namespace eval {my ns} {log e}\n\ + namespace inscope {my ns2} {log f}\n\ + namespace code {log g}\n", + Irules::BracedWord as u16, + ); + assert_eq!( + kept, + ["{100}", "{5}", "{1}", "{my ns}", "{my ns2}"], + "irules: the same delay, count, level and namespace names are \ + literals — the two dialects share one predicate and must not \ + drift" + ); + assert_eq!( + withdrawn, + [ + "{log a}", "{log b}", "{log c}", "{log d}", "{log e}", "{log f}", "{log g}", + ], + "irules: every evaluated argument is a script" + ); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); + } + + /// A mixed-signature command whose *layout* cannot be resolved keeps + /// the construct-wide script answer, and a resolvable one beside it + /// still yields its value. + /// + /// Three shapes reach a fallback in + /// `lang_helpers::tcl_family` that no other fixture does: + /// + /// - `namespace $sub {…}` — the subcommand is a substitution, not a + /// plain word, so no `ScriptSlots` row can be looked up at all. + /// - `switch -matchvar {m} -indexvar {i}` — both options consume the + /// word after them, so the scan runs off the end without finding a + /// subject. *Two* pairs, because one leaves the scan at a list of + /// two whose second word an invented subject 0 would call the arm + /// list — the same answer, so a single pair proves nothing. + /// - `switch bar {p} {puts c} {q} {puts d}` — the subject is a bare + /// `simple_word`, the one spelling neither the braced nor the `$v` + /// fixtures above exercise. *Two* arms, for the same reason: with + /// one, the trailing argument is the braced arm list either way, so + /// the subject's index does not change any answer. + /// + /// Pairing each with a resolvable line is what makes the claim + /// two-sided: the exact vectors below say `{my ns}`, `{p}` and `{q}` + /// are values *and* that `{puts b}`, `{m}` and `{i}` are not, so a + /// fallback answering "value" instead moves rows between the lists. + /// + /// The `switch` rows are Tcl-only by construction. The iRules grammar + /// models `switch`, so a well-formed one is never the generic + /// `command` this rule reads, and a malformed one lands under an + /// `ERROR` that `is_braced_literal_slot` rejects before the slot + /// table — the same asymmetry `SCRIPT_TAKING_COMMANDS` records for + /// its `for` and `switch` rows. + #[test] + #[cfg(any(feature = "tcl", feature = "irules"))] + fn tcl_family_unresolvable_slot_layouts_keep_the_script_answer() { + let mut ran = 0; + #[cfg(feature = "tcl")] + { + ran += 1; + let (kept, withdrawn, _) = braced_word_string_verdicts::( + "tcl", + b"namespace eval {my ns} {puts a}\n\ + namespace $sub {puts b}\n\ + switch -matchvar {m} -indexvar {i}\n\ + switch bar {p} {puts c} {q} {puts d}\n", + Tcl::BracedWord as u16, + ); + assert_eq!( + kept, + ["{my ns}", "{p}", "{q}"], + "tcl: a resolvable `namespace eval` name and both flat-form \ + patterns behind a bare-word subject are literals" + ); + assert_eq!( + withdrawn, + ["{puts a}", "{puts b}", "{m}", "{i}", "{puts c}", "{puts d}"], + "tcl: an unresolvable `namespace` subcommand and a `switch` \ + with no subject keep every braced argument a script" + ); + } + #[cfg(feature = "irules")] + { + ran += 1; + let (kept, withdrawn, _) = braced_word_string_verdicts::( + "irules", + b"namespace eval {my ns} {log a}\n\ + namespace $sub {log b}\n", + Irules::BracedWord as u16, + ); + assert_eq!( + kept, + ["{my ns}"], + "irules: the resolvable `namespace eval` name is a literal" + ); + assert_eq!( + withdrawn, + ["{log a}", "{log b}"], + "irules: the unresolvable subcommand keeps its argument a script" + ); + } + assert!( + ran > 0, + "neither tcl nor irules is enabled; this test asserted nothing" + ); + } } diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index c07006df8..4a0eb01e8 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -11,6 +11,9 @@ use crate::token_role::TokenRole; +use crate::lang_helpers::tcl_family::{ + ArgumentRoles, Dialect, SWITCH_COMMAND, fills_script_slot, namespace_script_slots, script_slots, +}; use crate::space_kind::SpaceKind; use crate::traits::Search; @@ -42,7 +45,7 @@ use crate::*; /// asymmetry is deliberate: guarding is free here because `Option` is /// already part of this signature's contract (#1059). #[inline] -fn node_text<'a>(code: &'a [u8], node: &Node) -> Option<&'a str> { +pub(crate) fn node_text<'a>(code: &'a [u8], node: &Node) -> Option<&'a str> { code.get(node.start_byte()..node.end_byte()) .and_then(|bytes| std::str::from_utf8(bytes).ok()) } @@ -351,91 +354,6 @@ pub struct BracedWordKinds { pub open_brace: u16, } -/// The core Tcl-family commands that evaluate a braced argument as a -/// *script* and that neither dialect's grammar models with a node of its -/// own (#1318). -/// -/// A command the grammar *does* model — `proc`, `if`, `while`, -/// `foreach`, `catch`, `try`, `namespace`, iRules' `when`, `for`, -/// `switch` and `dict for` / `dict update` / `dict with` — needs no -/// entry: its body is a child of that construct's own node rather than -/// of a generic `command`, which -/// [`Getter::generic_argument_command`] already answers `None` for. -/// `for` and `switch` appear here because the *Tcl* grammar models -/// neither (#467, #1264); the iRules grammar models both, so those two -/// rows are live for one dialect and inert for the other. -/// -/// Each entry is a command whose documented syntax puts a script in a -/// braced argument: -/// -/// | command | syntax | -/// | --- | --- | -/// | `after` | `after ms script` | -/// | `eval` | `eval arg ?arg …?` | -/// | `for` | `for start test next body` — all four are evaluated | -/// | `on` | `on code varList script`, a `try` handler clause | -/// | `switch` | `switch ?options? string pattern body ?pattern body …?` | -/// | `time` | `time script ?count?` | -/// | `trap` | `trap pattern varList script`, a `try` handler clause | -/// | `uplevel` | `uplevel ?level? arg ?arg …?` | -/// -/// `on` and `trap` are listed because the iRules grammar models -/// `on_handler` / `trap_handler` only *under* `try` (pinned by -/// `irules_try_handler_kinds_appear_only_under_try`), and the Tcl -/// grammar models neither a `trap` nor a second `on`; written outside -/// that shape they parse as generic commands. Their last argument is -/// the handler script. The pattern and variable list before it are -/// values, which only [`Getter::is_braced_literal_slot`] tells apart — -/// the `{}` operator this table decides still bills them as blocks. -/// -/// `lmap varname list body` is deliberately **not** listed even though -/// its body is a script: the list is per-command, not per-argument, so -/// listing it would bill `lmap i {1 2 3} {…}`'s *list* as a block — -/// the same spelling sensitivity this rule exists to remove, and worse -/// than the one occurrence its body gives up. `for` and `switch` have -/// no such argument (`for`'s four are all evaluated, `switch`'s braced -/// argument is the arm list). -/// -/// Subcommand-dispatched script takers (`dict for`, `interp eval`, -/// `trace add … {script}`) are absent for a related reason: the -/// leading word alone cannot tell `dict for` from `dict set`, and -/// admitting it would misclassify the far commoner value-taking -/// spellings. Tk callbacks (`bind`, `fileevent`, a `-command {…}` -/// option) are absent for the same reason as any user proc. -/// -/// This list is the whole of the heuristic, and it is knowingly -/// incomplete — see [`Getter::is_value_braced_word`] for what an -/// unlisted command defaults to and why. -const SCRIPT_TAKING_COMMANDS: [&str; 8] = [ - "after", - "eval", - "for", - "on", - SWITCH_COMMAND, - "time", - "trap", - "uplevel", -]; - -/// Named because two rules have to agree on it: `switch` takes a -/// script, and its *arm bodies* are scripts too even though the Tcl -/// grammar hangs them off a `command` named after the pattern -/// ([`Getter::is_switch_arm`]). Spelling it twice would let one drift. -const SWITCH_COMMAND: &str = "switch"; - -/// The `namespace` subcommands with a script argument: -/// `namespace eval ns arg ?arg …?`, `namespace inscope ns script ?arg …?` -/// and `namespace code script`. Every other subcommand takes values — -/// `export {pattern}`, `path {ns …}`, `ensemble create -map {dict}` — -/// which is what [`Getter::is_braced_literal_slot`] keys on. -const NAMESPACE_SCRIPT_SUBCOMMANDS: [&str; 3] = ["eval", "inscope", "code"]; - -/// The `try` handler clauses a grammar leaves as generic commands: -/// `on code varList script` and `trap pattern varList script`, whose -/// every argument but the last is a value -/// ([`Getter::is_braced_literal_slot`]). -const TRY_HANDLER_COMMANDS: [&str; 2] = ["on", "trap"]; - /// Per-language accessors that *name* and *classify* what a node is: /// the function or space name, the [`SpaceKind`] a node opens, and the /// [`TokenRole`] of a leaf. @@ -757,7 +675,8 @@ pub trait Getter { return false; }; if Self::command_leading_word(&command, code, kinds) - .is_some_and(|name| SCRIPT_TAKING_COMMANDS.contains(&name)) + .and_then(script_slots) + .is_some() { return false; } @@ -805,26 +724,29 @@ pub trait Getter { } /// Whether `word`, a braced word [`is_value_braced_word`] calls a - /// script, fills a slot whose documented syntax takes a *value*. Three - /// constructs hold both roles in one argument list: + /// script, fills a slot whose documented syntax takes a *value*. /// - /// | construct | value slots | script slot | - /// | --- | --- | --- | - /// | `proc name args body` | the `name` field | the body | - /// | `namespace sub ?arg …?` | any subcommand's but three | `eval`, `inscope`, `code` | - /// | `on code varList script`, `trap pattern varList script` | all but the last | the last | + /// [`is_value_braced_word`] answers by command name alone, which is + /// all Halstead's `{}` operator needs. Most of the names it recognises + /// have a mixed signature, though, so reusing that answer for the + /// string and dump classifiers made a *value* — a millisecond count, a + /// namespace name, a `switch` subject — a script (#1381 review). This + /// is the per-argument half: `proc` has one value slot the grammar + /// names as a field, and every other construct declares its layout in + /// one of the two tables in `lang_helpers::tcl_family`, which + /// `fills_script_slot` applies. /// - /// Each literal here was a string and a flat dump leaf before #1381, - /// and would otherwise have become a script under it: the dump - /// rendered `{my proc}` as a command named `my`, and - /// `namespace export {…}` and `namespace ensemble create -map {…}` + /// Each value here was a string and a flat dump leaf before #1381, and + /// would otherwise have become a script under it: the dump rendered + /// `{my proc}` as a command named `my` and `{100}` as one named `100`, + /// and `namespace export {…}` and `namespace ensemble create -map {…}` /// both occur in the Tcl 8.6 standard library. /// /// Two guards keep the construct-wide answer. A switch arm list parses /// as commands, so an arm whose *pattern* is spelled `proc`, - /// `namespace`, `on` or `trap` builds one of these shapes around what - /// are really arm bodies — [`is_switch_arm`] recognises it first. And - /// an owner holding a parse error has no argument positions worth + /// `namespace`, `after` or `switch` builds one of these shapes around + /// what are really arm bodies — [`is_switch_arm`] recognises it first. + /// And an owner holding a parse error has no argument positions worth /// trusting. The multi-line `try … trap` clause is out of reach /// entirely: the Tcl grammar leaves it inside an `ERROR` node, where /// no role signal survives. @@ -839,6 +761,13 @@ pub trait Getter { kinds: &BracedWordKinds, ) -> bool { let mut chain = ancestors.iter(word); + // A value slot is an *argument*, so it sits at least two levels + // below the root; neither grammar puts a braced word at the root + // or directly under it (a statement-position braced word is + // wrapped in a `command`, so `{a b}` alone on a line is still two + // deep). Both exits are therefore unreachable from a walk and are + // here because this is a `pub` trait method any node may be + // handed — `braced_slot_tests` calls it at both depths. let Some((parent, above_parent)) = chain.next() else { return false; }; @@ -856,64 +785,25 @@ pub trait Getter { { return false; } - if owner.kind_id() == kinds.namespace { - Self::namespace_subcommand_takes_values(&parent, code, kinds) + let dialect = Dialect { code, kinds }; + // At both pinned grammars a `word_list` has exactly two possible + // owners — `command`'s `arguments` field and `namespace`'s child + // — so the third arm is unreachable and stays as the answer a + // grammar that grew a third owner should get: no layout, hence + // the construct-wide script answer. + // `a_word_list_is_owned_only_by_a_command_or_a_namespace` fails + // when that stops being true, which is the event this arm exists + // for. + let roles = if owner.kind_id() == kinds.namespace { + namespace_script_slots(&parent, dialect).map(|slots| ArgumentRoles { slots, first: 1 }) + } else if owner.kind_id() == kinds.command { + Self::command_leading_word(&owner, code, kinds) + .and_then(script_slots) + .map(|slots| ArgumentRoles { slots, first: 0 }) } else { - Self::is_try_handler_value(word, &owner, code, kinds) - } - } - - /// Whether a `namespace` construct's `word_list` names a subcommand - /// whose arguments are values — anything but - /// `NAMESPACE_SCRIPT_SUBCOMMANDS`. A subcommand that is not a plain - /// word (`namespace $sub …`) is unresolvable and keeps the script - /// answer, as an unresolvable command name does. - #[must_use] - fn namespace_subcommand_takes_values( - word_list: &Node<'_>, - code: &[u8], - kinds: &BracedWordKinds, - ) -> bool { - let Some(subcommand) = word_list.child(0) else { - return false; - }; - if subcommand.kind_id() != kinds.simple_word { - return false; - } - let Some(subcommand) = node_text(code, &subcommand) else { - return false; - }; - !NAMESPACE_SCRIPT_SUBCOMMANDS.contains(&subcommand) - } - - /// Whether `word` is an argument of a generic `on` / `trap` command - /// other than its last, the handler script. The index comes from an - /// `O(log n)` cursor lookup, as in [`is_switch_arm_body`], not a - /// sibling scan. - /// - /// [`is_switch_arm_body`]: Self::is_switch_arm_body - #[must_use] - fn is_try_handler_value( - word: &Node<'_>, - command: &Node<'_>, - code: &[u8], - kinds: &BracedWordKinds, - ) -> bool { - let is_handler = command.kind_id() == kinds.command - && matches!( - Self::command_leading_word(command, code, kinds), - Some(name) if TRY_HANDLER_COMMANDS.contains(&name) - ); - if !is_handler { - return false; - } - let Some(arguments) = command.child_by_field_name("arguments") else { - return false; + None }; - let mut cursor = arguments.cursor(); - let index = cursor.goto_first_child_for_byte(word.start_byte()); - cursor.node().id() == word.id() - && matches!(index, Some(index) if index + 1 < arguments.child_count()) + roles.is_some_and(|roles| !fills_script_slot(word, &parent, roles, dialect)) } /// Whether `word`, an argument of a `switch` arm command @@ -956,6 +846,12 @@ pub trait Getter { /// [`Cursor::goto_first_child_for_byte`]: crate::node::Cursor::goto_first_child_for_byte #[must_use] fn is_switch_arm_body(word: &Node<'_>, command: &Node<'_>) -> bool { + // `is_value_braced_word` asks only for a word it has already + // placed inside this command's `arguments`, so the field is + // always there on that path. It is not always there on the + // method's own contract — an argument-less `switch` is a + // `command` with a `name` and nothing else — and such a command + // has no body position at all. let Some(arguments) = command.child_by_field_name("arguments") else { return false; }; @@ -1264,3 +1160,196 @@ mod ancestor_tests { ); } } + +/// The `is_braced_literal_slot` / `is_switch_arm_body` guards that no +/// Tcl-family walk can reach, and the grammar shape each rests on. +#[cfg(test)] +#[cfg(any(feature = "tcl", feature = "irules"))] +mod braced_slot_tests { + use super::{BracedWordKinds, Getter}; + use crate::Tcl; + use crate::node::{Ancestors, Node}; + use crate::test_support::for_each_node_with_chain; + use crate::traits::LanguageInfo; + + /// No node shallower than an argument can fill a value slot. + /// + /// A value slot is an argument, and both grammars hang an argument at + /// least two levels below the root — under a `word_list` under a + /// `command`, or under a construct's own node. A statement-position + /// braced word is no exception: `{a b}` alone on a line is a `command` + /// wrapping the braced word, never the braced word itself. So the root + /// and its direct children are exactly the depths + /// `is_braced_literal_slot`'s two chain exits answer for, and no walk + /// reaches either. + /// + /// They are still the method's contract — it is `pub`, takes any node, + /// and must not read a slot into something with no argument list above + /// it. Flipping either exit to `true` makes this fail. + fn assert_no_shallow_value_slot( + label: &str, + code: &[u8], + kinds: &BracedWordKinds, + ) { + let (mut roots, mut children) = (0, 0); + for_each_node_with_chain::(code, |node: &Node<'_>, chain| { + if chain.len() > 1 { + return; + } + for ancestors in [Ancestors::known(chain), Ancestors::unknown()] { + assert!( + !L::is_braced_literal_slot(node, code, ancestors, kinds), + "{label}: a {} at depth {} filled a value slot", + node.kind(), + chain.len() + ); + } + if chain.is_empty() { + roots += 1; + } else { + children += 1; + } + }); + assert_eq!(roots, 1, "{label}: exactly one root"); + assert!( + children > 1, + "{label}: the fixture must put several nodes directly under the \ + root, or the depth-1 exit goes unexercised" + ); + } + + #[test] + #[cfg(feature = "tcl")] + fn tcl_no_node_above_an_argument_fills_a_value_slot() { + assert_no_shallow_value_slot::( + "tcl", + b"{a b}\nproc {my proc} {} {}\nnamespace export {c d}\n", + &crate::lang_helpers::tcl::BRACED_WORD_KINDS, + ); + } + + #[test] + #[cfg(feature = "irules")] + fn irules_no_node_above_an_argument_fills_a_value_slot() { + assert_no_shallow_value_slot::( + "irules", + b"{a b}\nproc {my proc} {} {}\nnamespace export {c d}\n", + &crate::lang_helpers::irules::BRACED_WORD_KINDS, + ); + } + + /// A command with no argument list has no `switch`-arm body position. + /// + /// `is_value_braced_word` only ever asks about a word it has already + /// placed inside the command's `arguments`, so that field is always + /// present on the walk's path. It is absent for an argument-less + /// command — `switch` on a line by itself is a `command` carrying only + /// its `name` — and the method has to answer for one, because a + /// bodiless command has no even-indexed slot to be in. + /// + /// The second half is the same method's positive answer on a real arm + /// list, so the first cannot be satisfied by a constant. + #[test] + #[cfg(feature = "tcl")] + fn a_command_without_arguments_has_no_switch_arm_body() { + use crate::node::Tree; + + let code = b"switch\nswitch $v { p {puts a} }\n"; + let tree = Tree::new::(code); + let root = tree.get_root(); + let bare = root.child(0).expect("the bare `switch` command"); + let name = bare + .child_by_field_name("name") + .expect("even a bare command names itself"); + assert!( + bare.child_by_field_name("arguments").is_none(), + "the fixture's first command must carry no argument list" + ); + assert!( + !::is_switch_arm_body(&name, &bare), + "a command with no argument list has no body position" + ); + + let arm = find_arm_named_p(&root, code).expect("the arm command named `p`"); + let body = arm + .child_by_field_name("arguments") + .and_then(|args| args.child(0)) + .expect("the arm's body"); + assert!( + ::is_switch_arm_body(&body, &arm), + "the arm's sole argument is its body, or the assertion above is \ + satisfied by a constant" + ); + } + + /// The `command` a Tcl `switch` arm list spells its first pattern as. + #[cfg(feature = "tcl")] + fn find_arm_named_p<'a>(node: &Node<'a>, code: &[u8]) -> Option> { + let named_p = node.kind_id() == Tcl::Command as u16 + && node + .child_by_field_name("name") + .is_some_and(|name| name.utf8_text(code) == Some("p")); + if named_p { + return Some(*node); + } + node.children() + .find_map(|child| find_arm_named_p(&child, code)) + } + + /// Only a `command` and a `namespace` own a `word_list`, which is what + /// makes `is_braced_literal_slot`'s third owner arm unreachable. + /// + /// Checked against both pinned grammars' `node-types.json` when that + /// arm was written; this is the executable half, so a grammar bump + /// handing a `word_list` to a third construct fails here rather than + /// silently routing it to the no-layout answer. + fn assert_word_list_owners(label: &str, code: &[u8], kinds: &BracedWordKinds) { + let mut seen = 0; + for_each_node_with_chain::(code, |node: &Node<'_>, chain| { + if node.kind_id() != kinds.word_list { + return; + } + let owner = chain.last().expect("a word_list is never the root"); + assert!( + [kinds.command, kinds.namespace].contains(&owner.kind_id()), + "{label}: a word_list owned by {}", + owner.kind() + ); + seen += 1; + }); + assert!(seen > 2, "{label}: fixture reached only {seen} word_lists"); + } + + /// Both owners plus the modelled constructs either grammar spells, so + /// a new owner among them is what the assertion catches. + const WORD_LIST_OWNER_FIXTURE: &[u8] = b"proc p {x} { puts $x }\n\ + namespace eval ns { puts hi }\n\ + namespace export {a b}\n\ + if {$x} { puts a } else { puts b }\n\ + while {$x} { puts a }\n\ + foreach i {1 2} { puts $i }\n\ + catch { puts a } msg\n\ + eval {puts hi}\n\ + switch $v { p {puts a} }\n\ + lappend l {c d}\n"; + + #[test] + #[cfg(feature = "tcl")] + fn tcl_a_word_list_is_owned_only_by_a_command_or_a_namespace() { + assert_word_list_owners::( + "tcl", + WORD_LIST_OWNER_FIXTURE, + &crate::lang_helpers::tcl::BRACED_WORD_KINDS, + ); + } + + #[test] + #[cfg(feature = "irules")] + fn irules_a_word_list_is_owned_only_by_a_command_or_a_namespace() { + assert_word_list_owners::( + "irules", + WORD_LIST_OWNER_FIXTURE, + &crate::lang_helpers::irules::BRACED_WORD_KINDS, + ); + } +} diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index e16019936..83e9f3001 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -19,3 +19,7 @@ pub mod elixir; pub(crate) mod irules; pub mod python; pub mod tcl; +// Crate-private for the same reason as `irules` above: the braced-word +// slot rule is read by this crate's three classifiers and by nothing +// outside it. +pub(crate) mod tcl_family; diff --git a/big-code-analysis-ast/src/lang_helpers/tcl_family.rs b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs new file mode 100644 index 000000000..e53785321 --- /dev/null +++ b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs @@ -0,0 +1,513 @@ +//! Tcl and iRules: which braced arguments of a command hold a *script*. +//! +//! Neither dialect's grammar marks the difference. `{a b}` and +//! `{puts hi}` parse to one kind, so the role is recoverable only from +//! the enclosing command's leading word and the argument's position in +//! its list (grammar-dispatch §9). This module is that lookup: two +//! tables of documented Tcl 8.6 signatures, and the position arithmetic +//! [`Getter::is_value_braced_word`] and [`Getter::is_braced_literal_slot`] +//! read them through. +//! +//! It lives here rather than in `getter.rs` for the reason the module +//! above gives: the classifiers consult it, and all three of them — +//! `Getter::get_op_type_with_code`, `Checker::is_string_with_code` and +//! `Alterator::keeps_children` — must agree on the same bytes. +//! +//! [`Getter::is_value_braced_word`]: crate::getter::Getter::is_value_braced_word +//! [`Getter::is_braced_literal_slot`]: crate::getter::Getter::is_braced_literal_slot + +use crate::getter::{BracedWordKinds, node_text}; +use crate::node::Node; + +/// The core Tcl-family commands that evaluate a braced argument as a +/// *script* and that neither dialect's grammar models with a node of its +/// own (#1318). +/// +/// A command the grammar *does* model — `proc`, `if`, `while`, +/// `foreach`, `catch`, `try`, `namespace`, iRules' `when`, `for`, +/// `switch` and `dict for` / `dict update` / `dict with` — needs no +/// entry: its body is a child of that construct's own node rather than +/// of a generic `command`, which +/// `Getter::generic_argument_command` already answers `None` for. +/// `for` and `switch` appear here because the *Tcl* grammar models +/// neither (#467, #1264); the iRules grammar models both, so those two +/// rows are live for one dialect and inert for the other. +/// +/// Each entry is a command whose documented syntax puts a script in a +/// braced argument, paired with *which* of its arguments that is. Only +/// `eval` and `for` evaluate all of them; the other six mix the two +/// roles in one argument list, and the second column is what +/// `Getter::is_braced_literal_slot` reads to tell the halves apart +/// (#1381 review). Syntax and slot are both from the Tcl 8.6 manual +/// page named in the last column. +/// +/// | command | syntax | evaluated | page | +/// | --- | --- | --- | --- | +/// | `after` | `after ms ?script …?` | all but the first — argument 0 is a millisecond count or a `cancel` / `idle` / `info` subcommand word | `after(n)` | +/// | `eval` | `eval arg ?arg …?` | all | `eval(n)` | +/// | `for` | `for start test next body` | all four | `for(n)` | +/// | `on` | `on code varList script` | the last | `try(n)` | +/// | `switch` | `switch ?options? string pattern body ?pattern body …?` | the arm bodies, or the single braced arm list | `switch(n)` | +/// | `time` | `time script ?count?` | argument 0 only | `time(n)` | +/// | `trap` | `trap pattern varList script` | the last | `try(n)` | +/// | `uplevel` | `uplevel ?level? arg ?arg …?` | all but a leading level specifier | `uplevel(n)` | +/// +/// `on` and `trap` are listed because the iRules grammar models +/// `on_handler` / `trap_handler` only *under* `try` (pinned by +/// `irules_try_handler_kinds_appear_only_under_try`), and the Tcl +/// grammar models neither a `trap` nor a second `on`; written outside +/// that shape they parse as generic commands. Their last argument is +/// the handler script. The pattern and variable list before it are +/// values, which only `Getter::is_braced_literal_slot` tells apart — +/// the `{}` operator this table decides still bills them as blocks. +/// +/// `after cancel {…}` and `after info {…}` are the one place the slot +/// column knowingly over-reports: those arguments identify a *pending* +/// script by its text rather than supplying one to run. Calling them +/// scripts costs nothing a reader would notice — the text is real code +/// either way — and telling them apart needs the subcommand dispatch +/// the paragraph below rules out. +/// +/// `lmap varname list body` is deliberately **not** listed even though +/// its body is a script: the list is per-command, not per-argument, so +/// listing it would bill `lmap i {1 2 3} {…}`'s *list* as a block — +/// the same spelling sensitivity this rule exists to remove, and worse +/// than the one occurrence its body gives up. `for` and `switch` have +/// no such argument (`for`'s four are all evaluated, `switch`'s braced +/// argument is the arm list). +/// +/// Subcommand-dispatched script takers (`dict for`, `interp eval`, +/// `trace add … {script}`) are absent for a related reason: the +/// leading word alone cannot tell `dict for` from `dict set`, and +/// admitting it would misclassify the far commoner value-taking +/// spellings. Tk callbacks (`bind`, `fileevent`, a `-command {…}` +/// option) are absent for the same reason as any user proc. +/// +/// This list is the whole of the heuristic, and it is knowingly +/// incomplete — see `Getter::is_value_braced_word` for what an +/// unlisted command defaults to and why. +const SCRIPT_TAKING_COMMANDS: [(&str, ScriptSlots); 8] = [ + ("after", ScriptSlots::EveryButFirst), + ("eval", ScriptSlots::Every), + ("for", ScriptSlots::Every), + ("on", ScriptSlots::Last), + (SWITCH_COMMAND, ScriptSlots::SwitchArms), + ("time", ScriptSlots::Only(0)), + ("trap", ScriptSlots::Last), + ("uplevel", ScriptSlots::EveryButLeadingLevel), +]; + +/// Named because two rules have to agree on it: `switch` takes a +/// script, and its *arm bodies* are scripts too even though the Tcl +/// grammar hangs them off a `command` named after the pattern +/// (`Getter::is_switch_arm`). Spelling it twice would let one drift. +pub(crate) const SWITCH_COMMAND: &str = "switch"; + +/// The `namespace` subcommands with a script argument, and which of +/// their arguments it is (Tcl 8.6 `namespace(n)`): +/// `namespace eval ns ?arg …?` evaluates everything after the namespace +/// name, `namespace inscope ns script ?arg …?` evaluates the second +/// argument and appends the rest to it as list elements, and +/// `namespace code script` has no name argument at all. Every +/// subcommand *not* listed takes values throughout — `export {pattern}`, +/// `path {ns …}`, `ensemble create -map {dict}` — which is the +/// [`ScriptSlots::NoneOfThem`] default [`namespace_script_slots`] falls +/// back to. +const NAMESPACE_SCRIPT_SUBCOMMANDS: [(&str, ScriptSlots); 3] = [ + ("code", ScriptSlots::Every), + ("eval", ScriptSlots::EveryButFirst), + ("inscope", ScriptSlots::Only(1)), +]; + +/// The `switch` options that consume the word after them, so the +/// subject scan must step over it: `-matchvar varName` and +/// `-indexvar varName` (Tcl 8.6 `switch(n)`; both are `-regexp`-only). +/// The other four — `-exact`, `-glob`, `-regexp`, `-nocase` — are bare +/// flags and need no entry. +const SWITCH_OPTIONS_TAKING_A_WORD: [&str; 2] = ["-indexvar", "-matchvar"]; + +/// `switch`'s end-of-options marker: whatever follows it is the +/// subject, even a subject that itself begins with `-`. +const SWITCH_OPTION_TERMINATOR: &str = "--"; + +/// Which of a Tcl-family construct's arguments the interpreter +/// *evaluates*. Positions count from the construct's first argument, +/// so the subcommand of a `namespace` — which shares the one +/// `word_list` with the arguments — is not one of them. +/// +/// Every position a variant does not name holds a value: a millisecond +/// count, an iteration count, a namespace name, a `switch` subject or +/// pattern, a `try` handler's error code and variable list. The two +/// tables above pair one of these with each command whose signature +/// mixes the roles, and [`fills_script_slot`] applies it. +#[derive(Clone, Copy)] +pub(crate) enum ScriptSlots { + /// Every argument: `eval arg ?arg …?`, `for start test next body`, + /// `namespace code script`. + Every, + /// No argument. The default for a `namespace` subcommand the table + /// does not list, every one of which takes values throughout. + NoneOfThem, + /// Every argument but the first: `after ms ?script …?`, + /// `namespace eval ns ?arg …?`. + EveryButFirst, + /// Exactly one argument, by position: `time script ?count?` is + /// `Only(0)`, `namespace inscope ns script ?arg …?` is `Only(1)`. + Only(usize), + /// The last argument: `on code varList script` and + /// `trap pattern varList script`. + Last, + /// Every argument but a leading level specifier: + /// `uplevel ?level? arg ?arg …?`, where an argument 0 that does not + /// read as a level is the start of the script rather than a value + /// ([`reads_as_uplevel_level`]). + EveryButLeadingLevel, + /// The evaluated half of `switch ?options? string pattern body …` + /// ([`switch_arm_slot`]). + SwitchArms, +} + +/// One dialect's kind ids paired with the buffer the nodes under test +/// were parsed from. +/// +/// The two are a unit: `BracedWordKinds` names ids in *this* grammar and +/// `code` must be the exact source *this* node came from, the same-parse +/// precondition [`Getter`] documents. Threading them as one value keeps +/// every rule below at one context parameter and, as with the kind table +/// itself, stops a caller pairing a node with the wrong buffer. +/// +/// [`Getter`]: crate::getter::Getter +#[derive(Clone, Copy)] +pub(crate) struct Dialect<'a> { + /// The source `word` and its siblings were parsed from. + pub(crate) code: &'a [u8], + /// This grammar's braced-word kind ids. + pub(crate) kinds: &'a BracedWordKinds, +} + +/// The [`ScriptSlots`] of a generic command named `command`, or `None` +/// when no core command of that name evaluates a braced argument. +pub(crate) fn script_slots(command: &str) -> Option { + SCRIPT_TAKING_COMMANDS + .iter() + .find_map(|&(name, slots)| (name == command).then_some(slots)) +} + +/// The [`ScriptSlots`] of the subcommand a `namespace` construct's +/// `word_list` names — its first entry, before the arguments proper. +/// +/// `None` for a subcommand that is not a plain word +/// (`namespace $sub {…}`), which is unresolvable and keeps the script +/// answer, exactly as an unresolvable command name does. +pub(crate) fn namespace_script_slots( + word_list: &Node<'_>, + dialect: Dialect<'_>, +) -> Option { + let subcommand = word_list.child(0)?; + if subcommand.kind_id() != dialect.kinds.simple_word { + return None; + } + let subcommand = node_text(dialect.code, &subcommand)?; + Some( + NAMESPACE_SCRIPT_SUBCOMMANDS + .iter() + .find_map(|&(name, slots)| (name == subcommand).then_some(slots)) + .unwrap_or(ScriptSlots::NoneOfThem), + ) +} + +/// One construct's argument layout: which slots hold a script, and +/// where in the shared `word_list` its arguments begin. The two always +/// travel together and are both plain scalars, so they ride in one +/// value rather than as two same-shaped parameters. +#[derive(Clone, Copy)] +pub(crate) struct ArgumentRoles { + /// The evaluated slots, counted from `first`. + pub(crate) slots: ScriptSlots, + /// The index the construct's arguments start at — `1` for a + /// `namespace`, whose subcommand takes the slot before them, and + /// `0` for a generic command, which keeps its name in a field of + /// its own. + pub(crate) first: usize, +} + +/// Whether the braced `word` fills a slot of `arguments` that `roles` +/// names as evaluated. +/// +/// A word whose index cannot be resolved keeps the construct-wide +/// script answer, which is what every caller had before the slot table +/// existed. +pub(crate) fn fills_script_slot( + word: &Node<'_>, + arguments: &Node<'_>, + roles: ArgumentRoles, + dialect: Dialect<'_>, +) -> bool { + // `O(log n)` in the argument count, as in `is_switch_arm_body`, and + // for the same reason: every braced argument of a wide one-line + // command asks, so a sibling scan here is quadratic in the width of + // that line (#1381 review). + let mut cursor = arguments.cursor(); + let index = cursor.goto_first_child_for_byte(word.start_byte()); + // Unreachable from `is_braced_literal_slot`, which reaches here only + // for a word whose own parent is `arguments`. It is not decoration: + // `goto_first_child_for_byte` answers with a *neighbouring* child for + // a byte that starts no child, so a caller pairing a word with + // another command's argument list would otherwise be given that + // neighbour's slot. `is_switch_arm_body` carries the same id check + // for the same reason. + let Some(index) = index.filter(|_| cursor.node().id() == word.id()) else { + return true; + }; + // Also unreachable, and guarding the other half of the same pairing: + // `roles.first` is 1 only for a `namespace`, whose slot 0 holds the + // subcommand — and `namespace_script_slots` resolves a layout at all + // only when that slot is a `simple_word`, which a braced word is not. + // Loosen that guard and this is what stops slot 0 reading as the + // argument before the first. + let Some(position) = index.checked_sub(roles.first) else { + return true; + }; + match roles.slots { + ScriptSlots::Every => true, + ScriptSlots::NoneOfThem => false, + ScriptSlots::EveryButFirst => position > 0, + ScriptSlots::Only(evaluated) => position == evaluated, + ScriptSlots::Last => position + 1 == arguments.child_count() - roles.first, + ScriptSlots::EveryButLeadingLevel => { + position > 0 || !reads_as_uplevel_level(word, dialect.code) + } + // `switch` is a generic command, so `roles.first` is 0 and the + // absolute index is the position; the subject scan counts from + // the same origin, so it takes the index. + ScriptSlots::SwitchArms => switch_arm_slot(arguments, index, dialect), + } +} + +/// Whether a braced argument 0 of `uplevel` reads as a level specifier +/// rather than as the first word of the script. +/// +/// `uplevel` decides this from the argument's *value*, so the test is +/// on the text between the braces: an optional `#` prefix and then an +/// integer, whitespace-tolerant because `Tcl_GetInt` is (Tcl 8.6 +/// `uplevel(n)`). `uplevel {set x 2}` has no level and must stay whole. +/// +/// Unreadable text and text that is not brace-delimited share one exit +/// because neither is a level and the caller cannot tell them apart +/// anyway: `node_text` answers `None` for non-UTF-8 bytes, which +/// `Ast::parse` accepts, and a caller reaching this with a node that is +/// not a `braced_word` has no level either. +fn reads_as_uplevel_level(word: &Node<'_>, code: &[u8]) -> bool { + let Some(inner) = node_text(code, word) + .and_then(|text| text.strip_prefix('{')) + .and_then(|inner| inner.strip_suffix('}')) + else { + return false; + }; + let inner = inner.trim(); + let digits = inner.strip_prefix('#').map_or(inner, str::trim_start); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// Whether the argument at `index` of a `switch` command is one of the +/// arms it evaluates, rather than an option, an option's operand, the +/// subject, or a pattern. +fn switch_arm_slot(arguments: &Node<'_>, index: usize, dialect: Dialect<'_>) -> bool { + let Some(subject) = switch_subject_index(arguments, dialect) else { + // No subject: every word was a leading option or an option's + // operand, as in the truncated `switch -matchvar {m}`. Nothing + // says which half of the list this word is in, so keep the + // construct-wide script answer. + return true; + }; + if index <= subject { + return false; + } + // `switch ?options? string {pattern body ?pattern body …?}`: when + // exactly one argument follows the subject it is the braced arm + // list, a script whose interior `is_switch_arm` classifies. + if arguments.child_count() == subject + 2 { + return true; + } + // The flat spelling runs `pattern body pattern body …` from the + // subject, paired by index with no marker on either half, so an + // even offset from it is a body — the same parity + // `is_switch_arm_body` reads inside a braced arm list. + (index - subject).is_multiple_of(2) +} + +/// The index of a `switch` command's subject among `arguments`: the +/// first word that is neither a leading option nor an option's operand, +/// or the word after a `--` terminator. +/// +/// The scan stops at the first argument that is not a plain option +/// word, and a braced word never is one, so it ends at or before +/// whichever braced word the caller is asking about — its length is the +/// leading-option count, not the argument count. Every braced argument +/// of a wide one-line arm list asks, so an unbounded scan here would be +/// the quadratic `is_switch_arm_body` documents (#1381 review). +fn switch_subject_index(arguments: &Node<'_>, dialect: Dialect<'_>) -> Option { + let mut skip_operand = false; + for (index, child) in arguments.children().enumerate() { + if skip_operand { + skip_operand = false; + continue; + } + let Some(word) = (child.kind_id() == dialect.kinds.simple_word) + .then(|| node_text(dialect.code, &child)) + .flatten() + else { + return Some(index); + }; + if word == SWITCH_OPTION_TERMINATOR { + return Some(index + 1); + } + if !word.starts_with('-') { + return Some(index); + } + skip_operand = SWITCH_OPTIONS_TAKING_A_WORD.contains(&word); + } + None +} + +#[cfg(test)] +#[cfg(feature = "tcl")] +mod tests { + use super::{ + ArgumentRoles, Dialect, ScriptSlots, fills_script_slot, node_text, reads_as_uplevel_level, + }; + use crate::Tcl; + use crate::lang_helpers::tcl::BRACED_WORD_KINDS; + use crate::node::{Node, Tree}; + + /// Every `braced_word` in `code`, in source order. + fn braced_words<'a>(root: &Node<'a>, out: &mut Vec>) { + if root.kind_id() == Tcl::BracedWord as u16 { + out.push(*root); + } + for child in root.children() { + braced_words(&child, out); + } + } + + /// The first `word_list` in `code`, in source order. + fn first_word_list<'a>(root: &Node<'a>) -> Option> { + if root.kind_id() == Tcl::WordList as u16 { + return Some(*root); + } + root.children().find_map(|child| first_word_list(&child)) + } + + /// `fills_script_slot` keeps the construct-wide *script* answer for a + /// word it cannot place among the arguments it was handed. + /// + /// Neither pairing arises from `is_braced_literal_slot`, which only + /// ever asks about a word whose own parent is the argument list. Both + /// guards are still load-bearing, and the comments at each say why: + /// `goto_first_child_for_byte` answers with a *neighbouring* child for + /// a byte that starts no child, so without the id check a nested word + /// silently borrows its enclosing sibling's slot; and without the + /// `checked_sub` a `namespace`'s slot 0 would read as the argument + /// before its first. + /// + /// The unplaceable word is a *nested* one — `{b c}` inside + /// `{a {b c}}` — rather than one from another command, because a word + /// outside the list's byte range makes the cursor answer `None` and + /// never reaches the id check at all. Both rows assert `true` (script) + /// against a fixture whose placeable word answers `false`, so neither + /// is the constant either guard could be replaced by. + #[test] + fn an_unplaceable_word_keeps_the_script_answer() { + // `namespace {export} {a {b c}}`: a braced subcommand in slot 0 — + // the shape `namespace_script_slots` refuses, reconstructed here + // with the layout it would otherwise have produced — and a nested + // `{b c}` whose bytes fall inside slot 1 without being slot 1. + let code = b"namespace {export} {a {b c}}\n"; + let tree = Tree::new::(code); + let root = tree.get_root(); + let dialect = Dialect { + code, + kinds: &BRACED_WORD_KINDS, + }; + let mut words = Vec::new(); + braced_words(&root, &mut words); + let texts: Vec<&str> = words + .iter() + .map(|word| node_text(code, word).expect("ascii fixture")) + .collect(); + assert_eq!( + texts, + ["{export}", "{a {b c}}", "{b c}"], + "the fixture must hold the subcommand, slot 1, and a word \ + nested inside slot 1" + ); + let arguments = first_word_list(&root).expect("the namespace has an argument list"); + let roles = ArgumentRoles { + slots: ScriptSlots::NoneOfThem, + first: 1, + }; + + // `{export}` sits *before* the first argument. + assert!( + fills_script_slot(&words[0], &arguments, roles, dialect), + "a word before the construct's first argument keeps the script answer" + ); + // `{a {b c}}` is slot 1 — placeable, and `NoneOfThem` calls it a value. + assert!( + !fills_script_slot(&words[1], &arguments, roles, dialect), + "the fixture's placeable word must answer the other way, or the \ + rows around it are asserting a constant" + ); + // `{b c}` starts inside slot 1 but is not slot 1. + assert!( + fills_script_slot(&words[2], &arguments, roles, dialect), + "a word that is not one of these arguments keeps the script \ + answer, rather than borrowing the slot it is nested in" + ); + } + + /// `reads_as_uplevel_level` accepts what `Tcl_GetInt` accepts, and + /// nothing else (Tcl 8.6 `uplevel(n)`). + /// + /// The last row is the shared exit for text that is not a readable + /// `{…}`: a `simple_word` has no braces, and neither has a node whose + /// bytes are not UTF-8 — `Ast::parse` accepts those and `node_text` + /// answers `None` for them. Both mean "not a level" and no caller can + /// tell them apart, which is why they share one line. + /// + /// The `#` half of the rule has no row because neither pinned grammar + /// can spell it: `#` at a command position opens a comment, so `{#0}` + /// parses as a brace block holding a comment that swallows the rest of + /// the line, and the enclosing command lands under an `ERROR` that + /// `is_braced_literal_slot` rejects before the slot table. Real Tcl + /// accepts `uplevel {#0} {…}`, so the prefix stays in the rule against + /// a grammar that learns to parse it. + #[test] + fn a_level_specifier_is_an_integer_and_nothing_else() { + let code = b"uplevel {1} { 2 } {a} {} {1x} plain\n"; + let tree = Tree::new::(code); + let root = tree.get_root(); + let mut words = Vec::new(); + braced_words(&root, &mut words); + let read: Vec = words + .iter() + .map(|word| reads_as_uplevel_level(word, code)) + .collect(); + assert_eq!( + read, + [true, true, false, false, false], + "`{{1}}` and `{{ 2 }}` are levels; a word, an empty word and \ + `1x` are not" + ); + + let arguments = first_word_list(&root).expect("the command has an argument list"); + let plain = arguments + .children() + .find(|node| node.kind_id() == Tcl::SimpleWord as u16) + .expect("the fixture ends in a bare word"); + assert!( + !reads_as_uplevel_level(&plain, code), + "a word that is not brace-delimited carries no level" + ); + } +} diff --git a/tests/grammars/alterator_string_flattening.rs b/tests/grammars/alterator_string_flattening.rs index 3beb10a15..6d5c7d3f5 100644 --- a/tests/grammars/alterator_string_flattening.rs +++ b/tests/grammars/alterator_string_flattening.rs @@ -101,6 +101,24 @@ flatten_cases! { tcl_flattens_namespace_argument: LANG::Tcl, "namespace export {a b}\n", "f.tcl", "{a b}"; irules_flattens_braced_proc_name: LANG::Irules, "proc {my proc} {} {}\n", "f.irule", "{my proc}"; irules_flattens_namespace_argument: LANG::Irules, "namespace export {a b}\n", "f.irule", "{a b}"; + // The *value* argument of a command whose other arguments are + // scripts (#1381 review). Each of these was flattened before #1381 + // and became a nested `command` under it, so the dump grew a + // subtree the source does not contain: `{100}` rendered as a + // command named `100`, `{5}` as one named `5`. The script argument + // beside each is the negative half, asserted through + // `Checker::is_string_with_code` in `checker.rs` — this file can + // only observe the positive, since a flattened leaf is what it + // looks for. + tcl_flattens_after_delay: LANG::Tcl, "after {100} {puts a}\n", "f.tcl", "{100}"; + tcl_flattens_time_count: LANG::Tcl, "time {puts b} {5}\n", "f.tcl", "{5}"; + tcl_flattens_uplevel_level: LANG::Tcl, "uplevel {1} {puts c}\n", "f.tcl", "{1}"; + tcl_flattens_switch_subject: LANG::Tcl, "switch {foo} {p {puts d}}\n", "f.tcl", "{foo}"; + tcl_flattens_namespace_eval_name: LANG::Tcl, "namespace eval {my ns} {puts e}\n", "f.tcl", "{my ns}"; + irules_flattens_after_delay: LANG::Irules, "after {100} {log a}\n", "f.irule", "{100}"; + irules_flattens_time_count: LANG::Irules, "time {log b} {5}\n", "f.irule", "{5}"; + irules_flattens_uplevel_level: LANG::Irules, "uplevel {1} {log c}\n", "f.irule", "{1}"; + irules_flattens_namespace_eval_name: LANG::Irules, "namespace eval {my ns} {log d}\n", "f.irule", "{my ns}"; ruby_flattens_string_literal: LANG::Ruby, "s = \"hi\"\n", "f.rb", "\"hi\""; elixir_flattens_string_literal: LANG::Elixir, "s = \"hi\"\n", "f.ex", "\"hi\""; } From e878186434384db64541f7293c2bd1dc3c526b00 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 05:47:40 -0700 Subject: [PATCH 19/22] fix(ast): scope the Tcl slot-table error guard to the argument list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `is_braced_literal_slot` gated on `Node::has_error()`, which is transitive, so a parse error anywhere inside one argument withdrew the value-slot rescue from every sibling. `trap {pat} {v} {puts ]}` reported zero strings where main reported them, and via `keeps_children` the dump rendered `{100}` as a fabricated `command` named `100` — the defect the slot table was added to remove. `has_error` stays as the O(1) pre-filter; the argument list's own direct children are then scanned for ERROR/MISSING, since an error *inside* an argument moves no sibling's index while one occupying a slot moves every index after it. Deleting the old guard failed none of the 3361 tests, so both new tests are verified by revert: the old spelling fails exactly the new test, and removing the guard entirely fails exactly the other. Also from the same review pass: - `ScriptSlots::Last` used a raw `usize` subtraction that a `Last` row added to `NAMESPACE_SCRIPT_SUBCOMMANDS` would underflow; now `checked_sub`, matching the arm two lines above it. - The `crate::Tcl` import in `braced_slot_tests` was gated wider than its uses, so an `irules`-without-`tcl` build warned and ci.yml's workspace-wide `-D warnings` would have failed that leg. - `switch_subject_index` takes the caller's cursor rather than allocating a second one per braced word. - `Node::is_error` / `Node::is_missing` added; `has_error` now documents the transitivity trap, and `goto_first_child_for_byte` documents that it cannot return a zero-width child, which its two call sites resolve in opposite directions. - `FIXME(#1410)` anchors the C-family bool-terminal gap in source rather than only in the tracker; the Halstead/string divergence cites #1382. - CHANGELOG: #1381 also withdraws braced *conditions*, not only bodies, and #1396's list of siblings that still lose literal rows was missing Ruby. - Baseline: `Node<'a>` nom 33 -> 35, via `make self-scan-write-baseline-headroom`. --- .bca-baseline.toml | 2 +- CHANGELOG.md | 15 ++- big-code-analysis-ast/src/alterator.rs | 4 +- big-code-analysis-ast/src/getter.rs | 113 ++++++++++++++++-- big-code-analysis-ast/src/getter/tcl.rs | 3 +- .../src/lang_helpers/tcl_family.rs | 87 ++++++++++++-- big-code-analysis-ast/src/macros/kind_sets.rs | 7 ++ big-code-analysis-ast/src/node.rs | 38 ++++++ 8 files changed, 244 insertions(+), 25 deletions(-) diff --git a/.bca-baseline.toml b/.bca-baseline.toml index 5de8afd4a..68e8e0958 100644 --- a/.bca-baseline.toml +++ b/.bca-baseline.toml @@ -267,7 +267,7 @@ value = 120805.48245794396 path = "big-code-analysis-ast/src/node.rs" qualified = "Node<'a>" metric = "nom" -value = 33.0 +value = 35.0 [[entry]] path = "big-code-analysis-ast/src/parser.rs" diff --git a/CHANGELOG.md b/CHANGELOG.md index 95359a04a..78581e3cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -226,8 +226,11 @@ for historical reference. so its last body row was credited to nothing; a multi-row backtick command is a single node, so every interior row was. The wrapper is routed rather than the body, because a heredoc whose body is a single - empty row emits no body node at all. Tcl and iRules braced values - (`set x {a\n\nb}`) and C# interpolated strings still lose such rows. + empty row emits no body node at all. The sweep stopped at PHP: + measured on this branch, Ruby's backtick subshell — the exact twin of + the PHP shape fixed here — and `%w[…]`, Tcl and iRules braced values + (`set x {a\n\nb}`), and C# interpolated strings all still lose such + rows, each reporting `ploc 2, blank 1` of `sloc 3`. **Metric drift:** `loc.ploc` rises and `loc.blank` falls for PHP files containing these literals — by one for every nowdoc, empty rows or not. @@ -244,7 +247,13 @@ for historical reference. - **`bca find --type string` and `bca count --type string` no longer report a Tcl or iRules script body as a string literal** (#1381) — a - `proc` or `if` body, or an iRules `when` handler. A braced *value* is + `proc` or `if` body, or an iRules `when` handler. The withdrawal + covers every braced word the grammar hangs off a *modelled* construct + rather than off a generic command's argument list, so a braced + **condition** goes with it: `if {$x > 1}`, `while {$y}`, + `expr {$a + $b}` and `catch {…}` are no longer reported either. A + Tcl condition is an `expr` script, so that is the same call as the + body, but it is a wider change than "bodies". A braced *value* is still reported: `lappend x {a b}`, a braced `proc` name, the arguments of a `namespace` subcommand other than `eval` / `inscope` / `code`, and the pattern and variable list of an `on` / `trap` handler clause the diff --git a/big-code-analysis-ast/src/alterator.rs b/big-code-analysis-ast/src/alterator.rs index b8c27de3a..3fb2ff823 100644 --- a/big-code-analysis-ast/src/alterator.rs +++ b/big-code-analysis-ast/src/alterator.rs @@ -103,7 +103,9 @@ where /// pay for one grammar's ambiguity. `ancestors` is the chain the /// dump walk descended through, for [`Node::parent`]'s `O(depth)` /// reason (#1084) — off an unknown chain this question took - /// `Ast::dump` on 8 KB of nested Tcl braces from 7 ms to 391 ms. + /// `Ast::dump` on 8 KB of nested Tcl braces from 10 ms to 418 ms — + /// the run `ast.rs` records beside the chain this reads, not a + /// second measurement. /// #[inline] #[must_use] diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index 4a0eb01e8..e4788cf0f 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -12,7 +12,8 @@ use crate::token_role::TokenRole; use crate::lang_helpers::tcl_family::{ - ArgumentRoles, Dialect, SWITCH_COMMAND, fills_script_slot, namespace_script_slots, script_slots, + ArgumentRoles, Dialect, SWITCH_COMMAND, argument_slots_are_readable, fills_script_slot, + namespace_script_slots, script_slots, }; use crate::space_kind::SpaceKind; use crate::traits::Search; @@ -707,7 +708,13 @@ pub trait Getter { /// braces of the value slots `is_braced_literal_slot` adds still /// bill a `{}` operator, as they have since #1318. Moving that rule /// into the shared predicate changes `bca metrics` for Tcl, and is - /// its own measured change rather than a rider on this one. + /// its own measured change rather than a rider on this one — **it is + /// tracked as #1382**, where the measurement belongs. Concretely, + /// until it lands `after {100} {puts hi}` bills `N1 2` against bare + /// `after 100 {puts hi}`'s `1`, which is the score-moves-with-the- + /// delimiter property [`is_value_braced_word`]'s own rationale says + /// #1318 removed. An exception carried knowingly is still a §7 + /// disagreement, so it gets an issue rather than only a paragraph. /// /// [`is_value_braced_word`]: Self::is_value_braced_word /// [`is_braced_literal_slot`]: Self::is_braced_literal_slot @@ -746,11 +753,18 @@ pub trait Getter { /// as commands, so an arm whose *pattern* is spelled `proc`, /// `namespace`, `after` or `switch` builds one of these shapes around /// what are really arm bodies — [`is_switch_arm`] recognises it first. - /// And an owner holding a parse error has no argument positions worth - /// trusting. The multi-line `try … trap` clause is out of reach - /// entirely: the Tcl grammar leaves it inside an `ERROR` node, where - /// no role signal survives. + /// And an argument list whose own slot sequence is unreadable has no + /// positions worth trusting. The multi-line `try … trap` clause is out + /// of reach entirely: the Tcl grammar leaves it inside an `ERROR` + /// node, where no role signal survives. /// + /// That second guard is `tcl_family::argument_slots_are_readable`, + /// which asks the *slots* rather than the owner — asking + /// [`Node::has_error`] of the command withdrew every value slot of a + /// command whose only error sat inside one of its script arguments. + /// Its doc carries the measurement. + /// + /// [`Node::has_error`]: crate::Node::has_error /// [`is_value_braced_word`]: Self::is_value_braced_word /// [`is_switch_arm`]: Self::is_switch_arm #[must_use] @@ -780,7 +794,7 @@ pub trait Getter { return false; }; if parent.kind_id() != kinds.word_list - || owner.has_error() + || !argument_slots_are_readable(&owner, &parent) || Self::is_switch_arm(&owner, code, above_owner, kinds) { return false; @@ -1167,6 +1181,12 @@ mod ancestor_tests { #[cfg(any(feature = "tcl", feature = "irules"))] mod braced_slot_tests { use super::{BracedWordKinds, Getter}; + // Gated with the rows that name it: `Tcl` is read only by the + // `feature = "tcl"` helpers below, so the module's wider + // `any(tcl, irules)` gate would leave this unused — and `ci.yml` sets + // `RUSTFLAGS: "-D warnings"`, which turns that into a hard failure on + // an `irules`-without-`tcl` build. + #[cfg(feature = "tcl")] use crate::Tcl; use crate::node::{Ancestors, Node}; use crate::test_support::for_each_node_with_chain; @@ -1238,6 +1258,85 @@ mod braced_slot_tests { ); } + /// Every braced word of `code` that fills a value slot, in source + /// order. Tcl only: `is_braced_literal_slot` is a provided method, so + /// both dialects run the identical body over their own kind table. + /// + /// Walks with [`Ancestors::unknown`] rather than through + /// `for_each_node_with_chain`, whose fixtures must parse cleanly so a + /// walk cannot cover error recovery by accident — which is exactly + /// what the callers below are about. The two spellings answer + /// identically here; `assert_no_shallow_value_slot` above asserts + /// that on every node it visits. + #[cfg(feature = "tcl")] + fn value_slot_texts(code: &[u8]) -> Vec { + let tree = crate::node::Tree::new::(code); + tree.get_root() + .preorder() + .filter(|node| { + ::is_braced_literal_slot( + node, + code, + Ancestors::unknown(), + &crate::lang_helpers::tcl::BRACED_WORD_KINDS, + ) + }) + .map(|node| node.utf8_text(code).unwrap_or_default().to_owned()) + .collect() + } + + /// A parse error nested inside one argument's *script* leaves its + /// sibling *value* slots alone (#1381 review). + /// + /// `Node::has_error` is transitive, so asking it of the command + /// withdrew the rescue from every argument: `trap {pat} {v} {puts ]}` + /// lost both value slots, and the dump then rendered each as a + /// fabricated `command` named after its own text. What decides + /// whether the positions are readable is the argument list's own slot + /// sequence, and an error *inside* an argument moves no sibling's + /// index. Verified by reverting the guard to `owner.has_error()`: + /// the error row then comes back empty and this fails. + /// + /// Asserted as the *same* list for both fixtures rather than as a + /// non-empty one for the error row. The well-formed row is what pins + /// that the fixture still spells two value slots at all, so trimming + /// one out fails here instead of quietly reducing the claim to + /// "nothing changed". + #[test] + #[cfg(feature = "tcl")] + fn an_error_inside_a_script_argument_keeps_the_sibling_value_slots() { + let well_formed = value_slot_texts(b"trap {pat} {v} {puts j}\n"); + assert_eq!( + well_formed, + ["{pat}", "{v}"], + "the fixture must spell two value slots before an error is added" + ); + assert_eq!( + value_slot_texts(b"trap {pat} {v} {puts ]}\n"), + well_formed, + "an error inside the script argument moves no sibling's index" + ); + } + + /// The other half of the same rule: an `ERROR` token occupying a slot + /// of the argument list *does* shift every argument after it, so the + /// construct-wide answer is the right one there. + /// + /// `on code varList script` reads its script from the last slot, and + /// the stray `]` takes a slot of its own — so `{v}` is no longer the + /// second of three and the layout cannot be trusted. Without this row + /// the guard could be deleted outright and the test above would still + /// pass. + #[test] + #[cfg(feature = "tcl")] + fn an_error_occupying_a_slot_withdraws_the_layout() { + assert!( + value_slot_texts(b"on {code} ] {v} {puts j}\n").is_empty(), + "an ERROR token in the argument list makes every position \ + unreadable" + ); + } + /// A command with no argument list has no `switch`-arm body position. /// /// `is_value_braced_word` only ever asks about a word it has already diff --git a/big-code-analysis-ast/src/getter/tcl.rs b/big-code-analysis-ast/src/getter/tcl.rs index 04a6b89d9..bc3557eb5 100644 --- a/big-code-analysis-ast/src/getter/tcl.rs +++ b/big-code-analysis-ast/src/getter/tcl.rs @@ -113,7 +113,8 @@ impl Getter for TclCode { // operand here and not a string there; and the value slots // `is_braced_literal_slot` recognises (`proc {my proc}`, // `namespace export {…}`) are strings there while - // `get_op_type_with_code` still bills their `{` as a block. + // `get_op_type_with_code` still bills their `{` as a block — + // the third is #1382, whose measurement decides it. // // `Checker::is_call` needs no such follow-up. It calls // every `Command` a call, including the ones inside a value diff --git a/big-code-analysis-ast/src/lang_helpers/tcl_family.rs b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs index e53785321..492fd3e40 100644 --- a/big-code-analysis-ast/src/lang_helpers/tcl_family.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs @@ -17,7 +17,7 @@ //! [`Getter::is_braced_literal_slot`]: crate::getter::Getter::is_braced_literal_slot use crate::getter::{BracedWordKinds, node_text}; -use crate::node::Node; +use crate::node::{Cursor, Node}; /// The core Tcl-family commands that evaluate a braced argument as a /// *script* and that neither dialect's grammar models with a node of its @@ -231,15 +231,40 @@ pub(crate) struct ArgumentRoles { pub(crate) first: usize, } +/// Whether `arguments`' own slot sequence can be read positionally — the +/// precondition every rule below rests on (#1381 review). +/// +/// The obvious spelling, `owner.has_error()`, is wrong and was measured +/// wrong: [`Node::has_error`] is transitive, so a stray `]` buried in one +/// argument's *script* withdrew the layout from every sibling. +/// `after {100} {puts ]}` lost `{100}`, and the dump then rendered it as +/// a fabricated `command` named `100` — the defect the slot table exists +/// to remove. An error *inside* an argument moves no sibling's index; an +/// `ERROR` or MISSING token holding a slot of its own moves every index +/// after it, and that is the only shape that makes a position +/// meaningless. +/// +/// `has_error` stays as the `O(1)` pre-filter, so a well-formed command — +/// every command in a file that parses — never pays for the scan, and +/// the scan is bounded by the argument count when it does. +/// +/// [`Node::has_error`]: crate::Node::has_error +pub(crate) fn argument_slots_are_readable(owner: &Node<'_>, arguments: &Node<'_>) -> bool { + !owner.has_error() + || !arguments + .children() + .any(|slot| slot.is_error() || slot.is_missing()) +} + /// Whether the braced `word` fills a slot of `arguments` that `roles` /// names as evaluated. /// /// A word whose index cannot be resolved keeps the construct-wide /// script answer, which is what every caller had before the slot table /// existed. -pub(crate) fn fills_script_slot( - word: &Node<'_>, - arguments: &Node<'_>, +pub(crate) fn fills_script_slot<'t>( + word: &Node<'t>, + arguments: &Node<'t>, roles: ArgumentRoles, dialect: Dialect<'_>, ) -> bool { @@ -273,14 +298,27 @@ pub(crate) fn fills_script_slot( ScriptSlots::NoneOfThem => false, ScriptSlots::EveryButFirst => position > 0, ScriptSlots::Only(evaluated) => position == evaluated, - ScriptSlots::Last => position + 1 == arguments.child_count() - roles.first, + // `checked_sub` for the same reason the one above it has one: + // `Last` is reachable only from a table row paired with + // `first == 0` today, and a `Last` row added to + // `NAMESPACE_SCRIPT_SUBCOMMANDS` would pair it with `1` and + // underflow — a debug panic, or a release wrap to `usize::MAX` + // that answers `false` for every argument. `None` keeps the + // construct-wide script answer, as every other unresolvable + // position here does. + ScriptSlots::Last => arguments + .child_count() + .checked_sub(roles.first) + .is_none_or(|count| position + 1 == count), ScriptSlots::EveryButLeadingLevel => { position > 0 || !reads_as_uplevel_level(word, dialect.code) } // `switch` is a generic command, so `roles.first` is 0 and the // absolute index is the position; the subject scan counts from - // the same origin, so it takes the index. - ScriptSlots::SwitchArms => switch_arm_slot(arguments, index, dialect), + // the same origin, so it takes the index. The cursor is handed on + // rather than rebuilt: `Cursor::new` heap-allocates its stack, and + // every braced argument of a one-line arm list reaches here. + ScriptSlots::SwitchArms => switch_arm_slot(arguments, index, dialect, &mut cursor), } } @@ -312,8 +350,13 @@ fn reads_as_uplevel_level(word: &Node<'_>, code: &[u8]) -> bool { /// Whether the argument at `index` of a `switch` command is one of the /// arms it evaluates, rather than an option, an option's operand, the /// subject, or a pattern. -fn switch_arm_slot(arguments: &Node<'_>, index: usize, dialect: Dialect<'_>) -> bool { - let Some(subject) = switch_subject_index(arguments, dialect) else { +fn switch_arm_slot<'t>( + arguments: &Node<'t>, + index: usize, + dialect: Dialect<'_>, + cursor: &mut Cursor<'t>, +) -> bool { + let Some(subject) = switch_subject_index(arguments, dialect, cursor) else { // No subject: every word was a leading option or an option's // operand, as in the truncated `switch -matchvar {m}`. Nothing // says which half of the list this word is in, so keep the @@ -345,10 +388,30 @@ fn switch_arm_slot(arguments: &Node<'_>, index: usize, dialect: Dialect<'_>) -> /// whichever braced word the caller is asking about — its length is the /// leading-option count, not the argument count. Every braced argument /// of a wide one-line arm list asks, so an unbounded scan here would be -/// the quadratic `is_switch_arm_body` documents (#1381 review). -fn switch_subject_index(arguments: &Node<'_>, dialect: Dialect<'_>) -> Option { +/// the quadratic `is_switch_arm_body` documents (#1381 review). It runs +/// over the caller's cursor for the same reason. +/// +/// # The option list is resolved statically, and Tcl resolves it at run time +/// +/// A word that is not a `simple_word` is taken to be the subject. That +/// is right for the overwhelmingly common `switch $v {…}` and for a +/// braced subject, and wrong for a legal but vanishingly rare computed +/// *option*: Tcl substitutes each word before dispatching, so +/// `switch "-exact" $v a {…}` and `switch $opt $v a {…}` really do pass +/// an option there. Taken as the subject, every later index shifts by +/// one and the pattern/body parity below inverts — the bodies read as +/// literals and the patterns as scripts. The same static-resolution +/// limit `command_leading_word` documents for a command *name*, and +/// answered the same way: guessing keeps the common form right, where +/// returning `None` would surrender `switch $v` to the construct-wide +/// script answer. +fn switch_subject_index<'t>( + arguments: &Node<'t>, + dialect: Dialect<'_>, + cursor: &mut Cursor<'t>, +) -> Option { let mut skip_operand = false; - for (index, child) in arguments.children().enumerate() { + for (index, child) in arguments.children_with(cursor).enumerate() { if skip_operand { skip_operand = false; continue; diff --git a/big-code-analysis-ast/src/macros/kind_sets.rs b/big-code-analysis-ast/src/macros/kind_sets.rs index 2b06b7659..187fc95c5 100644 --- a/big-code-analysis-ast/src/macros/kind_sets.rs +++ b/big-code-analysis-ast/src/macros/kind_sets.rs @@ -210,6 +210,13 @@ macro_rules! go_bool_terminal_kinds { }; } +// FIXME(#1410): C and C++ are integer-truthy, so this set is missing the +// numeric literal kinds — `if (1)` scores no condition where `if (true)` +// scores one, within the same language. Name-keyed, so C, C++, Mozcpp and +// Objective-C are all affected, which makes this the largest of the three +// sets #1379 left behind (see `perl_bool_terminal_kinds!` below for the +// measurement). Deferred out of #1379 because the DeepSpeech corpus is +// C/C++ and the fix moves snapshots. #[macro_export] #[doc(hidden)] macro_rules! cpp_bool_terminal_kinds { diff --git a/big-code-analysis-ast/src/node.rs b/big-code-analysis-ast/src/node.rs index d2bb296b9..eae72ed22 100644 --- a/big-code-analysis-ast/src/node.rs +++ b/big-code-analysis-ast/src/node.rs @@ -131,11 +131,42 @@ impl<'a> Node<'a> { /// Checks if a node represents a syntax error or contains any syntax errors /// anywhere within it. + /// + /// Transitive, which is the trap: asking it of a construct to decide + /// whether that construct's *own* shape is readable also answers `true` + /// for an error arbitrarily deep inside one of its children. Where the + /// question is about a node's own slot sequence, ask [`is_error`] / + /// [`is_missing`] of each slot instead (#1381). + /// + /// [`is_error`]: Self::is_error + /// [`is_missing`]: Self::is_missing #[must_use] pub fn has_error(&self) -> bool { self.0.has_error() } + /// Whether this node *is* the `ERROR` node, as opposed to + /// [`has_error`](Self::has_error)'s "is, or contains one". + #[must_use] + #[inline] + pub fn is_error(&self) -> bool { + self.0.is_error() + } + + /// Whether the parser inserted this node during error recovery: a + /// zero-width token the source does not contain, standing in for one + /// the grammar required. + /// + /// A sibling of [`is_error`](Self::is_error) rather than a synonym — + /// a MISSING token is not an `ERROR` node, but it occupies a child + /// slot just the same, so a rule reading a child's *index* has to + /// account for both. + #[must_use] + #[inline] + pub fn is_missing(&self) -> bool { + self.0.is_missing() + } + /// An id unique to this node within its tree, stable for the tree's /// lifetime. Suitable as a map key for per-node walk state. #[inline] @@ -814,6 +845,13 @@ impl<'a> Cursor<'a> { /// Moves to the first child that ends after `byte` and returns its /// index in [`Node::children`] order, or `None` when no child does. /// + /// "Ends after" is strict, so a **zero-width** child at `byte` — a + /// MISSING token the parser inserted during error recovery — is + /// skipped and the cursor lands on its successor. That is the one way + /// this differs from `children().position(..)`, which finds such a + /// node; a caller resolving a node's own index must therefore check + /// [`Cursor::node`] against it, as both call sites here do. + /// /// The index is what makes this worth having over a sibling scan: /// tree-sitter stores a `repeat()` child list under balanced hidden /// nodes and skips each by its cached visible-child count, so this From c17c97978ab0580b92fe501cc7a5f7cffa1546cc Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 07:00:13 -0700 Subject: [PATCH 20/22] fix(metrics): resolve ::-qualified Tcl and iRules commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `::switch` is `switch` — a leading `::` names the global namespace, and inside a `namespace eval` body it is the spelling that guarantees the core command rather than a local proc shadowing it. `Getter::command_leading_word` stripped it, so the braced-word slot table, Halstead, `bca find --type string` and the AST dump all read the qualified form correctly. The four metrics that resolve a leading word *without* that table read it raw, and so scored the same bytes differently: - `::switch` / `::for` contributed nothing to cognitive or cyclomatic — Tcl models neither construct, so the leading word is their only seam. - `::incr` / `::append` / `::lappend` counted as an ABC branch instead of an assignment. - `::return` / `::error` / `::throw` / `::exit` were not exits. The strip now lives in one `strip_global_qualifier`, which both `command_leading_word` and `tcl_command_name` call, so the two halves cannot drift again. It stays leading-only: `ns::eval` is a different command living in `ns`, the direction `BRACED_WORD_VALUE_CASES` already pins for Halstead. iRules resolves its exit and mutator names in its own walkers rather than through `tcl_command_name`, so both were swept in the same change per grammar-dispatch.md. `irules_command_is_assignment` moves from a raw byte compare to the same resolve-then-match shape as its Tcl sibling; its leading word is still addressed by index rather than by the `name` field, which is left alone as a separate behaviour change. Twelve tests, six qualified and six namespaced controls, across both dialects. Verified by revert: perturbing the helper to the identity fails exactly the six qualified tests plus the two pre-existing `::eval` Halstead rows, and leaves all six controls passing. No integration snapshot moves — the corpora contain no Tcl. --- CHANGELOG.md | 23 +++++ big-code-analysis-ast/src/getter.rs | 17 ++-- big-code-analysis-ast/src/lang_helpers.rs | 9 +- big-code-analysis-ast/src/lang_helpers/tcl.rs | 9 ++ .../src/lang_helpers/tcl_family.rs | 22 +++++ src/metrics/abc.rs | 64 ++++++++++++++ src/metrics/abc/irules.rs | 21 +++-- src/metrics/cognitive.rs | 43 +++++++++ src/metrics/cyclomatic.rs | 39 ++++++++ src/metrics/nexits.rs | 88 ++++++++++++++++++- 10 files changed, 313 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78581e3cf..583757f9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,29 @@ for historical reference. ### Fixed +- **A `::`-qualified Tcl or iRules command now resolves to the core + command it names** in every metric, not just the ones reading the + braced-word slot table. `::switch` *is* `switch` — a leading `::` names + the global namespace, and inside a `namespace eval` body it is the + spelling that guarantees the core command over a shadowing proc — and + `Getter::command_leading_word` stripped it, so Halstead, `bca find + --type string` and the AST dump read the qualified form correctly. The + four metrics that resolve a leading word *without* that table did not: + `::switch` and `::for` contributed nothing to `cognitive` or + `cyclomatic`, `::incr` / `::append` / `::lappend` counted as an ABC + branch rather than an assignment, and `::return` / `::error` / + `::throw` / `::exit` were not counted as exits. The two halves + therefore disagreed on identical bytes. The strip now lives in one + shared helper both sides call. iRules resolves its exit and mutator + names in its own walkers rather than through Tcl's, so those were + swept in the same change. Only the *leading* qualifier is stripped: + `ns::eval` is a different command living in `ns` and is still not + promoted. **Metric drift:** on Tcl and iRules sources that spell a core + command with a leading `::`, `cognitive`, `cyclomatic` and `nexits` + rise, and ABC moves one count per mutator command from `branches` to + `assignments`. No integration snapshot moves — the corpora contain no + Tcl. + - **A grammar span reaching past end-of-input no longer counts as a line of code** (#1398), so `loc.ploc` and `loc.cloc` can no longer exceed a space's own row span. A childless zero-width recovery token placed one diff --git a/big-code-analysis-ast/src/getter.rs b/big-code-analysis-ast/src/getter.rs index e4788cf0f..84da891df 100644 --- a/big-code-analysis-ast/src/getter.rs +++ b/big-code-analysis-ast/src/getter.rs @@ -895,17 +895,12 @@ pub trait Getter { return None; } let text = node_text(code, &name)?; - // `::eval` *is* `eval` — a leading `::` names the global - // namespace, and inside a `namespace eval` body it is the - // spelling that guarantees the core command rather than a local - // proc shadowing it. Without this the qualified form fell to - // the value default and lost its block, so the score moved with - // how the author spelled a command that resolves identically. - // - // Only the *leading* qualifier is stripped: `ns::eval` is a - // different command living in `ns`, and must not be mistaken - // for the core one. - Some(text.strip_prefix("::").unwrap_or(text)) + // `::eval` *is* `eval`, and without the strip the qualified form + // fell to the value default and lost its block. The rule and the + // reason it stops at the *leading* qualifier live with the + // helper, which the four metrics that resolve a leading word + // without this table share (#1381 review). + Some(crate::lang_helpers::strip_global_qualifier(text)) } /// Whether `command` is really one `pattern body` pair of a Tcl diff --git a/big-code-analysis-ast/src/lang_helpers.rs b/big-code-analysis-ast/src/lang_helpers.rs index 83e9f3001..f0b26341b 100644 --- a/big-code-analysis-ast/src/lang_helpers.rs +++ b/big-code-analysis-ast/src/lang_helpers.rs @@ -21,5 +21,12 @@ pub mod python; pub mod tcl; // Crate-private for the same reason as `irules` above: the braced-word // slot rule is read by this crate's three classifiers and by nothing -// outside it. +// outside it. The one exception is re-exported below rather than by +// widening the module, which would publish the slot tables too. pub(crate) mod tcl_family; + +/// Re-exported because both dialects' metrics need it and the module +/// holding it is crate-private: iRules' nexits and ABC walkers resolve a +/// leading word themselves rather than through `tcl::tcl_command_name`, +/// so they normalise the `::` qualifier at their own call sites. +pub use tcl_family::strip_global_qualifier; diff --git a/big-code-analysis-ast/src/lang_helpers/tcl.rs b/big-code-analysis-ast/src/lang_helpers/tcl.rs index ea6e21403..3f12ce380 100644 --- a/big-code-analysis-ast/src/lang_helpers/tcl.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl.rs @@ -56,6 +56,14 @@ pub(crate) const BRACED_WORD_KINDS: BracedWordKinds = BracedWordKinds { /// Callers dispatch on the returned name so each `command` node resolves it /// exactly once per metric walk — the helpers below take the resolved /// identity as a precondition rather than re-deriving it. +/// +/// The name is normalised through +/// [`strip_global_qualifier`](crate::lang_helpers::strip_global_qualifier), +/// so `::switch` resolves as `switch` — the same rule +/// `Getter::command_leading_word` applies on the slot-table side. Without +/// it the two disagreed on identical bytes: Halstead and the dump read a +/// qualified `::switch` as a script while cognitive, cyclomatic, ABC and +/// nexits read it as an ordinary call and scored it zero. #[inline] #[must_use] pub fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a str> { @@ -67,4 +75,5 @@ pub fn tcl_command_name<'a>(node: &'a Node<'a>, code: &'a [u8]) -> Option<&'a st return None; } name.utf8_text(code) + .map(crate::lang_helpers::strip_global_qualifier) } diff --git a/big-code-analysis-ast/src/lang_helpers/tcl_family.rs b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs index 492fd3e40..d0dee1676 100644 --- a/big-code-analysis-ast/src/lang_helpers/tcl_family.rs +++ b/big-code-analysis-ast/src/lang_helpers/tcl_family.rs @@ -185,6 +185,28 @@ pub(crate) struct Dialect<'a> { pub(crate) kinds: &'a BracedWordKinds, } +/// A leading word with its global-namespace qualifier removed. +/// +/// `::switch` *is* `switch`: a leading `::` names the global namespace, +/// and inside a `namespace eval` body it is the spelling that guarantees +/// the core command rather than a local proc shadowing it. Any rule that +/// resolves a leading word against a table of core command names has to +/// normalise through here, or the score moves with how the author spelled +/// a command that resolves identically. Four metrics read the leading +/// word without the slot table — cognitive, cyclomatic, ABC and nexits — +/// and each scored the qualified spelling as an ordinary call until this +/// was shared (#1381 review). +/// +/// Only the *leading* qualifier is stripped. `ns::eval` is a different +/// command living in `ns` and must not be mistaken for the core one; +/// `BRACED_WORD_VALUE_CASES` in `src/metrics/halstead.rs` pins that +/// direction, and stripping every `::` segment would promote it. +#[inline] +#[must_use] +pub fn strip_global_qualifier(name: &str) -> &str { + name.strip_prefix("::").unwrap_or(name) +} + /// The [`ScriptSlots`] of a generic command named `command`, or `None` /// when no core command of that name evaluates a braced argument. pub(crate) fn script_slots(command: &str) -> Option { diff --git a/src/metrics/abc.rs b/src/metrics/abc.rs index 421568f3e..a498b9fcf 100644 --- a/src/metrics/abc.rs +++ b/src/metrics/abc.rs @@ -10913,6 +10913,45 @@ function f(int $a, int $b): int { ); } + #[test] + fn tcl_qualified_mutator_commands_count_assignment() { + // `::incr` is `incr` through the global namespace. Anchored on + // `branches_sum() == 0` as well as the assignment total: an + // unresolved name falls to the branch arm, so the two columns + // move in opposite directions and no single total can mask the + // regression (#1381 review). + check_metrics::( + "proc f {} {\n\ + ::incr x\n\ + ::append s \"hi\"\n\ + ::lappend lst 1\n\ + }", + "foo.tcl", + |metric| { + assert_eq!(metric.abc.assignments_sum(), 3); + assert_eq!(metric.abc.branches_sum(), 0); + assert_eq!(metric.abc.conditions_sum(), 0); + }, + ); + } + + #[test] + fn tcl_namespaced_mutator_command_stays_a_branch() { + // Control: only the *leading* qualifier names the core command, + // so `ns::incr` is a user proc and keeps the branch column — the + // direction `BRACED_WORD_VALUE_CASES` pins for `ns::eval`. + check_metrics::( + "proc f {} {\n\ + ns::incr x\n\ + }", + "foo.tcl", + |metric| { + assert_eq!(metric.abc.assignments_sum(), 0); + assert_eq!(metric.abc.branches_sum(), 1); + }, + ); + } + #[test] fn tcl_computed_command_name_is_not_an_assignment() { // A command whose leading word is computed (`$cmd args`) names no @@ -11317,6 +11356,31 @@ function f(int $a, int $b): int { ); } + /// `::incr` is `incr` through the global namespace. iRules resolves + /// the leading word in `irules_command_is_assignment` rather than + /// through `tcl_command_name`, so this pins the strip on that second + /// path (#1381 review). + #[test] + fn irules_abc_qualified_mutator_commands() { + check_metrics::( + "when X {\n ::incr x\n ::append s \"y\"\n ::lappend l 1\n}\n", + "foo.irule", + |metric| { + assert_eq!(metric.abc.assignments_sum(), 3); + assert_eq!(metric.abc.branches_sum(), 0); + }, + ); + } + + /// Control: `ns::incr` is a proc in `ns`, so it stays a branch. + #[test] + fn irules_abc_namespaced_mutator_command_stays_a_branch() { + check_metrics::("when X {\n ns::incr x\n}\n", "foo.irule", |metric| { + assert_eq!(metric.abc.assignments_sum(), 0); + assert_eq!(metric.abc.branches_sum(), 1); + }); + } + /// Generic (non-mutator) commands count as branches. #[test] fn irules_abc_branch_commands() { diff --git a/src/metrics/abc/irules.rs b/src/metrics/abc/irules.rs index bd1ea3ca7..89dff63c0 100644 --- a/src/metrics/abc/irules.rs +++ b/src/metrics/abc/irules.rs @@ -163,20 +163,23 @@ impl Abc for IrulesCode { // iRules mutator commands (same Tcl builtins; the dedicated `set` // production is handled separately in the impl, like Tcl). -const IRULES_ASSIGNMENT_COMMANDS: &[&[u8]] = &[b"incr", b"append", b"lappend"]; +const IRULES_ASSIGNMENT_COMMANDS: &[&str] = &["incr", "append", "lappend"]; -// iRules counterpart of `tcl_command_is_assignment`. +// iRules counterpart of `tcl_command_is_assignment`, and normalised the +// same way: `::incr` is `incr` through the global namespace, and scored +// as a branch rather than an assignment until the strip was shared +// (#1381 review). The leading word is still addressed by index rather +// than by the `name` field — a grammar-dispatch §3 smell this fix +// deliberately leaves alone, since changing the child selection is a +// behaviour change of its own. fn irules_command_is_assignment(node: &Node, code: &[u8]) -> bool { let Some(first) = node.child(0) else { return false; }; - let start = first.start_byte(); - let end = first.end_byte(); - if end > code.len() || start >= end { - return false; - } - let word = &code[start..end]; - IRULES_ASSIGNMENT_COMMANDS.contains(&word) + first + .utf8_text(code) + .map(crate::lang_helpers::strip_global_qualifier) + .is_some_and(|word| IRULES_ASSIGNMENT_COMMANDS.contains(&word)) } // iRules counterpart of `tcl_inspect_container` (Fitzpatrick Rule 9): a diff --git a/src/metrics/cognitive.rs b/src/metrics/cognitive.rs index 359f37e93..fef65443f 100644 --- a/src/metrics/cognitive.rs +++ b/src/metrics/cognitive.rs @@ -6402,6 +6402,49 @@ mod tests { ); } + #[test] + fn tcl_qualified_for_resolves_to_the_builtin() { + // `::for` is `for` reached through the global namespace, so it + // scores exactly what `tcl_for_cognitive` asserts for the bare + // spelling. Until the leading word was normalised, the qualified + // form resolved to no builtin and the loop scored 0 while + // Halstead and the dump still read its body as a script + // (#1381 review). + check_metrics::( + "proc f {n} { + ::for {set i 0} {$i < $n} {incr i} { + puts $i + } +}", + "foo.tcl", + |metric| { + assert_eq!(metric.cognitive.cognitive_sum(), 1); + assert_eq!(metric.cognitive.cognitive_max(), 1); + }, + ); + } + + #[test] + fn tcl_namespaced_for_is_not_the_builtin() { + // Control for the test above: only the *leading* qualifier names + // the global namespace. `ns::for` is a different command living + // in `ns`, so it scores nothing — the direction the `ns::eval` + // row of `BRACED_WORD_VALUE_CASES` pins for Halstead. The braced + // arguments are deliberately operator-free, as in + // `tcl_for_cyclomatic_name_gate`, so nothing inside them can + // supply the increment the leading word must. + check_metrics::( + "proc f {} { + ns::for {a} {b} {c} {d} +}", + "foo.tcl", + |metric| { + assert_eq!(metric.cognitive.cognitive_sum(), 0); + assert_eq!(metric.cognitive.cognitive_max(), 0); + }, + ); + } + #[test] fn tcl_for_cognitive_nested() { // The `for` also nests its body: constructs inside it pay the diff --git a/src/metrics/cyclomatic.rs b/src/metrics/cyclomatic.rs index fc9bc2aca..c2b2c3507 100644 --- a/src/metrics/cyclomatic.rs +++ b/src/metrics/cyclomatic.rs @@ -3976,6 +3976,45 @@ f() { ); } + #[test] + fn tcl_qualified_for_resolves_to_the_builtin() { + // `::for` is `for` through the global namespace, so it scores + // exactly what `tcl_for_cyclomatic` asserts for the bare word. + check_metrics::( + "proc f {n} { + ::for {set i 0} {$i < $n} {incr i} { + puts $i + } +}", + "foo.tcl", + |metric| { + // unit(1) + proc(base 1 + for 1) = sum 3, max 2. + assert_eq!(metric.cyclomatic.cyclomatic_sum(), 3); + assert_eq!(metric.cyclomatic.cyclomatic_max(), 2); + assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 3); + }, + ); + } + + #[test] + fn tcl_namespaced_for_is_not_the_builtin() { + // Control: `ns::for` lives in `ns` and is not the core loop, so + // it adds no decision — the same expectation + // `tcl_for_cyclomatic_name_gate` holds for `format`, and the + // braced arguments are operator-free for the same reason. + check_metrics::( + "proc f {} { + ns::for {a} {b} {c} {d} +}", + "foo.tcl", + |metric| { + assert_eq!(metric.cyclomatic.cyclomatic_sum(), 2); + assert_eq!(metric.cyclomatic.cyclomatic_max(), 1); + assert_eq!(metric.cyclomatic.cyclomatic_modified_sum(), 2); + }, + ); + } + #[test] fn tcl_irules_for_parity() { // iRules models `for` as a dedicated kind counted by the kind diff --git a/src/metrics/nexits.rs b/src/metrics/nexits.rs index 75fdb906c..bedfda0dc 100644 --- a/src/metrics/nexits.rs +++ b/src/metrics/nexits.rs @@ -395,9 +395,17 @@ impl Exit for IrulesCode { // proc. iRules flow commands (`event disable`, `TCP::close`, // `reject`, `drop`) remain deliberately uncounted as exits in // v1. + // Normalised for the same reason `tcl_command_name` is: `::return` + // is `return` through the global namespace. This walker resolves + // the name itself rather than through that helper, so the strip + // has to be repeated here (#1381 review). if node.kind_id() == Irules::Command && let Some(name) = node.child_by_field_name("name") - && matches!(name.utf8_text(code), Some("return" | "error")) + && matches!( + name.utf8_text(code) + .map(crate::lang_helpers::strip_global_qualifier), + Some("return" | "error") + ) { stats.exit += 1; } @@ -2034,6 +2042,48 @@ end", /// position (`puts error`) or inside a string are not exits. The /// leading word of a nested braced command (`{ARITH DIVZERO}`) is /// likewise a different command name and contributes nothing. + /// Every abrupt-exit builtin reached through the global namespace. + /// `::return` *is* `return`, so all four must count; the leading word + /// is the only seam these have, and an unstripped qualifier made each + /// read as an ordinary call (#1381 review). + #[test] + fn tcl_qualified_exits_are_exits() { + check_metrics::( + "proc f {x} { + if {$x < 0} { + ::error \"negative\" + } + if {$x == 0} { + ::throw {ARITH DIVZERO} \"div by zero\" + } + if {$x > 100} { + ::exit 1 + } + ::return $x +}", + "foo.tcl", + |metric| { + assert_eq!(metric.nexits.nexits_sum(), 4); + assert_eq!(metric.nexits.nexits_max(), 4); + }, + ); + } + + /// Control for the test above: only the *leading* qualifier resolves + /// to the core command, so a proc in `ns` is not an exit. + #[test] + fn tcl_namespaced_return_is_not_an_exit() { + check_metrics::( + "proc f {x} { + ns::return $x +}", + "foo.tcl", + |metric| { + assert_eq!(metric.nexits.nexits_sum(), 0); + }, + ); + } + #[test] fn tcl_error_in_argument_position_is_not_exit() { check_metrics::( @@ -3101,6 +3151,42 @@ end", /// set — TMOS runs a Tcl 8.4-derived interpreter with no such /// builtin, so the word can only ever name a user proc — and /// `error` in argument position is not an exit either. + /// iRules resolves the exit name in its own walker rather than + /// through `tcl_command_name`, so the `::` strip needs pinning on + /// that second path too — the sibling sweep grammar-dispatch.md + /// requires (#1381 review). + #[test] + fn irules_qualified_exits_are_exits() { + check_metrics::( + "proc f { x } { + if { $x < 0 } { + ::error \"negative\" + } + ::return $x +} +", + "foo.irule", + |metric| { + assert_eq!(metric.nexits.nexits_sum(), 2); + }, + ); + } + + /// Control: `ns::return` is a proc in `ns`, not the core command. + #[test] + fn irules_namespaced_return_is_not_an_exit() { + check_metrics::( + "proc f { x } { + ns::return $x +} +", + "foo.irule", + |metric| { + assert_eq!(metric.nexits.nexits_sum(), 0); + }, + ); + } + #[test] fn irules_throw_and_argument_position_error_are_not_exits() { check_metrics::( From 5f609d2066e33a709be3850e8068979c0c104dab Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 07:03:37 -0700 Subject: [PATCH 21/22] fix(metrics): make the zero-span clamp branch observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `clamp_line_sets_to_span`'s `span == 0` arm passes `(1, 0)` to `retain_range`, whose inverted-range path calls `words.clear()` — so every PLOC and CLOC row the space holds is discarded, and the parent never recovers it, because a child is clamped before `Ploc::merge` lifts it. The two assertions at the end of the function cannot see that: they compare against `span`, which the clear forces to `0 <= 0`. The one branch that destroys data was the one branch nothing observed. Assert the precondition before the clear instead, where the question can still be answered. No behaviour change in either profile — the branch still clears — but a walk arriving with a row now fails loudly in debug rather than losing it silently. The review that raised this called it a live bug. It does not reproduce: `span == 0` needs a space opened on a genuinely zero-width node at column 0, every language's `is_func_space` matches compound productions rather than a raw token or a recovery node, and a MISSING node is always a single terminal. The assertion now backs that with the whole corpus rather than with reasoning — it holds across every lib and integration target, none of which trips it. The justification comment is softened to match. It claimed a corpus measurement, and `line_set`'s own header retracts a claim of exactly that shape, in this same release, as having been wrong *and* load-bearing (#1398). `a_zero_span_keeps_no_row` becomes `a_zero_span_holding_rows_trips_the _guard`: seeding a populated set at zero span is now precisely the state the guard rejects. The clearing behaviour it used to assert is covered a layer down by `retain_range_inverted_empties_the_set`, which seeds `[0, 1, 400]` and asserts row 0 specifically is gone — the same discrimination, at the layer that owns it. A companion test pins the empty zero-span case every real walk takes. --- src/metrics/loc.rs | 80 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/metrics/loc.rs b/src/metrics/loc.rs index f22be7524..41dd74c27 100644 --- a/src/metrics/loc.rs +++ b/src/metrics/loc.rs @@ -832,7 +832,32 @@ impl Stats { // below is therefore a direct `Stats` test rather than a // fixture, per `.claude/rules/testing.md`: it is the only shape // that can tell the two spellings apart. + // + // The corpus measurement is the weaker half of that claim, and + // `line_set`'s module header retracts one of exactly this shape + // as having been wrong *and* load-bearing (#1398). The stronger + // half is structural: `span == 0` needs a space opened on a + // genuinely zero-width node at column 0, and every language's + // `is_func_space` matches compound productions — never a raw + // token, never `is_error` / `is_missing` — while a MISSING node + // is always a single terminal. + // + // Either way the absence is not what should hold the branch, + // because `retain_range(1, 0)` *clears* both line sets: whatever + // arrived is gone, and the parent never recovers it, since a + // child is clamped before `Ploc::merge` lifts it. That clear is + // also what makes the two assertions at the end of this function + // vacuous here — they compare against `span`, so both read + // `0 <= 0` no matter what was destroyed. Assert before the + // clear, where the question can still be answered. let (first, last) = if span == 0 { + debug_assert!( + self.ploc() == 0 && self.cloc() == 0, + "zero-span space reached the clamp holding {} ploc / {} cloc \ + row(s); clearing them here would lose them silently", + self.ploc(), + self.cloc() + ); (1, 0) } else { (start, self.sloc.end_line.saturating_sub(1)) @@ -11612,37 +11637,52 @@ class A { } } - /// The `span == 0` arm of [`Stats::clamp_line_sets_to_span`], which - /// no parsed input can reach with a populated line set. + /// The `span == 0` arm of [`Stats::clamp_line_sets_to_span`] clears + /// both line sets, so its precondition — that no walk arrives here + /// holding a row — is the thing worth pinning. Nothing else can: + /// the two assertions at the end of that function compare against + /// `span`, which the clear forces to `0 <= 0` whatever it destroyed. /// - /// Measured: spelling it `(0, 0)` instead of `(1, 0)` — retaining - /// row 0 of a span that covers no row — is byte-identical over - /// every corpus file and fails none of the lib suite, because the - /// three walks that arrive here (the empty file and the two - /// whitespace-only root contracts) all arrive with both line sets - /// already empty. A fixture therefore cannot cover this branch at - /// all; seeding `Stats` directly is the only shape that can - /// (`.claude/rules/testing.md`, "Pair any end-to-end test with a - /// direct unit test on the function whose contract is verified"). + /// This test used to assert the clearing itself, seeding row 0 + /// because it is the one row a `(0, 0)` spelling would wrongly keep. + /// That is now covered a layer down by + /// `line_set::tests::retain_range_inverted_empties_the_set`, which + /// seeds `[0, 1, 400]` and asserts row 0 specifically is gone — so + /// the seed here is free to become what it always described: the + /// state the guard exists to reject. /// - /// The seed is row 0 specifically: it is the one row `(0, 0)` would - /// wrongly keep, so a test seeding any other row would pass under - /// both spellings. + /// A fixture cannot reach this branch with a populated set at all — + /// `span == 0` needs a space opened on a genuinely zero-width node + /// at column 0, and every `is_func_space` matches compound + /// productions — so seeding `Stats` directly is the only shape that + /// can (`.claude/rules/testing.md`, "Pair any end-to-end test with a + /// direct unit test on the function whose contract is verified"). #[test] - fn a_zero_span_keeps_no_row() { + #[cfg(debug_assertions)] + #[should_panic(expected = "zero-span space reached the clamp holding")] + fn a_zero_span_holding_rows_trips_the_guard() { let mut stats = Stats::default(); // The empty file's span: `0..0`, covering no row at all. stats.init_unit_span(0, 0); stats.ploc.lines.insert(0); - stats.cloc.only_comment_line_starts.insert(0); - stats.cloc.code_comment_line_starts.insert(0); - assert_eq!((stats.sloc(), stats.ploc(), stats.cloc()), (0, 1, 1)); + assert_eq!((stats.sloc(), stats.ploc()), (0, 1)); + + stats.clamp_line_sets_to_span(); + } + + /// The companion to the guard above: an *empty* zero-span space is + /// the state every real walk arrives in, and it must pass through + /// the clamp untouched rather than trip the assertion. + #[test] + fn a_zero_span_without_rows_clamps_cleanly() { + let mut stats = Stats::default(); + stats.init_unit_span(0, 0); stats.clamp_line_sets_to_span(); assert_eq!( - (stats.ploc(), stats.cloc()), - (0, 0), + (stats.sloc(), stats.ploc(), stats.cloc()), + (0, 0, 0), "a span of no rows retains no row" ); } From a3041252bef9aecd25b10346d7321f448624acfb Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Sat, 12 Sep 2026 07:12:35 -0700 Subject: [PATCH 22/22] docs(ast): record the Tcl brace cost in the serialize depth bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `MAX_AST_SERIALIZE_DEPTH` bounds the `AstNode` tree *after* the alterator, and its doc quoted one number: the deepest AST across the ~8 000-file corpus, 188 levels. That is a count of AstNode levels, not a conversion rate from source nesting, and since #1381 the two came apart for the Tcl family — a script body keeps its children, so one brace spans `braced_word` -> `command` -> `word_list` before the next. Measured rather than asserted: 196 levels over 64 nested `eval` braces, three per brace, putting the effective ceiling near 170 nested braces rather than 512. `a_tcl_brace_level_costs_three_ast_levels` pins both figures, so the doc cannot drift from the grammar again. The first draft of this commit said four levels and 128 braces; the test is what caught it. Nothing here changes behaviour. The bound is not raised: it exists because serde's recursion overflows the native stack into `SIGABRT` rather than a catchable panic (#1056), and the constant is published from two crates. Coverage for what the bound does when reached: - `server_tests.rs` had no Tcl or iRules fixture at all. One now pins that a script body renders its children rather than collapsing to a verbatim leaf, and a second pins that a 250-brace input fails the *request* — 500, `serialize_failed` — rather than the process. The depth message never reaches the client, because `Format::encode` collapses the serializer error, so the token is the contract. - `fuzz/src/nested.rs` gains `NestLang::Tcl`, the first entry whose constructs nest scripts rather than expressions, with a seed pair. Appending it last keeps every committed seed decoding unchanged: each selects a language with `byte % N` and all eight use a byte below 4. `dump_error` and `space_error` now take the `Nesting` rather than its rendered bytes, so the language parsed is always the language rendered — handing Tcl source to the Rust grammar yields a shallow `ERROR` tree that reports no depth failure, which is the same "looks like coverage" shape the seed corpus already had once. --- big-code-analysis-ast/src/ast.rs | 76 ++++++++++ big-code-analysis-web/src/web/server_tests.rs | 124 +++++++++++++++++ fuzz/corpus/nested_depth/tcl_deep | 1 + fuzz/corpus/nested_depth/tcl_shallow | Bin 0 -> 3 bytes fuzz/src/nested.rs | 131 ++++++++++++++---- 5 files changed, 306 insertions(+), 26 deletions(-) create mode 100644 fuzz/corpus/nested_depth/tcl_deep create mode 100644 fuzz/corpus/nested_depth/tcl_shallow diff --git a/big-code-analysis-ast/src/ast.rs b/big-code-analysis-ast/src/ast.rs index a5864063a..629aec7e7 100644 --- a/big-code-analysis-ast/src/ast.rs +++ b/big-code-analysis-ast/src/ast.rs @@ -160,6 +160,21 @@ pub struct AstResponse { /// DeepSpeech, …) is 188 levels. It is also set well clear of the stack: /// the earliest measured overflow of any emitted format was 2 000 levels /// on a debug build's default 2 MiB thread. +/// +/// Read that 188 as a corpus figure, not a conversion rate. The depth +/// counted here is the **`AstNode` tree after [`Alterator::alterate`]**, +/// and how many levels one level of *source* nesting costs is a +/// per-language property of what that alterator flattens. The Tcl family +/// is the outlier: since #1381 a script body keeps its children, so one +/// brace level spans `braced_word` → `command` → `word_list` and costs +/// three levels here — 196 levels over 64 nested `eval` braces, measured +/// by `a_tcl_brace_level_costs_three_ast_levels` below — putting the +/// ceiling near 170 nested braces rather than 512. Nothing in the corpus +/// approaches either number — it holds no Tcl at all — but a caller +/// re-deriving this bound from the 188 alone would be reasoning about +/// the wrong quantity. +/// +/// [`Alterator::alterate`]: crate::alterator::Alterator::alterate pub const MAX_AST_SERIALIZE_DEPTH: usize = 512; /// Serializes an [`AstNode`]'s children one level deeper, refusing to @@ -602,6 +617,67 @@ mod tests { ); } + /// The deepest `children` chain in a tree, measured iteratively so a + /// pathological input cannot overflow the measurement itself. + fn ast_depth(root: &AstNode) -> usize { + let mut deepest = 0; + let mut stack = vec![(root, 1usize)]; + while let Some((node, depth)) = stack.pop() { + deepest = deepest.max(depth); + stack.extend(node.children.iter().map(|child| (child, depth + 1))); + } + deepest + } + + /// What one level of Tcl brace nesting costs in `AstNode` levels, and + /// therefore how far [`MAX_AST_SERIALIZE_DEPTH`] actually reaches for + /// this language. + /// + /// The bound's doc quotes a corpus figure of 188 levels, which is a + /// count of `AstNode` levels and not a conversion rate from source + /// nesting. Since #1381 a Tcl script body keeps its children, so the + /// two quantities came apart here more than anywhere else: this test + /// is what keeps the number in that doc honest, and what would notice + /// if an alterator change moved it again. + #[cfg(feature = "tcl")] + #[test] + fn a_tcl_brace_level_costs_three_ast_levels() { + // `eval` evaluates every argument, so each level keeps its + // children rather than flattening to a verbatim leaf. + let levels = 64; + let mut code = String::new(); + for _ in 0..levels { + code.push_str("eval {"); + } + code.push_str("puts hi"); + for _ in 0..levels { + code.push('}'); + } + + let root = build_ast::(code.as_bytes(), "deep.tcl"); + let depth = ast_depth(&root); + let per_level = depth / levels; + + // Measured 196 over 64 at the pinned grammar: `braced_word` -> + // `command` -> `word_list` before the next `braced_word`. The + // range admits one level of drift either way so a grammar bump + // reports the change rather than simply going red. + assert!( + (3..=4).contains(&per_level), + "a braced script level should cost 3 AstNode levels \ + (braced_word -> command -> word_list); measured {depth} \ + levels over {levels} braces = {per_level}" + ); + // The consequence worth stating in one place: the 512-level bound + // is a low-hundreds brace ceiling for Tcl, not a 512 one. + let ceiling = MAX_AST_SERIALIZE_DEPTH / per_level; + assert!( + (120..=180).contains(&ceiling), + "the effective Tcl brace ceiling should sit near 170, got \ + {ceiling} from {per_level} levels per brace" + ); + } + #[test] fn a_pathologically_deep_ast_errors_and_tears_down_without_overflowing() { // The chain is built directly rather than parsed, but this depth diff --git a/big-code-analysis-web/src/web/server_tests.rs b/big-code-analysis-web/src/web/server_tests.rs index 3a6252496..33e868e7f 100644 --- a/big-code-analysis-web/src/web/server_tests.rs +++ b/big-code-analysis-web/src/web/server_tests.rs @@ -186,6 +186,130 @@ async fn test_web_ast() { assert_eq!(res, expected); } +/// Collects every `type` in a rendered `/ast` tree, depth-first. +fn ast_node_types(node: &Value, out: &mut Vec) { + if let Some(kind) = node["type"].as_str() { + out.push(kind.to_owned()); + } + if let Some(children) = node["children"].as_array() { + for child in children { + ast_node_types(child, out); + } + } +} + +/// The first Tcl coverage `/ast` has had, pinning the shape #1381 +/// changed: a `proc` body is a `braced_word` that keeps its children, so +/// the commands inside it are nodes of their own rather than collapsing +/// into one verbatim leaf. +/// +/// Asserted structurally rather than as a whole-tree `json!` literal — +/// the claim is that the body has a command subtree, and a full literal +/// would pin every unrelated detail of the Tcl grammar with it. +#[actix_rt::test] +async fn test_web_ast_tcl_script_body_keeps_its_children() { + let app = test::init_service( + App::new().app_data(test_config()).service( + web::resource("/ast") + .guard(guard::Header("content-type", "application/json")) + .route(web::post().to(ast_parser)), + ), + ) + .await; + let req = test::TestRequest::post() + .uri("/ast") + .set_json(AstPayload { + id: "tcl-1".to_string(), + file_name: "foo.tcl".to_string(), + code: "proc f {} {\n puts hi\n}\n".to_string(), + comment: false, + span: true, + }) + .to_request(); + + let res: Value = test::call_and_read_body_json(&app, req).await; + assert_eq!(res["language"], json!("tcl")); + + let mut types = Vec::new(); + ast_node_types(&res["root"], &mut types); + assert!( + types.iter().any(|kind| kind == "procedure"), + "the `proc` must render as its own construct: {types:?}" + ); + // The load-bearing one: a flattened body would leave the + // `braced_word` childless, so the `puts` command would not appear at + // all. Its presence is what says the body kept its children. + assert!( + types.iter().any(|kind| kind == "command"), + "the body's `puts` must survive as a command node, not be \ + flattened into the braced_word leaf: {types:?}" + ); +} + +/// Deeply-braced Tcl reaches `MAX_AST_SERIALIZE_DEPTH` where no other +/// language in the corpus does, because a script body that keeps its +/// children costs roughly four `AstNode` levels per brace (#1381). +/// +/// What this pins is that the bound *holds*: the request fails with the +/// uniform error body rather than overflowing serde's native stack, +/// which aborts the process rather than raising a catchable panic +/// (#1056). The depth message itself never reaches the client — +/// `Format::encode` collapses the serializer error — so the observable +/// contract is the 500 and its `serialize_failed` token. +#[actix_rt::test] +async fn test_web_ast_deeply_braced_tcl_errors_rather_than_aborting() { + let app = test::init_service( + App::new().app_data(test_config()).service( + web::resource("/ast") + .guard(guard::Header("content-type", "application/json")) + .route(web::post().to(ast_parser)), + ), + ) + .await; + + // `eval` takes a script in every argument, so each level keeps its + // children: `command` -> `word_list` -> `braced_word` -> `command`. + // 250 levels is comfortably past the 512-level bound at ~3 levels + // each, and still a ~1.5 KB payload. + let depth = 250; + let mut code = String::new(); + for _ in 0..depth { + code.push_str("eval {"); + } + code.push_str("puts hi"); + for _ in 0..depth { + code.push('}'); + } + code.push('\n'); + + let req = test::TestRequest::post() + .uri("/ast") + .set_json(AstPayload { + id: "tcl-deep".to_string(), + file_name: "deep.tcl".to_string(), + code, + comment: false, + span: true, + }) + .to_request(); + + let resp = test::call_service(&app, req).await; + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a tree past the serialize bound must fail the request, not the \ + process", + ); + let body = test::read_body(resp).await; + assert_uniform_error_body(&body, "tcl-deep"); + let parsed: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + parsed["error_kind"], + json!("serialize_failed"), + "the depth breach must surface as the serialize_failed token", + ); +} + #[actix_rt::test] async fn test_web_ast_string() { let app = test::init_service( diff --git a/fuzz/corpus/nested_depth/tcl_deep b/fuzz/corpus/nested_depth/tcl_deep new file mode 100644 index 000000000..3b49221e5 --- /dev/null +++ b/fuzz/corpus/nested_depth/tcl_deep @@ -0,0 +1 @@ +ÿ \ No newline at end of file diff --git a/fuzz/corpus/nested_depth/tcl_shallow b/fuzz/corpus/nested_depth/tcl_shallow new file mode 100644 index 0000000000000000000000000000000000000000..472b45891410d62ccc6ddd799736a5b9323cd9ed GIT binary patch literal 3 KcmZQ!X8-^I9RLgf literal 0 HcmV?d00001 diff --git a/fuzz/src/nested.rs b/fuzz/src/nested.rs index a6693360b..e2690aff5 100644 --- a/fuzz/src/nested.rs +++ b/fuzz/src/nested.rs @@ -7,12 +7,20 @@ //! //! # Why the constructs are valid source //! -//! Every open/close pair below nests around an *expression* and leaves -//! the result parseable. That is not politeness — tree-sitter's error -//! recovery flattens badly-formed input, so an invalid generator would -//! produce a shallow tree with a large `ERROR` node and quietly stop -//! testing depth at all. Malformed bytes are already covered by the -//! per-language targets, which mutate freely. +//! Every open/close pair below nests around an *expression* — except +//! Tcl's, which nest around a *script*, because the language has no +//! expression form that nests outside `expr` — and leaves the result +//! parseable. That is not politeness — tree-sitter's error recovery +//! flattens badly-formed input, so an invalid generator would produce a +//! shallow tree with a large `ERROR` node and quietly stop testing depth +//! at all. Malformed bytes are already covered by the per-language +//! targets, which mutate freely. +//! +//! That failure is silent in both directions, which is why each language +//! whose constructs differ in kind carries a shallow-nest assertion as +//! well as a deep one: an `ERROR`-flattened tree serializes cleanly at +//! every depth, so "the deep nest failed to serialize" says nothing on +//! its own. //! //! # What depth is for //! @@ -47,10 +55,11 @@ pub const MAX_NESTING_DEPTH: usize = 512; /// Languages the generator knows how to nest. /// /// A subset of the fuzzed set: each entry needs a hand-written table of -/// constructs, and these four span the interesting variation — braces -/// versus indentation, and three different lambda spellings. +/// constructs, and these five span the interesting variation — braces +/// versus indentation, three different lambda spellings, and one +/// language whose nesting is scripts rather than expressions. // `Ord` so `seeds_cover_every_language` can compare the decoded set -// against an expected one, rather than asserting membership four times. +// against an expected one, rather than asserting membership five times. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum NestLang { /// Rust: block expressions nest, so it has the widest shape table. @@ -63,6 +72,17 @@ pub enum NestLang { /// Python: expression nesting only, since indentation-based blocks /// would make the generated source quadratic in the depth. Python, + /// Tcl: the one entry whose constructs nest *scripts* rather than + /// expressions, and the reason it is here. Since #1381 a braced + /// script body keeps its children, so one brace costs three + /// `AstNode` levels (`braced_word` → `command` → `word_list`) where + /// the expression languages spend one or two — which puts + /// `MAX_AST_SERIALIZE_DEPTH` within reach of far shallower source + /// than the other four can produce. Appended last deliberately: the + /// committed seeds select a language with `byte % N`, and every one + /// of them uses a byte below 4, so extending the modulus leaves + /// their decoding unchanged. + Tcl, } /// One nesting construct, rendered per language by [`Nesting::pair`]. @@ -92,11 +112,12 @@ const MAX_SHAPES: usize = 16; impl NestLang { /// Decode a language selector byte. See [`Nesting`]'s byte layout. fn from_byte(byte: u8) -> Self { - match byte % 4 { + match byte % 5 { 0 => Self::Rust, 1 => Self::Cpp, 2 => Self::Javascript, - _ => Self::Python, + 3 => Self::Python, + _ => Self::Tcl, } } } @@ -132,7 +153,7 @@ pub struct Nesting { /// /// | bytes | meaning | /// |---|---| -/// | 0 | language selector, `% 4` | +/// | 0 | language selector, `% 5` | /// | 1-2 | raw depth, little-endian `u16`, reduced in [`Nesting::render`] | /// | 3.. | one shape per byte, `% 5`, up to [`MAX_SHAPES`] | /// @@ -171,6 +192,7 @@ impl Nesting { NestLang::Cpp => LANG::Cpp, NestLang::Javascript => LANG::Javascript, NestLang::Python => LANG::Python, + NestLang::Tcl => LANG::Tcl, } } @@ -213,6 +235,10 @@ impl Nesting { NestLang::Cpp => (b"int main() { auto x = ", b"; }\n"), NestLang::Javascript => (b"let x = ", b";\n"), NestLang::Python => (b"x = ", b"\n"), + // The nest is already a complete script, so it needs no + // surrounding unit: every shape below is a command taking a + // braced script, and the leaf is a bare command word. + NestLang::Tcl => (b"", b"\n"), } } @@ -263,13 +289,26 @@ impl Nesting { (NestLang::Python, Shape::Call) => (b"f(", b")"), (NestLang::Python, Shape::Bracket) => (b"[", b"][0]"), (NestLang::Python, Shape::Lambda) => (b"(lambda: ", b")()"), + + // Every Tcl arm nests a *script*, not an expression: the + // language has no expression form that nests outside `expr`, + // and a braced script body is the shape whose depth cost + // this language was added to cover. `proc` is the `Lambda` + // spelling because it is the one that opens a function + // space, so it drives `MAX_SPACE_SERIALIZE_DEPTH` the way + // the other languages' closures do. + (NestLang::Tcl, Shape::Paren) => (b"uplevel 1 {", b"}"), + (NestLang::Tcl, Shape::Call) => (b"catch {", b"}"), + (NestLang::Tcl, Shape::Bracket) => (b"if {1} {", b"}"), + (NestLang::Tcl, Shape::Lambda) => (b"proc p {} {", b"}"), + (NestLang::Tcl, Shape::Block) => (b"eval {", b"}"), } } } #[cfg(test)] mod tests { - use big_code_analysis::{Ast, AstCfg, LANG, MetricsOptions, Source}; + use big_code_analysis::{Ast, AstCfg, MetricsOptions, Source}; use arbitrary::{Arbitrary, Unstructured}; @@ -293,13 +332,21 @@ mod tests { /// make these tests pass for the wrong reason. const DEPTH_ERROR: &str = "nesting is deeper than the serialization limit"; - /// Serialize an `AstNode` dump of `source`, returning the error text + /// Serialize an `AstNode` dump of `input`, returning the error text /// if it fails. - fn dump_error(source: Vec) -> Option { - let ast = Ast::parse(Source::from_bytes(LANG::Rust, source)).expect("rust is enabled"); + /// + /// Takes the `Nesting` rather than its rendered bytes so the language + /// parsed is always the language rendered. Passing the two separately + /// let a caller hand Tcl source to the Rust grammar, which parses to + /// a shallow `ERROR` tree and reports no depth failure — the exact + /// "looks like coverage" result this module's seeds already had once. + fn dump_error(input: &Nesting) -> Option { + let lang = input.lang(); + let ast = + Ast::parse(Source::from_bytes(lang, input.render())).expect("language is enabled"); let dump = ast.dump(AstCfg { id: String::new(), - language: LANG::Rust.name().to_owned(), + language: lang.name().to_owned(), comment: false, span: true, }); @@ -318,7 +365,7 @@ mod tests { // `depth` is reduced modulo the cap, so `MAX_NESTING_DEPTH` maps // to 1. The value one below it is the deepest reachable nest. let deepest = u16::try_from(MAX_NESTING_DEPTH - 1).expect("cap fits in u16"); - let error = dump_error(nesting(NestLang::Rust, deepest, &[Shape::Paren]).render()) + let error = dump_error(&nesting(NestLang::Rust, deepest, &[Shape::Paren])) .expect("the deepest generated nest must reach MAX_AST_SERIALIZE_DEPTH"); assert!(error.contains(DEPTH_ERROR), "unexpected failure: {error}"); @@ -326,7 +373,7 @@ mod tests { // holds for a generator that emits an unserializable tree at // *every* depth, which would say nothing about reaching a bound. assert_eq!( - dump_error(nesting(NestLang::Rust, 4, &[Shape::Paren]).render()), + dump_error(&nesting(NestLang::Rust, 4, &[Shape::Paren])), None, "a shallow nest must serialize cleanly" ); @@ -334,14 +381,45 @@ mod tests { /// Serialize the `FuncSpace` tree for `source`, returning the error /// text if it fails. - fn space_error(source: Vec) -> Option { - let space = Ast::parse(Source::from_bytes(LANG::Rust, source)) - .expect("rust is enabled") + fn space_error(input: &Nesting) -> Option { + let space = Ast::parse(Source::from_bytes(input.lang(), input.render())) + .expect("language is enabled") .metrics(MetricsOptions::default()) .expect("walker succeeds"); serde_json::to_vec(&space).err().map(|e| e.to_string()) } + /// The same measurement for Tcl, which is why the language was added + /// to this generator: a braced script body keeps its children since + /// #1381, so one brace costs three `AstNode` levels and the bound is + /// reachable from far shallower source than the expression languages + /// need. + /// + /// The shallow half matters more here than it does for Rust. This + /// module's whole hazard is that invalid source flattens into a + /// large `ERROR` node and quietly stops testing depth — and every + /// Tcl arm is a *script* nest, a construct class none of the other + /// four exercise. A shallow nest that serializes cleanly is what + /// says the generated braces really parse. + #[test] + fn tcl_script_nesting_exceeds_the_ast_serialize_bound() { + // A third of the cap, so the claim is specifically that Tcl needs + // fewer braces than the other languages need constructs. At three + // levels per brace this still clears 512. + let braces = u16::try_from(MAX_NESTING_DEPTH / 3).expect("cap fits in u16"); + let error = dump_error(&nesting(NestLang::Tcl, braces, &[Shape::Block])) + .expect("a third of the cap in braces must reach MAX_AST_SERIALIZE_DEPTH"); + assert!(error.contains(DEPTH_ERROR), "unexpected failure: {error}"); + + assert_eq!( + dump_error(&nesting(NestLang::Tcl, 4, &[Shape::Block])), + None, + "a shallow brace nest must parse and serialize cleanly; an \ + ERROR-flattened tree would serialize cleanly at every depth \ + and make the assertion above meaningless" + ); + } + /// The same measurement for the `FuncSpace` bound, which only the /// lambda shapes drive: a paren nests the AST without opening a /// function space, so a `Shape::Paren` nest would leave this bound @@ -349,7 +427,7 @@ mod tests { #[test] fn lambda_shapes_exceed_the_space_serialize_bound() { let deepest = u16::try_from(MAX_NESTING_DEPTH - 1).expect("cap fits in u16"); - let error = space_error(nesting(NestLang::Rust, deepest, &[Shape::Lambda]).render()) + let error = space_error(&nesting(NestLang::Rust, deepest, &[Shape::Lambda])) .expect("the deepest lambda nest must reach MAX_SPACE_SERIALIZE_DEPTH"); assert!(error.contains(DEPTH_ERROR), "unexpected failure: {error}"); @@ -358,7 +436,7 @@ mod tests { // the assertion above a claim about `FuncSpace` depth rather // than about depth in general. assert_eq!( - space_error(nesting(NestLang::Rust, deepest, &[Shape::Paren]).render()), + space_error(&nesting(NestLang::Rust, deepest, &[Shape::Paren])), None, "a paren nest opens no function spaces and must serialize cleanly" ); @@ -399,7 +477,7 @@ mod tests { // Non-vacuity: an empty directory would satisfy every assertion // below by having nothing to contradict them. assert!( - seeds >= 4, + seeds >= 5, "expected at least one seed per language, found {seeds}" ); assert_eq!( @@ -408,7 +486,8 @@ mod tests { NestLang::Rust, NestLang::Cpp, NestLang::Javascript, - NestLang::Python + NestLang::Python, + NestLang::Tcl ]), "the seed corpus does not reach every language" );