diff --git a/AGENTS.md b/AGENTS.md index 3e6238e..8b2782a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ subprocess. The division of responsibility: - Python-specific classification signals - Coverage data parsing (coverage.py) - Cyclomatic complexity calculation (lifted gaze-py McCabe; no radon) -- Test-to-assertion mapping (pytest) +- Test-to-assertion mapping (pytest and unittest) - File and project discovery ## Technology Stack @@ -73,7 +73,8 @@ subprocess. The division of responsibility: - **Scope analysis**: Python `symtable` module (stdlib) for global/nonlocal detection - **Inference**: [Astroid](https://github.com/pylint-dev/astroid) `>=3.0,<4` - (shipped) for caller-count inference in `analysis/inference.py` + (shipped) for caller-count inference in `analysis/inference.py` and + strategy-3 transitive-call pairing in `quality/pairing.py` - **Complexity**: lifted gaze-py McCabe implementation; no radon dependency - **Coverage**: [coverage.py](https://github.com/nedbat/coveragepy) `>=7.0,<8` (shipped runtime dependency) for parsing `.coverage` data files @@ -99,16 +100,22 @@ snake-eyes/ │ │ ├── detector.py # Python side-effect detector (analyze method) │ │ ├── complexity.py # McCabe cyclomatic complexity (complexity method) │ │ └── inference.py # astroid caller-count inference (classify_signals) -│ └── signals/ -│ ├── __init__.py -│ ├── _routing.py # effect-type → category routing (reconstructed from gaze-py) -│ ├── _types.py # SignalResult value type -│ ├── interface.py # interface source extractor (reconstructed from gaze-py) -│ ├── visibility.py # visibility source extractor (reconstructed from gaze-py) -│ ├── caller.py # caller_count source extractor (reconstructed from gaze-py) -│ ├── naming.py # naming_convention source extractor (reconstructed from gaze-py) -│ ├── docstring.py # docstring source extractor (reconstructed from gaze-py) -│ └── adapter.py # extract_signals fan-out (classify_signals method) +│ ├── signals/ +│ │ ├── __init__.py +│ │ ├── _routing.py # effect-type → category routing (reconstructed from gaze-py) +│ │ ├── _types.py # SignalResult value type +│ │ ├── interface.py # interface source extractor (reconstructed from gaze-py) +│ │ ├── visibility.py # visibility source extractor (reconstructed from gaze-py) +│ │ ├── caller.py # caller_count source extractor (reconstructed from gaze-py) +│ │ ├── naming.py # naming_convention source extractor (reconstructed from gaze-py) +│ │ ├── docstring.py # docstring source extractor (reconstructed from gaze-py) +│ │ └── adapter.py # extract_signals fan-out (classify_signals method) +│ └── quality/ +│ ├── __init__.py # re-exports run_test_mapping +│ ├── pairing.py # test-function pairing (3 strategies, lifted from gaze-py) +│ ├── assertions.py # assertion detection & classification (lifted from gaze-py) +│ ├── mapping.py # side-effect-type inference (test_mapping method) +│ └── pipeline.py # run_test_mapping orchestration (test_mapping method) ├── tests/ ├── .github/workflows/ # CI: ruff, mypy, pytest gates ├── pyproject.toml @@ -119,6 +126,7 @@ snake-eyes/ ``` Delivered in issue #4: `detector.py`, `complexity.py`, `coverage.py`, `_shared.py`, and the `analyze`, `complexity`, and `coverage` JSON-RPC methods. Delivered in issue #5: the `signals/` extractors, `analysis/inference.py` (astroid caller-count inference), and the `classify_signals` JSON-RPC method. +Delivered in issue #6: the `quality/` package (`pairing.py`, `assertions.py`, `mapping.py`, `pipeline.py`), and the `test_mapping` JSON-RPC method. ## Shell Commands diff --git a/README.md b/README.md index 6258add..8a9a3ee 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,15 @@ Snake Eyes is a Gaze-spawned subprocess. It speaks JSON-RPC 2.0 over stdin/stdou | `complexity` | Implemented | | `coverage` | Implemented | | `classify_signals` | Implemented | - -Capability flags advertised at handshake: `discover` and -`classify_signals` are `true`; `test_mapping` and `streaming` are `false`. -Side-effect detection and classification-signal extraction are -implemented. coverage.py (`>=7.0,<8`) and astroid (`>=3.0,<4`, -caller-count inference) are runtime dependencies (shipped). `radon` is -not used — cyclomatic complexity is computed via a lifted McCabe +| `test_mapping` | Implemented | + +Capability flags advertised at handshake: `discover`, `classify_signals`, +and `test_mapping` are `true`; `streaming` is `false`. +Side-effect detection, classification-signal extraction, and +test-to-assertion mapping are implemented. coverage.py (`>=7.0,<8`) and +astroid (`>=3.0,<4`, caller-count inference and strategy-3 transitive-call +pairing in `quality/pairing.py`) are runtime dependencies (shipped). +`radon` is not used — cyclomatic complexity is computed via a lifted McCabe implementation (no radon dependency). ## Installation @@ -66,16 +68,22 @@ snake-eyes/ │ │ ├── detector.py # Python side-effect detector (analyze method) │ │ ├── complexity.py # McCabe cyclomatic complexity (complexity method) │ │ └── inference.py # astroid caller-count inference (classify_signals) -│ └── signals/ -│ ├── __init__.py -│ ├── interface.py # interface source extractor (reconstructed from gaze-py) -│ ├── visibility.py # visibility source extractor (reconstructed from gaze-py) -│ ├── caller.py # caller_count source extractor (reconstructed from gaze-py) -│ ├── naming.py # naming_convention source extractor (reconstructed from gaze-py) -│ ├── docstring.py # docstring source extractor (reconstructed from gaze-py) -│ ├── _routing.py # effect-type → category routing (reconstructed from gaze-py) -│ ├── _types.py # SignalResult value type -│ └── adapter.py # extract_signals fan-out (classify_signals method) +│ ├── signals/ +│ │ ├── __init__.py +│ │ ├── interface.py # interface source extractor (reconstructed from gaze-py) +│ │ ├── visibility.py # visibility source extractor (reconstructed from gaze-py) +│ │ ├── caller.py # caller_count source extractor (reconstructed from gaze-py) +│ │ ├── naming.py # naming_convention source extractor (reconstructed from gaze-py) +│ │ ├── docstring.py # docstring source extractor (reconstructed from gaze-py) +│ │ ├── _routing.py # effect-type → category routing (reconstructed from gaze-py) +│ │ ├── _types.py # SignalResult value type +│ │ └── adapter.py # extract_signals fan-out (classify_signals method) +│ └── quality/ +│ ├── __init__.py # re-exports run_test_mapping +│ ├── pairing.py # test-function pairing (3 strategies, lifted from gaze-py) +│ ├── assertions.py # assertion detection & classification (lifted from gaze-py) +│ ├── mapping.py # side-effect-type inference (test_mapping method) +│ └── pipeline.py # run_test_mapping orchestration (test_mapping method) ├── tests/ ├── pyproject.toml └── NOTICE @@ -85,6 +93,8 @@ Delivered in issue #4: `detector.py`, `complexity.py`, `coverage.py`, `_shared.p and the `analyze`, `complexity`, and `coverage` JSON-RPC methods. Delivered in issue #5: the `signals/` extractors, `analysis/inference.py` (astroid caller-count inference), and the `classify_signals` JSON-RPC method. +Delivered in issue #6: the `quality/` package (`pairing.py`, `assertions.py`, +`mapping.py`, `pipeline.py`), and the `test_mapping` JSON-RPC method. ## Limits & Troubleshooting @@ -109,7 +119,8 @@ invalid, the `coverage` method returns an empty result (`[]`), never an error. ## License Apache 2.0 -- see [LICENSE](LICENSE). Portions of `detector.py`, -`complexity.py`, and the `signals/` extractors (`interface.py`, `visibility.py`, -`caller.py`, `naming.py`, `docstring.py`, `_routing.py`) are lifted or -reconstructed from gaze-py under Apache-2.0; see [NOTICE](NOTICE) for +`complexity.py`, `quality/pairing.py`, `quality/assertions.py`, and the +`signals/` extractors (`interface.py`, `visibility.py`, `caller.py`, +`naming.py`, `docstring.py`, `_routing.py`) are lifted or reconstructed from +gaze-py (Copyright Matt Peter, Apache 2.0); see [NOTICE](NOTICE) for attribution. diff --git a/openspec/changes/test-mapping/.openspec.yaml b/openspec/changes/test-mapping/.openspec.yaml new file mode 100644 index 0000000..ecf3b45 --- /dev/null +++ b/openspec/changes/test-mapping/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-31 diff --git a/openspec/changes/test-mapping/design.md b/openspec/changes/test-mapping/design.md new file mode 100644 index 0000000..60e3e76 --- /dev/null +++ b/openspec/changes/test-mapping/design.md @@ -0,0 +1,127 @@ +## Context + +Snake Eyes is a JSON-RPC analyzer that Gaze spawns as a subprocess. Issues #4 (analysis-methods) and #5 (classify-signals) have already landed: the detector (`analyze_path`), discovery (`discover`), the shared helpers (`_shared.derive_package`, `iter_source_files`), the effects taxonomy (`effects.SideEffectType`, `TIER_MAP`), and the astroid-based caller inference (`analysis/inference.py`) all exist in the tree. `astroid>=3.0,<4` is already a shipped runtime dependency. + +Today `initialize` advertises `test_mapping: false` and no such method exists. Gaze's universal scoring engine wants Python test-to-assertion mapping data — which tests exercise which production functions, through which assertions, against which observable side effects — so it can compute test-quality and contract-coverage metrics on the Go side. This change implements the analyzer half of that contract. + +The behavior is fully specified by issue #6, which states every decision is already made; this document records the technical approach and the rationale behind each choice, and defers CRAP/GazeCRAP/quadrant/contract-coverage computation to Gaze. + +## Goals / Non-Goals + +**Goals:** + +- Deliver a protocol-faithful `test_mapping` JSON-RPC method returning `{"mappings": [...]}` per Gaze analyzer protocol v1.1.0. +- Pair Python tests to production functions using three strategies with first-match-wins priority and integer confidence scores. +- Emit one mapping row per assertion, classified into the six allowed `assertion_type` values, each carrying an inferred `side_effect_type`. +- Reuse existing snake-eyes infrastructure (`discover`, `analyze_path`, `derive_package`, `astroid` inference) rather than reimplementing it. +- Keep output deterministic and independently testable at the per-module coverage targets. + +**Non-Goals:** + +- Computing CRAP, GazeCRAP, quadrants, fix strategies, or contract-coverage percentages (Gaze owns scoring). +- Lifting gaze-py `quality/pipeline.py` (the pipeline is written fresh against snake-eyes models). +- Running pytest or reading coverage data during `test_mapping`. +- Modifying the signal extractors or the `classify_signals` method (independent of #5). +- Streaming responses (`streaming` stays `false`). + +## Decisions + +### Decision: New `src/snake_eyes/quality/` package with a lift/write split + +Introduce a `quality/` package containing `pairing.py` (lifted), `assertions.py` (lifted), `mapping.py` (new), `pipeline.py` (new), and `__init__.py`. Pairing and assertion detection are lifted from gaze-py and adapted to snake-eyes `ast` usage and models; the pipeline and side-effect-type inference are written fresh. + +*Rationale:* The issue grants permission to lift gaze-py's proven pairing and assertion logic (retaining copyright) but explicitly forbids lifting `quality/pipeline.py`, which is coupled to gaze-py's scoring. Splitting concerns per file keeps the lifted code isolated (with provenance headers) and the snake-eyes-specific orchestration clean and separately coverable. + +*Alternatives considered:* A single `quality/mapping.py` module — rejected because it would blur lifted-vs-original provenance and make the 95% mapping / 85% pipeline coverage split unenforceable. Placing pairing under `analysis/` — rejected; `quality/` mirrors gaze-py's package boundary and keeps test-mapping concerns cohesive. + +### Decision: Three-strategy, first-match-wins pairing with integer confidence + +For each (test_function, test_file), evaluate strategies in priority order and stop at the first match, never emitting duplicate pairs for the same `(test_function, test_file, target_package, target_function)` key: (1) name convention — strip `test_`/`Test`/`test`/`Test` prefix, exact match confidence 90, case-only match confidence 70; (2) direct AST `Call` to the target name, confidence 80; (3) astroid transitive call graph via BFS with `depth_limit=5`, confidence 75. When a single strategy matches multiple same-named targets across different packages, each distinct `(target_package, target_function)` is paired, deterministically ordered by `analyze_path` order (sorted by `(file, line, name)`). `target_package` comes from the production file's dotted module path via the shared `_shared.derive_package` helper (same as `analyze`), or equivalently `FunctionRecord.package`. + +*Rationale:* First-match-wins yields the highest-confidence relationship deterministically and avoids duplicate rows. Confidence is emitted as an integer 0–100 to match the protocol; lifted gaze-py scores on a 0.0–1.0 scale are converted at the lift boundary (0.9 → 90). + +*Alternatives considered:* Emitting all matching strategies per pair (multi-row) — rejected; the protocol expects one relationship per (test, target) and multiplies assertion rows already. Keeping float confidence — rejected; violates protocol fidelity (`confidence` is int 0–100). + +### Decision: Reuse shipped astroid for strategy 3 with graceful degradation + +Strategy 3 builds a transitive call graph with astroid (already a dependency, reused as `analysis/inference.py` does) and BFS to `depth_limit=5`. The graph is built **lazily, on the first unpaired-test lookup — once per `run_test_mapping` request and then reused** for every subsequent lookup — mirroring `analysis/inference.py`, which builds its caller index once (`build_caller_index`) and reuses it (`CallerIndex.count`) rather than rebuilding per lookup — so a project is not re-parsed once per test, and a fully-paired project (every test paired by strategies 1–2) never triggers the astroid parse at all. Callees are matched by their resolved defining **file path**, not by `derive_package`'s dotted name (under a `src/` layout `derive_package` yields `src.sample.calculator`, which never equals astroid's inferred `sample.calculator`; `analysis/inference.py` established file-path matching for exactly this reason). To stay safe in the long-lived stdio server, strategy 3 clears `astroid.MANAGER` before building the graph (per-request isolation, so one request's tree cannot leak into another's) and again in a `finally` (bounding resident memory), mirroring `analysis/inference.py`. It catches a broadened exception set — astroid errors, `RecursionError`, `MemoryError`, and an unexpected catch-all — so pathological-but-parseable input degrades (strategy 3 skipped, strategies 1–2 still returned) rather than surfacing as `-32603`; the method never fails because of astroid, and each skip/degrade emits a one-line **stderr** diagnostic (stdout stays reserved for the JSON-RPC response), mirroring `analysis/inference.py`. The broadened catch SHALL NOT absorb `FileNotFoundError` — a non-existent root must still propagate to the handler as `-32602` (discovery, not the strategy-3 catch, is the source of that error). + +*Rationale:* Constitution V (Analysis Safety) and issue #6 both require graceful degradation: ambiguity/omission of one strategy must not fail the whole method. Reusing the existing dependency adds no new supply-chain surface. + +*Alternatives considered:* Adding a new call-graph library or bumping astroid to `<5` — rejected; astroid `>=3.0,<4` already ships and suffices. Making strategy 3 mandatory — rejected; it would make CI depend on astroid successfully importing an arbitrary fixture. + +### Decision: Bounded, guarded parsing for untrusted input (Constitution V) + +The pipeline treats analyzed source as untrusted. Test files are read only through the shared guarded reader (`_shared.iter_source_files` / `is_analyzable_file`), which enforces the 16 MiB `MAX_FILE_BYTES` byte cap and skips (rather than aborts on) files that exceed it — the same reader `analyze` uses. The `MAX_AST_DEPTH` recursion budget is **not** enforced by the reader (it yields the raw tree); it is a visitor-level guard, so the pipeline's own traversals must apply it: test-function collection reuses the depth-guarded `_shared.enumerate_functions_with_spans`, and the assertion walk is depth-guarded (or catches `RecursionError`). An over-deep-but-parseable test file is therefore skipped (no rows) rather than surfacing as `-32603`, matching `analyze`'s behavior. The astroid cache lifecycle and broadened exception handling from the strategy-3 decision are implemented **inline in `quality/pairing.py`**, leaving `analysis/inference.py` untouched (issue #6 forbids `classify_signals` changes); factoring them into a shared astroid-manager lifecycle helper is an optional, behavior-preserving follow-up that must re-run the classify_signals conformance tests. + +*Rationale:* Constitution V (Analysis Safety) is non-negotiable and treats analyzed source as untrusted input; the new test-file parse path and the second astroid entry point must restate the byte-cap/AST-depth and cache-isolation/memory guards or they silently regress. The guarded reader bounds the direct test-file parse, but astroid's transitive inference can still parse imported modules that exceed the 16 MiB per-file cap (as `analysis/inference.py` documents); the `MemoryError`/`RecursionError` degrade path — not the reader — is what bounds that worst case. + +*Alternatives considered:* Bare `ast.parse` on test files — rejected; bypasses the byte-cap guard and the depth-guarded traversal and reintroduces a DoS vector. Extracting a shared astroid-manager lifecycle helper reused by `analysis/inference.py` — deferred; it would modify a `classify_signals` (#5) file, which issue #6 places out of scope, so the inline-in-`pairing.py` discipline is primary and the shared helper is an optional behavior-preserving refactor. + +### Decision: One mapping row per assertion, classified into six types + +For each paired test function, collect every assertion and emit one mapping row per assertion (a test with three asserts → three rows, same target, differing `assertion_location`). Each assertion is classified into exactly one of `equality | error_check | membership | identity | comparison | generic`, covering both pytest bare `assert` and unittest `assert*` methods. `assertion_location` is formatted `path:line` (no column), path relative to `root_path`. + +*Rationale:* Gaze needs assertion-granular data to reason about what each test verifies. The six-value closed set is the protocol contract; a `generic` fallback guarantees every assertion is representable (Detection Accuracy — ambiguity over omission). + +*Alternatives considered:* One row per test (aggregating assertions) — rejected; loses the assertion-level signal Gaze consumes. Adding assertion subtypes beyond the six — rejected; violates protocol fidelity. + +### Decision: Infer `side_effect_type` from assertion kind plus target effects + +`mapping.py` infers each row's `side_effect_type` from the assertion type and the target function's detector-reported `side_effects` (never re-parsing ad hoc): `error_check` → `ErrorReturn` if the target has it, else `ErrorSignal` if present, else `ErrorReturn`; `equality`/`comparison`/`identity`/`membership` → `ReturnValue` if present, else the target's first P0 effect, else `ReturnValue`; `generic` → the target's first effect if any, else `ReturnValue`. "First P0 effect" iterates the target's effects in detector order and selects the first whose `TIER_MAP[type]` is `Tier.P0`. + +*Rationale:* Consuming detector output keeps a single source of truth for effects and honors Python-Native Analysis. The fallback chains guarantee a valid, non-empty `side_effect_type` even when the target has no detected effects, so no row is dropped for lack of a perfect match. + +*Alternatives considered:* Re-parsing the target inside `mapping.py` — rejected; duplicates detector logic and risks divergence. Emitting a null/empty effect type — rejected; violates protocol (every row carries a `side_effect_type`). + +### Decision: Fresh `run_test_mapping` pipeline orchestrating existing components + +`pipeline.py` exposes `run_test_mapping(root_path, patterns) -> list[dict]`: (1) `discover()`; (2) `analyze_path()` on the same root/patterns to get target effects; (3) parse test files and collect `FunctionDef`/`AsyncFunctionDef` whose names start with `test_`, plus methods named `test*` of classes subclassing `unittest.TestCase`; (4) pair; (5) detect assertions; (6) infer effect types; (7) serialize protocol dicts. It never runs pytest and never reads coverage. Pairing targets are restricted to production functions defined in `discover().source_files`: because `analyze_path` enumerates both source and test files, the pipeline filters analyzed records to those whose `file` is a discovered source file, so a test function is never emitted as a pairing target. + +*Rationale:* Reuses the analyzer's proven discovery and detection paths, keeping the pipeline thin and safe (static analysis only). Returning `list[dict]` (single-valued) keeps the wrapping responsibility in one place — see the next decision. + +*Alternatives considered:* Lifting gaze-py `quality/pipeline.py` — explicitly forbidden by the issue. Re-discovering/re-reading files independently — rejected; duplicates `_shared`/`discovery` behavior and risks inconsistency. + +### Decision: Deterministic ordering with an explicit ordering assertion + +`run_test_mapping` sorts `mappings` by a stable composite key `(test_file, test_function, assertion_line, assertion_col, target_package, target_function)`, where `assertion_line` is the **integer** source line compared numerically (so line 2 sorts before line 10 — a `path:line` string key would order `:10` before `:2`) and `assertion_col` is the integer column offset (or an equivalent stable collection index) that breaks ties when several assertions share one source line, making the key a total order. Tests assert byte-identical repeat runs, byte-identical output across two separate processes launched with `PYTHONHASHSEED=0` vs `1` (the pipeline's set-based de-dup and astroid BFS make iteration order hash-seed-sensitive; a same-process repeat cannot catch this), the concrete order of rows on a fixture whose assertions span single- and double-digit lines, and a two-assertions-on-one-line case that exercises the `assertion_col` tiebreaker. + +*Rationale:* Protocol Fidelity requires deterministic output. A prior analysis-methods review learning is that a byte-identical determinism test cannot catch a wrong-but-stable sort key (e.g., a copy-pasted sort key referencing the wrong field name); an explicit ordering-value assertion on a known fixture is required in addition. + +*Alternatives considered:* Relying on discovery/AST order only — rejected; not guaranteed stable across platforms. Byte-identical determinism test alone — rejected per the learning above. + +### Decision: Single-wrap response and standard handler error mapping + +`run_test_mapping` returns `list[dict]`; only the `_test_mapping` server handler wraps it into `{"mappings": [...]}`. The handler mirrors existing handlers: it calls `_validate_analysis_params`, runs the pipeline in a try/except, and maps `FileNotFoundError` → `RpcError(INVALID_PARAMS, ...)` (-32602). `"test_mapping"` is registered in `DEFAULT_DISPATCH`, and `protocol.initialize_result()` flips `test_mapping` to `True` (updating its docstring; discover/classify_signals stay True, streaming stays False). + +*Rationale:* Matches the established handler pattern for consistency and testability, and avoids the double-wrapping defect called out in prior learnings (parse/return contracts stay single-valued; the handler is the sole wrapper). + +*Alternatives considered:* Wrapping inside the pipeline — rejected; couples orchestration to the RPC envelope and risks double-wrapping. A bespoke validation path — rejected; `_validate_analysis_params` already enforces the param contract uniformly. + +## Risks / Trade-offs + +- **Astroid cannot import/parse an arbitrary target project** → Strategy 3 is optional and wrapped in broad exception handling; strategies 1–2 still return. CI does not require strategy 3 to fire on the fixture; the BFS helper is unit-tested with a mocked graph or a tiny on-disk package astroid can parse. +- **Name-convention pairing produces false positives** → First-match-wins priority orders exact name match (90) above weaker signals; case-only matches are demoted to 70; direct-call (80) and transitive (75) provide corroboration where names differ. +- **Confidence scale confusion (0.0–1.0 vs 0–100)** → Conversion happens once at the lift boundary; tests assert `confidence` is an `int` in [0, 100] and check specific values (90/80/75/70). +- **Wrong-but-stable sort key** → Explicit ordering-value assertion on a fixture in addition to a byte-identical determinism test. +- **Target function has no detected side effects** → Effect-type inference falls back to `ReturnValue` (or `ErrorReturn` for `error_check`), guaranteeing a valid row rather than a dropped one. +- **Duplicate mapping rows** → Pairing de-duplicates by (test_function, test_file, target_package, target_function); assertion rows are intentionally distinct by `assertion_location`. +- **Scope creep toward scoring** → Non-Goals and the constitution check explicitly exclude CRAP/quadrant/coverage math; the pipeline reads no coverage and runs no pytest. +- **Astroid cache contamination / unbounded memory in the long-lived server** → Strategy 3 builds the graph once per request, clears `astroid.MANAGER` before building and in a `finally` (per-request isolation + memory bounding), implemented inline in `quality/pairing.py` (leaving `analysis/inference.py` untouched); an isolation test asserts no cross-request contamination across two requests over different trees in one process. +- **Divergence between the two inline astroid-lifecycle copies** → The clear-before/`finally`-clear + broadened-catch + stderr-diagnostic discipline is duplicated inline in `quality/pairing.py` and `analysis/inference.py` (issue #6 forbids modifying the latter). Extracting a shared astroid-manager lifecycle helper is a **tracked, deferred follow-up** (behavior-preserving, must re-run the classify_signals conformance tests); it is recorded here so the divergence risk survives archival. +- **Hash-seed-dependent iteration order** → The composite sort key uses the integer assertion line, and a cross-subprocess `PYTHONHASHSEED=0` vs `1` test asserts byte-identical output (a same-process repeat cannot detect hash-seed sensitivity). +- **Fixture test file auto-collected by the host pytest run** → `tests/fixtures/sample_project/tests/test_calculator.py` matches pytest's default collection and would `ImportError` on the fixture's package; `tests/` excludes `tests/fixtures/` from collection (`collect_ignore_glob`/`--ignore`/`norecursedirs`), and a test asserts the fixture file is not collected. +- **Unguarded parsing of untrusted test files** → Test-file reading reuses the guarded reader (`iter_source_files`/`is_analyzable_file`) for the 16 MiB byte cap; the `MAX_AST_DEPTH` budget is applied by the pipeline's own depth-guarded traversals (`enumerate_functions_with_spans` + a guarded assertion walk, or a `RecursionError` catch), so oversized and over-deep files are skipped rather than exhausting resources or surfacing as `-32603`. + +## Migration Plan + +Purely additive; no rollback data migration is required. + +1. Commit the spec artifacts (proposal, design, specs, tasks) **before** any implementation. Implementation commits MUST NOT be combined with spec-artifact commits. +2. Implement in dependency order: `quality/pairing.py` and `quality/assertions.py` (with provenance headers) → `quality/mapping.py` → `quality/pipeline.py` → `server.py` handler + dispatch → `protocol.py` capability flip → fixtures and tests. +3. Run the CI-parity gate (uv sync --locked; ruff check; ruff format --check; mypy src/; pytest --cov --cov-fail-under=85) plus a manual stdio smoke test before marking tasks complete. +4. **Rollback:** revert the change set. The only externally observable behavior change is the `test_mapping` capability flag and the new method; reverting restores `test_mapping: false` and removes the method with no residual state. + +## Open Questions + +None. Issue #6 states "Do not ask clarifying questions. Every decision is already made below," and all defaults (protocol field names, lift boundaries, no scoring math, optional strategy 3) are fixed by the issue. diff --git a/openspec/changes/test-mapping/proposal.md b/openspec/changes/test-mapping/proposal.md new file mode 100644 index 0000000..7e69b60 --- /dev/null +++ b/openspec/changes/test-mapping/proposal.md @@ -0,0 +1,60 @@ +## Why + +Gaze's universal scoring engine needs Python test-to-assertion mapping data to compute test quality and contract-coverage metrics, but snake-eyes currently advertises `test_mapping: false` and exposes no such method. This change delivers the `test_mapping` JSON-RPC method so Gaze can learn which Python tests exercise which production functions, through which assertions, against which observable side effects. + +## What Changes + +- Add a new `test_mapping` JSON-RPC method that accepts `{"root_path", "patterns"}` and returns `{"mappings": [...]}` conforming to Gaze analyzer protocol v1.1.0. Empty projects and projects with no test/target pairs return `{"mappings": []}` (a success result, never an error). +- Add a new `src/snake_eyes/quality/` package implementing a three-strategy pairing engine, assertion detection, side-effect-type inference, and an orchestration pipeline. +- Pair tests to production functions with **first-match-wins** priority: (1) name convention (`test_foo`/`testFoo`/`TestFoo`↔`foo`; strip a `test_`/`Test`/`test`/`Test` prefix; confidence 90, case-only match 70), (2) direct AST call (confidence 80), (3) astroid transitive call graph via BFS depth-limit 5 (confidence 75). Unpaired tests emit no rows. +- Emit **one mapping row per assertion**: classify each assertion into exactly one of `equality | error_check | membership | identity | comparison | generic`, and infer a `side_effect_type` from the assertion kind plus the target function's detected effects. +- Flip the `test_mapping` capability flag from `false` to `true` in `initialize`. +- Lift and adapt pairing and assertion-detection logic from gaze-py (permission granted, copyright retained); write the pipeline and effect-type inference fresh against snake-eyes models. `confidence` is emitted as an integer 0–100 (lifted 0.0–1.0 scores are converted, e.g. 0.9 → 90). +- Add a `tests/fixtures/sample_project/` fixture exercising all three pairing strategies and every assertion type. +- No **BREAKING** changes: the method is additive and no existing method, model, or capability behavior changes. + +## Capabilities + +### New Capabilities + +- `test-mapping-method`: The `test_mapping` JSON-RPC method — request/param validation, dispatch registration, `{"mappings": [...]}` response shape, error mapping, and the `initialize` capability-flag flip. +- `test-pairing`: The three-strategy, first-match-wins test-to-target pairing engine (name convention, direct call, astroid transitive call graph) with de-duplication and graceful degradation when astroid is unavailable. +- `assertion-detection`: Per-paired-test assertion collection and classification of each assertion into the six allowed `assertion_type` values across pytest and unittest styles. +- `effect-type-mapping`: Inference of `side_effect_type` for each assertion row from the assertion type plus the target function's detector-reported side effects. +- `test-mapping-pipeline`: The `run_test_mapping(root_path, patterns)` orchestration that composes discovery, analysis, pairing, assertion detection, and effect inference into deterministically ordered protocol dictionaries. + +### Modified Capabilities + +None. `openspec/specs/` holds no archived capabilities; no existing requirement changes. + +### Removed Capabilities + +None. + +## Impact + +- **New files**: + - `src/snake_eyes/quality/__init__.py` + - `src/snake_eyes/quality/pairing.py` (lifted from gaze-py `quality/pairing.py`, adapted) + - `src/snake_eyes/quality/assertions.py` (lifted from gaze-py quality assertion helpers, adapted) + - `src/snake_eyes/quality/mapping.py` (new — side-effect-type inference) + - `src/snake_eyes/quality/pipeline.py` (new — `run_test_mapping` orchestration) + - `tests/fixtures/sample_project/` (fixture: `src/sample/__init__.py`, `src/sample/calculator.py`, `tests/test_calculator.py`) + - New test modules under `tests/` covering pairing, assertions, effect inference, pipeline, and JSON-RPC end-to-end. +- **Modified files**: + - `src/snake_eyes/server.py` — add `_test_mapping` handler; register `"test_mapping"` in `DEFAULT_DISPATCH`; reuse `_validate_analysis_params`; map `FileNotFoundError` → `INVALID_PARAMS` (-32602). + - `src/snake_eyes/protocol.py` — flip `test_mapping` to `True` in `initialize_result()` and update its docstring (discover, classify_signals, test_mapping True; streaming False). + - `tests/conftest.py` (or `[tool.pytest.ini_options]`) — exclude `tests/fixtures/` from host pytest collection so the `sample_project` test file is analyzed as fixture input, not run by the host test suite. + - `README.md` — add a `test_mapping | Implemented` status-table row; flip the capability sentence so only `streaming` is `false`; update the implemented-features prose line to include test-to-assertion mapping; add `src/snake_eyes/quality/` (with its four modules `pairing.py`, `assertions.py`, `mapping.py`, `pipeline.py`) to the project-structure tree; enumerate `pairing.py` and `assertions.py` in the License section as lifted from gaze-py; update the astroid dependency line so it credits strategy-3 transitive-call pairing in addition to caller-count inference; add a "Delivered in issue #6" note. + - `AGENTS.md` — add `src/snake_eyes/quality/` (with its four modules `pairing.py`, `assertions.py`, `mapping.py`, `pipeline.py`) to the Project Structure section; update the Technology Stack "Inference" bullet so astroid is credited for strategy-3 transitive-call pairing in `quality/pairing.py` in addition to caller-count inference; broaden the Architecture "Test-to-assertion mapping (pytest)" line to "(pytest and unittest)"; note delivery in issue #6. +- **Dependencies**: None added. `astroid>=3.0,<4` is already a runtime dependency (shipped by the classify-signals change) and is reused for strategy-3 transitive pairing exactly as `analysis/inference.py` uses it. `coverage.py` is untouched — this change does not run pytest or read coverage data. +- **Provenance**: `pairing.py` and `assertions.py` retain the gaze-py copyright header (`# Copyright Matt Peter (gaze-py, https://github.com/mpeter/gaze-py). Apache 2.0.`) plus an Apache-2.0 §4(b) change notice. gaze-py `quality/pipeline.py` is explicitly **not** lifted. `NOTICE` already attributes gaze-py. +- **Protocol contract**: Consumes detector output (`analyze_path` → `FunctionRecord.side_effects`) and `discover()`; produces `{"mappings": [...]}`. snake-eyes still computes no CRAP/GazeCRAP/quadrants/contract-coverage percentages — scoring remains Gaze's responsibility. + +## Constitution Alignment + +- **I. Protocol Fidelity — PASS**: Field names, the six `assertion_type` values, integer `confidence` (0–100), `assertion_location` `path:line` format, and the `{"mappings": [...]}` envelope match Gaze analyzer protocol v1.1.0 exactly. Output is deterministically ordered by a stable key. Empty results return a success `{"mappings": []}`, never an error. +- **II. Detection Accuracy — PASS**: Three pairing strategies maximize true pairs; ambiguity is preferred over omission via a `generic` assertion fallback and an effect-type fallback chain, so uncertain-but-real relationships are still reported rather than silently dropped. +- **III. Python-Native Analysis — PASS**: Pairing and assertion detection use the stdlib `ast`; transitive pairing reuses `astroid` (already shipped) as `analysis/inference.py` does. No Python semantics are reimplemented. +- **IV. Testability — PASS**: Each module is independently testable; per-file coverage targets are pairing 90%+, assertions 90%+, mapping 95%+, pipeline 85%+, with the project gate held at 85%. A protocol-conformance end-to-end test asserts `test_mapping: true` and the response shape. +- **V. Analysis Safety — PASS**: Analyzed source is parsed statically only — the pipeline never executes analyzed code and never runs pytest. Test files are parsed through the shared guarded reader (`iter_source_files`/`is_analyzable_file`), which enforces the 16 MiB byte cap; because `MAX_AST_DEPTH` is a visitor-level guard rather than reader-enforced, the pipeline's own traversals enforce it (reusing the depth-guarded `enumerate_functions_with_spans` for test-function collection and a depth-guarded assertion walk, or catching `RecursionError`); oversized or over-deep files are skipped. Strategy-3 astroid clears `astroid.MANAGER` before building and in a `finally` (per-request isolation + memory bounding) and catches a broadened exception set (astroid errors, `RecursionError`, `MemoryError`, and an unexpected catch-all) so pathological input degrades to strategies 1–2 rather than surfacing as an internal error (-32603); `FileNotFoundError` is not absorbed and propagates to -32602. These guards are encoded as requirements, scenarios, and tasks, not asserted in prose alone. diff --git a/openspec/changes/test-mapping/specs/assertion-detection/spec.md b/openspec/changes/test-mapping/specs/assertion-detection/spec.md new file mode 100644 index 0000000..2dd9b53 --- /dev/null +++ b/openspec/changes/test-mapping/specs/assertion-detection/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: One mapping row per assertion + +For each paired test function, the detector SHALL collect every assertion in the test body and produce one mapping row per assertion. Rows from the same test share the same `target_function` and `target_package` but carry distinct `assertion_location` values. + +#### Scenario: Test with multiple assertions yields multiple rows +- **WHEN** a paired test function contains three assertions +- **THEN** three mapping rows are produced, all with the same target but different `assertion_location` values + +### Requirement: Assertion node identification + +The detector SHALL identify assertion nodes within a paired test's body as: every `ast.Assert` statement; every call whose callee is a unittest `assert*` method (e.g. `self.assertEqual(...)`) or a pytest/bare `raises`/`warns` callable; and every `with` item using `pytest.raises(...)`, `raises(...)`, or `self.assertRaises(...)`. Traversal SHALL descend into nested `with`, `for`, `if`, `while`, and `try` blocks of the test body, but SHALL NOT descend into nested function or class definitions. A `raises`/`warns`/`assertRaises` call that appears as the context expression of a `with` item SHALL be counted once — as the `with` item — and SHALL NOT additionally be counted as a bare call. Each identified node SHALL produce exactly one mapping row. + +#### Scenario: A pytest.raises context manager counts as one assertion +- **WHEN** a paired test body contains `with pytest.raises(ValueError): ...` +- **THEN** exactly one assertion row is produced with `assertion_type` `error_check` + +### Requirement: Assertion type classification + +Each assertion SHALL be classified into exactly one `assertion_type` from the closed set `equality | error_check | membership | identity | comparison | generic`, supporting both pytest bare-`assert` expressions and unittest `assert*` methods, according to a fixed classification table that is exhaustive over recognized pytest and unittest forms. Any recognized assertion form not matched by a more specific rule SHALL be classified as `generic`. + +#### Scenario: Equality assertions +- **WHEN** an assertion is `assert x == y`, `assertEqual(...)`, `assertEquals(...)`, `assertAlmostEqual(...)`, `assertDictEqual(...)`, `assertListEqual(...)`, `assertMultiLineEqual(...)`, `assertCountEqual(...)`, or `assertSequenceEqual(...)` +- **THEN** its `assertion_type` is `equality` + +#### Scenario: Comparison assertions +- **WHEN** an assertion is `assert x != y`, `assert x < y` / `>` / `<=` / `>=`, `assertNotEqual(...)`, `assertNotAlmostEqual(...)`, or an `assertLess*`/`assertGreater*` method +- **THEN** its `assertion_type` is `comparison` + +#### Scenario: Identity assertions +- **WHEN** an assertion is `assert x is y`, `assert x is not y`, `assertIs(...)`, `assertIsNot(...)`, `assertIsNone(...)`, or `assertIsNotNone(...)` +- **THEN** its `assertion_type` is `identity` + +#### Scenario: Membership assertions +- **WHEN** an assertion is `assert x in y`, `assert x not in y`, `assertIn(...)`, or `assertNotIn(...)` +- **THEN** its `assertion_type` is `membership` + +#### Scenario: Error-check assertions +- **WHEN** an assertion uses `pytest.raises(...)`, `unittest`'s `assertRaises(...)`, `assertRaisesRegex(...)`, `assertRaisesRegexp(...)`, `pytest.warns(...)`, or a bare `raises(...)` +- **THEN** its `assertion_type` is `error_check` + +#### Scenario: Generic fallback +- **WHEN** an assertion is any other `assert`, `assertTrue(...)`, or `assertFalse(...)` +- **THEN** its `assertion_type` is `generic` + +### Requirement: Assertion location format + +Each assertion row SHALL record `assertion_location` as `path:line`, where `path` is relative to `root_path` and `line` is the assertion's source line; no column component is required. + +#### Scenario: Location is root-relative path plus line +- **WHEN** an assertion occurs at line 10 of `tests/test_ops.py` +- **THEN** `assertion_location` is `tests/test_ops.py:10` diff --git a/openspec/changes/test-mapping/specs/effect-type-mapping/spec.md b/openspec/changes/test-mapping/specs/effect-type-mapping/spec.md new file mode 100644 index 0000000..4233539 --- /dev/null +++ b/openspec/changes/test-mapping/specs/effect-type-mapping/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: Effect-type inference consumes detector output + +For each assertion row, the mapping module SHALL infer `side_effect_type` from the assertion type together with the target function's side effects as reported by the detector (`FunctionRecord.side_effects`). It SHALL NOT re-parse the target function ad hoc. "First P0 effect" means the target's first side effect, in detector order, whose tier in `TIER_MAP` is `P0`. + +#### Scenario: Inference reads detected effects, not a fresh parse +- **WHEN** inferring `side_effect_type` for a row whose target function was analyzed +- **THEN** the inference uses that target's `side_effects` from the detector output rather than re-parsing the source + +### Requirement: Error-check effect inference + +For an `error_check` assertion, `side_effect_type` SHALL be `ErrorReturn` if the target has an `ErrorReturn` effect; otherwise `ErrorSignal` if the target has an `ErrorSignal` effect; otherwise `ErrorReturn` as a fallback. + +#### Scenario: Target has ErrorReturn +- **WHEN** the assertion type is `error_check` and the target's effects include `ErrorReturn` +- **THEN** `side_effect_type` is `ErrorReturn` + +#### Scenario: Target has ErrorSignal but not ErrorReturn +- **WHEN** the assertion type is `error_check`, the target has no `ErrorReturn`, but has `ErrorSignal` +- **THEN** `side_effect_type` is `ErrorSignal` + +#### Scenario: Target has neither error effect +- **WHEN** the assertion type is `error_check` and the target has neither `ErrorReturn` nor `ErrorSignal` +- **THEN** `side_effect_type` falls back to `ErrorReturn` + +### Requirement: Value-assertion effect inference + +For an `equality`, `comparison`, `identity`, or `membership` assertion, `side_effect_type` SHALL be `ReturnValue` if the target has a `ReturnValue` effect; otherwise the target's first P0 effect; otherwise `ReturnValue` as a fallback. + +#### Scenario: Target returns a value +- **WHEN** the assertion type is `equality` and the target's effects include `ReturnValue` +- **THEN** `side_effect_type` is `ReturnValue` + +#### Scenario: Target has no ReturnValue but has a P0 effect +- **WHEN** the assertion type is `comparison`, the target has no `ReturnValue`, but has at least one P0 effect +- **THEN** `side_effect_type` is the target's first P0 effect + +#### Scenario: Target has no usable effect +- **WHEN** the assertion type is `membership` and the target has no `ReturnValue` and no P0 effect +- **THEN** `side_effect_type` falls back to `ReturnValue` + +### Requirement: Generic effect inference + +For a `generic` assertion, `side_effect_type` SHALL be the target's first side effect if the target has any; otherwise `ReturnValue`. + +#### Scenario: Target has at least one effect +- **WHEN** the assertion type is `generic` and the target has one or more side effects +- **THEN** `side_effect_type` is the target's first side effect + +#### Scenario: Target has no effects +- **WHEN** the assertion type is `generic` and the target has no side effects +- **THEN** `side_effect_type` falls back to `ReturnValue` diff --git a/openspec/changes/test-mapping/specs/test-mapping-method/spec.md b/openspec/changes/test-mapping/specs/test-mapping-method/spec.md new file mode 100644 index 0000000..a17905e --- /dev/null +++ b/openspec/changes/test-mapping/specs/test-mapping-method/spec.md @@ -0,0 +1,61 @@ +## ADDED Requirements + +### Requirement: test_mapping JSON-RPC method + +The server SHALL expose a `test_mapping` JSON-RPC method that accepts a params object of the form `{"root_path": , "patterns": }` and returns a result object `{"mappings": [...]}` conforming to Gaze analyzer protocol v1.1.0. The method SHALL be registered in the server dispatch table under the key `"test_mapping"`. + +#### Scenario: Valid request returns a mappings envelope +- **WHEN** a `test_mapping` request is dispatched with a valid `root_path` and `patterns` +- **THEN** the result is an object whose sole top-level key is `mappings`, and its value is a JSON array + +#### Scenario: Project with no tests returns empty mappings +- **WHEN** a `test_mapping` request targets a root that contains no test files +- **THEN** the result is `{"mappings": []}` returned as a success result, not a JSON-RPC error + +#### Scenario: Project with no test/target pairs returns empty mappings +- **WHEN** test files exist but none can be paired to a production function +- **THEN** the result is `{"mappings": []}` returned as a success result, not a JSON-RPC error + +### Requirement: Capability advertisement + +The `initialize` result SHALL advertise `test_mapping` as `true`, while leaving `discover` and `classify_signals` as `true` and `streaming` as `false`. + +#### Scenario: initialize reports test_mapping enabled +- **WHEN** an `initialize` request is handled +- **THEN** the returned `capabilities` object contains `test_mapping: true`, `discover: true`, `classify_signals: true`, and `streaming: false` + +### Requirement: Parameter validation and error mapping + +The `test_mapping` handler SHALL validate params using the shared analysis-params validator and SHALL return JSON-RPC error code `INVALID_PARAMS` (-32602) when params are not an object, `root_path` is not a string, `patterns` is not a list of strings, or the resolved root path does not exist. + +#### Scenario: Missing root_path is rejected +- **WHEN** a `test_mapping` request omits `root_path` or provides a non-string `root_path` +- **THEN** the server responds with JSON-RPC error code -32602 (INVALID_PARAMS) + +#### Scenario: Non-existent root path is rejected +- **WHEN** a `test_mapping` request provides a `root_path` that is not an existing directory +- **THEN** the pipeline's `FileNotFoundError` is mapped to JSON-RPC error code -32602 (INVALID_PARAMS) + +### Requirement: Mapping row field contract + +Each element of `mappings` SHALL be an object containing the keys `test_function`, `test_file`, `assertion_location`, `assertion_type`, `target_function`, `target_package`, `side_effect_type`, and `confidence`. `confidence` SHALL be an integer in the inclusive range 0–100. `assertion_type` SHALL be exactly one of `equality`, `error_check`, `membership`, `identity`, `comparison`, or `generic`. `assertion_location` SHALL be formatted `path:line` with the path relative to `root_path`. `test_file` and every path-valued field SHALL be a POSIX path relative to `root_path`, consistent with `assertion_location`, so that output is independent of the absolute location of the analyzed project. For a test defined as a method of a `unittest.TestCase` subclass, `test_function` SHALL be class-qualified as `ClassName.method_name` (e.g. `T.test_x`); for a top-level test function it SHALL be the bare function name. This keeps the de-duplication key unambiguous when two same-named `test_*` methods are defined in different classes within one file. + +#### Scenario: Row carries all required keys with correct types +- **WHEN** a mapping row is produced for a paired test assertion +- **THEN** the row contains every required key, `confidence` is an integer in [0, 100], and `assertion_type` is one of the six allowed values + +#### Scenario: assertion_location is a root-relative path with a line number +- **WHEN** a mapping row references an assertion at a given source line +- **THEN** `assertion_location` equals `:` with no column component + +#### Scenario: test_file is a root-relative path +- **WHEN** a mapping row references a test defined under `root_path` +- **THEN** `test_file` is a POSIX path relative to `root_path` with no absolute prefix + +### Requirement: Supersedes prior capability assertions on archive + +Because OpenSpec archival is sequential, the latest archived `initialize` snapshot governs the effective capability set. This change's `test_mapping: true` SHALL supersede the `test_mapping: false` assertions carried by the prior unarchived changes (`scaffold-and-protocol`, `taxonomy-and-discovery`, `analysis-methods`, and `classify-signals`). This reconciliation is recorded so that archiving the full chain does not leave conflicting `initialize` capability scenarios in `openspec/specs/`. + +#### Scenario: Latest archived capability snapshot wins +- **WHEN** this change is archived after `classify-signals` +- **THEN** the governing `initialize` capability snapshot advertises `test_mapping: true`, superseding every prior `test_mapping: false` assertion diff --git a/openspec/changes/test-mapping/specs/test-mapping-pipeline/spec.md b/openspec/changes/test-mapping/specs/test-mapping-pipeline/spec.md new file mode 100644 index 0000000..eb0b423 --- /dev/null +++ b/openspec/changes/test-mapping/specs/test-mapping-pipeline/spec.md @@ -0,0 +1,69 @@ +## ADDED Requirements + +### Requirement: Orchestration entry point + +The pipeline SHALL expose `run_test_mapping(root_path, patterns) -> list[dict]` that composes, in order: file discovery, target analysis (to obtain side effects on targets), test-function collection, pairing, assertion detection, effect-type inference, and serialization into protocol mapping dictionaries. + +#### Scenario: Pipeline over the sample project yields mapping rows +- **WHEN** `run_test_mapping` runs over the `sample_project` fixture +- **THEN** it returns at least two mapping rows, each containing all required protocol keys + +### Requirement: Test-function collection + +The pipeline SHALL collect as test functions all `FunctionDef` and `AsyncFunctionDef` nodes whose names start with `test_`, plus methods whose names start with `test` that are defined on classes subclassing `unittest.TestCase`. A class is recognized as a `unittest.TestCase` subclass by a shallow match on its base names (`TestCase` or `unittest.TestCase`); transitively- or alias-derived subclasses are a documented known limitation (an acknowledged false negative). + +#### Scenario: Pytest-style and unittest-style tests are both collected +- **WHEN** a test file contains a top-level `def test_add(): ...` and a `class T(unittest.TestCase)` with a `def test_x(self): ...` method +- **THEN** both `test_add` and `T.test_x` are collected as test functions + +### Requirement: Pairing targets restricted to source files + +The pipeline SHALL treat only production functions defined in `discover().source_files` as pairing targets; functions defined in test files SHALL NOT be pairing targets. Because `analyze_path` enumerates both source and test files, the pipeline SHALL filter analyzed `FunctionRecord`s to those whose `file` is a discovered source file before pairing. + +#### Scenario: Test-module helper is never a target +- **WHEN** a test module defines a helper named `add` and a production module also defines `add` +- **THEN** tests pair only to the production `add`, and the test-module `add` is never emitted as a `target_function` + +### Requirement: Bounded, guarded parsing of untrusted test files + +The pipeline SHALL read test files through the shared guarded reader (`iter_source_files` / `is_analyzable_file`), which enforces the `MAX_FILE_BYTES` (16 MiB) byte cap; a test file exceeding the byte cap SHALL be skipped. The `MAX_AST_DEPTH` recursion budget is a visitor-level guard (not enforced by the reader), so the pipeline's own traversals SHALL enforce it: test-function collection SHALL reuse the depth-guarded `enumerate_functions_with_spans`, and the assertion walk SHALL be depth-guarded or SHALL catch `RecursionError`. An over-deep-but-parseable test file SHALL be skipped (yielding no rows) rather than aborting the request, and no untrusted input SHALL surface as an internal error (-32603). + +#### Scenario: Oversized or over-deep test file is skipped +- **WHEN** a test file exceeds the byte cap or AST-depth budget +- **THEN** the pipeline skips that file and continues, producing rows for the remaining files without raising an internal error + +### Requirement: Static-analysis-only execution + +The pipeline SHALL perform static analysis only: it SHALL NOT execute analyzed code, SHALL NOT run pytest, and SHALL NOT read coverage data. + +#### Scenario: No execution or coverage access during mapping +- **WHEN** `run_test_mapping` processes a project +- **THEN** it parses source statically and never invokes pytest, never imports/executes analyzed modules, and never reads `.coverage`/coverage.json + +### Requirement: Deterministic ordering + +The pipeline SHALL order `mappings` by a stable composite key `(test_file, test_function, assertion_line, assertion_col, target_package, target_function)`, where `assertion_line` is the integer source line of the assertion compared numerically (so line 2 sorts before line 10, which a `path:line` string would not) and `assertion_col` is the integer column offset (or an equivalent stable collection index) that breaks ties when several assertions share one source line, making the key a total order. Output SHALL be byte-identical across repeated runs on the same input, including across separate processes launched with different `PYTHONHASHSEED` values. + +#### Scenario: Repeated runs are byte-identical +- **WHEN** `run_test_mapping` is executed twice on the same unchanged project +- **THEN** the two serialized results are byte-identical + +#### Scenario: Output is stable across hash seeds +- **WHEN** the server serializes `test_mapping` output in two separate processes launched with `PYTHONHASHSEED=0` and `PYTHONHASHSEED=1` over the same project +- **THEN** the two serialized results are byte-identical + +#### Scenario: Rows appear in numeric-line order on a fixture +- **WHEN** `run_test_mapping` runs over a fixture whose assertions span single- and double-digit lines (e.g. lines 2 and 10) +- **THEN** the rows appear in ascending order of the composite key with the line compared numerically, asserted against the expected concrete ordering (not only byte-equality) + +### Requirement: Single-valued return contract + +`run_test_mapping` SHALL return a `list[dict]` (an empty list when there are no mappings). Wrapping the list into the `{"mappings": [...]}` response envelope SHALL be performed only by the server handler, not by the pipeline. + +#### Scenario: Pipeline returns a bare list +- **WHEN** `run_test_mapping` completes +- **THEN** it returns a `list[dict]` and does not itself wrap the result in a `{"mappings": ...}` object + +#### Scenario: Empty project returns an empty list +- **WHEN** `run_test_mapping` runs over a project with no pairable tests +- **THEN** it returns `[]` diff --git a/openspec/changes/test-mapping/specs/test-pairing/spec.md b/openspec/changes/test-mapping/specs/test-pairing/spec.md new file mode 100644 index 0000000..66ed685 --- /dev/null +++ b/openspec/changes/test-mapping/specs/test-pairing/spec.md @@ -0,0 +1,69 @@ +## ADDED Requirements + +### Requirement: Name-convention pairing (Priority 1) + +The pairing engine SHALL pair a test function to a production function by name convention, stripping a leading `test_`/`Test` (or `test`/`Test` camel prefix) from the test name and matching the remainder to a production function name. An exact-name match SHALL yield confidence 90; a match that differs only by letter case SHALL yield confidence 70. + +#### Scenario: Exact name match +- **WHEN** a test named `test_add` is evaluated against a production function `add` +- **THEN** they are paired with `confidence` 90 + +#### Scenario: Case-only match +- **WHEN** a test named `test_Add` is evaluated against a production function `add` and no exact-case match exists +- **THEN** they are paired with `confidence` 70 + +### Requirement: Direct-call pairing (Priority 2) + +When name convention does not pair a test, the engine SHALL pair the test to a production function whose name appears as a direct `Call` in the test function's AST, yielding confidence 80. + +#### Scenario: Test directly calls the target +- **WHEN** a test named `test_it_works` contains a call to `divide` and does not name-match any production function +- **THEN** it is paired to `divide` with `confidence` 80 + +### Requirement: Transitive-call pairing (Priority 3) + +When neither name convention nor direct call pairs a test, the engine SHALL attempt to pair via an astroid transitive call graph using breadth-first search with a depth limit of 5, yielding confidence 75. The transitive call graph SHALL be built once per `test_mapping` request and reused for every unpaired-test lookup (mirroring `build_caller_index`/`CallerIndex.count` in `analysis/inference.py`), never rebuilt per test. Callees SHALL be matched by their resolved defining file path (not by a `derive_package` dotted name, which under a `src/` layout would not equal astroid's inferred module name). To preserve per-request isolation and bound memory in the long-lived stdio server, the engine SHALL clear `astroid.MANAGER` before building the graph and again in a `finally` block, mirroring `analysis/inference.py`. This strategy SHALL degrade gracefully: it SHALL catch the broadened exception set (astroid errors, `RecursionError`, `MemoryError`, and an unexpected-error catch-all) and, on any such error, skip strategy 3 while still returning results from strategies 1 and 2 — no strategy-3 failure SHALL surface as an internal error (-32603). Each skip or degrade SHALL emit a one-line diagnostic to stderr (stdout is reserved for JSON-RPC), mirroring `analysis/inference.py`. The broadened catch SHALL NOT absorb `FileNotFoundError`, which propagates so a non-existent root is reported as -32602 by discovery, not by the strategy-3 catch. + +#### Scenario: Transitive call within depth limit +- **WHEN** a test calls a helper that (within 5 hops) calls the target production function, and no earlier strategy matched +- **THEN** the test is paired to the target with `confidence` 75 + +#### Scenario: Target beyond the depth limit is not paired by strategy 3 +- **WHEN** the target production function is reachable from the test only through a call chain longer than 5 hops +- **THEN** strategy 3 does not pair the test to that target + +#### Scenario: Astroid unavailable degrades without failing +- **WHEN** astroid cannot import or parse the analyzed project during strategy 3 +- **THEN** strategy 3 is skipped and any pairs found by strategies 1 and 2 are still returned + +#### Scenario: Astroid cache is isolated per request +- **WHEN** strategy 3 runs across two `test_mapping` requests over different project trees within the same long-lived process +- **THEN** the astroid manager cache is cleared before building each request's graph and released afterward, so inference from one tree cannot contaminate the other + +#### Scenario: Pathological input degrades without an internal error +- **WHEN** strategy 3 raises a `RecursionError`, `MemoryError`, or any unexpected error on pathological-but-parseable input +- **THEN** the strategy is skipped, strategies 1 and 2 results are returned, and no internal error (-32603) surfaces + +### Requirement: First-match-wins and de-duplication + +The engine SHALL evaluate strategies in priority order and stop at the first strategy that pairs a given test, and SHALL NOT emit duplicate pairs for the same `(test_function, test_file, target_package, target_function)` key. When a single strategy matches several same-named production functions across different packages, the engine SHALL pair each distinct `(target_package, target_function)` deterministically in `analyze_path` order (sorted by `(file, line, name)`). Unpaired tests SHALL produce no mapping rows (no confidence-0 rows). + +#### Scenario: Highest-priority strategy wins +- **WHEN** a test both name-matches and directly calls the same target +- **THEN** a single pair is produced using the name-convention confidence (90), not a duplicate + +#### Scenario: Unpaired test produces no rows +- **WHEN** a test neither name-matches nor calls (directly or transitively) any production function +- **THEN** no mapping row is emitted for that test + +#### Scenario: Same-named targets in different packages are disambiguated +- **WHEN** a test name-matches a function `add` that is defined in two different packages +- **THEN** a distinct mapping row is emitted for each `(target_package, target_function)`, ordered deterministically in `analyze_path` order + +### Requirement: Target package derivation + +The engine SHALL derive `target_package` from the production file's dotted module path using the shared `derive_package` helper (the same helper used by the `analyze` method); equivalently, it MAY read `FunctionRecord.package`, which already carries the same `derive_package` value from analysis. + +#### Scenario: Package derived from module path +- **WHEN** a target function is defined in `src/sample/calculator.py` +- **THEN** `target_package` is the dotted module path produced by `derive_package` for that file diff --git a/openspec/changes/test-mapping/tasks.md b/openspec/changes/test-mapping/tasks.md new file mode 100644 index 0000000..ed5c1a6 --- /dev/null +++ b/openspec/changes/test-mapping/tasks.md @@ -0,0 +1,95 @@ +## 1. Package scaffold & dependencies + +- [x] 1.1 Create the `src/snake_eyes/quality/` package with an `__init__.py` (re-export `run_test_mapping` for ergonomic imports, matching the `analysis/`/`signals/` convention) +- [x] 1.2 Confirm `astroid>=3.0,<4` is already present in `pyproject.toml` (shipped by classify-signals) and add NO new dependency; do NOT bump the astroid pin to `<5` +- [x] 1.3 Confirm `NOTICE` already attributes gaze-py; no NOTICE change required unless a new lifted file needs a distinct attribution line + +## 2. Pairing engine (capability: test-pairing) + +- [x] 2.1 Lift gaze-py `quality/pairing.py` to `src/snake_eyes/quality/pairing.py`, retaining the gaze-py copyright header and adding the Apache-2.0 §4(b) change notice; annotate to satisfy `mypy --strict` and mark the astroid import `# type: ignore[import-untyped]` as `analysis/inference.py` does +- [x] 2.2 Adapt the lifted code to snake-eyes stdlib `ast` usage and models; convert lifted 0.0–1.0 confidence scores to integer 0–100 (e.g. 0.9 → 90) +- [x] 2.3 Implement Priority 1 name-convention pairing: strip a leading `test_`/`Test`/`test`/`Test` (snake or camel) prefix, exact match confidence 90, case-only match confidence 70; define the case-only tie-break as the first candidate in `analyze_path` order +- [x] 2.4 Implement Priority 2 direct-call pairing: target name appears as a `Call` in the test AST, confidence 80 +- [x] 2.5 Implement Priority 3 transitive pairing: astroid call-graph BFS with `depth_limit=5`, confidence 75, reusing the astroid approach from `analysis/inference.py`; build the call graph ONCE per `run_test_mapping` — lazily, on the first unpaired-test lookup — and reuse it for all subsequent lookups (mirror `build_caller_index` built once + `CallerIndex.count` reuse; no per-test rebuild; a fully-paired project never triggers the astroid parse); match callees by resolved defining file path (not `derive_package`'s dotted name); clear `astroid.MANAGER` before that single build and again in a `finally` (per-request isolation + memory bounding); catch a broadened exception set (astroid errors, `RecursionError`, `MemoryError`, catch-all) so it skips gracefully with no method failure, WITHOUT absorbing `FileNotFoundError`; emit a one-line stderr diagnostic on skip/degrade (stdout stays clean for JSON-RPC) +- [x] 2.6 Enforce first-match-wins ordering and de-duplicate pairs by `(test_function, test_file, target_package, target_function)`; when one strategy matches multiple same-named targets across packages, pair each distinct `(target_package, target_function)` deterministically in `analyze_path` order; emit no rows for unpaired tests +- [x] 2.7 Derive `target_package` via `_shared.derive_package` (same helper as `analyze`), or equivalently read `FunctionRecord.package` +- [x] 2.8 Implement the astroid manager lifecycle (clear-cache on entry + `finally`) INLINE in `quality/pairing.py`, leaving `analysis/inference.py` UNTOUCHED (issue #6 scope: no classify_signals changes). A shared helper is OPTIONAL only; if extracted it MUST be behavior-preserving and the classify_signals conformance tests MUST be re-run + +## 3. Assertion detection (capability: assertion-detection) + +- [x] 3.1 Lift the gaze-py assertion helpers to `src/snake_eyes/quality/assertions.py`, retaining copyright header and adding the Apache-2.0 §4(b) change notice; annotate for `mypy --strict` +- [x] 3.2 Define the assertion node set (`ast.Assert`; `Call`s whose callee ∈ the `assert*`/`raises`/`warns` set; `with pytest.raises(...)`/`raises(...)`/`self.assertRaises(...)`) and traversal scope (test body incl. nested `with`/`for`/`if`/`try`, excluding nested def/class bodies); a `raises`/`warns`/`assertRaises` call used as a `with`-item context expression is counted ONCE (as the with-item), never additionally as a bare call +- [x] 3.3 Implement the exhaustive classification table mapping pytest bare `assert` and every recognized unittest `assert*` form to exactly one of `equality | error_check | membership | identity | comparison | generic` (incl. `assertIsNot`/`assertIsNotNone` → identity, `assertRaisesRegex`/`pytest.warns` → error_check, `assert*Equal` family → equality, `assertNotIn`/`not in` → membership); anything unlisted → `generic`; classify by an explicit name-keyed map (NOT an `endswith("Equal")` heuristic) so `assertNotEqual`/`assertNotAlmostEqual` resolve to `comparison`, not the equality family +- [x] 3.4 Collect every assertion in a paired test and emit one row per assertion with `assertion_location` formatted `path:line` relative to `root_path` + +## 4. Effect-type inference (capability: effect-type-mapping) + +- [x] 4.1 Create `src/snake_eyes/quality/mapping.py` that consumes the target's `FunctionRecord.side_effects` from the detector (no ad hoc re-parsing); look up tiers via `TIER_MAP.get(...)` with a safe fallback (no `KeyError` on an unexpected type string) +- [x] 4.2 Implement the `error_check` chain: `ErrorReturn` if present, else `ErrorSignal` if present, else `ErrorReturn` +- [x] 4.3 Implement the value-assertion chain (`equality`/`comparison`/`identity`/`membership`): `ReturnValue` if present, else first P0 effect (via `effects.TIER_MAP`), else `ReturnValue` +- [x] 4.4 Implement the `generic` chain: first detected effect if any, else `ReturnValue` + +## 5. Pipeline (capability: test-mapping-pipeline) + +- [x] 5.1 Create `src/snake_eyes/quality/pipeline.py` exposing `run_test_mapping(root_path, patterns) -> list[dict]` +- [x] 5.2 Compose the stages: `discover()`, `analyze_path()` for target effects, then test-function collection +- [x] 5.3 Restrict pairing targets to production functions defined in `discover().source_files` (filter `analyze_path` records by `file ∈ source_files`); test functions are never targets +- [x] 5.4 Collect test functions: `FunctionDef`/`AsyncFunctionDef` named `test_*`, plus `test*` methods of `unittest.TestCase` subclasses (detected by a shallow scan of each top-level `ClassDef`'s base names for `TestCase`/`unittest.TestCase`, mirroring the detector's Exception-base matching — a step that complements `enumerate_functions_with_spans`, which yields only name/span and no class context; transitive/aliased subclasses are a documented known limitation, i.e. an acknowledged false negative); reuse the depth-guarded `enumerate_functions_with_spans` for function enumeration +- [x] 5.5 Parse test files through the shared guarded reader (`iter_source_files`/`is_analyzable_file`), which provides the `MAX_FILE_BYTES` byte cap only; enforce the `MAX_AST_DEPTH` budget in the pipeline's OWN traversals (the reader does not — depth is a visitor-level guard): use the depth-guarded `enumerate_functions_with_spans` for collection and guard the assertion walk (or catch `RecursionError`) so an over-deep-but-parseable test file is SKIPPED without failing the request or surfacing as -32603 +- [x] 5.6 Run pairing → assertion detection → effect-type inference → serialize to protocol dicts +- [x] 5.7 Sort `mappings` by the composite key `(test_file, test_function, assertion_line, assertion_col, target_package, target_function)` comparing the line numerically (line 2 before line 10), where `assertion_col` is the integer column offset (or an equivalent stable collection index) that breaks ties when multiple assertions share one source line to the same target, making the key a total order; emit `test_file` and all path-valued fields root-relative POSIX; return `[]` when empty +- [x] 5.8 Ensure the pipeline runs no pytest, executes no analyzed code, and reads no coverage data + +## 6. Server & protocol wiring (capability: test-mapping-method) + +- [x] 6.1 Add a `_test_mapping` handler in `server.py` that calls `_validate_analysis_params`, runs `run_test_mapping`, and wraps the list into `{"mappings": [...]}` +- [x] 6.2 Map `FileNotFoundError` from the pipeline to `RpcError(INVALID_PARAMS, ...)` (-32602), mirroring existing handlers +- [x] 6.3 Register `"test_mapping"` in `DEFAULT_DISPATCH` +- [x] 6.4 Flip `test_mapping` to `True` in `protocol.initialize_result()` and update its docstring (discover, classify_signals, test_mapping True; streaming False) +- [x] 6.5 Update existing capability assertions to `test_mapping: True`: `tests/test_classify_signals_method.py` (`caps["test_mapping"]`), `tests/test_discover_method.py`, and the exact-dict assertion in `tests/test_protocol.py`; and remove `test_mapping` from the `test_reserved_methods_not_implemented` parametrize list in `tests/test_server.py` (once registered it returns -32602, not the asserted -32601 METHOD_NOT_FOUND, mirroring the analyze/complexity/coverage/classify_signals precedent) + +## 7. Fixtures + +- [x] 7.1 Create `tests/fixtures/sample_project/src/sample/__init__.py` and `calculator.py` (`add(a, b)`, `divide(a, b)` raising `ZeroDivisionError`, `Counter.inc` mutating `self`) +- [x] 7.2 Create `tests/fixtures/sample_project/tests/test_calculator.py` exercising: the name strategy (`test_add`), a case-only name match (`test_Add`↔`add`), a direct-call test on `divide` without a name match, a `pytest.raises` error check, a `unittest.TestCase` subclass whose `test_*` method PAIRS to a production target (e.g. `test_inc`↔`Counter.inc`) so a row is emitted, a multi-assertion test whose assertions span single- and double-digit lines (e.g. 2 and 10); add `pyproject.toml` only if needed for imports (the two-assertions-on-one-source-line tiebreaker case is NOT placed in this on-disk fixture — `ruff format` would reflow it onto separate lines — but is constructed in-test via `tmp_path`; see 8.5) +- [x] 7.3 Exclude `tests/fixtures/` from host pytest collection (`collect_ignore_glob = ["fixtures/*"]` in `tests/conftest.py`, or `--ignore=tests/fixtures`/`norecursedirs` in `[tool.pytest.ini_options]`) so the fixture test file is analyzed as input, not executed; prefer the `conftest.py` `collect_ignore_glob` option so the protected `--cov-fail-under` flags already in `addopts` are not disturbed; ensure all fixture sources are ruff-format-clean (the `ruff format --check` gate covers `tests/`) + +## 8. Tests + +- [x] 8.1 Pairing: `test_add`↔`add` conf 90; `test_Add`↔`add` conf 70; `test_it_works` calling `divide`↔`divide` conf 80; a no-match test yields no row; a test that both name-matches AND calls the same target yields a single row at conf 90 (first-match-wins); a same-named helper in a test module is never a target; assert every emitted row's `confidence` satisfies `isinstance(confidence, int)` and `0 <= confidence <= 100` (guarding against a protocol-illegal float such as `90.0`) +- [x] 8.2 Strategy-3 BFS helper (mocked call graph or tiny on-disk package astroid can parse): target reachable in ≤5 hops pairs at conf 75; target at 6 hops does NOT pair via strategy 3; do NOT require strategy 3 to fire on the fixture in CI +- [x] 8.3 Assertions: a parametrized table asserting each of the six `assertion_type` values against representative pytest AND unittest forms (incl. `assertIsNot`/`assertIsNotNone`, `assertRaisesRegex`, `assert*Equal` family, `assertNotIn`/`not in`); plus node-identification count/scope assertions: a `with pytest.raises(...)` block yields EXACTLY one row (never double-counted as with-item + bare call), an N-assertion test yields EXACTLY N rows, an assertion nested in `with`/`for`/`if`/`while`/`try` IS collected, and an assertion nested in a `def`/`class` body is EXCLUDED +- [x] 8.4 Effect-type: exercise each branch with hand-built `FunctionRecord.side_effects` (`error_check` → `ErrorReturn`/`ErrorSignal`/fallback; value → `ReturnValue`/first-P0/fallback; `generic` → first-effect/fallback), asserting the exact `side_effect_type` +- [x] 8.5 Pipeline: `sample_project` → ≥2 rows with all required keys; the unittest-style test method is collected; determinism via byte-identical repeat run PLUS an explicit ordering-value assertion on the fixture whose assertions span lines 2 and 10 (proving numeric, not lexicographic, ordering); construct a two-assertions-on-one-line case via `tmp_path` and assert the two rows order deterministically by the `col_offset`/collection-index tiebreaker; assert a fixture row's `target_package` equals `derive_package(file)` (guarding the `src/`-layout dotted-name path) +- [x] 8.6 JSON-RPC end-to-end: `initialize` reports `test_mapping: true`; `test_mapping` returns a `{"mappings": [...]}` result; every returned row's `confidence` is an `int` in [0,100]; missing/invalid `root_path` → -32602 +- [x] 8.7 Cross-subprocess determinism: spawn `python -m snake_eyes --stdio` twice with `PYTHONHASHSEED=0` and `=1` over `sample_project` and assert byte-identical stdout (mirroring `tests/test_classify_signals_method.py`) +- [x] 8.8 Empty-result contracts: pipeline returns `[]` on a project with test files but zero pairs; e2e `test_mapping` returns a `{"mappings": []}` result object (assert `result` present, `error` absent) for a no-test-files project +- [x] 8.9 Safety: an oversized/over-deep test file is skipped (byte-cap/AST-depth) without an internal error; strategy-3 pathological input degrades to strategies 1–2 with no -32603; a non-existent `root_path` propagates `FileNotFoundError` → -32602 and is NOT absorbed by strategy-3's broad catch +- [x] 8.10 Fixture isolation: assert `tests/fixtures/sample_project/tests/test_calculator.py` is not collected by the host pytest run (host collection count unaffected) +- [x] 8.11 Meet per-file coverage targets: `pairing.py` ≥90%, `assertions.py` ≥90%, `mapping.py` ≥95%, `pipeline.py` ≥85%, project gate ≥85% +- [x] 8.12 Astroid cache isolation: two `test_mapping` requests over different trees in one process do not contaminate each other (`astroid.MANAGER` cleared before + after each strategy-3 build) +- [x] 8.13 Static-only sentinel: a behavioral test proving `run_test_mapping` runs no pytest, executes no analyzed code, and reads no coverage (mirroring the classify-signals sentinel test — behavioral, not a comment) +- [x] 8.14 Same-named multi-package disambiguation: two packages each defining `add` yield exactly one row per `(target_package, target_function)` (build the two-package tree inline via `tmp_path`; the single-package `sample_project` does not provision this) +- [x] 8.15 Assert emitted `test_file` (and path-valued fields) are root-relative POSIX, not absolute + +## 9. Documentation + +- [x] 9.1 Update `README.md`: add a `test_mapping | Implemented` status-table row, flip the capability sentence (`streaming` is `false`; `test_mapping` now `true`), update the implemented-features prose line to include test-to-assertion mapping, add `src/snake_eyes/quality/` to the project-structure tree enumerating its four modules (`pairing.py`, `assertions.py`, `mapping.py`, `pipeline.py`), and add a "Delivered in issue #6" note +- [x] 9.2 Update `AGENTS.md` Project Structure to list `src/snake_eyes/quality/` (enumerating `pairing.py`, `assertions.py`, `mapping.py`, `pipeline.py`) and note issue #6 delivery; broaden the Architecture "Test-to-assertion mapping (pytest)" line to "(pytest and unittest)" +- [x] 9.3 Extend the `README.md` License section to enumerate the newly-lifted `pairing.py` and `assertions.py` among the files lifted from gaze-py (Matt Peter, Apache 2.0), mirroring the classify-signals precedent +- [x] 9.4 Verify lifted `pairing.py` and `assertions.py` carry both the gaze-py copyright header and the Apache-2.0 §4(b) change notice +- [x] 9.5 Update the astroid-purpose docs to reflect its second consumer: the `README.md` astroid dependency line and the `AGENTS.md` Technology Stack "Inference" bullet must note astroid also backs strategy-3 transitive-call pairing in `quality/pairing.py` (mirroring the classify-signals precedent that updated the Technology Stack when astroid's scope changed) + +## 10. Verification (CI parity gate) + +- [x] 10.1 `uv sync --locked` +- [x] 10.2 `uv run ruff check src/ tests/` +- [x] 10.3 `uv run ruff format --check src/ tests/` +- [x] 10.4 `uv run mypy src/` +- [x] 10.5 `uv run pytest --cov=snake_eyes --cov-report=term-missing --cov-fail-under=85` +- [x] 10.6 Manual stdio smoke test: `initialize` shows `test_mapping: true`; `test_mapping` on `sample_project` returns mapping rows +- [x] 10.7 `openspec validate test-mapping --strict` +- [x] 10.8 Constitution check — confirm PASS for all five principles (Protocol Fidelity, Detection Accuracy, Python-Native Analysis, Testability, Analysis Safety) + + + diff --git a/pyproject.toml b/pyproject.toml index 611fb60..bf58df5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,3 +51,4 @@ branch = true [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--cov=snake_eyes --cov-report=term-missing --cov-fail-under=85" +markers = ["slow: marks tests that spawn subprocesses (deselect with -m 'not slow')"] diff --git a/src/snake_eyes/protocol.py b/src/snake_eyes/protocol.py index dda9606..b8025c4 100644 --- a/src/snake_eyes/protocol.py +++ b/src/snake_eyes/protocol.py @@ -75,8 +75,8 @@ def initialize_result() -> dict[str, Any]: Returns a plain dict with the exact keys ``analyzer_name``, ``language``, ``language_version`` (from ``sys.version_info``), ``protocol_version`` (``"1.1.0"``), and ``capabilities`` with all four - flags present (``discover`` and ``classify_signals`` ``True``; - ``test_mapping`` and ``streaming`` ``False``). + flags present (``discover``, ``classify_signals``, and ``test_mapping`` + ``True``; ``streaming`` ``False``). """ major, minor, micro = sys.version_info[:3] return { @@ -86,7 +86,7 @@ def initialize_result() -> dict[str, Any]: "protocol_version": PROTOCOL_VERSION, "capabilities": { "discover": True, - "test_mapping": False, + "test_mapping": True, "classify_signals": True, "streaming": False, }, diff --git a/src/snake_eyes/quality/__init__.py b/src/snake_eyes/quality/__init__.py new file mode 100644 index 0000000..a21c216 --- /dev/null +++ b/src/snake_eyes/quality/__init__.py @@ -0,0 +1,5 @@ +"""Test-mapping pipeline for snake-eyes (quality package).""" + +from .pipeline import run_test_mapping + +__all__ = ["run_test_mapping"] diff --git a/src/snake_eyes/quality/assertions.py b/src/snake_eyes/quality/assertions.py new file mode 100644 index 0000000..c54273c --- /dev/null +++ b/src/snake_eyes/quality/assertions.py @@ -0,0 +1,367 @@ +# Copyright Matt Peter (gaze-py, https://github.com/mpeter/gaze-py). Apache 2.0. +# Modified 2026 by zero-dot-force: adapted for snake-eyes ast/models; +# exhaustive name-keyed classification table (not endswith heuristic); +# traversal scope and de-dup rules aligned with Gaze analyzer protocol v1.1.0; +# annotated for mypy --strict. +"""Assertion node identification and classification for snake-eyes. + +Lifted from gaze-py ``quality/assertions.py`` and adapted for the snake-eyes +protocol (Gaze analyzer protocol v1.1.0). + +Identifies assertion nodes in a test function's AST body and classifies each +into one of six ``assertion_type`` values: +``equality | comparison | identity | membership | error_check | generic`` + +Public API: +- ``collect_assertions(func_node, rel_path) -> list[AssertionInfo]`` +""" + +from __future__ import annotations + +import ast +from dataclasses import dataclass + +from ..analysis._shared import MAX_AST_DEPTH + +# --------------------------------------------------------------------------- +# Classification tables (exhaustive name-keyed, not endswith heuristics) +# --------------------------------------------------------------------------- + +_EQUALITY_METHODS: frozenset[str] = frozenset( + { + "assertEqual", + "assertEquals", + "assertAlmostEqual", + "assertDictEqual", + "assertListEqual", + "assertMultiLineEqual", + "assertCountEqual", + "assertSequenceEqual", + } +) + +_COMPARISON_METHODS: frozenset[str] = frozenset( + { + "assertNotEqual", + "assertNotAlmostEqual", + "assertLess", + "assertLessEqual", + "assertGreater", + "assertGreaterEqual", + } +) + +_IDENTITY_METHODS: frozenset[str] = frozenset( + { + "assertIs", + "assertIsNot", + "assertIsNone", + "assertIsNotNone", + } +) + +_MEMBERSHIP_METHODS: frozenset[str] = frozenset( + { + "assertIn", + "assertNotIn", + } +) + +_ERROR_CHECK_METHODS: frozenset[str] = frozenset( + { + "assertRaises", + "assertRaisesRegex", + "assertRaisesRegexp", + } +) + +_WARNS_METHODS: frozenset[str] = frozenset( + { + "assertWarns", + "assertWarnsRegex", + } +) + +# Callee names that identify an error-check when used as a plain call or with-item +_ERROR_CHECK_CALLEES: frozenset[str] = frozenset({"raises", "warns"}) +# Callees that indicate raises/warns as pytest.raises / pytest.warns attrs +_PYTEST_ATTRS: frozenset[str] = frozenset({"raises", "warns"}) + + +# --------------------------------------------------------------------------- +# Public data type +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class AssertionInfo: + """A single assertion within a test function.""" + + assertion_type: str # one of the six types + assertion_location: str # "rel_path:line" + line: int + col: int # col_offset for tiebreaking + + +# --------------------------------------------------------------------------- +# Callee classification helpers +# --------------------------------------------------------------------------- + + +def _classify_by_method_name(method_name: str) -> str | None: + """Return assertion_type for a known unittest assert* method, or None.""" + if method_name in _EQUALITY_METHODS: + return "equality" + if method_name in _COMPARISON_METHODS: + return "comparison" + if method_name in _IDENTITY_METHODS: + return "identity" + if method_name in _MEMBERSHIP_METHODS: + return "membership" + if method_name in _ERROR_CHECK_METHODS: + return "error_check" + if method_name in _WARNS_METHODS: + return "error_check" + # assertTrue, assertFalse, and any other assert* → generic + if method_name.startswith("assert"): + return "generic" + return None + + +def _is_raises_warns_call(node: ast.Call) -> bool: + """Return True if *node* is raises(...)/warns(...)/pytest.raises/pytest.warns.""" + fn = node.func + if isinstance(fn, ast.Name) and fn.id in _ERROR_CHECK_CALLEES: + return True + if ( + isinstance(fn, ast.Attribute) + and fn.attr in _PYTEST_ATTRS + and isinstance(fn.value, ast.Name) + and fn.value.id == "pytest" + ): + return True + return False + + +def _is_assert_raises_call(node: ast.Call) -> bool: + """Return True if *node* is self.assertRaises(...) / similar.""" + fn = node.func + if isinstance(fn, ast.Attribute) and fn.attr in _ERROR_CHECK_METHODS: + return True + return False + + +def _classify_assert_stmt(node: ast.Assert) -> str: + """Classify a bare ``assert`` statement by its test expression.""" + test = node.test + if isinstance(test, ast.Compare): + ops = test.ops + if ops: + op = ops[0] + if isinstance(op, ast.Eq): + return "equality" + if isinstance(op, (ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE)): + return "comparison" + if isinstance(op, (ast.Is, ast.IsNot)): + return "identity" + if isinstance(op, (ast.In, ast.NotIn)): + return "membership" + return "generic" + + +def _classify_call(node: ast.Call) -> str | None: + """Return assertion_type if this Call is an assertion; else None.""" + fn = node.func + + # self.assertXxx(...) or bare assertXxx(...) + if isinstance(fn, ast.Attribute): + atype = _classify_by_method_name(fn.attr) + if atype is not None: + return atype + # pytest.raises / pytest.warns + if fn.attr in _PYTEST_ATTRS and isinstance(fn.value, ast.Name): + if fn.value.id == "pytest": + return "error_check" + + # bare function call: raises(...), warns(...) + if isinstance(fn, ast.Name): + if fn.id in _ERROR_CHECK_CALLEES: + return "error_check" + + return None + + +# --------------------------------------------------------------------------- +# Context-expression skip set: calls that are WITH-item context expressions +# (these are collected as with-items, NOT as bare calls) +# --------------------------------------------------------------------------- + + +def _collect_with_context_calls(stmts: list[ast.stmt]) -> set[int]: + """Return id() of Call nodes used as with-item context expressions. + + These calls should be counted once as the with-item, never additionally + as a bare call in visit_Call. + """ + ids: set[int] = set() + for stmt in stmts: + _collect_context_calls_stmt(stmt, ids) + return ids + + +def _collect_context_calls_stmt(stmt: ast.stmt, ids: set[int]) -> None: + """Recursively collect with-item context Call ids from *stmt*.""" + if isinstance(stmt, ast.With): + for item in stmt.items: + ctx = item.context_expr + if isinstance(ctx, ast.Call) and ( + _is_raises_warns_call(ctx) or _is_assert_raises_call(ctx) + ): + ids.add(id(ctx)) + for child in stmt.body: + _collect_context_calls_stmt(child, ids) + elif isinstance(stmt, (ast.For, ast.While)): + for child in stmt.body: + _collect_context_calls_stmt(child, ids) + if hasattr(stmt, "orelse"): + for child in stmt.orelse: + _collect_context_calls_stmt(child, ids) + elif isinstance(stmt, ast.If): + for child in stmt.body: + _collect_context_calls_stmt(child, ids) + for child in stmt.orelse: + _collect_context_calls_stmt(child, ids) + elif isinstance(stmt, ast.Try): + for child in stmt.body: + _collect_context_calls_stmt(child, ids) + for handler in stmt.handlers: + for child in handler.body: + _collect_context_calls_stmt(child, ids) + for child in stmt.orelse: + _collect_context_calls_stmt(child, ids) + for child in stmt.finalbody: + _collect_context_calls_stmt(child, ids) + + +# --------------------------------------------------------------------------- +# AST visitor +# --------------------------------------------------------------------------- + + +class _AssertionVisitor: + """Collect assertions from a function body (no nested defs/classes).""" + + def __init__( + self, + rel_path: str, + skip_call_ids: set[int], + max_depth: int, + ) -> None: + self._rel_path = rel_path + self._skip_call_ids = skip_call_ids + self._max_depth = max_depth + self._depth = 0 + self.assertions: list[AssertionInfo] = [] + + def _loc(self, node: ast.AST) -> str: + line = getattr(node, "lineno", 0) + return f"{self._rel_path}:{line}" + + def _col(self, node: ast.AST) -> int: + return int(getattr(node, "col_offset", 0)) + + def _line(self, node: ast.AST) -> int: + return int(getattr(node, "lineno", 0)) + + def visit_stmts(self, stmts: list[ast.stmt]) -> None: + self._depth += 1 + if self._depth > self._max_depth: + raise RecursionError("AST depth budget exceeded in assertion visitor") + for stmt in stmts: + self.visit_stmt(stmt) + self._depth -= 1 + + def visit_stmt(self, stmt: ast.stmt) -> None: + if isinstance(stmt, ast.Assert): + atype = _classify_assert_stmt(stmt) + self.assertions.append( + AssertionInfo( + assertion_type=atype, + assertion_location=self._loc(stmt), + line=self._line(stmt), + col=self._col(stmt), + ) + ) + elif isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call): + call = stmt.value + if id(call) not in self._skip_call_ids: + call_atype = _classify_call(call) + if call_atype is not None: + self.assertions.append( + AssertionInfo( + assertion_type=call_atype, + assertion_location=self._loc(call), + line=self._line(call), + col=self._col(call), + ) + ) + elif isinstance(stmt, ast.With): + for item in stmt.items: + ctx = item.context_expr + if isinstance(ctx, ast.Call) and ( + _is_raises_warns_call(ctx) or _is_assert_raises_call(ctx) + ): + self.assertions.append( + AssertionInfo( + assertion_type="error_check", + assertion_location=self._loc(ctx), + line=self._line(ctx), + col=self._col(ctx), + ) + ) + # Descend into with body + self.visit_stmts(stmt.body) + elif isinstance(stmt, (ast.For, ast.While)): + self.visit_stmts(stmt.body) + if stmt.orelse: + self.visit_stmts(stmt.orelse) + elif isinstance(stmt, ast.If): + self.visit_stmts(stmt.body) + if stmt.orelse: + self.visit_stmts(stmt.orelse) + elif isinstance(stmt, ast.Try): + self.visit_stmts(stmt.body) + for handler in stmt.handlers: + self.visit_stmts(handler.body) + if stmt.orelse: + self.visit_stmts(stmt.orelse) + if stmt.finalbody: + self.visit_stmts(stmt.finalbody) + # Do NOT descend into FunctionDef / AsyncFunctionDef / ClassDef + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def collect_assertions( + func_node: ast.FunctionDef | ast.AsyncFunctionDef, + rel_path: str, +) -> list[AssertionInfo]: + """Collect assertions from a test function node. + + *rel_path* is the test file path relative to the project root (POSIX), + used to format ``assertion_location`` as ``rel_path:line``. + + Traverses the direct body (nested ``with``/``for``/``if``/``while``/``try`` + included); does NOT descend into nested ``def``/``class`` bodies. + """ + body: list[ast.stmt] = list(func_node.body) + skip_ids = _collect_with_context_calls(body) + visitor = _AssertionVisitor(rel_path, skip_ids, max_depth=MAX_AST_DEPTH) + try: + visitor.visit_stmts(body) + except RecursionError: + pass + return visitor.assertions diff --git a/src/snake_eyes/quality/mapping.py b/src/snake_eyes/quality/mapping.py new file mode 100644 index 0000000..0a12e0d --- /dev/null +++ b/src/snake_eyes/quality/mapping.py @@ -0,0 +1,73 @@ +"""Effect-type inference for test mapping rows. + +FRESH — not lifted from gaze-py. No provenance header required. + +Infers ``side_effect_type`` from an assertion's ``assertion_type`` and the +target function's ``FunctionRecord.side_effects`` (detector output — no +re-parsing). Consumes ``TIER_MAP`` safely (``get``) with no ``KeyError`` on +unknown type strings. + +Public API: +- ``infer_side_effect_type(assertion_type, target_effects) -> str`` +""" + +from __future__ import annotations + +from ..analysis.effects import TIER_MAP, SideEffectType, Tier +from ..analysis.models import Effect + +# Assertion types that form the value-assertion group +_VALUE_TYPES: frozenset[str] = frozenset( + {"equality", "comparison", "identity", "membership"} +) + + +def _effect_type_str(et: SideEffectType) -> str: + return str(et) + + +def infer_side_effect_type( + assertion_type: str, + target_effects: tuple[Effect, ...], +) -> str: + """Infer ``side_effect_type`` from assertion kind and target side effects. + + Chains: + - ``error_check``: ``ErrorReturn`` if present, else ``ErrorSignal`` if + present, else ``"ErrorReturn"`` (fallback). + - ``equality``/``comparison``/``identity``/``membership``: ``ReturnValue`` + if present, else first P0 effect, else ``"ReturnValue"`` (fallback). + - ``generic``: first detected effect if any, else ``"ReturnValue"`` + (fallback). + """ + effect_types: list[str] = [e.type for e in target_effects] + + if assertion_type == "error_check": + err_return = _effect_type_str(SideEffectType.ErrorReturn) + err_signal = _effect_type_str(SideEffectType.ErrorSignal) + if err_return in effect_types: + return err_return + if err_signal in effect_types: + return err_signal + return err_return + + if assertion_type in _VALUE_TYPES: + ret_val = _effect_type_str(SideEffectType.ReturnValue) + if ret_val in effect_types: + return ret_val + # First P0 effect + for effect in target_effects: + try: + se_type = SideEffectType(effect.type) + tier = TIER_MAP.get(se_type) + if tier == Tier.P0: + return effect.type + except ValueError: + # Unknown type string — safe fallback per spec + continue + return ret_val + + # generic + if effect_types: + return effect_types[0] + return _effect_type_str(SideEffectType.ReturnValue) diff --git a/src/snake_eyes/quality/pairing.py b/src/snake_eyes/quality/pairing.py new file mode 100644 index 0000000..68ed2d6 --- /dev/null +++ b/src/snake_eyes/quality/pairing.py @@ -0,0 +1,442 @@ +# Copyright Matt Peter (gaze-py, https://github.com/mpeter/gaze-py). Apache 2.0. +# Modified 2026 by zero-dot-force: adapted for snake-eyes ast/models; lifted +# 0.0-1.0 confidence scores converted to int 0-100; astroid transitive call +# graph (strategy 3) ported with snake-eyes MANAGER lifecycle inline; +# Go-only patterns removed; integrated with snake-eyes protocol models. +"""Test-function pairing engine for snake-eyes. + +Lifted from gaze-py ``quality/pairing.py`` and adapted for the snake-eyes +protocol (Gaze analyzer protocol v1.1.0). + +Three-strategy, first-match-wins pairing: + +1. **Name convention** -- strip leading ``test_``/``Test``/``test``/``Test`` + prefix; exact match → confidence 90, case-only match → confidence 70. +2. **Direct call** -- target name appears as a ``Call`` in the test AST + → confidence 80. +3. **Transitive call graph** -- astroid BFS with ``depth_limit=5`` + → confidence 75; built once per ``run_test_mapping`` request (lazy). + +Public API: +- ``pair_tests(test_functions, target_records, test_trees, root_abs, + graph_files) -> list[PairedResult]`` +""" + +from __future__ import annotations + +import ast +import pathlib +import sys +from collections import deque +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import astroid # type: ignore[import-untyped] +from astroid import nodes as astroid_nodes +from astroid.exceptions import AstroidError # type: ignore[import-untyped] +from astroid.util import Uninferable # type: ignore[import-untyped] + +from ..analysis._shared import derive_package, is_analyzable_file + +if TYPE_CHECKING: + from ..analysis.models import FunctionRecord + +# Broadened exception tuple for strategy 3 (mirrors analysis/inference.py). +# FileNotFoundError is intentionally excluded so discover() errors propagate. +_STRATEGY3_DEGRADE: tuple[type[BaseException], ...] = ( + AstroidError, + RecursionError, + MemoryError, +) + +# --------------------------------------------------------------------------- +# Public data types +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PairedResult: + """A single (test, target) pairing.""" + + test_function: str + test_file: str + target_function: str + target_package: str + target_file: str + confidence: int # 0–100 integer per protocol + + +# --------------------------------------------------------------------------- +# Name-stripping helpers (Strategy 1) +# --------------------------------------------------------------------------- + + +def _strip_test_prefix(name: str) -> str | None: + """Return the name with a leading test prefix stripped, or None.""" + # snake_case: test_foo -> foo + if name.startswith("test_") and len(name) > len("test_"): + return name[len("test_") :] + # snake_case short: testfoo -> foo (bare 'test' prefix without underscore) + if name.startswith("test") and len(name) > len("test") and not name[4:5] == "_": + return name[len("test") :] + # CamelCase: TestFoo -> Foo (class method style with leading capital) + if name.startswith("Test") and len(name) > len("Test"): + return name[len("Test") :] + return None + + +def _name_match( + test_name: str, + target_records: list[FunctionRecord], +) -> list[tuple[FunctionRecord, int]]: + """Return list of (record, confidence) for name-convention strategy.""" + # Extract the bare name (without class qualifier) for matching + bare = test_name.split(".")[-1] + stripped = _strip_test_prefix(bare) + if stripped is None: + return [] + results: list[tuple[FunctionRecord, int]] = [] + for rec in target_records: + if rec.name == stripped: + results.append((rec, 90)) + elif rec.name.lower() == stripped.lower(): + results.append((rec, 70)) + return results + + +# --------------------------------------------------------------------------- +# Direct-call helpers (Strategy 2) +# --------------------------------------------------------------------------- + + +def _direct_call_names(tree: ast.Module, func_name: str) -> set[str]: + """Return the set of callee names directly called in *func_name*.""" + # Find the function node — bare name or Class.method + target_node: ast.FunctionDef | ast.AsyncFunctionDef | None = None + bare = func_name.split(".")[-1] + class_part = func_name.split(".")[0] if "." in func_name else None + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and class_part and node.name == class_part: + for item in node.body: + if ( + isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + and item.name == bare + ): + target_node = item + break + elif ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == func_name + and class_part is None + ): + target_node = node + + if target_node is None: + return set() + + names: set[str] = set() + for child in ast.walk(target_node): + if isinstance(child, ast.Call): + fn = child.func + if isinstance(fn, ast.Name): + names.add(fn.id) + elif isinstance(fn, ast.Attribute): + names.add(fn.attr) + return names + + +def _direct_call_match( + test_name: str, + test_tree: ast.Module, + target_records: list[FunctionRecord], +) -> list[tuple[FunctionRecord, int]]: + """Return list of (record, confidence=80) for direct-call strategy.""" + called = _direct_call_names(test_tree, test_name) + results: list[tuple[FunctionRecord, int]] = [] + for rec in target_records: + if rec.name in called: + results.append((rec, 80)) + return results + + +# --------------------------------------------------------------------------- +# Transitive call graph (Strategy 3) — built lazily, once per request +# --------------------------------------------------------------------------- + + +@dataclass +class _CallGraph: + """Maps normalized file path → set of normalized callee file paths.""" + + edges: dict[str, set[str]] + + def reachable_files(self, start_file: str, depth_limit: int = 5) -> set[str]: + """BFS from *start_file*; return all files reachable within depth_limit.""" + visited: set[str] = set() + queue: deque[tuple[str, int]] = deque() + queue.append((start_file, 0)) + while queue: + current, depth = queue.popleft() + if current in visited or depth > depth_limit: + continue + visited.add(current) + for neighbour in self.edges.get(current, set()): + if neighbour not in visited and depth + 1 <= depth_limit: + queue.append((neighbour, depth + 1)) + return visited + + +def _normalize_path(p: str) -> str: + return str(pathlib.Path(p).resolve()) + + +def _build_call_graph( + root_abs: str, + graph_files: list[str], +) -> _CallGraph | None: + """Build an astroid call graph; return None on any degrade.""" + try: + manager = astroid.MANAGER + manager.clear_cache() + + edges: dict[str, set[str]] = {} + # list of (norm_path, module) pairs + parsed: list[tuple[str, Any]] = [] + + root = pathlib.Path(root_abs) + + for rel in graph_files: + abs_path = root / rel + if not is_analyzable_file(abs_path, label=rel): + continue + try: + modname = derive_package(rel) + module = manager.ast_from_file(str(abs_path), modname, source=True) + norm = _normalize_path(str(abs_path)) + parsed.append((norm, module)) + except FileNotFoundError: + raise + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 skipping {rel}:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + continue + + analyzed_norms: set[str] = {norm for norm, _ in parsed} + + # Build the set of function/method names defined anywhere in the analyzed + # project. Inference is attempted only for call sites whose unqualified + # callee name is in this set, so astroid does not parse ambient + # stdlib/site-packages modules (which bypass the 16 MiB byte-cap) to + # resolve external callees that would never produce an in-project edge. + # Mirrors analysis/inference.py _build() defined_names pattern. + modules = [module for _, module in parsed] + defined_names: set[str] = set() + for module in modules: + for func in module.nodes_of_class( + (astroid_nodes.FunctionDef, astroid_nodes.AsyncFunctionDef) + ): + defined_names.add(func.name) + + for source_norm, module in parsed: + try: + call_iter = module.nodes_of_class(astroid_nodes.Call) + except Exception: + continue + for call in call_iter: + fn = call.func + callee_name: str | None = None + if isinstance(fn, astroid_nodes.Name): + callee_name = fn.name + elif isinstance(fn, astroid_nodes.Attribute): + callee_name = fn.attrname + if callee_name is None: + continue + # Pre-filter: skip callee names not defined in the project. + # Bounds transitive parsing to in-project resolution and removes + # the infer-then-discard waste for external (pytest/mock/stdlib) + # call sites. + if callee_name not in defined_names: + continue + try: + inferred = list(call.func.infer()) + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 skipping" + f" uninferable call site {callee_name!r}:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + continue + for candidate in inferred: + try: + if candidate is Uninferable: + continue + if not isinstance( + candidate, + ( + astroid_nodes.FunctionDef, + astroid_nodes.AsyncFunctionDef, + ), + ): + continue + callee_file = getattr(candidate.root(), "file", None) + if callee_file is None: + continue + callee_norm = _normalize_path(callee_file) + if callee_norm in analyzed_norms: + edges.setdefault(source_norm, set()).add(callee_norm) + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 skipping candidate:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + continue + + return _CallGraph(edges=edges) + + except FileNotFoundError: + raise + except _STRATEGY3_DEGRADE as exc: + print( + f"snake-eyes: test_mapping strategy-3 call graph build failed:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + return None + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 call graph build failed" + f" (unexpected): {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + return None + + +def _transitive_match( + test_file_norm: str, + call_graph: _CallGraph, + target_records: list[FunctionRecord], + root_abs: str, + depth_limit: int = 5, +) -> list[tuple[FunctionRecord, int]]: + """Match via BFS call graph reachability, confidence 75. + + *root_abs* is used to resolve root-relative ``rec.file`` paths to absolute + paths before normalizing, so the lookup is CWD-independent. + """ + reachable = call_graph.reachable_files(test_file_norm, depth_limit) + results: list[tuple[FunctionRecord, int]] = [] + root = pathlib.Path(root_abs) + for rec in target_records: + # rec.file is always root-relative in production; resolve via root_abs. + rec_norm = _normalize_path(str(root / rec.file)) + if rec_norm in reachable: + results.append((rec, 75)) + return results + + +# --------------------------------------------------------------------------- +# Main pairing function +# --------------------------------------------------------------------------- + + +def pair_tests( + test_functions: list[tuple[str, str]], # (name, test_file) + target_records: list[FunctionRecord], + test_trees: dict[str, ast.Module], # test_file -> AST + root_abs: str, + graph_files: list[str], +) -> list[PairedResult]: + """Pair test functions to production functions using three strategies. + + Returns one ``PairedResult`` per unique ``(test_function, test_file, + target_package, target_function)`` key, in first-match-wins priority order. + """ + results: list[PairedResult] = [] + seen: set[tuple[str, str, str, str]] = set() + + # Lazy call graph for strategy 3: None = disabled/failed, False = not yet tried + _graph: _CallGraph | None | bool = False + + def _get_graph() -> _CallGraph | None: + nonlocal _graph + if _graph is False: + try: + _graph = _build_call_graph(root_abs, graph_files) + except FileNotFoundError: + raise + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 disabled:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + _graph = None + # _graph is now None or a _CallGraph (bool False was the sentinel) + if isinstance(_graph, _CallGraph): + return _graph + return None + + try: + for test_name, test_file in test_functions: + tree = test_trees.get(test_file) + if tree is None: + continue + + paired: list[tuple[FunctionRecord, int]] = [] + + # Strategy 1: name convention + paired = _name_match(test_name, target_records) + + # Strategy 2: direct call (only if strategy 1 found nothing) + if not paired: + paired = _direct_call_match(test_name, tree, target_records) + + # Strategy 3: transitive call graph (only if strategies 1+2 empty) + if not paired: + try: + graph = _get_graph() + if graph is not None: + test_abs = str((pathlib.Path(root_abs) / test_file).resolve()) + paired = _transitive_match( + test_abs, graph, target_records, root_abs + ) + except FileNotFoundError: + raise + except Exception as exc: + print( + f"snake-eyes: test_mapping strategy-3 lookup failed:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + + # Emit one result per distinct (target_package, target_function), de-dup + for rec, confidence in paired: + key = (test_name, test_file, rec.package, rec.name) + if key in seen: + continue + seen.add(key) + results.append( + PairedResult( + test_function=test_name, + test_file=test_file, + target_function=rec.name, + target_package=rec.package, + target_file=rec.file, + confidence=confidence, + ) + ) + finally: + # Release astroid cache after the request; bounds memory in long-lived server. + # Clear unconditionally — a clear on an untouched manager is a cheap no-op. + try: + astroid.MANAGER.clear_cache() + except Exception: # pragma: no cover + pass + + return results diff --git a/src/snake_eyes/quality/pipeline.py b/src/snake_eyes/quality/pipeline.py new file mode 100644 index 0000000..71ea785 --- /dev/null +++ b/src/snake_eyes/quality/pipeline.py @@ -0,0 +1,265 @@ +"""Test-mapping pipeline for snake-eyes. + +FRESH — not lifted from gaze-py. No provenance header required. + +Orchestrates discovery → analysis → test-function collection → pairing → +assertion detection → effect-type inference → serialisation. + +Public API: +- ``run_test_mapping(root_path, patterns) -> list[dict]`` +""" + +from __future__ import annotations + +import ast +import pathlib +import sys +from typing import Any + +from ..analysis._shared import ( + BROADENED_EXCEPTIONS, + enumerate_functions_with_spans, + iter_source_files, +) +from ..analysis.detector import analyze_path +from ..discovery import discover +from .assertions import collect_assertions +from .mapping import infer_side_effect_type +from .pairing import pair_tests + +# --------------------------------------------------------------------------- +# Test-function collection helpers +# --------------------------------------------------------------------------- + + +def _is_testcase_subclass(class_node: ast.ClassDef) -> bool: + """Return True if *class_node* appears to subclass unittest.TestCase.""" + for base in class_node.bases: + if isinstance(base, ast.Name) and base.id == "TestCase": + return True + if ( + isinstance(base, ast.Attribute) + and base.attr == "TestCase" + and isinstance(base.value, ast.Name) + and base.value.id == "unittest" + ): + return True + return False + + +def _collect_test_functions( + tree: ast.Module, +) -> list[str]: + """Collect test function names from an AST module. + + Returns bare names for top-level ``test_*`` functions and + ``ClassName.method`` for ``test*`` methods of ``unittest.TestCase`` + subclasses. + """ + names: list[str] = [] + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name.startswith("test_"): + names.append(node.name) + elif isinstance(node, ast.ClassDef): + if _is_testcase_subclass(node): + for item in node.body: + if isinstance( + item, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and item.name.startswith("test"): + names.append(f"{node.name}.{item.name}") + return names + + +def _get_func_node( + tree: ast.Module, + test_name: str, +) -> ast.FunctionDef | ast.AsyncFunctionDef | None: + """Look up the AST node for *test_name* (bare or Class.method).""" + if "." in test_name: + class_name, method_name = test_name.split(".", 1) + for node in ast.iter_child_nodes(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + for item in node.body: + if ( + isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + and item.name == method_name + ): + return item + return None + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name == test_name: + return node + return None + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + + +def run_test_mapping( + root_path: str, + patterns: list[str] | None, +) -> list[dict[str, Any]]: + """Run the full test-mapping pipeline and return a list of mapping rows. + + Each row is a dict with exactly these keys: + ``test_function, test_file, assertion_location, assertion_type, + target_function, target_package, side_effect_type, confidence``. + + Returns ``[]`` when there are no pairs. + + Raises ``FileNotFoundError`` when ``root_path`` is not a directory + (caller maps to -32602). + + Static only: never runs pytest, reads coverage, or executes analyzed code. + """ + # 1. Discover source and test files + disc = discover(root_path, patterns) + source_files_set: set[str] = set(disc.source_files) + + # 2. Analyze all production (source) files for side effects + all_records = analyze_path(root_path, patterns) + # Filter to production files only (never test files) + target_records = [r for r in all_records if r.file in source_files_set] + + root = pathlib.Path(root_path).resolve() + + # 3. Parse test files and collect test functions + test_files: list[str] = list(disc.test_files) + test_trees: dict[str, ast.Module] = {} + test_functions: list[tuple[str, str]] = [] # (name, test_file) + + for rel_path, _source, tree in iter_source_files(root_path, test_files): + # Guard depth: use enumerate_functions_with_spans which is depth-guarded + try: + _ = enumerate_functions_with_spans(tree) + except RecursionError: + print( + f"snake-eyes: test_mapping skipping {rel_path}:" + " AST depth budget exceeded", + file=sys.stderr, + ) + continue + except BROADENED_EXCEPTIONS as exc: + print( + f"snake-eyes: test_mapping skipping {rel_path}:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + continue + + test_trees[rel_path] = tree + funcs = _collect_test_functions(tree) + for fn in funcs: + test_functions.append((fn, rel_path)) + + if not test_functions or not target_records: + return [] + + # 4. Pair tests to production functions. + # The graph node set includes BOTH source AND test files so that test-file + # nodes have outgoing edges for transitive BFS (strategy 3). Target + # candidacy is still restricted to source-only `target_records` above. + graph_files = list(disc.source_files) + list(disc.test_files) + pairs = pair_tests( + test_functions=test_functions, + target_records=target_records, + test_trees=test_trees, + root_abs=str(root), + graph_files=graph_files, + ) + + if not pairs: + return [] + + # 5. Collect assertions and build mapping rows + rows: list[dict[str, Any]] = [] + + for pair in pairs: + pair_tree = test_trees.get(pair.test_file) + if pair_tree is None: + continue + + func_node = _get_func_node(pair_tree, pair.test_function) + if func_node is None: + continue + + # Collect assertions — guard depth + try: + assertions = collect_assertions(func_node, pair.test_file) + except RecursionError: + print( + f"snake-eyes: test_mapping skipping assertions in" + f" {pair.test_file}/{pair.test_function}: depth budget exceeded", + file=sys.stderr, + ) + continue + except BROADENED_EXCEPTIONS as exc: + print( + f"snake-eyes: test_mapping skipping assertions in" + f" {pair.test_file}/{pair.test_function}:" + f" {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + continue + + # Find target record for side-effect inference + target_rec = next( + ( + r + for r in target_records + if r.name == pair.target_function and r.package == pair.target_package + ), + None, + ) + target_effects = target_rec.side_effects if target_rec is not None else () + + for assertion in assertions: + side_effect_type = infer_side_effect_type( + assertion.assertion_type, target_effects + ) + rows.append( + { + "test_function": pair.test_function, + "test_file": pair.test_file, + "assertion_location": assertion.assertion_location, + "assertion_type": assertion.assertion_type, + "target_function": pair.target_function, + "target_package": pair.target_package, + "side_effect_type": side_effect_type, + "confidence": pair.confidence, + # Internal tiebreaker fields (stripped before return) + "_line": assertion.line, + "_col": assertion.col, + } + ) + + # 6. Sort by composite key (numeric line, col for tiebreaking) + rows.sort( + key=lambda r: ( + r["test_file"], + r["test_function"], + r["_line"], + r["_col"], + r["target_package"], + r["target_function"], + ) + ) + + # Strip internal tiebreaker fields + return [ + { + "test_function": r["test_function"], + "test_file": r["test_file"], + "assertion_location": r["assertion_location"], + "assertion_type": r["assertion_type"], + "target_function": r["target_function"], + "target_package": r["target_package"], + "side_effect_type": r["side_effect_type"], + "confidence": r["confidence"], + } + for r in rows + ] diff --git a/src/snake_eyes/server.py b/src/snake_eyes/server.py index 32a8856..e60e921 100644 --- a/src/snake_eyes/server.py +++ b/src/snake_eyes/server.py @@ -27,6 +27,7 @@ shutdown_result, to_json, ) +from .quality import run_test_mapping from .signals.adapter import extract_signals SHUTDOWN_METHOD = "shutdown" @@ -132,6 +133,15 @@ def _classify_signals(params: dict[str, Any] | None) -> dict[str, Any]: return {"signals": signals} +def _test_mapping(params: dict[str, Any] | None) -> dict[str, Any]: + root_path, patterns = _validate_analysis_params(params) + try: + mappings = run_test_mapping(root_path, patterns) + except FileNotFoundError as err: + raise RpcError(INVALID_PARAMS, str(err)) from err + return {"mappings": mappings} + + DEFAULT_DISPATCH: Mapping[str, Handler] = { "initialize": _initialize, SHUTDOWN_METHOD: _shutdown, @@ -140,6 +150,7 @@ def _classify_signals(params: dict[str, Any] | None) -> dict[str, Any]: "complexity": _complexity, "coverage": _coverage, "classify_signals": _classify_signals, + "test_mapping": _test_mapping, } diff --git a/tests/conftest.py b/tests/conftest.py index ae5187c..93cbab1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,10 @@ import json from typing import Any +# Exclude fixture directories from host pytest collection so fixture test files +# are analyzed as static input, not executed as tests. +collect_ignore_glob = ["fixtures/*"] + def req(method: str, id: int | str = 1, **params: Any) -> str: """Serialize a JSON-RPC 2.0 request object to a single-line string.""" diff --git a/tests/fixtures/sample_project/src/sample/__init__.py b/tests/fixtures/sample_project/src/sample/__init__.py new file mode 100644 index 0000000..febca4f --- /dev/null +++ b/tests/fixtures/sample_project/src/sample/__init__.py @@ -0,0 +1 @@ +"""Sample package for snake-eyes test fixtures.""" diff --git a/tests/fixtures/sample_project/src/sample/calculator.py b/tests/fixtures/sample_project/src/sample/calculator.py new file mode 100644 index 0000000..74be8f0 --- /dev/null +++ b/tests/fixtures/sample_project/src/sample/calculator.py @@ -0,0 +1,24 @@ +"""Sample calculator module for snake-eyes test fixtures.""" + + +def add(a: int, b: int) -> int: + """Return the sum of a and b.""" + return a + b + + +def divide(a: int, b: int) -> float: + """Return a divided by b; raises ZeroDivisionError when b is zero.""" + if b == 0: + raise ZeroDivisionError("division by zero") + return a / b + + +class Counter: + """A simple counter that tracks a running total.""" + + def __init__(self) -> None: + self.value: int = 0 + + def inc(self) -> None: + """Increment the counter by one.""" + self.value += 1 diff --git a/tests/fixtures/sample_project/tests/test_calculator.py b/tests/fixtures/sample_project/tests/test_calculator.py new file mode 100644 index 0000000..6d7e739 --- /dev/null +++ b/tests/fixtures/sample_project/tests/test_calculator.py @@ -0,0 +1,66 @@ +"""Fixture test file for snake-eyes test-mapping pipeline tests. + +Not executed by the host pytest run (excluded via collect_ignore_glob). +Analyzed as static input by test_test_mapping_method.py. +""" + +from __future__ import annotations + +import unittest + +from sample.calculator import Counter, add, divide + + +def test_add(a: int = 1, b: int = 2) -> None: + """Name-strategy pair: test_add <-> add (confidence 90).""" + result = add(a, b) + assert result == a + b + + +def test_Add(a: int = 2, b: int = 3) -> None: + """Case-only name match: test_Add <-> add (confidence 70).""" + result = add(a, b) + assert result == a + b + + +def test_it_divides(a: int = 10, b: int = 2) -> None: + """Direct-call strategy: calls divide() but name doesn't match (confidence 80).""" + result = divide(a, b) + assert result == 5.0 + + +def test_divide_error() -> None: + """Error-check assertion using pytest.raises.""" + import pytest + + with pytest.raises(ZeroDivisionError): + divide(1, 0) + + +def test_multi_assertion() -> None: + """Multi-assertion test spanning lines 2 and 10 of the function body. + + Line 2 (within body): first assert + Line 10 (within body): second assert — triggers numeric sort check. + """ + # assertion on an early line of this function body + assert add(1, 2) == 3 + # pad to push the next assertion further down + _ = add(0, 0) + _ = add(0, 0) + _ = add(0, 0) + _ = add(0, 0) + _ = add(0, 0) + _ = add(0, 0) + _ = add(0, 0) + # assertion on a later line (line 10+ of the body) + assert add(3, 4) == 7 + + +class TestCounter(unittest.TestCase): + """unittest.TestCase subclass — test_inc pairs to Counter.inc.""" + + def test_inc(self) -> None: + c = Counter() + c.inc() + self.assertEqual(c.value, 1) diff --git a/tests/test_classify_signals_method.py b/tests/test_classify_signals_method.py index 809142a..0c58dc7 100644 --- a/tests/test_classify_signals_method.py +++ b/tests/test_classify_signals_method.py @@ -38,7 +38,7 @@ def test_initialize_advertises_classify_signals(tmp_path: Path) -> None: resp = responses(_run(req("initialize", root_path=str(tmp_path)) + "\n"))[0] caps = resp["result"]["capabilities"] assert caps["classify_signals"] is True - assert caps["test_mapping"] is False + assert caps["test_mapping"] is True assert caps["streaming"] is False diff --git a/tests/test_discover_method.py b/tests/test_discover_method.py index d6082b0..c9911c8 100644 --- a/tests/test_discover_method.py +++ b/tests/test_discover_method.py @@ -83,6 +83,6 @@ def test_initialize_reports_discover_true() -> None: assert response["result"] == initialize_result() capabilities = response["result"]["capabilities"] assert capabilities["discover"] is True - assert capabilities["test_mapping"] is False + assert capabilities["test_mapping"] is True assert capabilities["classify_signals"] is True assert capabilities["streaming"] is False diff --git a/tests/test_protocol.py b/tests/test_protocol.py index e2dee56..47c7ba4 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -86,7 +86,7 @@ def test_initialize_result_schema() -> None: assert result["capabilities"] == { "discover": True, - "test_mapping": False, + "test_mapping": True, "classify_signals": True, "streaming": False, } diff --git a/tests/test_server.py b/tests/test_server.py index 48bebb8..32f93cf 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -231,9 +231,8 @@ def test_invalid_params_root_path_wrong_type(root_path: object) -> None: @pytest.mark.parametrize( "method", [ - # analyze/complexity/coverage/classify_signals are now implemented + # analyze/complexity/coverage/classify_signals/test_mapping are now implemented # (they return -32602 without params rather than METHOD_NOT_FOUND) - "test_mapping", "analyze/stream", ], ) diff --git a/tests/test_test_mapping_method.py b/tests/test_test_mapping_method.py new file mode 100644 index 0000000..f17364f --- /dev/null +++ b/tests/test_test_mapping_method.py @@ -0,0 +1,1779 @@ +"""Tests for the test_mapping JSON-RPC method and underlying pipeline. + +Covers tasks 8.1–8.15 from tasks.md: + +8.1 Pairing strategy confidence values +8.2 Strategy-3 BFS depth limit +8.3 Assertion types (all six) + node identification +8.4 Effect-type inference branches +8.5 Pipeline on sample_project +8.6 JSON-RPC end-to-end +8.7 Cross-subprocess determinism +8.8 Empty-result contracts +8.9 Safety (oversized/over-deep/FileNotFoundError) +8.10 Fixture isolation +8.11 Per-file coverage targets (enforced by running all branches) +8.12 Astroid cache isolation +8.13 Static-only sentinel +8.14 Same-named multi-package disambiguation +8.15 Path-valued fields are root-relative POSIX +""" + +from __future__ import annotations + +import ast +import io +import json +import os +import subprocess +import sys +import textwrap +from pathlib import Path +from unittest import mock + +import pytest +from conftest import req, responses + +from snake_eyes.analysis._shared import derive_package +from snake_eyes.analysis.effects import SideEffectType +from snake_eyes.analysis.models import Effect, FunctionRecord +from snake_eyes.protocol import INVALID_PARAMS +from snake_eyes.quality.assertions import collect_assertions +from snake_eyes.quality.mapping import infer_side_effect_type +from snake_eyes.quality.pairing import pair_tests +from snake_eyes.quality.pipeline import run_test_mapping +from snake_eyes.server import Server + +# --------------------------------------------------------------------------- +# Fixtures and helpers +# --------------------------------------------------------------------------- + +SAMPLE_PROJECT = Path(__file__).parent / "fixtures" / "sample_project" + + +def _run_server(raw: str) -> str: + stdout = io.StringIO() + server = Server(io.StringIO(raw), stdout, io.StringIO()) + with pytest.raises(SystemExit) as exc: + server.run() + assert exc.value.code == 0 + return stdout.getvalue() + + +def _parse_func(source: str) -> ast.FunctionDef | ast.AsyncFunctionDef: + """Parse *source* and return the first top-level function node.""" + tree = ast.parse(textwrap.dedent(source)) + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return node + raise AssertionError("no function found") + + +# --------------------------------------------------------------------------- +# 8.1 Pairing confidence values +# --------------------------------------------------------------------------- + + +class TestPairing: + """8.1 — pairing strategy confidence values.""" + + def _target_rec( + self, + tmp_path: Path, + func_name: str = "add", + ) -> FunctionRecord: + src = tmp_path / "m.py" + src.write_text(f"def {func_name}(a, b):\n return a + b\n") + return FunctionRecord( + name=func_name, + package=derive_package("m.py"), + file="m.py", + line=1, + ) + + def test_exact_name_match_confidence_90(self, tmp_path: Path) -> None: + rec = self._target_rec(tmp_path, "add") + (tmp_path / "test_m.py").write_text("def test_add():\n assert True\n") + tree = ast.parse("def test_add():\n assert True\n") + pairs = pair_tests( + [("test_add", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert len(pairs) == 1 + assert pairs[0].confidence == 90 + assert isinstance(pairs[0].confidence, int) + + def test_case_only_match_confidence_70(self, tmp_path: Path) -> None: + rec = self._target_rec(tmp_path, "add") + (tmp_path / "test_m.py").write_text("def test_Add():\n assert True\n") + tree = ast.parse("def test_Add():\n assert True\n") + pairs = pair_tests( + [("test_Add", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert len(pairs) == 1 + assert pairs[0].confidence == 70 + assert isinstance(pairs[0].confidence, int) + + def test_direct_call_confidence_80(self, tmp_path: Path) -> None: + (tmp_path / "m.py").write_text("def divide(a, b):\n return a / b\n") + rec = FunctionRecord( + name="divide", package=derive_package("m.py"), file="m.py", line=1 + ) + test_src = ( + "def test_it_works():\n result = divide(10, 2)\n assert result == 5\n" + ) + (tmp_path / "test_m.py").write_text(test_src) + tree = ast.parse(test_src) + pairs = pair_tests( + [("test_it_works", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert len(pairs) == 1 + assert pairs[0].confidence == 80 + assert isinstance(pairs[0].confidence, int) + + def test_no_match_yields_no_row(self, tmp_path: Path) -> None: + rec = self._target_rec(tmp_path, "compute") + tree = ast.parse("def test_nothing():\n assert True\n") + pairs = pair_tests( + [("test_nothing", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert pairs == [] + + def test_first_match_wins_single_row(self, tmp_path: Path) -> None: + """Same target name-matched AND called → single row, confidence 90.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + test_src = "def test_add():\n result = add(1, 2)\n assert result == 3\n" + tree = ast.parse(test_src) + pairs = pair_tests( + [("test_add", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert len(pairs) == 1 + assert pairs[0].confidence == 90 + + def test_helper_in_test_module_is_not_a_target(self, tmp_path: Path) -> None: + """A function defined in a test file is never a pairing target.""" + # target_records only contains production functions; test functions + # are never passed as targets (pipeline filters by source_files) + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + tree = ast.parse("def test_add():\n assert True\n") + pairs = pair_tests( + [("test_add", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + # Only production record (m.py:add) is a target + assert all(p.target_file == "m.py" for p in pairs) + + def test_confidence_is_int_not_float(self, tmp_path: Path) -> None: + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + tree = ast.parse("def test_add():\n assert True\n") + pairs = pair_tests( + [("test_add", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + for p in pairs: + assert isinstance(p.confidence, int) + assert 0 <= p.confidence <= 100 + + def test_confidence_range_all_strategies(self, tmp_path: Path) -> None: + """Confidence values produced by all three strategies are integers in [0, 100]. + + Drives real pair_tests calls to observe actual strategy outputs. + """ + # Strategy 1 exact (90) + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + rec_exact = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + tree_exact = ast.parse("def test_add():\n assert True\n") + pairs_90 = pair_tests( + [("test_add", "test_m.py")], + [rec_exact], + {"test_m.py": tree_exact}, + str(tmp_path), + ["m.py"], + ) + # Strategy 1 case-only (70) + tree_70 = ast.parse("def test_Add():\n assert True\n") + pairs_70 = pair_tests( + [("test_Add", "test_m.py")], + [rec_exact], + {"test_m.py": tree_70}, + str(tmp_path), + ["m.py"], + ) + # Strategy 2 direct call (80) + src_80 = "def test_it():\n add(1, 2)\n assert True\n" + tree_80 = ast.parse(src_80) + (tmp_path / "test_m.py").write_text(src_80) + pairs_80 = pair_tests( + [("test_it", "test_m.py")], + [rec_exact], + {"test_m.py": tree_80}, + str(tmp_path), + ["m.py"], + ) + all_pairs = pairs_90 + pairs_70 + pairs_80 + for p in all_pairs: + assert isinstance(p.confidence, int), ( + f"confidence must be int, got {type(p.confidence)}" + ) + assert 0 <= p.confidence <= 100, f"confidence {p.confidence} out of range" + confidences = {p.confidence for p in all_pairs} + # Should observe at least 90 and 70 from strategy 1, and 80 from strategy 2 + assert 90 in confidences, f"Expected 90 in confidences, got {confidences}" + assert 70 in confidences, f"Expected 70 in confidences, got {confidences}" + assert 80 in confidences, f"Expected 80 in confidences, got {confidences}" + + +# --------------------------------------------------------------------------- +# 8.2 Strategy-3 BFS depth limit (mocked call graph) +# --------------------------------------------------------------------------- + + +class TestStrategy3BFS: + """8.2 — strategy-3 BFS depth 5 pairs, depth 6 does not.""" + + def test_strategy3_transitive_across_files_real_pipeline( + self, tmp_path: Path + ) -> None: + """HIGH-1: strategy 3 pairs via transitive call chain across separate files. + + Only strategy 3 (transitive BFS) can pair test_nothing → target. + Drives the real ``run_test_mapping`` (no hand-built _CallGraph injection). + + Layout: + target_mod.py (SOURCE): target() <- production target + tests/helpers.py (TEST file, non-source): helper() -> target() + tests/test_main.py (TEST file): test_nothing() -> helper() + + ``target_records`` contains only ``target`` (source files only). + Strategy 1 fails: 'test_nothing' has no name-convention match to 'target'. + Strategy 2 fails: test_nothing calls helper(), which is NOT in target_records. + Strategy 3 must pair test_main.py → helpers.py → target_mod.py → target + with confidence 75. + """ + (tmp_path / "target_mod.py").write_text("def target():\n return 42\n") + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + # helpers.py lives in tests/ so it is discovered as a test file, NOT a source + # file, and its helper() function is never added to target_records. + (tests_dir / "helpers.py").write_text( + "from target_mod import target\n\n\ndef helper():\n return target()\n" + ) + # test_main.py: test_nothing calls helper() (not target directly) + (tests_dir / "test_main.py").write_text( + "from tests.helpers import helper\n" + "\n" + "\n" + "def test_nothing():\n" + " helper()\n" + " assert True\n" + ) + + rows = run_test_mapping(str(tmp_path), None) + target_rows = [r for r in rows if r["target_function"] == "target"] + # Strategy 3 must have found target via transitive BFS. + assert target_rows, ( + "Strategy 3 failed to pair test_nothing -> helper -> target. " + "Verify test files are included in the graph node set (HIGH-1 fix)." + ) + for row in target_rows: + assert row["confidence"] == 75, ( + f"Expected confidence 75 for transitive pair, got {row['confidence']}" + ) + + def test_strategy3_degrade_no_internal_error(self, tmp_path: Path) -> None: + """Strategy-3 failure degrades gracefully without surfacing -32603.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + tree = ast.parse("def test_nothing():\n assert True\n") + # patch _build_call_graph to raise unexpectedly + with mock.patch( + "snake_eyes.quality.pairing._build_call_graph", + side_effect=RuntimeError("astroid blew up"), + ): + pairs = pair_tests( + [("test_nothing", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + # No exception, just no pairs + assert isinstance(pairs, list) + + +# --------------------------------------------------------------------------- +# 8.3 Assertions: all six types + node identification +# --------------------------------------------------------------------------- + + +class TestAssertionTypes: + """8.3 — all six assertion_type values + node identification.""" + + @pytest.mark.parametrize( + "src,expected_type", + [ + # equality — bare assert == + ("def f():\n assert x == y\n", "equality"), + # equality — assertEqual + ("def f():\n self.assertEqual(a, b)\n", "equality"), + # equality — assertEquals + ("def f():\n self.assertEquals(a, b)\n", "equality"), + # equality — assertAlmostEqual + ("def f():\n self.assertAlmostEqual(a, b)\n", "equality"), + # equality — assertDictEqual + ("def f():\n self.assertDictEqual(a, b)\n", "equality"), + # equality — assertListEqual + ("def f():\n self.assertListEqual(a, b)\n", "equality"), + # equality — assertMultiLineEqual + ("def f():\n self.assertMultiLineEqual(a, b)\n", "equality"), + # equality — assertCountEqual + ("def f():\n self.assertCountEqual(a, b)\n", "equality"), + # equality — assertSequenceEqual + ("def f():\n self.assertSequenceEqual(a, b)\n", "equality"), + # comparison — != + ("def f():\n assert x != y\n", "comparison"), + # comparison — < + ("def f():\n assert x < y\n", "comparison"), + # comparison — > + ("def f():\n assert x > y\n", "comparison"), + # comparison — <= + ("def f():\n assert x <= y\n", "comparison"), + # comparison — >= + ("def f():\n assert x >= y\n", "comparison"), + # comparison — assertNotEqual + ("def f():\n self.assertNotEqual(a, b)\n", "comparison"), + # comparison — assertNotAlmostEqual + ("def f():\n self.assertNotAlmostEqual(a, b)\n", "comparison"), + # comparison — assertLess + ("def f():\n self.assertLess(a, b)\n", "comparison"), + # comparison — assertLessEqual + ("def f():\n self.assertLessEqual(a, b)\n", "comparison"), + # comparison — assertGreater + ("def f():\n self.assertGreater(a, b)\n", "comparison"), + # comparison — assertGreaterEqual + ("def f():\n self.assertGreaterEqual(a, b)\n", "comparison"), + # identity — is + ("def f():\n assert x is y\n", "identity"), + # identity — is not + ("def f():\n assert x is not y\n", "identity"), + # identity — assertIs + ("def f():\n self.assertIs(a, b)\n", "identity"), + # identity — assertIsNot + ("def f():\n self.assertIsNot(a, b)\n", "identity"), + # identity — assertIsNone + ("def f():\n self.assertIsNone(a)\n", "identity"), + # identity — assertIsNotNone + ("def f():\n self.assertIsNotNone(a)\n", "identity"), + # membership — in + ("def f():\n assert x in y\n", "membership"), + # membership — not in + ("def f():\n assert x not in y\n", "membership"), + # membership — assertIn + ("def f():\n self.assertIn(a, b)\n", "membership"), + # membership — assertNotIn + ("def f():\n self.assertNotIn(a, b)\n", "membership"), + # generic — assertTrue + ("def f():\n self.assertTrue(x)\n", "generic"), + # generic — assertFalse + ("def f():\n self.assertFalse(x)\n", "generic"), + # generic — bare assert (no operator) + ("def f():\n assert x\n", "generic"), + ], + ) + def test_assertion_type(self, src: str, expected_type: str) -> None: + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == expected_type + + def test_error_check_pytest_raises_as_with(self) -> None: + """with pytest.raises(...) yields exactly one error_check row.""" + src = ( + "def f():\n" + " import pytest\n" + " with pytest.raises(ValueError):\n" + " pass\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_error_check_assertRaises_unittest(self) -> None: + """self.assertRaises(...) as plain call → error_check.""" + src = "def f():\n self.assertRaises(ValueError, fn)\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_error_check_assertRaisesRegex(self) -> None: + src = "def f():\n self.assertRaisesRegex(ValueError, 'x')\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_error_check_assertRaisesRegexp(self) -> None: + src = "def f():\n self.assertRaisesRegexp(ValueError, 'x')\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_error_check_pytest_warns(self) -> None: + src = "def f():\n with pytest.warns(UserWarning):\n pass\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_error_check_bare_raises(self) -> None: + src = "def f():\n with raises(ValueError):\n pass\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_pytest_raises_as_with_counts_once(self) -> None: + """with pytest.raises(...) counts EXACTLY once, not also as a bare call.""" + src = "def f():\n with pytest.raises(ValueError):\n do_thing()\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + # Must be exactly 1 row (the with-item), not 2 (with-item + bare call) + assert len(assertions) == 1 + + def test_n_assertions_yield_n_rows(self) -> None: + src = "def f():\n assert a == b\n assert c != d\n assert e is f_\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 3 + + def test_nested_in_for_collected(self) -> None: + src = "def f():\n for x in items:\n assert x > 0\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_nested_in_with_collected(self) -> None: + src = "def f():\n with ctx():\n assert True\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_nested_in_if_collected(self) -> None: + src = "def f():\n if cond:\n assert True\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_nested_in_while_collected(self) -> None: + src = "def f():\n while cond:\n assert True\n break\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_nested_in_try_collected(self) -> None: + src = ( + "def f():\n" + " try:\n" + " assert True\n" + " except Exception:\n" + " pass\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_nested_in_def_excluded(self) -> None: + src = "def f():\n def inner():\n assert True\n assert False\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + # Only the outer assert False is collected + assert len(assertions) == 1 + assert assertions[0].assertion_type == "generic" + + def test_nested_in_class_excluded(self) -> None: + src = ( + "def f():\n" + " class Inner:\n" + " def m(self):\n" + " assert True\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 0 + + def test_assertion_location_format(self) -> None: + src = "def f():\n assert True\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert assertions[0].assertion_location == "tests/test_f.py:2" + + +# --------------------------------------------------------------------------- +# 8.4 Effect-type inference branches +# --------------------------------------------------------------------------- + + +class TestEffectTypeInference: + """8.4 — each branch of effect-type inference with hand-built side_effects.""" + + def _effect(self, et: SideEffectType) -> Effect: + return Effect(type=str(et), description="test") + + def test_error_check_has_ErrorReturn(self) -> None: + effects = ( + self._effect(SideEffectType.ErrorReturn), + self._effect(SideEffectType.ErrorSignal), + ) + result = infer_side_effect_type("error_check", effects) + assert result == str(SideEffectType.ErrorReturn) + + def test_error_check_no_ErrorReturn_has_ErrorSignal(self) -> None: + effects = (self._effect(SideEffectType.ErrorSignal),) + result = infer_side_effect_type("error_check", effects) + assert result == str(SideEffectType.ErrorSignal) + + def test_error_check_fallback(self) -> None: + result = infer_side_effect_type("error_check", ()) + assert result == str(SideEffectType.ErrorReturn) + + def test_value_has_ReturnValue(self) -> None: + effects = ( + self._effect(SideEffectType.ReturnValue), + self._effect(SideEffectType.ReceiverMutation), + ) + for atype in ("equality", "comparison", "identity", "membership"): + result = infer_side_effect_type(atype, effects) + assert result == str(SideEffectType.ReturnValue) + + def test_value_no_ReturnValue_uses_first_P0(self) -> None: + effects = (self._effect(SideEffectType.ReceiverMutation),) + result = infer_side_effect_type("equality", effects) + assert result == str(SideEffectType.ReceiverMutation) + + def test_value_no_ReturnValue_first_P0_not_first_effect(self) -> None: + """Non-P0 effect first, P0 second — must return the P0, not the first.""" + from snake_eyes.analysis.effects import TIER_MAP, Tier + + # Find a non-P0 effect type to put first + non_p0_type = next( + et + for et in SideEffectType + if TIER_MAP.get(et) != Tier.P0 and et not in (SideEffectType.ReturnValue,) + ) + p0_type = SideEffectType.ReceiverMutation # known P0 + assert TIER_MAP.get(p0_type) == Tier.P0 + effects = ( + self._effect(non_p0_type), # non-P0 first + self._effect(p0_type), # P0 second + ) + result = infer_side_effect_type("equality", effects) + # Must be the P0 effect, not the non-P0 that came first + assert result == str(p0_type), f"Expected first P0 ({p0_type}), got {result!r}" + + def test_value_fallback_ReturnValue(self) -> None: + result = infer_side_effect_type("equality", ()) + assert result == str(SideEffectType.ReturnValue) + + def test_generic_first_effect(self) -> None: + effects = (self._effect(SideEffectType.GlobalMutation),) + result = infer_side_effect_type("generic", effects) + assert result == str(SideEffectType.GlobalMutation) + + def test_generic_fallback(self) -> None: + result = infer_side_effect_type("generic", ()) + assert result == str(SideEffectType.ReturnValue) + + def test_unknown_type_string_safe(self) -> None: + """TIER_MAP.get() is safe on unknown type strings — no KeyError.""" + effects = (Effect(type="UnknownFutureEffect", description="x"),) + result = infer_side_effect_type("equality", effects) + # Falls through to ReturnValue fallback + assert result == str(SideEffectType.ReturnValue) + + +# --------------------------------------------------------------------------- +# 8.5 Pipeline on sample_project +# --------------------------------------------------------------------------- + + +class TestPipeline: + """8.5 — pipeline integration on the sample_project fixture.""" + + def test_pipeline_returns_ge_2_rows(self) -> None: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + assert len(rows) >= 2 + + def test_all_required_keys_present(self) -> None: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + required = { + "test_function", + "test_file", + "assertion_location", + "assertion_type", + "target_function", + "target_package", + "side_effect_type", + "confidence", + } + for row in rows: + assert set(row.keys()) == required + + def test_unittest_method_is_collected(self) -> None: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + # TestCounter.test_inc should appear + funcs = {r["test_function"] for r in rows} + assert any("test_inc" in f for f in funcs) + + def test_numeric_ordering_not_lexicographic(self, tmp_path: Path) -> None: + """Assertions at lines 9 and 10 are ordered numerically (9<10), not lexically. + + Lexicographic ordering would put "10" before "9"; numeric ordering puts 9 first. + This test catches string-sort regressions. + """ + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + # Craft a test function where assertions land on lines 9 and 10 of the file. + # Lines 1-8: header + padding; line 9: assert 1; line 10: assert 2. + lines = [ + "def test_add():", # line 1 + " _ = 1", # line 2 + " _ = 2", # line 3 + " _ = 3", # line 4 + " _ = 4", # line 5 + " _ = 5", # line 6 + " _ = 6", # line 7 + " _ = 7", # line 8 + " assert add(1, 2) == 3", # line 9 + " assert add(2, 3) == 5", # line 10 + ] + (tests / "test_prod.py").write_text("\n".join(lines) + "\n") + rows = run_test_mapping(str(tmp_path), None) + add_rows = [r for r in rows if r["target_function"] == "add"] + assert len(add_rows) == 2, f"Expected 2 rows, got {len(add_rows)}" + line_nums = [int(r["assertion_location"].split(":")[-1]) for r in add_rows] + # Numeric order: 9 before 10 + assert line_nums == sorted(line_nums), ( + f"Lines not sorted numerically: {line_nums}" + ) + # Discriminating check: numeric 9 < 10, but lexicographic "10" < "9" + assert line_nums[0] == 9 and line_nums[1] == 10, ( + f"Expected [9, 10], got {line_nums}" + ) + + def test_target_package_equals_derive_package(self) -> None: + """target_package matches derive_package for a fixture source file.""" + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + # Find a row targeting calculator.py functions + calc_rows = [r for r in rows if "calculator" in r["target_package"]] + assert calc_rows, "Expected at least one row targeting calculator functions" + expected_package = derive_package("src/sample/calculator.py") + for row in calc_rows: + assert row["target_package"] == expected_package, ( + f"Expected target_package == {expected_package!r}," + f" got {row['target_package']!r}" + ) + + def test_same_line_tiebreaker(self, tmp_path: Path) -> None: + """Col-offset tiebreaker is genuinely exercised: 4 rows, 2 packages. + + Layout: + pkg_a/calc.py def add(a, b) + pkg_b/calc.py def add(a, b) + tests/test_calc.py + def test_add(): + assert add(1,1)==2; assert add(1,1) in (2,) # noqa: E702 + + Each assert on the single physical line pairs to ``add`` in BOTH + packages → 4 rows sharing (test_file, test_function, assertion_line). + + With the _col sort key the sequence groups col-first: + equality / equality / membership / membership + Without _col it would group by target_package order: + equality / membership / equality / membership (or similar) + + This confirms the _col component is load-bearing. + """ + # Two packages each defining add + pkg_a = tmp_path / "pkg_a" + pkg_a.mkdir() + (pkg_a / "__init__.py").write_text("") + (pkg_a / "calc.py").write_text("def add(a, b):\n return a + b\n") + pkg_b = tmp_path / "pkg_b" + pkg_b.mkdir() + (pkg_b / "__init__.py").write_text("") + (pkg_b / "calc.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + # Two assertions on ONE physical line. The semicolon is deliberate and + # safe (ruff-format doesn't reformat files in tmp_path). + test_src = ( + "def test_add():\n" + " assert add(1,1)==2; assert add(1,1) in (2,) # noqa: E702\n" + ) + (tests / "test_calc.py").write_text(test_src) + + rows = run_test_mapping(str(tmp_path), None) + add_rows = [r for r in rows if r["target_function"] == "add"] + + # With two packages, strategy-1 name-match fires for both → 4 rows + assert len(add_rows) == 4, ( + f"Expected 4 rows (2 assertions × 2 packages)," + f" got {len(add_rows)}: {add_rows}" + ) + + # The _col tiebreaker means all col-0 rows (equality) come before all + # col-N rows (membership), regardless of target_package insertion order. + types = [r["assertion_type"] for r in add_rows] + assert types == ["equality", "equality", "membership", "membership"], ( + f"Expected col-sorted sequence" + f" ['equality','equality','membership','membership']," + f" got {types}" + ) + + def test_two_package_disambiguation(self, tmp_path: Path) -> None: + """8.14 — two packages each defining add yield one row per package.""" + pkg_a = tmp_path / "pkg_a" + pkg_a.mkdir() + (pkg_a / "__init__.py").write_text("") + (pkg_a / "math.py").write_text("def add(a, b):\n return a + b\n") + pkg_b = tmp_path / "pkg_b" + pkg_b.mkdir() + (pkg_b / "__init__.py").write_text("") + (pkg_b / "math.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_math.py").write_text("def test_add():\n assert True\n") + rows = run_test_mapping(str(tmp_path), None) + add_rows = [r for r in rows if r["target_function"] == "add"] + # Should have rows for both packages + packages = {r["target_package"] for r in add_rows} + assert len(packages) >= 2 + + def test_helper_in_test_module_never_a_target_pipeline( + self, tmp_path: Path + ) -> None: + """MED-D: test-file helper with same name as source function is never a target. + + The pipeline filters targets to source_files only, so a helper ``add`` + defined in the test file must never appear as a target row. + """ + (tmp_path / "calc.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + # Test file defines both test_add() AND a helper add() with the same name + (tests / "test_calc.py").write_text( + "def add(a, b):\n" + " return a + b\n" + "\n" + "\n" + "def test_add():\n" + " assert add(1, 2) == 3\n" + ) + rows = run_test_mapping(str(tmp_path), None) + add_rows = [r for r in rows if r["target_function"] == "add"] + assert add_rows, "Expected at least one row pairing test_add to source add" + expected_pkg = derive_package("calc.py") + # Every row must point to the SOURCE module (calc.py), never the test file + for row in add_rows: + assert row["target_package"] == expected_pkg, ( + f"target_package should be {expected_pkg!r}," + f" got {row['target_package']!r} — test-file add must never be a target" + ) + + def test_empty_result_no_pairs(self, tmp_path: Path) -> None: + """Pipeline returns [] for a project with test files but no pairs.""" + (tmp_path / "prod.py").write_text("def compute_xyz():\n return 42\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_unrelated.py").write_text( + "def test_totally_unrelated():\n assert True\n" + ) + rows = run_test_mapping(str(tmp_path), None) + assert rows == [] + + +# --------------------------------------------------------------------------- +# 8.6 JSON-RPC end-to-end +# --------------------------------------------------------------------------- + + +class TestJsonRpcEndToEnd: + """8.6 — JSON-RPC e2e tests for test_mapping.""" + + def test_initialize_advertises_test_mapping_true(self, tmp_path: Path) -> None: + resp = responses( + _run_server(req("initialize", root_path=str(tmp_path)) + "\n") + )[0] + caps = resp["result"]["capabilities"] + assert caps["test_mapping"] is True + + def test_test_mapping_returns_mappings_key(self) -> None: + resp = responses( + _run_server(req("test_mapping", root_path=str(SAMPLE_PROJECT)) + "\n") + )[0] + assert "result" in resp + assert "error" not in resp + assert "mappings" in resp["result"] + assert isinstance(resp["result"]["mappings"], list) + + def test_confidence_is_int_in_range(self) -> None: + resp = responses( + _run_server(req("test_mapping", root_path=str(SAMPLE_PROJECT)) + "\n") + )[0] + for row in resp["result"]["mappings"]: + assert isinstance(row["confidence"], int) + assert 0 <= row["confidence"] <= 100 + + def test_missing_root_path_returns_32602(self) -> None: + resp = responses( + _run_server(req("test_mapping", root_path="/nonexistent/path/xyz") + "\n") + )[0] + assert resp["error"]["code"] == INVALID_PARAMS + + def test_no_params_returns_32602(self) -> None: + raw = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "test_mapping"}) + resp = responses(_run_server(raw + "\n"))[0] + assert resp["error"]["code"] == INVALID_PARAMS + + +# --------------------------------------------------------------------------- +# 8.7 Cross-subprocess determinism +# --------------------------------------------------------------------------- + + +class TestCrossSubprocessDeterminism: + """8.7 — byte-identical output with PYTHONHASHSEED=0 vs PYTHONHASHSEED=1.""" + + @pytest.mark.slow + def test_byte_identical_across_hash_seeds(self) -> None: + request = ( + json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "test_mapping", + "params": {"root_path": str(SAMPLE_PROJECT)}, + } + ) + + "\n" + ) + + def _spawn(seed: str) -> str: + env = dict(os.environ) + env["PYTHONHASHSEED"] = seed + result = subprocess.run( + [sys.executable, "-m", "snake_eyes", "--stdio"], + input=request, + capture_output=True, + text=True, + env=env, + timeout=60, + ) + return result.stdout + + out0 = _spawn("0") + out1 = _spawn("1") + assert out0 == out1, "Output differs across PYTHONHASHSEED values" + + +# --------------------------------------------------------------------------- +# 8.8 Empty-result contracts +# --------------------------------------------------------------------------- + + +class TestEmptyResults: + """8.8 — empty result contracts.""" + + def test_pipeline_returns_empty_list_no_pairs(self, tmp_path: Path) -> None: + (tmp_path / "prod.py").write_text("def xyz():\n pass\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_x.py").write_text("def test_abc():\n assert True\n") + rows = run_test_mapping(str(tmp_path), None) + assert rows == [] + + def test_e2e_empty_mappings_no_error(self, tmp_path: Path) -> None: + """test_mapping returns {"mappings": []} for a no-test project, no error key.""" + resp = responses( + _run_server(req("test_mapping", root_path=str(tmp_path)) + "\n") + )[0] + assert "result" in resp + assert "error" not in resp + assert resp["result"]["mappings"] == [] + + +# --------------------------------------------------------------------------- +# 8.9 Safety +# --------------------------------------------------------------------------- + + +class TestSafety: + """8.9 — oversized/over-deep files, FileNotFoundError propagation.""" + + def test_oversized_test_file_skipped_no_crash(self, tmp_path: Path) -> None: + """An oversized test file is skipped without surfacing as -32603.""" + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + # Write a test file that exceeds MAX_FILE_BYTES + # We mock is_analyzable_file to simulate skipping an oversized file + with mock.patch( + "snake_eyes.analysis._shared.is_analyzable_file", return_value=False + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + def test_filenotfounderror_propagates(self) -> None: + """A non-existent root_path → FileNotFoundError, not absorbed by strategy-3.""" + with pytest.raises(FileNotFoundError): + run_test_mapping("/totally/nonexistent/path/abc123", None) + + def test_e2e_filenotfounderror_maps_to_32602(self) -> None: + resp = responses( + _run_server( + req("test_mapping", root_path="/totally/nonexistent/abc") + "\n" + ) + )[0] + assert resp["error"]["code"] == INVALID_PARAMS + + def test_strategy3_pathological_degrades_gracefully(self, tmp_path: Path) -> None: + """Strategy-3 RecursionError/MemoryError degrades, no -32603.""" + (tmp_path / "prod.py").write_text("def xyz():\n pass\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_x.py").write_text("def test_nothing():\n assert True\n") + with mock.patch( + "snake_eyes.quality.pairing._build_call_graph", + side_effect=MemoryError("OOM"), + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + +# --------------------------------------------------------------------------- +# 8.10 Fixture isolation +# --------------------------------------------------------------------------- + + +def test_fixture_not_collected_by_host_pytest() -> None: + """The fixture test file is NOT collected by the host pytest run.""" + # Run pytest --collect-only on the tests/ dir and check the fixture is absent + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + str(Path(__file__).parent), + ], + capture_output=True, + text=True, + timeout=60, + ) + assert "test_calculator.py" not in result.stdout + assert "fixtures/sample_project" not in result.stdout + + +# --------------------------------------------------------------------------- +# 8.12 Astroid cache isolation +# --------------------------------------------------------------------------- + + +class TestAstroidCacheIsolation: + """8.12 — astroid.MANAGER cleared before + after each strategy-3 build.""" + + def test_two_requests_do_not_contaminate(self, tmp_path: Path) -> None: + """Two run_test_mapping calls over different trees don't share astroid state.""" + # Project 1 + p1 = tmp_path / "p1" + p1.mkdir() + (p1 / "m.py").write_text("def add(a, b):\n return a + b\n") + t1 = p1 / "tests" + t1.mkdir() + (t1 / "test_m.py").write_text("def test_add():\n assert True\n") + # Project 2 + p2 = tmp_path / "p2" + p2.mkdir() + (p2 / "n.py").write_text("def multiply(a, b):\n return a * b\n") + t2 = p2 / "tests" + t2.mkdir() + (t2 / "test_n.py").write_text("def test_multiply():\n assert True\n") + rows1 = run_test_mapping(str(p1), None) + rows2 = run_test_mapping(str(p2), None) + # Results should be independent + funcs1 = {r["target_function"] for r in rows1} + funcs2 = {r["target_function"] for r in rows2} + assert "add" in funcs1 + assert "multiply" in funcs2 + # No cross-contamination: multiply should not appear in rows1 + assert "multiply" not in funcs1 + assert "add" not in funcs2 + + +# --------------------------------------------------------------------------- +# 8.13 Static-only sentinel +# --------------------------------------------------------------------------- + + +class TestStaticOnlySentinel: + """8.13 — run_test_mapping never runs pytest, executes code, or reads coverage.""" + + def test_no_pytest_subprocess(self) -> None: + """run_test_mapping does not spawn pytest as a subprocess.""" + with mock.patch("subprocess.run") as mock_run: + with mock.patch("subprocess.Popen") as mock_popen: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + mock_run.assert_not_called() + mock_popen.assert_not_called() + assert isinstance(rows, list) + + def test_no_coverage_read(self) -> None: + """run_test_mapping does not call parse_coverage.""" + with mock.patch("snake_eyes.coverage.parse_coverage") as mock_cov: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + mock_cov.assert_not_called() + assert isinstance(rows, list) + + def test_no_exec_or_eval(self) -> None: + """run_test_mapping does not call exec or eval on analyzed code.""" + exec_called = [] + eval_called = [] + + def patched_exec(*args: object, **kwargs: object) -> None: + exec_called.append(args) + + def patched_eval(*args: object, **kwargs: object) -> None: + eval_called.append(args) + + import builtins + + with mock.patch.object(builtins, "exec", patched_exec): + with mock.patch.object(builtins, "eval", patched_eval): + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + assert exec_called == [], "exec was called during test mapping" + assert eval_called == [], "eval was called during test mapping" + assert isinstance(rows, list) + + +# --------------------------------------------------------------------------- +# 8.15 Path-valued fields are root-relative POSIX +# --------------------------------------------------------------------------- + + +class TestPathFields: + """8.15 — test_file and path-valued fields are root-relative POSIX.""" + + def test_test_file_is_relative_posix(self) -> None: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + for row in rows: + path = row["test_file"] + assert not path.startswith("/"), f"test_file is absolute: {path}" + assert "\\" not in path, f"test_file uses backslash: {path}" + + def test_assertion_location_is_relative_posix(self) -> None: + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + for row in rows: + loc = row["assertion_location"] + path_part = loc.rsplit(":", 1)[0] + assert not path_part.startswith("/"), ( + f"assertion_location path is absolute: {loc}" + ) + assert "\\" not in path_part + + +# --------------------------------------------------------------------------- +# Additional coverage tests +# --------------------------------------------------------------------------- + + +class TestPairingCoverage: + """Additional tests to hit pairing.py coverage targets (≥90%).""" + + def test_strip_prefix_bare_test_no_underscore(self, tmp_path: Path) -> None: + """testFoo (no underscore) is stripped to Foo → case-only match.""" + (tmp_path / "m.py").write_text("def foo(a, b):\n return a + b\n") + rec = FunctionRecord( + name="foo", package=derive_package("m.py"), file="m.py", line=1 + ) + # testFoo → strips 'test' → 'Foo', matches 'foo' case-insensitive + tree = ast.parse("def testFoo():\n assert True\n") + pairs = pair_tests( + [("testFoo", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert len(pairs) == 1 + assert pairs[0].confidence == 70 + + def test_strip_prefix_no_test_prefix(self, tmp_path: Path) -> None: + """A function without a test prefix does not match by name convention.""" + rec = FunctionRecord( + name="foo", package=derive_package("m.py"), file="m.py", line=1 + ) + tree = ast.parse("def helper():\n assert True\n") + pairs = pair_tests( + [("helper", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + # No name match — helper has no test prefix + name_pairs = [p for p in pairs if p.confidence == 90 or p.confidence == 70] + assert name_pairs == [] + + def test_direct_call_class_method(self, tmp_path: Path) -> None: + """Direct-call matching finds calls in class method bodies.""" + (tmp_path / "m.py").write_text("def divide(a, b):\n return a / b\n") + rec = FunctionRecord( + name="divide", package=derive_package("m.py"), file="m.py", line=1 + ) + # Class method calling divide + test_src = ( + "class TestOps:\n" + " def test_it_divides(self):\n" + " result = divide(10, 2)\n" + " assert result == 5\n" + ) + (tmp_path / "test_m.py").write_text(test_src) + tree = ast.parse(test_src) + # Use the class-qualified form + pairs = pair_tests( + [("TestOps.test_it_divides", "test_m.py")], + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + assert any(p.target_function == "divide" for p in pairs) + + def test_no_tree_for_test_file(self, tmp_path: Path) -> None: + """If a test file has no tree, no pairs are emitted.""" + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + # test_m.py not in test_trees dict + pairs = pair_tests( + [("test_add", "test_m.py")], + [rec], + {}, # empty: no tree for test_m.py + str(tmp_path), + ["m.py"], + ) + assert pairs == [] + + +class TestAssertionsCoverage: + """Additional tests to hit assertions.py coverage targets (≥90%).""" + + def test_for_orelse_collected(self) -> None: + src = ( + "def f():\n" + " for x in items:\n" + " pass\n" + " else:\n" + " assert True\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_while_orelse_collected(self) -> None: + src = ( + "def f():\n while cond:\n pass\n else:\n assert True\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_try_finally_collected(self) -> None: + src = "def f():\n try:\n pass\n finally:\n assert True\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_try_orelse_collected(self) -> None: + src = ( + "def f():\n" + " try:\n" + " pass\n" + " except Exception:\n" + " pass\n" + " else:\n" + " assert True\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_if_orelse_collected(self) -> None: + src = "def f():\n if cond:\n pass\n else:\n assert True\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + def test_assertRaises_as_with_item_not_double_counted(self) -> None: + src = "def f():\n with self.assertRaises(ValueError):\n do_it()\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_assertWarns_method(self) -> None: + src = "def f():\n self.assertWarns(UserWarning)\n" + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + assert assertions[0].assertion_type == "error_check" + + def test_nested_with_in_try_collected(self) -> None: + src = ( + "def f():\n" + " try:\n" + " with ctx():\n" + " assert True\n" + " except Exception:\n" + " pass\n" + ) + func_node = _parse_func(src) + assertions = collect_assertions(func_node, "tests/test_f.py") + assert len(assertions) == 1 + + +class TestPipelineCoverage: + """Additional tests to hit pipeline.py coverage targets (≥85%).""" + + def test_unittest_testcase_attribute_form(self, tmp_path: Path) -> None: + """unittest.TestCase (attribute form) is detected as TestCase subclass.""" + (tmp_path / "m.py").write_text("def inc():\n pass\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text( + "import unittest\n" + "class TestOps(unittest.TestCase):\n" + " def test_inc(self):\n" + " self.assertTrue(True)\n" + ) + rows = run_test_mapping(str(tmp_path), None) + funcs = {r["test_function"] for r in rows} + # test_inc should be collected as a TestCase method + assert any("test_inc" in f for f in funcs) or rows == [] + + def test_get_func_node_class_method(self, tmp_path: Path) -> None: + """Pipeline collects assertions from class.method test functions.""" + rows = run_test_mapping(str(SAMPLE_PROJECT), None) + # TestCounter.test_inc should produce rows + class_method_rows = [r for r in rows if "." in r["test_function"]] + # Should have at least the TestCounter.test_inc row + assert len(class_method_rows) >= 1 + + def test_no_test_files_returns_empty(self, tmp_path: Path) -> None: + """Pipeline returns [] when there are no test files.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + rows = run_test_mapping(str(tmp_path), None) + assert rows == [] + + def test_empty_test_functions_no_target(self, tmp_path: Path) -> None: + """Pipeline returns [] when test file has no test_ functions.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def helper():\n pass\n") + rows = run_test_mapping(str(tmp_path), None) + assert rows == [] + + def test_depth_guard_skips_over_deep_test(self, tmp_path: Path) -> None: + """Over-deep test files are skipped without surfacing as -32603.""" + + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + # Write a normal test file (the depth guard is tested via mock) + (tests / "test_m.py").write_text("def test_add():\n assert True\n") + with mock.patch( + "snake_eyes.analysis._shared.enumerate_functions_with_spans", + side_effect=RecursionError("depth exceeded"), + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + +class TestStrategy3ActualGraph: + """Tests that exercise the actual astroid call graph building code.""" + + def test_strategy3_builds_and_finds_target(self, tmp_path: Path) -> None: + """Trigger strategy 3 with an actual tiny project astroid can parse.""" + # Two files: prod.py with helper() calling target(), test calls helper() + (tmp_path / "prod.py").write_text( + "def target():\n return 42\n\n\ndef helper():\n return target()\n" + ) + (tmp_path / "test_prod.py").write_text( + "from prod import helper\n\n\n" + "def test_nothing_by_name():\n helper()\n assert True\n" + ) + # Run the full pipeline - strategy 3 may or may not fire depending on astroid + rows = run_test_mapping(str(tmp_path), None) + # Just verify no crash + assert isinstance(rows, list) + + def test_strategy3_exception_in_lookup_degrades(self, tmp_path: Path) -> None: + """Exception in strategy-3 lookup is caught and degraded.""" + (tmp_path / "m.py").write_text("def xyz():\n pass\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_nothing():\n assert True\n") + + # Use a real _CallGraph that raises on reachable_files + from snake_eyes.quality.pairing import _CallGraph + + broken_graph = _CallGraph(edges={}) + + def bad_reachable(start: str, depth_limit: int = 5) -> set[str]: + raise RuntimeError("graph error") + + broken_graph.reachable_files = bad_reachable # type: ignore[method-assign] + + with mock.patch( + "snake_eyes.quality.pairing._build_call_graph", + return_value=broken_graph, + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + def test_dedup_key_prevents_duplicate_pairs(self, tmp_path: Path) -> None: + """De-dup key (test_function, test_file, package, target) prevents dups.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + rec = FunctionRecord( + name="add", package=derive_package("m.py"), file="m.py", line=1 + ) + # Two test functions that would both map to the same pair + test_src = ( + "def test_add():\n assert True\ndef test_add2():\n assert True\n" + ) + tree = ast.parse(test_src) + pairs = pair_tests( + [("test_add", "test_m.py"), ("test_add", "test_m.py")], # duplicate! + [rec], + {"test_m.py": tree}, + str(tmp_path), + ["m.py"], + ) + # Dedup: second identical (test_add, test_m.py, pkg, add) is dropped + matching = [p for p in pairs if p.test_function == "test_add"] + assert len(matching) == 1 + + def test_astroid_module_error_during_build(self, tmp_path: Path) -> None: + """Error during ast_from_file in _build_call_graph is caught per-file.""" + import astroid as _astroid # type: ignore[import-untyped] + + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_nothing():\n assert True\n") + + def raise_for_rel(path: str, modname: str, source: bool = False) -> object: + raise RuntimeError("simulated parse error") + + with mock.patch.object(_astroid.MANAGER, "ast_from_file", raise_for_rel): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + def test_strategy3_outer_exception_degrades(self, tmp_path: Path) -> None: + """Outer Exception in _build_call_graph is caught.""" + (tmp_path / "m.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_nothing():\n assert True\n") + + with mock.patch( + "snake_eyes.quality.pairing.astroid.MANAGER.clear_cache", + side_effect=RuntimeError("cache clear failed"), + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + +class TestPipelineCoverage2: + """Additional pipeline coverage tests.""" + + def test_testcase_bare_name_detected(self, tmp_path: Path) -> None: + """class Foo(TestCase) (bare name, not unittest.TestCase) is detected.""" + (tmp_path / "prod.py").write_text("def inc():\n pass\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text( + "from unittest import TestCase\n\n\n" + "class TestOps(TestCase):\n" + " def test_inc(self):\n" + " self.assertTrue(True)\n" + ) + rows = run_test_mapping(str(tmp_path), None) + funcs = {r["test_function"] for r in rows} + assert any("test_inc" in f for f in funcs) or rows == [] + + def test_get_func_node_returns_none_for_missing_class(self, tmp_path: Path) -> None: + """_get_func_node returns None when class not found.""" + from snake_eyes.quality.pipeline import _get_func_node + + tree = ast.parse("def test_x():\n assert True\n") + result = _get_func_node(tree, "NonExistentClass.test_x") + assert result is None + + def test_get_func_node_returns_none_for_missing_func(self, tmp_path: Path) -> None: + """_get_func_node returns None when top-level function not found.""" + from snake_eyes.quality.pipeline import _get_func_node + + tree = ast.parse("def test_x():\n assert True\n") + result = _get_func_node(tree, "test_nonexistent") + assert result is None + + def test_assertion_recursion_skipped(self, tmp_path: Path) -> None: + """RecursionError in collect_assertions skips the pair's assertions.""" + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_add():\n assert True\n") + with mock.patch( + "snake_eyes.quality.pipeline.collect_assertions", + side_effect=RecursionError("depth exceeded"), + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + def test_assertion_broadened_exception_skipped(self, tmp_path: Path) -> None: + """BROADENED_EXCEPTIONS in collect_assertions skips the pair's assertions.""" + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_add():\n assert True\n") + with mock.patch( + "snake_eyes.quality.pipeline.collect_assertions", + side_effect=OSError("IO error"), + ): + rows = run_test_mapping(str(tmp_path), None) + assert isinstance(rows, list) + + def test_pair_tree_none_continues(self, tmp_path: Path) -> None: + """If pair.test_file has no tree entry, the pair is skipped.""" + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_add():\n assert True\n") + # Patch pair_tests to return a pair whose test_file has no tree + from snake_eyes.quality.pairing import PairedResult + + fake_pair = PairedResult( + test_function="test_add", + test_file="tests/nonexistent.py", + target_function="add", + target_package="prod", + target_file="prod.py", + confidence=90, + ) + with mock.patch( + "snake_eyes.quality.pipeline.pair_tests", return_value=[fake_pair] + ): + rows = run_test_mapping(str(tmp_path), None) + assert rows == [] + + +class TestPairingCoverage2: + """Additional pairing coverage (strategy-3 code paths).""" + + def test_transitive_match_finds_target(self, tmp_path: Path) -> None: + """_transitive_match returns confidence 75 for reachable files.""" + + from snake_eyes.quality.pairing import _CallGraph, _transitive_match + + # Create a real file so _normalize_path can resolve it + f = tmp_path / "prod.py" + f.write_text("def add(a, b):\n return a + b\n") + + norm_prod = str(f.resolve()) + norm_test = str((tmp_path / "test_prod.py").resolve()) + + graph = _CallGraph(edges={norm_test: {norm_prod}}) + # Use a root-relative file path — consistent with production behaviour. + rec = FunctionRecord( + name="add", + package=derive_package("prod.py"), + file="prod.py", + line=1, + ) + results = _transitive_match(norm_test, graph, [rec], str(tmp_path)) + assert any(r[1] == 75 for r in results) + + def test_strategy3_finalizer_clears_cache_on_success(self, tmp_path: Path) -> None: + """After successful strategy-3 (or any run), astroid cache is cleared.""" + import astroid as _astroid # type: ignore[import-untyped] + + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + tests = tmp_path / "tests" + tests.mkdir() + (tests / "test_m.py").write_text("def test_add():\n assert True\n") + + clear_calls = [] + original = _astroid.MANAGER.clear_cache + + def spy_clear() -> None: + clear_calls.append(1) + original() + + with mock.patch.object(_astroid.MANAGER, "clear_cache", spy_clear): + run_test_mapping(str(tmp_path), None) + # finally-clear is unconditional, so clear_calls must be non-empty. + assert clear_calls + + def test_build_call_graph_with_parseable_project(self, tmp_path: Path) -> None: + """_build_call_graph runs on a tiny parseable project.""" + from snake_eyes.quality.pairing import _build_call_graph + + (tmp_path / "prod.py").write_text( + "def add(a, b):\n return a + b\n\n\n" + "def helper():\n return add(1, 2)\n" + ) + graph = _build_call_graph(str(tmp_path), ["prod.py"]) + assert graph is not None + + def test_build_call_graph_missing_file_skips_gracefully( + self, tmp_path: Path + ) -> None: + """Missing files are skipped per-file (not a fatal error in the graph build).""" + from snake_eyes.quality.pairing import _build_call_graph + + # nonexistent.py is skipped with a diagnostic; returns a graph with no edges + graph = _build_call_graph(str(tmp_path), ["nonexistent.py"]) + assert graph is not None + assert graph.edges == {} + + def test_name_match_test_prefix_without_underscore(self, tmp_path: Path) -> None: + """test prefix (no underscore) is stripped for matching.""" + from snake_eyes.quality.pairing import _name_match + + rec = FunctionRecord(name="foo", package="pkg", file="m.py", line=1) + # testFoo -> 'Foo' -> case-only match with 'foo' + results = _name_match("testFoo", [rec]) + assert any(r[1] in (70, 90) for r in results) + + def test_name_match_no_prefix_no_match(self) -> None: + """A function with no test prefix returns empty.""" + from snake_eyes.quality.pairing import _name_match + + rec = FunctionRecord(name="foo", package="pkg", file="m.py", line=1) + results = _name_match("helper", [rec]) + assert results == [] + + +class TestPairingStrategy3Paths: + """Tests that exercise specific strategy-3 code paths in pairing.py.""" + + def test_astroid_error_during_build_degrades(self, tmp_path: Path) -> None: + """AstroidError during call graph build degrades gracefully.""" + from snake_eyes.quality.pairing import _build_call_graph + + (tmp_path / "prod.py").write_text("def add(a, b):\n return a + b\n") + + import astroid as _astroid # type: ignore[import-untyped] + from astroid.exceptions import AstroidError # type: ignore[import-untyped] + + with mock.patch.object( + _astroid.MANAGER, + "clear_cache", + side_effect=AstroidError("test error"), + ): + result = _build_call_graph(str(tmp_path), ["prod.py"]) + assert result is None + + def test_strategy3_triggers_when_s1_s2_fail(self, tmp_path: Path) -> None: + """Strategy 3 triggers when neither s1 nor s2 find a match.""" + # Create a project with a test that doesn't match by name or direct call + prod_file = tmp_path / "prod.py" + prod_file.write_text("def compute():\n return 42\n") + + from snake_eyes.quality.pairing import _CallGraph + + norm_test = str((tmp_path / "test_prod.py").resolve()) + norm_prod = str(prod_file.resolve()) + graph = _CallGraph(edges={norm_test: {norm_prod}}) + + # Use a root-relative file path — consistent with production behaviour. + # _transitive_match resolves it via root_abs / rec.file. + rec = FunctionRecord( + name="compute", + package=derive_package("prod.py"), + file="prod.py", + line=1, + ) + # test_nothing: no name match, no direct call to compute + tree = ast.parse("def test_nothing():\n assert True\n") + + with mock.patch( + "snake_eyes.quality.pairing._build_call_graph", return_value=graph + ): + pairs = pair_tests( + [("test_nothing", "test_prod.py")], + [rec], + {"test_prod.py": tree}, + str(tmp_path), + ["prod.py"], + ) + # Strategy 3 should have found compute via the mocked graph + assert any(p.confidence == 75 for p in pairs) + + def test_strategy3_file_not_found_propagates_from_pair_tests( + self, tmp_path: Path + ) -> None: + """FileNotFoundError from _get_graph propagates (not absorbed).""" + rec = FunctionRecord(name="compute", package="pkg", file="prod.py", line=1) + tree = ast.parse("def test_nothing():\n assert True\n") + + with mock.patch( + "snake_eyes.quality.pairing._build_call_graph", + side_effect=FileNotFoundError("root not found"), + ): + with pytest.raises(FileNotFoundError): + pair_tests( + [("test_nothing", "test_prod.py")], + [rec], + {"test_prod.py": tree}, + str(tmp_path), + ["prod.py"], + ) + + +class TestPairingCoverage3: + """More pairing coverage: attribute calls, edge cases.""" + + def test_strip_prefix_Test_capital(self) -> None: + """TestFoo (capital Test prefix) is stripped to Foo.""" + from snake_eyes.quality.pairing import _strip_test_prefix + + result = _strip_test_prefix("TestFoo") + assert result == "Foo" + + def test_build_call_graph_attribute_calls(self, tmp_path: Path) -> None: + """Attribute-form calls (obj.method()) are handled in the call graph.""" + from snake_eyes.quality.pairing import _build_call_graph + + # File with an attribute-form call + (tmp_path / "prod.py").write_text( + "class Calculator:\n" + " def add(self, a, b):\n" + " return a + b\n\n\n" + "def run():\n" + " calc = Calculator()\n" + " return calc.add(1, 2)\n" + ) + graph = _build_call_graph(str(tmp_path), ["prod.py"]) + assert graph is not None + + def test_build_call_graph_with_module_call(self, tmp_path: Path) -> None: + """Module-level calls (module.func()) covered in call graph build.""" + from snake_eyes.quality.pairing import _build_call_graph + + (tmp_path / "a.py").write_text("def compute():\n return 1\n") + (tmp_path / "b.py").write_text( + "import a\n\n\ndef run():\n return a.compute()\n" + ) + graph = _build_call_graph(str(tmp_path), ["a.py", "b.py"]) + assert graph is not None + + def test_name_match_TestFoo_prefix(self, tmp_path: Path) -> None: + """TestFoo (capital T prefix) strips to Foo, case-matches foo.""" + from snake_eyes.quality.pairing import _name_match + + rec = FunctionRecord(name="foo", package="pkg", file="m.py", line=1) + results = _name_match("TestFoo", [rec]) + # 'TestFoo' -> strip 'Test' -> 'Foo', case-only matches 'foo' + assert any(r[1] == 70 for r in results) + + def test_direct_call_names_empty_class_body(self) -> None: + """Class with no methods matching bare name returns empty set.""" + from snake_eyes.quality.pairing import _direct_call_names + + src = "class TestOps:\n pass\n" + tree = ast.parse(src) + result = _direct_call_names(tree, "TestOps.test_nonexistent") + assert result == set() + + def test_direct_call_names_attribute_call(self) -> None: + """Direct call via obj.method() is collected as 'method'.""" + from snake_eyes.quality.pairing import _direct_call_names + + src = "def test_x():\n obj.method()\n" + tree = ast.parse(src) + result = _direct_call_names(tree, "test_x") + assert "method" in result + + def test_reachable_files_bfs_stops_at_depth(self) -> None: + """BFS reachability stops at depth_limit.""" + from snake_eyes.quality.pairing import _CallGraph + + # Chain: a -> b -> c -> d -> e -> f + graph = _CallGraph( + edges={ + "a": {"b"}, + "b": {"c"}, + "c": {"d"}, + "d": {"e"}, + "e": {"f"}, + } + ) + # depth_limit=4: a, b, c, d, e are reachable (depth 0-4), f is not + reachable = graph.reachable_files("a", depth_limit=4) + assert "e" in reachable + assert "f" not in reachable + + def test_reachable_files_cycle_handled(self) -> None: + """BFS handles cycles gracefully.""" + from snake_eyes.quality.pairing import _CallGraph + + graph = _CallGraph(edges={"a": {"b"}, "b": {"a"}}) + reachable = graph.reachable_files("a", depth_limit=5) + assert "a" in reachable + assert "b" in reachable