From 7406059f5e120c401da7268c1544a04421b3368b Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Wed, 12 Aug 2026 00:45:24 +0000 Subject: [PATCH 1/2] fix(report): flag a partial LLM failure as degraded, not only a total one _llm_runtime_status() only set degraded when every LLM call failed (succeeded == 0). A rate-limited provider that drops a single batch (e.g. semantic_security_discovery hits a 429) still has succeeded > 0, so the scan reported a normal risk_assessment even though the security-critical analyzer never ran. Widen the condition to succeeded < attempted, so any dropped batch degrades the scan and the existing fail-closed floor (CAUTION instead of SAFE) applies to a partial pass too. Updated the two degraded-scan messages to say how many of the calls failed instead of assuming all of them did. Covers request 3 of #303 (surface incompleteness in the verdict). Request 1 (configurable concurrency) shipped in #305; request 2 (retry with backoff) is left to the already-open #29. Refs #303 Signed-off-by: Amir Fathi --- src/skillspector/nodes/report.py | 24 +++++++++++-------- tests/nodes/test_report.py | 40 ++++++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index fb7dc5525..cb3fc7951 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -578,12 +578,13 @@ def _llm_runtime_status( """Return ``(attempted, succeeded, degraded)`` from the LLM call log. ``degraded`` is True when the LLM stage was requested and at least one call - was attempted, but every call failed at runtime — meaning the report - reflects static analysis only despite a deep scan being requested. + was attempted, but not every call succeeded: a dropped or throttled batch + (e.g. a 429) leaves the same coverage gap as a full failure, so a partial + pass is degraded too, not just a total one. """ attempted = len(llm_call_log) succeeded = sum(1 for r in llm_call_log if r.get("ok")) - degraded = bool(use_llm and attempted > 0 and succeeded == 0) + degraded = bool(use_llm and attempted > 0 and succeeded < attempted) return attempted, succeeded, degraded @@ -591,12 +592,13 @@ def _llm_degradation_notice( use_llm: bool, llm_call_log: Sequence[Mapping[str, object]] ) -> str | None: """Return a human-readable degraded-scan warning, or None if not degraded.""" - attempted, _succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log) + attempted, succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log) if not degraded: return None + failed = attempted - succeeded return ( - f"LLM analysis was requested but all {attempted} LLM call(s) failed - " - "results reflect STATIC analysis only." + f"LLM analysis was requested but {failed} of {attempted} LLM call(s) failed - " + "results reflect STATIC analysis only for the affected batch(es)." ) @@ -611,7 +613,7 @@ def _build_metadata( llm_available, llm_error = is_llm_available() attempted, succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log) # meta_analysis_applied reflects whether the LLM meta-analysis effectively - # ran: requested, available, and not fully degraded (every call failing). + # ran in full: requested, available, and every attempted call succeeded. meta_analysis_applied = use_llm and llm_available and not degraded meta: dict[str, object] = { @@ -619,7 +621,8 @@ def _build_metadata( "skillspector_version": skillspector_version, "llm_requested": use_llm, # llm_available reflects runtime truth: the binary/credentials were - # available AND the stage was not fully degraded (every call failing). + # available AND every attempted call succeeded (a dropped batch is + # coverage the caller did not actually get, same as none at all). "llm_available": llm_available and not degraded, "meta_analysis_applied": meta_analysis_applied, # A list (including an empty list) makes observability explicit. Empty @@ -638,9 +641,10 @@ def _build_metadata( {str(r.get("error")) for r in llm_call_log if not r.get("ok") and r.get("error")} ) detail = f" Reasons: {'; '.join(reasons)}" if reasons else "" + failed = attempted - succeeded meta["llm_error"] = ( - f"LLM analysis was requested but all {attempted} LLM call(s) failed; " - f"results reflect static analysis only.{detail}" + f"LLM analysis was requested but {failed} of {attempted} LLM call(s) failed; " + f"results reflect static analysis only for the affected batch(es).{detail}" ) elif use_llm and not llm_available: meta["llm_error"] = llm_error diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 19a528502..9d22a63d1 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -781,8 +781,13 @@ def test_report_llm_degraded_when_all_calls_failed(monkeypatch: pytest.MonkeyPat assert "static analysis only" in meta["llm_error"] -def test_report_not_degraded_when_some_calls_succeeded(monkeypatch: pytest.MonkeyPatch) -> None: - """At least one successful LLM call -> not degraded, llm_available stays True.""" +def test_report_degraded_when_some_calls_fail(monkeypatch: pytest.MonkeyPatch) -> None: + """A dropped/throttled batch degrades the scan even though other calls succeeded. + + A rate-limited provider can 429 one batch (e.g. the security-discovery + analyzer) while the rest of the fan-out succeeds; that is still a coverage + gap and must not read as a clean, fully-analyzed scan. + """ monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) state: SkillspectorState = { "filtered_findings": [], @@ -797,10 +802,11 @@ def test_report_not_degraded_when_some_calls_succeeded(monkeypatch: pytest.Monke ], } meta = _meta_from_json_report(state) - assert meta["llm_available"] is True - assert "llm_degraded" not in meta + assert meta["llm_available"] is False # a dropped batch is not full coverage + assert meta["llm_degraded"] is True assert meta["llm_calls_attempted"] == 2 assert meta["llm_calls_succeeded"] == 1 + assert "1 of 2" in meta["llm_error"] def test_report_not_degraded_when_no_llm_calls(monkeypatch: pytest.MonkeyPatch) -> None: @@ -980,6 +986,32 @@ def test_degraded_scan_floors_recommendation_at_caution() -> None: assert result["risk_recommendation"] == "CAUTION" # but never SAFE when degraded +def test_partial_llm_failure_also_floors_recommendation_at_caution() -> None: + """A rate-limited provider dropping one batch must not read as a clean scan. + + Matches the reported failure: llm_calls_attempted=4, llm_calls_succeeded=3 + (one batch 429'd and was dropped), yet the report emitted a plain SAFE + verdict because only an all-calls-failed scan was treated as degraded. + """ + state: SkillspectorState = { + "filtered_findings": [], # static score 0 -> would be SAFE + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [ + llm_call_record("semantic_security_discovery", ok=False, error="429 rate limited"), + llm_call_record("semantic_developer_intent", ok=True), + llm_call_record("semantic_quality_policy", ok=True), + llm_call_record("meta_analyzer", ok=True), + ], + } + result = report(state) + assert result["risk_score"] == 0 # score is left honest + assert result["risk_recommendation"] == "CAUTION" # never SAFE on a partial pass + + def test_non_degraded_clean_scan_stays_safe() -> None: """Without degradation, a clean scan still reports SAFE (no over-flooring).""" state: SkillspectorState = { From f80318aba7e1accda1963f8e2d434d8dd591dbcb Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Wed, 12 Aug 2026 12:25:59 +0000 Subject: [PATCH 2/2] fix(report): mark a batch record failed on any dropped batch, and stop meta-analysis fields inheriting other analyzers' failures Two gaps from review on #362: 1. llm_call_log records were built with ok=bool(outcome.successful) or not outcome.failures, so an analyzer with one succeeded batch and one dropped/429'd batch still recorded ok=True. In that exact case succeeded == attempted at the report layer and the scan stayed SAFE, defeating the partial-coverage fix. Now the record is ok=not outcome.failures: any dropped batch marks the whole record failed. Applied identically in the three semantic analyzers and meta_analyzer, the four call sites that build this record. 2. meta_analysis_applied and the llm_available field were derived from the aggregate `degraded` flag, which pools every LLM-backed node together. That let a different analyzer's dropped batch force meta_analysis_applied=False, filtering_mode="heuristic" and llm_available=False even when meta_analyzer's own call fully succeeded, misstating two independent contracts (meta-analysis ran vs. some coverage was lost) as one boolean. Both fields now derive from is_llm_available() plus meta_analyzer's own llm_call_log record only; the coverage loss from other analyzers still surfaces through llm_degraded / llm_calls_attempted / llm_calls_succeeded, unchanged. Verified: test_partial_batch_failure_records_llm_failure (renamed from ..._records_llm_success, now pins ok=False) and three new report-level tests, run red against the pre-fix code (3 of 4 failed) and green after. tests/nodes/test_report.py: 66 passed. Full suite in Docker (python:3.12-slim): 1947 passed, 13 skipped, 4 xfailed, 0 failed. ruff lint and format-check both pass. Signed-off-by: Amir Fathi --- .../analyzers/semantic_developer_intent.py | 6 +- .../analyzers/semantic_quality_policy.py | 6 +- .../analyzers/semantic_security_discovery.py | 6 +- src/skillspector/nodes/meta_analyzer.py | 9 +- src/skillspector/nodes/report.py | 29 +++- .../test_semantic_developer_intent.py | 11 +- tests/nodes/test_report.py | 147 +++++++++++++++++- 7 files changed, 194 insertions(+), 20 deletions(-) diff --git a/src/skillspector/nodes/analyzers/semantic_developer_intent.py b/src/skillspector/nodes/analyzers/semantic_developer_intent.py index e67e03e48..491fca5c2 100644 --- a/src/skillspector/nodes/analyzers/semantic_developer_intent.py +++ b/src/skillspector/nodes/analyzers/semantic_developer_intent.py @@ -213,7 +213,11 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "inspection_ledger": events, "analyzer_status_events": [status], "llm_call_log": [ - llm_call_record(ANALYZER_ID, ok=bool(outcome.successful) or not outcome.failures) + # A record is ok only when every submitted batch succeeded. A + # partial batch failure (e.g. one file's batch 429'd while + # another's succeeded) is still lost coverage, so it must not + # read as ok=True just because some batches came back. + llm_call_record(ANALYZER_ID, ok=not outcome.failures) ], "inference_usage": analyzer.inference_usage, } diff --git a/src/skillspector/nodes/analyzers/semantic_quality_policy.py b/src/skillspector/nodes/analyzers/semantic_quality_policy.py index 2778da524..bed90d026 100644 --- a/src/skillspector/nodes/analyzers/semantic_quality_policy.py +++ b/src/skillspector/nodes/analyzers/semantic_quality_policy.py @@ -182,7 +182,11 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "inspection_ledger": events, "analyzer_status_events": [status], "llm_call_log": [ - llm_call_record(ANALYZER_ID, ok=bool(outcome.successful) or not outcome.failures) + # A record is ok only when every submitted batch succeeded. A + # partial batch failure (e.g. one file's batch 429'd while + # another's succeeded) is still lost coverage, so it must not + # read as ok=True just because some batches came back. + llm_call_record(ANALYZER_ID, ok=not outcome.failures) ], "inference_usage": analyzer.inference_usage, } diff --git a/src/skillspector/nodes/analyzers/semantic_security_discovery.py b/src/skillspector/nodes/analyzers/semantic_security_discovery.py index 09bf2b2ae..cef1ae43a 100644 --- a/src/skillspector/nodes/analyzers/semantic_security_discovery.py +++ b/src/skillspector/nodes/analyzers/semantic_security_discovery.py @@ -179,7 +179,11 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "inspection_ledger": all_events, "analyzer_status_events": [status], "llm_call_log": [ - llm_call_record(ANALYZER_ID, ok=bool(outcome.successful) or not outcome.failures) + # A record is ok only when every submitted batch succeeded. A + # partial batch failure (e.g. one file's batch 429'd while + # another's succeeded) is still lost coverage, so it must not + # read as ok=True just because some batches came back. + llm_call_record(ANALYZER_ID, ok=not outcome.failures) ], "inference_usage": analyzer.inference_usage, } diff --git a/src/skillspector/nodes/meta_analyzer.py b/src/skillspector/nodes/meta_analyzer.py index 34c980652..46935b3b4 100644 --- a/src/skillspector/nodes/meta_analyzer.py +++ b/src/skillspector/nodes/meta_analyzer.py @@ -716,10 +716,11 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: "inspection_ledger": ledger_events, "analyzer_status_events": [status], "llm_call_log": [ - llm_call_record( - "meta_analyzer", - ok=bool(detailed.successful) or not detailed.failures, - ) + # A record is ok only when every submitted batch succeeded. A + # partial batch failure (e.g. one file's batch 429'd while + # another's succeeded) is still lost coverage, so it must not + # read as ok=True just because some batches came back. + llm_call_record("meta_analyzer", ok=not detailed.failures) ], "inference_usage": analyzer.inference_usage, } diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index cb3fc7951..543f136e2 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -610,20 +610,33 @@ def _build_metadata( ) -> dict[str, object]: """Build the metadata section shared by all output formats.""" llm_call_log = llm_call_log or [] - llm_available, llm_error = is_llm_available() + provider_available, llm_error = is_llm_available() attempted, succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log) - # meta_analysis_applied reflects whether the LLM meta-analysis effectively - # ran in full: requested, available, and every attempted call succeeded. - meta_analysis_applied = use_llm and llm_available and not degraded + + # meta_analyzer's own record, independent of whether a DIFFERENT + # LLM-backed node (a semantic_* analyzer) lost coverage to a dropped + # batch. A missing record means meta_analyzer never ran (e.g. there were + # no findings to filter), which is not itself a failure, so it reads as + # vacuously ok. When it does run it always emits exactly one record. + meta_analyzer_records = [r for r in llm_call_log if r.get("node") == "meta_analyzer"] + meta_analyzer_ok = all(bool(r.get("ok")) for r in meta_analyzer_records) + + # meta_analysis_applied / llm_available answer "did meta-analysis itself + # run": the provider was available and meta_analyzer's own call (if it + # ran) succeeded. A different analyzer's partial batch loss is a + # coverage gap, reported separately below via llm_degraded / + # llm_calls_attempted / llm_calls_succeeded, and must not flip these two + # fields false on its own - that conflated two independent contracts + # (meta-analysis ran vs. some coverage was lost) into one boolean. + meta_analysis_applied = use_llm and provider_available and meta_analyzer_ok meta: dict[str, object] = { "has_executable_scripts": has_executable_scripts, "skillspector_version": skillspector_version, "llm_requested": use_llm, # llm_available reflects runtime truth: the binary/credentials were - # available AND every attempted call succeeded (a dropped batch is - # coverage the caller did not actually get, same as none at all). - "llm_available": llm_available and not degraded, + # available AND meta_analyzer's own call (if it ran) succeeded. + "llm_available": provider_available and meta_analyzer_ok, "meta_analysis_applied": meta_analysis_applied, # A list (including an empty list) makes observability explicit. Empty # means the provider/transport supplied no counters; it is never an @@ -646,7 +659,7 @@ def _build_metadata( f"LLM analysis was requested but {failed} of {attempted} LLM call(s) failed; " f"results reflect static analysis only for the affected batch(es).{detail}" ) - elif use_llm and not llm_available: + elif use_llm and not provider_available: meta["llm_error"] = llm_error return meta diff --git a/tests/nodes/analyzers/test_semantic_developer_intent.py b/tests/nodes/analyzers/test_semantic_developer_intent.py index 10ab00f28..8190b82ac 100644 --- a/tests/nodes/analyzers/test_semantic_developer_intent.py +++ b/tests/nodes/analyzers/test_semantic_developer_intent.py @@ -253,7 +253,14 @@ def test_success_records_ok_true(self) -> None: assert result["llm_call_log"] == [{"node": ANALYZER_ID, "ok": True, "error": None}] @patch(MOCK_PATCH_TARGET, _mock_get_chat_model) - def test_partial_batch_failure_records_llm_success(self) -> None: + def test_partial_batch_failure_records_llm_failure(self) -> None: + """One batch succeeding does not hide another batch's dropped coverage. + + Regression for the case where a two-file run has one file batch + succeed and the other 429 / time out: the record must be ok=False so + the report can detect the coverage gap, not ok=True just because + `outcome.successful` was non-empty. + """ from skillspector.llm_analyzer_base import LLMAnalyzerBase async def partially_succeeds(self, batches, **_kwargs): @@ -267,7 +274,7 @@ async def partially_succeeds(self, batches, **_kwargs): with patch.object(LLMAnalyzerBase, "arun_batches", partially_succeeds): result = node({"file_cache": {"first.py": "print(1)", "second.py": "print(2)"}}) - assert result["llm_call_log"] == [{"node": ANALYZER_ID, "ok": True, "error": None}] + assert result["llm_call_log"] == [{"node": ANALYZER_ID, "ok": False, "error": None}] @patch(MOCK_PATCH_TARGET) def test_exception_records_ok_false(self, mock_get_model: MagicMock) -> None: diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 9d22a63d1..7fc611be1 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -754,7 +754,14 @@ def _meta_from_json_report(state: SkillspectorState) -> dict: def test_report_llm_degraded_when_all_calls_failed(monkeypatch: pytest.MonkeyPatch) -> None: - """use_llm requested + every LLM call failed -> llm_available False, llm_degraded True.""" + """use_llm requested + every semantic-analyzer call failed -> llm_degraded True. + + llm_available/meta_analysis_applied are about the provider and + meta_analyzer's OWN call specifically (see the meta_analysis_applied + tests below); none of these three failures is a meta_analyzer record, + so those two fields stay True here and the failure surfaces only via + llm_degraded / llm_calls_attempted / llm_calls_succeeded / llm_error. + """ # Pre-flight reports available (binary/creds present); the failure is at runtime. monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) state: SkillspectorState = { @@ -772,7 +779,7 @@ def test_report_llm_degraded_when_all_calls_failed(monkeypatch: pytest.MonkeyPat } meta = _meta_from_json_report(state) assert meta["llm_requested"] is True - assert meta["llm_available"] is False # degraded -> not actually available + assert meta["llm_available"] is True # provider ok; meta_analyzer never ran/failed assert meta["llm_degraded"] is True assert meta["llm_calls_attempted"] == 3 assert meta["llm_calls_succeeded"] == 0 @@ -802,13 +809,79 @@ def test_report_degraded_when_some_calls_fail(monkeypatch: pytest.MonkeyPatch) - ], } meta = _meta_from_json_report(state) - assert meta["llm_available"] is False # a dropped batch is not full coverage + # Neither record is meta_analyzer, so llm_available/meta_analysis_applied + # are untouched by this coverage gap; llm_degraded is the signal for it. + assert meta["llm_available"] is True assert meta["llm_degraded"] is True assert meta["llm_calls_attempted"] == 2 assert meta["llm_calls_succeeded"] == 1 assert "1 of 2" in meta["llm_error"] +def test_report_meta_analysis_applied_survives_other_analyzer_partial_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """meta_analyzer succeeding is independent of a different analyzer's coverage loss. + + Regression for the reviewed gap: a semantic analyzer dropping a batch + forced meta_analysis_applied=False and llm_available=False even though + meta_analyzer's own call succeeded in full, which misstated two + independent contracts (meta-analysis ran vs. some coverage was lost) as + one boolean. Matches the reported 3/4 scenario: 3 calls succeed + (including meta_analyzer), 1 semantic-analyzer batch is dropped. + """ + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [ + llm_call_record("semantic_security_discovery", ok=False, error="429 rate limited"), + llm_call_record("semantic_developer_intent", ok=True), + llm_call_record("semantic_quality_policy", ok=True), + llm_call_record("meta_analyzer", ok=True), + ], + } + meta = _meta_from_json_report(state) + assert meta["meta_analysis_applied"] is True + assert meta["llm_available"] is True + assert "filtering_mode" not in meta + # The lost coverage is still visible, just not through these two fields. + assert meta["llm_degraded"] is True + assert meta["llm_calls_attempted"] == 4 + assert meta["llm_calls_succeeded"] == 3 + + +def test_report_meta_analysis_not_applied_when_meta_analyzer_itself_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """meta_analyzer's own failure still zeros meta_analysis_applied/llm_available. + + This is the other half of the independent-contracts fix: the two fields + are not blind to meta_analyzer - they just ignore everyone ELSE. + """ + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [ + llm_call_record("semantic_security_discovery", ok=True), + llm_call_record("meta_analyzer", ok=False, error="claude empty stdout"), + ], + } + meta = _meta_from_json_report(state) + assert meta["meta_analysis_applied"] is False + assert meta["llm_available"] is False + assert meta["filtering_mode"] == "heuristic" + + def test_report_not_degraded_when_no_llm_calls(monkeypatch: pytest.MonkeyPatch) -> None: """use_llm True but no LLM calls attempted (e.g. empty skill) -> not degraded.""" monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) @@ -1012,6 +1085,74 @@ def test_partial_llm_failure_also_floors_recommendation_at_caution() -> None: assert result["risk_recommendation"] == "CAUTION" # never SAFE on a partial pass +def test_analyzer_partial_batch_failure_flows_through_to_report_degraded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end: an analyzer-level partial batch failure reaches the report as degraded. + + Regression for the reviewed gap: llm_call_log records were built with + ``ok=bool(outcome.successful) or not outcome.failures``, so a batch + dropping while a sibling batch in the same analyzer succeeded still + recorded ok=True and the report never saw the coverage loss (the exact + two-file/one-429 case ``test_partial_batch_failure_records_llm_failure`` + in test_semantic_developer_intent.py now pins). This drives the real + ``semantic_developer_intent`` node through a mocked partial-batch outcome + and feeds its ACTUAL llm_call_log output into report(), rather than + hand-constructing the log the way the report-only tests above do. + """ + from unittest.mock import MagicMock, patch + + from skillspector.llm_analyzer_base import ( + BatchExecutionResult, + BatchFailure, + LLMAnalyzerBase, + ) + from skillspector.nodes.analyzers.semantic_developer_intent import node as di_node + + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + + def _mock_get_chat_model(*_args: object, **_kwargs: object) -> MagicMock: + mock_llm = MagicMock() + mock_llm.with_structured_output.return_value = MagicMock() + return mock_llm + + async def partially_succeeds(self: LLMAnalyzerBase, batches: list, **_kwargs: object) -> list: + successful = [(batches[0], [])] + self._last_batch_outcome = BatchExecutionResult( + successful=successful, + failures=[BatchFailure(batches[1], "TimeoutError")], + ) + return successful + + with ( + patch("skillspector.llm_analyzer_base.get_chat_model", _mock_get_chat_model), + patch.object(LLMAnalyzerBase, "arun_batches", partially_succeeds), + ): + analyzer_result = di_node({"file_cache": {"first.py": "print(1)", "second.py": "print(2)"}}) + + # The analyzer's own record reflects the dropped batch... + assert analyzer_result["llm_call_log"] == [ + {"node": "semantic_developer_intent", "ok": False, "error": None} + ] + + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": analyzer_result["llm_call_log"], + } + result = report(state) + meta = json.loads(result["report_body"])["metadata"] + + # ...and the report-level verdict reflects it too: never a plain SAFE on + # a multi-batch analyzer that silently lost coverage. + assert meta["llm_degraded"] is True + assert result["risk_recommendation"] == "CAUTION" + + def test_non_degraded_clean_scan_stays_safe() -> None: """Without degradation, a clean scan still reports SAFE (no over-flooring).""" state: SkillspectorState = {