From 2868709619fe543d33dc5ed33df1d5f1b320e27f Mon Sep 17 00:00:00 2001 From: Keshav Pradeep <32313895+keshprad@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:53:16 -0700 Subject: [PATCH] chore: public OSS release 2.9.3 Signed-off-by: Keshav Pradeep <32313895+keshprad@users.noreply.github.com> --- CHANGELOG.md | 4 ++ docs/release/skillspector-2.9.3.md | 47 +++++++++++++++ pyproject.toml | 2 +- src/skillspector/inspection_ledger.py | 24 ++++++-- src/skillspector/llm_analyzer_base.py | 24 ++------ .../nodes/analyzers/mcp_tool_poisoning.py | 58 ++++++++++++------- src/skillspector/nodes/meta_analyzer.py | 25 ++------ .../nodes/test_finalize_inspection_ledger.py | 29 ++++++++++ tests/nodes/test_llm_analyzer_base.py | 7 ++- tests/nodes/test_meta_analyzer.py | 58 ++++++++++++++++++- tests/nodes/test_report.py | 46 +++++++++++++++ tests/nodes/test_semantic_quality_policy.py | 10 +++- tests/test_inspection_ledger.py | 33 +++++++++-- tests/test_mcp_tool_poisoning.py | 6 ++ uv.lock | 2 +- 15 files changed, 296 insertions(+), 79 deletions(-) create mode 100644 docs/release/skillspector-2.9.3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 566f09523..f65e2fd38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### 2.9.3 (Tuesday, August 11, 2026) +### Features/Bug Fixes +* fix(llm): surface invalid responses as degraded (skipped, non-fatal, incomplete) +--- ### 2.9.2 (Monday, August 10, 2026) ### Features/Bug Fixes * fix(llm): retry malformed structured responses diff --git a/docs/release/skillspector-2.9.3.md b/docs/release/skillspector-2.9.3.md new file mode 100644 index 000000000..921a77dd7 --- /dev/null +++ b/docs/release/skillspector-2.9.3.md @@ -0,0 +1,47 @@ +# SkillSpector v2.9.3 + +Released: 2026-08-11 + +## Summary + +This patch makes malformed structured LLM responses non-fatal during analysis. Affected analysis work is now recorded as skipped so reports clearly show degraded, incomplete results while preserving the remaining analysis output. + +## Highlights + +- Improve resilience to malformed structured responses from LLM-backed analyzers without masking the affected analysis outcome. + +## Added + +- None. + +## Changed + +- Analysis ledger and analyzer status handling consistently represent malformed structured-response batches as skipped and degraded rather than failed. + +## Fixed + +- Preserve the input findings and incomplete-analysis provenance when a malformed structured response exhausts retry handling. + +## Security + +- None. + +## Breaking Changes and Migration + +- None. + +## Deprecations + +- None. + +## Validation + +- `uv run --locked --extra dev pytest tests/nodes/test_llm_analyzer_base.py tests/nodes/test_meta_analyzer.py tests/nodes/test_finalize_inspection_ledger.py tests/test_inspection_ledger.py tests/test_mcp_tool_poisoning.py` — passed. + +## Known Limitations + +- Malformed structured responses remain unavailable for analysis; this release surfaces their impact as incomplete rather than producing findings for the affected work. + +## References + +- `CHANGELOG.md` diff --git a/pyproject.toml b/pyproject.toml index fe27d95bb..6522d4731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "skillspector" -version = "2.9.2" +version = "2.9.3" description = "SkillSpector: Security scanner for AI agent skills (Claude Code, Cursor, and similar). Scans skills for vulnerabilities, malicious patterns, and security risks before installation. Supports Git repos, URLs, zips, and local directories; runs static pattern checks and optional LLM semantic analysis; outputs terminal, JSON, and Markdown reports with risk scoring." readme = "README.md" license = "Apache-2.0" diff --git a/src/skillspector/inspection_ledger.py b/src/skillspector/inspection_ledger.py index 00482a71d..0b3cf2041 100644 --- a/src/skillspector/inspection_ledger.py +++ b/src/skillspector/inspection_ledger.py @@ -102,6 +102,15 @@ class LedgerReason(StrEnum): } +def outcome_for_llm_batch_failure(reason: LedgerReason) -> LedgerOutcome: + """Return the terminal ledger outcome for an exhausted LLM batch.""" + return ( + LedgerOutcome.SKIPPED + if reason is LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID + else LedgerOutcome.FAILED + ) + + class PlannedWorkTarget(TypedDict): """One analyzer work item expected to have a terminal ledger row.""" @@ -279,11 +288,12 @@ def ledger_event( raise ValueError("non-completed producers cannot reference findings") elif outcome is LedgerOutcome.COMPLETED and not set(emitted_ids).issubset(input_ids): raise ValueError("completed meta events must emit a subset of input findings") - elif outcome is LedgerOutcome.FAILED and emitted_ids != input_ids: - raise ValueError("failed meta events must pass every input finding through") - elif outcome is not LedgerOutcome.COMPLETED and outcome is not LedgerOutcome.FAILED: + elif outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED): + if emitted_ids != input_ids: + raise ValueError("failed or skipped meta events must pass every input finding through") + elif outcome is not LedgerOutcome.COMPLETED: if input_ids or emitted_ids: - raise ValueError("skipped meta events cannot reference findings") + raise ValueError("non-completed meta events cannot reference findings") work_identity = analyzer_id or f"{record_type.value}:{phase}" event: InspectionLedgerEvent = { @@ -614,7 +624,11 @@ def accounting_error(path: object = None) -> None: and not set(emitted_ids).issubset(input_ids) ): accounting_error(event.get("path")) - if is_meta and outcome == LedgerOutcome.FAILED and emitted_ids != input_ids: + if ( + is_meta + and outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED) + and emitted_ids != input_ids + ): accounting_error(event.get("path")) for finding_id in [*input_ids, *emitted_ids]: if finding_id not in findings_by_id: diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index 0420ff3da..0fbf8d23d 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -44,8 +44,9 @@ InspectionLedgerEvent, LedgerOutcome, LedgerReason, - analyzer_status_event, + analyzer_status_for_events, ledger_event, + outcome_for_llm_batch_failure, ) from skillspector.llm_utils import ( StructuredOutputParseError, @@ -320,7 +321,7 @@ def ledger_events_for_batches( events.append( ledger_event( analyzer_id=analyzer_id, - outcome=LedgerOutcome.FAILED, + outcome=outcome_for_llm_batch_failure(failure.reason), phase="semantic", path=path, start_line=start_line, @@ -330,24 +331,7 @@ def ledger_events_for_batches( ) ) - status = analyzer_status_event( - analyzer_id=analyzer_id, - status=( - "failed" - if any(event["outcome"] is LedgerOutcome.FAILED for event in events) - else "completed" - ), - planned_work=[ - { - "work_id": event["work_id"], - "path": event["path"], - "start_line": event["start_line"], - "end_line": event["end_line"], - } - for event in events - ], - ) - return events, status + return events, analyzer_status_for_events(analyzer_id, events) # --------------------------------------------------------------------------- diff --git a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py index a233ee168..9c7f817b2 100644 --- a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py +++ b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py @@ -30,7 +30,9 @@ LedgerOutcome, LedgerReason, analyzer_status_event, + analyzer_status_for_events, ledger_event, + outcome_for_llm_batch_failure, ) from skillspector.llm_analyzer_base import Batch, LLMAnalyzerBase from skillspector.models import Finding @@ -736,11 +738,12 @@ def _check_tp4( list[Finding], LLMCallRecord | None, str | None, + LedgerReason | None, list[InferenceUsageRecord], ]: """TP4: LLM-based description-behavior mismatch detection. - Returns ``(findings, record, error_class, inference_usage)`` where + Returns ``(findings, record, error_class, failure_reason, inference_usage)`` where *record* is the LLM-call telemetry for ``llm_call_log`` — or ``None`` when no LLM call was attempted (no description / no executable code), so an intentional no-op is never counted as a degraded LLM stage. Token usage is @@ -752,7 +755,7 @@ def _check_tp4( manifest: dict = state.get("manifest") or {} description = manifest.get("description") if not description or not isinstance(description, str) or not description.strip(): - return [], None, None, [] + return [], None, None, None, [] triggers = manifest.get("triggers") or [] permissions = manifest.get("permissions") @@ -774,7 +777,7 @@ def _check_tp4( code_parts.append(f"### {path} ({file_type})\n{content}") if not code_parts: - return [], None, None, [] + return [], None, None, None, [] code_contents = "\n\n".join(code_parts) @@ -824,6 +827,7 @@ def _check_tp4( error=f"TP4 LLM batch failed: {failure.error_class}", ), failure.error_class, + failure.reason, cast(list[InferenceUsageRecord], analyzer.inference_usage), ) result = outcome.successful[0][1][0] @@ -832,11 +836,23 @@ def _check_tp4( ok_record = llm_call_record(ANALYZER_ID, ok=True) if not result.is_mismatch: - return [], ok_record, None, cast(list[InferenceUsageRecord], analyzer.inference_usage) + return ( + [], + ok_record, + None, + None, + cast(list[InferenceUsageRecord], analyzer.inference_usage), + ) confidence = result.confidence if confidence < 0.5: - return [], ok_record, None, cast(list[InferenceUsageRecord], analyzer.inference_usage) + return ( + [], + ok_record, + None, + None, + cast(list[InferenceUsageRecord], analyzer.inference_usage), + ) severity = "HIGH" if confidence >= 0.7 else "MEDIUM" @@ -868,6 +884,7 @@ def _check_tp4( ], ok_record, None, + None, cast(list[InferenceUsageRecord], analyzer.inference_usage), ) @@ -880,11 +897,12 @@ def _check_tp4( [], llm_call_record(ANALYZER_ID, ok=False, error=str(exc)), type(exc).__name__, + LedgerReason.LLM_BATCH_FAILED, cast(list[InferenceUsageRecord], analyzer.inference_usage) if analyzer is not None else [], ) - return [], None, None, [] + return [], None, None, None, [] # --------------------------------------------------------------------------- @@ -946,36 +964,32 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: tp4_record: LLMCallRecord | None = None tp4_findings: list[Finding] = [] tp4_error_class: str | None = None + tp4_failure_reason: LedgerReason | None = None tp4_usage: list[InferenceUsageRecord] = [] if state.get("use_llm", True): - tp4_findings, tp4_record, tp4_error_class, tp4_usage = _check_tp4(state) + tp4_findings, tp4_record, tp4_error_class, tp4_failure_reason, tp4_usage = _check_tp4(state) findings.extend(tp4_findings) logger.info("%s: %d findings", ANALYZER_ID, len(findings)) if tp4_record is not None: + tp4_event_outcome = ( + LedgerOutcome.COMPLETED + if tp4_record["ok"] + else outcome_for_llm_batch_failure(tp4_failure_reason or LedgerReason.LLM_BATCH_FAILED) + ) tp4_event = ledger_event( analyzer_id=ANALYZER_ID, - outcome=LedgerOutcome.COMPLETED if tp4_record["ok"] else LedgerOutcome.FAILED, + outcome=tp4_event_outcome, phase="semantic", path="SKILL.md", - reason=None if tp4_record["ok"] else LedgerReason.LLM_BATCH_FAILED, + reason=( + None if tp4_record["ok"] else tp4_failure_reason or LedgerReason.LLM_BATCH_FAILED + ), emitted_finding_ids=[finding.finding_id for finding in tp4_findings], error_class=tp4_error_class, ) ledger.append(tp4_event) - status = analyzer_status_event( - analyzer_id=ANALYZER_ID, - status="failed" if tp4_record is not None and not tp4_record["ok"] else "completed", - planned_work=[ - { - "work_id": event["work_id"], - "path": event["path"], - "start_line": event["start_line"], - "end_line": event["end_line"], - } - for event in ledger - ], - ) + status = analyzer_status_for_events(ANALYZER_ID, ledger) result: AnalyzerNodeResponse = { "findings": findings, "inspection_ledger": ledger, diff --git a/src/skillspector/nodes/meta_analyzer.py b/src/skillspector/nodes/meta_analyzer.py index e42a278cd..34c980652 100644 --- a/src/skillspector/nodes/meta_analyzer.py +++ b/src/skillspector/nodes/meta_analyzer.py @@ -34,8 +34,10 @@ LedgerOutcome, LedgerReason, analyzer_status_event, + analyzer_status_for_events, inspection_work_id, ledger_event, + outcome_for_llm_batch_failure, ) from skillspector.llm_analyzer_base import ( Batch, @@ -559,7 +561,7 @@ def _meta_ledger_response( events.append( ledger_event( analyzer_id="meta_analyzer", - outcome=LedgerOutcome.FAILED, + outcome=outcome_for_llm_batch_failure(failure.reason), phase="meta", path=batch.file_path, start_line=batch.start_line if batch.end_line is not None else None, @@ -570,24 +572,9 @@ def _meta_ledger_response( error_class=failure.error_class, ) ) - status = analyzer_status_event( - analyzer_id="meta_analyzer", - status=( - "failed" - if any(event["outcome"] is LedgerOutcome.FAILED for event in events) - else "completed" - ), - planned_work=[ - { - "work_id": event["work_id"], - "path": event["path"], - "start_line": event["start_line"], - "end_line": event["end_line"], - } - for event in events - ], - ) - return events, status + if not events: + return events, analyzer_status_event(analyzer_id="meta_analyzer", status="completed") + return events, analyzer_status_for_events("meta_analyzer", events) def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: diff --git a/tests/nodes/test_finalize_inspection_ledger.py b/tests/nodes/test_finalize_inspection_ledger.py index d9da0766b..4e1828242 100644 --- a/tests/nodes/test_finalize_inspection_ledger.py +++ b/tests/nodes/test_finalize_inspection_ledger.py @@ -162,6 +162,35 @@ def test_meta_failure_preserves_primary_coverage_but_fails_execution() -> None: assert effective_ids == [finding.finding_id] +def test_skipped_meta_event_that_drops_findings_is_a_fatal_accounting_error() -> None: + """Finalization rejects malformed skipped meta rows that bypass the factory.""" + finding = Finding(rule_id="P1", message="unsafe", file="SKILL.md") + skipped_meta = ledger_event( + outcome=LedgerOutcome.SKIPPED, + phase="meta", + analyzer_id="meta_analyzer", + reason=LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID, + path="SKILL.md", + input_finding_ids=[finding.finding_id], + emitted_finding_ids=[finding.finding_id], + ) + skipped_meta["emitted_finding_ids"] = [] + + completeness, _ = finalize_ledger( + { + "components": ["SKILL.md"], + "findings": [finding], + "inspection_ledger": [skipped_meta], + } + ) + + assert completeness["execution_successful"] is False + assert completeness["ledger_exceptions"][0]["reason_code"] == ( + LedgerReason.FINDING_ACCOUNTING_ERROR + ) + assert completeness["ledger_exceptions"][0]["fatal"] is True + + def test_json_round_trip_keeps_failed_ledger_work_fatal() -> None: """Deserialized StrEnum values must retain failure semantics.""" state = json.loads( diff --git a/tests/nodes/test_llm_analyzer_base.py b/tests/nodes/test_llm_analyzer_base.py index 30a3a9ba5..a6fb0daea 100644 --- a/tests/nodes/test_llm_analyzer_base.py +++ b/tests/nodes/test_llm_analyzer_base.py @@ -27,7 +27,7 @@ from langchain_openai import ChatOpenAI from pydantic import ValidationError -from skillspector.inspection_ledger import LedgerReason, finalize_ledger +from skillspector.inspection_ledger import LedgerOutcome, LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import ( API_CONNECTION_MAX_RETRIES, DEFAULT_MAX_LLM_CONCURRENCY, @@ -1290,6 +1290,8 @@ def test_safe_failure_reason_is_preserved_in_ledger_events(self) -> None: ), ) + assert events[0]["outcome"] is LedgerOutcome.SKIPPED + assert status["status"] == "degraded" assert events[0]["reason_code"] == LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID assert ( events[0]["message"] @@ -1311,6 +1313,9 @@ def test_safe_failure_reason_is_preserved_in_ledger_events(self) -> None: assert completeness["ledger_exceptions"][0]["message"] == ( "LLM returned a malformed structured response after bounded retries." ) + assert completeness["ledger_exceptions"][0]["fatal"] is False + assert completeness["execution_successful"] is True + assert completeness["is_complete"] is False def test_successful_unchunked_retry_has_one_terminal_outcome(self) -> None: """A retry does not create duplicate work IDs or fatal unaccounted work.""" diff --git a/tests/nodes/test_meta_analyzer.py b/tests/nodes/test_meta_analyzer.py index b87b57eec..cabbe4037 100644 --- a/tests/nodes/test_meta_analyzer.py +++ b/tests/nodes/test_meta_analyzer.py @@ -24,7 +24,7 @@ from unittest.mock import AsyncMock, MagicMock, patch -from skillspector.inspection_ledger import LedgerReason, finalize_ledger +from skillspector.inspection_ledger import LedgerOutcome, LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import Batch, BatchExecutionResult, BatchFailure from skillspector.models import Finding from skillspector.nodes.meta_analyzer import ( @@ -202,11 +202,11 @@ def test_mixed_batches_distinguish_retained_and_filtered_findings(self) -> None: for event in events ] - def test_failed_batch_preserves_safe_failure_reason(self) -> None: + def test_connection_failure_remains_fatal(self) -> None: failed = _lineage_finding("failed", "failed.py", 3) failed_batch = Batch(file_path="failed.py", content="failed", findings=[failed]) - events, _ = _meta_ledger_response( + events, status = _meta_ledger_response( [failed_batch], BatchExecutionResult( failures=[ @@ -220,8 +220,60 @@ def test_failed_batch_preserves_safe_failure_reason(self) -> None: [failed], ) + assert events[0]["outcome"] is LedgerOutcome.FAILED assert events[0]["reason_code"] == LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED assert events[0]["message"] == "LLM connection failed after bounded retries." + assert status["status"] == "failed" + + completeness, _ = finalize_ledger( + { + "components": ["failed.py"], + "findings": [failed], + "effective_finding_ids": [failed.finding_id], + "inspection_ledger": events, + "analyzer_status_events": [status], + } + ) + + assert completeness["execution_successful"] is False + assert completeness["ledger_exceptions"][0]["fatal"] is True + + def test_structured_response_failure_is_nonfatal_and_degraded(self) -> None: + failed = _lineage_finding("failed", "failed.py", 3) + failed_batch = Batch(file_path="failed.py", content="failed", findings=[failed]) + + events, status = _meta_ledger_response( + [failed_batch], + BatchExecutionResult( + failures=[ + BatchFailure( + batch=failed_batch, + error_class="ValidationError", + reason=LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID, + ) + ] + ), + [failed], + ) + + assert events[0]["outcome"] is LedgerOutcome.SKIPPED + assert events[0]["input_finding_ids"] == [failed.finding_id] + assert events[0]["emitted_finding_ids"] == [failed.finding_id] + assert status["status"] == "degraded" + + completeness, _ = finalize_ledger( + { + "components": ["failed.py"], + "findings": [failed], + "effective_finding_ids": [failed.finding_id], + "inspection_ledger": events, + "analyzer_status_events": [status], + } + ) + + assert completeness["execution_successful"] is True + assert completeness["is_complete"] is False + assert completeness["ledger_exceptions"][0]["fatal"] is False def test_overlapping_batches_do_not_reaccount_completed_finding(self) -> None: shared = _lineage_finding("shared", "complete.py", 1) diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 74dff645d..19a528502 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -475,6 +475,52 @@ def test_report_output_format_markdown(self) -> None: assert "## Components" in body assert "## Issues" in body + def test_report_markdown_lists_nonfatal_llm_validation_exception(self) -> None: + """A non-fatal structured-output failure remains visible in the report.""" + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "skill_path": None, + "output_format": "markdown", + "execution_successful": True, + "analysis_completeness": { + "coverage_percent": 0.0, + "fully_inspected_files": 0, + "partially_inspected_files": 0, + "entirely_uninspected_files": 1, + "is_complete": False, + "execution_successful": True, + "ledger_exceptions": [ + { + "reason_code": "llm_structured_response_invalid", + "path": "SKILL.md", + "message": "LLM returned a malformed structured response after bounded retries.", + "fatal": False, + } + ], + "scope_exclusions": [], + "analyzer_statuses": [ + { + "analyzer_id": "semantic_quality_policy", + "status": "degraded", + "planned_work": [], + } + ], + "limitations": ["Analyzer semantic_quality_policy status: degraded."], + }, + } + + body = report(state)["report_body"] + + assert "| Execution | successful |" in body + assert "### Ledger Exceptions" in body + assert "llm_structured_response_invalid" in body + assert "`SKILL.md`" in body + assert "### Analyzer Statuses" in body + assert "### Limitations" in body + def test_report_output_format_terminal(self) -> None: """output_format terminal produces Rich-formatted output.""" state: SkillspectorState = { diff --git a/tests/nodes/test_semantic_quality_policy.py b/tests/nodes/test_semantic_quality_policy.py index ba294f494..d22c9a0d3 100644 --- a/tests/nodes/test_semantic_quality_policy.py +++ b/tests/nodes/test_semantic_quality_policy.py @@ -25,7 +25,7 @@ from langchain_core.outputs import ChatGeneration, LLMResult from langchain_core.runnables import Runnable, RunnableConfig -from skillspector.inspection_ledger import finalize_ledger +from skillspector.inspection_ledger import LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import LLMAnalysisResult, LLMFinding from skillspector.llm_utils import AgentCLIChatModel from skillspector.models import Finding @@ -330,7 +330,7 @@ def test_post_response_value_error_preserves_provider_usage(self) -> None: ) assert completeness["execution_successful"] is False - def test_post_response_value_error_without_usage_uses_failed_fallback(self) -> None: + def test_post_response_value_error_without_usage_records_partial_coverage(self) -> None: provider = MagicMock() provider.complete.return_value = "not valid structured JSON" cli_model = AgentCLIChatModel(provider, "gpt-5.6-sol", 1024) @@ -341,7 +341,11 @@ def test_post_response_value_error_without_usage_uses_failed_fallback(self) -> N assert result["findings"] == [] assert result["inference_usage"] == [] assert result["inspection_ledger"] - assert result["analyzer_status_events"][0]["status"] == "failed" + assert result["inspection_ledger"][0]["outcome"] == "skipped" + assert result["inspection_ledger"][0]["reason_code"] == ( + LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID + ) + assert result["analyzer_status_events"][0]["status"] == "degraded" # --------------------------------------------------------------------------- diff --git a/tests/test_inspection_ledger.py b/tests/test_inspection_ledger.py index ac8d7d78b..e8d73cc9c 100644 --- a/tests/test_inspection_ledger.py +++ b/tests/test_inspection_ledger.py @@ -11,6 +11,7 @@ analyzer_status_for_events, inspection_work_id, ledger_event, + outcome_for_llm_batch_failure, ) @@ -68,6 +69,21 @@ def test_analyzer_status_for_events_summarizes_terminal_work() -> None: } +@pytest.mark.parametrize( + ("reason", "expected_outcome"), + [ + (LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID, LedgerOutcome.SKIPPED), + (LedgerReason.LLM_BATCH_FAILED, LedgerOutcome.FAILED), + (LedgerReason.LLM_CONNECTION_RETRIES_EXHAUSTED, LedgerOutcome.FAILED), + ], +) +def test_outcome_for_llm_batch_failure_preserves_failure_policy( + reason: LedgerReason, expected_outcome: LedgerOutcome +) -> None: + """Only exhausted malformed structured responses are non-fatal.""" + assert outcome_for_llm_batch_failure(reason) is expected_outcome + + def test_failed_producer_ledger_event_cannot_reference_findings() -> None: """Failed producers do not claim findings they did not successfully emit.""" with pytest.raises(ValueError, match="cannot reference findings"): @@ -95,13 +111,22 @@ def test_completed_meta_event_emits_a_subset_of_its_inputs() -> None: assert event["emitted_finding_ids"] == ["finding-a"] -def test_failed_meta_event_must_pass_every_input_through() -> None: - """Failed meta work is fail-closed and preserves every input ID.""" +@pytest.mark.parametrize( + ("outcome", "reason"), + [ + (LedgerOutcome.FAILED, LedgerReason.LLM_BATCH_FAILED), + (LedgerOutcome.SKIPPED, LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID), + ], +) +def test_unprocessed_meta_event_must_pass_every_input_through( + outcome: LedgerOutcome, reason: LedgerReason +) -> None: + """Failed or skipped meta work is fail-closed and preserves every input ID.""" event = ledger_event( - outcome=LedgerOutcome.FAILED, + outcome=outcome, phase="meta", analyzer_id="meta_analyzer", - reason=LedgerReason.LLM_BATCH_FAILED, + reason=reason, path="SKILL.md", input_finding_ids=["finding-a", "finding-b"], emitted_finding_ids=["finding-a", "finding-b"], diff --git a/tests/test_mcp_tool_poisoning.py b/tests/test_mcp_tool_poisoning.py index 05b979890..feb33bbaa 100644 --- a/tests/test_mcp_tool_poisoning.py +++ b/tests/test_mcp_tool_poisoning.py @@ -26,6 +26,7 @@ import yaml from pydantic import BaseModel +from skillspector.inspection_ledger import LedgerOutcome, LedgerReason from skillspector.llm_utils import AgentCLIChatModel from skillspector.nodes.analyzers import mcp_tool_poisoning @@ -718,7 +719,12 @@ def test_persistently_malformed_response_returns_empty(self, monkeypatch: pytest tp4 = [f for f in result["findings"] if f.rule_id == "TP4"] assert len(tp4) == 0 assert structured_llm.calls == 4 + assert result["inspection_ledger"][1]["outcome"] is LedgerOutcome.SKIPPED assert result["inspection_ledger"][1]["error_class"] == "ValidationError" + assert result["inspection_ledger"][1]["reason_code"] is ( + LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID + ) + assert result["analyzer_status_events"][0]["status"] == "degraded" def test_malformed_response_is_retried(self, monkeypatch: pytest.MonkeyPatch): state = _make_state("mcp_mismatched_skill", use_llm=True) diff --git a/uv.lock b/uv.lock index 53992dd10..c4f6b6a88 100644 --- a/uv.lock +++ b/uv.lock @@ -2675,7 +2675,7 @@ wheels = [ [[package]] name = "skillspector" -version = "2.9.2" +version = "2.9.3" source = { editable = "." } dependencies = [ { name = "boto3" },