Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
### 2.9.3 (Tuesday, August 11, 2026)
### Features/Bug Fixes
* fix(llm): surface invalid responses as degraded (skipped, non-fatal, incomplete)
---
### 2.9.2 (Monday, August 10, 2026)
### Features/Bug Fixes
* fix(llm): retry malformed structured responses
Expand Down
47 changes: 47 additions & 0 deletions docs/release/skillspector-2.9.3.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# SkillSpector v2.9.3

Released: 2026-08-11

## Summary

This patch makes malformed structured LLM responses non-fatal during analysis. Affected analysis work is now recorded as skipped so reports clearly show degraded, incomplete results while preserving the remaining analysis output.

## Highlights

- Improve resilience to malformed structured responses from LLM-backed analyzers without masking the affected analysis outcome.

## Added

- None.

## Changed

- Analysis ledger and analyzer status handling consistently represent malformed structured-response batches as skipped and degraded rather than failed.

## Fixed

- Preserve the input findings and incomplete-analysis provenance when a malformed structured response exhausts retry handling.

## Security

- None.

## Breaking Changes and Migration

- None.

## Deprecations

- None.

## Validation

- `uv run --locked --extra dev pytest tests/nodes/test_llm_analyzer_base.py tests/nodes/test_meta_analyzer.py tests/nodes/test_finalize_inspection_ledger.py tests/test_inspection_ledger.py tests/test_mcp_tool_poisoning.py` — passed.

## Known Limitations

- Malformed structured responses remain unavailable for analysis; this release surfaces their impact as incomplete rather than producing findings for the affected work.

## References

- `CHANGELOG.md`
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "skillspector"
version = "2.9.2"
version = "2.9.3"
description = "SkillSpector: Security scanner for AI agent skills (Claude Code, Cursor, and similar). Scans skills for vulnerabilities, malicious patterns, and security risks before installation. Supports Git repos, URLs, zips, and local directories; runs static pattern checks and optional LLM semantic analysis; outputs terminal, JSON, and Markdown reports with risk scoring."
readme = "README.md"
license = "Apache-2.0"
Expand Down
24 changes: 19 additions & 5 deletions src/skillspector/inspection_ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ class LedgerReason(StrEnum):
}


def outcome_for_llm_batch_failure(reason: LedgerReason) -> LedgerOutcome:
"""Return the terminal ledger outcome for an exhausted LLM batch."""
return (
LedgerOutcome.SKIPPED
if reason is LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID
else LedgerOutcome.FAILED
)


class PlannedWorkTarget(TypedDict):
"""One analyzer work item expected to have a terminal ledger row."""

Expand Down Expand Up @@ -279,11 +288,12 @@ def ledger_event(
raise ValueError("non-completed producers cannot reference findings")
elif outcome is LedgerOutcome.COMPLETED and not set(emitted_ids).issubset(input_ids):
raise ValueError("completed meta events must emit a subset of input findings")
elif outcome is LedgerOutcome.FAILED and emitted_ids != input_ids:
raise ValueError("failed meta events must pass every input finding through")
elif outcome is not LedgerOutcome.COMPLETED and outcome is not LedgerOutcome.FAILED:
elif outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED):
if emitted_ids != input_ids:
raise ValueError("failed or skipped meta events must pass every input finding through")
elif outcome is not LedgerOutcome.COMPLETED:
if input_ids or emitted_ids:
raise ValueError("skipped meta events cannot reference findings")
raise ValueError("non-completed meta events cannot reference findings")

work_identity = analyzer_id or f"{record_type.value}:{phase}"
event: InspectionLedgerEvent = {
Expand Down Expand Up @@ -614,7 +624,11 @@ def accounting_error(path: object = None) -> None:
and not set(emitted_ids).issubset(input_ids)
):
accounting_error(event.get("path"))
if is_meta and outcome == LedgerOutcome.FAILED and emitted_ids != input_ids:
if (
is_meta
and outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED)
and emitted_ids != input_ids
):
accounting_error(event.get("path"))
for finding_id in [*input_ids, *emitted_ids]:
if finding_id not in findings_by_id:
Expand Down
24 changes: 4 additions & 20 deletions src/skillspector/llm_analyzer_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@
InspectionLedgerEvent,
LedgerOutcome,
LedgerReason,
analyzer_status_event,
analyzer_status_for_events,
ledger_event,
outcome_for_llm_batch_failure,
)
from skillspector.llm_utils import (
StructuredOutputParseError,
Expand Down Expand Up @@ -320,7 +321,7 @@ def ledger_events_for_batches(
events.append(
ledger_event(
analyzer_id=analyzer_id,
outcome=LedgerOutcome.FAILED,
outcome=outcome_for_llm_batch_failure(failure.reason),
phase="semantic",
path=path,
start_line=start_line,
Expand All @@ -330,24 +331,7 @@ def ledger_events_for_batches(
)
)

status = analyzer_status_event(
analyzer_id=analyzer_id,
status=(
"failed"
if any(event["outcome"] is LedgerOutcome.FAILED for event in events)
else "completed"
),
planned_work=[
{
"work_id": event["work_id"],
"path": event["path"],
"start_line": event["start_line"],
"end_line": event["end_line"],
}
for event in events
],
)
return events, status
return events, analyzer_status_for_events(analyzer_id, events)


# ---------------------------------------------------------------------------
Expand Down
58 changes: 36 additions & 22 deletions src/skillspector/nodes/analyzers/mcp_tool_poisoning.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@
LedgerOutcome,
LedgerReason,
analyzer_status_event,
analyzer_status_for_events,
ledger_event,
outcome_for_llm_batch_failure,
)
from skillspector.llm_analyzer_base import Batch, LLMAnalyzerBase
from skillspector.models import Finding
Expand Down Expand Up @@ -736,11 +738,12 @@ def _check_tp4(
list[Finding],
LLMCallRecord | None,
str | None,
LedgerReason | None,
list[InferenceUsageRecord],
]:
"""TP4: LLM-based description-behavior mismatch detection.

Returns ``(findings, record, error_class, inference_usage)`` where
Returns ``(findings, record, error_class, failure_reason, inference_usage)`` where
*record* is the LLM-call telemetry for ``llm_call_log`` — or ``None`` when
no LLM call was attempted (no description / no executable code), so an
intentional no-op is never counted as a degraded LLM stage. Token usage is
Expand All @@ -752,7 +755,7 @@ def _check_tp4(
manifest: dict = state.get("manifest") or {}
description = manifest.get("description")
if not description or not isinstance(description, str) or not description.strip():
return [], None, None, []
return [], None, None, None, []

triggers = manifest.get("triggers") or []
permissions = manifest.get("permissions")
Expand All @@ -774,7 +777,7 @@ def _check_tp4(
code_parts.append(f"### {path} ({file_type})\n{content}")

if not code_parts:
return [], None, None, []
return [], None, None, None, []

code_contents = "\n\n".join(code_parts)

Expand Down Expand Up @@ -824,6 +827,7 @@ def _check_tp4(
error=f"TP4 LLM batch failed: {failure.error_class}",
),
failure.error_class,
failure.reason,
cast(list[InferenceUsageRecord], analyzer.inference_usage),
)
result = outcome.successful[0][1][0]
Expand All @@ -832,11 +836,23 @@ def _check_tp4(
ok_record = llm_call_record(ANALYZER_ID, ok=True)

if not result.is_mismatch:
return [], ok_record, None, cast(list[InferenceUsageRecord], analyzer.inference_usage)
return (
[],
ok_record,
None,
None,
cast(list[InferenceUsageRecord], analyzer.inference_usage),
)

confidence = result.confidence
if confidence < 0.5:
return [], ok_record, None, cast(list[InferenceUsageRecord], analyzer.inference_usage)
return (
[],
ok_record,
None,
None,
cast(list[InferenceUsageRecord], analyzer.inference_usage),
)

severity = "HIGH" if confidence >= 0.7 else "MEDIUM"

Expand Down Expand Up @@ -868,6 +884,7 @@ def _check_tp4(
],
ok_record,
None,
None,
cast(list[InferenceUsageRecord], analyzer.inference_usage),
)

Expand All @@ -880,11 +897,12 @@ def _check_tp4(
[],
llm_call_record(ANALYZER_ID, ok=False, error=str(exc)),
type(exc).__name__,
LedgerReason.LLM_BATCH_FAILED,
cast(list[InferenceUsageRecord], analyzer.inference_usage)
if analyzer is not None
else [],
)
return [], None, None, []
return [], None, None, None, []


# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -946,36 +964,32 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse:
tp4_record: LLMCallRecord | None = None
tp4_findings: list[Finding] = []
tp4_error_class: str | None = None
tp4_failure_reason: LedgerReason | None = None
tp4_usage: list[InferenceUsageRecord] = []
if state.get("use_llm", True):
tp4_findings, tp4_record, tp4_error_class, tp4_usage = _check_tp4(state)
tp4_findings, tp4_record, tp4_error_class, tp4_failure_reason, tp4_usage = _check_tp4(state)
findings.extend(tp4_findings)

logger.info("%s: %d findings", ANALYZER_ID, len(findings))
if tp4_record is not None:
tp4_event_outcome = (
LedgerOutcome.COMPLETED
if tp4_record["ok"]
else outcome_for_llm_batch_failure(tp4_failure_reason or LedgerReason.LLM_BATCH_FAILED)
)
tp4_event = ledger_event(
analyzer_id=ANALYZER_ID,
outcome=LedgerOutcome.COMPLETED if tp4_record["ok"] else LedgerOutcome.FAILED,
outcome=tp4_event_outcome,
phase="semantic",
path="SKILL.md",
reason=None if tp4_record["ok"] else LedgerReason.LLM_BATCH_FAILED,
reason=(
None if tp4_record["ok"] else tp4_failure_reason or LedgerReason.LLM_BATCH_FAILED
),
emitted_finding_ids=[finding.finding_id for finding in tp4_findings],
error_class=tp4_error_class,
)
ledger.append(tp4_event)
status = analyzer_status_event(
analyzer_id=ANALYZER_ID,
status="failed" if tp4_record is not None and not tp4_record["ok"] else "completed",
planned_work=[
{
"work_id": event["work_id"],
"path": event["path"],
"start_line": event["start_line"],
"end_line": event["end_line"],
}
for event in ledger
],
)
status = analyzer_status_for_events(ANALYZER_ID, ledger)
result: AnalyzerNodeResponse = {
"findings": findings,
"inspection_ledger": ledger,
Expand Down
25 changes: 6 additions & 19 deletions src/skillspector/nodes/meta_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,10 @@
LedgerOutcome,
LedgerReason,
analyzer_status_event,
analyzer_status_for_events,
inspection_work_id,
ledger_event,
outcome_for_llm_batch_failure,
)
from skillspector.llm_analyzer_base import (
Batch,
Expand Down Expand Up @@ -559,7 +561,7 @@ def _meta_ledger_response(
events.append(
ledger_event(
analyzer_id="meta_analyzer",
outcome=LedgerOutcome.FAILED,
outcome=outcome_for_llm_batch_failure(failure.reason),
phase="meta",
path=batch.file_path,
start_line=batch.start_line if batch.end_line is not None else None,
Expand All @@ -570,24 +572,9 @@ def _meta_ledger_response(
error_class=failure.error_class,
)
)
status = analyzer_status_event(
analyzer_id="meta_analyzer",
status=(
"failed"
if any(event["outcome"] is LedgerOutcome.FAILED for event in events)
else "completed"
),
planned_work=[
{
"work_id": event["work_id"],
"path": event["path"],
"start_line": event["start_line"],
"end_line": event["end_line"],
}
for event in events
],
)
return events, status
if not events:
return events, analyzer_status_event(analyzer_id="meta_analyzer", status="completed")
return events, analyzer_status_for_events("meta_analyzer", events)


def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse:
Expand Down
29 changes: 29 additions & 0 deletions tests/nodes/test_finalize_inspection_ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,35 @@ def test_meta_failure_preserves_primary_coverage_but_fails_execution() -> None:
assert effective_ids == [finding.finding_id]


def test_skipped_meta_event_that_drops_findings_is_a_fatal_accounting_error() -> None:
"""Finalization rejects malformed skipped meta rows that bypass the factory."""
finding = Finding(rule_id="P1", message="unsafe", file="SKILL.md")
skipped_meta = ledger_event(
outcome=LedgerOutcome.SKIPPED,
phase="meta",
analyzer_id="meta_analyzer",
reason=LedgerReason.LLM_STRUCTURED_RESPONSE_INVALID,
path="SKILL.md",
input_finding_ids=[finding.finding_id],
emitted_finding_ids=[finding.finding_id],
)
skipped_meta["emitted_finding_ids"] = []

completeness, _ = finalize_ledger(
{
"components": ["SKILL.md"],
"findings": [finding],
"inspection_ledger": [skipped_meta],
}
)

assert completeness["execution_successful"] is False
assert completeness["ledger_exceptions"][0]["reason_code"] == (
LedgerReason.FINDING_ACCOUNTING_ERROR
)
assert completeness["ledger_exceptions"][0]["fatal"] is True


def test_json_round_trip_keeps_failed_ledger_work_fatal() -> None:
"""Deserialized StrEnum values must retain failure semantics."""
state = json.loads(
Expand Down
Loading
Loading