From 7ce882179167a2c0ad92e68291ab18d48ab53cae Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 10 Aug 2026 16:16:01 +0100 Subject: [PATCH 1/2] fix(analyzers): HIGH SC8 when skill ships __pycache__ or .pyc Close the silent bytecode skip described in #356: discovery excludes __pycache__ and treats .pyc as binary, so presence alone must fail the score even before full disassembly exists. Signed-off-by: Alex J Lennon Co-authored-by: Cursor --- README.md | 7 +- .../nodes/analyzers/pattern_defaults.py | 4 + .../analyzers/static_patterns_supply_chain.py | 92 ++++++++++++++++++- .../analyzers/test_sc8_shipped_bytecode.py | 41 +++++++++ 4 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 tests/nodes/analyzers/test_sc8_shipped_bytecode.py diff --git a/README.md b/README.md index 2d4d64c1b..b5ee4e0bf 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ SkillSpector is part of the [NVIDIA Verified Skills pipeline](https://docs.nvidi ## Features - **Multi-format input**: Scan Git repos, URLs, zip files, directories, or single files -- **68 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning +- **69 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning - **Two-stage analysis**: Fast static analysis + optional LLM semantic evaluation - **Live vulnerability lookups**: SC4 queries [OSV.dev](https://osv.dev) for real-time CVE data with automatic offline fallback - **Multiple output formats**: Terminal, JSON, Markdown, and SARIF reports @@ -352,7 +352,7 @@ claude mcp add skillspector -- skillspector mcp ## Vulnerability Patterns -SkillSpector detects **68 vulnerability patterns** across 17 categories: +SkillSpector detects **69 vulnerability patterns** across 17 categories: ### Prompt Injection (5 patterns) @@ -389,7 +389,7 @@ SkillSpector detects **68 vulnerability patterns** across 17 categories: | PE2 | Sudo/Root Execution | MEDIUM | Invoking elevated system privileges | | PE3 | Credential Access | HIGH | Reading SSH keys, tokens, passwords | -### Supply Chain (6 patterns) +### Supply Chain (7+ patterns) | ID | Pattern | Severity | Description | |----|---------|----------|-------------| @@ -399,6 +399,7 @@ SkillSpector detects **68 vulnerability patterns** across 17 categories: | SC4 | Known Vulnerable Dependencies | HIGH | Dependencies with known CVEs (live OSV.dev lookup) | | SC5 | Abandoned Dependencies | MEDIUM | Unmaintained packages without security updates | | SC6 | Typosquatting | HIGH | Package names similar to popular packages | +| SC8 | Shipped Python Bytecode | HIGH | `__pycache__` / `.pyc` present (discovery skips; malicious bytecode bypass) | ### Excessive Agency (4 patterns) diff --git a/src/skillspector/nodes/analyzers/pattern_defaults.py b/src/skillspector/nodes/analyzers/pattern_defaults.py index bb0a7f2b9..ebafcd01d 100644 --- a/src/skillspector/nodes/analyzers/pattern_defaults.py +++ b/src/skillspector/nodes/analyzers/pattern_defaults.py @@ -91,6 +91,7 @@ class PatternCategory(StrEnum): "SC5": "Dependency appears abandoned or unmaintained. Abandoned packages no longer receive security patches, leaving known and future vulnerabilities unaddressed.", "SC6": "Package name closely resembles a popular package, suggesting possible typosquatting. Attackers publish malicious packages with similar names to trick developers into installing them.", "SC7": "Code pulls a container image with signature or registry verification disabled (--disable-content-trust, DOCKER_CONTENT_TRUST=0, --insecure-registry). This accepts tampered or unverified images and is a container supply-chain risk.", + "SC8": "Skill ships Python bytecode (__pycache__/ or .pyc/.pyo). Discovery skips these paths, so malicious bytecode can score SAFE while decoy sources look clean.", # Trigger Abuse "TR1": "Skill uses overly broad trigger patterns that match common words or phrases, causing it to activate in unintended contexts and potentially shadow other skills.", "TR2": "Skill trigger shadows a common built-in command or another skill's trigger, potentially intercepting requests meant for trusted functionality.", @@ -181,6 +182,7 @@ class PatternCategory(StrEnum): "SC5": PatternCategory.SUPPLY_CHAIN.value, "SC6": PatternCategory.SUPPLY_CHAIN.value, "SC7": PatternCategory.SUPPLY_CHAIN.value, + "SC8": PatternCategory.SUPPLY_CHAIN.value, "TR1": PatternCategory.TRIGGER_ABUSE.value, "TR2": PatternCategory.TRIGGER_ABUSE.value, "TR3": PatternCategory.TRIGGER_ABUSE.value, @@ -259,6 +261,7 @@ class PatternCategory(StrEnum): "SC5": "Abandoned Dependency", "SC6": "Typosquatting Dependency", "SC7": "Untrusted Container Image", + "SC8": "Shipped Python Bytecode", "TR1": "Overly Broad Trigger", "TR2": "Shadow Command Trigger", "TR3": "Keyword Baiting Trigger", @@ -344,6 +347,7 @@ class PatternCategory(StrEnum): "SC5": "Replace the abandoned dependency with an actively maintained alternative. Check the package's repository for last commit date and open issues.", "SC6": "Verify the package name is correct and not a typosquatting variant. Compare against the official package name on PyPI or npm.", "SC7": "Keep image signature verification (Docker Content Trust / cosign) and registry TLS enabled. Pull only signed images from trusted registries; never disable content-trust or use insecure registries in skill code.", + "SC8": "Do not ship __pycache__/ or .pyc/.pyo in skills. Delete bytecode before packaging; if presence is intentional for a lab fixture, quarantine it outside the skill install path.", # Trigger Abuse "TR1": "Use specific, narrow trigger patterns that match only the skill's intended use case. Avoid single-word or common-phrase triggers.", "TR2": "Choose triggers that do not conflict with built-in commands or other skills. Prefix with a unique namespace if necessary.", diff --git a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py index f781d7e52..c84ce25e6 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py +++ b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py @@ -13,13 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Static patterns: supply chain (SC1–SC7) and trigger analysis (TR1–TR3). +"""Static patterns: supply chain (SC1–SC8) and trigger analysis (TR1–TR3). SC1–SC3: regex-based pattern matching (original implementation). SC4: Known vulnerable dependencies — live OSV.dev lookup with static fallback. SC5: Abandoned dependencies — flags known-abandoned or archived packages. SC6: Typosquatting — flags package names similar to popular packages. SC7: Untrusted container image — flags image signature / registry-verification bypass. +SC8: Shipped Python bytecode — flags __pycache__/ and *.pyc/*.pyo that discovery skips. TR1–TR3: Trigger analysis — flags overly broad, shadowing, or baiting triggers. Node and analyze() in one module. @@ -27,9 +28,11 @@ from __future__ import annotations +import os import re import sys import tomllib +from pathlib import Path from urllib.parse import urlparse from packaging.requirements import InvalidRequirement, Requirement @@ -1185,13 +1188,86 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find return findings + +# --------------------------------------------------------------------------- +# SC8: Shipped Python bytecode (closes silent __pycache__ / .pyc skip) +# --------------------------------------------------------------------------- + +# Still skip heavy/vendor trees for SC8, but *do* descend into __pycache__. +_SC8_SKIP_DIRS = frozenset({".git", "node_modules", ".venv", "venv", ".tox", ".pytest_cache"}) +_SC8_BYTECODE_SUFFIXES = (".pyc", ".pyo") + + +def _analyze_shipped_bytecode(skill_path: str) -> list[Finding]: + """Emit SC8 when a skill ships __pycache__ dirs or .pyc/.pyo files. + + ``build_context`` excludes ``__pycache__`` from inventory and + ``static_runner`` treats ``.pyc`` as binary, so malicious bytecode can + otherwise score SAFE. Presence alone is a HIGH supply-chain signal; + full disassembly can come later. + """ + findings: list[Finding] = [] + if not skill_path or not isinstance(skill_path, str): + return findings + root = Path(skill_path) + if not root.is_dir(): + return findings + + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = sorted(name for name in dirnames if name not in _SC8_SKIP_DIRS) + rel_dir = Path(dirpath).relative_to(root).as_posix() + if rel_dir == ".": + rel_dir = "" + + for dirname in list(dirnames): + if dirname != "__pycache__": + continue + rel = f"{rel_dir}/{dirname}/" if rel_dir else f"{dirname}/" + af = AnalyzerFinding( + rule_id="SC8", + message="Skill ships a __pycache__ directory that normal discovery skips", + severity=Severity.HIGH, + location=Location(file=rel, start_line=1), + confidence=0.95, + tags=[PatternCategory.SUPPLY_CHAIN.value], + matched_text=rel, + context=( + "Python may load .pyc from this directory even when decoy " + ".py sources look clean (PEP 552 UNCHECKED_HASH)." + ), + ) + findings.append(analyzer_finding_to_finding(af)) + + for filename in sorted(filenames): + lower = filename.lower() + if not lower.endswith(_SC8_BYTECODE_SUFFIXES): + continue + rel = f"{rel_dir}/{filename}" if rel_dir else filename + af = AnalyzerFinding( + rule_id="SC8", + message="Skill ships Python bytecode (.pyc/.pyo) that normal analysis skips", + severity=Severity.HIGH, + location=Location(file=rel, start_line=1), + confidence=0.95, + tags=[PatternCategory.SUPPLY_CHAIN.value], + matched_text=filename, + context=( + "Bytecode is excluded from content analysis; a malicious " + ".pyc can execute while source decoys remain clean." + ), + ) + findings.append(analyzer_finding_to_finding(af)) + + return findings + + # --------------------------------------------------------------------------- # Graph node # --------------------------------------------------------------------------- def node(state: SkillspectorState) -> AnalyzerNodeResponse: - """Run supply_chain patterns (SC1–SC6) and trigger analysis (TR1–TR3).""" + """Run supply_chain patterns (SC1–SC8) and trigger analysis (TR1–TR3).""" # SC1–SC3 via static_runner response = static_runner.run_static_patterns_with_ledger(state, [sys.modules[__name__]]) findings = response["findings"] @@ -1263,6 +1339,18 @@ def record_extra_findings( f"{ANALYZER_ID}_triggers", ) + # SC8: shipped bytecode / __pycache__ (discovery otherwise skips these) + skill_path = state.get("skill_path") or "" + if isinstance(skill_path, str) and skill_path.strip(): + bytecode_findings = _analyze_shipped_bytecode(skill_path) + findings.extend(bytecode_findings) + if bytecode_findings: + record_extra_findings( + skill_path, + bytecode_findings, + f"{ANALYZER_ID}_bytecode", + ) + logger.info("%s: %d findings", ANALYZER_ID, len(findings)) response["analyzer_status_events"] = [ analyzer_status_for_events(ANALYZER_ID, response["inspection_ledger"]) diff --git a/tests/nodes/analyzers/test_sc8_shipped_bytecode.py b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py new file mode 100644 index 000000000..8e159ddd5 --- /dev/null +++ b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path + +from skillspector.nodes.analyzers import static_patterns_supply_chain as supply_chain + + +def test_sc8_flags_pycache_and_pyc(tmp_path: Path) -> None: + cache = tmp_path / "scripts" / "__pycache__" + cache.mkdir(parents=True) + (cache / "evil.cpython-312.pyc").write_bytes(b"\x00") + (tmp_path / "orphan.pyc").write_bytes(b"\x00") + (tmp_path / "clean.py").write_text("print('ok')\n", encoding="utf-8") + + findings = supply_chain._analyze_shipped_bytecode(str(tmp_path)) + rule_ids = {f.rule_id for f in findings} + assert rule_ids == {"SC8"} + paths = {f.file for f in findings} + assert "scripts/__pycache__/" in paths + assert "scripts/__pycache__/evil.cpython-312.pyc" in paths + assert "orphan.pyc" in paths + assert all(f.severity == "HIGH" for f in findings) + + +def test_sc8_clean_tree_has_no_findings(tmp_path: Path) -> None: + (tmp_path / "SKILL.md").write_text("# demo\n", encoding="utf-8") + (tmp_path / "main.py").write_text("x = 1\n", encoding="utf-8") + assert supply_chain._analyze_shipped_bytecode(str(tmp_path)) == [] From 3d12791f8782aa4e61bfe110fd2954abbd2608e4 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Tue, 11 Aug 2026 21:44:19 -0700 Subject: [PATCH 2/2] fix(sc8): enforce fail-closed bytecode verdict Signed-off-by: Narendran Raghavan --- .../analyzers/static_patterns_supply_chain.py | 13 ++++++---- src/skillspector/nodes/report.py | 15 ++++++++++- .../analyzers/test_sc8_shipped_bytecode.py | 25 +++++++++++++++++++ tests/nodes/test_report.py | 7 ++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py index c84ce25e6..63a6b55e2 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py +++ b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py @@ -1188,7 +1188,6 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find return findings - # --------------------------------------------------------------------------- # SC8: Shipped Python bytecode (closes silent __pycache__ / .pyc skip) # --------------------------------------------------------------------------- @@ -1277,7 +1276,7 @@ def record_extra_findings( extra_findings: list[Finding], fallback_analyzer_id: str, ) -> None: - """Attach dependency/manifest findings to the matching completed work item.""" + """Attach supplemental findings to the matching completed work item.""" if not extra_findings: return finding_ids = [finding.finding_id for finding in extra_findings] @@ -1344,10 +1343,14 @@ def record_extra_findings( if isinstance(skill_path, str) and skill_path.strip(): bytecode_findings = _analyze_shipped_bytecode(skill_path) findings.extend(bytecode_findings) - if bytecode_findings: + for finding_path in sorted({finding.file.rstrip("/") for finding in bytecode_findings}): record_extra_findings( - skill_path, - bytecode_findings, + finding_path, + [ + finding + for finding in bytecode_findings + if finding.file.rstrip("/") == finding_path + ], f"{ANALYZER_ID}_bytecode", ) diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index fb7dc5525..68c0760a8 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -156,6 +156,11 @@ def _severity_to_sarif_level(severity: str) -> Literal["error", "warning", "note _MAX_OCCURRENCES_PER_RULE = 3 _DIMINISHING_WEIGHTS = (1.0, 0.5, 0.25) +# Some findings describe artifacts whose unanalyzed contents can execute. Their +# presence must block installation even when ordinary confidence-weighted, +# per-rule scoring would otherwise keep the aggregate below the CLI threshold. +_RISK_SCORE_FLOORS_BY_RULE_ID = {"SC8": 51} + def _compute_risk_score( findings: list[Finding], @@ -220,7 +225,15 @@ def _compute_risk_score( score += contribution - final_score = min(100, max(0, int(score))) + score_floor = max( + ( + _RISK_SCORE_FLOORS_BY_RULE_ID.get(f.rule_id, 0) + for f in sorted_findings + if max(0.0, min(1.0, f.confidence)) > 0.0 + ), + default=0, + ) + final_score = min(100, max(score_floor, int(score))) severity_band = "LOW" for threshold, band in _RISK_SEVERITY_BANDS: diff --git a/tests/nodes/analyzers/test_sc8_shipped_bytecode.py b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py index 8e159ddd5..aa45f2db7 100644 --- a/tests/nodes/analyzers/test_sc8_shipped_bytecode.py +++ b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py @@ -13,8 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from pathlib import Path +from typer.testing import CliRunner + +from skillspector.cli import app from skillspector.nodes.analyzers import static_patterns_supply_chain as supply_chain @@ -39,3 +43,24 @@ def test_sc8_clean_tree_has_no_findings(tmp_path: Path) -> None: (tmp_path / "SKILL.md").write_text("# demo\n", encoding="utf-8") (tmp_path / "main.py").write_text("x = 1\n", encoding="utf-8") assert supply_chain._analyze_shipped_bytecode(str(tmp_path)) == [] + + +def test_sc8_single_pyc_blocks_install_and_cli_exit(tmp_path: Path) -> None: + (tmp_path / "SKILL.md").write_text( + "---\nname: shipped-bytecode\n---\n# Shipped bytecode\n", encoding="utf-8" + ) + (tmp_path / "payload.pyc").write_bytes(b"\x00") + + result = CliRunner().invoke( + app, + ["scan", str(tmp_path), "--format", "json", "--no-llm"], + ) + + assert result.exit_code == 1, result.output + report = json.loads(result.output) + assert report["risk_assessment"] == { + "score": 51, + "severity": "HIGH", + "recommendation": "DO_NOT_INSTALL", + } + assert any(issue["id"] == "SC8" for issue in report["issues"]) diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 74dff645d..2f40e7d7b 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -84,6 +84,13 @@ def test_single_finding_partial_confidence_scales_score(self) -> None: score, _, _ = _compute_risk_score(findings, False) assert score == 12 # 25 * 1.0 * 0.5 = 12.5 -> int(12.5) = 12 + def test_shipped_bytecode_enforces_blocking_risk_floor(self) -> None: + findings = [_finding("SC8", "HIGH", confidence=0.95, file="payload.pyc")] + score, band, recommendation = _compute_risk_score(findings, False) + assert score == 51 + assert band == "HIGH" + assert recommendation == "DO_NOT_INSTALL" + def test_unknown_severity_defaults_to_low_points(self) -> None: f = _finding("R1", "LOW") f.severity = ""