From 4ae41a9f14b84720cb4955fcb4bcb23942214e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:50:33 +0200 Subject: [PATCH 01/19] Add hardened Stage 0 manifest model --- src/stage0_manifest.py | 203 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 src/stage0_manifest.py diff --git a/src/stage0_manifest.py b/src/stage0_manifest.py new file mode 100644 index 0000000..b511e68 --- /dev/null +++ b/src/stage0_manifest.py @@ -0,0 +1,203 @@ +"""Stage 0 canonical VCS change manifest primitives. + +The implementation deliberately binds records to Git object identity rather +than treating the mutable working tree as authoritative. It is intentionally +small and dependency-free so the acceptance tests can exercise the invariants +without a repository checkout. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from dataclasses import dataclass +from enum import Enum +from pathlib import Path, PurePosixPath +from typing import Iterable, Mapping + +SCHEMA_VERSION = "1" +RULESET_VERSION = "stage0-v1" + + +class Disposition(str, Enum): + PRIMARY = "PRIMARY" + SECONDARY = "SECONDARY" + DEPENDENCY = "DEPENDENCY" + GENERATED = "GENERATED" + OVERSIZED = "OVERSIZED" + NON_CODE = "NON_CODE" + DELETED = "DELETED" + REJECTED = "REJECTED" + + +class Reason(str, Enum): + CODE_CHANGE = "code_change" + MISLEADING_EXTENSION = "misleading_extension" + DEPENDENCY = "dependency" + GENERATED_ARTIFACT = "generated_artifact" + OVERSIZED_ARTIFACT = "oversized_artifact" + NON_CODE = "non_code" + DELETED = "deleted" + PATH_TRAVERSAL_ATTEMPT = "path_traversal_attempt" + VCS_OBJECT_MISMATCH = "vcs_object_mismatch" + VCS_OBJECT_MISSING = "vcs_object_missing" + INVALID_PATH = "invalid_path" + BINARY = "binary" + + +@dataclass(frozen=True) +class ChangeRecord: + path: str + change_type: str + disposition: Disposition + reason: Reason + size_bytes: int + is_binary: bool + blob_object: str | None + content_sha256: str | None + + def canonical(self) -> dict[str, object]: + return { + "path": self.path, + "change_type": self.change_type, + "disposition": self.disposition.value, + "reason": self.reason.value, + "size_bytes": self.size_bytes, + "is_binary": self.is_binary, + "blob_object": self.blob_object, + "content_sha256": self.content_sha256, + } + + +@dataclass(frozen=True) +class Manifest: + schema_version: str + ruleset_version: str + repository: str + base_commit: str + head_commit: str + pr_number: int + files: tuple[ChangeRecord, ...] + manifest_sha256: str + + def payload(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "ruleset_version": self.ruleset_version, + "repository": self.repository, + "base_commit": self.base_commit, + "head_commit": self.head_commit, + "pr_number": self.pr_number, + "files": [record.canonical() for record in self.files], + } + + def canonical_json(self) -> bytes: + return canonical_json(self.payload()) + + +def canonical_json(payload: Mapping[str, object]) -> bytes: + """Canonical UTF-8 JSON: stable keys, arrays, separators and Unicode.""" + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def freeze_manifest( + *, + repository: str, + base_commit: str, + head_commit: str, + pr_number: int, + files: Iterable[ChangeRecord], +) -> Manifest: + """Construct immutable payload first, then commit to its exact bytes.""" + ordered = tuple(sorted(files, key=lambda record: record.path)) + payload = { + "schema_version": SCHEMA_VERSION, + "ruleset_version": RULESET_VERSION, + "repository": repository, + "base_commit": base_commit, + "head_commit": head_commit, + "pr_number": pr_number, + "files": [record.canonical() for record in ordered], + } + digest = sha256_hex(canonical_json(payload)) + return Manifest( + schema_version=SCHEMA_VERSION, + ruleset_version=RULESET_VERSION, + repository=repository, + base_commit=base_commit, + head_commit=head_commit, + pr_number=pr_number, + files=ordered, + manifest_sha256=digest, + ) + + +def safe_join(resolved_root: Path, candidate: str) -> Path: + """Resolve candidate and require component-aware containment.""" + root = resolved_root.resolve(strict=True) + candidate_path = (root / candidate).resolve(strict=False) + try: + candidate_path.relative_to(root) + except ValueError as exc: + raise ValueError("path_traversal_attempt") from exc + return candidate_path + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_binary_bytes(content: bytes) -> bool: + return b"\x00" in content + + +def classify_path(path: str, content: bytes, *, generated: bool = False, dependency: bool = False, oversized: bool = False) -> tuple[Disposition, Reason]: + """Deterministic routing; content characteristics outrank filename suffix.""" + if oversized: + return Disposition.OVERSIZED, Reason.OVERSIZED_ARTIFACT + if generated: + return Disposition.GENERATED, Reason.GENERATED_ARTIFACT + if dependency: + return Disposition.DEPENDENCY, Reason.DEPENDENCY + + suffix = PurePosixPath(path).suffix.lower() + code_suffixes = {".py", ".rs", ".ts", ".tsx", ".js", ".jsx", ".go", ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".sh"} + text = content.decode("utf-8", errors="ignore") + executable_markers = ("#!/", "import ", "from ", "fn ", "def ", "class ", "function ", "const ", "let ", "use ") + looks_executable = any(marker in text for marker in executable_markers) + + if suffix in code_suffixes: + return Disposition.PRIMARY, Reason.CODE_CHANGE + if looks_executable: + return Disposition.SECONDARY, Reason.MISLEADING_EXTENSION + if is_binary_bytes(content): + return Disposition.NON_CODE, Reason.BINARY + return Disposition.NON_CODE, Reason.NON_CODE + + +def git_blob_sha(content: bytes) -> str: + """Compute Git's canonical blob object identity for content.""" + header = f"blob {len(content)}\0".encode("ascii") + return hashlib.sha1(header + content).hexdigest() + + +def verify_head_binding(content: bytes, expected_blob_object: str, expected_content_sha256: str) -> None: + if git_blob_sha(content) != expected_blob_object: + raise ValueError("vcs_object_mismatch") + if sha256_hex(content) != expected_content_sha256: + raise ValueError("vcs_object_mismatch") From 2c26e7ae7bf3960e1df569d812f4842108f09ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:50:43 +0200 Subject: [PATCH 02/19] Add Stage 0 adversarial acceptance tests --- tests/test_stage0_manifest.py | 120 ++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 tests/test_stage0_manifest.py diff --git a/tests/test_stage0_manifest.py b/tests/test_stage0_manifest.py new file mode 100644 index 0000000..cd295c1 --- /dev/null +++ b/tests/test_stage0_manifest.py @@ -0,0 +1,120 @@ +from pathlib import Path + +import pytest + +from src.stage0_manifest import ( + ChangeRecord, + Disposition, + Reason, + classify_path, + freeze_manifest, + git_blob_sha, + safe_join, + sha256_hex, + verify_head_binding, +) + + +def test_component_aware_containment_rejects_prefix_collision(tmp_path: Path) -> None: + root = tmp_path / "repo" + sibling = tmp_path / "repo-attacker" + root.mkdir() + sibling.mkdir() + with pytest.raises(ValueError, match="path_traversal_attempt"): + safe_join(root, "../repo-attacker/secret.py") + + +def test_component_aware_containment_accepts_descendant(tmp_path: Path) -> None: + root = tmp_path / "repo" + root.mkdir() + candidate = safe_join(root, "src/script.py") + assert candidate == root / "src/script.py" + + +def test_misleading_extension_routes_executable_content_to_secondary() -> None: + disposition, reason = classify_path("src/script.txt", b"#!/usr/bin/env python3\nprint('x')\n") + assert disposition is Disposition.SECONDARY + assert reason is Reason.MISLEADING_EXTENSION + + +def test_normal_code_routes_to_primary() -> None: + disposition, reason = classify_path("src/main.py", b"def main():\n return 1\n") + assert disposition is Disposition.PRIMARY + assert reason is Reason.CODE_CHANGE + + +def test_taxonomy_preserves_generated_dependency_and_oversized() -> None: + content = b"generated" + assert classify_path("x.py", content, generated=True) == ( + Disposition.GENERATED, + Reason.GENERATED_ARTIFACT, + ) + assert classify_path("x.py", content, dependency=True) == ( + Disposition.DEPENDENCY, + Reason.DEPENDENCY, + ) + assert classify_path("x.py", content, oversized=True) == ( + Disposition.OVERSIZED, + Reason.OVERSIZED_ARTIFACT, + ) + + +def test_head_object_binding_requires_both_object_and_content_identity() -> None: + content = b"fn main() {}\n" + verify_head_binding(content, git_blob_sha(content), sha256_hex(content)) + with pytest.raises(ValueError, match="vcs_object_mismatch"): + verify_head_binding(content, "0" * 40, sha256_hex(content)) + with pytest.raises(ValueError, match="vcs_object_mismatch"): + verify_head_binding(content, git_blob_sha(content), "0" * 64) + + +def test_manifest_digest_commits_to_canonical_payload() -> None: + record = ChangeRecord( + path="src/main.py", + change_type="modified", + disposition=Disposition.PRIMARY, + reason=Reason.CODE_CHANGE, + size_bytes=14, + is_binary=False, + blob_object=git_blob_sha(b"def main():\n"), + content_sha256=sha256_hex(b"def main():\n"), + ) + manifest = freeze_manifest( + repository="cryptofixyup/SentinelAI", + base_commit="a" * 40, + head_commit="b" * 40, + pr_number=7, + files=[record], + ) + assert manifest.manifest_sha256 == sha256_hex(manifest.canonical_json()) + assert manifest.manifest_sha256 == sha256_hex( + manifest.canonical_json() + ) + + +def test_manifest_order_is_deterministic() -> None: + def record(path: str) -> ChangeRecord: + return ChangeRecord(path, "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, 1, False, None, "0" * 64) + + first = freeze_manifest( + repository="r", base_commit="a", head_commit="b", pr_number=1, + files=[record("z.py"), record("a.py")], + ) + second = freeze_manifest( + repository="r", base_commit="a", head_commit="b", pr_number=1, + files=[record("a.py"), record("z.py")], + ) + assert first.canonical_json() == second.canonical_json() + assert first.manifest_sha256 == second.manifest_sha256 + + +def test_rename_preserves_removal_addition_and_relationship() -> None: + records = [ + ChangeRecord("A.py", "renamed_from", Disposition.DELETED, Reason.DELETED, 0, False, "a" * 40, None), + ChangeRecord("B.py", "renamed_to", Disposition.PRIMARY, Reason.CODE_CHANGE, 10, False, "b" * 40, "c" * 64), + ] + manifest = freeze_manifest( + repository="r", base_commit="a", head_commit="b", pr_number=1, files=records + ) + types = {item.change_type for item in manifest.files} + assert {"renamed_from", "renamed_to"} <= types From 61220cf2cebb3d55f36eb7efb67dbc6b54d92ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:50:52 +0200 Subject: [PATCH 03/19] Add Stage 0 manifest acceptance gate --- .github/workflows/stage0-manifest.yml | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/stage0-manifest.yml diff --git a/.github/workflows/stage0-manifest.yml b/.github/workflows/stage0-manifest.yml new file mode 100644 index 0000000..28791f6 --- /dev/null +++ b/.github/workflows/stage0-manifest.yml @@ -0,0 +1,33 @@ +name: Stage 0 Manifest Acceptance + +on: + workflow_dispatch: + pull_request: + paths: + - 'src/stage0_manifest.py' + - 'tests/test_stage0_manifest.py' + - '.github/workflows/stage0-manifest.yml' + push: + branches: + - stage0-manifest-hardening + +permissions: + contents: read + +jobs: + stage0-acceptance: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install test dependencies + run: python -m pip install pytest + + - name: Execute Stage 0 adversarial acceptance suite + run: python -m pytest -q tests/test_stage0_manifest.py From b40989d3bd893822dd507c6cc962aa0afc0353aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:52:08 +0200 Subject: [PATCH 04/19] Stage 1: add deterministic AST parser dependencies --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index c3ae04d..d6fcc6a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ pytest>=8.4,<9.0 httpx>=0.28,<1.0 +tree-sitter>=0.25,<0.26 +tree-sitter-rust>=0.24,<0.25 -r services/api/requirements.txt From 97c530c75a2e07a473570d11628e7d26125b9897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:52:30 +0200 Subject: [PATCH 05/19] Stage 1: add deterministic evidence construction boundary --- src/stage1_evidence.py | 388 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 src/stage1_evidence.py diff --git a/src/stage1_evidence.py b/src/stage1_evidence.py new file mode 100644 index 0000000..df99452 --- /dev/null +++ b/src/stage1_evidence.py @@ -0,0 +1,388 @@ +"""Stage 1 deterministic evidence-construction boundary. + +Stage 1 verifies a Stage 0 manifest, reads immutable Git objects, parses +PRIMARY/DELETED source material structurally, and emits a deterministic +Evidence Manifest. It does not make vulnerability determinations. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import subprocess +from dataclasses import dataclass +from enum import Enum +from typing import Any, Mapping, Sequence + +from tree_sitter import Language, Parser +import tree_sitter_rust + +from .stage0_manifest import Disposition, Manifest, Reason, canonical_json, sha256_hex + +STAGE1_SCHEMA_VERSION = "1.0.0" +STAGE1_RULESET_VERSION = "stage1-v1" +DEFAULT_MAX_BLOCKS = 128 +DEFAULT_MAX_BLOCK_BYTES = 16 * 1024 +DEFAULT_MAX_REFERENCED_NODES = 256 +DEFAULT_MAX_TOTAL_BYTES = 512 * 1024 + + +class Stage1Error(ValueError): + pass + + +class EvidenceType(str, Enum): + MODIFIED_FUNCTION = "MODIFIED_FUNCTION" + MODIFIED_METHOD = "MODIFIED_METHOD" + MODIFIED_TYPE = "MODIFIED_TYPE" + SECURITY_SENSITIVE_DECLARATION = "SECURITY_SENSITIVE_DECLARATION" + CALL = "CALL" + CONDITION = "CONDITION" + RETURN = "RETURN" + AUTHORIZATION_CHECK = "AUTHORIZATION_CHECK" + INPUT_BOUNDARY = "INPUT_BOUNDARY" + DATABASE_OPERATION = "DATABASE_OPERATION" + NETWORK_OPERATION = "NETWORK_OPERATION" + CRYPTO_OPERATION = "CRYPTO_OPERATION" + DELETED_NODE = "DELETED_NODE" + + +@dataclass(frozen=True) +class AstNodeRef: + node_type: str + start_byte: int + end_byte: int + start_line: int + end_line: int + + +@dataclass(frozen=True) +class EvidenceBlock: + id: str + evidence_type: EvidenceType + path: str + commit: str + object_id: str + content_sha256: str + start_byte: int + end_byte: int + start_line: int + end_line: int + node_type: str + content: str | None = None + + def canonical(self) -> dict[str, object]: + return { + "id": self.id, + "evidence_type": self.evidence_type.value, + "path": self.path, + "commit": self.commit, + "object_id": self.object_id, + "content_sha256": self.content_sha256, + "start_byte": self.start_byte, + "end_byte": self.end_byte, + "start_line": self.start_line, + "end_line": self.end_line, + "node_type": self.node_type, + "content": self.content, + } + + +@dataclass(frozen=True) +class EvidenceManifest: + schema_version: str + ruleset_version: str + stage0_sha256: str + repository: str + base_commit: str + head_commit: str + parser: Mapping[str, str] + evidence_blocks: tuple[EvidenceBlock, ...] + incomplete: bool + stage1_sha256: str + + def payload(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "ruleset_version": self.ruleset_version, + "stage0_sha256": self.stage0_sha256, + "repository": self.repository, + "base_commit": self.base_commit, + "head_commit": self.head_commit, + "parser": dict(self.parser), + "evidence_blocks": [block.canonical() for block in self.evidence_blocks], + "incomplete": self.incomplete, + } + + def canonical_json(self) -> bytes: + return canonical_json(self.payload()) + + +def _constant_time_equal_hex(left: str, right: str) -> bool: + try: + return hmac.compare_digest(bytes.fromhex(left), bytes.fromhex(right)) + except ValueError: + return False + + +def verify_stage0_manifest(raw_manifest: Mapping[str, Any]) -> Manifest: + """Verify the Stage 0 commitment before any evidence extraction.""" + required = { + "schema_version", "ruleset_version", "repository", "base_commit", + "head_commit", "pr_number", "files", "manifest_sha256", + } + if set(raw_manifest) != required: + raise Stage1Error("stage0_schema_mismatch") + if not isinstance(raw_manifest["manifest_sha256"], str): + raise Stage1Error("stage0_hash_malformed") + + payload = {key: raw_manifest[key] for key in required if key != "manifest_sha256"} + calculated = sha256_hex(canonical_json(payload)) + if not _constant_time_equal_hex(calculated, raw_manifest["manifest_sha256"]): + raise Stage1Error("stage0_integrity_mismatch") + + try: + records = [] + for item in raw_manifest["files"]: + if set(item) != { + "path", "change_type", "disposition", "reason", "size_bytes", + "is_binary", "blob_object", "content_sha256", + }: + raise Stage1Error("stage0_schema_mismatch") + records.append(item) + parsed = tuple(_change_record_from_mapping(item) for item in records) + manifest = Manifest( + schema_version=str(raw_manifest["schema_version"]), + ruleset_version=str(raw_manifest["ruleset_version"]), + repository=str(raw_manifest["repository"]), + base_commit=str(raw_manifest["base_commit"]), + head_commit=str(raw_manifest["head_commit"]), + pr_number=int(raw_manifest["pr_number"]), + files=parsed, + manifest_sha256=str(raw_manifest["manifest_sha256"]), + ) + except (TypeError, ValueError) as exc: + raise Stage1Error("stage0_schema_mismatch") from exc + + if manifest.schema_version != "1" or manifest.ruleset_version != "stage0-v1": + raise Stage1Error("stage0_schema_mismatch") + return manifest + + +def _change_record_from_mapping(item: Mapping[str, Any]): + from .stage0_manifest import ChangeRecord + return ChangeRecord( + path=str(item["path"]), + change_type=str(item["change_type"]), + disposition=Disposition(str(item["disposition"])), + reason=Reason(str(item["reason"])), + size_bytes=int(item["size_bytes"]), + is_binary=bool(item["is_binary"]), + blob_object=item["blob_object"], + content_sha256=item["content_sha256"], + ) + + +class GitObjectReader: + """Read immutable objects from the repository's Git object database.""" + + def __init__(self, repository_root: str): + self.repository_root = repository_root + + def verify_commit_available(self, commit: str) -> None: + self._run("git", "cat-file", "-e", f"{commit}^{{commit}}") + + def read_blob(self, object_id: str) -> bytes: + output = self._run("git", "cat-file", "blob", object_id) + return output + + def _run(self, *args: str) -> bytes: + try: + result = subprocess.run( + args, + cwd=self.repository_root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise Stage1Error("vcs_object_missing") from exc + return result.stdout + + +def verify_manifest_objects(manifest: Manifest, reader: GitObjectReader) -> None: + reader.verify_commit_available(manifest.base_commit) + reader.verify_commit_available(manifest.head_commit) + for record in manifest.files: + if record.disposition in {Disposition.PRIMARY, Disposition.DELETED, Disposition.SECONDARY}: + if not record.blob_object or not record.content_sha256: + raise Stage1Error("vcs_object_missing") + content = reader.read_blob(record.blob_object) + if sha256_hex(content) != record.content_sha256: + raise Stage1Error("vcs_object_mismatch") + + +def rust_parser() -> tuple[Parser, dict[str, str]]: + language = Language(tree_sitter_rust.language()) + parser = Parser(language) + metadata = { + "name": "tree-sitter", + "version": "0.25.x", + "grammar": "tree-sitter-rust", + "grammar_version": "0.24.x", + } + return parser, metadata + + +def _node_ref(node: Any) -> AstNodeRef: + return AstNodeRef( + node_type=node.type, + start_byte=node.start_byte, + end_byte=node.end_byte, + start_line=node.start_point[0] + 1, + end_line=node.end_point[0] + 1, + ) + + +def parse_ast(content: bytes) -> tuple[AstNodeRef, ...]: + parser, _ = rust_parser() + tree = parser.parse(content) + refs: list[AstNodeRef] = [] + + def visit(node: Any) -> None: + refs.append(_node_ref(node)) + for child in node.children: + visit(child) + + visit(tree.root_node) + return tuple(refs) + + +def _is_candidate(node: AstNodeRef) -> EvidenceType | None: + mapping = { + "function_item": EvidenceType.MODIFIED_FUNCTION, + "struct_item": EvidenceType.MODIFIED_TYPE, + "enum_item": EvidenceType.MODIFIED_TYPE, + "trait_item": EvidenceType.MODIFIED_TYPE, + "impl_item": EvidenceType.MODIFIED_TYPE, + "call_expression": EvidenceType.CALL, + "if_expression": EvidenceType.CONDITION, + "match_expression": EvidenceType.CONDITION, + "return_expression": EvidenceType.RETURN, + } + return mapping.get(node.node_type) + + +def deterministic_evidence_id(index: int) -> str: + if index < 0: + raise Stage1Error("invalid_evidence_index") + return f"E-{index + 1:06d}" + + +def build_evidence_blocks( + *, + manifest: Manifest, + reader: GitObjectReader, + max_blocks: int = DEFAULT_MAX_BLOCKS, + max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, + max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, +) -> tuple[tuple[EvidenceBlock, ...], bool]: + if min(max_blocks, max_block_bytes, max_referenced_nodes, max_total_bytes) <= 0: + raise Stage1Error("invalid_evidence_limits") + + candidates: list[tuple[str, str, str, bytes, AstNodeRef, EvidenceType]] = [] + for record in manifest.files: + if record.disposition not in {Disposition.PRIMARY, Disposition.DELETED}: + continue + if not record.blob_object or not record.content_sha256: + raise Stage1Error("vcs_object_missing") + content = reader.read_blob(record.blob_object) + if sha256_hex(content) != record.content_sha256: + raise Stage1Error("vcs_object_mismatch") + refs = parse_ast(content) + for ref in refs: + evidence_type = _is_candidate(ref) + if evidence_type is not None: + candidates.append((record.path, record.change_type, record.blob_object, content, ref, evidence_type)) + + candidates.sort(key=lambda item: (item[0], item[4].start_byte, item[4].end_byte, item[4].node_type, item[5].value)) + incomplete = len(candidates) > max_blocks + blocks: list[EvidenceBlock] = [] + total_bytes = 0 + for candidate in candidates[:max_blocks]: + path, change_type, object_id, content, ref, evidence_type = candidate + block_bytes = content[ref.start_byte:ref.end_byte] + if len(block_bytes) > max_block_bytes or total_bytes + len(block_bytes) > max_total_bytes: + incomplete = True + break + if len(blocks) >= max_referenced_nodes: + incomplete = True + break + text = block_bytes.decode("utf-8", errors="strict") + if change_type == "deleted" or evidence_type in {EvidenceType.MODIFIED_FUNCTION, EvidenceType.MODIFIED_TYPE} and change_type == "renamed_from": + evidence_type = EvidenceType.DELETED_NODE if change_type == "deleted" else evidence_type + blocks.append(EvidenceBlock( + id=deterministic_evidence_id(len(blocks)), + evidence_type=evidence_type, + path=path, + commit=manifest.base_commit if change_type in {"deleted", "renamed_from"} else manifest.head_commit, + object_id=object_id, + content_sha256=sha256_hex(content), + start_byte=ref.start_byte, + end_byte=ref.end_byte, + start_line=ref.start_line, + end_line=ref.end_line, + node_type=ref.node_type, + content=text, + )) + total_bytes += len(block_bytes) + + return tuple(blocks), incomplete + + +def build_evidence_manifest( + raw_stage0_manifest: Mapping[str, Any], + repository_root: str, + *, + max_blocks: int = DEFAULT_MAX_BLOCKS, + max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, + max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, + max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, +) -> EvidenceManifest: + manifest = verify_stage0_manifest(raw_stage0_manifest) + reader = GitObjectReader(repository_root) + verify_manifest_objects(manifest, reader) + blocks, incomplete = build_evidence_blocks( + manifest=manifest, + reader=reader, + max_blocks=max_blocks, + max_block_bytes=max_block_bytes, + max_referenced_nodes=max_referenced_nodes, + max_total_bytes=max_total_bytes, + ) + payload = { + "schema_version": STAGE1_SCHEMA_VERSION, + "ruleset_version": STAGE1_RULESET_VERSION, + "stage0_sha256": manifest.manifest_sha256, + "repository": manifest.repository, + "base_commit": manifest.base_commit, + "head_commit": manifest.head_commit, + "parser": rust_parser()[1], + "evidence_blocks": [block.canonical() for block in blocks], + "incomplete": incomplete, + } + digest = sha256_hex(canonical_json(payload)) + return EvidenceManifest( + schema_version=STAGE1_SCHEMA_VERSION, + ruleset_version=STAGE1_RULESET_VERSION, + stage0_sha256=manifest.manifest_sha256, + repository=manifest.repository, + base_commit=manifest.base_commit, + head_commit=manifest.head_commit, + parser=rust_parser()[1], + evidence_blocks=blocks, + incomplete=incomplete, + stage1_sha256=digest, + ) From a40116b7bbdda8c2f900f88de239a89cf4b198fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:52:48 +0200 Subject: [PATCH 06/19] Stage 1: add deterministic evidence boundary regression suite --- tests/test_stage1_evidence.py | 147 ++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/test_stage1_evidence.py diff --git a/tests/test_stage1_evidence.py b/tests/test_stage1_evidence.py new file mode 100644 index 0000000..45ba015 --- /dev/null +++ b/tests/test_stage1_evidence.py @@ -0,0 +1,147 @@ +import hashlib +import json +import subprocess +from pathlib import Path + +import pytest + +from src.stage0_manifest import ChangeRecord, Disposition, Reason, freeze_manifest, git_blob_sha, sha256_hex +from src.stage1_evidence import ( + EvidenceType, + GitObjectReader, + Stage1Error, + build_evidence_manifest, + deterministic_evidence_id, + parse_ast, + verify_stage0_manifest, +) + + +def make_manifest(content: bytes, *, disposition=Disposition.PRIMARY, change_type="modified"): + record = ChangeRecord( + path="src/main.rs", + change_type=change_type, + disposition=disposition, + reason=Reason.CODE_CHANGE, + size_bytes=len(content), + is_binary=False, + blob_object=git_blob_sha(content), + content_sha256=sha256_hex(content), + ) + return freeze_manifest( + repository="test/repo", + base_commit="a" * 40, + head_commit="b" * 40, + pr_number=1, + files=[record], + ) + + +def manifest_json(manifest): + payload = manifest.payload() + payload["manifest_sha256"] = manifest.manifest_sha256 + return payload + + +def test_stage0_manifest_verification_rejects_tampering(): + manifest = make_manifest(b"fn main() {}\n") + raw = manifest_json(manifest) + raw["head_commit"] = "c" * 40 + with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): + verify_stage0_manifest(raw) + + +def test_stage0_manifest_verification_rejects_unknown_disposition(): + manifest = make_manifest(b"fn main() {}\n") + raw = manifest_json(manifest) + raw["files"][0]["disposition"] = "VULNERABLE" + with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): + verify_stage0_manifest(raw) + + +def test_evidence_id_is_deterministic(): + assert [deterministic_evidence_id(i) for i in range(3)] == ["E-000001", "E-000002", "E-000003"] + + +def test_rust_ast_ranges_are_valid(): + content = b"fn main() { let x = 1; if x > 0 { return; } }\n" + refs = parse_ast(content) + assert refs + for ref in refs: + assert 0 <= ref.start_byte <= ref.end_byte <= len(content) + assert ref.start_line <= ref.end_line + + +def test_secondary_dependency_and_non_code_are_not_stage1_evidence(): + content = b"#!/usr/bin/env python3\nprint('x')\n" + for disposition in (Disposition.SECONDARY, Disposition.DEPENDENCY, Disposition.NON_CODE): + manifest = make_manifest(content, disposition=disposition) + assert all( + disposition not in {Disposition.SECONDARY, Disposition.DEPENDENCY, Disposition.NON_CODE} + for _ in [] + ) + + +def test_stage1_hash_is_reproducible_for_identical_git_objects(tmp_path: Path): + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Stage1 Test"], cwd=repo, check=True) + source = repo / "src" + source.mkdir() + path = source / "main.rs" + content = b"fn main() { let x = 1; }\n" + path.write_bytes(content) + subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() + record = ChangeRecord("src/main.rs", "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, len(content), False, blob, sha256_hex(content)) + manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) + raw = manifest_json(manifest) + first = build_evidence_manifest(raw, str(repo)) + second = build_evidence_manifest(raw, str(repo)) + assert first.canonical_json() == second.canonical_json() + assert first.stage1_sha256 == second.stage1_sha256 + assert first.evidence_blocks + + +def test_working_tree_mutation_does_not_change_head_bound_evidence(tmp_path: Path): + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Stage1 Test"], cwd=repo, check=True) + source = repo / "src" + source.mkdir() + path = source / "main.rs" + original = b"fn main() { let x = 1; }\n" + path.write_bytes(original) + subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() + record = ChangeRecord("src/main.rs", "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, len(original), False, blob, sha256_hex(original)) + manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) + raw = manifest_json(manifest) + first = build_evidence_manifest(raw, str(repo)) + path.write_bytes(b"fn main() { panic!(\"working tree mutation\"); }\n") + second = build_evidence_manifest(raw, str(repo)) + assert first.stage1_sha256 == second.stage1_sha256 + assert first.evidence_blocks[0].content == "fn main() { let x = 1; }\n" + + +def test_stage1_content_is_not_authoritative_identity(): + content = b"fn main() {}\n" + digest = hashlib.sha256(content).hexdigest() + assert digest == sha256_hex(content) + assert git_blob_sha(content) != digest + + +def test_evidence_types_do_not_make_security_claims(): + assert EvidenceType.AUTHORIZATION_CHECK.value == "AUTHORIZATION_CHECK" + assert EvidenceType.CALL.value == "CALL" + + +def test_parser_malformed_source_is_explicitly_structural(): + refs = parse_ast(b"fn broken( {\n") + assert refs From 52dcb3f40dab9ad4e1de362f63241cbf40566778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:52:56 +0200 Subject: [PATCH 07/19] Stage 1: add deterministic evidence CLI --- src/stage1_cli.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/stage1_cli.py diff --git a/src/stage1_cli.py b/src/stage1_cli.py new file mode 100644 index 0000000..259000b --- /dev/null +++ b/src/stage1_cli.py @@ -0,0 +1,26 @@ +"""Stage 1 CLI: verify a Stage 0 manifest and emit deterministic evidence.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .stage1_evidence import build_evidence_manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build deterministic Stage 1 evidence from a Stage 0 manifest") + parser.add_argument("manifest", type=Path) + parser.add_argument("repository", type=Path) + args = parser.parse_args() + raw = json.loads(args.manifest.read_text(encoding="utf-8")) + evidence = build_evidence_manifest(raw, str(args.repository)) + output = evidence.payload() + output["stage1_sha256"] = evidence.stage1_sha256 + print(json.dumps(output, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 5bf3b2cd977f6614fe53e5aa437246c5a2a30873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:53:01 +0200 Subject: [PATCH 08/19] Stage 1: add deterministic evidence CI gate --- .github/workflows/stage1-evidence.yml | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/stage1-evidence.yml diff --git a/.github/workflows/stage1-evidence.yml b/.github/workflows/stage1-evidence.yml new file mode 100644 index 0000000..009e7b2 --- /dev/null +++ b/.github/workflows/stage1-evidence.yml @@ -0,0 +1,33 @@ +name: Stage 1 Evidence Boundary + +on: + pull_request: + paths: + - 'src/stage0_manifest.py' + - 'src/stage1_evidence.py' + - 'src/stage1_cli.py' + - 'tests/test_stage0_manifest.py' + - 'tests/test_stage1_evidence.py' + - 'requirements-dev.txt' + - '.github/workflows/stage1-evidence.yml' + push: + branches: + - stage1-evidence-boundary + - main + +permissions: + contents: read + +jobs: + stage1: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install -r requirements-dev.txt + - name: Run Stage 0 and Stage 1 regression suites + run: python -m pytest -q tests/test_stage0_manifest.py tests/test_stage1_evidence.py From 23e0359ec03afcb9ab2cb3b36b28f4e78a32e490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:53:35 +0200 Subject: [PATCH 09/19] Stage 1: tighten deterministic evidence regression suite --- tests/test_stage1_evidence.py | 143 ++++++++++++---------------------- 1 file changed, 51 insertions(+), 92 deletions(-) diff --git a/tests/test_stage1_evidence.py b/tests/test_stage1_evidence.py index 45ba015..01cb413 100644 --- a/tests/test_stage1_evidence.py +++ b/tests/test_stage1_evidence.py @@ -1,40 +1,30 @@ -import hashlib -import json -import subprocess from pathlib import Path import pytest from src.stage0_manifest import ChangeRecord, Disposition, Reason, freeze_manifest, git_blob_sha, sha256_hex -from src.stage1_evidence import ( - EvidenceType, - GitObjectReader, - Stage1Error, - build_evidence_manifest, - deterministic_evidence_id, - parse_ast, - verify_stage0_manifest, -) - - -def make_manifest(content: bytes, *, disposition=Disposition.PRIMARY, change_type="modified"): - record = ChangeRecord( - path="src/main.rs", - change_type=change_type, - disposition=disposition, - reason=Reason.CODE_CHANGE, - size_bytes=len(content), - is_binary=False, - blob_object=git_blob_sha(content), - content_sha256=sha256_hex(content), - ) - return freeze_manifest( - repository="test/repo", - base_commit="a" * 40, - head_commit="b" * 40, - pr_number=1, - files=[record], - ) +from src.stage1_evidence import EvidenceType, Stage1Error, build_evidence_manifest, deterministic_evidence_id, parse_ast, verify_stage0_manifest + + +def git_repo(tmp_path: Path, content: bytes): + import subprocess + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Stage1 Test"], cwd=repo, check=True) + (repo / "src").mkdir() + (repo / "src/main.rs").write_bytes(content) + subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-qm", "fixture"], cwd=repo, check=True) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() + return repo, commit, blob + + +def make_manifest(commit: str, blob: str, content: bytes, *, disposition=Disposition.PRIMARY, change_type="modified"): + record = ChangeRecord("src/main.rs", change_type, disposition, Reason.CODE_CHANGE, len(content), False, blob, sha256_hex(content)) + return freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) def manifest_json(manifest): @@ -43,16 +33,20 @@ def manifest_json(manifest): return payload -def test_stage0_manifest_verification_rejects_tampering(): - manifest = make_manifest(b"fn main() {}\n") +def test_stage0_manifest_verification_rejects_tampering(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + manifest = make_manifest(commit, blob, content) raw = manifest_json(manifest) raw["head_commit"] = "c" * 40 with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): verify_stage0_manifest(raw) -def test_stage0_manifest_verification_rejects_unknown_disposition(): - manifest = make_manifest(b"fn main() {}\n") +def test_stage0_manifest_verification_rejects_unknown_disposition(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + manifest = make_manifest(commit, blob, content) raw = manifest_json(manifest) raw["files"][0]["disposition"] = "VULNERABLE" with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): @@ -67,81 +61,46 @@ def test_rust_ast_ranges_are_valid(): content = b"fn main() { let x = 1; if x > 0 { return; } }\n" refs = parse_ast(content) assert refs - for ref in refs: - assert 0 <= ref.start_byte <= ref.end_byte <= len(content) - assert ref.start_line <= ref.end_line + assert all(0 <= ref.start_byte <= ref.end_byte <= len(content) for ref in refs) -def test_secondary_dependency_and_non_code_are_not_stage1_evidence(): - content = b"#!/usr/bin/env python3\nprint('x')\n" - for disposition in (Disposition.SECONDARY, Disposition.DEPENDENCY, Disposition.NON_CODE): - manifest = make_manifest(content, disposition=disposition) - assert all( - disposition not in {Disposition.SECONDARY, Disposition.DEPENDENCY, Disposition.NON_CODE} - for _ in [] - ) +def test_non_primary_dispositions_are_not_emitted(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + for disposition, reason in [(Disposition.SECONDARY, Reason.MISLEADING_EXTENSION), (Disposition.DEPENDENCY, Reason.DEPENDENCY), (Disposition.NON_CODE, Reason.NON_CODE)]: + record = ChangeRecord("src/main.rs", "modified", disposition, reason, len(content), False, blob, sha256_hex(content)) + manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) + evidence = build_evidence_manifest(manifest_json(manifest), str(repo)) + assert evidence.evidence_blocks == () def test_stage1_hash_is_reproducible_for_identical_git_objects(tmp_path: Path): - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.name", "Stage1 Test"], cwd=repo, check=True) - source = repo / "src" - source.mkdir() - path = source / "main.rs" content = b"fn main() { let x = 1; }\n" - path.write_bytes(content) - subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() - blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() - record = ChangeRecord("src/main.rs", "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, len(content), False, blob, sha256_hex(content)) - manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) - raw = manifest_json(manifest) - first = build_evidence_manifest(raw, str(repo)) - second = build_evidence_manifest(raw, str(repo)) + repo, commit, blob = git_repo(tmp_path, content) + manifest = make_manifest(commit, blob, content) + first = build_evidence_manifest(manifest_json(manifest), str(repo)) + second = build_evidence_manifest(manifest_json(manifest), str(repo)) assert first.canonical_json() == second.canonical_json() assert first.stage1_sha256 == second.stage1_sha256 assert first.evidence_blocks def test_working_tree_mutation_does_not_change_head_bound_evidence(tmp_path: Path): - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=repo, check=True) - subprocess.run(["git", "config", "user.name", "Stage1 Test"], cwd=repo, check=True) - source = repo / "src" - source.mkdir() - path = source / "main.rs" original = b"fn main() { let x = 1; }\n" - path.write_bytes(original) - subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) - commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() - blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() - record = ChangeRecord("src/main.rs", "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, len(original), False, blob, sha256_hex(original)) - manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) - raw = manifest_json(manifest) - first = build_evidence_manifest(raw, str(repo)) - path.write_bytes(b"fn main() { panic!(\"working tree mutation\"); }\n") - second = build_evidence_manifest(raw, str(repo)) + repo, commit, blob = git_repo(tmp_path, original) + manifest = make_manifest(commit, blob, original) + first = build_evidence_manifest(manifest_json(manifest), str(repo)) + (repo / "src/main.rs").write_bytes(b"fn main() { panic!(\"working tree mutation\"); }\n") + second = build_evidence_manifest(manifest_json(manifest), str(repo)) assert first.stage1_sha256 == second.stage1_sha256 - assert first.evidence_blocks[0].content == "fn main() { let x = 1; }\n" - - -def test_stage1_content_is_not_authoritative_identity(): - content = b"fn main() {}\n" - digest = hashlib.sha256(content).hexdigest() - assert digest == sha256_hex(content) - assert git_blob_sha(content) != digest + assert first.evidence_blocks[0].content == original.decode() -def test_evidence_types_do_not_make_security_claims(): +def test_stage1_does_not_promote_security_semantics(): assert EvidenceType.AUTHORIZATION_CHECK.value == "AUTHORIZATION_CHECK" assert EvidenceType.CALL.value == "CALL" -def test_parser_malformed_source_is_explicitly_structural(): +def test_malformed_rust_is_structural_not_security_semantic(): refs = parse_ast(b"fn broken( {\n") assert refs From 9090dcfd0bb3d6f858d540c9fa02dcd05247f60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:19 +0200 Subject: [PATCH 10/19] Stage 1: document evidence construction contract --- docs/STAGE1_EVIDENCE_CONTRACT.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/STAGE1_EVIDENCE_CONTRACT.md diff --git a/docs/STAGE1_EVIDENCE_CONTRACT.md b/docs/STAGE1_EVIDENCE_CONTRACT.md new file mode 100644 index 0000000..d837142 --- /dev/null +++ b/docs/STAGE1_EVIDENCE_CONTRACT.md @@ -0,0 +1,29 @@ +# Stage 1 Evidence Construction Contract + +Stage 1 is a deterministic evidence-construction boundary. It extracts and describes source evidence but does not determine whether that evidence constitutes a vulnerability. + +## Boundary + +Stage 0 manifest -> verify canonical payload and `manifest_sha256` -> verify commit/object bindings -> deterministic router -> AST extraction for `PRIMARY` and `DELETED` -> bounded evidence construction -> canonical Evidence Manifest -> `stage1_sha256`. + +`SECONDARY`, `DEPENDENCY`, and `NON_CODE` records do not enter AST evidence construction. + +## Authority + +The exact Git object referenced by the verified Stage 0 record is authoritative. The working tree is an execution environment only. Evidence content is materialized from the immutable Git object after binding verification. + +## Evidence identity + +Evidence IDs are deterministic and assigned after canonical ordering by path, byte range, node type, and evidence type. Each evidence block binds to commit, Git object ID, SHA-256 content hash, and an exact byte range. + +## Bounds + +Evidence construction has explicit limits for block count, block size, referenced nodes, and total payload bytes. Exceeding a limit produces an explicit incomplete state; content is never silently truncated and treated as complete evidence. + +## Security boundary + +Structural labels such as `CALL`, `CONDITION`, `RETURN`, and `AUTHORIZATION_CHECK` describe extracted evidence. They are not vulnerability conclusions. Stage 2 is the first layer permitted to reason about claims supported by the evidence. + +## Reproducibility + +Identical Stage 0 input, Git objects, parser metadata, and Stage 1 ruleset must produce byte-identical evidence payloads and `stage1_sha256`. Parser or ruleset changes are expected to change the committed artifact. From e23e2612b8e6ac8e85d4bea02fc91ecb0008fc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:22 +0200 Subject: [PATCH 11/19] Stage 1: mark source package explicitly --- src/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/__init__.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 From 6bf06c19a1a2dacfbcb419542fb46c040292f4b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:26 +0200 Subject: [PATCH 12/19] Stage 1: add import smoke test --- tests/test_stage1_smoke.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/test_stage1_smoke.py diff --git a/tests/test_stage1_smoke.py b/tests/test_stage1_smoke.py new file mode 100644 index 0000000..45505f9 --- /dev/null +++ b/tests/test_stage1_smoke.py @@ -0,0 +1,5 @@ +from src.stage1_evidence import deterministic_evidence_id + + +def test_stage1_import_and_id_smoke(): + assert deterministic_evidence_id(0) == "E-000001" From 6dc73320b74da3a0d962d5243e9c8a494ff6702e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:30 +0200 Subject: [PATCH 13/19] Stage 1: record implementation gate status --- docs/STAGE1_STATUS.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/STAGE1_STATUS.md diff --git a/docs/STAGE1_STATUS.md b/docs/STAGE1_STATUS.md new file mode 100644 index 0000000..c3f41f4 --- /dev/null +++ b/docs/STAGE1_STATUS.md @@ -0,0 +1,7 @@ +# Stage 1 Status + +Stage 1 remains gated on the hardened Stage 0 contract. This branch contains the deterministic evidence-construction implementation and regression suite, but Stage 1 is not accepted until CI verifies the complete boundary. + +Acceptance state: PENDING CI + +Stage 2 LLM integration: BLOCKED From 7196d8198622baa083dea2623cf74ed0b5a35a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:34 +0200 Subject: [PATCH 14/19] Stage 1: record execution gate --- docs/STAGE1_NEXT.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/STAGE1_NEXT.md diff --git a/docs/STAGE1_NEXT.md b/docs/STAGE1_NEXT.md new file mode 100644 index 0000000..0e29bc4 --- /dev/null +++ b/docs/STAGE1_NEXT.md @@ -0,0 +1 @@ +Stage 1 implementation is prepared on the Stage 0 hardening line. The acceptance gate remains CI-bound; no Stage 2 integration is authorized until the deterministic evidence boundary passes. \ No newline at end of file From ceef8e083c14236b02816b3dec7de01094daae7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:37 +0200 Subject: [PATCH 15/19] Stage 1: freeze execution gate --- docs/STAGE1_GATE.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/STAGE1_GATE.md diff --git a/docs/STAGE1_GATE.md b/docs/STAGE1_GATE.md new file mode 100644 index 0000000..00600a7 --- /dev/null +++ b/docs/STAGE1_GATE.md @@ -0,0 +1 @@ +Stage 1 gate: CI must pass before Stage 2. No LLM integration is part of this boundary. \ No newline at end of file From 3a91a937f77f9d49ceef65030d7f0b56c495a8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:41 +0200 Subject: [PATCH 16/19] Stage 1: mark implementation line --- docs/STAGE1_READY.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/STAGE1_READY.md diff --git a/docs/STAGE1_READY.md b/docs/STAGE1_READY.md new file mode 100644 index 0000000..afaf396 --- /dev/null +++ b/docs/STAGE1_READY.md @@ -0,0 +1 @@ +Stage 1 deterministic evidence boundary implementation line. \ No newline at end of file From 8336719db821667e865b6bd874ae749f4e1e9332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:54:45 +0200 Subject: [PATCH 17/19] Stage 1: implementation status --- docs/STAGE1_IMPLEMENTATION.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/STAGE1_IMPLEMENTATION.md diff --git a/docs/STAGE1_IMPLEMENTATION.md b/docs/STAGE1_IMPLEMENTATION.md new file mode 100644 index 0000000..135278a --- /dev/null +++ b/docs/STAGE1_IMPLEMENTATION.md @@ -0,0 +1 @@ +Implementation status is CI-gated. \ No newline at end of file From 38b9c0a377c64bc0f5c06661b2fdceefd091ae0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:57:05 +0200 Subject: [PATCH 18/19] Harden Stage 0 rename identity and binary classification --- src/stage0_manifest.py | 78 +++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/src/stage0_manifest.py b/src/stage0_manifest.py index b511e68..4e849a7 100644 --- a/src/stage0_manifest.py +++ b/src/stage0_manifest.py @@ -1,17 +1,9 @@ -"""Stage 0 canonical VCS change manifest primitives. - -The implementation deliberately binds records to Git object identity rather -than treating the mutable working tree as authoritative. It is intentionally -small and dependency-free so the acceptance tests can exercise the invariants -without a repository checkout. -""" +"""Stage 0 canonical VCS change manifest primitives.""" from __future__ import annotations import hashlib import json -import os -import stat from dataclasses import dataclass from enum import Enum from pathlib import Path, PurePosixPath @@ -57,6 +49,7 @@ class ChangeRecord: is_binary: bool blob_object: str | None content_sha256: str | None + rename_group_id: str | None = None def canonical(self) -> dict[str, object]: return { @@ -68,6 +61,7 @@ def canonical(self) -> dict[str, object]: "is_binary": self.is_binary, "blob_object": self.blob_object, "content_sha256": self.content_sha256, + "rename_group_id": self.rename_group_id, } @@ -98,29 +92,25 @@ def canonical_json(self) -> bytes: def canonical_json(payload: Mapping[str, object]) -> bytes: - """Canonical UTF-8 JSON: stable keys, arrays, separators and Unicode.""" - return json.dumps( - payload, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") def sha256_hex(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def freeze_manifest( - *, - repository: str, - base_commit: str, - head_commit: str, - pr_number: int, - files: Iterable[ChangeRecord], -) -> Manifest: - """Construct immutable payload first, then commit to its exact bytes.""" - ordered = tuple(sorted(files, key=lambda record: record.path)) +def rename_group_id(base_commit: str, head_commit: str, source_path: str, destination_path: str) -> str: + payload = { + "base_commit": base_commit, + "head_commit": head_commit, + "source_path": source_path, + "destination_path": destination_path, + } + return sha256_hex(canonical_json(payload)) + + +def freeze_manifest(*, repository: str, base_commit: str, head_commit: str, pr_number: int, files: Iterable[ChangeRecord]) -> Manifest: + ordered = tuple(sorted(files, key=lambda record: (record.path, record.change_type, record.rename_group_id or ""))) payload = { "schema_version": SCHEMA_VERSION, "ruleset_version": RULESET_VERSION, @@ -131,20 +121,10 @@ def freeze_manifest( "files": [record.canonical() for record in ordered], } digest = sha256_hex(canonical_json(payload)) - return Manifest( - schema_version=SCHEMA_VERSION, - ruleset_version=RULESET_VERSION, - repository=repository, - base_commit=base_commit, - head_commit=head_commit, - pr_number=pr_number, - files=ordered, - manifest_sha256=digest, - ) + return Manifest(SCHEMA_VERSION, RULESET_VERSION, repository, base_commit, head_commit, pr_number, ordered, digest) def safe_join(resolved_root: Path, candidate: str) -> Path: - """Resolve candidate and require component-aware containment.""" root = resolved_root.resolve(strict=True) candidate_path = (root / candidate).resolve(strict=False) try: @@ -163,41 +143,37 @@ def sha256_file(path: Path) -> str: def is_binary_bytes(content: bytes) -> bool: + try: + content.decode("utf-8") + except UnicodeDecodeError: + return True return b"\x00" in content def classify_path(path: str, content: bytes, *, generated: bool = False, dependency: bool = False, oversized: bool = False) -> tuple[Disposition, Reason]: - """Deterministic routing; content characteristics outrank filename suffix.""" if oversized: return Disposition.OVERSIZED, Reason.OVERSIZED_ARTIFACT if generated: return Disposition.GENERATED, Reason.GENERATED_ARTIFACT if dependency: return Disposition.DEPENDENCY, Reason.DEPENDENCY - + if is_binary_bytes(content): + return Disposition.NON_CODE, Reason.BINARY suffix = PurePosixPath(path).suffix.lower() code_suffixes = {".py", ".rs", ".ts", ".tsx", ".js", ".jsx", ".go", ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".sh"} - text = content.decode("utf-8", errors="ignore") + text = content.decode("utf-8") executable_markers = ("#!/", "import ", "from ", "fn ", "def ", "class ", "function ", "const ", "let ", "use ") - looks_executable = any(marker in text for marker in executable_markers) - if suffix in code_suffixes: return Disposition.PRIMARY, Reason.CODE_CHANGE - if looks_executable: + if any(marker in text for marker in executable_markers): return Disposition.SECONDARY, Reason.MISLEADING_EXTENSION - if is_binary_bytes(content): - return Disposition.NON_CODE, Reason.BINARY return Disposition.NON_CODE, Reason.NON_CODE def git_blob_sha(content: bytes) -> str: - """Compute Git's canonical blob object identity for content.""" - header = f"blob {len(content)}\0".encode("ascii") - return hashlib.sha1(header + content).hexdigest() + return hashlib.sha1(f"blob {len(content)}\0".encode("ascii") + content).hexdigest() def verify_head_binding(content: bytes, expected_blob_object: str, expected_content_sha256: str) -> None: - if git_blob_sha(content) != expected_blob_object: - raise ValueError("vcs_object_mismatch") - if sha256_hex(content) != expected_content_sha256: + if git_blob_sha(content) != expected_blob_object or sha256_hex(content) != expected_content_sha256: raise ValueError("vcs_object_mismatch") From b379ef59f8122826256359b0e98259d13166c7ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?cryptofixyup=F0=9F=94=A5?= <199330534+cryptofixyup@users.noreply.github.com> Date: Sun, 6 Sep 2026 10:14:59 +0200 Subject: [PATCH 19/19] Harden Stage 1 Git-object provenance boundary --- src/stage1_evidence.py | 276 +++++++++++++++++++++------------- tests/test_stage1_evidence.py | 138 +++++++++++++---- 2 files changed, 281 insertions(+), 133 deletions(-) diff --git a/src/stage1_evidence.py b/src/stage1_evidence.py index df99452..3181c6a 100644 --- a/src/stage1_evidence.py +++ b/src/stage1_evidence.py @@ -1,24 +1,25 @@ -"""Stage 1 deterministic evidence-construction boundary. +"""Deterministic Stage 1 evidence-construction boundary. -Stage 1 verifies a Stage 0 manifest, reads immutable Git objects, parses -PRIMARY/DELETED source material structurally, and emits a deterministic -Evidence Manifest. It does not make vulnerability determinations. +Stage 1 verifies the Stage 0 commitment, reads source bytes from the Git +object database, parses Rust structurally, and emits a committed evidence +payload. It never trusts the working tree for source provenance and never +makes security conclusions. """ from __future__ import annotations import hashlib import hmac -import json import subprocess from dataclasses import dataclass from enum import Enum -from typing import Any, Mapping, Sequence +from importlib.metadata import PackageNotFoundError, version +from typing import Any, Mapping from tree_sitter import Language, Parser import tree_sitter_rust -from .stage0_manifest import Disposition, Manifest, Reason, canonical_json, sha256_hex +from .stage0_manifest import ChangeRecord, Disposition, Manifest, Reason, canonical_json, sha256_hex STAGE1_SCHEMA_VERSION = "1.0.0" STAGE1_RULESET_VERSION = "stage1-v1" @@ -29,22 +30,16 @@ class Stage1Error(ValueError): - pass + """Fail-closed Stage 1 boundary error.""" class EvidenceType(str, Enum): MODIFIED_FUNCTION = "MODIFIED_FUNCTION" MODIFIED_METHOD = "MODIFIED_METHOD" MODIFIED_TYPE = "MODIFIED_TYPE" - SECURITY_SENSITIVE_DECLARATION = "SECURITY_SENSITIVE_DECLARATION" CALL = "CALL" CONDITION = "CONDITION" RETURN = "RETURN" - AUTHORIZATION_CHECK = "AUTHORIZATION_CHECK" - INPUT_BOUNDARY = "INPUT_BOUNDARY" - DATABASE_OPERATION = "DATABASE_OPERATION" - NETWORK_OPERATION = "NETWORK_OPERATION" - CRYPTO_OPERATION = "CRYPTO_OPERATION" DELETED_NODE = "DELETED_NODE" @@ -54,7 +49,9 @@ class AstNodeRef: start_byte: int end_byte: int start_line: int + start_column: int end_line: int + end_column: int @dataclass(frozen=True) @@ -64,13 +61,16 @@ class EvidenceBlock: path: str commit: str object_id: str + source_sha256: str content_sha256: str start_byte: int end_byte: int start_line: int + start_column: int end_line: int + end_column: int node_type: str - content: str | None = None + content: str def canonical(self) -> dict[str, object]: return { @@ -79,11 +79,14 @@ def canonical(self) -> dict[str, object]: "path": self.path, "commit": self.commit, "object_id": self.object_id, + "source_sha256": self.source_sha256, "content_sha256": self.content_sha256, "start_byte": self.start_byte, "end_byte": self.end_byte, "start_line": self.start_line, + "start_column": self.start_column, "end_line": self.end_line, + "end_column": self.end_column, "node_type": self.node_type, "content": self.content, } @@ -97,7 +100,7 @@ class EvidenceManifest: repository: str base_commit: str head_commit: str - parser: Mapping[str, str] + parser: tuple[tuple[str, str], ...] evidence_blocks: tuple[EvidenceBlock, ...] incomplete: bool stage1_sha256: str @@ -127,31 +130,31 @@ def _constant_time_equal_hex(left: str, right: str) -> bool: def verify_stage0_manifest(raw_manifest: Mapping[str, Any]) -> Manifest: - """Verify the Stage 0 commitment before any evidence extraction.""" + """Verify the exact Stage 0 canonical commitment before extraction.""" required = { "schema_version", "ruleset_version", "repository", "base_commit", "head_commit", "pr_number", "files", "manifest_sha256", } - if set(raw_manifest) != required: + if set(raw_manifest) != required or not isinstance(raw_manifest.get("manifest_sha256"), str): + raise Stage1Error("stage0_schema_mismatch") + if not isinstance(raw_manifest.get("files"), list): raise Stage1Error("stage0_schema_mismatch") - if not isinstance(raw_manifest["manifest_sha256"], str): - raise Stage1Error("stage0_hash_malformed") - payload = {key: raw_manifest[key] for key in required if key != "manifest_sha256"} + payload = {key: raw_manifest[key] for key in raw_manifest if key != "manifest_sha256"} calculated = sha256_hex(canonical_json(payload)) if not _constant_time_equal_hex(calculated, raw_manifest["manifest_sha256"]): raise Stage1Error("stage0_integrity_mismatch") + expected_fields = { + "path", "change_type", "disposition", "reason", "size_bytes", + "is_binary", "blob_object", "content_sha256", "rename_group_id", + } + records: list[ChangeRecord] = [] try: - records = [] for item in raw_manifest["files"]: - if set(item) != { - "path", "change_type", "disposition", "reason", "size_bytes", - "is_binary", "blob_object", "content_sha256", - }: + if not isinstance(item, Mapping) or set(item) != expected_fields: raise Stage1Error("stage0_schema_mismatch") - records.append(item) - parsed = tuple(_change_record_from_mapping(item) for item in records) + records.append(_change_record_from_mapping(item)) manifest = Manifest( schema_version=str(raw_manifest["schema_version"]), ruleset_version=str(raw_manifest["ruleset_version"]), @@ -159,10 +162,12 @@ def verify_stage0_manifest(raw_manifest: Mapping[str, Any]) -> Manifest: base_commit=str(raw_manifest["base_commit"]), head_commit=str(raw_manifest["head_commit"]), pr_number=int(raw_manifest["pr_number"]), - files=parsed, - manifest_sha256=str(raw_manifest["manifest_sha256"]), + files=tuple(records), + manifest_sha256=raw_manifest["manifest_sha256"], ) except (TypeError, ValueError) as exc: + if isinstance(exc, Stage1Error): + raise raise Stage1Error("stage0_schema_mismatch") from exc if manifest.schema_version != "1" or manifest.ruleset_version != "stage0-v1": @@ -170,22 +175,25 @@ def verify_stage0_manifest(raw_manifest: Mapping[str, Any]) -> Manifest: return manifest -def _change_record_from_mapping(item: Mapping[str, Any]): - from .stage0_manifest import ChangeRecord - return ChangeRecord( - path=str(item["path"]), - change_type=str(item["change_type"]), - disposition=Disposition(str(item["disposition"])), - reason=Reason(str(item["reason"])), - size_bytes=int(item["size_bytes"]), - is_binary=bool(item["is_binary"]), - blob_object=item["blob_object"], - content_sha256=item["content_sha256"], - ) +def _change_record_from_mapping(item: Mapping[str, Any]) -> ChangeRecord: + try: + return ChangeRecord( + path=str(item["path"]), + change_type=str(item["change_type"]), + disposition=Disposition(str(item["disposition"])), + reason=Reason(str(item["reason"])), + size_bytes=int(item["size_bytes"]), + is_binary=bool(item["is_binary"]), + blob_object=item["blob_object"], + content_sha256=item["content_sha256"], + rename_group_id=item["rename_group_id"], + ) + except (KeyError, TypeError, ValueError) as exc: + raise Stage1Error("stage0_schema_mismatch") from exc class GitObjectReader: - """Read immutable objects from the repository's Git object database.""" + """Read and verify immutable Git objects; never reads source from the worktree.""" def __init__(self, repository_root: str): self.repository_root = repository_root @@ -193,9 +201,27 @@ def __init__(self, repository_root: str): def verify_commit_available(self, commit: str) -> None: self._run("git", "cat-file", "-e", f"{commit}^{{commit}}") + def resolve_path_blob(self, commit: str, path: str) -> str: + try: + output = self._run("git", "rev-parse", f"{commit}:{path}") + object_id = output.decode("ascii").strip() + except UnicodeDecodeError as exc: + raise Stage1Error("vcs_object_missing") from exc + if len(object_id) != 40 or any(ch not in "0123456789abcdef" for ch in object_id): + raise Stage1Error("vcs_object_missing") + return object_id + def read_blob(self, object_id: str) -> bytes: - output = self._run("git", "cat-file", "blob", object_id) - return output + return self._run("git", "cat-file", "blob", object_id) + + def read_bound_blob(self, *, commit: str, path: str, expected_object: str, expected_sha256: str) -> bytes: + resolved = self.resolve_path_blob(commit, path) + if resolved != expected_object: + raise Stage1Error("vcs_object_mismatch") + content = self.read_blob(expected_object) + if sha256_hex(content) != expected_sha256: + raise Stage1Error("vcs_object_mismatch") + return content def _run(self, *args: str) -> bytes: try: @@ -211,27 +237,48 @@ def _run(self, *args: str) -> bytes: return result.stdout +def _record_commit(record: ChangeRecord, manifest: Manifest) -> str: + if record.change_type in {"deleted", "renamed_from", "copied_from"} or record.disposition is Disposition.DELETED: + return manifest.base_commit + return manifest.head_commit + + def verify_manifest_objects(manifest: Manifest, reader: GitObjectReader) -> None: reader.verify_commit_available(manifest.base_commit) reader.verify_commit_available(manifest.head_commit) for record in manifest.files: - if record.disposition in {Disposition.PRIMARY, Disposition.DELETED, Disposition.SECONDARY}: - if not record.blob_object or not record.content_sha256: + if not record.blob_object or not record.content_sha256: + if record.disposition in {Disposition.PRIMARY, Disposition.DELETED}: raise Stage1Error("vcs_object_missing") - content = reader.read_blob(record.blob_object) - if sha256_hex(content) != record.content_sha256: - raise Stage1Error("vcs_object_mismatch") + continue + commit = _record_commit(record, manifest) + content = reader.read_bound_blob( + commit=commit, + path=record.path, + expected_object=record.blob_object, + expected_sha256=record.content_sha256, + ) + if len(content) != record.size_bytes: + raise Stage1Error("vcs_object_mismatch") -def rust_parser() -> tuple[Parser, dict[str, str]]: +def rust_parser() -> tuple[Parser, tuple[tuple[str, str], ...]]: language = Language(tree_sitter_rust.language()) parser = Parser(language) - metadata = { - "name": "tree-sitter", - "version": "0.25.x", - "grammar": "tree-sitter-rust", - "grammar_version": "0.24.x", - } + try: + binding_version = version("tree-sitter") + except PackageNotFoundError as exc: + raise Stage1Error("parser_metadata_unavailable") from exc + try: + grammar_version = version("tree-sitter-rust") + except PackageNotFoundError as exc: + raise Stage1Error("parser_metadata_unavailable") from exc + metadata = ( + ("name", "tree-sitter"), + ("binding_version", binding_version), + ("grammar", "tree-sitter-rust"), + ("grammar_version", grammar_version), + ) return parser, metadata @@ -241,13 +288,18 @@ def _node_ref(node: Any) -> AstNodeRef: start_byte=node.start_byte, end_byte=node.end_byte, start_line=node.start_point[0] + 1, + start_column=node.start_point[1], end_line=node.end_point[0] + 1, + end_column=node.end_point[1], ) def parse_ast(content: bytes) -> tuple[AstNodeRef, ...]: parser, _ = rust_parser() tree = parser.parse(content) + if tree.root_node.has_error: + raise Stage1Error("parse_error") + refs: list[AstNodeRef] = [] def visit(node: Any) -> None: @@ -260,7 +312,7 @@ def visit(node: Any) -> None: def _is_candidate(node: AstNodeRef) -> EvidenceType | None: - mapping = { + return { "function_item": EvidenceType.MODIFIED_FUNCTION, "struct_item": EvidenceType.MODIFIED_TYPE, "enum_item": EvidenceType.MODIFIED_TYPE, @@ -270,98 +322,112 @@ def _is_candidate(node: AstNodeRef) -> EvidenceType | None: "if_expression": EvidenceType.CONDITION, "match_expression": EvidenceType.CONDITION, "return_expression": EvidenceType.RETURN, - } - return mapping.get(node.node_type) + }.get(node.node_type) + + +def evidence_id(*, source_sha256: str, file: str, commit: str, object_id: str, node: AstNodeRef, evidence_type: EvidenceType) -> str: + hasher = hashlib.sha256() + fields = ( + b"stage1-evidence-v2\0", + source_sha256.encode("ascii"), b"\0", + file.encode("utf-8"), b"\0", + commit.encode("ascii"), b"\0", + object_id.encode("ascii"), b"\0", + evidence_type.value.encode("ascii"), b"\0", + node.node_type.encode("utf-8"), b"\0", + node.start_byte.to_bytes(8, "little"), + node.end_byte.to_bytes(8, "little"), + ) + for field in fields: + hasher.update(field) + return f"E-{hasher.hexdigest()}" def deterministic_evidence_id(index: int) -> str: + """Legacy test helper; not used for authoritative evidence identity.""" if index < 0: raise Stage1Error("invalid_evidence_index") return f"E-{index + 1:06d}" -def build_evidence_blocks( - *, - manifest: Manifest, - reader: GitObjectReader, - max_blocks: int = DEFAULT_MAX_BLOCKS, - max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, - max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, - max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, -) -> tuple[tuple[EvidenceBlock, ...], bool]: +def build_evidence_blocks(*, manifest: Manifest, reader: GitObjectReader, max_blocks: int = DEFAULT_MAX_BLOCKS, max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES) -> tuple[tuple[EvidenceBlock, ...], bool]: if min(max_blocks, max_block_bytes, max_referenced_nodes, max_total_bytes) <= 0: raise Stage1Error("invalid_evidence_limits") - candidates: list[tuple[str, str, str, bytes, AstNodeRef, EvidenceType]] = [] + candidates: list[tuple[str, str, str, str, bytes, AstNodeRef, EvidenceType]] = [] for record in manifest.files: if record.disposition not in {Disposition.PRIMARY, Disposition.DELETED}: continue if not record.blob_object or not record.content_sha256: raise Stage1Error("vcs_object_missing") - content = reader.read_blob(record.blob_object) - if sha256_hex(content) != record.content_sha256: - raise Stage1Error("vcs_object_mismatch") + commit = _record_commit(record, manifest) + content = reader.read_bound_blob( + commit=commit, + path=record.path, + expected_object=record.blob_object, + expected_sha256=record.content_sha256, + ) refs = parse_ast(content) for ref in refs: evidence_type = _is_candidate(ref) if evidence_type is not None: - candidates.append((record.path, record.change_type, record.blob_object, content, ref, evidence_type)) + candidates.append((record.path, record.change_type, commit, record.blob_object, content, ref, evidence_type)) - candidates.sort(key=lambda item: (item[0], item[4].start_byte, item[4].end_byte, item[4].node_type, item[5].value)) + candidates.sort(key=lambda item: (item[0], item[2], item[4][item[5].start_byte:item[5].end_byte], item[5].start_byte, item[5].end_byte, item[5].node_type, item[6].value, item[3])) incomplete = len(candidates) > max_blocks + candidates = candidates[:max_blocks] + blocks: list[EvidenceBlock] = [] total_bytes = 0 - for candidate in candidates[:max_blocks]: - path, change_type, object_id, content, ref, evidence_type = candidate - block_bytes = content[ref.start_byte:ref.end_byte] - if len(block_bytes) > max_block_bytes or total_bytes + len(block_bytes) > max_total_bytes: + for path, change_type, commit, object_id, content, ref, evidence_type in candidates: + if len(blocks) >= max_referenced_nodes: incomplete = True break - if len(blocks) >= max_referenced_nodes: + if ref.start_byte > ref.end_byte or ref.end_byte > len(content): + raise Stage1Error("invalid_ast_span") + block_bytes = content[ref.start_byte:ref.end_byte] + if len(block_bytes) > max_block_bytes or total_bytes + len(block_bytes) > max_total_bytes: incomplete = True break - text = block_bytes.decode("utf-8", errors="strict") - if change_type == "deleted" or evidence_type in {EvidenceType.MODIFIED_FUNCTION, EvidenceType.MODIFIED_TYPE} and change_type == "renamed_from": - evidence_type = EvidenceType.DELETED_NODE if change_type == "deleted" else evidence_type + try: + text = block_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise Stage1Error("source_not_utf8") from exc + source_hash = sha256_hex(content) + block_hash = sha256_hex(block_bytes) + if change_type in {"deleted", "renamed_from"}: + evidence_type = EvidenceType.DELETED_NODE blocks.append(EvidenceBlock( - id=deterministic_evidence_id(len(blocks)), + id=evidence_id(source_sha256=source_hash, file=path, commit=commit, object_id=object_id, node=ref, evidence_type=evidence_type), evidence_type=evidence_type, path=path, - commit=manifest.base_commit if change_type in {"deleted", "renamed_from"} else manifest.head_commit, + commit=commit, object_id=object_id, - content_sha256=sha256_hex(content), + source_sha256=source_hash, + content_sha256=block_hash, start_byte=ref.start_byte, end_byte=ref.end_byte, start_line=ref.start_line, + start_column=ref.start_column, end_line=ref.end_line, + end_column=ref.end_column, node_type=ref.node_type, content=text, )) total_bytes += len(block_bytes) + ids = [block.id for block in blocks] + if len(ids) != len(set(ids)): + raise Stage1Error("duplicate_evidence_id") return tuple(blocks), incomplete -def build_evidence_manifest( - raw_stage0_manifest: Mapping[str, Any], - repository_root: str, - *, - max_blocks: int = DEFAULT_MAX_BLOCKS, - max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, - max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, - max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES, -) -> EvidenceManifest: +def build_evidence_manifest(raw_stage0_manifest: Mapping[str, Any], repository_root: str, *, max_blocks: int = DEFAULT_MAX_BLOCKS, max_block_bytes: int = DEFAULT_MAX_BLOCK_BYTES, max_referenced_nodes: int = DEFAULT_MAX_REFERENCED_NODES, max_total_bytes: int = DEFAULT_MAX_TOTAL_BYTES) -> EvidenceManifest: manifest = verify_stage0_manifest(raw_stage0_manifest) reader = GitObjectReader(repository_root) verify_manifest_objects(manifest, reader) - blocks, incomplete = build_evidence_blocks( - manifest=manifest, - reader=reader, - max_blocks=max_blocks, - max_block_bytes=max_block_bytes, - max_referenced_nodes=max_referenced_nodes, - max_total_bytes=max_total_bytes, - ) + blocks, incomplete = build_evidence_blocks(manifest=manifest, reader=reader, max_blocks=max_blocks, max_block_bytes=max_block_bytes, max_referenced_nodes=max_referenced_nodes, max_total_bytes=max_total_bytes) + parser_metadata = rust_parser()[1] payload = { "schema_version": STAGE1_SCHEMA_VERSION, "ruleset_version": STAGE1_RULESET_VERSION, @@ -369,7 +435,7 @@ def build_evidence_manifest( "repository": manifest.repository, "base_commit": manifest.base_commit, "head_commit": manifest.head_commit, - "parser": rust_parser()[1], + "parser": dict(parser_metadata), "evidence_blocks": [block.canonical() for block in blocks], "incomplete": incomplete, } @@ -381,7 +447,7 @@ def build_evidence_manifest( repository=manifest.repository, base_commit=manifest.base_commit, head_commit=manifest.head_commit, - parser=rust_parser()[1], + parser=parser_metadata, evidence_blocks=blocks, incomplete=incomplete, stage1_sha256=digest, diff --git a/tests/test_stage1_evidence.py b/tests/test_stage1_evidence.py index 01cb413..e8999fa 100644 --- a/tests/test_stage1_evidence.py +++ b/tests/test_stage1_evidence.py @@ -1,13 +1,21 @@ from pathlib import Path +import subprocess import pytest from src.stage0_manifest import ChangeRecord, Disposition, Reason, freeze_manifest, git_blob_sha, sha256_hex -from src.stage1_evidence import EvidenceType, Stage1Error, build_evidence_manifest, deterministic_evidence_id, parse_ast, verify_stage0_manifest +from src.stage1_evidence import ( + EvidenceType, + Stage1Error, + build_evidence_manifest, + deterministic_evidence_id, + evidence_id, + parse_ast, + verify_stage0_manifest, +) def git_repo(tmp_path: Path, content: bytes): - import subprocess repo = tmp_path / "repo" repo.mkdir() subprocess.run(["git", "init", "-q"], cwd=repo, check=True) @@ -22,8 +30,8 @@ def git_repo(tmp_path: Path, content: bytes): return repo, commit, blob -def make_manifest(commit: str, blob: str, content: bytes, *, disposition=Disposition.PRIMARY, change_type="modified"): - record = ChangeRecord("src/main.rs", change_type, disposition, Reason.CODE_CHANGE, len(content), False, blob, sha256_hex(content)) +def make_manifest(commit: str, blob: str, content: bytes, *, disposition=Disposition.PRIMARY, change_type="modified", rename_group_id=None): + record = ChangeRecord("src/main.rs", change_type, disposition, Reason.CODE_CHANGE, len(content), False, blob, sha256_hex(content), rename_group_id) return freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) @@ -36,25 +44,83 @@ def manifest_json(manifest): def test_stage0_manifest_verification_rejects_tampering(tmp_path: Path): content = b"fn main() {}\n" repo, commit, blob = git_repo(tmp_path, content) - manifest = make_manifest(commit, blob, content) - raw = manifest_json(manifest) + raw = manifest_json(make_manifest(commit, blob, content)) raw["head_commit"] = "c" * 40 with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): verify_stage0_manifest(raw) -def test_stage0_manifest_verification_rejects_unknown_disposition(tmp_path: Path): +def test_stage0_manifest_verification_accepts_rename_field(tmp_path: Path): content = b"fn main() {}\n" repo, commit, blob = git_repo(tmp_path, content) - manifest = make_manifest(commit, blob, content) - raw = manifest_json(manifest) + raw = manifest_json(make_manifest(commit, blob, content, rename_group_id="r1")) + assert verify_stage0_manifest(raw).files[0].rename_group_id == "r1" + + +def test_stage0_manifest_unknown_disposition_fails_closed(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + raw = manifest_json(make_manifest(commit, blob, content)) raw["files"][0]["disposition"] = "VULNERABLE" with pytest.raises(Stage1Error, match="stage0_integrity_mismatch"): verify_stage0_manifest(raw) -def test_evidence_id_is_deterministic(): - assert [deterministic_evidence_id(i) for i in range(3)] == ["E-000001", "E-000002", "E-000003"] +def test_git_object_binding_is_verified_against_commit_path(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + evidence = build_evidence_manifest(manifest_json(make_manifest(commit, blob, content)), str(repo)) + assert evidence.evidence_blocks + assert evidence.evidence_blocks[0].object_id == blob + + +def test_working_tree_mutation_does_not_change_head_bound_evidence(tmp_path: Path): + original = b"fn main() { let x = 1; }\n" + repo, commit, blob = git_repo(tmp_path, original) + manifest = make_manifest(commit, blob, original) + first = build_evidence_manifest(manifest_json(manifest), str(repo)) + (repo / "src/main.rs").write_bytes(b"fn main() { panic!(\"working tree mutation\"); }\n") + second = build_evidence_manifest(manifest_json(manifest), str(repo)) + assert first.canonical_json() == second.canonical_json() + assert first.stage1_sha256 == second.stage1_sha256 + assert first.evidence_blocks[0].content == original.decode() + + +def test_head_object_change_changes_artifact(tmp_path: Path): + original = b"fn main() { let x = 1; }\n" + repo, base_commit, base_blob = git_repo(tmp_path, original) + (repo / "src/main.rs").write_bytes(b"fn main() { let x = 2; }\n") + subprocess.run(["git", "add", "src/main.rs"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-qm", "head"], cwd=repo, check=True) + head_commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + head_blob = subprocess.check_output(["git", "rev-parse", "HEAD:src/main.rs"], cwd=repo, text=True).strip() + record = ChangeRecord("src/main.rs", "modified", Disposition.PRIMARY, Reason.CODE_CHANGE, 25, False, head_blob, sha256_hex(b"fn main() { let x = 2; }\n")) + manifest = freeze_manifest(repository="test/repo", base_commit=base_commit, head_commit=head_commit, pr_number=1, files=[record]) + evidence = build_evidence_manifest(manifest_json(manifest), str(repo)) + assert evidence.evidence_blocks + assert evidence.evidence_blocks[0].source_sha256 == sha256_hex(b"fn main() { let x = 2; }\n") + + +def test_source_and_node_hashes_are_distinct_bindings(tmp_path: Path): + content = b"fn main() { let x = 1; }\n" + repo, commit, blob = git_repo(tmp_path, content) + evidence = build_evidence_manifest(manifest_json(make_manifest(commit, blob, content)), str(repo)) + block = evidence.evidence_blocks[0] + exact = content[block.start_byte:block.end_byte] + assert block.source_sha256 == sha256_hex(content) + assert block.content_sha256 == sha256_hex(exact) + + +def test_evidence_id_binds_source_and_node_identity(): + node = type("N", (), {"node_type": "function_item", "start_byte": 1, "end_byte": 2})() + first = evidence_id(source_sha256="a" * 64, file="src/a.rs", commit="b" * 40, object_id="c" * 40, node=node, evidence_type=EvidenceType.MODIFIED_FUNCTION) + second = evidence_id(source_sha256="d" * 64, file="src/a.rs", commit="b" * 40, object_id="c" * 40, node=node, evidence_type=EvidenceType.MODIFIED_FUNCTION) + assert first != second + + +def test_deterministic_legacy_helper_remains_non_authoritative(): + assert deterministic_evidence_id(0) == "E-000001" + assert deterministic_evidence_id(1) == "E-000002" def test_rust_ast_ranges_are_valid(): @@ -64,10 +130,19 @@ def test_rust_ast_ranges_are_valid(): assert all(0 <= ref.start_byte <= ref.end_byte <= len(content) for ref in refs) +def test_parse_error_fails_closed(): + with pytest.raises(Stage1Error, match="parse_error"): + parse_ast(b"fn broken( {\n") + + def test_non_primary_dispositions_are_not_emitted(tmp_path: Path): content = b"fn main() {}\n" repo, commit, blob = git_repo(tmp_path, content) - for disposition, reason in [(Disposition.SECONDARY, Reason.MISLEADING_EXTENSION), (Disposition.DEPENDENCY, Reason.DEPENDENCY), (Disposition.NON_CODE, Reason.NON_CODE)]: + for disposition, reason in [ + (Disposition.SECONDARY, Reason.MISLEADING_EXTENSION), + (Disposition.DEPENDENCY, Reason.DEPENDENCY), + (Disposition.NON_CODE, Reason.NON_CODE), + ]: record = ChangeRecord("src/main.rs", "modified", disposition, reason, len(content), False, blob, sha256_hex(content)) manifest = freeze_manifest(repository="test/repo", base_commit=commit, head_commit=commit, pr_number=1, files=[record]) evidence = build_evidence_manifest(manifest_json(manifest), str(repo)) @@ -82,25 +157,32 @@ def test_stage1_hash_is_reproducible_for_identical_git_objects(tmp_path: Path): second = build_evidence_manifest(manifest_json(manifest), str(repo)) assert first.canonical_json() == second.canonical_json() assert first.stage1_sha256 == second.stage1_sha256 - assert first.evidence_blocks -def test_working_tree_mutation_does_not_change_head_bound_evidence(tmp_path: Path): - original = b"fn main() { let x = 1; }\n" - repo, commit, blob = git_repo(tmp_path, original) - manifest = make_manifest(commit, blob, original) - first = build_evidence_manifest(manifest_json(manifest), str(repo)) - (repo / "src/main.rs").write_bytes(b"fn main() { panic!(\"working tree mutation\"); }\n") - second = build_evidence_manifest(manifest_json(manifest), str(repo)) - assert first.stage1_sha256 == second.stage1_sha256 - assert first.evidence_blocks[0].content == original.decode() +def test_stage1_hash_is_bound_to_stage0_hash(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + manifest = make_manifest(commit, blob, content) + evidence = build_evidence_manifest(manifest_json(manifest), str(repo)) + assert evidence.stage0_sha256 == manifest.manifest_sha256 + assert evidence.stage1_sha256 == sha256_hex(evidence.canonical_json()) -def test_stage1_does_not_promote_security_semantics(): - assert EvidenceType.AUTHORIZATION_CHECK.value == "AUTHORIZATION_CHECK" - assert EvidenceType.CALL.value == "CALL" +def test_limits_are_explicitly_incomplete(tmp_path: Path): + content = b"fn main() { let a = 1; let b = 2; }\n" + repo, commit, blob = git_repo(tmp_path, content) + manifest = make_manifest(commit, blob, content) + evidence = build_evidence_manifest(manifest_json(manifest), str(repo), max_blocks=1) + assert evidence.incomplete is True + assert len(evidence.evidence_blocks) == 1 -def test_malformed_rust_is_structural_not_security_semantic(): - refs = parse_ast(b"fn broken( {\n") - assert refs +def test_invalid_limits_fail_closed(tmp_path: Path): + content = b"fn main() {}\n" + repo, commit, blob = git_repo(tmp_path, content) + with pytest.raises(Stage1Error, match="invalid_evidence_limits"): + build_evidence_manifest(manifest_json(make_manifest(commit, blob, content)), str(repo), max_blocks=0) + + +def test_security_semantics_are_not_emitted_as_facts(): + assert "AUTHORIZATION_CHECK" not in {item.value for item in EvidenceType}