ci(#983): add pre-commit hook to verify URL #sha256= integrity pins

.pre-commit-config.yaml

@@ -58,3 +58,9 @@ repos:
         language: script
         files: ^(eval/measurements/|eval/lint-measurements)
         pass_filenames: false
+      - id: verify-sha256-pins
+        name: verify URL sha256 integrity pins
+        entry: ./hack/verify-sha256-pins
+        language: script
+        files: ^(harness/|\.fullsend/config\.yaml|hack/verify-sha256-pins)
+        pass_filenames: false

Makefile

@@ -62,6 +62,7 @@ script-test:
 	$(call run-timed,bash scripts/validate-code-output-test.sh)
 	$(call run-timed,bash scripts/gitlint-forbidden-type-scope-test.sh)
 	$(call run-timed,bash hack/lint-agent-docs-test.sh)
+	$(call run-timed,bash hack/verify-sha256-pins-test.sh)
 	$(call run-timed,bash eval/lint-measurements-test.sh)
 	$(call run-timed,bash .github/scripts/check-e2e-authorization-test.sh)
 	$(call run-timed,bash .github/scripts/select-eval-agents-test.sh)

hack/verify-sha256-pins

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Verify that URL #sha256= integrity fragments match actual content.
+#
+# Scans harness/*.yaml and .fullsend/config.yaml for URL+#sha256=HASH
+# patterns, fetches each URL, and compares the computed sha256 against
+# the pinned hash.  Field-agnostic: any YAML string value matching
+# the pattern is checked, so future fields are covered automatically.
+#
+# Container image digests (@sha256:) are ignored.
+#
+# Usage:
+#   ./hack/verify-sha256-pins
+#
+# Environment:
+#   REPO_ROOT      — override repo root (default: parent of script dir)
+#   VERIFY_FETCH   — override fetch command (default: curl -sfL);
+#                    receives the URL as $1, must write content to stdout
+set -euo pipefail
+
+REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}"
+FETCH_CMD="${VERIFY_FETCH:-curl -sfL}"
+
+errors=0
+checked=0
+
+echo "Verifying sha256 pins..."
+echo "================================================"
+
+# Collect YAML files to scan.
+files=()
+shopt -s nullglob
+for f in "$REPO_ROOT"/harness/*.yaml; do
+  files+=("$f")
+done
+shopt -u nullglob
+[[ -f "$REPO_ROOT/.fullsend/config.yaml" ]] && files+=("$REPO_ROOT/.fullsend/config.yaml")
+
+if [[ ${#files[@]} -eq 0 ]]; then
+  echo "  No YAML files to scan"
+  echo ""
+  echo "OK: no sha256 pins to verify"
+  exit 0
+fi
+
+for file in "${files[@]}"; do
+  relpath="${file#"$REPO_ROOT/"}"
+
+  # Find lines containing URL #sha256= fragments (not @sha256: image digests).
+  # grep -n gives "LINENO:LINE"; we split on the first colon.
+  while IFS= read -r grepline; do
+    lineno="${grepline%%:*}"
+    line="${grepline#*:}"
+
+    # Extract every URL+#sha256=HEX token on the line.
+    # The URL ends at the first whitespace, quote, or end of string after
+    # the hex hash.  Use grep -oE to pull each match individually.
+    while IFS= read -r url_with_hash; do
+      url="${url_with_hash%%#sha256=*}"
+      expected="${url_with_hash##*#sha256=}"
+
+      # Skip placeholder/template values (e.g. <sha256sum>).
+      if [[ ! "$expected" =~ ^[0-9a-fA-F]+$ ]]; then
+        continue
+      fi
+
+      # Fetch the URL content to a temp file so we can detect fetch failures
+      # separately from hash computation.
+      local_tmp="$(mktemp)"
+      if ! $FETCH_CMD "$url" > "$local_tmp" 2>/dev/null; then
+        rm -f "$local_tmp"
+        echo "  ERROR: ${relpath}:${lineno}: failed to fetch ${url}"
+        errors=$((errors + 1))
+        continue
+      fi
+      actual="$(sha256sum "$local_tmp" | awk '{print $1}')"
+      rm -f "$local_tmp"
+
+      if [[ "$actual" != "$expected" ]]; then
+        echo "  MISMATCH: ${relpath}:${lineno}"
+        echo "    url:      ${url}"
+        echo "    expected: ${expected}"
+        echo "    actual:   ${actual}"
+        errors=$((errors + 1))
+      else
+        echo "  ${relpath}:${lineno}: OK"
+      fi
+      checked=$((checked + 1))
+    done < <(echo "$line" | grep -oE 'https?://[^[:space:]"'"'"']*#sha256=[0-9a-fA-F]+' || true)
+  done < <(grep -n '#sha256=' "$file" | grep -v '@sha256:' || true)
+done
+
+echo ""
+echo "================================================"
+if [[ $errors -gt 0 ]]; then
+  echo "FAILED: ${errors} error(s) found"
+  exit 1
+else
+  echo "OK: ${checked} sha256 pin(s) verified"
+fi

hack/verify-sha256-pins-test.sh

@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+# verify-sha256-pins-test.sh — Tests for hack/verify-sha256-pins
+#
+# Run from the repo root:
+#   bash hack/verify-sha256-pins-test.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+LINTER="${SCRIPT_DIR}/verify-sha256-pins"
+
+FAILURES=0
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+# Create content files for mock fetching.
+CONTENT_A="harness content alpha"
+printf '%s' "${CONTENT_A}" > "${WORKDIR}/content-a.txt"
+HASH_A="$(sha256sum "${WORKDIR}/content-a.txt" | awk '{print $1}')"
+
+CONTENT_B="harness content beta"
+printf '%s' "${CONTENT_B}" > "${WORKDIR}/content-b.txt"
+HASH_B="$(sha256sum "${WORKDIR}/content-b.txt" | awk '{print $1}')"
+
+WRONG_HASH="0000000000000000000000000000000000000000000000000000000000000000"
+
+# Mock fetch script: maps test URLs to local fixture files.
+MOCK_FETCH="${WORKDIR}/mock-fetch"
+cat > "${MOCK_FETCH}" <<MOCKEOF
+#!/bin/bash
+case "\$1" in
+  https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml)
+    cat "${WORKDIR}/content-a.txt" ;;
+  https://raw.githubusercontent.com/example/repo/abc123/file-b.yaml)
+    cat "${WORKDIR}/content-b.txt" ;;
+  https://raw.githubusercontent.com/example/repo/abc123/unreachable.yaml)
+    exit 1 ;;
+  *) exit 1 ;;
+esac
+MOCKEOF
+chmod +x "${MOCK_FETCH}"
+
+# run_case NAME EXPECTED_EXIT [EXPECTED_OUTPUT_SUBSTRING]
+#
+# Expects the case directory to have been set up before calling.
+run_case() {
+  local name="$1" expected_exit="$2" expected_substring="${3:-}"
+  local case_dir="${WORKDIR}/${name}"
+
+  local output
+  local actual_exit=0
+  output="$(REPO_ROOT="${case_dir}" VERIFY_FETCH="${MOCK_FETCH}" "${LINTER}" 2>&1)" || actual_exit=$?
+
+  if [[ "${actual_exit}" != "${expected_exit}" ]]; then
+    echo "FAIL: ${name} (exit ${actual_exit}, expected ${expected_exit})"
+    echo "${output}" | sed 's/^/    /'
+    FAILURES=$((FAILURES + 1))
+    return
+  fi
+
+  if [[ -n "${expected_substring}" ]] && [[ "${output}" != *"${expected_substring}"* ]]; then
+    echo "FAIL: ${name} (missing expected output: '${expected_substring}')"
+    echo "${output}" | sed 's/^/    /'
+    FAILURES=$((FAILURES + 1))
+    return
+  fi
+
+  echo "PASS: ${name}"
+}
+
+# ---- Test: correct hash passes ----
+name="correct-hash-passes"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+agent: agents/test.md
+base: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${HASH_A}
+EOF
+run_case "${name}" 0 "OK"
+
+# ---- Test: wrong hash fails ----
+name="wrong-hash-fails"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+agent: agents/test.md
+base: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${WRONG_HASH}
+EOF
+run_case "${name}" 1 "MISMATCH"
+
+# ---- Test: local paths silently skipped ----
+name="local-paths-skipped"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+agent: agents/test.md
+policy: policies/base.yaml
+openshell:
+  profiles:
+    - profiles/fullsend-vertex-ai.yaml
+providers:
+  - providers/vertex-ai.yaml
+EOF
+run_case "${name}" 0 "0 sha256 pin(s) verified"
+
+# ---- Test: container image digests ignored ----
+name="image-digest-ignored"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+agent: agents/test.md
+image: ghcr.io/fullsend-ai/fullsend-code@sha256:9743bc7b6e451e0bcea25ae4a67e0c040c296f1fee04c08988ae80c53fafcfe6
+EOF
+run_case "${name}" 0 "0 sha256 pin(s) verified"
+
+# ---- Test: field-agnostic scanning ----
+name="field-agnostic"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+agent: agents/test.md
+base: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${HASH_A}
+skills:
+  - https://raw.githubusercontent.com/example/repo/abc123/file-b.yaml#sha256=${HASH_B}
+EOF
+run_case "${name}" 0 "2 sha256 pin(s) verified"
+
+# ---- Test: .fullsend/config.yaml scanned ----
+name="fullsend-config-scanned"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness" "${case_dir}/.fullsend"
+# Empty harness dir (no .yaml files)
+cat > "${case_dir}/.fullsend/config.yaml" <<EOF
+version: "1"
+agents:
+    - source: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${HASH_A}
+EOF
+run_case "${name}" 0 ".fullsend/config.yaml"
+
+# ---- Test: fetch failure reports error ----
+name="fetch-failure"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+source: https://raw.githubusercontent.com/example/repo/abc123/unreachable.yaml#sha256=${HASH_A}
+EOF
+run_case "${name}" 1 "failed to fetch"
+
+# ---- Test: placeholder hashes skipped ----
+name="placeholder-skipped"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+base: https://raw.githubusercontent.com/fullsend-ai/agents/<SHA>/harness/review.yaml#sha256=<sha256sum>
+EOF
+run_case "${name}" 0 "0 sha256 pin(s) verified"
+
+# ---- Test: mismatch output includes url and hashes ----
+name="mismatch-output-details"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+base: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${WRONG_HASH}
+EOF
+run_case "${name}" 1 "expected: ${WRONG_HASH}"
+
+# ---- Test: mixed valid and invalid ----
+name="mixed-valid-invalid"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}/harness"
+cat > "${case_dir}/harness/test.yaml" <<EOF
+---
+base: https://raw.githubusercontent.com/example/repo/abc123/file-a.yaml#sha256=${HASH_A}
+source: https://raw.githubusercontent.com/example/repo/abc123/file-b.yaml#sha256=${WRONG_HASH}
+EOF
+run_case "${name}" 1 "MISMATCH"
+
+# ---- Test: no yaml files ----
+name="no-yaml-files"
+case_dir="${WORKDIR}/${name}"
+mkdir -p "${case_dir}"
+run_case "${name}" 0 "no sha256 pins to verify"
+
+echo ""
+if [[ ${FAILURES} -gt 0 ]]; then
+  echo "${FAILURES} test(s) failed"
+  exit 1
+fi
+echo "All tests passed"