fix(ci): pin-resolver probes ancestry for reusable pins — orphan-pin class (#782)

scripts/check-action-pins-resolve.sh

@@ -42,14 +42,44 @@
 # green — a fail-open that announces itself is not a fake gate; a fail-open
 # that hides is.
 #
+# ── Orphan pins: reachable ≠ consumable (issue #782) ───────────────────────
+# For REUSABLE-WORKFLOW pins (owner/repo/.github/workflows/*.yml@sha), object
+# existence is necessary but NOT sufficient. GitHub resolves a called workflow
+# only at a commit reachable from the repo's default branch. A real commit
+# object that is an ancestor of nothing — squash-merge orphan, deleted unmerged
+# branch, remote-branch-only ref — answers 200 at the commits endpoint AND at
+# contents/<path>?ref=…, yet Actions fails at graph resolution with
+# "workflow was not found", reporting jobs.total_count == 0: no check run,
+# often not even a red one. Measured in-estate 2026-09: four such SHAs
+# (7fdc2705…, 892497fe…, 46960521…, plus the non-object 5b1d0022…) account
+# for 61 dead workflow-run rows with ZERO alive rows — all passing this gate's
+# old existence predicate.
+#
+# So reusable-workflow pins get a second probe, server-side:
+#     compare/<default>...<sha>  = behind|identical → ancestor; consumable
+#                                = ahead|diverged   → NOT an ancestor of the
+#                                                     default branch; a
+#                                                     DETERMINATE negative
+# Local forms are unusable: for-each-ref --contains passes remote-branch
+# orphans, and merge-base --is-ancestor lies under shallow clones. The compare
+# call is one request and needs no clone at all. (The pin-writer,
+# scripts/apply-workflow-pins-remote.sh, already enforces the same rule with
+# compare/<sha>...main ∈ {identical, ahead} — identical semantics, reversed
+# direction.)
+#
+# The ancestry probe applies ONLY to reusable-workflow pins: ordinary action
+# pins (@sha on an action repo) are fetched by object id at run time, and
+# non-default-branch commits are a working, legitimate pattern there.
+#
 # Rate limiting is not expected to bite: with GITHUB_TOKEN the limit is 1,000
 # requests/hour/repo, and the largest estate repo carries well under 100 unique
 # pins (only unique (repo,sha) pairs are queried, not every occurrence).
 #
 # USAGE:  check-action-pins-resolve.sh [path]     # default: current directory
 #         GH_TOKEN / GITHUB_TOKEN respected for auth.
-# EXIT:   0 = all pins resolve (or only indeterminate results)
-#         1 = at least one pin determinately does not exist
+# EXIT:   0 = all pins consumable (or only indeterminate results)
+#         1 = at least one pin determinately unusable: does not exist, or is
+#             an orphan no default-branch ref can reach (reusable pins only)
 
 TARGET="${1:-.}"
 WORKFLOW_DIR="$TARGET/.github/workflows"
@@ -59,15 +89,17 @@
   exit 0
 fi
 
-# ── Collect unique (repo, sha) pairs ────────────────────────────────────────
+# ── Collect unique (repo, sha, kind) pairs ──────────────────────────────────
 # Handles `owner/repo@sha` and `owner/repo/sub/path@sha` (reusable workflows
 # and composite subpaths both pin at the repository level).
 # Skips local (`./`) and docker:// refs, which have no upstream commit.
+# kind = R: the ref points at a reusable workflow file (.github/workflows/*.yml
+# in the repo) — those get the ancestry probe (see header); kind = A otherwise.
 pairs="$(
   grep -rhoE '\buses:[[:space:]]*[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+@[0-9a-f]{40}' \
     "$WORKFLOW_DIR" 2>/dev/null \
   | sed -E 's/.*uses:[[:space:]]*//' \
-  | awk -F'@' '{ split($1, p, "/"); print p[1] "/" p[2] "\t" $2 }' \
+  | awk -F'@' '{ split($1, p, "/"); k = ($1 ~ /\.github\/workflows\/[^\/]+\.ya?ml$/) ? "R" : "A"; print p[1] "/" p[2] "\t" $2 "\t" k }' \
   | sort -u
 )"
 
@@ -79,28 +111,99 @@
 total=$(printf '%s\n' "$pairs" | wc -l | tr -d ' ')
 echo "Checking $total unique action pin(s) resolve upstream…"
 
-api() { # api <path> -> prints body, returns curl-visible HTTP code in $HTTP
-  local path="$1" auth=()
+api() { # api <path> — HTTP=response code, BODY=response body (both globals).
+  # The ancestry probe below needs the body (compare status, default_branch),
+  # so the code is captured via -w on the final line of stdout.
+  local path="$1" auth=() out
   [ -n "${GH_TOKEN:-${GITHUB_TOKEN:-}}" ] && \
     auth=(-H "Authorization: Bearer ${GH_TOKEN:-$GITHUB_TOKEN}")
-  HTTP="$(curl -sS -o /dev/null -w '%{http_code}' \
+  out="$(curl -sS -w $'\n%{http_code}' \
     -H "Accept: application/vnd.github+json" \
     -H "X-GitHub-Api-Version: 2022-11-28" \
-    "${auth[@]}" "https://api.github.com/$path" 2>/dev/null)" || HTTP="000"
+    "${auth[@]}" "https://api.github.com/$path" 2>/dev/null)" || out=""
+  # Newline test must be a BUILTIN: `printf | grep -q` under pipefail lets
+  # grep exit on the first match and kill printf with SIGPIPE for any body
+  # over the 64 KB pipe buffer, turning HTTP into the whole response blob
+  # (measured live: 146 KB codeql-action compare body).
+  if [ "$out" != "${out%$'\n'*}" ]; then
+    HTTP="${out##*$'\n'}"
+    BODY="${out%$'\n'*}"
+  elif [ -n "$out" ]; then
+    HTTP="$out"  # bare code from a minimal server (test stub): no body
+    BODY=""
+  else
+    HTTP="000"; BODY=""
+  fi
+}
+
+# json_field <name> — first string-valued "name":"value" pair in $BODY.
+# Sufficient here: repos/<repo> defines default_branch exactly once, and in
+# /compare the top-level status precedes the commits/files arrays, so the
+# first occurrence IS the verdict. Deliberately no jq dependency: this gate
+# also runs in minimal local shells.
+json_field() {
+  printf '%s\n' "$BODY" \
+    | grep -oE "\"$1\"[[:space:]]*:[[:space:]]*\"[^\"]+\"" \
+    | head -1 \
+    | sed -E 's/^.*:[[:space:]]*"([^"]+)"$/\1/'
+}
+
+# default_branch <repo> — echo the repo's default branch, or "" if the probe
+# was indeterminate. pairs arrive repo-sorted (sort -u above), so a one-entry
+# cache hits every adjacent repeat exactly.
+last_db_repo=""; last_db=""
+default_branch() {
+  local repo="$1"
+  if [ "$last_db_repo" != "$repo" ]; then
+    last_db_repo="$repo"; last_db=""
+    api "repos/$repo"
+    [ "$HTTP" = "200" ] && last_db="$(json_field default_branch)"
+  fi
+  printf '%s' "$last_db"
 }
 
 bad=0
 unverified=0
 bad_list=""
 unver_list=""
 
-while IFS=$'\t' read -r repo sha; do
+while IFS=$'\t' read -r repo sha kind; do
   [ -z "$repo" ] && continue
 
   api "repos/$repo/commits/$sha"
   case "$HTTP" in
     200)
-      : # resolves — good
+      if [ "$kind" = "R" ]; then
+        # Reusable-workflow pin (issue #782): the object exists, but Actions
+        # will still refuse it at graph resolution unless it is an ancestor of
+        # the repo's default branch. Probe ancestry server-side.
+        db="$(default_branch "$repo")"
+        if [ -z "$db" ]; then
+          unverified=$((unverified + 1))
+          unver_list="${unver_list}  ancestry: default-branch probe indeterminate — $repo@$sha"$'\n'
+        else
+          api "repos/$repo/compare/$db...$sha"
+          status=""
+          [ "$HTTP" = "200" ] && status="$(json_field status)"
+          case "$status" in
+            behind|identical)
+              : # ancestor of the default branch — consumable
+              ;;
+            ahead|diverged)
+              # Determinate negative: a real commit the resolver cannot reach.
+              # ahead/diverged => NOT an ancestor of $db (an ancestor would
+              # report behind/identical).
+              bad=$((bad + 1))
+              bad_list="${bad_list}  NOT-ANCESTOR    $repo@$sha  (compare $db → ${status})"$'\n'
+              ;;
+            *)
+              unverified=$((unverified + 1))
+              unver_list="${unver_list}  ancestry: compare probe indeterminate (HTTP $HTTP) — $repo@$sha  (base: $db)"$'\n'
+              ;;
+          esac
+        fi
+      fi
+      # Action pins (kind=A): object existence is sufficient at run time.
       ;;
     404|422)
       # Determinate negative from the commits endpoint. Disambiguate:
@@ -147,6 +250,13 @@
   echo "  · REPO-NOT-FOUND — the action is gone. Vendor the logic into this repo"
   echo "                     and call it with 'run:' (see hyperpolymath/tangle#84),"
   echo "                     or repoint at the live repository name."
+  echo "  · NOT-ANCESTOR   — the reusable-workflow pin names a real commit that"
+  echo "                     the resolver cannot reach from the default branch"
+  echo "                     (squash-merge orphan, deleted unmerged branch, or"
+  echo "                     remote-branch-only ref — 61 estate rows dead this way,"
+  echo "                     issue #782). Repin to a commit ON the default branch:"
+  echo "                     the merge commit, never a PR head — a PR head orphans"
+  echo "                     at squash-merge — and re-run this gate to confirm."
   exit 1
 fi
 

scripts/tests/check-action-pins-resolve-test.sh

@@ -36,18 +36,32 @@
 bad() { echo "  ❌ $1"; fail=$((fail + 1)); }
 
 # ── the curl stub ───────────────────────────────────────────────────────────
-# The script calls: curl -sS -o /dev/null -w '%{http_code}' … <url>
-# so the stub simply prints the code the case wants for that URL shape.
+# The script calls: curl -sS -w $'\n%{http_code}' <headers> <url>, so the stub
+# prints the body the case wants, a newline, and then the HTTP code — exactly
+# the real api()'s capture contract (last line = code).
 mkdir -p "$TMP/bin"
 cat > "$TMP/bin/curl" <<'STUB'
 #!/usr/bin/env bash
 url="${!#}"
+code="200"; body=""
+[ -n "${STUB_URLLOG:-}" ] && printf '%s\n' "$url" >> "$STUB_URLLOG"
+pad=""
+# STUB_BIGBODY=1 emits a large PRETTY-PRINTED body (like GitHub's): an early
+# newline (a piped `grep -q` matches and exits at once) followed by >64 KB of
+# payload the stranded printf can then no longer write — EPIPE/SIGPIPE, and
+# under pipefail the branch flips and the whole blob lands in $HTTP.
+[ "${STUB_BIGBODY:-0}" = "1" ] && pad="$(printf '%*s' 150000 '' | tr ' ' x)"
+mkbody() { # mkbody <key> <value> — compact one-liner, or PRETTY + 150 KB pad
+  if [ -n "$pad" ]; then printf '{\n  "%s": "%s",\n  "pad": "%s"\n}' "$1" "$2" "$pad"
+  else printf '{"%s":"%s"}' "$1" "$2"; fi
+}
 case "$url" in
-  */commits/*) code="${STUB_COMMITS:-200}" ;;
-  *)           code="${STUB_REPO:-200}" ;;
+  */compare/*) code="${STUB_COMPARE:-200}"; body="$(mkbody status "${STUB_COMPARE_STATUS:-behind}")" ;;
+  */commits/*) code="${STUB_COMMITS:-200}";  body="$(mkbody sha object)" ;;
+  *)           code="${STUB_REPO:-200}";     body="$(mkbody default_branch "${STUB_BRANCH:-main}")" ;;
 esac
 [ "$code" = "NETFAIL" ] && exit 22
-printf '%s' "$code"
+printf '%s\n%s' "$body" "$code"
 STUB
 chmod +x "$TMP/bin/curl"
 export PATH="$TMP/bin:$PATH"
@@ -68,6 +82,18 @@
 YAML
 }
 
+# mk_reusable <dir> <sha> — a target tree with ONE SHA-pinned reusable
+# workflow call, the shape the #782 orphan class requires.
+mk_reusable() {
+  local d="$1" sha="$2"; rm -rf "$d"; mkdir -p "$d/.github/workflows"
+  cat > "$d/.github/workflows/caller.yml" <<YAML
+name: caller
+jobs:
+  scan:
+    uses: hyperpolymath/standards/.github/workflows/secret-scanner-reusable.yml@${sha}
+YAML
+}
+
 # expect <name> <expected-rc> <expected-substring> <target-dir>
 expect() {
   local name=$1 want_rc=$2 want_sub=$3 target=$4 out rc
@@ -135,6 +161,98 @@
 STUB_COMMITS=404 STUB_REPO=500 \
   expect "a determinate negative with an unconfirmable repo is indeterminate" 0 "::warning::UNVERIFIED" "$TMP/r"
 
+echo
+echo "== orphan reusable pins — the four #782 witness SHAs =="
+# Measured 2026-09 (estate census of 435 repos x 4 reusable workflows): every
+# one of these answered 200 at the commits endpoint, and every row calling it
+# died at graph resolution — 61 dead rows, zero alive. The four provenances:
+#
+#   7fdc2705…  squash-merge orphan: pin captured the PR head; the squash
+#              discarded it. compare/main = diverged.
+#   892497fe…  deleted unmerged branch; 0 PRs reference it. diverged.
+#   46960521…  reachable from a LIVE remote branch, but not an ancestor of
+#              main. compare/main = ahead.
+#   5b1d0022…  prefix corruption — not an object at all. commits 404.
+#
+# The old predicate passed the first three, which is exactly the class this
+# gate now exists to fail on.
+SHA_W1=7fdc27050000000000000000000000000000000000
+SHA_W2=892497fe0000000000000000000000000000000000
+SHA_W3=4696052100000000000000000000000000000000
+SHA_W4=5b1d00220000000000000000000000000000000000
+SHA_OK=81dbf2dd00000000000000000000000000000000
+
+mk_reusable "$TMP/w1" "$SHA_W1"
+STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged \
+  expect "witness 1 (7fdc2705… squash-merge orphan) fails as NOT-ANCESTOR" 1 "NOT-ANCESTOR" "$TMP/w1"
+
+STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged \
+  expect "witness 1's failure line names the compare verdict" 1 "compare main → diverged" "$TMP/w1"
+
+mk_reusable "$TMP/w2" "$SHA_W2"
+STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged \
+  expect "witness 2 (892497fe… deleted branch) fails as NOT-ANCESTOR" 1 "NOT-ANCESTOR" "$TMP/w2"
+
+mk_reusable "$TMP/w3" "$SHA_W3"
+STUB_COMMITS=200 STUB_COMPARE_STATUS=ahead \
+  expect "witness 3 (46960521… remote-branch-only, compare=ahead) fails as NOT-ANCESTOR" 1 "NOT-ANCESTOR" "$TMP/w3"
+
+mk_reusable "$TMP/w4" "$SHA_W4"
+STUB_COMMITS=404 STUB_REPO=200 \
+  expect "witness 4 (5b1d0022… not an object) is still SHA-NOT-FOUND" 1 "SHA-NOT-FOUND" "$TMP/w4"
+
+STUB_COMMITS=200 STUB_COMPARE=403 \
+  expect "an indeterminate compare probe does NOT fail the build" 0 "::warning::UNVERIFIED" "$TMP/w3"
+
+STUB_COMMITS=200 STUB_COMPARE=403 \
+  expect "an indeterminate compare probe announces itself" 0 "ancestry: compare probe indeterminate" "$TMP/w3"
+
+echo
+echo "== ancestry probe semantics =="
+
+mk_reusable "$TMP/ok" "$SHA_OK"
+STUB_COMMITS=200 STUB_COMPARE_STATUS=behind \
+  expect "a reusable pin that IS an ancestor passes (behind)" 0 "resolve upstream." "$TMP/ok"
+
+STUB_COMMITS=200 STUB_COMPARE_STATUS=identical \
+  expect "a reusable pin AT the default-branch tip passes (identical)" 0 "resolve upstream." "$TMP/ok"
+
+STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged STUB_REPO=404 \
+  expect "an unprobeable default branch is indeterminate, not a verdict" 0 "ancestry: default-branch probe indeterminate" "$TMP/ok"
+
+# The single most important guard against overreach: ordinary ACTION pins are
+# fetched by object id at run time; a non-default-branch action commit works.
+# The ancestry probe must NOT fire on them.
+STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged \
+  expect "an action pin is never ancestry-probed (would false-fail real usage)" 0 "resolve upstream." "$TMP/r"
+
+# …and prove it structurally: with only action pins present, the compare
+# endpoint must not appear in the stub's access log at all.
+: > "$TMP/urllog"
+STUB_URLLOG="$TMP/urllog" STUB_COMMITS=200 \
+  expect "action-only fixture passes" 0 "All 1 verifiable action pin(s) resolve upstream." "$TMP/r"
+if grep -q "compare" "$TMP/urllog" 2>/dev/null; then
+  bad "compare endpoint was called for an action-only fixture"
+else
+  ok "compare endpoint untouched for an action-only fixture"
+fi
+
+echo
+echo "== large response bodies vs pipefail (the SIGPIPE trap) =="
+# api() isolates the HTTP code from a body that can exceed 64 KB. If the
+# isolation uses a `printf | grep -q` pipeline, grep exits on first match,
+# printf dies on SIGPIPE, pipefail flips the branch, and the ENTIRE blob
+# lands in the HTTP variable — making a healthy 200 look like an unknown
+# status. Caught live against a 146 KB codeql-action compare body.
+STUB_BIGBODY=1 STUB_COMMITS=200 \
+  expect "a >64k commits body still parses its status code" 0 "All 1 verifiable action pin(s) resolve upstream." "$TMP/r"
+
+STUB_BIGBODY=1 STUB_COMMITS=200 STUB_COMPARE_STATUS=behind \
+  expect "a >64k compare body still yields its ancestry status" 0 "resolve upstream." "$TMP/ok"
+
+STUB_BIGBODY=1 STUB_COMMITS=200 STUB_COMPARE_STATUS=diverged \
+  expect "a >64k compare body still fails loud as NOT-ANCESTOR" 1 "NOT-ANCESTOR" "$TMP/w1"
+
 echo
 echo "== the happy path =="