ci(#388): fail roll-up when tests are skipped on pull_request_target

.github/scripts/check-rollup-result-test.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# check-rollup-result-test.sh — Tests for check-rollup-result.sh
+#
+# Run from the repo root: bash .github/scripts/check-rollup-result-test.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROLLUP_SCRIPT="${SCRIPT_DIR}/check-rollup-result.sh"
+FAILURES=0
+TESTS=0
+
+run_rollup() {
+  EVENT_NAME="$1" \
+  GATE_RESULT="$2" DETECT_RESULT="$3" TESTS_RESULT="$4" \
+  CROSS_REPO="${5:-}" \
+  bash "${ROLLUP_SCRIPT}" 2>/dev/null
+}
+
+assert_pass() {
+  local test_name="$1"
+  shift
+  TESTS=$((TESTS + 1))
+  if run_rollup "$@" >/dev/null; then
+    echo "PASS: ${test_name}"
+  else
+    echo "FAIL: ${test_name} — expected exit 0"
+    FAILURES=$((FAILURES + 1))
+  fi
+}
+
+assert_fail() {
+  local test_name="$1"
+  shift
+  TESTS=$((TESTS + 1))
+  if run_rollup "$@" >/dev/null; then
+    echo "FAIL: ${test_name} — expected exit 1 but got 0"
+    FAILURES=$((FAILURES + 1))
+  else
+    echo "PASS: ${test_name}"
+  fi
+}
+
+# --- Test cases ---
+#                                       event_name            gate      detect    tests     cross_repo
+
+assert_pass "push, all success"         push                  skipped   success   success   ""
+assert_fail "push, tests failure"       push                  skipped   success   failure   ""
+assert_pass "merge_group, all success"  merge_group           skipped   success   success   ""
+assert_fail "merge_group, tests fail"   merge_group           skipped   success   failure   ""
+
+assert_pass "PRT, all success"          pull_request_target   success   success   success   ""
+assert_fail "PRT, detect skipped"       pull_request_target   success   skipped   skipped   ""
+assert_fail "PRT, all skipped"          pull_request_target   skipped   skipped   skipped   ""
+
+assert_fail "gate cancelled"            pull_request_target   cancelled skipped   skipped   ""
+assert_fail "detect failure"            push                  skipped   failure   skipped   ""
+assert_fail "detect cancelled"          push                  skipped   cancelled skipped   ""
+assert_fail "tests cancelled"           push                  skipped   success   cancelled ""
+
+assert_pass "cross-repo, all success"   push                  skipped   success   success   true
+assert_fail "cross-repo, tests skipped" push                  skipped   success   skipped   true
+assert_pass "same-repo, tests skipped"  push                  skipped   success   skipped   ""
+
+# --- Summary ---
+echo ""
+echo "=== ${TESTS} tests, ${FAILURES} failures ==="
+if [[ "${FAILURES}" -gt 0 ]]; then
+  exit 1
+fi

.github/scripts/check-rollup-result.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# check-rollup-result.sh — Decide whether the functional-tests-complete
+# roll-up job should pass or fail.
+#
+# Inputs (env vars):
+#   EVENT_NAME    — github.event_name
+#   GATE_RESULT   — needs.gate.result
+#   DETECT_RESULT — needs.detect.result
+#   TESTS_RESULT  — needs.functional-tests.result
+#   CROSS_REPO    — 'true' when invoked from a different repository
+#
+# Exit 0 = pass, exit 1 = fail.
+
+set -euo pipefail
+
+EVENT_NAME="${EVENT_NAME:-}"
+GATE_RESULT="${GATE_RESULT:-}"
+DETECT_RESULT="${DETECT_RESULT:-}"
+TESTS_RESULT="${TESTS_RESULT:-}"
+CROSS_REPO="${CROSS_REPO:-}"
+
+if [ "$GATE_RESULT" = "failure" ] || [ "$GATE_RESULT" = "cancelled" ]; then
+  echo "::error::Gate job ${GATE_RESULT}"
+  exit 1
+fi
+
+if [ "$EVENT_NAME" = "pull_request_target" ] && [ "$DETECT_RESULT" = "skipped" ]; then
+  echo "::error::Detect was ${DETECT_RESULT} on pull_request_target — tests were not authorized to run"
+  exit 1
+fi
+
+if [ "$DETECT_RESULT" = "failure" ] || [ "$DETECT_RESULT" = "cancelled" ]; then
+  echo "::error::Detect job ${DETECT_RESULT}"
+  exit 1
+fi
+
+if [ "$TESTS_RESULT" = "failure" ] || [ "$TESTS_RESULT" = "cancelled" ]; then
+  echo "::error::One or more functional tests ${TESTS_RESULT}"
+  exit 1
+fi
+
+if [ "$CROSS_REPO" = "true" ] && [ "$TESTS_RESULT" = "skipped" ]; then
+  echo "::error::Cross-repo call completed with zero functional tests — agents list was empty or detect was skipped"
+  exit 1
+fi

.github/workflows/functional-tests.yml

@@ -400,27 +400,22 @@ jobs:
     if: always()
     runs-on: ubuntu-24.04
     timeout-minutes: 1
+    permissions:
+      contents: read
     steps:
+      # Base branch only -- a PR must not be able to rewrite this script
+      # to force-pass its own roll-up check.
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+          sparse-checkout: .github/scripts
+
       - name: Check results
         env:
+          EVENT_NAME: ${{ github.event_name }}
           GATE_RESULT: ${{ needs.gate.result }}
           DETECT_RESULT: ${{ needs.detect.result }}
           TESTS_RESULT: ${{ needs.functional-tests.result }}
           CROSS_REPO: ${{ github.repository != 'fullsend-ai/agents' }}
-        run: |
-          if [ "$GATE_RESULT" = "failure" ] || [ "$GATE_RESULT" = "cancelled" ]; then
-            echo "::error::Gate job ${GATE_RESULT}"
-            exit 1
-          fi
-          if [ "$DETECT_RESULT" = "failure" ] || [ "$DETECT_RESULT" = "cancelled" ]; then
-            echo "::error::Detect job ${DETECT_RESULT}"
-            exit 1
-          fi
-          if [ "$TESTS_RESULT" = "failure" ] || [ "$TESTS_RESULT" = "cancelled" ]; then
-            echo "::error::One or more functional tests ${TESTS_RESULT}"
-            exit 1
-          fi
-          if [ "$CROSS_REPO" = "true" ] && [ "$TESTS_RESULT" = "skipped" ]; then
-            echo "::error::Cross-repo call completed with zero functional tests — agents list was empty or detect was skipped"
-            exit 1
-          fi
+        run: bash .github/scripts/check-rollup-result.sh

Makefile

@@ -62,5 +62,6 @@ script-test:
 	$(call run-timed,bash .github/scripts/select-eval-agents-test.sh)
 	$(call run-timed,python3 scripts/process-fix-result-test.py)
 	$(call run-timed,bash eval/scripts/scrub-eval-results-test.sh)
+	$(call run-timed,bash .github/scripts/check-rollup-result-test.sh)
 
 test: script-test