feat(review): add confidence annotation to review verdicts

agents/review.md

@@ -256,6 +256,7 @@ fields such as `outcome`, `summary`, `prior_review_sha`, or
 | `reason`    | string  | conditional     | One of: `tool-failure`, `missing-context`, `ambiguous-findings`, `token-limit`, `time-budget` |
 | `label_actions` | object | no | Contextual label recommendations (see `issue-labels` skill) |
 | `risk_assessment` | object | no | Risk assessment from the risk-assessment sub-agent (see `pr-risk-assessment` skill) |
+| `confidence` | string | no | One of: `high`, `medium`, `low`. Verdict confidence from the `pr-review` skill step 6g. Set on any action except `failure` (the schema rejects it on `failure`). |
 
 **Required fields per action:**
 

schemas/review-result.schema.json

@@ -15,6 +15,11 @@
     "repo": { "type": "string", "pattern": "^[^/]+/[^/]+$" },
     "head_sha": { "type": "string", "pattern": "^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$" },
     "body": { "type": "string", "minLength": 1 },
+    "confidence": {
+      "description": "Reviewer confidence in the verdict: how strongly the evidence and sub-agent agreement support this action. See skills/pr-review §6g. Never set when action is failure.",
+      "type": "string",
+      "enum": ["high", "medium", "low"]
+    },
     "findings": {
       "type": "array",
       "items": { "$ref": "#/$defs/finding" },
@@ -82,6 +87,10 @@
       "if": { "properties": { "action": { "const": "failure" } }, "required": ["action"] },
       "then": { "required": ["reason"] }
     },
+    {
+      "if": { "properties": { "action": { "const": "failure" } }, "required": ["action"] },
+      "then": { "not": { "required": ["confidence"] } }
+    },
     {
       "if": {
         "properties": { "action": { "const": "approve" } },

scripts/post-review-test.sh

@@ -1293,6 +1293,87 @@ run_body_test "label-actions-plus-action-hints-has-labels-section" \
 run_body_test "label-actions-plus-action-hints-has-next-steps" \
   "${LABEL_PLUS_HINTS_JSON}" "**Next steps:**"
 
+# confidence present → body gets a "**Confidence:** <value>" annotation
+CONFIDENCE_JSON='{"action":"comment","pr_number":99,"repo":"test-org/test-repo","head_sha":"abcdef0123456789abcdef0123456789abcdef01","body":"Some notes","confidence":"high"}'
+
+run_body_test "confidence-present-appends-annotation" \
+  "${CONFIDENCE_JSON}" "**Confidence:** high"
+
+# confidence absent → no annotation appended
+NO_CONFIDENCE_JSON='{"action":"comment","pr_number":99,"repo":"test-org/test-repo","head_sha":"abcdef0123456789abcdef0123456789abcdef01","body":"Some notes"}'
+
+run_body_count_test "confidence-absent-no-annotation" \
+  "${NO_CONFIDENCE_JSON}" "**Confidence:**" "0"
+
+# Confidence after a post-script verdict override: the posted value must
+# name the agent's original action, not look like confidence in `comment`.
+run_body_test_with_env() {
+  local test_name="$1"
+  local json_content="$2"
+  local expected_body_pattern="$3"
+  local extra_env="$4"
+
+  local run_dir="${TMPDIR}/run-${test_name}"
+  mkdir -p "${run_dir}/iteration-1/output"
+  echo "${json_content}" > "${run_dir}/iteration-1/output/agent-result.json"
+  : > "${GH_LOG}"
+  rm -f "${TMPDIR}/last-result.json"
+
+  local exit_code=0
+  # shellcheck disable=SC2030,SC2031
+  (
+    cd "${run_dir}"
+    export PATH="${MOCK_BIN}:${PATH}"
+    export REVIEW_TOKEN="fake-token"
+    export PR_NUMBER="99"
+    export REPO_FULL_NAME="test-org/test-repo"
+    export PR_URL="https://github.com/test-org/test-repo/pull/99"
+    export FULLSEND_FORGE="github"
+    export REVIEW_FINDING_SEVERITY_THRESHOLD="low"
+    eval "${extra_env}"
+    bash "${POST_SCRIPT}"
+  ) > "${TMPDIR}/stdout-${test_name}.log" 2>&1 || exit_code=$?
+
+  if [[ ${exit_code} -ne 0 ]]; then
+    echo "FAIL: ${test_name} — exit code ${exit_code}"
+    cat "${TMPDIR}/stdout-${test_name}.log"
+    FAILURES=$((FAILURES + 1))
+    return
+  fi
+
+  if [[ ! -f "${TMPDIR}/last-result.json" ]]; then
+    echo "FAIL: ${test_name} — no result file captured"
+    FAILURES=$((FAILURES + 1))
+    return
+  fi
+
+  local body
+  body="$(jq -r '.body' "${TMPDIR}/last-result.json")"
+  if ! echo "${body}" | grep -qF "${expected_body_pattern}"; then
+    echo "FAIL: ${test_name} — expected body pattern '${expected_body_pattern}' not found"
+    echo "Actual body:"
+    echo "${body}"
+    FAILURES=$((FAILURES + 1))
+    return
+  fi
+
+  echo "PASS: ${test_name}"
+}
+
+APPROVE_CONFIDENCE_JSON='{"action":"approve","pr_number":99,"repo":"test-org/test-repo","head_sha":"abcdef0123456789abcdef0123456789abcdef01","body":"Looks good to me","confidence":"high"}'
+
+run_body_test_with_env "confidence-protected-path-scopes-original-verdict" \
+  "${APPROVE_CONFIDENCE_JSON}" \
+  "**Confidence:** high (agent verdict: approve — downgraded by protected-path check)" \
+  'export MOCK_PR_FILES="skills/pr-review/SKILL.md"; export REVIEW_PROTECTED_PATHS="skills/"'
+
+FILTERED_CONFIDENCE_JSON='{"action":"request-changes","pr_number":99,"repo":"test-org/test-repo","head_sha":"abcdef0123456789abcdef0123456789abcdef01","body":"Please fix nits","confidence":"medium","findings":[{"severity":"low","category":"style","file":"a.go","description":"nit"}]}'
+
+run_body_test_with_env "confidence-severity-filter-scopes-original-verdict" \
+  "${FILTERED_CONFIDENCE_JSON}" \
+  "**Confidence:** medium (agent verdict: request-changes — downgraded by severity filter)" \
+  'export REVIEW_FINDING_SEVERITY_THRESHOLD="high"; export MOCK_PR_FILES="src/main.go"'
+
 # ---------------------------------------------------------------------------
 # REVIEW_PROTECTED_PATHS override tests
 # Verify that setting REVIEW_PROTECTED_PATHS overrides the default list.

scripts/post-review.sh

@@ -578,6 +578,8 @@ if jq -e '.findings' "${RESULT_FILE}" >/dev/null 2>&1; then
       if [ "${original_action}" = "request-changes" ] || [ "${original_action}" = "reject" ]; then
         echo "All findings removed by severity filter — downgrading '${original_action}' to 'comment'"
         jq 'del(.findings) | .action = "comment"' "${FILTERED_RESULT}" > "${DOWNGRADE_RESULT}"
+        CONFIDENCE_AGENT_ACTION="${original_action}"
+        CONFIDENCE_DOWNGRADE_REASON="severity filter"
       else
         jq 'del(.findings)' "${FILTERED_RESULT}" > "${DOWNGRADE_RESULT}"
       fi
@@ -590,6 +592,11 @@ fi
 
 ACTION=$(jq -r '.action' "${RESULT_FILE}")
 # ACTION retains the original value for the entire script — not re-read after protected-path downgrade.
+# CONFIDENCE_AGENT_ACTION / CONFIDENCE_DOWNGRADE_REASON record a verdict the
+# post-script overrode so the confidence annotation can name the agent's
+# original action (severity-filter sets them above; protected-path below).
+CONFIDENCE_AGENT_ACTION="${CONFIDENCE_AGENT_ACTION:-}"
+CONFIDENCE_DOWNGRADE_REASON="${CONFIDENCE_DOWNGRADE_REASON:-}"
 
 # ---------------------------------------------------------------------------
 # Protected-path check: the review agent must not approve PRs that touch
@@ -683,6 +690,8 @@ if [ "${ACTION}" = "approve" ]; then
         "${RESULT_FILE}" > "${MODIFIED_RESULT}"
       RESULT_FILE="${MODIFIED_RESULT}"
       DOWNGRADED=true
+      CONFIDENCE_AGENT_ACTION="${ACTION}"
+      CONFIDENCE_DOWNGRADE_REASON="protected-path check"
     fi
   fi
 fi
@@ -792,6 +801,25 @@ if [[ "${HAS_LABEL_ACTIONS}" == "true" ]]; then
   fi
 fi
 
+# ---------------------------------------------------------------------------
+# Append confidence annotation to body (skips failure, which has no body)
+# ---------------------------------------------------------------------------
+
+CONFIDENCE=$(jq -r '.confidence // empty' "${RESULT_FILE}")
+if [ -n "${CONFIDENCE}" ] && [ "${ACTION}" != "failure" ]; then
+  if [ -n "${CONFIDENCE_DOWNGRADE_REASON}" ]; then
+    CONFIDENCE_NOTICE=$'\n\n---\n'"**Confidence:** ${CONFIDENCE} (agent verdict: ${CONFIDENCE_AGENT_ACTION} — downgraded by ${CONFIDENCE_DOWNGRADE_REASON})"
+  else
+    CONFIDENCE_NOTICE=$'\n\n---\n'"**Confidence:** ${CONFIDENCE}"
+  fi
+  CONFIDENCE_RESULT=$(mktemp)
+  CLEANUP_FILES+=("${CONFIDENCE_RESULT}")
+  jq --arg notice "${CONFIDENCE_NOTICE}" \
+    '.body = (.body + $notice)' \
+    "${RESULT_FILE}" > "${CONFIDENCE_RESULT}"
+  RESULT_FILE="${CONFIDENCE_RESULT}"
+fi
+
 # ---------------------------------------------------------------------------
 # Append action-hints footer (request-changes only)
 # ---------------------------------------------------------------------------

scripts/post-review.src.sh

@@ -168,6 +168,8 @@ if jq -e '.findings' "${RESULT_FILE}" >/dev/null 2>&1; then
       if [ "${original_action}" = "request-changes" ] || [ "${original_action}" = "reject" ]; then
         echo "All findings removed by severity filter — downgrading '${original_action}' to 'comment'"
         jq 'del(.findings) | .action = "comment"' "${FILTERED_RESULT}" > "${DOWNGRADE_RESULT}"
+        CONFIDENCE_AGENT_ACTION="${original_action}"
+        CONFIDENCE_DOWNGRADE_REASON="severity filter"
       else
         jq 'del(.findings)' "${FILTERED_RESULT}" > "${DOWNGRADE_RESULT}"
       fi
@@ -180,6 +182,11 @@ fi
 
 ACTION=$(jq -r '.action' "${RESULT_FILE}")
 # ACTION retains the original value for the entire script — not re-read after protected-path downgrade.
+# CONFIDENCE_AGENT_ACTION / CONFIDENCE_DOWNGRADE_REASON record a verdict the
+# post-script overrode so the confidence annotation can name the agent's
+# original action (severity-filter sets them above; protected-path below).
+CONFIDENCE_AGENT_ACTION="${CONFIDENCE_AGENT_ACTION:-}"
+CONFIDENCE_DOWNGRADE_REASON="${CONFIDENCE_DOWNGRADE_REASON:-}"
 
 # ---------------------------------------------------------------------------
 # Protected-path check: the review agent must not approve PRs that touch
@@ -273,6 +280,8 @@ if [ "${ACTION}" = "approve" ]; then
         "${RESULT_FILE}" > "${MODIFIED_RESULT}"
       RESULT_FILE="${MODIFIED_RESULT}"
       DOWNGRADED=true
+      CONFIDENCE_AGENT_ACTION="${ACTION}"
+      CONFIDENCE_DOWNGRADE_REASON="protected-path check"
     fi
   fi
 fi
@@ -382,6 +391,25 @@ if [[ "${HAS_LABEL_ACTIONS}" == "true" ]]; then
   fi
 fi
 
+# ---------------------------------------------------------------------------
+# Append confidence annotation to body (skips failure, which has no body)
+# ---------------------------------------------------------------------------
+
+CONFIDENCE=$(jq -r '.confidence // empty' "${RESULT_FILE}")
+if [ -n "${CONFIDENCE}" ] && [ "${ACTION}" != "failure" ]; then
+  if [ -n "${CONFIDENCE_DOWNGRADE_REASON}" ]; then
+    CONFIDENCE_NOTICE=$'\n\n---\n'"**Confidence:** ${CONFIDENCE} (agent verdict: ${CONFIDENCE_AGENT_ACTION} — downgraded by ${CONFIDENCE_DOWNGRADE_REASON})"
+  else
+    CONFIDENCE_NOTICE=$'\n\n---\n'"**Confidence:** ${CONFIDENCE}"
+  fi
+  CONFIDENCE_RESULT=$(mktemp)
+  CLEANUP_FILES+=("${CONFIDENCE_RESULT}")
+  jq --arg notice "${CONFIDENCE_NOTICE}" \
+    '.body = (.body + $notice)' \
+    "${RESULT_FILE}" > "${CONFIDENCE_RESULT}"
+  RESULT_FILE="${CONFIDENCE_RESULT}"
+fi
+
 # ---------------------------------------------------------------------------
 # Append action-hints footer (request-changes only)
 # ---------------------------------------------------------------------------

skills/pr-review/SKILL.md

@@ -897,6 +897,14 @@ When merging
 - Combine descriptions if they add complementary detail
 - Keep the more specific remediation
 - Preserve `actionable: true` if either finding had it
+- If the merged findings disagreed on severity, attach an internal
+  `merged_from` array on the merged finding listing each input
+  severity, e.g. `merged_from: [{severity: low}, {severity: high}]`.
+  Carry this field through 6c–6f so step 6g can measure the gap.
+  Strip `merged_from` before writing `agent-result.json` — it is not
+  part of the output schema. This applies only to the same-category
+  merges here in 6b, not to the distinct-category findings preserved
+  in 6c.
 
 #### 6c. Preserve distinct-category findings
 
@@ -988,7 +996,10 @@ budget section), skip the challenger: keep the merged finding set from
      that removes all findings is unlikely — an empty result more likely
      indicates a parsing error or context truncation.
    - Otherwise, replace the challenged subset with the challenger's
-     `adjudicated_findings` (then re-append anything withheld).
+     `adjudicated_findings` (then re-append anything withheld). Copy
+     each finding's internal `merged_from` (if present) from the
+     pre-challenger finding that shares category and location — the
+     challenger is not shown that field, and 6g still needs it.
    - Log any `removed_findings` for transparency but do not include
      them in the final review.
 
@@ -1205,6 +1216,73 @@ require action, because `comment` (COMMENTED review state) does not
 block the PR. When the summary language and the verdict action
 contradict each other, escalate the verdict to match the language.
 
+#### 6g. Determine confidence level
+
+After the outcome is fixed (6f), set an optional `confidence` value
+(`high`, `medium`, or `low`) describing how strongly the evidence and
+sub-agent agreement support the verdict. Confidence is advisory: it does
+not change the action, it only annotates the verdict for the human
+reviewer and for downstream graduated-approval work (see
+[`graduated-approval-policy.md`](https://github.com/fullsend-ai/fullsend/blob/main/docs/problems/graduated-approval-policy.md)).
+Omit `confidence` entirely for the `failure` action.
+
+Confidence is two steps that must not be mixed: pick a band from
+evidence, then apply action ceilings that can only lower it.
+
+**Step 1 — evidence band.** Evaluate only the evidence conditions below,
+in order: low first, then medium, then high. Assign the first band whose
+condition holds. Do not consider the action (`comment-only`, `reject`,
+`approve`) in this step.
+
+**Low** (checked first). Assign if any of:
+
+- The challenger pass failed and you fell back to the pre-challenger
+  finding set (a `sub-agent-failure` info finding is present, see 6d).
+- A 6b merge combined findings that disagreed on severity by two or more
+  levels (read `merged_from` on the merged finding; for example
+  `{severity: low}` and `{severity: high}`), and that finding drives the
+  verdict.
+- The verdict rests on a finding the challenger downgraded, or on a
+  reconciliation (6e-1) that resolved a direct contradiction between
+  sub-agents.
+- Required PR context was missing or partial.
+
+**Medium** (checked next). Assign if no low condition holds and any of:
+
+- A 6b merge combined findings that disagreed on severity by exactly one
+  level (read `merged_from`).
+- The verdict rests on a single finding with no corroboration from a
+  second sub-agent or from the challenger.
+
+**High** (checked last). Assign only if no low or medium condition holds
+and:
+
+- No detected conflict survived synthesis: no `sub-agent-failure`
+  finding, no `merged_from` severity disagreement in any 6b merge, and no
+  reconciliation contradiction. This is *absence of detected conflict*,
+  not positive corroboration. Sub-agents that examined disjoint areas do
+  not corroborate each other, so high additionally requires that each
+  finding driving the verdict was either raised by more than one
+  sub-agent or confirmed by the challenger.
+- For an `approve` with no findings, high is appropriate when all
+  dimension sub-agents ran and returned without error.
+
+**Step 2 — action ceilings.** After step 1, apply these caps. A ceiling
+may only lower the band; it never raises it.
+
+- `comment-only`: cap at medium unless the single driving medium finding
+  was raised by more than one sub-agent AND survived the challenger
+  unchanged. Only then may the step-1 band of high stand.
+- `reject`: cap at medium unless the architectural objection was raised
+  independently by more than one sub-agent or explicitly confirmed by
+  the challenger. Only then may the step-1 band of high stand.
+
+**Provisional boundaries.** The one-level and two-level severity-gap
+splits above are provisional heuristics, not calibrated thresholds. Per
+`graduated-approval-policy.md`, confidence bands should ultimately be
+derived from observed review outcomes; treat this rubric as a starting
+point pending eval-case calibration.
+
 ### 7. Produce the review result
 
 Compose the review comment using this structure:
@@ -1281,6 +1359,10 @@ The table below lists the **additional** required fields per action:
 | failure         | `failure`         | `reason` (body optional)                                                                      |
 | reject          | `reject`          | `body`, `head_sha`, `findings[]`                                                              |
 
+`confidence` (`high`/`medium`/`low`, from step 6g) is an optional field on
+every action except `failure`. Include it when you have determined a band;
+the schema rejects it on `failure`.
+
 #### Pipeline mode (`$FULLSEND_OUTPUT_DIR` is set)
 
 Write the result to `$FULLSEND_OUTPUT_DIR/agent-result.json` following