From a22a2b382645083bab0bc4f0b8c431e1aa916050 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 16 Feb 2026 15:09:33 +0000 Subject: [PATCH 1/2] Transpose rows with columns --- .github/workflows/probe.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/probe.yml b/.github/workflows/probe.yml index ce463d5..2fb85ee 100644 --- a/.github/workflows/probe.yml +++ b/.github/workflows/probe.yml @@ -244,28 +244,24 @@ jobs: continue lines.append(f'### {title}') lines.append('') - # Header row: Server | test1 | test2 | ... - hdr = '| Server | ' + ' | '.join(f'`{short(tid)}`' for tid in cat_tests) + ' |' - sep = '|---' + ''.join('|:---:' for _ in cat_tests) + '|' + # Header row: Test | Expected | Server1 | Server2 | ... + hdr = '| Test | Expected | ' + ' | '.join(f'**{n}**' for n in names) + ' |' + sep = '|---|---' + ''.join('|:---:' for _ in names) + '|' lines.append(hdr) lines.append(sep) - # Expected row - exp_cells = [] + # One row per test for tid in cat_tests: first = lookup[names[0]][tid] - exp_cells.append(first['expected']) - lines.append('| **Expected** | ' + ' | '.join(exp_cells) + ' |') - # Server rows - for n in names: + expected = first['expected'] cells = [] - for tid in cat_tests: + for n in names: r = lookup[n].get(tid) if not r: cells.append('—') else: icon = '✅' if r['verdict'] == 'Pass' else ('⚠️' if r['verdict'] == 'Warn' else '❌') cells.append(f"{icon}`{r['got']}`") - lines.append(f"| **{n}** | " + ' | '.join(cells) + ' |') + lines.append(f"| `{short(tid)}` | {expected} | " + ' | '.join(cells) + ' |') lines.append('') lines.append(f"Commit: {commit_id[:7]}") From a8843700ef2f71ab379f0e972e4df1e7cda7d7ab Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 16 Feb 2026 15:29:52 +0000 Subject: [PATCH 2/2] Add acceptance criteria to merge a PR with framework modifications --- .github/workflows/probe-comment.yml | 2 +- .github/workflows/probe.yml | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/probe-comment.yml b/.github/workflows/probe-comment.yml index 3f504a5..a6ba7de 100644 --- a/.github/workflows/probe-comment.yml +++ b/.github/workflows/probe-comment.yml @@ -10,7 +10,7 @@ jobs: name: Post PR Comment if: > github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' + github.event.workflow_run.conclusion != 'cancelled' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/probe.yml b/.github/workflows/probe.yml index 2fb85ee..d4009e1 100644 --- a/.github/workflows/probe.yml +++ b/.github/workflows/probe.yml @@ -199,6 +199,18 @@ jobs: print('::warning::No probe results found — nothing to report') sys.exit(0) + # ── Baseline gate ───────────────────────────────────────────── + BASELINE_TESTS = {'COMP-BASELINE', 'COMP-POST-CL-BODY'} + baseline_failures = [] + for sv in server_data: + for r in sv['results']: + if r['id'] in BASELINE_TESTS and r['verdict'] == 'Fail': + baseline_failures.append(f"{sv['name']}: {r['id']} — got {r['got']}") + + if baseline_failures: + for f in baseline_failures: + print(f'::error::{f}') + # ── Write data.js ──────────────────────────────────────────── output = { 'commit': {'id': commit_id, 'message': commit_msg, 'timestamp': commit_time}, @@ -238,6 +250,17 @@ jobs: def short(tid): return re.sub(r'^(RFC\d+-[\d.]+-|COMP-|SMUG-|MAL-|NORM-)', '', tid) + # Baseline status + if baseline_failures: + lines.append('### ❌ Baseline Failed') + lines.append('') + for bf in baseline_failures: + lines.append(f'- `{bf}`') + lines.append('') + else: + lines.append('### ✅ Baseline Passed') + lines.append('') + for cat_name, title in [('Compliance', 'Compliance'), ('Smuggling', 'Smuggling'), ('MalformedInput', 'Malformed Input'), ('Normalization', 'Header Normalization')]: cat_tests = [tid for tid in test_ids if lookup[names[0]][tid]['category'] == cat_name] if not cat_tests: @@ -268,6 +291,9 @@ jobs: with open('probe-comment.md', 'w') as f: f.write('\n'.join(lines)) + + if baseline_failures: + sys.exit(1) PYEOF # ── Upload / publish ─────────────────────────────────────────── @@ -284,7 +310,7 @@ jobs: run: echo '${{ github.event.number }}' > pr-number.txt - name: Upload PR comment - if: github.event_name == 'pull_request' && steps.changes.outputs.servers != '[]' + if: always() && github.event_name == 'pull_request' && steps.changes.outputs.servers != '[]' uses: actions/upload-artifact@v4 with: name: probe-pr-comment