From ca9e2b809f91019fc96de195ce6490e2f3792637 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 07:56:28 -0700 Subject: [PATCH 01/11] gateway tests: kill the full mutants run's survivors; delete dead breaker threshold() Real gaps, each test verified to fail under its mutant by hand: - reset_before_reading (D80): each conjunct alone false keeps the request unresent. - plan_stream_usage_injection: a body not opening with an object is never planned. - ModelScanner::feed: a comma disarms a root `message` that was not an object. - escaped_at: an escaped `"type"` opening quote (odd run) is not a member. - UsageTail: explicit exact-capacity and wrap-straddling usage event test (BIL-15). Equivalent: UsageTail::push's `if rest > 0` only skipped an empty copy; removed. Dead: CircuitBreakerConfig::threshold had no callers; deleted. Five listed survivors (scan_buffered guard, root_open, escaped_at ==, push rest<0, push head*len) were already killed by existing lib tests: the run shared one CARGO_TARGET_DIR across --jobs 3, so builds raced and tested other jobs' mutants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- crates/gateway/src/circuit_breaker.rs | 9 ---- crates/gateway/src/peek.rs | 37 ++++++++++++++ crates/gateway/src/proxy.rs | 74 +++++++++++++++++++++++++-- 3 files changed, 108 insertions(+), 12 deletions(-) diff --git a/crates/gateway/src/circuit_breaker.rs b/crates/gateway/src/circuit_breaker.rs index 022432e6..269b3937 100644 --- a/crates/gateway/src/circuit_breaker.rs +++ b/crates/gateway/src/circuit_breaker.rs @@ -149,15 +149,6 @@ impl CircuitBreakerConfig { self.half_open_permits = permits; self } - - /// Get the failure threshold from the policy. - #[allow(dead_code)] - fn threshold(&self) -> u32 { - match &self.failure_policy { - FailurePolicy::Consecutive { threshold } => *threshold, - FailurePolicy::Windowed { threshold, .. } => *threshold, - } - } } /// Lock-free circuit breaker. diff --git a/crates/gateway/src/peek.rs b/crates/gateway/src/peek.rs index 301781a4..8c955f32 100644 --- a/crates/gateway/src/peek.rs +++ b/crates/gateway/src/peek.rs @@ -2037,4 +2037,41 @@ mod mutation_gaps { let model = scan_buffered(br#"{"mod\u0065l":"x","model":"m"}"#); assert!(model.duplicate_model); } + + /// Only a body whose root is an object is ever rewritten: one that merely contains an object + /// after a leading value (trailing garbage a provider rejects) gets no splice, which would + /// otherwise land inside the leading value. + /// claim: BIL-2 + #[test] + fn a_body_that_does_not_open_with_an_object_is_never_planned() { + assert_eq!(plan_stream_usage_injection(br#"{"stream":true}"#), Some(1)); + assert_eq!(plan_stream_usage_injection(br#"[]{"stream":true}"#), None); + assert_eq!(plan_stream_usage_injection(br#" 0{"stream":true}"#), None); + } + + /// The response scanner reads raw provider bytes: a root `message` whose value was not an + /// object arms no nesting once a `,` ends it, so a malformed body's bare object at the root is + /// not read as the `message` object and cannot supply the billed model. + /// claim: BIL-13 + #[test] + fn a_comma_disarms_a_root_message_that_was_not_an_object() { + let mut s = ModelScanner::for_response(); + s.feed(br#"{"message":"x",{"model":"not-billed"}}"#); + assert_eq!(s.take_model(), None); + // The armed path, for contrast. + let mut s = ModelScanner::for_response(); + s.feed(br#"{"message":{"model":"billed"}}"#); + assert_eq!(s.take_model().as_deref(), Some("billed")); + } + + /// A `"type"` whose opening quote is escaped (one backslash) sits inside a string, here a key + /// spelled `\"type`, so it is no member; three backslashes are still an odd run. + /// claim: T3 + #[test] + fn an_escaped_type_quote_is_not_a_type_member() { + let hit = |b: &[u8]| has_typed_member(b, (0, b.len()), &["image"]); + assert!(hit(br#"{"type":"image"}"#)); + assert!(!hit(br#"{"\"type":"image"}"#)); + assert!(!hit(br#"{"\\\"type":"image"}"#)); + } } diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index 5da59252..e9970689 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -173,10 +173,10 @@ impl UsageTail { } let first = (USAGE_TAIL_CAP - self.head).min(data.len()); self.buf[self.head..self.head + first].copy_from_slice(&data[..first]); + // The wrapped remainder; empty (a no-op copy) when the chunk fit before the end. No `if`: + // a guard here could only skip an empty copy, so it had no behaviour to test. let rest = data.len() - first; - if rest > 0 { - self.buf[..rest].copy_from_slice(&data[first..]); - } + self.buf[..rest].copy_from_slice(&data[first..]); self.head = (self.head + data.len()) % USAGE_TAIL_CAP; } @@ -8775,4 +8775,72 @@ mod mutation_gaps { assert_eq!(v["stream_options"]["include_usage"], true); assert_eq!(v["stream_options"]["include_obfuscation"], false); } + + /// D80 resends only when **every** condition holds: a reused connection, pingora's + /// `ReusedOnly` verdict, a read error, and a reset or abort at the root. With any one false the + /// request may have reached a server, so it is never sent twice. + /// claim: REL-1, REL-11 + #[test] + fn a_reset_before_reading_needs_every_condition() { + use std::io::{Error as Io, ErrorKind as K}; + let err = |etype: T, kind: K, reused_only: bool| { + let mut e = Error::because(etype, "upstream", Io::from(kind)); + if reused_only { + e.retry = pingora_core::RetryType::ReusedOnly; + } + e + }; + let reset = err(T::ReadError, K::ConnectionReset, true); + assert!(reset_before_reading(&reset, true)); + let aborted = err(T::ReadError, K::ConnectionAborted, true); + assert!(reset_before_reading(&aborted, true)); + // A fresh connection: the reset says nothing about an idle-close race. + assert!(!reset_before_reading(&reset, false)); + // Pingora did not judge it retryable on reuse. + let decided = err(T::ReadError, K::ConnectionReset, false); + assert!(!reset_before_reading(&decided, true)); + // Not a read: a write that met the reset may have delivered part of the request. + let write = err(T::WriteError, K::ConnectionReset, true); + assert!(!reset_before_reading(&write, true)); + // A timeout is a liveness verdict on a peer that may have taken the request. + let timed_out = err(T::ReadError, K::TimedOut, true); + assert!(!reset_before_reading(&timed_out, true)); + } + + /// The usage tail at the exact moment it becomes a ring and across its wrap: a chunk that + /// lands flush on the end (no remainder, `head` back to 0), then a final usage event that + /// straddles the end. Any slip here truncates or misorders the event billing reads. + /// claim: BIL-15 + #[test] + fn the_usage_tail_wraps_at_exactly_its_capacity() { + let stream: Vec = (0..2 * USAGE_TAIL_CAP).map(|i| (i % 253) as u8).collect(); + let mut tail = UsageTail::default(); + tail.push(&stream[..USAGE_TAIL_CAP]); + assert!( + tail.ring(), + "exactly the cap is a ring with nothing dropped" + ); + assert_eq!(tail.head, 0); + tail.push(&stream[USAGE_TAIL_CAP..USAGE_TAIL_CAP + 10]); + tail.push(&stream[USAGE_TAIL_CAP + 10..2 * USAGE_TAIL_CAP - 3]); + assert_eq!(tail.head, USAGE_TAIL_CAP - 3); + tail.push(&stream[2 * USAGE_TAIL_CAP - 3..]); + assert_eq!(tail.head, 0, "flush on the end wraps head to the front"); + assert_eq!(tail.contiguous(), &stream[USAGE_TAIL_CAP..]); + + // Outgrown (compacted to head 0), then filled to 5 bytes short of the end, so the usage + // event's first 5 bytes land at the end and the rest wrap to the front. + let mut tail = UsageTail::default(); + tail.push(&stream[..USAGE_TAIL_CAP + 5]); + tail.push(&stream[USAGE_TAIL_CAP + 5..2 * USAGE_TAIL_CAP - 5]); + assert_eq!(tail.head, USAGE_TAIL_CAP - 10); + tail.push(&stream[2 * USAGE_TAIL_CAP - 5..]); + assert_eq!(tail.head, USAGE_TAIL_CAP - 5); + let event = br#"data: {"usage":{"total_tokens":7}}"#; + tail.push(event); + assert_eq!(tail.head, event.len() - 5); + let mut whole = stream.clone(); + whole.extend_from_slice(event); + assert_eq!(tail.contiguous(), &whole[whole.len() - USAGE_TAIL_CAP..]); + } } From 51dc939a5177f822606611d30e66da5c702b059c Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 08:15:10 -0700 Subject: [PATCH 02/11] ci(deep): mutants in 32 two-stage shards (lib tests first, survivors against the full suite) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/workflows/deep.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deep.yml b/.github/workflows/deep.yml index bd42462c..f061fa11 100644 --- a/.github/workflows/deep.yml +++ b/.github/workflows/deep.yml @@ -14,13 +14,13 @@ jobs: # job stays well under GitHub's 6-hour ceiling. A surviving mutant is code no test checks: the job # fails and lists it. mutants: - name: mutants (shard ${{ matrix.shard }}/8) + name: mutants (shard ${{ matrix.shard }}/32) runs-on: ubuntu-latest timeout-minutes: 300 strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] steps: - uses: actions/checkout@v7 - uses: jdx/mise-action@v4 @@ -28,15 +28,36 @@ jobs: - uses: taiki-e/install-action@v2 with: tool: nextest,cargo-mutants + # Two stages per shard: every mutant against the fast lib unit tests, then only the + # survivors against the full suite. Stage 2's missed.txt is the real list. - run: | - cargo mutants -p beyond-ai --test-tool nextest --shard ${{ matrix.shard }}/8 \ - --file crates/gateway/src/usage.rs --file crates/gateway/src/translate.rs \ + FILES="--file crates/gateway/src/usage.rs --file crates/gateway/src/translate.rs \ --file crates/gateway/src/proxy.rs --file crates/gateway/src/peek.rs \ - --file crates/gateway/src/circuit_breaker.rs --file crates/gateway/src/route.rs \ - --jobs 2 --timeout-multiplier 2 /dev/null | sort -u > survivors.txt || true + echo "stage 1 survivors: $(wc -l < survivors.txt)" + if [ -s survivors.txt ]; then + RE=$(sed -E 's/^[^:]+:[0-9]+:[0-9]+: //' survivors.txt | sed -E 's/[][(){}.*+?^$|\\]/\\&/g' | paste -sd'|') + cargo mutants -p beyond-ai --test-tool nextest $FILES --re "$RE" \ + --jobs 2 --timeout-multiplier 3 --minimum-test-timeout 600 --output s2 /dev/null || true + [ "${rc:-0}" = 3 ] && exit 0 + exit "${rc:-0}" + fi - name: missed mutants if: failure() run: cat mutants.out/missed.txt 2>/dev/null || true + - name: upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutants-shard-${{ matrix.shard }} + path: | + s1/mutants.out/*.txt + s2/mutants.out/*.txt + if-no-files-found: ignore # The property tests (translator, usage parsers, body scanners) at 50x the PR case count. Every # property runs all 100,000 cases: PROPS_SECS (each property's wall-clock box, 60s by default) is # raised past anything a property needs, and the `deep` nextest profile lifts `ci`'s 180s From 0284803a3bfb43ffa86fb0ef7a7b1f5bb210b97d Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 11:16:48 -0700 Subject: [PATCH 03/11] ci: mutants jobs build without debug info (pass/fail only; debug builds filled disks and slowed every mutant) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/workflows/ci.yml | 4 ++++ .github/workflows/deep.yml | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71c28a2d..84aeb5da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -318,6 +318,10 @@ jobs: with: tool: nextest,cargo-mutants - name: mutate the lines this PR changed + # No debug info: mutants only need pass/fail, and each mutant rebuilds the test binaries. + env: + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_TEST_DEBUG: 0 run: | git diff "origin/${{ github.base_ref }}...HEAD" -- crates/gateway/src crates/providers/src > "$RUNNER_TEMP/pr.diff" if [ ! -s "$RUNNER_TEMP/pr.diff" ]; then echo "no gateway or catalog source changes"; exit 0; fi diff --git a/.github/workflows/deep.yml b/.github/workflows/deep.yml index f061fa11..a09c293d 100644 --- a/.github/workflows/deep.yml +++ b/.github/workflows/deep.yml @@ -28,9 +28,15 @@ jobs: - uses: taiki-e/install-action@v2 with: tool: nextest,cargo-mutants + # No debug info: mutants only need pass/fail, and each mutant rebuilds the test binaries, so + # full debug info writes several GB per mutant (it saturated the local SSD, and likely filled + # runner disks: shards were cancelled mid-run). # Two stages per shard: every mutant against the fast lib unit tests, then only the # survivors against the full suite. Stage 2's missed.txt is the real list. - - run: | + - env: + CARGO_PROFILE_DEV_DEBUG: 0 + CARGO_PROFILE_TEST_DEBUG: 0 + run: | FILES="--file crates/gateway/src/usage.rs --file crates/gateway/src/translate.rs \ --file crates/gateway/src/proxy.rs --file crates/gateway/src/peek.rs \ --file crates/gateway/src/circuit_breaker.rs --file crates/gateway/src/route.rs" From ae37afdb8839ef0a489ab20a48ad3391ccee5241 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 13:27:28 -0700 Subject: [PATCH 04/11] ci(deep): dispatch can target specific mutants shards and skip the other jobs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/workflows/deep.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deep.yml b/.github/workflows/deep.yml index a09c293d..374dea17 100644 --- a/.github/workflows/deep.yml +++ b/.github/workflows/deep.yml @@ -6,6 +6,15 @@ on: schedule: - cron: "0 7 * * 0" workflow_dispatch: + inputs: + shards: + description: "JSON list of mutants shards to run (default: all 32)" + required: false + default: "" + only_mutants: + description: "Skip the property and catalog jobs" + type: boolean + default: false env: CARGO_TERM_COLOR: always CARGO_INCREMENTAL: "0" @@ -20,7 +29,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + shard: ${{ fromJSON(inputs.shards || '[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]') }} steps: - uses: actions/checkout@v7 - uses: jdx/mise-action@v4 @@ -71,6 +80,7 @@ jobs: # kill, so a property that somehow runs away still reports how many cases it ran. properties: name: property tests (deep) + if: ${{ !inputs.only_mutants }} runs-on: ubuntu-latest timeout-minutes: 330 steps: @@ -90,6 +100,7 @@ jobs: # ahead, so the row is removed before the vendor's shutdown. catalog-retirements: name: catalog retirements (strict) + if: ${{ !inputs.only_mutants }} runs-on: ubuntu-latest timeout-minutes: 30 steps: From 46810d811218158961201b255fb24c3511b7fd87 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 14:40:45 -0700 Subject: [PATCH 05/11] ci(deep): select stage-2 mutants with --in-diff on a synthetic survivor diff --re over mutant names skipped most stage-1 survivors (26 of 41 in one shard), so stage 2 never re-tested them against the full suite. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/scripts/survivor-diff.py | 15 +++++++++++++++ .github/workflows/deep.yml | 6 ++++-- 2 files changed, 19 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/survivor-diff.py diff --git a/.github/scripts/survivor-diff.py b/.github/scripts/survivor-diff.py new file mode 100755 index 00000000..067736e2 --- /dev/null +++ b/.github/scripts/survivor-diff.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# Turn a cargo-mutants survivor list (file:line:col: desc) into a unified diff that marks exactly +# those lines as added, for `cargo mutants --in-diff`. Usage: survivor-diff.py SRC survivors.txt > d.diff +import sys, collections, difflib +src, surv = sys.argv[1], sys.argv[2] +lines = collections.defaultdict(set) +for l in open(surv): + l = l.strip() + if not l: continue + f, ln = l.split(':')[0], int(l.split(':')[1]) + lines[f].add(ln) +for f, lns in sorted(lines.items()): + new = open(f"{src}/{f}").read().splitlines(keepends=True) + old = [x for i, x in enumerate(new, 1) if i not in lns] + sys.stdout.writelines(difflib.unified_diff(old, new, f"a/{f}", f"b/{f}", n=0)) diff --git a/.github/workflows/deep.yml b/.github/workflows/deep.yml index 374dea17..8508ded3 100644 --- a/.github/workflows/deep.yml +++ b/.github/workflows/deep.yml @@ -54,8 +54,10 @@ jobs: cat s1/mutants.out/missed.txt s1/mutants.out/timeout.txt 2>/dev/null | sort -u > survivors.txt || true echo "stage 1 survivors: $(wc -l < survivors.txt)" if [ -s survivors.txt ]; then - RE=$(sed -E 's/^[^:]+:[0-9]+:[0-9]+: //' survivors.txt | sed -E 's/[][(){}.*+?^$|\\]/\\&/g' | paste -sd'|') - cargo mutants -p beyond-ai --test-tool nextest $FILES --re "$RE" \ + # Select exactly the survivors' lines with a synthetic diff: `--re` matches names + # unreliably and skipped most stage-1 survivors. + python3 .github/scripts/survivor-diff.py . survivors.txt > survivors.diff + cargo mutants -p beyond-ai --test-tool nextest --in-diff survivors.diff \ --jobs 2 --timeout-multiplier 3 --minimum-test-timeout 600 --output s2 /dev/null || true [ "${rc:-0}" = 3 ] && exit 0 From 9ba7be462fa31960bc48dabb3b3ce36fdab55dd7 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 15:49:05 -0700 Subject: [PATCH 06/11] gateway tests: kill the proxy/usage/route/breaker/peek survivors of the full mutants run 53 listed survivors (translate.rs left to a parallel pass), every kill verified by hand-applying the mutant and running the targeted test. Real gaps (new tests): - catalog_error_relay: a same-endpoint vendor's foreign error (DeepSeek) is put in the client's envelope; a sub-resource error (count_tokens) is relayed byte for byte. The D100 test had drifted off this path when grok moved to Responses. - catalog_chat_relay: an OpenAI Chat stream is a byte relay (no identity bridge). - request_filter's up-front body reservations: a declared body that fits the budget once but not twice is refused before a byte is read (headerless and header-won walks), and a buffered /{provider} body is refused before the upstream sees it. - walk_reads_body / walk_reads_tools / candidate_path_is_responses unit tests. - breaker: a generation-0 probe success closes it; Debug prints the config. - value_end: a separator is no value; disable_openrouter_compression shapes; empty model span rewrite; a cancelled full-body attempt reaps as Finished. Dead or simplified: redundant STATE_OPEN arms (breaker), splice_out's no-op guard, InputTally::run's unreachable (None, Some) arm, estimate_stream_output's event counter (deltas == 0 is the whole guard), clip_catalog_name -> floor_char_boundary, catalog_chat_relay's client check (any other client is translated anyway), request_summary's body moved to a tested summary_line. Equivalent, excluded in .cargo/mutants.toml with reasons: pack's | vs ^, the TALLY index | vs ^, close()'s flag clear, remove_items kat vs >=, request_summary delegation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .cargo/mutants.toml | 18 +++ crates/gateway/src/circuit_breaker.rs | 41 ++++++- crates/gateway/src/peek.rs | 22 +++- crates/gateway/src/proxy.rs | 110 ++++++++++++++---- crates/gateway/src/route.rs | 100 ++++++++++++++++ crates/gateway/src/usage.rs | 14 +-- crates/gateway/tests/reliability_lifecycle.rs | 73 ++++++++++++ .../gateway/tests/translate_error_envelope.rs | 59 ++++++++++ crates/gateway/tests/translate_response.rs | 31 +++++ 9 files changed, 429 insertions(+), 39 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 5635750d..23172b64 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -21,4 +21,22 @@ exclude_re = [ # The same guard's operator mutation (it is the only `>=` in `logging` today; re-check this # exclusion if another one is added there). 'replace >= with < in ::logging', + # `pack` ORs fields masked into disjoint bit ranges, where `|` and `^` are the same operation. + 'replace \| with \^ in CircuitBreaker::pack', + # `k < l`: `k` is a dropped item and `l` a kept one, so `k == l` never happens. + 'replace < with <= in remove_items', + # `state << 3 | class`: the shifted state's low three bits are zero and a class is below 8, so + # `|` and `^` agree (the TALLY index in `run` and `step`). + 'replace \| with \^ in InputTally::(run|step)', + # `close` clears COLON and ESCAPE (distinct bits, so `^` is `|`). ESCAPE is never set when it + # runs, and in valid JSON a `,` `:` `]` or `}` follows every closing quote, so `Structure` + # re-sets COLON before reading it: clearing neither (`&`) changes no count on a valid body. + 'replace \| with (&|\^) in InputTally::close', + # The drained `max_tokens` is the last member only when `max_completion_tokens` precedes it + # (both are present on this path), so the byte before it is always the separating comma. + '''replace match guard body\[comma\] == b',' with true in rename_max_tokens''', + # `model_span` and the `stream_options` value are distinct root values: `s == at` never holds. + 'replace > with >= in force_include_usage', + # A one-line delegation to `summary_line` (tested); pingora calls it only on its own error lines. + 'replace ::request_summary -> String', ] diff --git a/crates/gateway/src/circuit_breaker.rs b/crates/gateway/src/circuit_breaker.rs index 269b3937..e1b66d53 100644 --- a/crates/gateway/src/circuit_breaker.rs +++ b/crates/gateway/src/circuit_breaker.rs @@ -451,7 +451,7 @@ impl CircuitBreaker { // Consecutive, an expired window, or HALF_OPEN (a probe succeeded → close the // circuit): reset the counts and re-anchor the window. (STATE_CLOSED | STATE_HALF_OPEN, _) => Self::pack(STATE_CLOSED, 0, 0, now), - (STATE_OPEN, _) => return, // Shouldn't record success while open + // OPEN records no success; any other word is invalid. _ => return, }; @@ -533,7 +533,7 @@ impl CircuitBreaker { STATE_HALF_OPEN if Self::stale(scope, failures) => return, // Keep the generation; the next half-open bumps it. STATE_HALF_OPEN => Self::pack(STATE_OPEN, failures, 0, now), - STATE_OPEN => return, + // OPEN is already open; any other word is invalid. _ => return, }; @@ -597,7 +597,7 @@ impl CircuitBreaker { } STATE_HALF_OPEN if Self::stale(scope, failures) => return, STATE_HALF_OPEN => Self::pack(STATE_OPEN, failures, 0, now), - STATE_OPEN => return, + // OPEN is already open; any other word is invalid. _ => return, }; @@ -1704,4 +1704,39 @@ mod mutation_gaps { assert_eq!(cb.state(), CircuitState::Closed { failure_count: 0 }); assert!(cb.allow().is_ok()); } + + /// A probe generation wraps through 0 (the 14-bit field), and a probe of generation 0 still + /// closes the breaker on success: the no-op fast path is for a clean CLOSED word only. + /// claim: R6, REL-6 + #[test] + fn a_generation_zero_probe_success_closes_the_breaker() { + // Opening at t = 16383 seeds the generation at the field's max, so half-opening wraps it + // to 0. + fn clock() -> u64 { + FAILURE_MASK + } + let cb = CircuitBreaker::with_clock( + CircuitBreakerConfig::consecutive(1) + .reset_timeout(Duration::ZERO) + .half_open_permits(1), + clock, + ); + cb.record_failure(); + let probe = cb.allow().expect("half-open probe"); + assert_eq!(probe, Permit::probe(0), "the generation wrapped to 0"); + cb.record_success_for(probe); + assert_eq!(cb.state(), CircuitState::Closed { failure_count: 0 }); + } + + /// The breaker's `Debug` prints its config, not an empty struct. + /// claim: R6 + #[test] + fn debug_names_the_breaker_and_its_config() { + let cb = CircuitBreaker::new(CircuitBreakerConfig::consecutive(7)); + let s = format!("{cb:?}"); + assert!( + s.starts_with("CircuitBreaker {") && s.contains("threshold: 7"), + "{s}" + ); + } } diff --git a/crates/gateway/src/peek.rs b/crates/gateway/src/peek.rs index 8c955f32..19432b1d 100644 --- a/crates/gateway/src/peek.rs +++ b/crates/gateway/src/peek.rs @@ -869,11 +869,10 @@ fn splice_out(body: &mut Vec, cuts: &[(usize, usize)]) { }; for (k, &(_, end)) in cuts.iter().enumerate() { let next = cuts.get(k.saturating_add(1)).map_or(body.len(), |c| c.0); - // Spans are ascending and disjoint (`remove_items` builds them in order): `end <= next`. - if end < next { - body.copy_within(end..next, write); - write = write.saturating_add(next.saturating_sub(end)); - } + // Spans are ascending and disjoint (`remove_items` builds them in order): `end <= next`, + // and two touching spans copy nothing. + body.copy_within(end..next, write); + write = write.saturating_add(next.saturating_sub(end)); } body.truncate(write); } @@ -2074,4 +2073,17 @@ mod mutation_gaps { assert!(!hit(br#"{"\"type":"image"}"#)); assert!(!hit(br#"{"\\\"type":"image"}"#)); } + + /// A value cannot be empty: a separator where a value should start ends none, so `{"a":}` and + /// `{"a":,"b":1}` are malformed rather than members with an empty value. + /// claim: SEC-21 + #[test] + fn a_separator_where_a_value_starts_is_no_value() { + for b in [&b","[..], b"}", b"]", b" "] { + assert_eq!(value_end(b, 0), None, "{:?}", b); + } + assert_eq!(value_end(b"12,", 0), Some(2)); + assert_eq!(root_members(br#"{"a":}"#), None); + assert_eq!(root_members(br#"{"a":,"b":1}"#), None); + } } diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index e9970689..932a47e4 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -2785,18 +2785,21 @@ fn is_v1_models_list(session: &Session) -> bool { && (path == "/v1/models" || path == "/v1/models/") } +/// [`ProxyHttp::request_summary`]'s line: method, path without the query, and host. +fn summary_line(req: &pingora::http::RequestHeader) -> String { + let host = req + .headers + .get(http::header::HOST) + .and_then(|v| v.to_str().ok()) + .or_else(|| req.uri.host()) + .unwrap_or(""); + format!("{} {}, Host: {host}", req.method, req.uri.path()) +} + /// Cap a caller-supplied model name before echoing it in an error. The peek is 64 KiB; we do not /// want that in a JSON error body. fn clip_catalog_name(name: &str) -> &str { - const MAX: usize = 128; - if name.len() <= MAX { - return name; - } - let mut end = MAX; - while end > 0 && !name.is_char_boundary(end) { - end -= 1; - } - &name[..end] + &name[..name.floor_char_boundary(128)] } /// Resolve the provider name for a request whose first path segment matched no known/config @@ -2965,12 +2968,12 @@ fn catalog_error_relay(auto: &ModelRouting, status: Option, streaming: bool && !catalog_translating(auto) } -/// A Chat Completions client served by a Chat Completions candidate of a vendor other than OpenAI, -/// whose stream may repeat what OpenAI's sends once (see `translate::ChatIdentity`). +/// A catalog walk (not a sub-resource) served by a Chat Completions candidate of a vendor other +/// than OpenAI, whose stream may repeat what OpenAI's sends once (see `translate::ChatIdentity`). +/// It decides only for a Chat Completions client: any other is translated off that candidate +/// anyway (`catalog_translating`), which builds the same bridge. fn catalog_chat_relay(auto: &ModelRouting) -> bool { - auto.translate - .as_ref() - .is_some_and(|t| t.client == route::Endpoint::ChatCompletions) + auto.translate.is_some() && auto.candidate_at(auto.candidate).is_some_and(|c| { c.provider != providers::ProviderId::OpenAi && route::Endpoint::of_upstream_path(c.path) == route::Endpoint::ChatCompletions @@ -3256,14 +3259,7 @@ impl ProxyHttp for AiProxy { /// **with** the query, which is where a Gemini-style `?key=` credential (a virtual key or a /// BYO Google key) lives. Logged without the query. fn request_summary(&self, session: &Session, _ctx: &Self::CTX) -> String { - let req = session.req_header(); - let host = req - .headers - .get(http::header::HOST) - .and_then(|v| v.to_str().ok()) - .or_else(|| req.uri.host()) - .unwrap_or(""); - format!("{} {}, Host: {host}", req.method, req.uri.path()) + summary_line(session.req_header()) } fn allow_spawning_subrequest(&self, _session: &Session, _ctx: &Self::CTX) -> bool { @@ -8311,6 +8307,76 @@ mod tests { assert_eq!(clip_catalog_name("gpt-4o-mini"), "gpt-4o-mini"); let long = "x".repeat(200); assert_eq!(clip_catalog_name(&long).len(), 128); + // Byte 128 falls inside a two-byte character: the cut backs off to its start. + let wide = format!("{}{}", "x".repeat(127), "é".repeat(10)); + assert_eq!(clip_catalog_name(&wide), "x".repeat(127)); + } + + /// Pingora's own error lines print the request without its query, where a `?key=` credential + /// rides. + /// claim: SEC-4 + #[test] + fn the_summary_line_leaves_out_the_query() { + let mut req = pingora::http::RequestHeader::build( + http::Method::POST, + b"/v1beta/models/g:generateContent?key=secret", + None, + ) + .unwrap(); + req.insert_header("host", "gw.example").unwrap(); + assert_eq!( + summary_line(&req), + "POST /v1beta/models/g:generateContent, Host: gw.example" + ); + } + + /// The compression opt-out is spliced only just inside a non-empty root object; an empty + /// object, an array, or a body that already names `plugins` goes as sent. + /// claim: CAT-3 + /// defect: D109 + #[test] + fn compression_is_disabled_only_inside_a_non_empty_object() { + let off = |b: &[u8]| disable_openrouter_compression(b.to_vec()); + for same in [ + &b"{}"[..], + b" { } ", + b"[1]", + br#"[{"a":1}]"#, + b"", + b" ", + br#"{"plugins":[],"a":1}"#, + ] { + assert_eq!(off(same), same, "{}", String::from_utf8_lossy(same)); + } + let spliced = |pre: &str, rest: &str| { + let mut v = pre.as_bytes().to_vec(); + v.extend_from_slice(OPENROUTER_NO_COMPRESSION); + v.extend_from_slice(rest.as_bytes()); + v + }; + assert_eq!(off(br#"{"a":1}"#), spliced("{", r#""a":1}"#)); + assert_eq!(off(br#" {"a":1}"#), spliced(" {", r#""a":1}"#)); + } + + /// An empty `model` value (a zero-width span) is still rewritten. + /// claim: R1 + #[test] + fn an_empty_model_span_is_rewritten() { + let body = br#"{"model":""}"#.to_vec(); + assert_eq!( + apply_model_rewrite(body, (10, 10), b"gpt-4o"), + br#"{"model":"gpt-4o"}"# + ); + } + + /// A full-body attempt that was cancelled rather than panicked is just finished: only a + /// panic has a payload to log. + /// claim: REL-17 + #[tokio::test] + async fn a_cancelled_full_body_attempt_is_finished_not_a_panic() { + let cancelled = tokio::spawn(std::future::pending::<()>()); + cancelled.abort(); + assert_eq!(reap_attempt(cancelled, "rid").await, Reaped::Finished); } #[test] diff --git a/crates/gateway/src/route.rs b/crates/gateway/src/route.rs index 30d373ae..7afa9b61 100644 --- a/crates/gateway/src/route.rs +++ b/crates/gateway/src/route.rs @@ -1209,4 +1209,104 @@ mod tests { assert_eq!(azure.auth.header(), "api-key"); assert_eq!(azure.pool_auth[0].value.expose(), "azure-secret"); } + + fn catalog_row(model: &str) -> ModelRoute { + *providers::catalog::MODEL_ROUTES + .iter() + .find(|r| r.model == model) + .expect("catalog row") + } + + /// A header-won walk reads the body only where the body decides something. On gpt-4o-mini + /// (images, files, tools, every candidate serving structured outputs and files) it decides + /// nothing; each refusal or unserved capability alone makes it read; an embeddings row never. + /// claim: CAT-6 + #[test] + fn walk_reads_body_only_when_the_body_decides_something() { + use providers::catalog::{IN_FILE, IN_IMAGE, IN_TEXT, ModelCard, TOOLS}; + use providers::{Candidate, ProviderId}; + let base = catalog_row("gpt-4o-mini"); + assert!(!walk_reads_body(&base)); + assert!(!walk_reads_body(&catalog_row("text-embedding-3-small"))); + + const BEDROCK: &[Candidate] = &[Candidate { + provider: ProviderId::Bedrock, + upstream_model: "m", + path: "/v1/chat/completions", + }]; + const NO_FILES: &[Candidate] = &[Candidate { + provider: ProviderId::OpenRouter, + upstream_model: "x-ai/grok-build-0.1", + path: "/api/v1/chat/completions", + }]; + let card = |input, features| ModelCard { + input, + features, + ..base.card + }; + for (what, row, want) in [ + ( + "no image input", + ModelRoute { + card: card(IN_TEXT | IN_FILE, TOOLS), + ..base + }, + true, + ), + ( + "no tools", + ModelRoute { + card: card(IN_TEXT | IN_IMAGE | IN_FILE, 0), + ..base + }, + true, + ), + ( + "a candidate refusing structured outputs", + ModelRoute { + candidates: BEDROCK, + ..base + }, + true, + ), + ( + "a candidate refusing the card's file input", + ModelRoute { + candidates: NO_FILES, + ..base + }, + true, + ), + ( + "a candidate refusing files on a card without them", + ModelRoute { + card: card(IN_TEXT | IN_IMAGE, TOOLS), + candidates: NO_FILES, + ..base + }, + false, + ), + ] { + assert_eq!(walk_reads_body(&row), want, "{what}"); + } + } + + /// Only a Messages client on a Chat Completions row with a Responses arm counts tools. + /// claim: TOOL-1 + #[test] + fn walk_reads_tools_only_for_a_messages_client() { + let row = catalog_row("gpt-4o-mini"); + assert!(walk_reads_tools(&row, "/v1/messages")); + assert!(!walk_reads_tools(&row, "/v1/chat/completions")); + assert!(!walk_reads_tools(&row, "/v1/responses")); + } + + /// Session-state Responses never walks onto a candidate that is not a Responses path. + /// claim: SES-3, E3 + #[test] + fn candidate_path_is_responses_reads_the_path() { + assert!(candidate_path_is_responses("/v1/responses")); + assert!(!candidate_path_is_responses("/v1/chat/completions")); + assert!(!candidate_path_is_responses("/v1/messages")); + } } diff --git a/crates/gateway/src/usage.rs b/crates/gateway/src/usage.rs index 90d27d6b..81c92aa8 100644 --- a/crates/gateway/src/usage.rs +++ b/crates/gateway/src/usage.rs @@ -1266,12 +1266,8 @@ impl InputTally { units += u32::from(e >> 4); sa = e & 15; } - (None, Some(y)) => { - let e = quad(sb, y); - units += u32::from(e >> 4); - sb = e & 15; - } - (None, None) => break, + // The cut is past the middle, so `a` is the longer half: `qb` never outlasts `qa`. + (None, _) => break, } } for (state, rest) in [(&mut sa, qa.remainder()), (&mut sb, qb.remainder())] { @@ -1348,7 +1344,7 @@ pub fn stream_carried_error(tail: &[u8]) -> bool { /// Runs once, on the cut-short path only, so it parses each line into a `Value` rather than /// maintaining a typed view per wire. pub fn estimate_stream_output(tail: &[u8], total_bytes: u64) -> u64 { - let (mut events, mut deltas, mut text) = (0u64, 0u64, 0u64); + let (mut deltas, mut text) = (0u64, 0u64); for line in sse_lines(tail) { let Some(payload) = strip_sse_data(line) else { continue; @@ -1356,14 +1352,14 @@ pub fn estimate_stream_output(tail: &[u8], total_bytes: u64) -> u64 { let Ok(v) = serde_json::from_slice::(payload) else { continue; }; - events += 1; let n = delta_text_len(&v); if n > 0 { deltas += 1; text += n; } } - if events == 0 || tail.is_empty() { + // No delta, no text (an empty tail has neither, which keeps `sampled` nonzero below). + if deltas == 0 { return 0; } // The bytes before the tail are extrapolated at 90%: a stream opens with heavier preamble diff --git a/crates/gateway/tests/reliability_lifecycle.rs b/crates/gateway/tests/reliability_lifecycle.rs index 8d0134d6..41e1791d 100644 --- a/crates/gateway/tests/reliability_lifecycle.rs +++ b/crates/gateway/tests/reliability_lifecycle.rs @@ -718,3 +718,76 @@ async fn a_body_past_the_memory_budget_is_a_retryable_503() { let ok = send(chat(200 * 1024)).await.unwrap(); assert_eq!(ok.status().as_u16(), 200); } + +/// The up-front reservation refuses, before reading a byte of it, a body the budget can hold once +/// but not twice: a catalog walk holds a large body twice (the peek and its full-body re-run), +/// whether the body or the `x-beyond-model` header names the row. A buffered `/{provider}` body +/// is held once, and is refused before the upstream sees the request. +/// claim: SEC-19 +/// defect: D35 +#[tokio::test] +async fn a_large_body_reserves_every_copy_it_will_hold_before_it_is_read() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .config_line("max_buffered_body_bytes = 1048576") + .start() + .await; + let chat = |pad: usize| { + format!( + r#"{{"model":"gpt-4o-mini","messages":[{{"role":"user","content":"{}"}}]}}"#, + "x".repeat(pad) + ) + }; + let send = |path: &'static str, header: Option<&'static str>, body: String| { + let mut req = test_client() + .post(format!("{}{path}", gw.url())) + .header("authorization", format!("Bearer {}", vkey(&sk, 35))) + .header("content-type", "application/json"); + if let Some(model) = header { + req = req.header("x-beyond-model", model); + } + req.body(body).send() + }; + // 600 KiB fits the 1 MiB budget once, not twice. Declared and never sent: the refusal comes + // from the declared length, before a byte of the body is read. + for (what, header) in [ + ("body-named", ""), + ("header-named", "x-beyond-model: gpt-4o-mini\r\n"), + ] { + let addr = gw.url().trim_start_matches("http://").to_owned(); + let mut s = tokio::net::TcpStream::connect(&addr).await.unwrap(); + let head = format!( + "POST /v1/chat/completions HTTP/1.1\r\nhost: gw\r\nauthorization: Bearer {}\r\n\ + content-type: application/json\r\n{header}content-length: {}\r\n\r\n", + vkey(&sk, 35), + 600 * 1024 + ); + s.write_all(head.as_bytes()).await.unwrap(); + let mut buf = vec![0u8; 4096]; + let n = tokio::time::timeout(Duration::from_secs(5), s.read(&mut buf)) + .await + .unwrap_or_else(|_| panic!("{what}: no answer until the body arrives")) + .unwrap(); + let text = String::from_utf8_lossy(&buf[..n]); + assert!( + text.starts_with("HTTP/1.1 503") && text.contains("too many large request bodies"), + "{what}: {text}" + ); + } + // 1.5 MiB on `/openai/…`, buffered for `stream_options`: past the budget even once. + let resp = send("/openai/v1/chat/completions", None, chat(1536 * 1024)) + .await + .unwrap(); + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap_or_default(); + assert!( + status == 503 && text.contains("too many large request bodies"), + "provider route: status {status}: {text}" + ); + assert_eq!( + mock.hits(), + 0, + "every refusal came before an upstream attempt" + ); +} diff --git a/crates/gateway/tests/translate_error_envelope.rs b/crates/gateway/tests/translate_error_envelope.rs index 228533b5..ee191b5a 100644 --- a/crates/gateway/tests/translate_error_envelope.rs +++ b/crates/gateway/tests/translate_error_envelope.rs @@ -98,3 +98,62 @@ async fn an_error_already_in_the_clients_envelope_is_relayed_untouched() { assert_eq!(resp.status().as_u16(), 400); assert_eq!(resp.text().await.unwrap(), OPENAI_SHAPED); } + +/// The same-endpoint half of D100: a Chat client on a row whose candidate is a non-OpenAI Chat +/// Completions vendor (DeepSeek) gets that vendor's foreign error shape re-encoded in OpenAI's +/// envelope, with no translation in play. +/// claim: T6 +/// defect: D100 +#[tokio::test] +async fn a_same_endpoint_vendors_foreign_error_arrives_in_the_clients_envelope() { + let (pubkey, sk) = test_keypair(137); + let mock = MockUpstream::start(Mode::Raw(400, "application/json", XAI_400)).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["deepseek"]) + .start() + .await; + let resp = test_client() + .post(format!("{}/v1/chat/completions", gw.url())) + .header( + "authorization", + format!("Bearer {}", billing_vkey(&sk, 137)), + ) + .header("content-type", "application/json") + .body(r#"{"model":"deepseek-v4-pro","messages":[{"role":"user","content":"hi"}]}"#) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 400); + let v: Value = resp.json().await.unwrap(); + assert_eq!(v["error"]["type"], "invalid_request_error", "{v}"); + assert_eq!(v["error"]["code"], "invalid_image", "{v}"); +} + +/// A sub-resource is its provider's own API and is never re-encoded: Anthropic's error on +/// `/v1/messages/count_tokens` reaches the client byte for byte. +/// claim: T6 +#[tokio::test] +async fn a_sub_resource_error_is_relayed_untouched() { + const ANTHROPIC_400: &str = r#"{"type":"error","error":{"type":"invalid_request_error","message":"messages: field required"}}"#; + let (pubkey, sk) = test_keypair(138); + let mock = MockUpstream::start(Mode::Raw(400, "application/json", ANTHROPIC_400)).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["anthropic"]) + .start() + .await; + let resp = test_client() + .post(format!("{}/v1/messages/count_tokens", gw.url())) + .header("x-api-key", billing_vkey(&sk, 138)) + .header("anthropic-version", "2023-06-01") + .header("content-type", "application/json") + .body(r#"{"model":"claude-opus-4-8","messages":[{"role":"user","content":"hi"}]}"#) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 400); + assert_eq!(resp.text().await.unwrap(), ANTHROPIC_400); + assert_eq!( + mock.captured().expect("forwarded").path, + "/v1/messages/count_tokens" + ); +} diff --git a/crates/gateway/tests/translate_response.rs b/crates/gateway/tests/translate_response.rs index a3903556..14525532 100644 --- a/crates/gateway/tests/translate_response.rs +++ b/crates/gateway/tests/translate_response.rs @@ -363,6 +363,37 @@ async fn chat_client_on_openrouter_gets_each_identity_field_once() { ); } +/// A Chat client served by OpenAI itself is a byte relay: OpenAI's stream is the reference the +/// identity bridge repairs others toward, so nothing in it is dropped, even a field on every chunk. +/// claim: S2 +#[tokio::test] +async fn chat_client_on_openai_gets_its_stream_byte_for_byte() { + const OPENAI_SSE: &str = concat!( + r#"data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}"#, + "\n\n", + r#"data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"role":"assistant","content":"!"},"finish_reason":"stop"}]}"#, + "\n\n", + r#"data: {"id":"c1","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":[],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}"#, + "\n\ndata: [DONE]\n\n", + ); + let nats_port = unused_nats_port(); + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Raw(200, "text/event-stream", OPENAI_SSE)).await; + let gw = Gateway::builder(nats_port, &mock.authority(), &b64(&pubkey)) + .providers(&["openai", "openrouter"]) + .start() + .await; + let body = + r#"{"model":"gpt-4o-mini","stream":true,"messages":[{"role":"user","content":"hi"}]}"#; + let (status, text) = post(&gw, "/v1/chat/completions", &vkey(&sk), body.to_owned()).await; + assert_eq!(status, 200, "{text}\n{}", gw.log()); + assert_eq!(text, OPENAI_SSE); + assert_eq!( + mock.captured().expect("OpenAI served").path, + "/v1/chat/completions" + ); +} + /// A Chat client on a Claude row: `prompt_tokens` is the whole prompt, cache included, and every /// chunk carries `created` (strictly typed clients reject a chunk without it). /// claim: S2 From 2674923987e84bcedc695424fef44e9de4d46b4e Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 16:15:48 -0700 Subject: [PATCH 07/11] gateway tests: kill translate.rs's survivors of the full mutants run 92 survivors in translate.rs: 88 killed by new tests, 3 removed as dead code, 1 excluded as equivalent. No defects found. Tests (request, response/stream, and pure-table unit tests) pin: the reasoning-value table and both Claude relay rules, assistant refusals in both directions, each legacy function form alone (and the modern field winning), effort/budget tables, tool-result part mapping, strict-schema nesting, json_object both ways, custom grammar flattening, allowed_tools both ways, session-field edge inputs, the FNV-1a vectors, and Chat <-> Messages/Responses stream details (role chunk, summary separator, item_id-only calls, nameless calls, escaped quotes, DONE-only streams). Dead: message_text's Null arm (same as `_`), anthropic_system_text's String arm (only ever called on an array), and the cache_control copy in the Responses instructions helper (only its text was ever read). Equivalent: ArgsEnd::whole's memo `||` (either operator, same answer). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .cargo/mutants.toml | 5 + crates/gateway/src/translate.rs | 211 +++++-- crates/gateway/src/translate_request_tests.rs | 515 ++++++++++++++++++ .../gateway/src/translate_response_tests.rs | 302 ++++++++++ 4 files changed, 986 insertions(+), 47 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 23172b64..87195d55 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -39,4 +39,9 @@ exclude_re = [ 'replace > with >= in force_include_usage', # A one-line delegation to `summary_line` (tested); pingora calls it only on its own error lines. 'replace ::request_summary -> String', + # `ArgsEnd::whole`'s first line is a memo: `invalid` (a closed value that failed to parse, which + # no later byte can fix) and `failed_at` (this exact length already failed) each skip a re-parse + # whose answer is the same `false`. Either operator gives the same result, only slower. (It is + # the only `||` in `whole`; re-check this exclusion if another one is added there.) + 'replace \|\| with && in ArgsEnd::whole', ] diff --git a/crates/gateway/src/translate.rs b/crates/gateway/src/translate.rs index ae7703a9..be11066f 100644 --- a/crates/gateway/src/translate.rs +++ b/crates/gateway/src/translate.rs @@ -2670,7 +2670,6 @@ fn message_text(m: &Value) -> Option { .collect(); Some(t) } - Some(Value::Null) | None => None, _ => None, } } @@ -2946,7 +2945,7 @@ fn anthropic_system_to_openai(sys: &Value) -> Option { } Some(json!({ "role": "system", "content": parts })) } else { - let t = anthropic_system_text(sys)?; + let t = anthropic_system_text(blocks); (!t.is_empty()).then(|| json!({ "role": "system", "content": t })) } } @@ -2954,18 +2953,13 @@ fn anthropic_system_to_openai(sys: &Value) -> Option { } } -fn anthropic_system_text(sys: &Value) -> Option { - match sys { - Value::String(s) => Some(s.clone()), - Value::Array(blocks) => Some( - blocks - .iter() - .filter_map(|b| b.get("text").and_then(Value::as_str).or_else(|| b.as_str())) - .collect::>() - .join("\n"), - ), - _ => None, - } +/// A system prompt's text blocks, one per line. +fn anthropic_system_text(blocks: &[Value]) -> String { + blocks + .iter() + .filter_map(|b| b.get("text").and_then(Value::as_str).or_else(|| b.as_str())) + .collect::>() + .join("\n") } fn anthropic_tool_to_openai(t: &Value) -> Option { @@ -4212,12 +4206,7 @@ fn openai_req_to_responses(v: &Value, openai: OpenAiModel) -> Value { let role = m.get("role").and_then(Value::as_str).unwrap_or(""); match role { "system" | "developer" if !in_conversation => { - instructions.extend( - openai_system_to_responses_blocks(m) - .iter() - .filter_map(|b| b.get("text").and_then(Value::as_str)) - .map(str::to_owned), - ); + instructions.extend(openai_system_texts(m).map(str::to_owned)); } // Responses takes system and developer messages in `input`; a mid-conversation one // stays where the client put it. @@ -5147,34 +5136,19 @@ fn responses_part_to_openai(part: &Value) -> Option { } } -fn openai_system_to_responses_blocks(m: &Value) -> Vec { - match m.get("content") { - Some(Value::String(s)) => { - let mut b = json!({ "type": "text", "text": s }); - if let Some(obj) = b.as_object_mut() { - copy_cache_control(obj, m); - } - vec![b] - } - Some(Value::Array(parts)) => parts +/// A Chat Completions system or developer message's text parts (Responses `instructions` is one +/// string: Anthropic's `cache_control` has nowhere to go). +fn openai_system_texts(m: &Value) -> impl Iterator { + let (one, parts) = match m.get("content") { + Some(Value::String(s)) => (Some(s.as_str()), &[][..]), + Some(Value::Array(parts)) => (None, parts.as_slice()), + _ => (None, &[][..]), + }; + one.into_iter().chain( + parts .iter() - .filter_map(|p| { - let text = p - .get("text") - .and_then(Value::as_str) - .or_else(|| p.as_str())?; - let mut b = json!({ "type": "text", "text": text }); - if let Some(obj) = b.as_object_mut() { - copy_cache_control(obj, p); - if !obj.contains_key("cache_control") { - copy_cache_control(obj, m); - } - } - Some(b) - }) - .collect(), - _ => Vec::new(), - } + .filter_map(|p| p.get("text").and_then(Value::as_str).or_else(|| p.as_str())), + ) } /// A Chat Completions response for a Responses client. Items come in the order OpenAI emits @@ -10045,4 +10019,147 @@ mod mutation_gaps { assert_eq!(out["stop_reason"], native, "{finish}"); } } + + /// Which reasoning-control values ask for reasoning, each documented "off" spelling included, + /// and an object that says off in any one of its keys. + /// claim: TRN-7 + #[test] + fn reasoning_value_on_reads_every_off_spelling() { + for (v, on) in [ + (Value::Null, false), + (json!(false), false), + (json!(true), true), + (json!("none"), false), + (json!("high"), true), + (json!({}), true), + (json!({"effort": "none"}), false), + (json!({"effort": "high"}), true), + (json!({"enabled": false}), false), + (json!({"enabled": true}), true), + (json!({"type": "disabled"}), false), + (json!({"type": "enabled"}), true), + (json!({"effort": "high", "enabled": false}), false), + (json!({"effort": "high", "type": "disabled"}), false), + (json!({"enabled": true, "type": "disabled"}), false), + (json!(7), true), + ] { + assert_eq!(reasoning_value_on(&v), on, "{v}"); + } + } + + /// The hashed ids (tool ids, email `user_id`s) repeat across builds and processes only if the + /// hash is the published FNV-1a, not merely deterministic. + /// claim: TRN-23 + #[test] + fn fnv1a64_matches_the_published_vectors() { + assert_eq!(fnv1a64(b""), 0xcbf2_9ce4_8422_2325); + assert_eq!(fnv1a64(b"a"), 0xaf63_dc4c_8601_ec8c); + assert_eq!(fnv1a64(b"foobar"), 0x8594_4171_f739_67e8); + } + + /// The effort ↔ budget tables, every row and boundary, and the effort a native `thinking` + /// object stands for. + /// claim: T5 + #[test] + fn effort_and_budget_tables_hold_row_by_row() { + for (effort, budget) in [ + ("minimal", 1024), + ("low", 1024), + ("medium", 4096), + ("high", 8192), + ("xhigh", 16384), + ("max", 16384), + ("other", 4096), + ] { + assert_eq!(budget_for_effort(effort), budget, "{effort}"); + } + for (budget, effort) in [ + (0, "low"), + (1024, "low"), + (1025, "medium"), + (4096, "medium"), + (4097, "high"), + (8192, "high"), + (8193, "xhigh"), + ] { + assert_eq!(effort_from_budget(budget), effort, "{budget}"); + } + for (thinking, effort) in [ + (json!({"type": "disabled"}), Some("none")), + (json!({"type": "adaptive"}), Some("high")), + (json!({"type": "adaptive", "effort": "max"}), Some("xhigh")), + ( + json!({"type": "enabled", "budget_tokens": 2000}), + Some("medium"), + ), + (json!({"type": "enabled"}), Some("medium")), + (json!({"type": "between_tools"}), None), + ] { + assert_eq!(effort_from_thinking(&thinking), effort, "{thinking}"); + } + } + + /// A data URI is base64 inline data only with a media type and a payload. + /// claim: T3 + #[test] + fn a_data_uri_needs_both_a_media_type_and_data() { + assert_eq!( + parse_data_uri("data:image/png;base64,AAAA"), + Some(("image/png", "AAAA")) + ); + assert_eq!(parse_data_uri("data:;base64,AAAA"), None); + assert_eq!(parse_data_uri("data:image/png;base64,"), None); + assert_eq!(parse_data_uri("data:image/svg+xml;utf8,"), None); + } + + /// Tool-call arguments as a JSON string or an object become the input object; anything else + /// is `{}` (Messages' `input` must be an object). + /// claim: T1 + #[test] + fn tool_arguments_become_an_input_object() { + assert_eq!(parse_arguments(Some(&json!(r#"{"a":1}"#))), json!({"a": 1})); + assert_eq!(parse_arguments(Some(&json!({"a": 1}))), json!({"a": 1})); + assert_eq!(parse_arguments(Some(&json!([1]))), json!({})); + assert_eq!(parse_arguments(Some(&json!(3))), json!({})); + assert_eq!(parse_arguments(None), json!({})); + } + + /// OpenAI strict mode: every object, at any depth (properties, items, combinators), closes + /// `additionalProperties` and requires every property; `type: object` alone or `properties` + /// alone makes a schema an object. + /// claim: TRN-11, T4 + #[test] + fn openai_strict_schema_checks_every_nested_object() { + let strict = json!({"type": "object", "properties": {"a": {"type": "string"}}, + "required": ["a"], "additionalProperties": false}); + let open = json!({"type": "object", "properties": {"a": {"type": "string"}}}); + assert!(openai_strict_schema(&strict)); + assert!(!openai_strict_schema(&open)); + assert!(!openai_strict_schema(&json!({"type": "object"}))); + assert!(!openai_strict_schema(&json!({"properties": {}}))); + assert!(openai_strict_schema(&json!({"type": "string"}))); + for key in ["items", "anyOf", "oneOf", "allOf", "prefixItems"] { + let one = if key == "items" { + open.clone() + } else { + json!([open]) + }; + assert!( + !openai_strict_schema(&json!({"type": "array", key: one})), + "{key}" + ); + let ok = if key == "items" { + strict.clone() + } else { + json!([strict]) + }; + assert!( + openai_strict_schema(&json!({"type": "array", key: ok})), + "{key}" + ); + } + for key in ["$defs", "definitions"] { + assert!(!openai_strict_schema(&json!({key: {"x": open}})), "{key}"); + } + } } diff --git a/crates/gateway/src/translate_request_tests.rs b/crates/gateway/src/translate_request_tests.rs index 76e2c6e3..b0a40993 100644 --- a/crates/gateway/src/translate_request_tests.rs +++ b/crates/gateway/src/translate_request_tests.rs @@ -2362,3 +2362,518 @@ fn hosted_web_search_is_dropped_leaving_responses_unless_chosen() { "{v}" ); } + +// ---- gaps a cargo-mutants run found no test constraining ---------------------------------------- + +/// `stop` as one string is a one-element `stop_sequences`; `modalities: ["text"]` is the default +/// and is not forwarded (only a non-text modality is, for the provider to reject by name). +/// claim: E2 +#[test] +fn a_string_stop_and_text_only_modalities_onto_messages() { + let v = c2m( + &chat(json!({"stop": "END", "modalities": ["text"]})), + "claude-haiku-4-5", + ); + assert_eq!(v["stop_sequences"], json!(["END"]), "{v}"); + assert!(v.get("modalities").is_none(), "{v}"); +} + +/// With thinking off, an assistant message that holds nothing but its thinking keeps it: emptied, +/// it would be a 400 of its own. +/// claim: TRN-7 +/// defect: D79 +#[test] +fn an_all_thinking_assistant_message_keeps_its_thinking_when_thinking_is_off() { + let body = chat(json!({"reasoning_effort": "none", "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "", + "thinking": [{"type": "thinking", "thinking": "x", "signature": "SIG"}]}, + {"role": "user", "content": "again"}, + ]})); + let v = c2m(&body, "claude-haiku-4-5"); + assert_eq!(v["thinking"], json!({"type": "disabled"}), "{v}"); + assert_eq!( + v["messages"][1]["content"], + json!([{"type": "thinking", "thinking": "x", "signature": "SIG"}]), + "{v}" + ); +} + +/// The same-wire Claude relay applies both of its rules to one body: the reasoning control goes +/// (the tool turn has nothing signed to replay), and then, the request no longer thinking, so does +/// the unreplayable thinking the assistant message carried. +/// claim: TRN-7 +/// defect: D14, D79 +#[test] +fn the_claude_chat_relay_applies_both_reasoning_rules_to_one_body() { + let chat = json!({"model": "anthropic/claude-sonnet-4", "reasoning_effort": "high", "messages": [ + {"role": "user", "content": "weather in Paris?"}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "toolu_01", "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}}], + "reasoning_details": [{"type": "reasoning.text", "text": "unsigned", + "format": "anthropic-claude-v1", "index": 0}]}, + {"role": "tool", "tool_call_id": "toolu_01", "content": "sunny"}, + ]}); + let relayed: Value = serde_json::from_slice(&claude_chat_relay_reasoning( + serde_json::to_vec(&chat).unwrap(), + "anthropic/claude-sonnet-4", + )) + .unwrap(); + assert!(relayed.get("reasoning_effort").is_none(), "{relayed}"); + assert!( + relayed["messages"][1].get("reasoning_details").is_none(), + "{relayed}" + ); +} + +/// The `refusal` field stands in for an assistant turn's content only when the content is empty +/// (`""`, `[]` or null); beside real content it adds nothing. +/// claim: TRN-22 +#[test] +fn an_assistant_refusal_fills_only_empty_content() { + let turn = |content: Value| { + let body = chat(json!({"messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": content, "refusal": "I can't help with that."}, + {"role": "user", "content": "ok"}, + ]})); + c2m(&body, "claude-haiku-4-5")["messages"][1]["content"].clone() + }; + for empty in [json!(""), json!([]), Value::Null] { + assert_eq!(turn(empty.clone()), "I can't help with that.", "{empty}"); + } + assert_eq!(turn(json!("Sure.")), "Sure."); +} + +/// Each legacy form alone (`functions`, a `function_call` choice, a `function` result in history) +/// is rewritten; a body that also sends the modern field keeps the modern one. +/// claim: T1 +#[test] +fn each_legacy_function_form_alone_is_rewritten_and_the_modern_field_wins() { + let schema = weather_schema(); + let tool = + json!({"type": "function", "function": {"name": "get_weather", "parameters": schema}}); + + let only_functions = + chat(json!({"functions": [{"name": "get_weather", "parameters": schema}]})); + assert_eq!( + c2m(&only_functions, "claude-haiku-4-5")["tools"][0]["name"], + "get_weather" + ); + + let only_choice = chat(json!({"tools": [tool], "function_call": {"name": "get_weather"}})); + assert_eq!( + c2m(&only_choice, "claude-haiku-4-5")["tool_choice"], + json!({"type": "tool", "name": "get_weather"}) + ); + + // A history that opens on a function result (a truncated transcript): still a tool result. + let only_history = chat(json!({"messages": [ + {"role": "function", "name": "get_weather", "content": "sunny"}, + ]})); + let v = c2m(&only_history, "claude-haiku-4-5"); + assert_eq!(v["messages"][0]["content"][0]["type"], "tool_result", "{v}"); + assert_eq!( + v["messages"][0]["content"][0]["tool_use_id"], "call_legacy_0", + "{v}" + ); + + let both = chat(json!({ + "tools": [tool], + "functions": [{"name": "legacy_fn", "parameters": schema}], + "tool_choice": "required", + "function_call": {"name": "legacy_fn"}, + })); + let v = c2m(&both, "claude-haiku-4-5"); + assert_eq!(v["tools"].as_array().map(Vec::len), Some(1), "{v}"); + assert_eq!(v["tools"][0]["name"], "get_weather", "{v}"); + assert_eq!(v["tool_choice"], json!({"type": "any"}), "{v}"); +} + +/// A same-role message with empty content merges as nothing: an empty text block is a 400. +/// claim: TRN-3 +#[test] +fn an_empty_same_role_message_adds_no_block() { + let body = chat(json!({"messages": [ + {"role": "user", "content": "Hello"}, + {"role": "user", "content": ""}, + ]})); + let v = c2m(&body, "claude-haiku-4-5"); + assert_eq!(v["messages"][0]["content"], "Hello", "{v}"); +} + +/// `cache_control` set inside a tool's `function` object (where some clients put it) reaches the +/// Messages tool. (Its `ttl` tells it from the marker the gateway would add on its own.) +/// claim: TRN-4 +#[test] +fn a_function_level_cache_control_reaches_the_messages_tool() { + let body = chat(json!({"tools": [{"type": "function", "function": { + "name": "get_weather", "parameters": weather_schema(), + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + }}]})); + let v = c2m(&body, "claude-haiku-4-5"); + assert_eq!( + v["tools"][0]["cache_control"], + json!({"type": "ephemeral", "ttl": "1h"}), + "{v}" + ); +} + +/// Effort onto budget thinking, row by row of its table, and an explicit `reasoning_effort` wins +/// over a native `thinking.budget_tokens`. On a 4.7+ model `max` stays `max` (it aliases `xhigh` +/// for the table, but those models take both). +/// claim: T5 +#[test] +fn effort_maps_to_its_budget_and_wins_over_a_native_budget() { + let budget = |extra: Value| { + let v = c2m( + &chat(with(json!({"max_tokens": 64000}), extra)), + "claude-haiku-4-5", + ); + v["thinking"]["budget_tokens"].clone() + }; + assert_eq!(budget(json!({"reasoning_effort": "low"})), 1024); + assert_eq!(budget(json!({"reasoning_effort": "minimal"})), 1024); + assert_eq!(budget(json!({"reasoning_effort": "medium"})), 4096); + assert_eq!(budget(json!({"reasoning_effort": "high"})), 8192); + assert_eq!(budget(json!({"reasoning_effort": "xhigh"})), 16384); + assert_eq!(budget(json!({"reasoning_effort": "max"})), 16384); + assert_eq!( + budget(json!({"thinking": {"type": "enabled", "budget_tokens": 2000}})), + 2000 + ); + assert_eq!( + budget(json!({"reasoning_effort": "high", + "thinking": {"type": "enabled", "budget_tokens": 2000}})), + 8192 + ); + let v = c2m(&chat(json!({"reasoning_effort": "max"})), "claude-opus-4-8"); + assert_eq!(v["output_config"]["effort"], "max", "{v}"); +} + +/// A tool message's parts onto a `tool_result`: all text flattens to one string; any non-text part +/// keeps them as blocks, each mapped (a `file` becomes a `document`, a text part loses fields +/// Messages would reject), an image alone included. +/// claim: T3 +#[test] +fn tool_message_parts_map_onto_the_tool_result() { + let result = |content: Value| { + let body = chat(json!({"messages": [ + {"role": "user", "content": "go"}, + {"role": "assistant", "content": null, "tool_calls": [{"id": "call_1", + "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "call_1", "content": content}, + ]})); + c2m(&body, "claude-haiku-4-5")["messages"][2]["content"][0]["content"].clone() + }; + assert_eq!( + result(json!([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}])), + "ab" + ); + let png = "data:image/png;base64,iVBORw0KGgo="; + assert_eq!( + result(json!([{"type": "image_url", "image_url": {"url": png}}])), + json!([{"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": "iVBORw0KGgo="}}]) + ); + assert_eq!( + result(json!([ + {"type": "text", "text": "report", "annotations": []}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", + "filename": "r.pdf"}}, + ])), + json!([ + {"type": "text", "text": "report"}, + {"type": "document", "title": "r.pdf", "source": {"type": "base64", + "media_type": "application/pdf", "data": "JVBERi0="}}, + ]) + ); +} + +/// A user text part with no text is dropped, not forwarded half-formed. +/// claim: T3 +#[test] +fn a_user_text_part_without_text_is_dropped() { + let body = chat(json!({"messages": [{"role": "user", "content": [ + {"type": "text", "text": null}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + ]}]})); + let v = c2m(&body, "claude-haiku-4-5"); + assert_eq!( + v["messages"][0]["content"], + json!([{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}]), + "{v}" + ); +} + +/// `input` that is a string saying "item_reference" is a one-shot prompt, not session state; an +/// `input` array that cannot be read fails closed. +/// claim: E3 +#[test] +fn session_field_reads_item_references_only_from_a_well_formed_input_array() { + let said = br#"{"store": false, "input": "what is an item_reference?"}"#; + assert_eq!(responses_session_field(said, false), None); + let malformed = br#"{"store": false, "input": [{"type": "item_reference" "id": "msg_1"}]}"#; + assert_eq!(responses_session_field(malformed, false), Some("store")); + // An `input` that never says `item_reference` is never walked (D215), so its malformed items + // are the provider's to reject. + let unwalked = br#"{"store": false, "input": [{"type": "message" "role": "user"}]}"#; + assert_eq!(responses_session_field(unwalked, false), None); +} + +/// A body whose only reasoning-shaped key is an assistant `thinking` array is still read: with no +/// reasoning asked, a budget Claude model gets it without that thinking. +/// claim: TRN-7 +/// defect: D79 +#[test] +fn the_claude_chat_relay_reads_a_body_that_says_only_thinking() { + let chat = json!({"model": "anthropic/claude-sonnet-4", "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello", + "thinking": [{"type": "thinking", "thinking": "x", "signature": "SIG"}]}, + {"role": "user", "content": "again"}, + ]}); + let relayed: Value = serde_json::from_slice(&claude_chat_relay_reasoning( + serde_json::to_vec(&chat).unwrap(), + "anthropic/claude-sonnet-4", + )) + .unwrap(); + assert!( + relayed["messages"][1].get("thinking").is_none(), + "{relayed}" + ); +} + +/// Onto Chat: an empty system string is no system message, and `tool_choice: none` stays `none`. +/// claim: E1 +#[test] +fn an_empty_system_and_tool_choice_none_onto_chat() { + let v = m2c( + &anth(json!({"system": "", + "tools": [{"name": "get_weather", "input_schema": weather_schema()}], + "tool_choice": {"type": "none"}})), + "gpt-4o", + ); + assert_eq!(roles(&v), ["user"], "{v}"); + assert_eq!(v["tool_choice"], "none", "{v}"); +} + +/// JSON mode (`json_object`) crosses between Chat's `response_format` and Responses' +/// `text.format`, both ways. +/// claim: T4 +#[test] +fn json_mode_crosses_between_chat_and_responses() { + let v = r2c( + &json!({"model": "m", "input": "hi", "text": {"format": {"type": "json_object"}}}), + "gpt-4o", + ); + assert_eq!(v["response_format"], json!({"type": "json_object"}), "{v}"); + let v = c2r( + &chat(json!({"response_format": {"type": "json_object"}})), + "gpt-5-pro", + ); + assert_eq!(v["text"]["format"], json!({"type": "json_object"}), "{v}"); +} + +/// Onto Responses, the defaults are not forwarded (`n: 1`, `modalities: ["text"]`), and `user` is +/// hashed only past OpenAI's 64-character cap. +/// claim: E3 +#[test] +fn chat_defaults_and_a_64_character_user_onto_responses() { + let user = "u".repeat(64); + let v = c2r( + &chat(json!({"n": 1, "modalities": ["text"], "user": user})), + "gpt-5-pro", + ); + assert!(v.get("n").is_none(), "{v}"); + assert!(v.get("modalities").is_none(), "{v}"); + assert_eq!(v["user"], user.as_str(), "{v}"); +} + +/// An assistant refusal onto Responses: the `refusal` field of an empty turn, and a refusal part, +/// are `refusal` content; a refusal part in a user message is just text there. +/// claim: TRN-22 +#[test] +fn refusals_onto_responses_stay_refusals_only_from_the_assistant() { + let v = c2r( + &chat(json!({"messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": null, "refusal": "No."}, + {"role": "assistant", "content": [{"type": "refusal", "refusal": "Still no."}]}, + {"role": "user", "content": [{"type": "refusal", "refusal": "quoted"}]}, + ]})), + "gpt-5-pro", + ); + let input = &v["input"]; + assert_eq!( + input[1]["content"], + json!([{"type": "refusal", "refusal": "No."}]), + "{v}" + ); + assert_eq!( + input[2]["content"], + json!([{"type": "refusal", "refusal": "Still no."}]), + "{v}" + ); + assert_eq!( + input[3]["content"], + json!([{"type": "input_text", "text": "quoted"}]), + "{v}" + ); +} + +/// A Chat `custom` tool's nested grammar format becomes Responses' flat one; a function tool never +/// takes a format. +/// claim: TRN-14 +#[test] +fn a_custom_tool_grammar_flattens_onto_responses() { + let v = c2r( + &chat(json!({"tools": [ + {"type": "custom", "custom": {"name": "patch", "format": {"type": "grammar", + "grammar": {"syntax": "lark", "definition": "start: \"x\""}}}}, + {"type": "custom", "custom": {"name": "free", "format": null}}, + {"type": "function", "function": {"name": "f", "parameters": weather_schema(), + "format": {"type": "text"}}}, + ]})), + "gpt-5-pro", + ); + assert_eq!( + v["tools"][0], + json!({"type": "custom", "name": "patch", "format": {"type": "grammar", + "syntax": "lark", "definition": "start: \"x\""}}), + "{v}" + ); + assert_eq!( + v["tools"][1], + json!({"type": "custom", "name": "free"}), + "{v}" + ); + assert!(v["tools"][2].get("format").is_none(), "{v}"); +} + +/// A Responses `custom` tool wrapped for Messages: its description, if any, then a blank line, then +/// what `input` is; with none, the explanation alone. +/// claim: TRN-14 +#[test] +fn a_wrapped_custom_tool_description_has_no_leading_blank_line() { + let desc = |tool: Value| { + let body = json!({"model": "m", "input": "hi", "tools": [tool]}); + let v = req( + Endpoint::Responses, + Endpoint::Messages, + &body, + "claude-haiku-4-5", + ); + v["tools"][0]["description"].as_str().unwrap().to_owned() + }; + let bare = desc(json!({"type": "custom", "name": "apply_patch"})); + assert!(bare.starts_with("`input` is"), "{bare:?}"); + let described = desc(json!({"type": "custom", "name": "apply_patch", "description": "Edit."})); + assert!( + described.starts_with("Edit.\n\n`input` is"), + "{described:?}" + ); +} + +/// `allowed_tools` crosses both ways: Chat nests mode and list, Responses flattens them, and a +/// choice already in the other's shape is mapped or kept, never emptied. +/// claim: T1 +#[test] +fn allowed_tools_cross_in_both_directions() { + let tools = json!([{"type": "function", "function": {"name": "get_weather", + "parameters": weather_schema()}}]); + let nested = json!({"type": "allowed_tools", "allowed_tools": {"mode": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}]}}); + let v = c2r( + &chat(json!({"tools": tools, "tool_choice": nested})), + "gpt-5-pro", + ); + assert_eq!( + v["tool_choice"], + json!({"type": "allowed_tools", "mode": "required", + "tools": [{"type": "function", "name": "get_weather"}]}), + "{v}" + ); + // A Responses client that sent Chat's nested shape: passed as sent. + let body = json!({"model": "m", "input": "hi", "tool_choice": nested, + "tools": [{"type": "function", "name": "get_weather", "parameters": weather_schema()}]}); + assert_eq!(r2c(&body, "gpt-4o")["tool_choice"], nested); +} + +/// A Responses tool already in Chat's nested shape passes whole, even beside a top-level `name`. +/// claim: T1 +#[test] +fn a_nested_responses_tool_passes_whole() { + let tool = json!({"type": "function", "name": "get_weather", + "function": {"name": "get_weather", "parameters": weather_schema()}}); + let v = r2c( + &json!({"model": "m", "input": "hi", "tools": [tool]}), + "gpt-4o", + ); + assert_eq!(v["tools"][0], tool, "{v}"); +} + +/// A developer message stays `developer` on a native OpenAI Chat row (only other hosts get +/// `system`). +/// claim: TRN-16 +#[test] +fn a_responses_developer_message_stays_developer_on_native_openai_chat() { + let body = json!({"model": "m", "input": [ + {"role": "developer", "content": "be terse"}, + {"role": "user", "content": "hi"}, + ]}); + assert_eq!(roles(&r2c(&body, "gpt-4o")), ["developer", "user"]); +} + +/// A gateway reasoning item opens the assistant message after it, not only a tool call. +/// claim: T2, TRN-8 +#[test] +fn gateway_reasoning_before_an_assistant_message_rides_that_message() { + let body = json!({"model": "m", "store": false, "reasoning": {"effort": "high"}, "input": [ + {"role": "user", "content": "hi"}, + {"type": "reasoning", "id": "rs_gw18f2a0001", + "summary": [{"type": "summary_text", "text": "t"}], "encrypted_content": "SIG"}, + {"type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "hello"}]}, + {"role": "user", "content": "again"}, + ]}); + let v = req( + Endpoint::Responses, + Endpoint::Messages, + &body, + "claude-sonnet-4-5", + ); + assert_eq!( + v["messages"][1]["content"][0], + json!({"type": "thinking", "thinking": "t", "signature": "SIG"}), + "{v}" + ); +} + +/// OpenAI's own reasoning item is kept on a relay even when its text happens to say `rs_gw`. +/// claim: TRN-20 +#[test] +fn an_openai_reasoning_item_that_mentions_rs_gw_is_not_stripped() { + let body = br#"{"model":"gpt-5","input":[{"type":"reasoning","id":"rs_68ab","summary":[{"type":"summary_text","text":"ids like rs_gw1"}],"encrypted_content":"gAAAA"}]}"#; + assert_eq!(strip_gateway_reasoning(body.to_vec()), body); +} + +/// A signed thinking part in a Responses assistant message reaches Messages as that block. +/// claim: T2 +#[test] +fn a_thinking_part_in_a_responses_message_reaches_messages() { + let block = json!({"type": "thinking", "thinking": "t", "signature": "SIG"}); + let body = json!({"model": "m", "store": false, "reasoning": {"effort": "high"}, "input": [ + {"role": "user", "content": "hi"}, + {"type": "message", "role": "assistant", + "content": [block, {"type": "output_text", "text": "hello"}]}, + {"role": "user", "content": "again"}, + ]}); + let v = req( + Endpoint::Responses, + Endpoint::Messages, + &body, + "claude-sonnet-4-5", + ); + assert_eq!(v["messages"][1]["content"][0], block, "{v}"); +} diff --git a/crates/gateway/src/translate_response_tests.rs b/crates/gateway/src/translate_response_tests.rs index 37cf311e..bf40a349 100644 --- a/crates/gateway/src/translate_response_tests.rs +++ b/crates/gateway/src/translate_response_tests.rs @@ -2756,3 +2756,305 @@ fn an_empty_args_call_does_not_hold_parallel_calls_until_the_end() { .collect(); assert_eq!(args, r#"{"city":"Paris"}"#); } + +// ---- gaps a cargo-mutants run found no test constraining ---------------------------------------- + +/// One Chat Completions chunk as an SSE event. +fn chat_chunk(delta: Value, finish: Option<&str>) -> String { + let v = json!({"id": "chatcmpl-1", "model": "gpt-5", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]}); + format!("data: {v}\n\n") +} + +/// One Responses event, named by its `type`. +fn resp_event(v: Value) -> String { + format!("event: {}\ndata: {v}\n\n", v["type"].as_str().unwrap()) +} + +/// A Messages response onto Chat: no text is `""`, unless the turn called a tool (then `null`). +/// claim: TRN-22 +#[test] +fn an_empty_messages_turn_is_empty_content_unless_it_called_a_tool() { + let usage = json!({"input_tokens": 1, "output_tokens": 1}); + let chat = json_resp( + Messages, + Chat, + &anthropic_msg(json!([]), "end_turn", usage.clone()), + ); + assert_eq!(chat["choices"][0]["message"]["content"], "", "{chat}"); + let call = json!([{"type": "tool_use", "id": "toolu_1", "name": "f", "input": {}}]); + let chat = json_resp(Messages, Chat, &anthropic_msg(call, "tool_use", usage)); + assert!(chat["choices"][0]["message"]["content"].is_null(), "{chat}"); +} + +/// A Chat message whose content is a list of parts, a refusal part among them: text blocks on +/// Messages with a `refusal` stop, and `output_text` and `refusal` parts on Responses. +/// claim: TRN-22 +#[test] +fn chat_content_parts_and_a_refusal_part_reach_messages_and_responses() { + let message = json!({"role": "assistant", "content": [ + {"type": "text", "text": "Partly."}, + {"type": "refusal", "refusal": "Not that part."}, + ]}); + let body = chat_completion(message, "stop", json!({})); + let anth = json_resp(Chat, Messages, &body); + assert_eq!( + anth["content"], + json!([{"type": "text", "text": "Partly."}, {"type": "text", "text": "Not that part."}]) + ); + assert_eq!(anth["stop_reason"], "refusal"); + let resp = json_resp(Chat, Responses, &body); + assert_eq!( + resp["output"][0]["content"], + json!([ + {"type": "output_text", "text": "Partly.", "annotations": []}, + {"type": "refusal", "refusal": "Not that part."}, + ]), + "{resp}" + ); +} + +/// A Chat tool turn with `content: ""` has no message item on Responses, only its call. +/// claim: TRN-22 +#[test] +fn an_empty_chat_content_is_no_responses_message_item() { + let message = json!({"role": "assistant", "content": "", "tool_calls": [{"id": "call_1", + "type": "function", "function": {"name": "f", "arguments": "{}"}}]}); + let resp = json_resp( + Chat, + Responses, + &chat_completion(message, "tool_calls", json!({})), + ); + let types: Vec<&str> = resp["output"] + .as_array() + .unwrap() + .iter() + .map(|i| i["type"].as_str().unwrap()) + .collect(); + assert_eq!(types, ["function_call"], "{resp}"); +} + +/// A Responses stream onto Chat: one role chunk first, summary parts joined by a blank line, and +/// argument deltas that name their call by `item_id` alone still land on it. +/// claim: E1 +#[test] +fn a_responses_stream_onto_chat_opens_once_and_finds_calls_by_item_id() { + let src = [ + resp_event( + json!({"type": "response.created", "response": {"id": "resp_1", + "created_at": 1_790_000_000, "model": "gpt-5", "output": []}}), + ), + resp_event( + json!({"type": "response.output_item.added", "output_index": 0, + "item": {"id": "rs_1", "type": "reasoning", "summary": []}}), + ), + resp_event( + json!({"type": "response.reasoning_summary_part.added", "item_id": "rs_1", + "output_index": 0, "summary_index": 0}), + ), + resp_event( + json!({"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", + "output_index": 0, "summary_index": 0, "delta": "a"}), + ), + resp_event( + json!({"type": "response.reasoning_summary_part.added", "item_id": "rs_1", + "output_index": 0, "summary_index": 1}), + ), + resp_event( + json!({"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", + "output_index": 0, "summary_index": 1, "delta": "b"}), + ), + resp_event( + json!({"type": "response.output_item.added", "output_index": 1, + "item": {"id": "fc_1", "type": "function_call", "call_id": "call_1", + "name": "get_weather", "arguments": ""}}), + ), + resp_event( + json!({"type": "response.function_call_arguments.delta", "item_id": "fc_1", + "delta": "{\"city\":"}), + ), + resp_event( + json!({"type": "response.function_call_arguments.delta", "item_id": "fc_1", + "delta": "\"Paris\"}"}), + ), + resp_event( + json!({"type": "response.completed", "response": {"id": "resp_1", + "status": "completed", "output": [], "usage": {"input_tokens": 1, + "output_tokens": 1}}}), + ), + ] + .concat(); + let evs = stream(Responses, Chat, &src); + let chunks = chunks(&evs); + let roles: Vec = chunks + .iter() + .enumerate() + .filter(|(_, c)| c.pointer("/choices/0/delta/role").is_some()) + .map(|(i, _)| i) + .collect(); + assert_eq!(roles, [0], "{evs:#?}"); + let reasoning: String = chunks + .iter() + .filter_map(|c| c.pointer("/choices/0/delta/reasoning_content")?.as_str()) + .collect(); + assert_eq!(reasoning, "a\n\nb"); + assert_eq!( + chat_tool_calls(&evs), + [( + "call_1".to_owned(), + "get_weather".to_owned(), + r#"{"city":"Paris"}"#.to_owned() + )] + ); +} + +/// A streamed call opens once its name is known, never nameless; one that never names itself +/// still shows at the end; and `finish_reason: "stop"` beside a call is a `tool_use` stop. +/// claim: T1, TRN-12 +#[test] +fn a_chat_stream_call_opens_named_and_stops_as_tool_use() { + let src = [ + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"arguments": ""}}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 0, "function": {"name": "get_weather"}}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 0, + "function": {"arguments": "{\"city\":\"Paris\"}"}}]}), + None, + ), + // A delta with nothing new is no event. + chat_chunk( + json!({"tool_calls": [{"index": 0, "function": {"arguments": ""}}]}), + Some("stop"), + ), + "data: [DONE]\n\n".to_owned(), + ] + .concat(); + let evs = stream(Chat, Messages, &src); + assert_eq!( + anthropic_content(&evs), + [ + json!({"type": "tool_use", "id": "call_1", "name": "get_weather", + "input": {"city": "Paris"}}) + ] + ); + assert_eq!(named(&evs, "content_block_delta").len(), 1, "{evs:#?}"); + assert_eq!(stop_reason(&evs), "tool_use"); + + // A call that never names itself, then text: the text block closes before the call opens at + // the end (Messages blocks are sequential). + let src = [ + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", + "function": {"arguments": "{}"}}]}), + None, + ), + chat_chunk(json!({"content": "hi"}), Some("tool_calls")), + "data: [DONE]\n\n".to_owned(), + ] + .concat(); + let evs = stream(Chat, Messages, &src); + assert_eq!( + anthropic_content(&evs), + [ + json!({"type": "text", "text": "hi"}), + json!({"type": "tool_use", "id": "call_1", "name": "", "input": {}}), + ] + ); + let blocks: Vec<(&str, u64)> = evs + .iter() + .filter(|(n, _)| n == "content_block_start" || n == "content_block_stop") + .map(|(n, v)| (n.as_str(), v["index"].as_u64().unwrap())) + .collect(); + assert_eq!( + blocks, + [ + ("content_block_start", 0), + ("content_block_stop", 0), + ("content_block_start", 1), + ("content_block_stop", 1), + ] + ); +} + +/// An upstream stream that is only `[DONE]` still opens with `message_start` on Messages. +/// claim: E2 +#[test] +fn a_done_only_chat_stream_still_starts_the_message() { + let evs = stream(Chat, Messages, "data: [DONE]\n\n"); + let names: Vec<&str> = evs.iter().map(|(n, _)| n.as_str()).collect(); + assert_eq!(names, ["message_start", "message_delta", "message_stop"]); +} + +/// A live call whose arguments hold an escaped quote is seen to end where its JSON does, so the +/// next parallel call streams live rather than waiting, whole, for the end of the stream. +/// claim: TRN-12 +#[test] +fn an_escaped_quote_does_not_hide_where_a_call_ends() { + let src = [ + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": "say", "arguments": r#"{"q":"a\"b"}"#}}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 1, "id": "call_2", "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":"}}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 1, "function": {"arguments": "\"Paris\"}"}}]}), + Some("tool_calls"), + ), + "data: [DONE]\n\n".to_owned(), + ] + .concat(); + let evs = stream(Chat, Messages, &src); + let second: Vec<&Value> = named(&evs, "content_block_delta") + .into_iter() + .filter(|v| v["index"] == 1) + .collect(); + assert_eq!(second.len(), 2, "{evs:#?}"); + assert_eq!(anthropic_content(&evs)[0]["input"]["q"], r#"a"b"#); +} + +/// Onto Responses: a custom call whose deltas carry `custom` but no `type` is still a +/// `custom_tool_call`, and an OpenRouter `reasoning.summary` detail streams as summary text. +/// claim: TRN-14, T2 +#[test] +fn a_typeless_custom_call_and_a_reasoning_summary_reach_responses() { + let src = [ + chat_chunk( + json!({"reasoning_details": [{"type": "reasoning.summary", "summary": "plan", + "index": 0}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", + "custom": {"name": "apply_patch", "input": "*** Begin"}}]}), + Some("tool_calls"), + ), + "data: [DONE]\n\n".to_owned(), + ] + .concat(); + let evs = stream(Chat, Responses, &src); + let summary: String = named(&evs, "response.reasoning_summary_text.delta") + .iter() + .filter_map(|v| v["delta"].as_str()) + .collect(); + assert_eq!(summary, "plan", "{evs:#?}"); + let done = one(&evs, "response.completed"); + let types: Vec<&str> = done["response"]["output"] + .as_array() + .unwrap() + .iter() + .map(|i| i["type"].as_str().unwrap()) + .collect(); + assert_eq!(types, ["reasoning", "custom_tool_call"], "{done}"); +} From fae8cb8d4ebbcd780ae63c14f2fbefc0b0513ba6 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 18:32:50 -0700 Subject: [PATCH 08/11] ci(deep): stage 2 keeps --file and adds an exact anchored --re Without --file, --in-diff falls back to mutants.toml's examine_globs and skips route.rs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/workflows/deep.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deep.yml b/.github/workflows/deep.yml index 8508ded3..30e78b5d 100644 --- a/.github/workflows/deep.yml +++ b/.github/workflows/deep.yml @@ -54,10 +54,12 @@ jobs: cat s1/mutants.out/missed.txt s1/mutants.out/timeout.txt 2>/dev/null | sort -u > survivors.txt || true echo "stage 1 survivors: $(wc -l < survivors.txt)" if [ -s survivors.txt ]; then - # Select exactly the survivors' lines with a synthetic diff: `--re` matches names - # unreliably and skipped most stage-1 survivors. + # Select exactly the survivors: `--in-diff` narrows to their lines, an anchored `--re` to + # their full names (a description alone matches the same text elsewhere). Keep $FILES: + # without it the toml's examine_globs apply and route.rs is skipped. python3 .github/scripts/survivor-diff.py . survivors.txt > survivors.diff - cargo mutants -p beyond-ai --test-tool nextest --in-diff survivors.diff \ + RE=$(python3 -c "import re,sys; print('|'.join('^'+re.escape(l.strip())+'\$' for l in open(sys.argv[1]) if l.strip()))" survivors.txt) + cargo mutants -p beyond-ai --test-tool nextest $FILES --in-diff survivors.diff --re "$RE" \ --jobs 2 --timeout-multiplier 3 --minimum-test-timeout 600 --output s2 /dev/null || true [ "${rc:-0}" = 3 ] && exit 0 From c0153000f3aa6f9001729788e5e643f019e5bd82 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 19:37:07 -0700 Subject: [PATCH 09/11] gateway tests: kill translate.rs's second-pass mutants survivors 84 survivors of the full mutants run in translate.rs: 67 killed by tests (5 already by existing tests, 2 more by existing tests once the key packing was factored out), 10 removed as dead or equivalent code, 7 excluded as equivalent. No defects found. New tests pin: both bridge overflow bounds one byte past their size (and MAX_STREAM_OUTPUT against the catalog), the 8 MiB thinking cap in HeldThinking and Gather, Gather::close, the raw-JSON keep rules and negative integers in KeepInputs, looks_like_error / in_client_envelope / status_error_type / anthropic_error_type tables, clip at a char boundary, error_info's message sources and the overflow code, the plain-text document mapping, Responses body created/text/content-null rules, OpenRouter reasoning.summary onto Responses, interrupted thinking never folded into a later signature, an unterminated last event, byte-exact relay role cuts (LF, CRLF, unterminated), a repeated call id without a role, nothing after message_stop onto Chat, every Messages stream event shape onto Chat, and Responses refusal deltas. Dead: map_response's and request's `a == b` arms (same as `_`), KeepInputs' visit_string/visit_none/visit_some (serde_json's deserialize_any never calls them), error_info's `is_object` guard on a parsed raw error, chat_finish's tool_calls arm (responses_status reads only length and content_filter), Usage::from_chat's `r > 0` (adds 0), and flush's Responses `!errored` guard (error marks it completed). ChatIdentity's key packing is one fn now. Excluded: KeepInputs expecting/visit_unit, `|` vs `^` in ChatIdentity::key, and assembled's clone-skipping guard. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .cargo/mutants.toml | 7 + crates/gateway/src/translate.rs | 291 ++++++++++++++++-- crates/gateway/src/translate_request_tests.rs | 13 + .../gateway/src/translate_response_tests.rs | 245 +++++++++++++++ 4 files changed, 531 insertions(+), 25 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 87195d55..c8989235 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -44,4 +44,11 @@ exclude_re = [ # whose answer is the same `false`. Either operator gives the same result, only slower. (It is # the only `||` in `whole`; re-check this exclusion if another one is added there.) 'replace \|\| with && in ArgsEnd::whole', + # `KeepInputs::expecting` only words a parse error's message, which `parse_body` discards (it + # keeps `None`); `visit_unit`'s `Value::Null` is `Value::default()`. + "replace for KeepInputs>::(expecting|visit_unit)", + # `ChatIdentity::key` packs three masked fields into disjoint bit ranges: `|` and `^` agree. + 'replace \| with \^ in ChatIdentity::key', + # `map_response` clones a same-wire body as is: the guard only skips that clone. + 'replace match guard client == Endpoint::Responses with false in SseBridge::assembled', ] diff --git a/crates/gateway/src/translate.rs b/crates/gateway/src/translate.rs index be11066f..8c0bbb40 100644 --- a/crates/gateway/src/translate.rs +++ b/crates/gateway/src/translate.rs @@ -376,7 +376,6 @@ fn map_request(from: Endpoint, to: Endpoint, v: &Value, up: Upstream) -> (Value, (Endpoint::Messages, Endpoint::Responses) => { openai_req_to_responses(&anthropic_req_to_openai(v, up), up.openai) } - (a, b) if a == b => v.clone(), _ => v.clone(), }; (out, false) @@ -454,7 +453,6 @@ fn map_response( (Endpoint::Responses, Endpoint::Messages) => { openai_resp_to_anthropic(&responses_resp_to_openai(v)) } - (a, b) if a == b => v.clone(), _ => v.clone(), } } @@ -574,6 +572,8 @@ impl<'de> serde::de::DeserializeSeed<'de> for KeepInputs { } } +/// Only the methods serde_json's `deserialize_any` calls: a string comes through `visit_str` +/// (borrowed or copied, never `visit_string`'s owned form), and JSON has no `Option` to visit. impl<'de> serde::de::Visitor<'de> for KeepInputs { type Value = Value; @@ -601,22 +601,10 @@ impl<'de> serde::de::Visitor<'de> for KeepInputs { Ok(Value::String(s.to_owned())) } - fn visit_string(self, s: String) -> Result { - Ok(Value::String(s)) - } - fn visit_unit(self) -> Result { Ok(Value::Null) } - fn visit_none(self) -> Result { - Ok(Value::Null) - } - - fn visit_some>(self, d: D) -> Result { - d.deserialize_any(self) - } - fn visit_seq>(self, mut seq: A) -> Result { let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); while let Some(v) = seq.next_element_seed(KeepInputs)? { @@ -753,8 +741,10 @@ fn error_info(v: &Value) -> ErrorInfo { if let Some(meta) = metadata.as_ref() { let provider = meta.get("provider_name").and_then(Value::as_str); let inner = match meta.get("raw") { + // JSON that is not an object names no type or message, so it is quoted whole, as text + // that is not JSON is. Some(Value::String(raw)) => match serde_json::from_str::(raw) { - Ok(parsed) if parsed.is_object() => { + Ok(parsed) => { if typ.is_none() { typ = parsed .get("error") @@ -765,7 +755,7 @@ fn error_info(v: &Value) -> ErrorInfo { } plain_error_message(&parsed).or_else(|| Some(clip(raw))) } - _ => Some(clip(raw)), + Err(_) => Some(clip(raw)), }, Some(raw @ Value::Object(_)) => plain_error_message(raw), _ => None, @@ -3384,11 +3374,11 @@ fn finish_anthropic_stop(stop: &'static str, refused: bool, called: bool) -> &'s } } -/// A Chat `finish_reason` kept past its chunk. +/// A Chat `finish_reason` kept past its chunk, as far as [`responses_status`] tells reasons apart +/// (a tool call's `tool_calls` is a completed response, like `stop`). fn chat_finish(reason: &str) -> &'static str { match reason { "length" => "length", - "tool_calls" | "function_call" => "tool_calls", "content_filter" => "content_filter", _ => "stop", } @@ -3478,9 +3468,8 @@ impl Usage { // must be shown the output it is billed. let r = reasoning.unwrap_or(0); // Saturating: provider numbers are untrusted and overflow-checks are on (D87). - let outside = r > 0 - && u64_at(u, "/total_tokens") - == Some(prompt.saturating_add(completion).saturating_add(r)); + let outside = + u64_at(u, "/total_tokens") == Some(prompt.saturating_add(completion).saturating_add(r)); Self { uncached: prompt .saturating_sub(cache_read) @@ -5598,10 +5587,15 @@ impl ChatIdentity { dropped } + /// (choice, entry, kind) packed into disjoint bit ranges: 24 bits, 32 bits, 8 bits. + fn key(choice: u64, entry: u64, kind: u64) -> u64 { + (choice & 0xff_ffff) << 40 | (entry & 0xffff_ffff) << 8 | kind + } + /// The packed key for (choice, entry, kind), when neither `sent` nor `pending` holds it yet: /// the field is new and should be kept (and recorded). `None` for a repeat. fn seen(sent: &[u64], pending: &[u64], choice: u64, entry: u64, kind: u64) -> Option { - let key = (choice & 0xff_ffff) << 40 | (entry & 0xffff_ffff) << 8 | kind; + let key = Self::key(choice, entry, kind); (!sent.contains(&key) && !pending.contains(&key)).then_some(key) } @@ -5617,7 +5611,7 @@ impl ChatIdentity { if !obj.get(field).is_some_and(Value::is_string) { return false; } - let key = (choice & 0xff_ffff) << 40 | (entry & 0xffff_ffff) << 8 | kind; + let key = Self::key(choice, entry, kind); if self.sent.contains(&key) { obj.remove(field); return true; @@ -6030,9 +6024,10 @@ impl SseBridge { } } Endpoint::Messages if !self.errored => self.oai_to_ant.finish(&mut out), - Endpoint::Responses if !self.errored => self.oai_to_resp.finish(&mut out), + // After an error this is a no-op: `OaiToResp::error` marked the response completed. + Endpoint::Responses => self.oai_to_resp.finish(&mut out), // Embeddings never stream and never translate. - Endpoint::Messages | Endpoint::Responses | Endpoint::Embeddings => {} + Endpoint::Messages | Endpoint::Embeddings => {} } out } @@ -10162,4 +10157,250 @@ mod mutation_gaps { assert!(!openai_strict_schema(&json!({key: {"x": open}})), "{key}"); } } + + /// Each bound is crossed one byte past its documented size, not at it: an unterminated event + /// at 32 MiB, and gathered output at the catalog's largest answer times the widest token. + /// claim: SEC-19 + #[test] + fn a_bridge_overflows_one_byte_past_each_bound() { + let largest = providers::catalog::MODEL_ROUTES + .iter() + .map(|r| r.card.max_output_tokens as usize) + .max() + .unwrap(); + assert_eq!(MAX_STREAM_OUTPUT, largest * 128); + let mut b = SseBridge::new(Endpoint::Messages, Endpoint::ChatCompletions); + b.buf = vec![b' '; 32 << 20]; + assert_eq!(b.overflow(), None); + b.buf.push(b' '); + assert_eq!(b.overflow(), Some("sse_event")); + b.buf = Vec::new(); + b.oai_to_resp.held = MAX_STREAM_OUTPUT; + assert_eq!(b.overflow(), None); + b.oai_to_resp.held += 1; + assert_eq!(b.overflow(), Some("sse_output")); + } + + /// A thinking block is held whole up to 8 MiB and dropped one byte past it, in both the + /// Messages-stream holder and the OpenRouter gatherer. + /// claim: T2 + #[test] + fn thinking_is_held_up_to_8_mib_and_dropped_past_it() { + let cap = 8 << 20; + let at_cap = "a".repeat(cap); + let mut held = HeldThinking::default(); + held.push(&at_cap[..cap - 1]); + held.push("a"); + assert!(!held.overflow); + assert_eq!(held.text.len(), cap); + held.push("a"); + assert!(held.overflow && held.text.is_empty()); + let mut held = HeldThinking::default(); + held.push(&at_cap); + held.push("a"); + assert!(held.overflow, "one push past the cap"); + + let mut done = Vec::new(); + let mut g = Gather::default(); + g.text(None, &at_cap, &mut done); + g.sign("S", &mut done); + assert_eq!(done.len(), 1); + assert_eq!(done[0]["thinking"].as_str().map(str::len), Some(cap)); + let mut done = Vec::new(); + let mut g = Gather::default(); + g.text(None, &at_cap, &mut done); + g.text(None, "a", &mut done); + g.sign("S", &mut done); + g.text(None, &at_cap, &mut done); + g.text(None, "a", &mut done); + g.close(&mut done); + assert!(done.is_empty(), "{done:?}"); + } + + /// An unsigned gathered block closes as a `thinking` block; with nothing gathered, closing + /// adds nothing. + /// claim: T2 + #[test] + fn closing_a_gather_emits_only_a_block_it_holds() { + let mut done = Vec::new(); + Gather::default().close(&mut done); + assert!(done.is_empty()); + let mut g = Gather::default(); + g.text(Some(0), "hm", &mut done); + g.close(&mut done); + assert_eq!(done, [json!({"type": "thinking", "thinking": "hm"})]); + } + + /// Only a number a `Value` keeps as an `f64`, at any depth, makes tool input keep its text; an + /// object stands for kept text only when that is its one member. + /// claim: T1 + #[test] + fn only_a_float_keeps_tool_input_text() { + assert!(has_float(&json!([1.5]))); + assert!(has_float(&json!({"a": [1, {"b": [2.5]}]}))); + assert!(!has_float(&json!({"a": [1, -2, "1.5"]}))); + assert_eq!(exact_value(r#"{"a": 1}"#, json!({"a": 1})), json!({"a": 1})); + let kept = exact_value(r#"{"a": [1.10]}"#, json!({"a": [1.1]})); + assert_eq!(raw_json(&kept), Some(r#"{"a": [1.10]}"#)); + let mut two = kept.as_object().unwrap().clone(); + two.insert("b".into(), json!(2)); + assert_eq!(raw_json(&Value::Object(two)), None); + // Negative integers survive the input-keeping parse as integers. + let v = parse_body( + Endpoint::Messages, + br#"{"temperature": -1, "input": {"n": -5}}"#, + ) + .unwrap(); + assert_eq!(v, json!({"temperature": -1, "input": {"n": -5}})); + } + + /// What is and is not an upstream error body. + /// claim: T6 + #[test] + fn looks_like_error_needs_the_whole_signal() { + for (v, err) in [ + // A Responses object is an error only when it failed *and* carries the error. + ( + json!({"object": "response", "status": "failed", "error": {"code": "x"}}), + true, + ), + ( + json!({"object": "response", "status": "failed", "error": null}), + false, + ), + ( + json!({"object": "response", "status": "completed", "error": {"code": "x"}}), + false, + ), + // A string `error` beside a success field is not an error. + (json!({"error": "warn", "choices": []}), false), + (json!({"error": "warn", "content": []}), false), + (json!({"error": "warn", "output": []}), false), + (json!({"error": "warn", "type": "message"}), false), + (json!({"error": "boom"}), true), + (json!({"error": {"message": "boom"}, "choices": []}), true), + ] { + assert_eq!(looks_like_error(&v), err, "{v}"); + } + } + + /// Anthropic's error envelope needs `type: "error"` and both an error `type` and `message`. + /// claim: T6 + #[test] + fn the_messages_envelope_needs_every_field() { + let full = json!({"type": "error", "error": {"type": "x", "message": "m"}}); + assert!(in_client_envelope(&full, Endpoint::Messages)); + for v in [ + json!({"error": {"type": "x", "message": "m"}}), + json!({"type": "error", "error": {"message": "m"}}), + json!({"type": "error", "error": {"type": "x"}}), + ] { + assert!(!in_client_envelope(&v, Endpoint::Messages), "{v}"); + } + assert!(in_client_envelope( + &json!({"error": {"message": "m"}}), + Endpoint::ChatCompletions + )); + } + + /// The error type a status means when the body names none, row by row. + /// claim: T6 + #[test] + fn status_error_types_hold_row_by_row() { + use Endpoint::{ChatCompletions as Chat, Messages}; + for (client, status, typ) in [ + (Messages, 400, "invalid_request_error"), + (Messages, 401, "authentication_error"), + (Messages, 402, "billing_error"), + (Messages, 403, "permission_error"), + (Messages, 404, "not_found_error"), + (Messages, 413, "request_too_large"), + (Messages, 429, "rate_limit_error"), + (Messages, 500, "api_error"), + (Messages, 504, "timeout_error"), + (Messages, 529, "overloaded_error"), + (Chat, 401, "invalid_request_error"), + (Chat, 429, "rate_limit_error"), + (Chat, 504, "api_error"), + ] { + assert_eq!( + status_error_type(status, client), + typ, + "{client:?} {status}" + ); + } + } + + /// OpenAI error types and codes onto Anthropic's closed set, row by row. + /// claim: T6 + #[test] + fn anthropic_error_types_hold_row_by_row() { + for (typ, code, want) in [ + ("server_error", None, "api_error"), + ("insufficient_quota", None, "billing_error"), + ( + "invalid_request_error", + Some("insufficient_quota"), + "billing_error", + ), + ("requests", None, "rate_limit_error"), + ("x", Some("rate_limit_exceeded"), "rate_limit_error"), + ( + "invalid_request_error", + Some("invalid_api_key"), + "authentication_error", + ), + ( + "invalid_request_error", + Some("model_not_found"), + "not_found_error", + ), + ( + "invalid_request_error", + Some("other"), + "invalid_request_error", + ), + ] { + let code = code.map(|c| json!(c)); + assert_eq!( + anthropic_error_type(typ, code.as_ref()), + want, + "{typ} {code:?}" + ); + } + } + + /// An error message past 4096 bytes is cut at the last character boundary at or before it. + /// claim: T6 + #[test] + fn clip_cuts_at_a_character_boundary() { + assert_eq!(clip(&"a".repeat(4096)), "a".repeat(4096)); + assert_eq!(clip(&"a".repeat(5000)), format!("{}…", "a".repeat(4096))); + let wide = format!("{}é{}", "a".repeat(4095), "b".repeat(10)); + assert_eq!(clip(&wide), format!("{}…", "a".repeat(4095))); + } + + /// An error body with no message anywhere is quoted whole (a choice that finished with an + /// error says so instead); OpenRouter's provider error as an object is quoted after its + /// wrapper's message; and a message that does not say overflow gets no overflow code. + /// claim: T6, TRN-18 + #[test] + fn error_messages_come_from_wherever_the_body_keeps_them() { + let chat = Endpoint::ChatCompletions; + let out = map_error(&json!({"foo": "bar"}), chat, 500); + assert_eq!(out["error"]["message"], r#"{"foo":"bar"}"#, "{out}"); + let ended = json!({"choices": [{"finish_reason": "error"}]}); + assert_eq!( + map_error(&ended, chat, 200)["error"]["message"], + "The upstream ended the response with an error." + ); + let wrapped = json!({"error": {"message": "Provider returned error", "code": 400, + "metadata": {"provider_name": "Acme", "raw": {"error": {"message": "inner"}}}}}); + assert_eq!( + map_error(&wrapped, chat, 400)["error"]["message"], + "Provider returned error (Acme): inner" + ); + let plain = map_error(&json!({"error": {"message": "bad"}}), chat, 400); + assert_eq!(plain["error"]["code"], Value::Null, "{plain}"); + } } diff --git a/crates/gateway/src/translate_request_tests.rs b/crates/gateway/src/translate_request_tests.rs index b0a40993..8039c538 100644 --- a/crates/gateway/src/translate_request_tests.rs +++ b/crates/gateway/src/translate_request_tests.rs @@ -2877,3 +2877,16 @@ fn a_thinking_part_in_a_responses_message_reaches_messages() { ); assert_eq!(v["messages"][1]["content"][0], block, "{v}"); } + +/// A plain-text document onto Chat Completions is its text (alone, the content string). +/// claim: T3 +#[test] +fn a_plain_text_document_becomes_a_text_part() { + let doc = json!({"type": "document", + "source": {"type": "text", "media_type": "text/plain", "data": "the notes"}}); + let v = m2c( + &anth(json!({"messages": [{"role": "user", "content": [doc]}]})), + "gpt-5", + ); + assert_eq!(v["messages"][0]["content"], "the notes", "{v}"); +} diff --git a/crates/gateway/src/translate_response_tests.rs b/crates/gateway/src/translate_response_tests.rs index bf40a349..ee1acebd 100644 --- a/crates/gateway/src/translate_response_tests.rs +++ b/crates/gateway/src/translate_response_tests.rs @@ -3058,3 +3058,248 @@ fn a_typeless_custom_call_and_a_reasoning_summary_reach_responses() { .collect(); assert_eq!(types, ["reasoning", "custom_tool_call"], "{done}"); } + +// ---- gaps the second cargo-mutants pass found no test constraining ----------------------------- + +/// A Responses body onto Chat: a `created_at` of 0 is no timestamp (the gateway's own clock +/// stands in), a message item's string `content` is its text, and content is `null` only when +/// the turn called a tool or refused. +/// claim: TRN-23, TRN-22 +#[test] +fn a_responses_body_reaches_chat_with_created_text_and_null_only_beside_a_call() { + let body = |output: Value| { + json!({"id": "resp_1", "object": "response", "created_at": 0, "status": "completed", + "model": "gpt-5", "output": output, "usage": {"input_tokens": 1, "output_tokens": 1}}) + }; + let chat = json_resp( + Responses, + Chat, + &body(json!([{"type": "message", "role": "assistant", "content": "hello"}])), + ); + assert!(chat["created"].as_u64().unwrap() > 0, "{chat}"); + assert_eq!(chat["choices"][0]["message"]["content"], "hello", "{chat}"); + let chat = json_resp(Responses, Chat, &body(json!([]))); + assert_eq!(chat["choices"][0]["message"]["content"], "", "{chat}"); + let call = + json!([{"type": "function_call", "call_id": "call_1", "name": "f", "arguments": "{}"}]); + let chat = json_resp(Responses, Chat, &body(call)); + assert!(chat["choices"][0]["message"]["content"].is_null(), "{chat}"); +} + +/// An OpenRouter `reasoning.summary` detail on a Chat body is the Responses reasoning item's +/// summary text. +/// claim: T2 +#[test] +fn an_openrouter_reasoning_summary_reaches_a_responses_reasoning_item() { + let message = json!({"role": "assistant", "content": "x", "reasoning_details": [ + {"type": "reasoning.summary", "summary": "plan", "index": 0}]}); + let resp = json_resp( + Chat, + Responses, + &chat_completion(message, "stop", json!({})), + ); + let item = &resp["output"][0]; + assert_eq!(item["type"], "reasoning", "{resp}"); + assert_eq!(item["summary"][0]["text"], "plan", "{resp}"); +} + +/// Reasoning text that visible text interrupts is never signed with a later block's signature: +/// the signed block onto Messages holds only its own text. +/// claim: T2 +#[test] +fn interrupted_thinking_is_not_folded_into_the_next_signed_block() { + let detail = |extra: Value| { + let mut d = json!({"type": "reasoning.text", "format": "anthropic-claude-v1", "index": 0}); + d.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + json!({"reasoning_details": [d]}) + }; + let src = [ + chat_chunk(detail(json!({"text": "hm"})), None), + chat_chunk(json!({"content": "hi"}), None), + chat_chunk( + detail(json!({"text": "ok", "signature": "S"})), + Some("stop"), + ), + "data: [DONE]\n\n".to_owned(), + ] + .concat(); + let evs = stream(Chat, Messages, &src); + let thinking: Vec = anthropic_content(&evs) + .into_iter() + .filter(|b| b["type"] == "thinking") + .collect(); + assert_eq!(thinking.len(), 1, "{evs:#?}"); + assert_eq!(thinking[0]["thinking"], "ok"); + assert_eq!(thinking[0]["signature"], "S"); +} + +/// A stream whose last event has no terminating blank line still has that event read at the +/// end: its finish reason ends the message cleanly. +/// claim: T7 +#[test] +fn the_last_event_needs_no_terminating_blank_line() { + let last = chat_chunk(json!({}), Some("stop")); + let src = [ + chat_chunk(json!({"content": "hi"}), None), + last.trim_end().to_owned(), + ] + .concat(); + let evs = stream(Chat, Messages, &src); + assert!(named(&evs, "error").is_empty(), "{evs:#?}"); + assert_eq!(stop_reason(&evs), "end_turn"); +} + +/// A repeated `role` is cut out of the relayed event's own bytes: spacing, key order and the +/// line ending stay as the provider sent them, and so does a last event with no terminator. +/// claim: S2 +#[test] +fn a_relay_cuts_a_repeated_role_byte_for_byte() { + let event = |content: &str| { + format!( + r#"data: {{"id": "c", "choices": [{{"index": 0, "delta": {{"role": "assistant", "content": "{content}"}}}}]}}"# + ) + }; + let cut = |content: &str| { + format!( + r#"data: {{"id": "c", "choices": [{{"index": 0, "delta": {{"content": "{content}"}}}}]}}"# + ) + }; + for end in ["\n\n", "\r\n\r\n"] { + let first = format!("{}{end}", event("a")); + let second = format!("{}{end}", event("b")); + let mut b = SseBridge::new(Chat, Chat); + assert_eq!( + feeds(&mut b, &[first.as_bytes(), second.as_bytes()]), + [first.clone(), format!("{}{end}", cut("b"))], + "{end:?}" + ); + } + let first = format!("{}\n\n", event("a")); + let mut b = SseBridge::new(Chat, Chat); + let mut out = b.feed(first.as_bytes(), false); + out.extend(b.feed(event("b").as_bytes(), true)); + assert_eq!( + String::from_utf8(out).unwrap(), + format!("{first}{}", cut("b")) + ); +} + +/// A relayed chunk whose only identity field is a repeated tool call `id` (no `role` anywhere) +/// still has the repeat dropped. +/// claim: S2 +#[test] +fn a_relay_drops_a_repeated_call_id_in_a_chunk_without_a_role() { + let src = [ + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": "f", "arguments": ""}}]}), + None, + ), + chat_chunk( + json!({"tool_calls": [{"index": 0, "id": "call_1", "function": {"arguments": "{}"}}]}), + None, + ), + ] + .concat(); + let evs = stream(Chat, Chat, &src); + let ids: Vec<&Value> = chunks(&evs) + .iter() + .filter_map(|c| c.pointer("/choices/0/delta/tool_calls/0/id")) + .collect(); + assert_eq!(ids, [&json!("call_1")], "{evs:#?}"); +} + +/// Nothing follows a Messages upstream's `message_stop` onto Chat: an error written after it +/// is not a second ending after `[DONE]`. +/// claim: T7 +#[test] +fn nothing_follows_message_stop_onto_chat() { + let src = format!( + "{}event: error\ndata: {}\n\n", + claude_turn(&[json!({"type": "text", "text": "hi"})], "end_turn"), + json!({"type": "error", "error": {"type": "overloaded_error", "message": "late"}}) + ); + let evs = stream(Messages, Chat, &src); + assert_eq!(evs.last().unwrap().1, "[DONE]", "{evs:#?}"); + assert_eq!(evs.iter().filter(|(_, v)| *v == "[DONE]").count(), 1); + assert!( + chunks(&evs).iter().all(|c| c.get("error").is_none()), + "{evs:#?}" + ); +} + +/// A Messages stream onto Chat, read event by event: chunks carry a minted id and a timestamp +/// even with no `message_start`; a text block that opens with text streams it; an unnamed +/// error object is the upstream's error; and a refusal's explanation is the Chat refusal. +/// claim: TRN-23, TRN-22, T6 +#[test] +fn a_messages_stream_onto_chat_reads_every_event_shape() { + let src = ant_sse(&[ + json!({"type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": "Hi"}}), + json!({"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": " there"}}), + json!({"type": "content_block_stop", "index": 0}), + json!({"type": "message_delta", "delta": {"stop_reason": "refusal", + "stop_details": {"type": "refusal", "explanation": "Declined."}}, + "usage": {"output_tokens": 3}}), + json!({"type": "message_stop"}), + ]); + let evs = stream(Messages, Chat, &src); + for c in chunks(&evs) { + assert!(c["id"].as_str().is_some_and(|id| !id.is_empty()), "{c}"); + assert!(c["created"].as_u64().unwrap() > 0, "{c}"); + } + let text: String = chunks(&evs) + .iter() + .filter_map(|c| c.pointer("/choices/0/delta/content")?.as_str()) + .collect(); + assert_eq!(text, "Hi there"); + let refusal: String = chunks(&evs) + .iter() + .filter_map(|c| c.pointer("/choices/0/delta/refusal")?.as_str()) + .collect(); + assert_eq!(refusal, "Declined."); + + let src = format!( + "{ANT_START}data: {}\n\n", + json!({"error": {"type": "overloaded_error", "message": "busy"}}) + ); + let evs = stream(Messages, Chat, &src); + let errors: Vec<&Value> = chunks(&evs) + .into_iter() + .filter(|c| c.get("error").is_some()) + .collect(); + assert_eq!(errors.len(), 1, "{evs:#?}"); + assert_eq!(errors[0]["error"]["message"], "busy"); +} + +/// A Responses stream's refusal deltas reach a Chat client as refusal deltas. +/// claim: TRN-22 +#[test] +fn a_responses_refusal_streams_onto_chat() { + let src = [ + resp_event( + json!({"type": "response.created", "response": {"id": "resp_1", + "created_at": 1_790_000_000, "model": "gpt-5", "output": []}}), + ), + resp_event(json!({"type": "response.refusal.delta", "item_id": "msg_1", + "output_index": 0, "content_index": 0, "delta": "No"})), + resp_event(json!({"type": "response.refusal.delta", "item_id": "msg_1", + "output_index": 0, "content_index": 0, "delta": "pe."})), + resp_event( + json!({"type": "response.completed", "response": {"id": "resp_1", + "status": "completed", "output": [], "usage": {"input_tokens": 1, + "output_tokens": 1}}}), + ), + ] + .concat(); + let evs = stream(Responses, Chat, &src); + let refusal: String = chunks(&evs) + .iter() + .filter_map(|c| c.pointer("/choices/0/delta/refusal")?.as_str()) + .collect(); + assert_eq!(refusal, "Nope.", "{evs:#?}"); +} From 8f3f6ff9b8483e9826d949ec4591df5a8df7cfd1 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 20:12:08 -0700 Subject: [PATCH 10/11] gateway tests: kill the rest of proxy/usage/peek/breaker/route's full-run survivors 78 survivors of the full beyond-ai suite (proxy.rs, usage.rs, peek.rs, circuit_breaker.rs, route.rs). Every kill verified by hand-applying the mutant and running the targeted test. Real gaps (52, new tests): - Large-body walks: every pool key is walked (30 keys) and the attempt bound's last attempt is the client's answer (150 keys, a 429 not a 502); each candidate of a three-candidate walk gets its own refused-stream resend, then fails over. - Catalog walks: a refused stream is resent once; a reused-connection reset is resent on the same candidate; a cooling provider is skipped from any catalog slot; a rewrite ahead of `model` (developer->system, Claude-on-Chat reasoning cut, gateway reasoning cut) is rescanned so the model is re-spelled in place; same-wire relays keep reasoning_details (non-Claude host) and explicit nulls (Anthropic); a Responses stream's stream_options gets no include_usage. - Body budget: bodies within the 64 KiB replay buffer are not charged; a chunked body that outgrows the budget is a 503, not a 413; a body declared at exactly MAX_REQUEST_BODY is read; a translation needing exactly the budget is a 503, not a 413; client_write_timeout_secs = 0 disables the timeout. - Headers and errors: a managed key beside mixed BYO credentials is not ambiguous; a BYO ?key= is forwarded; an allowed anthropic-beta line goes out untouched; a rewritten JSON error is chunked to an HTTP/1.1 client; a non-JSON error keeps its Content-Length; a streaming request's account remedy is neutralized. - Billing: message_start behind a few KiB of pings is still read from the head; a final multi-line SSE event without a blank line bills; skipped strings (escaped quotes, non-letter keys) end at their own closing quote in the input tally. - Units: h2_body_unsent, Redact with no key / a whole body, the unanswered-attempt 502 (Reaped::unanswered), CircuitOpen's Display, str_is on an escaped value, has_typed_member's span end, an escaped stream_options in the injection planner, error_body returns the static table body (REJECT_BODIES is now a static, so the pointer check holds). Dead or simplified (11): ModelScanner's and the planner's depth guards on ':'/',', the planner's whitespace skip and key-skip condition, scan_buffered's key-skip condition, peek_body_model's empty-chunk arm and `complete` guard, the walked-arms fold (now control::Walk::mask), and a stream-only candidate's force_stream usage flag (the usage splice every Chat stream gets already asks for it; ARCHITECTURE.md updated). BODY_PEEK_LIMIT's six comparisons are one past_replay_buffer helper. Equivalent, excluded in .cargo/mutants.toml with reasons (14): str_is's raw fast path, unserved's and the TALLY tables' | vs ^, de_service_tier's Ok(None) visitors, UsageTail::push at exactly the cap, release_body(0), settle_health's undecided arm, retain_managed_client_headers' (None, _) arm, mask_all's at *= (same hits, quadratic), Ctx::deref (DerefMut's supertrait; a panicking body passes the integration suites). Already removed before this pass: 1 (the breaker's STATE_OPEN arm). Found, not fixed: a managed 402 that fails over to the next candidate is never read, so its key is never cooled (D180 cools only a relayed one) and every later request pays a round trip to the unfunded provider first. Cooling needs the body (OpenRouter's "requires more credits" 402 is not unfunded), which the failover discards. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .cargo/mutants.toml | 34 ++++ crates/gateway/ARCHITECTURE.md | 7 +- crates/gateway/src/circuit_breaker.rs | 7 + crates/gateway/src/control.rs | 7 + crates/gateway/src/peek.rs | 73 +++++--- crates/gateway/src/proxy.rs | 157 +++++++++++++---- crates/gateway/src/usage.rs | 32 ++++ crates/gateway/tests/billing_streams.rs | 49 ++++++ crates/gateway/tests/body_memory.rs | 26 +++ crates/gateway/tests/cancellation.rs | 38 +++++ crates/gateway/tests/catalog_body.rs | 144 ++++++++++++++++ crates/gateway/tests/claims_reliability.rs | 89 ++++++++++ crates/gateway/tests/claims_security.rs | 59 +++++++ crates/gateway/tests/errors_account_remedy.rs | 131 +++++++++++++++ .../gateway/tests/reliability_large_body.rs | 41 +++++ crates/gateway/tests/reliability_lifecycle.rs | 159 ++++++++++++++++++ .../gateway/tests/reliability_unfunded_key.rs | 51 ++++++ 17 files changed, 1045 insertions(+), 59 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index c8989235..ecbc7712 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -51,4 +51,38 @@ exclude_re = [ 'replace \| with \^ in ChatIdentity::key', # `map_response` clones a same-wire body as is: the guard only skips that clone. 'replace match guard client == Endpoint::Responses with false in SseBridge::assembled', + # `str_is`'s raw compare is a fast path: `escaped_key_is` decodes a string with no escape to + # itself, so skipping the raw compare (`false`) gives the same answer, only slower. + '''replace match guard !inner\.contains\(&b'\\\\'\) with false in str_is''', + # `unserved` ORs `1 << i` for distinct indices `i`, where `|` and `^` are the same operation. + 'replace \| with \^ in unserved', + # The const `TALLY` / `TALLY4` tables pack `state << 3 | class` and `next | add << 4` from + # disjoint bit ranges, where `|` and `^` agree. A const initializer's mutant names no function, + # only the file and position. + 'usage\.rs:\d+:\d+: replace \| with \^$', + # `de_service_tier`'s visitors for a non-string tier answer `Ok(None)`, which is `Option`'s + # default. + 'replace de_service_tier:: for V>::visit_(bool|i64|u64|f64|none|unit) -> Result with Ok\(Default::default\(\)\)', + # A push that leaves the buffer at exactly the cap: `>=` "compacts" it with a self-copy and a + # no-op truncate, and `ring()` reads it as a ring either way. + 'replace > with >= in UsageTail::push', + # `release_body` with nothing held: `BodyBudget::release(0)` changes nothing; the guard only skips + # an atomic. + 'replace > with >= in Held::release_body', + # `settle_health`'s undecided arm stops the work at the cap or the body's end. Without it the + # verdict stays pending, and a pending verdict only re-reads the same capped prefix (it never + # counts a failure from it), so nothing observable changes. (The guard's `||` is the only one in + # `settle_health`; re-check this exclusion if another one is added there.) + 'replace match guard end_of_stream \|\| pending\.prefix\.len\(\) >= HEALTH_PREFIX_CAP with false in AiProxy::settle_health', + 'replace \|\| with && in AiProxy::settle_health', + # No `anthropic-beta` header: the rebuild finds no token to keep and removes a header that is not + # there. The same headers, one lookup more. + 'delete match arm \(None, _\) in retain_managed_client_headers', + # `mask_all` with `at *= …`: `at` stays 0, so each search restarts at the front, where every hit + # so far now reads `[redacted]` (or its `*` padding) and no longer matches: the same hits, found + # in quadratic time. Only a pool key spelled like the marker itself could tell them apart. + 'replace \+= with \*= in mask_all', + # `Ctx` implements `Deref` as `DerefMut`'s supertrait: every hook holds the context mutably, so + # nothing calls `deref` itself (a panicking body passes the integration suites). + 'replace ::deref -> &Self::Target', ] diff --git a/crates/gateway/ARCHITECTURE.md b/crates/gateway/ARCHITECTURE.md index 76c181e9..4af04559 100644 --- a/crates/gateway/ARCHITECTURE.md +++ b/crates/gateway/ARCHITECTURE.md @@ -510,9 +510,10 @@ stays a byte relay. `/{provider}/…` never translates. **Stream-only candidates.** Together serves Qwen3.6 Plus, Qwen3.7 Plus, Qwen3.7 Max and Qwen3.8 Flash only as streams: a request without `"stream": true` is a 400 `streaming_required` (D147). `providers::catalog::stream_only` names those candidates. When one serves an attempt whose client -did not ask for a stream, the body goes out with `"stream":true,"stream_options":{"include_usage":true}` -spliced first and any `stream` / `stream_options` the client sent cut out by span -(`translate::force_stream`; onto a Messages path only `stream`). Every other byte is unchanged. +did not ask for a stream, the body goes out with `"stream":true` spliced first and any `stream` the +client sent cut out by span (`translate::force_stream`). Its usage is asked for by the splice every +Chat Completions stream gets (an injected `stream_options`, or the client's own made to include +usage). Every other byte is unchanged. The streamed answer is assembled into the client's ordinary JSON body: `TranslateState::assemble` gives the attempt a `SseBridge::assembling` bridge. That is the bridge a Responses client's stream already uses, which diff --git a/crates/gateway/src/circuit_breaker.rs b/crates/gateway/src/circuit_breaker.rs index e1b66d53..10d79a74 100644 --- a/crates/gateway/src/circuit_breaker.rs +++ b/crates/gateway/src/circuit_breaker.rs @@ -1739,4 +1739,11 @@ mod mutation_gaps { "{s}" ); } + + /// A refusal reads as what it is when it reaches a log line or an error chain. + /// claim: R6 + #[test] + fn an_open_circuit_says_so() { + assert_eq!(CircuitOpen.to_string(), "circuit breaker is open"); + } } diff --git a/crates/gateway/src/control.rs b/crates/gateway/src/control.rs index fa0ccdc3..4183bb3b 100644 --- a/crates/gateway/src/control.rs +++ b/crates/gateway/src/control.rs @@ -142,6 +142,13 @@ impl Walk { (i < self.len).then_some(self.indices[i as usize]) } + /// The walked catalog indices as a bitmask (bit `orig` set for `arms[orig]`). + pub fn mask(self) -> u8 { + (0..self.len) + .filter_map(|i| self.catalog_index(i)) + .fold(0u8, |m, orig| m | (1 << orig)) + } + /// `ProviderId::index` bytes in walk order — the exact-match cache's per-arm discriminator. pub fn provider_ids(self, candidates: &[Candidate]) -> ([u8; MAX_CANDIDATES], u8) { let mut ids = [0u8; MAX_CANDIDATES]; diff --git a/crates/gateway/src/peek.rs b/crates/gateway/src/peek.rs index 19432b1d..50808949 100644 --- a/crates/gateway/src/peek.rs +++ b/crates/gateway/src/peek.rs @@ -120,12 +120,6 @@ impl ModelScanner { (self.depth == 1 && self.root_is_object) || (self.depth == 2 && self.in_message) } - /// The depth whose `:` / `,` punctuation drives `expect_key`. Mirrors [`Self::at_key_level`]. - #[inline] - fn key_depth(&self) -> u32 { - if self.in_message { 2 } else { 1 } - } - #[allow(clippy::indexing_slicing, clippy::arithmetic_side_effects)] pub fn feed(&mut self, bytes: &[u8]) { // Proof for the allow: every index is `i`/`j` with `i < n` (the loop condition, or a `memchr` @@ -235,8 +229,11 @@ impl ModelScanner { } self.depth = self.depth.saturating_sub(1); } - b':' if self.depth == self.key_depth() => self.expect_key = false, - b',' if self.depth == self.key_depth() => { + // Unguarded by depth: a `:` or `,` inside a nested value is always followed by + // the key-level `,` or `}` that closes it before the next key-level string, and + // that one sets `expect_key` again. + b':' => self.expect_key = false, + b',' => { self.expect_key = true; self.last_key_is_model = false; if self.depth == 1 { @@ -286,12 +283,9 @@ pub fn plan_stream_usage_injection(body: &[u8]) -> Option { { return None; } - let mut i = 0; - while i < n && body[i].is_ascii_whitespace() { - i += 1; - } // Must be a JSON object; anything else (array, scalar, garbage) we never rewrite. - if i >= n || body[i] != b'{' { + let i = body.iter().position(|b| !b.is_ascii_whitespace())?; + if body[i] != b'{' { return None; } let insert_at = i + 1; @@ -312,10 +306,10 @@ pub fn plan_stream_usage_injection(body: &[u8]) -> Option { let mut j = i; while j < n { if in_string { - // Fast path: inside a string we're not capturing (any non-root-key string — message - // content, system prompts, base64 images), jump straight to the next `"`/`\` with a - // SIMD search instead of inspecting every byte. Mirrors the skip in `ModelScanner::feed`. - if !capturing_key && !escaped { + // Fast path: jump straight to the next `"`/`\` with a SIMD search instead of + // inspecting every byte, a root key included (it is sliced out of the body at its + // closing quote). Mirrors the skip in `ModelScanner::feed`. + if !escaped { match memchr::memchr2(b'"', b'\\', &body[j..]) { Some(rel) => j += rel, None => break, // rest of the body is skippable string content @@ -376,8 +370,10 @@ pub fn plan_stream_usage_injection(body: &[u8]) -> Option { } b'[' => depth += 1, b'}' | b']' => depth = depth.saturating_sub(1), - b':' if depth == 1 => expect_key = false, - b',' if depth == 1 => { + // Unguarded by depth, as in `ModelScanner::feed`: a nested `:` or `,` is always + // followed by the root `,` or `}` that closes its value before the next root key. + b':' => expect_key = false, + b',' => { expect_key = true; last_key_is_stream = false; } @@ -518,9 +514,10 @@ pub fn scan_buffered(body: &[u8]) -> BufferedScan { let mut j = i; while j < n { if in_string { - // SIMD skip over any string we aren't capturing — message content, system prompts, - // base64 images. Mirrors both functions this replaces. - if !capturing_key && !capturing_model && !escaped { + // SIMD skip over any string whose bytes we don't accumulate — message content, system + // prompts, base64 images, and keys too, which are sliced out of the body at their + // closing quote. + if !capturing_model && !escaped { match memchr::memchr2(b'"', b'\\', &body[j..]) { Some(rel) => j += rel, None => break, // rest of the body is skippable string content @@ -2086,4 +2083,36 @@ mod mutation_gaps { assert_eq!(root_members(br#"{"a":}"#), None); assert_eq!(root_members(br#"{"a":,"b":1}"#), None); } + + /// A string value spelled with an escape is decoded before it is compared, as the provider's + /// parser decodes it. + /// claim: T3 + #[test] + fn an_escaped_string_value_is_compared_decoded() { + // `"image"`, spelled out so the escape survives as written. + let quoted = format!(r#""ima{}u0067e""#, '\\'); + let b = quoted.as_bytes(); + assert!(str_is(b, (0, b.len()), "image")); + assert!(!str_is(b, (0, b.len()), "imag")); + } + + /// A `"type"` member counts only when its value starts inside the span: a span that ends at + /// the key reads no value past it. + /// claim: T3 + #[test] + fn a_type_value_past_the_span_is_not_read() { + let b = br#"{"type":"image"}"#; + assert!(has_typed_member(b, (0, b.len()), &["image"])); + assert!(!has_typed_member(b, (1, 7), &["image"])); + } + + /// A root `stream_options` spelled with an escape decodes to `stream_options` at the provider, + /// so the client already controls usage there too: nothing to inject. + /// claim: BIL-2 + #[test] + fn an_escaped_stream_options_key_means_no_injection() { + let body = format!(r#"{{"stream":true,"stream{}u005foptions":{{}}}}"#, '\\'); + assert_eq!(plan_stream_usage_injection(body.as_bytes()), None); + assert_eq!(plan_stream_usage_injection(br#"{"stream":true}"#), Some(1)); + } } diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index 932a47e4..8c045b87 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -1113,7 +1113,7 @@ impl RequestCtx { /// Kept as a table so `reject_bodies_are_valid_json` can walk it and assert each entry parses, /// carries the `type` and `message` it claims, and is reachable — a hand-written JSON literal is /// exactly the thing that rots silently otherwise. -pub const REJECT_BODIES: [(&str, &str, &str); 16] = [ +pub static REJECT_BODIES: [(&str, &str, &str); 16] = [ ( "invalid_request_error", "unknown provider", @@ -1205,7 +1205,7 @@ pub const REJECT_BODIES: [(&str, &str, &str); 16] = [ /// `pub` for the bench target (`benches/unit.rs`), which measures it against the `serde_json` /// construction it replaced. Not part of the crate's intended surface. pub fn error_body(typ: &str, msg: &str) -> Bytes { - for (t, m, body) in REJECT_BODIES { + for &(t, m, body) in &REJECT_BODIES { if t == typ && m == msg { return Bytes::from_static(body.as_bytes()); } @@ -1689,14 +1689,7 @@ impl AiProxy { Err(e) => e, // The attempt ended without a response or an error (it panicked, say). // Never leave the client waiting on a connection pingora would keep alive. - Ok(_) => pingora_core::Error::explain( - pingora_core::ErrorType::HTTPStatus(502), - if reaped == Reaped::Panicked { - "full-body attempt panicked" - } else { - "full-body attempt ended without a response" - }, - ), + Ok(_) => reaped.unanswered(), }); } } @@ -1802,7 +1795,7 @@ impl AiProxy { let Some(budget) = self.state.body_budget.as_ref() else { return Ok(None); }; - if body.len() <= BODY_PEEK_LIMIT { + if !past_replay_buffer(body.len()) { return Ok(None); } let heap = translate::translation_heap(body); @@ -2224,6 +2217,13 @@ fn body_replayable(session: &mut Session) -> bool { /// truncates), so [`ModelRouting::replay`] carries our copy for `request_body_filter` to prepend. const BODY_PEEK_LIMIT: usize = 64 * 1024; +/// Whether a body of `n` bytes is past [`BODY_PEEK_LIMIT`]: pingora cannot replay it from its own +/// buffer, so every copy the gateway holds of it is charged to the process body budget. Below it, +/// concurrency bounds the bodies in memory. +fn past_replay_buffer(n: usize) -> bool { + n > BODY_PEEK_LIMIT +} + /// The `x-beyond-model` header, viewed as a catalog lookup. Present-but-unknown is distinct from /// absent: the header wins, so an unknown header is a 404 rather than a fall-through to the body. #[derive(Clone, Copy)] @@ -2251,7 +2251,8 @@ fn catalog_from_header(session: &Session) -> CatalogHeader { /// What `peek_body_model` read before a catalog row could be chosen. struct BodyPeek { model: Option, - /// The full pre-rewrite body, when the peek read it to the end. + /// The full pre-rewrite body, when the peek read it to the end. Read only once `over_cap` and + /// `over_budget` are known to be false. complete: Option>, /// The whole body was read and it outgrew pingora's 64 KiB retry buffer: pingora has nothing /// to send and nothing left to read. The caller re-runs the request with `complete` as its body @@ -2303,15 +2304,16 @@ async fn peek_body_model( break; } match session.read_request_body().await? { - Some(chunk) if !chunk.is_empty() => { + // An empty chunk feeds nothing, adds nothing, and re-reserves the same total (a no-op). + Some(chunk) => { scanner.feed(&chunk); buf.extend_from_slice(&chunk); - if buf.len() > BODY_PEEK_LIMIT && !held.reserve_body(buf.len().saturating_mul(2)) { + if past_replay_buffer(buf.len()) && !held.reserve_body(buf.len().saturating_mul(2)) + { over_budget = true; break; } } - Some(_) => {} None => break, } } @@ -2319,7 +2321,7 @@ async fn peek_body_model( let done = session.as_mut().is_body_done(); Ok(BodyPeek { model: scanner.take_model(), - complete: (done && !over_budget).then_some(buf), + complete: done.then_some(buf), relay: truncated && done, over_cap: over_cap && !done, over_budget, @@ -2519,6 +2521,21 @@ enum Reaped { StillRunning, } +impl Reaped { + /// The 502 for an attempt that ended with neither a response nor an error, saying whether it + /// panicked. + fn unanswered(self) -> Box { + pingora_core::Error::explain( + pingora_core::ErrorType::HTTPStatus(502), + if self == Reaped::Panicked { + "full-body attempt panicked" + } else { + "full-body attempt ended without a response" + }, + ) + } +} + /// Wait, at most [`ABANDONED_ATTEMPT_GRACE`], for an attempt's task to end, and say how it did. A /// panic is logged at error with the request id (D207): the subrequest dies with it, and the /// client would otherwise get a 502 no log line explains. @@ -3753,7 +3770,7 @@ impl ProxyHttp for AiProxy { // whole body in hand the gateway can hash it for the cache and read Responses session // fields. let responses = route::is_responses_path(session.req_header().uri.path()); - let large = declared_len.is_none_or(|n| n > BODY_PEEK_LIMIT); + let large = declared_len.is_none_or(past_replay_buffer); let mut peeked = false; // Taken before any full read (see `SlotGuard`), handed to the request context below. let mut early_slot: Option> = None; @@ -3780,7 +3797,7 @@ impl ProxyHttp for AiProxy { } // A large body is held twice (here and in its `FullBody` re-run): reserve // both before reading a byte of it. - if let Some(n) = declared_len.filter(|n| *n > BODY_PEEK_LIMIT) + if let Some(n) = declared_len.filter(|&n| past_replay_buffer(n)) && !ctx.held.reserve_body(n.saturating_mul(2)) { return self.reject_body_memory(session, &request_id).await; @@ -3846,7 +3863,7 @@ impl ProxyHttp for AiProxy { Ok(guard) => early_slot = guard, Err(()) => return self.reject_tenant_busy(session, &request_id).await, } - if let Some(n) = declared_len.filter(|n| *n > BODY_PEEK_LIMIT) + if let Some(n) = declared_len.filter(|&n| past_replay_buffer(n)) && !ctx.held.reserve_body(n.saturating_mul(2)) { return self.reject_body_memory(session, &request_id).await; @@ -4093,9 +4110,7 @@ impl ProxyHttp for AiProxy { (None, Some(b)) => route::unserved(arms, b), (None, None) => 0, }; - let walked = (0..walk.len) - .filter_map(|i| walk.catalog_index(i)) - .fold(0u8, |m, orig| m | (1 << orig)); + let walked = walk.mask(); if dispatchable & walked & !unserved != 0 { dispatchable &= !unserved; } @@ -4331,7 +4346,7 @@ impl ProxyHttp for AiProxy { // reserves as it grows (`request_body_filter`). if (inject_eligible || background_check) && model_route.is_none() - && let Some(n) = declared_len.filter(|n| *n > BODY_PEEK_LIMIT) + && let Some(n) = declared_len.filter(|&n| past_replay_buffer(n)) && !ctx.held.reserve_body(n) { return self.reject_body_memory(session, &request_id).await; @@ -4629,7 +4644,7 @@ impl ProxyHttp for AiProxy { input_tally: usage::InputTally::default(), tally_eager: managed && model_route.is_none() - && declared_len.is_none_or(|n| n > BODY_PEEK_LIMIT), + && declared_len.is_none_or(past_replay_buffer), resp_bytes: 0, upstream_phase: UpstreamPhase::None, redact: None, @@ -5320,7 +5335,7 @@ impl ProxyHttp for AiProxy { // A large body buffered for a rewrite (the `/{provider}` usage splice; a catalog // walk's large body is a `FullBody` re-run, already reserved) holds budget as it // grows. Tagged downstream: the client's size, not the provider's health. - if rc.req_buf.len() > BODY_PEEK_LIMIT && !held.reserve_body(rc.req_buf.len()) { + if past_replay_buffer(rc.req_buf.len()) && !held.reserve_body(rc.req_buf.len()) { self.state.metrics.rejection(Rejection::BodyMemory).inc(); return Err( gateway_error(503, "too many large request bodies in flight").into_down(), @@ -5464,15 +5479,15 @@ impl ProxyHttp for AiProxy { changed |= translate::thinking_off_for_tools(&mut buf, tool_thinking); } // A candidate that answers only streams, for a client that did not ask - // for one: ask it for the stream (with its usage) and assemble the answer - // into the client's own body (D147). Per attempt, like the tools: a - // failover candidate gets the client's body as it came. + // for one: ask it for the stream and assemble the answer into the + // client's own body (D147). Per attempt, like the tools: a failover + // candidate gets the client's body as it came. Its usage is asked for by + // the splice below, as on every Chat Completions stream (`inject_at`, or + // `force_include_usage` on the client's own `stream_options`); a Messages + // body takes no `stream_options`. t.assemble = stream_only && to != route::Endpoint::Responses - && translate::force_stream( - &mut buf, - to == route::Endpoint::ChatCompletions, - ); + && translate::force_stream(&mut buf, false); changed |= t.assemble; } if changed { @@ -8385,7 +8400,7 @@ mod tests { // build, so the thing to guard is that they still *say* what the `error_type` in the log // line and the metric label claim. A drifting literal would ship a response whose `type` // contradicts the reason we rejected for. - for (typ, msg, body) in REJECT_BODIES { + for &(typ, msg, body) in &REJECT_BODIES { let v: serde_json::Value = serde_json::from_str(body).unwrap_or_else(|e| panic!("{body} is not JSON: {e}")); assert_eq!(v["error"]["type"], typ, "type mismatch in {body}"); @@ -8396,6 +8411,7 @@ mod tests { assert_eq!(body, built, "constant diverges from the built body"); // The lookup must find it rather than falling through to the allocating branch. assert_eq!(error_body(typ, msg), Bytes::from_static(body.as_bytes())); + assert_eq!(error_body(typ, msg).as_ptr(), body.as_ptr(), "{typ}: {msg}"); } } @@ -8449,7 +8465,7 @@ mod tests { ); // Every tabulated message must also appear at a call site, catching the reverse drift (a // table entry left behind after its rejection was removed). Twice: the table and the caller. - for (_, msg, _) in REJECT_BODIES { + for &(_, msg, _) in &REJECT_BODIES { assert!( src.matches(&format!("\"{msg}\"")).count() >= 2, "REJECT_BODIES entry {msg:?} has no reject_boxed call site" @@ -8909,4 +8925,77 @@ mod mutation_gaps { whole.extend_from_slice(event); assert_eq!(tail.contiguous(), &whole[whole.len() - USAGE_TAIL_CAP..]); } + + /// A body write that met an HTTP/2 stream already closed (D248) is pingora's `H2Error` with a + /// capacity context, found anywhere down the cause chain; another type or context is not it. + /// claim: REL-22 + #[test] + fn only_an_h2_capacity_failure_is_an_unsent_body() { + for ctx in ["cannot reserve capacity", "while waiting for capacity"] { + assert!(h2_body_unsent(&Error::explain(T::H2Error, ctx)), "{ctx}"); + let wrapped = Error::because( + T::WriteError, + "writing the body", + Error::explain(T::H2Error, ctx), + ); + assert!(h2_body_unsent(&wrapped), "caused by {ctx}"); + assert!( + !h2_body_unsent(&Error::explain(T::WriteError, ctx)), + "not H2: {ctx}" + ); + } + assert!(!h2_body_unsent(&Error::explain(T::H2Error, "stream reset"))); + assert!(!h2_body_unsent(&Error::new(T::H2Error))); + } + + /// With no key to scrub (none sent, or an empty one) and no body to hold, a chunk is relayed as + /// it came, never searched; a body held whole is held whether or not there is a key. + /// claim: SEC-7 + #[test] + fn redact_holds_a_whole_body_even_with_no_key() { + let chunk = Bytes::from_static(br#"{"error":{"message":"x"}}"#); + let empty = memchr::memmem::Finder::new(b""); + let mut r = Redact::default(); + let got = r.feed(std::slice::from_ref(&empty), Some(chunk.clone()), false); + assert_eq!(got.map(|b| b.as_ptr()), Some(chunk.as_ptr())); + let mut r = Redact { + whole: true, + status: 400, + ..Redact::default() + }; + assert_eq!(r.feed(&[], Some(chunk.clone()), false), Some(Bytes::new())); + assert_eq!(r.feed(&[], None, true), Some(chunk)); + } + + /// An attempt that ended unanswered is a 502 that says whether it panicked. + /// claim: REL-1 + /// defect: D207 + #[test] + fn an_unanswered_attempt_says_whether_it_panicked() { + let context = |e: Box| e.context.as_ref().map(|c| c.as_str().to_owned()); + let panicked = Reaped::Panicked.unanswered(); + assert_eq!(panicked.etype(), &T::HTTPStatus(502)); + assert_eq!( + context(panicked).as_deref(), + Some("full-body attempt panicked") + ); + for quiet in [Reaped::Finished, Reaped::StillRunning] { + assert_eq!( + context(quiet.unanswered()).as_deref(), + Some("full-body attempt ended without a response") + ); + } + } + + /// An `anthropic-beta` line whose every token is allowed goes out exactly as the client wrote + /// it, spacing included: only a value with a token to drop is rebuilt. + /// claim: SEC-6 + #[test] + fn an_allowed_beta_line_is_forwarded_untouched() { + let value = "prompt-caching-2024-07-31, interleaved-thinking-2025-05-14"; + let mut req = pingora::http::RequestHeader::build("POST", b"/v1/messages", None).unwrap(); + req.insert_header("anthropic-beta", value).unwrap(); + retain_managed_client_headers(&mut req).unwrap(); + assert_eq!(req.headers.get("anthropic-beta").unwrap(), value); + } } diff --git a/crates/gateway/src/usage.rs b/crates/gateway/src/usage.rs index 81c92aa8..132fc6de 100644 --- a/crates/gateway/src/usage.rs +++ b/crates/gateway/src/usage.rs @@ -2818,4 +2818,36 @@ data: {\"type\":\"message_delta\",\"usage\":{\"input_tokens\":40}}\n\n"; assert_eq!(estimate_body_output(tail.as_bytes(), n), 29); assert_eq!(estimate_body_output(tail.as_bytes(), 2 * n), 59); } + + /// A string that is not text is skipped whole whatever it holds: an escaped quote inside a + /// skipped value or key, and a key that leaves the trie on a non-letter, end where their + /// closing quote does, so the text after them still counts. + /// claim: BIL-20 + #[test] + fn skipped_strings_end_at_their_own_closing_quote() { + let text = r#""content":"hello world""#; + for envelope in [ + r#""model":"a\"b""#, + r#""\"":1"#, + r#""X":"y""#, + r#""tX":"y""#, + ] { + let body = format!(r#"{{{envelope},"messages":[{{{text}}}]}}"#); + let mut t = InputTally::default(); + t.feed(body.as_bytes()); + assert_eq!(t.estimate_tokens(), 2, "{body}"); + } + } + + /// A stream's last event is read even when no blank line ends it, on the slow path that reads + /// an event's `data:` lines joined (D126). + /// claim: BIL-1 + #[test] + fn a_final_multi_line_event_without_a_blank_line_still_bills() { + let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n\ +data: {\"choices\":[],\"usage\":\n\ +data: {\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}}\n"; + let u = openai_stream(sse).expect("usage"); + assert_eq!((u.input_tokens, u.output_tokens), (7, 2)); + } } diff --git a/crates/gateway/tests/billing_streams.rs b/crates/gateway/tests/billing_streams.rs index 9f7f0be2..4e9cd245 100644 --- a/crates/gateway/tests/billing_streams.rs +++ b/crates/gateway/tests/billing_streams.rs @@ -343,3 +343,52 @@ async fn duplicate_or_escaped_stream_options_cannot_turn_off_exact_metering() { ); } } + +/// `message_start` (input and cache counts) need not be the first bytes of an Anthropic stream: a +/// few KiB of `ping`s ahead of it still leave it inside the head the gateway keeps, so a stream far +/// longer than the tail bills its exact input. +/// claim: BIL-8 +#[tokio::test] +async fn a_message_start_behind_pings_is_still_read_from_the_head() { + let mut sse = String::new(); + while sse.len() < 4 * 1024 { + sse.push_str("event: ping\ndata: {\"type\":\"ping\"}\n\n"); + } + sse.push_str("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-opus-4-8\",\"content\":[],\"usage\":{\"input_tokens\":1234,\"cache_read_input_tokens\":56,\"cache_creation_input_tokens\":0,\"output_tokens\":1}}}\n\n"); + while sse.len() < 160 * 1024 { + sse.push_str("event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello hello hello\"}}\n\n"); + } + sse.push_str("event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":300}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"); + let (pubkey, sk) = test_keypair(63); + let mock = MockUpstream::start(Mode::Raw( + 200, + "text/event-stream", + Box::leak(sse.into_boxed_str()), + )) + .await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["anthropic"]) + .start() + .await; + let resp = test_client() + .post(format!("{}/anthropic/v1/messages", gw.url())) + .header("x-api-key", billing_vkey(&sk, 64)) + .header("anthropic-version", "2023-06-01") + .header("content-type", "application/json") + .body(r#"{"model":"claude-opus-4-8","max_tokens":512,"stream":true,"messages":[{"role":"user","content":"hi"}]}"#) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + let _ = resp.text().await; + let row = usage_row_of(&gw).await; + assert_eq!( + ( + row["input_tokens"].as_u64(), + row["cache_read_tokens"].as_u64(), + row["output_tokens"].as_u64() + ), + (Some(1234), Some(56), Some(300)), + "{row}" + ); +} diff --git a/crates/gateway/tests/body_memory.rs b/crates/gateway/tests/body_memory.rs index e8998c3b..1eca500a 100644 --- a/crates/gateway/tests/body_memory.rs +++ b/crates/gateway/tests/body_memory.rs @@ -223,3 +223,29 @@ async fn large_bodies_that_fit_the_budget_still_translate() { assert_eq!(status, 200, "{text}"); assert_eq!(mock.hits(), 2); } + +/// The 413 is for a translation that needs more than the whole budget. One that needs exactly the +/// budget fits it, so it is not a 413; it cannot be held while the walk holds the body's own two +/// copies, which is the budget's retryable 503. +/// claim: SEC-19 +/// defect: D216 +#[tokio::test] +async fn a_translation_needing_exactly_the_budget_is_not_a_413() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::AnthropicJson).await; + let body = format!( + r#"{{"model":"claude-opus-4-8","store":false,"input":[{}]}}"#, + tiny_objects(100 * 1024) + ); + let heap = beyond_ai::translate::translation_heap(body.as_bytes()); + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["anthropic"]) + .config_line(&format!("max_buffered_body_bytes = {heap}")) + .start() + .await; + let (status, text, _) = send(&gw, &sk, "/v1/responses", body).await; + assert!( + status == 503 && text.contains("too many large request bodies"), + "{status}: {text}" + ); +} diff --git a/crates/gateway/tests/cancellation.rs b/crates/gateway/tests/cancellation.rs index 5a7941d8..fc80efbd 100644 --- a/crates/gateway/tests/cancellation.rs +++ b/crates/gateway/tests/cancellation.rs @@ -370,3 +370,41 @@ async fn a_key_walk_retry_replays_the_body_exactly_once() { "the usage injection must still be spliced exactly once on the retried attempt", ); } + +/// The same on a catalog walk: a pooled connection the provider closed with the request unread is +/// the connection's failure, not the candidate's, so the walk retries the same candidate on a fresh +/// connection, even with no other candidate to walk to. +/// claim: REL-1 +/// defect: D80 +#[tokio::test] +async fn a_catalog_walk_resends_a_reused_connection_reset_before_reading() { + let (port, fired) = closes_reused_connections_unread().await; + let (pubkey, sk) = test_keypair(1); + let gw = Gateway::builder( + unused_nats_port(), + &format!("127.0.0.1:{port}"), + &b64(&pubkey), + ) + .providers(&["openai"]) + .start() + .await; + let vkey = billing_vkey(&sk, 80); + let client = reqwest::Client::new(); + for i in 0..6 { + let resp = client + .post(format!("{}/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {vkey}")) + .header("content-type", "application/json") + .body(body()) + .send() + .await + .unwrap_or_else(|e| panic!("request {i}: {e}; log:\n{}", gw.log())); + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap_or_default(); + assert_eq!(status, 200, "request {i}: {text}\n{}", gw.log()); + } + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst) > 0, + "no pooled connection was reused, so the scenario did not fire" + ); +} diff --git a/crates/gateway/tests/catalog_body.rs b/crates/gateway/tests/catalog_body.rs index d6fb066e..4c88a534 100644 --- a/crates/gateway/tests/catalog_body.rs +++ b/crates/gateway/tests/catalog_body.rs @@ -133,3 +133,147 @@ async fn a_duplicate_model_is_refused_before_the_upstream_with_a_json_400() { ); assert_eq!(mock.hits(), 0, "the upstream got the request"); } + +/// POST `body` (sent as written) to `path` with a managed key; the status. +async fn send_raw(gw: &Gateway, sk: &ed25519_dalek::SigningKey, path: &str, body: &str) -> u16 { + let resp = test_client() + .post(format!("{}{path}", gw.url())) + .header("authorization", format!("Bearer {}", managed_key(sk))) + .header("x-api-key", managed_key(sk)) + .header("anthropic-version", "2023-06-01") + .header("content-type", "application/json") + .body(body.to_owned()) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + let _ = resp.bytes().await; + status +} + +/// What the upstream received, parsed. +fn sent(mock: &MockUpstream) -> serde_json::Value { + let cap = mock.captured().expect("a request reached the upstream"); + serde_json::from_slice(&cap.body) + .unwrap_or_else(|e| panic!("{e}: {}", String::from_utf8_lossy(&cap.body))) +} + +/// A same-wire rewrite that moves bytes ahead of `model` (a `developer` role spelled `system`, a +/// Claude-on-Chat body re-encoded without its reasoning control) is followed by a fresh scan, so +/// the model is still re-spelled where it now is and the body stays valid JSON. +/// claim: SEC-21, TRN-16 +/// defect: D173 +#[tokio::test] +async fn a_rewrite_ahead_of_the_model_still_respells_the_model() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["deepseek", "openrouter"]) + .provider_authority("openrouter", &mock.authority()) + .start() + .await; + // DeepSeek does not read `developer`: the role shrinks by three bytes ahead of `model`. + let developer = r#"{"messages":[{"role":"developer","content":"be brief"},{"role":"user","content":"hi"}],"model":"deepseek-flash","temperature":0.5}"#; + assert_eq!( + send_raw(&gw, &sk, "/v1/chat/completions", developer).await, + 200 + ); + let v = sent(&mock); + assert_eq!( + (v["model"].as_str(), v["messages"][0]["role"].as_str()), + (Some("deepseek-flash"), Some("system")), + "{v}" + ); + // A Claude row on OpenRouter's Chat endpoint: a tool turn with no thinking to replay goes + // without `reasoning_effort`, and the body is re-encoded with `model` after `messages`. + let claude = r#"{"model":"claude-opus-4-8","reasoning_effort":"high","messages":[{"role":"user","content":"weather?"},{"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"w","arguments":"{}"}}]},{"role":"tool","tool_call_id":"c1","content":"sunny"}],"tools":[{"type":"function","function":{"name":"w","parameters":{"type":"object"}}}]}"#; + assert_eq!( + send_raw(&gw, &sk, "/v1/chat/completions", claude).await, + 200 + ); + let v = sent(&mock); + assert_eq!(v["model"], "anthropic/claude-opus-4.8", "{v}"); + assert!(v.get("reasoning_effort").is_none(), "{v}"); +} + +/// A same-wire relay is byte for byte except where a host needs otherwise: an assistant turn's +/// `reasoning_details` reaches a non-Claude Chat host as sent (only a Claude model behind Chat +/// loses unreplayable reasoning), and an explicit `null` reaches Anthropic's Messages endpoint as +/// sent (only a Chat host that is not OpenAI's has its root nulls dropped). +/// claim: CAT-9, TRN-16 +/// defect: D101 +#[tokio::test] +async fn a_same_wire_relay_keeps_what_its_host_reads() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let anthropic = MockUpstream::start(Mode::AnthropicJson).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["deepseek", "anthropic"]) + .provider_authority("anthropic", &anthropic.authority()) + .start() + .await; + let chat = r#"{"model":"deepseek-flash","messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"x","reasoning_details":[{"type":"reasoning.text","text":"t"}]},{"role":"user","content":"more"}]}"#; + assert_eq!(send_raw(&gw, &sk, "/v1/chat/completions", chat).await, 200); + let cap = mock.captured().unwrap(); + assert_eq!(String::from_utf8_lossy(&cap.body), chat); + + let messages = r#"{"model":"claude-opus-4-8","max_tokens":16,"temperature":null,"messages":[{"role":"user","content":"hi"}]}"#; + assert_eq!(send_raw(&gw, &sk, "/v1/messages", messages).await, 200); + let v = sent(&anthropic); + assert!( + v.as_object().unwrap().get("temperature") == Some(&serde_json::Value::Null), + "{v}" + ); +} + +/// `stream_options` is made to ask for usage only on a candidate that streams Chat Completions; a +/// Responses stream's own `stream_options` reaches its Responses upstream as the client sent it. +/// claim: BIL-2, CAT-9 +/// defect: D06 +#[tokio::test] +async fn a_responses_streams_options_are_not_given_include_usage() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["openai"]) + .start() + .await; + let body = r#"{"model":"gpt-5.4","stream":true,"stream_options":{"include_obfuscation":false},"store":false,"input":"hi"}"#; + send_raw(&gw, &sk, "/v1/responses", body).await; + let cap = mock.captured().unwrap(); + assert_eq!(cap.path, "/v1/responses"); + let v: serde_json::Value = serde_json::from_slice(&cap.body).unwrap(); + assert_eq!( + v["stream_options"], + serde_json::json!({"include_obfuscation": false}), + "{v}" + ); +} + +/// The reasoning items the gateway minted from Claude's thinking are cut out of a Responses body +/// ahead of `model` on its way to a Responses upstream (here xAI's, on a row with no Responses arm, +/// where no signed-id check makes the cut first); the body is scanned again after the cut, so +/// `model` is still re-spelled where it now is and nothing after it moves. +/// claim: SEC-21, CAT-9 +/// defect: D50 +#[tokio::test] +async fn a_reasoning_cut_ahead_of_the_model_still_respells_the_model() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["xai"]) + .start() + .await; + // Instructions after `model`, longer than the cut: a stale span would land inside them. + let instructions = "Answer in French. ".repeat(12); + let body = format!( + r#"{{"input":[{{"type":"reasoning","id":"rs_gw18f2c3a4b5d60001","encrypted_content":"rs_gw:EqQBsig==","summary":[]}},{{"role":"user","content":"hi"}}],"model":"grok-4.3","instructions":"{instructions}","store":false}}"# + ); + send_raw(&gw, &sk, "/v1/responses", &body).await; + let cap = mock.captured().unwrap(); + assert_eq!(cap.path, "/v1/responses"); + let v = sent(&mock); + assert_eq!(v["model"], "grok-4.3", "{v}"); + assert_eq!(v["instructions"], instructions.as_str(), "{v}"); + assert_eq!(v["input"].as_array().map(Vec::len), Some(1), "{v}"); +} diff --git a/crates/gateway/tests/claims_reliability.rs b/crates/gateway/tests/claims_reliability.rs index c8c056d5..f9e7a064 100644 --- a/crates/gateway/tests/claims_reliability.rs +++ b/crates/gateway/tests/claims_reliability.rs @@ -2031,3 +2031,92 @@ async fn rss_plateaus_under_sustained_load_and_large_bodies() { round {WARM} (limit {limit}); per round: {rss:?}" ); } + +/// On a catalog walk, a provider that keeps refusing streams gets the same one resend as on its +/// provider route, then the walk moves on (here, with no other candidate, the error stands): two +/// refusals, never a loop of them. +/// claim: REL-22 +/// defect: D91 +#[tokio::test] +async fn a_catalog_walk_resends_a_refused_stream_once() { + let (pubkey, sk) = test_keypair(224); + let key = billing_vkey(&sk, 2207); + let (port, served, refused, task) = refusing_h2_upstream(Refuse::Always).await; + let gw = Gateway::builder( + unused_nats_port(), + &format!("127.0.0.1:{port}"), + &b64(&pubkey), + ) + .providers(&["openai"]) + .tls_upstream() + .upstream_http2(true) + .start() + .await; + let resp = post(&gw, "/v1/chat/completions", &key, CHAT).await; + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap_or_default(); + task.abort(); + assert_eq!( + ( + status, + refused.load(Ordering::SeqCst), + served.load(Ordering::SeqCst) + ), + (502, 2, 0), + "one send and one resend: {text}" + ); +} + +/// A large body's walk gives every candidate its own one resend of a refused stream, then fails +/// over: the first two candidates refuse every stream, the third refuses only its first, and the +/// client gets the third's answer. Each refusal is counted against its own candidate, never +/// another's. +/// claim: REL-22, REL-21 +/// defect: D72 +#[tokio::test] +async fn each_candidate_of_a_large_body_walk_gets_one_refused_stream_resend() { + let (pubkey, sk) = test_keypair(225); + let key = billing_vkey(&sk, 2208); + let large = format!( + r#"{{"model":"deepseek-flash","messages":[{{"role":"user","content":"{}"}}]}}"#, + "x".repeat(200 * 1024) + ); + let (p0, served0, refused0, t0) = refusing_h2_upstream(Refuse::Always).await; + let (p1, served1, refused1, t1) = refusing_h2_upstream(Refuse::Always).await; + let (p2, served2, refused2, t2) = refusing_h2_upstream(Refuse::FirstStream).await; + let gw = Gateway::builder( + unused_nats_port(), + &format!("127.0.0.1:{p0}"), + &b64(&pubkey), + ) + .providers(&["deepseek", "together", "openrouter"]) + .provider_authority("deepseek", &format!("127.0.0.1:{p0}")) + .provider_authority("together", &format!("127.0.0.1:{p1}")) + .provider_authority("openrouter", &format!("127.0.0.1:{p2}")) + .tls_upstream() + .upstream_http2(true) + .start() + .await; + let resp = post(&gw, "/v1/chat/completions", &key, &large).await; + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap_or_default(); + for t in [t0, t1, t2] { + t.abort(); + } + let counts = |served: &AtomicUsize, refused: &AtomicUsize| { + ( + refused.load(Ordering::SeqCst), + served.load(Ordering::SeqCst), + ) + }; + assert_eq!(status, 200, "{text}\n{}", gw.log()); + assert_eq!( + [ + counts(&served0, &refused0), + counts(&served1, &refused1), + counts(&served2, &refused2) + ], + [(2, 0), (2, 0), (1, 1)], + "(refused, served) per candidate" + ); +} diff --git a/crates/gateway/tests/claims_security.rs b/crates/gateway/tests/claims_security.rs index ae802286..874dba3c 100644 --- a/crates/gateway/tests/claims_security.rs +++ b/crates/gateway/tests/claims_security.rs @@ -1667,3 +1667,62 @@ async fn bare_v1_byo_routes_by_the_forwarded_credential() { "an Anthropic key reached OpenAI" ); } + +/// A managed key anywhere makes a bare `/v1` request managed: BYO credentials for two providers +/// beside it are not ambiguous, because every credential location is stripped and the pool key +/// goes instead. The request is served, and no client credential reaches the provider. +/// claim: SEC-11 +/// defect: D82 +#[tokio::test] +async fn a_managed_key_beside_mixed_byo_credentials_is_not_ambiguous() { + let openai = MockUpstream::start(Mode::Json).await; + let (pubkey, sk) = test_keypair(82); + let gw = Gateway::builder(unused_nats_port(), &openai.authority(), &b64(&pubkey)) + .providers(&["openai"]) + .start() + .await; + let managed = format!("Bearer {}", billing_vkey(&sk, 821)); + let status = bare_byo( + &gw, + "/v1/chat/completions", + &[ + ("authorization", &managed), + ("x-api-key", "stray"), + ("x-api-key", "sk-proj-openai-byo"), + ], + ) + .await; + assert_eq!((status, openai.hits()), (200, 1), "{}", gw.log()); + let cap = openai.captured().unwrap(); + assert!(cap.x_api_key.is_none(), "{:?}", cap.x_api_key); + assert_ne!(cap.authorization.as_deref(), Some(managed.as_str())); +} + +/// A BYO `?key=` query parameter is the caller's own credential on its own account: forwarded with +/// the rest of the path, untouched. Only a managed (virtual) key is stripped from the query. +/// claim: SEC-11 +#[tokio::test] +async fn a_byo_key_query_parameter_is_forwarded() { + let mock = MockUpstream::start(Mode::Json).await; + let (pubkey, _sk) = test_keypair(82); + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .providers(&["openai"]) + .start() + .await; + let resp = test_client() + .post(format!( + "{}/openai/v1/chat/completions?key=byo-secret&x=1", + gw.url() + )) + .header("authorization", "Bearer sk-proj-openai-byo") + .header("content-type", "application/json") + .body(CHAT) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + assert_eq!( + mock.captured().unwrap().path, + "/v1/chat/completions?key=byo-secret&x=1" + ); +} diff --git a/crates/gateway/tests/errors_account_remedy.rs b/crates/gateway/tests/errors_account_remedy.rs index fd48a99f..c32ae0be 100644 --- a/crates/gateway/tests/errors_account_remedy.rs +++ b/crates/gateway/tests/errors_account_remedy.rs @@ -179,3 +179,134 @@ async fn a_request_over_the_tpm_limit_is_told_to_shrink_not_to_retry() { "the account behind the gateway stays out: {text}" ); } + +/// A streaming request's provider error is a JSON body like any other: the account remedy in it is +/// neutralized the same way, rather than relayed because the client asked for a stream. +/// claim: T6 +/// defect: D174 +#[tokio::test] +async fn a_streaming_requests_account_remedy_is_neutralized_too() { + let (pubkey, sk) = test_keypair(174); + let mock = MockUpstream::start(Mode::Raw( + 429, + "application/json", + OPENROUTER_SHARED_POOL_429, + )) + .await; + let gw = Gateway::builder( + unused_nats_port(), + &GatewayBuilder::dead_authority(), + &b64(&pubkey), + ) + .providers(&["openrouter"]) + .provider_authority("openrouter", &mock.authority()) + .start() + .await; + let key = billing_vkey(&sk, 174); + let resp = test_client() + .post(format!("{}/openrouter/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {key}")) + .header("content-type", "application/json") + .body(r#"{"model":"anthropic/claude-opus-4.8","stream":true,"messages":[{"role":"user","content":"hi"}]}"#) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap(); + let leaks = remedy_leaks(&text); + assert!( + status == 429 && leaks.is_empty(), + "{status}, leaks {leaks:?}: {text}\n{}", + gw.log() + ); +} + +/// The neutralized error is held whole and loses its `Content-Length`; on an HTTP/1.1 upstream it +/// goes to the client chunked, so the connection stays open for the client's next request. +/// claim: T6 +/// defect: D174 +#[tokio::test] +async fn a_rewritten_error_is_chunked_to_an_http1_client() { + let (pubkey, sk) = test_keypair(174); + let mock = MockUpstream::start(Mode::Raw( + 429, + "application/json", + OPENROUTER_SHARED_POOL_429, + )) + .await; + let gw = Gateway::builder( + unused_nats_port(), + &GatewayBuilder::dead_authority(), + &b64(&pubkey), + ) + .providers(&["openrouter"]) + .provider_authority("openrouter", &mock.authority()) + .start() + .await; + let key = billing_vkey(&sk, 174); + let body = + r#"{"model":"anthropic/claude-opus-4.8","messages":[{"role":"user","content":"hi"}]}"#; + let resp = test_client() + .post(format!("{}/openrouter/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {key}")) + .header("content-type", "application/json") + .body(body) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 429); + assert_eq!(resp.version(), reqwest::Version::HTTP_11); + let te = resp + .headers() + .get("transfer-encoding") + .map(|v| v.to_str().unwrap().to_owned()); + let length = resp.headers().get("content-length").cloned(); + let text = resp.text().await.unwrap(); + assert_eq!((te.as_deref(), length), (Some("chunked"), None), "{text}"); + assert!(remedy_leaks(&text).is_empty(), "{text}"); +} + +/// Only a JSON error is held whole for the rewrite: a plain-text one (an edge's HTML or text +/// page) has no remedy to find and is relayed as sent, its `Content-Length` kept. +/// claim: T6 +/// defect: D174 +#[tokio::test] +async fn a_non_json_error_is_relayed_with_its_length() { + const PLAIN: &str = "upstream connect error or disconnect/reset before headers"; + let (pubkey, sk) = test_keypair(174); + let mock = MockUpstream::start(Mode::Raw(400, "text/plain", PLAIN)).await; + let gw = Gateway::builder( + unused_nats_port(), + &GatewayBuilder::dead_authority(), + &b64(&pubkey), + ) + .providers(&["openrouter"]) + .provider_authority("openrouter", &mock.authority()) + .start() + .await; + let key = billing_vkey(&sk, 174); + let resp = test_client() + .post(format!("{}/openrouter/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {key}")) + .header("content-type", "application/json") + .body( + r#"{"model":"anthropic/claude-opus-4.8","messages":[{"role":"user","content":"hi"}]}"#, + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 400); + let length = resp + .headers() + .get("content-length") + .map(|v| v.to_str().unwrap().to_owned()); + let te = resp.headers().get("transfer-encoding").cloned(); + let text = resp.text().await.unwrap(); + assert_eq!(text, PLAIN); + assert_eq!( + (length, te), + (Some(PLAIN.len().to_string()), None), + "{}", + gw.log() + ); +} diff --git a/crates/gateway/tests/reliability_large_body.rs b/crates/gateway/tests/reliability_large_body.rs index 12a317c6..88bad8fa 100644 --- a/crates/gateway/tests/reliability_large_body.rs +++ b/crates/gateway/tests/reliability_large_body.rs @@ -340,3 +340,44 @@ async fn a_large_body_failover_charges_the_breaker_it_left() { gw.log() ); } + +/// A large body's 429 key walk reaches every pool key, however many: thirty keys, the last one +/// served, thirty uploads. The bound on a full-body walk's attempts guards a looping bug, not a +/// long legitimate walk; and when a walk does reach it (one hundred and fifty keys, all +/// throttled), the last attempt it allows is the client's answer, the provider's own 429, not a +/// synthetic 502. +/// claim: REL-21, R5 +/// defect: D81 +#[tokio::test] +async fn a_large_body_walks_every_pool_key_up_to_the_attempt_bound() { + let (pubkey, sk) = test_keypair(1); + for (n, served_by) in [(30usize, Some("Bearer sk-k029")), (150, None)] { + let keys: Vec = (0..n).map(|i| format!("sk-k{i:03}")).collect(); + let keys: Vec<&str> = keys.iter().map(String::as_str).collect(); + let primary = ReplyUpstream::start(move |_, req| { + if served_by.is_some() && req.authorization.as_deref() == served_by { + Reply::ok() + } else { + Reply::json(429, r#"{"error":{"message":"slow down"}}"#) + } + }) + .await; + let gw = Gateway::builder(unused_nats_port(), &primary.authority(), &b64(&pubkey)) + .providers(&["openai"]) + .pool_keys("openai", &keys) + .start() + .await; + let out = send(&gw, &vkey(&sk), LARGE).await; + let want = if served_by.is_some() { + (200, n) + } else { + (429, 144) + }; + assert_eq!( + (out.status, primary.hits()), + want, + "{n} keys: {out:?}\n{}", + gw.log() + ); + } +} diff --git a/crates/gateway/tests/reliability_lifecycle.rs b/crates/gateway/tests/reliability_lifecycle.rs index 41e1791d..b3694f18 100644 --- a/crates/gateway/tests/reliability_lifecycle.rs +++ b/crates/gateway/tests/reliability_lifecycle.rs @@ -791,3 +791,162 @@ async fn a_large_body_reserves_every_copy_it_will_hold_before_it_is_read() { "every refusal came before an upstream attempt" ); } + +/// A body pingora can replay from its own 64 KiB buffer is bounded by concurrency, not charged to +/// the body budget: 60 KiB and exactly 64 KiB are served under a budget smaller than two copies of +/// either. Only a body past the buffer reserves (the test above). +/// claim: SEC-19 +/// defect: D35 +#[tokio::test] +async fn a_body_within_the_replay_buffer_is_not_charged_to_the_budget() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .config_line("max_buffered_body_bytes = 100000") + .start() + .await; + let chat = |len: usize| { + let empty = r#"{"model":"gpt-4o-mini","messages":[{"role":"user","content":""}]}"#; + let body = format!( + r#"{{"model":"gpt-4o-mini","messages":[{{"role":"user","content":"{}"}}]}}"#, + "x".repeat(len - empty.len()) + ); + assert_eq!(body.len(), len); + body + }; + for len in [60 * 1024, 64 * 1024] { + let resp = test_client() + .post(format!("{}/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {}", vkey(&sk, 35))) + .header("content-type", "application/json") + .body(chat(len)) + .send() + .await + .unwrap(); + let status = resp.status().as_u16(); + let text = resp.text().await.unwrap_or_default(); + assert_eq!(status, 200, "{len} B: {text}"); + } + assert_eq!(mock.hits(), 2); +} + +/// A chunked body (no `Content-Length`, so nothing to reserve up front) that outgrows the budget +/// while the walk reads it is the budget's retryable 503, not "request body too large": it is far +/// below the size cap. +/// claim: SEC-19 +/// defect: D35 +#[tokio::test] +async fn a_chunked_body_that_outgrows_the_budget_is_a_503_not_a_413() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .config_line("max_buffered_body_bytes = 1048576") + .start() + .await; + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", gw.port)) + .await + .unwrap(); + let head = format!( + "POST /v1/chat/completions HTTP/1.1\r\nhost: gw\r\nauthorization: Bearer {}\r\n\ + content-type: application/json\r\ntransfer-encoding: chunked\r\n\r\n", + vkey(&sk, 35) + ); + s.write_all(head.as_bytes()).await.unwrap(); + // 700 KiB in 64 KiB chunks: held twice, past the 1 MiB budget half way through. + let open = br#"{"model":"gpt-4o-mini","messages":[{"role":"user","content":""#; + let mut body = open.to_vec(); + body.resize(700 * 1024, b'x'); + body.extend_from_slice(br#""}]}"#); + for chunk in body.chunks(64 * 1024) { + let frame = [format!("{:x}\r\n", chunk.len()).as_bytes(), chunk, b"\r\n"].concat(); + // The gateway may answer, and stop reading, before the last chunk. + if s.write_all(&frame).await.is_err() { + break; + } + } + let _ = s.write_all(b"0\r\n\r\n").await; + let mut buf = vec![0u8; 4096]; + let n = tokio::time::timeout(Duration::from_secs(10), s.read(&mut buf)) + .await + .expect("an answer") + .unwrap(); + let text = String::from_utf8_lossy(&buf[..n]); + assert!( + text.starts_with("HTTP/1.1 503") && text.contains("too many large request bodies"), + "{text}" + ); + assert_eq!(mock.hits(), 0); +} + +/// The size cap is inclusive: a body declared at exactly `MAX_REQUEST_BODY` (100 MiB) is read, not +/// refused up front the way one byte more is. +/// claim: SEC-19 +#[tokio::test] +async fn a_body_declared_at_the_size_cap_is_not_refused_up_front() { + let (pubkey, sk) = test_keypair(1); + let mock = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .start() + .await; + const CAP: usize = 100 * 1024 * 1024; + for (len, refused) in [(CAP + 1, true), (CAP, false)] { + let mut s = tokio::net::TcpStream::connect(("127.0.0.1", gw.port)) + .await + .unwrap(); + let head = format!( + "POST /v1/chat/completions HTTP/1.1\r\nhost: gw\r\nauthorization: Bearer {}\r\n\ + content-type: application/json\r\ncontent-length: {len}\r\n\r\n{{\"model\":\"gpt-4o-mini\"", + vkey(&sk, 35) + ); + s.write_all(head.as_bytes()).await.unwrap(); + // Refused up front, the answer comes at once; read, the walk waits for the rest of it. + let mut buf = vec![0u8; 4096]; + let answer = tokio::time::timeout(Duration::from_secs(2), s.read(&mut buf)).await; + let text = match answer { + Ok(Ok(n)) => String::from_utf8_lossy(&buf[..n]).into_owned(), + _ => String::new(), + }; + assert_eq!( + text.starts_with("HTTP/1.1 413"), + refused, + "{len} B declared: {text:?}" + ); + } +} + +/// `client_write_timeout_secs = 0` disables the downstream write timeout rather than making it +/// zero: a client that is slow to start reading a large stream still gets every byte of it. +/// claim: REL-10 +/// defect: D42 +#[tokio::test] +async fn a_zero_client_write_timeout_disables_it() { + let (pubkey, _sk) = test_keypair(1); + let big = { + let event = "data: {\"choices\":[{\"delta\":{\"content\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}}]}\n\n"; + bytes::Bytes::from(event.repeat((32 << 20) / event.len())) + }; + let want = big.len(); + let mock = ReplyUpstream::start(move |_, _| Reply::Full { + status: 200, + content_type: "text/event-stream", + body: big.clone(), + }) + .await; + let gw = Gateway::builder(unused_nats_port(), &mock.authority(), &b64(&pubkey)) + .config_line("client_write_timeout_secs = 0") + .start() + .await; + let resp = test_client() + .post(format!("{}/openai/v1/chat/completions", gw.url())) + .header("authorization", "Bearer sk-byo-test") + .header("content-type", "application/json") + .body(STREAM_BODY) + .send() + .await + .unwrap(); + assert_eq!(resp.status().as_u16(), 200); + // Let the socket buffers fill before reading: every write after that has to wait. + tokio::time::sleep(Duration::from_millis(500)).await; + let got = resp.bytes().await.map(|b| b.len()); + assert_eq!(got.ok(), Some(want), "{}", gw.log()); +} diff --git a/crates/gateway/tests/reliability_unfunded_key.rs b/crates/gateway/tests/reliability_unfunded_key.rs index 2ed410a2..92402dbe 100644 --- a/crates/gateway/tests/reliability_unfunded_key.rs +++ b/crates/gateway/tests/reliability_unfunded_key.rs @@ -210,3 +210,54 @@ async fn each_cooled_key_is_counted_with_its_reason() { } assert!(failures.is_empty(), "{}", failures.join("\n")); } + +/// The cooldown is per candidate wherever it sits in the row: an unfunded provider at the row's +/// second catalog slot, walked first by `x-beyond-order`, is passed over for the next requests +/// just as one in the first slot is. (Its out-of-quota 429 is relayed, and cools its key.) +/// claim: REL-4 +/// defect: D180 +#[tokio::test] +async fn an_unfunded_candidate_cools_off_from_any_catalog_slot() { + let (pubkey, sk) = test_keypair(180); + let key = billing_vkey(&sk, 181); + let unfunded = MockUpstream::start(Mode::Raw(429, "application/json", OPENAI_NO_QUOTA)).await; + let openai = MockUpstream::start(Mode::Json).await; + let gw = Gateway::builder(unused_nats_port(), &openai.authority(), &b64(&pubkey)) + .providers(&["openai", "openrouter"]) + .provider_authority("openrouter", &unfunded.authority()) + .start() + .await; + let mut served = Vec::new(); + for _ in 0..3 { + let resp = test_client() + .post(format!("{}/v1/chat/completions", gw.url())) + .header("authorization", format!("Bearer {key}")) + .header("content-type", "application/json") + .header("x-beyond-order", "openrouter,openai") + .body(r#"{"model":"gpt-4o-mini","messages":[{"role":"user","content":"hi"}]}"#) + .send() + .await + .unwrap(); + let by = resp + .headers() + .get("x-beyond-provider") + .map(|v| v.to_str().unwrap().to_owned()); + served.push((resp.status().as_u16(), by)); + } + let by = |p: &str| Some(p.to_owned()); + assert_eq!( + served, + [ + (429, by("openrouter")), + (200, by("openai")), + (200, by("openai")) + ], + "{}", + gw.log() + ); + assert_eq!( + unfunded.hits(), + 1, + "OpenRouter was tried after it said it had no quota" + ); +} From 69d61744f22398a87a8fdf04b8a41a11743ba7d7 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Sat, 3 Oct 2026 22:04:42 -0700 Subject: [PATCH 11/11] gateway tests: split the load-flaky replica breaker test; exclude four equivalent mutants The replica breaker test opened A's breaker with a 2s reset and then asserted A routed away for three turns: on a loaded host the turns outran the reset and A's half-open probe went to the healed primary. That flake also made cargo-mutants count mutants as caught when nothing else failed. Now two tests: open-phase routing with a reset far past the test, and recovery with a 1s reset, polled. ChatIdentity::key's & -> ^ keeps keys distinct (XOR with a mask is a bijection in range); tally_eager (now its own fn) only picks where the same input estimate is made. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .cargo/mutants.toml | 11 +++- crates/gateway/src/proxy.rs | 10 +++- crates/gateway/tests/replicas.rs | 97 ++++++++++++++++++++++---------- verify/defects.toml | 2 +- 4 files changed, 83 insertions(+), 37 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index ecbc7712..77132521 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -47,8 +47,15 @@ exclude_re = [ # `KeepInputs::expecting` only words a parse error's message, which `parse_body` discards (it # keeps `None`); `visit_unit`'s `Value::Null` is `Value::default()`. "replace for KeepInputs>::(expecting|visit_unit)", - # `ChatIdentity::key` packs three masked fields into disjoint bit ranges: `|` and `^` agree. - 'replace \| with \^ in ChatIdentity::key', + # `ChatIdentity::key` packs three masked fields into disjoint bit ranges: `|` and `^` agree. The + # key is only compared for equality, and XOR with a field's mask is a bijection on the values in + # range, so `&` -> `^` keeps every key distinct too. + 'replace (\||&) with \^ in ChatIdentity::key', + # `tally_eager` only picks where the input estimate is made: as the body streams, or in `logging` + # from the copy that is still held (pingora's retry buffer or the `FullBody` parent). Both tally + # the same request text; widening it (`||`, or `true`) only tallies eagerly where the held copy + # would have done. + 'replace (&& with \|\| in tally_eager|tally_eager -> bool with true)', # `map_response` clones a same-wire body as is: the guard only skips that clone. 'replace match guard client == Endpoint::Responses with false in SseBridge::assembled', # `str_is`'s raw compare is a fast path: `escaped_key_is` decodes a string with no escape to diff --git a/crates/gateway/src/proxy.rs b/crates/gateway/src/proxy.rs index 8c045b87..622c10da 100644 --- a/crates/gateway/src/proxy.rs +++ b/crates/gateway/src/proxy.rs @@ -2224,6 +2224,12 @@ fn past_replay_buffer(n: usize) -> bool { n > BODY_PEEK_LIMIT } +/// [`RequestCtx::tally_eager`]: a managed provider-routed body that may outgrow the retry buffer +/// is the one case where no copy is left for `logging` to tally. +fn tally_eager(managed: bool, catalog: bool, declared_len: Option) -> bool { + managed && !catalog && declared_len.is_none_or(past_replay_buffer) +} + /// The `x-beyond-model` header, viewed as a catalog lookup. Present-but-unknown is distinct from /// absent: the header wins, so an unknown header is a 404 rather than a fall-through to the body. #[derive(Clone, Copy)] @@ -4642,9 +4648,7 @@ impl ProxyHttp for AiProxy { }), request_id, input_tally: usage::InputTally::default(), - tally_eager: managed - && model_route.is_none() - && declared_len.is_none_or(past_replay_buffer), + tally_eager: tally_eager(managed, model_route.is_some(), declared_len), resp_bytes: 0, upstream_phase: UpstreamPhase::None, redact: None, diff --git a/crates/gateway/tests/replicas.rs b/crates/gateway/tests/replicas.rs index 1b88004b..27fe6abe 100644 --- a/crates/gateway/tests/replicas.rs +++ b/crates/gateway/tests/replicas.rs @@ -19,6 +19,8 @@ mod common; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use common::*; @@ -486,41 +488,19 @@ async fn a_failover_pin_returns_to_the_primary_once_it_is_healthy() { } /// The residual divergence, stated: a replica whose breaker for the preferred host is open routes -/// the session to the next candidate, and only while the breaker is open. The other replica, whose -/// breaker never opened, keeps serving the preferred host; once the open breaker's probe succeeds, -/// both are back on it. +/// the session to the next candidate, while the other replica, whose breaker never opened, keeps +/// serving the preferred host. The reset is far past the test's length so the breaker stays open +/// however slowly a loaded host runs the turns (a short reset let it half-open mid-loop). /// claim: R4, R6 /// defect: D253 #[tokio::test] -async fn a_replica_with_an_open_breaker_routes_away_only_while_it_is_open() { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; +async fn a_replica_with_an_open_breaker_routes_away_while_the_other_keeps_the_host() { let nats_port = unused_nats_port(); let (pubkey, sk) = test_keypair(50); - let failing = Arc::new(AtomicBool::new(true)); - let primary = ReplyUpstream::start({ - let failing = failing.clone(); - move |_, _| { - if failing.load(Ordering::Relaxed) { - Reply::json(500, r#"{"error":{"message":"outage"}}"#) - } else { - Reply::json(200, OK_JSON) - } - } - }) - .await; + let (primary, failing) = flaky_primary().await; let fallback = MockUpstream::start(Mode::Json).await; - let replica = || { - Gateway::builder(nats_port, &primary.authority(), &b64(&pubkey)) - .providers(&["openai", "openrouter"]) - .provider_authority("openrouter", &fallback.authority()) - .config_line("circuit_breaker_threshold = 2") - .config_line("circuit_breaker_window_secs = 60") - .config_line("circuit_breaker_reset_secs = 2") - .start() - }; - let a = replica().await; - let b = replica().await; + let a = breaker_replica(nats_port, &primary, &fallback, &pubkey, 600).await; + let b = breaker_replica(nats_port, &primary, &fallback, &pubkey, 600).await; let key = billing_vkey(&sk, 5001); // Only A meets the outage: two failed-over turns open its breaker for the primary. for _ in 0..2 { @@ -541,10 +521,29 @@ async fn a_replica_with_an_open_breaker_routes_away_only_while_it_is_open() { ); } assert_eq!(primary.hits(), hits + 3, "only B reached the primary"); - // Past the reset the half-open probe succeeds and A is back on the primary. +} + +/// The other half: once the open breaker's half-open probe succeeds, the replica that routed away +/// is back on the preferred host, and both replicas agree again. Polled, so a loaded host only +/// makes it slower. +/// claim: R4, R6 +/// defect: D253 +#[tokio::test] +async fn a_replica_returns_to_the_host_once_its_breaker_probe_succeeds() { + let nats_port = unused_nats_port(); + let (pubkey, sk) = test_keypair(51); + let (primary, failing) = flaky_primary().await; + let fallback = MockUpstream::start(Mode::Json).await; + let a = breaker_replica(nats_port, &primary, &fallback, &pubkey, 1).await; + let b = breaker_replica(nats_port, &primary, &fallback, &pubkey, 1).await; + let key = billing_vkey(&sk, 5002); + for _ in 0..2 { + assert_eq!(post_auto(&a, &key).await, "openrouter"); + } + failing.store(false, Ordering::Relaxed); let start = Instant::now(); let mut on_a = String::new(); - while start.elapsed() < Duration::from_secs(10) { + while start.elapsed() < Duration::from_secs(20) { on_a = post_auto(&a, &key).await; if on_a == "openai" { break; @@ -558,6 +557,42 @@ async fn a_replica_with_an_open_breaker_routes_away_only_while_it_is_open() { } } +/// A primary that answers 500 while the flag is set, then 200. +async fn flaky_primary() -> (ReplyUpstream, Arc) { + let failing = Arc::new(AtomicBool::new(true)); + let primary = ReplyUpstream::start({ + let failing = failing.clone(); + move |_, _| { + if failing.load(Ordering::Relaxed) { + Reply::json(500, r#"{"error":{"message":"outage"}}"#) + } else { + Reply::json(200, OK_JSON) + } + } + }) + .await; + (primary, failing) +} + +/// A replica preferring `primary` (openai) with `fallback` as openrouter, whose breaker opens after +/// two failures and half-opens after `reset_secs`. +async fn breaker_replica( + nats_port: u16, + primary: &ReplyUpstream, + fallback: &MockUpstream, + pubkey: &[u8], + reset_secs: u64, +) -> Gateway { + Gateway::builder(nats_port, &primary.authority(), &b64(pubkey)) + .providers(&["openai", "openrouter"]) + .provider_authority("openrouter", &fallback.authority()) + .config_line("circuit_breaker_threshold = 2") + .config_line("circuit_breaker_window_secs = 60") + .config_line(&format!("circuit_breaker_reset_secs = {reset_secs}")) + .start() + .await +} + /// `tenant_max_in_flight` is per process: with a cap of 1 on each of two replicas, a tenant holding /// its one slot on A is refused a second there (429) and still admitted on B. N replicas admit /// N × the cap, so the overspend bound scales with the replica count. diff --git a/verify/defects.toml b/verify/defects.toml index accb8dc3..197b961c 100644 --- a/verify/defects.toml +++ b/verify/defects.toml @@ -2484,7 +2484,7 @@ claims = ["R4"] state = "fixed" test = "a_pinned_conversation_stays_on_one_provider_across_replicas" fix = "a0cc11f" -note = "Fixed statelessly (no shared store, no NATS, no load-balancer affinity): session pins are computed every turn and never remembered; the per-pod pin table is gone. smart.rs preferred(): the row's leading run of equally-preferred first-party hosts (providers with their own model-id prefix: Anthropic + Bedrock, never an aggregator) ordered by rendezvous hash of the pin key (tenant, vpc, key_id, row) and provider name, then catalog order; only candidates unusable on the pod (unkeyed, every key cooling, body unserved) move back, and upstream_peer skips an open breaker. No TTFT, probe or failure history, so a failure fails over for that request only. The TTFT ranker now orders only walks with no caller identity. An error-in-200 now counts against the serving candidate's breaker (settle_health), since the pin no longer learns from failures. Residual divergence, documented in ARCHITECTURE.md Running several replicas: per-pod breakers (and key cooling) during a real outage. Also tested: a_pooled_session_reaches_one_provider_on_every_replica (20 alternating turns, 10 sessions), a_failover_pin_returns_to_the_primary_once_it_is_healthy, a_replica_with_an_open_breaker_routes_away_only_while_it_is_open, smart::tests::every_pod_computes_the_same_pin, a_failure_does_not_move_the_next_turn. Follow-up: the live TEN-2+SEC-15 pin_isolation cells look for keys pinned to two different providers on anthropic+openrouter; with computed pins every key lands on anthropic, so those cells need rework (not run here)." +note = "Fixed statelessly (no shared store, no NATS, no load-balancer affinity): session pins are computed every turn and never remembered; the per-pod pin table is gone. smart.rs preferred(): the row's leading run of equally-preferred first-party hosts (providers with their own model-id prefix: Anthropic + Bedrock, never an aggregator) ordered by rendezvous hash of the pin key (tenant, vpc, key_id, row) and provider name, then catalog order; only candidates unusable on the pod (unkeyed, every key cooling, body unserved) move back, and upstream_peer skips an open breaker. No TTFT, probe or failure history, so a failure fails over for that request only. The TTFT ranker now orders only walks with no caller identity. An error-in-200 now counts against the serving candidate's breaker (settle_health), since the pin no longer learns from failures. Residual divergence, documented in ARCHITECTURE.md Running several replicas: per-pod breakers (and key cooling) during a real outage. Also tested: a_pooled_session_reaches_one_provider_on_every_replica (20 alternating turns, 10 sessions), a_failover_pin_returns_to_the_primary_once_it_is_healthy, a_replica_with_an_open_breaker_routes_away_while_the_other_keeps_the_host, a_replica_returns_to_the_host_once_its_breaker_probe_succeeds, smart::tests::every_pod_computes_the_same_pin, a_failure_does_not_move_the_next_turn. Follow-up: the live TEN-2+SEC-15 pin_isolation cells look for keys pinned to two different providers on anthropic+openrouter; with computed pins every key lands on anthropic, so those cells need rework (not run here)." [[defect]] id = "D251"