From d769b1906e587555f2bdc68feaaec1218ac3664c Mon Sep 17 00:00:00 2001 From: Sergi Torres Albert Date: Tue, 28 Jul 2026 23:56:50 +0200 Subject: [PATCH] fix: keep max_tokens at 512 and stop killing slow generations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the cap to 512 and moves the fix to where the defect actually was. The cap was never wasting anything: across the ten recorded generations in bob/sessions/Sprint_1/baseline_eval.md the longest output was 415 tokens and all ten ended on a complete sentence. Lowering it to 320 would have cut real prose to buy latency headroom. The real problem is that HARD_TIMEOUT_SECONDS was the MVP's *SLA* number (<8s P95) used as a *kill* threshold, so no band existed in which a generation is late but working: a conditioned call measured at 8.4s worst case got retried and ultimately failed rather than returned, surfacing as an intermittent 503. The timeout goes to 15s (~1.8x that worst case). The SLA is unchanged and still tracked — now measured on results rather than enforced by discarding them. The client abort in frontend/src/lib/api.ts goes 10s -> 20s for the same reason: at 10s the browser would cancel requests the backend was about to answer, showing a timeout for a generation that had in fact succeeded. Decision by the repo owner (2026-07-28): the demo is an edited video, not a live timed run, so complete prose is worth more than a shorter worst case. Verified: pytest backend/tests/test_generation_params.py -q -> 3 passed; npx vitest run (frontend) -> 107 passed. Co-Authored-By: Claude Opus 5 --- ai_pipeline/autoria_ai/generator.py | 26 ++++++++++++++----------- backend/app/services/watsonx_client.py | 11 ++++++++++- backend/tests/test_generation_params.py | 2 +- frontend/src/lib/api.ts | 10 ++++++++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/ai_pipeline/autoria_ai/generator.py b/ai_pipeline/autoria_ai/generator.py index a697896..3448474 100644 --- a/ai_pipeline/autoria_ai/generator.py +++ b/ai_pipeline/autoria_ai/generator.py @@ -42,23 +42,27 @@ # an unknown key is dropped in silence, not rejected. ``max_new_tokens`` is the # *text-generation* spelling and has no effect on ``ModelInference.chat()``, # which is what we call — see backend/tests/test_generation_params.py. -# 320, not the 512 originally written here, because 512 does not fit the -# latency budget. Measured against real Watsonx (eu-de, llama-3-3-70b, n=5 per -# cell, hard timeout lifted so the true duration shows): +# The cap stays at 512 because nothing is wasted there: across ten recorded +# generations (bob/sessions/Sprint_1/baseline_eval.md) the longest output was +# 415 tokens and all ten ended on a complete sentence. Lowering the cap would +# cut real prose, not slack. +# +# Measured against real Watsonx (eu-de, llama-3-3-70b, n=5 per cell, hard +# timeout lifted so the true duration shows): # # cap=512 vanilla median 4.5s worst 5.9s (~196 words) -# autoria median 7.2s worst 8.4s (~313 words) <- over the 8s cap +# autoria median 7.2s worst 8.4s (~313 words) # cap=320 vanilla median 5.0s worst 5.3s (~242 words) # autoria median 5.4s worst 5.6s (~223 words) # -# The conditioned branch straddles HARD_TIMEOUT_SECONDS at 512: it sometimes -# returns and sometimes exhausts all four attempts, and because a failed -# AutorIA branch cannot be degraded away (the passport needs it), that surfaces -# as an intermittent 503 from POST /api/generate. 320 keeps a comfortable -# margin under both the 8s timeout and the 10s client-side abort in -# frontend/src/lib/api.ts, and still yields ~220 words per column. +# At 512 the conditioned branch runs past the *old* 8s HARD_TIMEOUT_SECONDS +# often enough to matter, and a failed AutorIA branch cannot be degraded away +# because the passport needs it — that surfaced as intermittent 503s. The fix +# is on the timeout side, not the cap side: see HARD_TIMEOUT_SECONDS in +# backend/app/services/watsonx_client.py. A generation that is merely slow +# should arrive late, not die. _GENERATION_PARAMS: dict[str, Any] = { - "max_tokens": 320, + "max_tokens": 512, "temperature": 0.7, "top_p": 0.9, } diff --git a/backend/app/services/watsonx_client.py b/backend/app/services/watsonx_client.py index af4c790..c61bc0f 100644 --- a/backend/app/services/watsonx_client.py +++ b/backend/app/services/watsonx_client.py @@ -20,7 +20,16 @@ logger = logging.getLogger(__name__) -HARD_TIMEOUT_SECONDS = 8.0 +# 15s, not the 8s this used to be. The old value was the MVP's *SLA* number +# ("side-by-side <8s P95", docs/MVP.md) used as a *kill* threshold, which left +# no band in which a generation is late but still working: a conditioned call +# measured at 8.4s worst case was retried and ultimately failed rather than +# returned, surfacing as an intermittent 503 (#106). +# +# 15s is ~1.8x the worst conditioned generation observed at max_tokens=512 +# (8.4s, eu-de, llama-3-3-70b). The SLA is unchanged and still tracked; it is +# now measured on results instead of enforced by killing them. +HARD_TIMEOUT_SECONDS = 15.0 # Three retries after the first failure → four attempts total. _RETRY_DELAYS_SECONDS: tuple[float, ...] = (1.0, 2.0, 4.0) _DEFAULT_URL = "https://us-south.ml.cloud.ibm.com" diff --git a/backend/tests/test_generation_params.py b/backend/tests/test_generation_params.py index 171dbae..28beebc 100644 --- a/backend/tests/test_generation_params.py +++ b/backend/tests/test_generation_params.py @@ -64,7 +64,7 @@ def test_generation_params_match_chat_schema(): def test_output_cap_is_declared(): """The output cap must be present — an absent key falls back to 1024.""" - assert _GENERATION_PARAMS.get("max_tokens") == 320 + assert _GENERATION_PARAMS.get("max_tokens") == 512 assert "max_new_tokens" not in _GENERATION_PARAMS diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index f35fcd3..7fe3cd7 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -179,9 +179,15 @@ export function slugifyAuthorName(name: string): string { /** * POST /api/generate — run vanilla + AutorIA generation in parallel on the backend. * - * Uses AbortController with a 10 s timeout so a hanging Watsonx call resolves + * Uses AbortController with a 20 s timeout so a hanging Watsonx call resolves * to a timeout state rather than an infinite spinner (design-system §1 / §7.4). * + * 20 s, not the 10 s this used to be, so the client outlives the server: the + * backend allows a single Watsonx attempt 15 s (HARD_TIMEOUT_SECONDS) and the + * worst conditioned generation measured at max_tokens=512 took 8.4 s. A 10 s + * abort here would cancel requests the backend was about to answer, and the + * user would see a timeout for a generation that actually succeeded. + * * Return contract: * 200 → resolves with GenerateResponse * AbortError → throws with name "AbortError" (timeout path) @@ -192,7 +198,7 @@ export async function generateText( prompt: string, ): Promise { const controller = new AbortController(); - const timerId = setTimeout(() => controller.abort(), 10_000); + const timerId = setTimeout(() => controller.abort(), 20_000); try { const body: GenerateRequest = { author_id: authorId, prompt };