diff --git a/ai_pipeline/autoria_ai/generator.py b/ai_pipeline/autoria_ai/generator.py index a697896..3448474 100644 --- a/ai_pipeline/autoria_ai/generator.py +++ b/ai_pipeline/autoria_ai/generator.py @@ -42,23 +42,27 @@ # an unknown key is dropped in silence, not rejected. ``max_new_tokens`` is the # *text-generation* spelling and has no effect on ``ModelInference.chat()``, # which is what we call — see backend/tests/test_generation_params.py. -# 320, not the 512 originally written here, because 512 does not fit the -# latency budget. Measured against real Watsonx (eu-de, llama-3-3-70b, n=5 per -# cell, hard timeout lifted so the true duration shows): +# The cap stays at 512 because nothing is wasted there: across ten recorded +# generations (bob/sessions/Sprint_1/baseline_eval.md) the longest output was +# 415 tokens and all ten ended on a complete sentence. Lowering the cap would +# cut real prose, not slack. +# +# Measured against real Watsonx (eu-de, llama-3-3-70b, n=5 per cell, hard +# timeout lifted so the true duration shows): # # cap=512 vanilla median 4.5s worst 5.9s (~196 words) -# autoria median 7.2s worst 8.4s (~313 words) <- over the 8s cap +# autoria median 7.2s worst 8.4s (~313 words) # cap=320 vanilla median 5.0s worst 5.3s (~242 words) # autoria median 5.4s worst 5.6s (~223 words) # -# The conditioned branch straddles HARD_TIMEOUT_SECONDS at 512: it sometimes -# returns and sometimes exhausts all four attempts, and because a failed -# AutorIA branch cannot be degraded away (the passport needs it), that surfaces -# as an intermittent 503 from POST /api/generate. 320 keeps a comfortable -# margin under both the 8s timeout and the 10s client-side abort in -# frontend/src/lib/api.ts, and still yields ~220 words per column. +# At 512 the conditioned branch runs past the *old* 8s HARD_TIMEOUT_SECONDS +# often enough to matter, and a failed AutorIA branch cannot be degraded away +# because the passport needs it — that surfaced as intermittent 503s. The fix +# is on the timeout side, not the cap side: see HARD_TIMEOUT_SECONDS in +# backend/app/services/watsonx_client.py. A generation that is merely slow +# should arrive late, not die. _GENERATION_PARAMS: dict[str, Any] = { - "max_tokens": 320, + "max_tokens": 512, "temperature": 0.7, "top_p": 0.9, } diff --git a/backend/app/services/watsonx_client.py b/backend/app/services/watsonx_client.py index af4c790..c61bc0f 100644 --- a/backend/app/services/watsonx_client.py +++ b/backend/app/services/watsonx_client.py @@ -20,7 +20,16 @@ logger = logging.getLogger(__name__) -HARD_TIMEOUT_SECONDS = 8.0 +# 15s, not the 8s this used to be. The old value was the MVP's *SLA* number +# ("side-by-side <8s P95", docs/MVP.md) used as a *kill* threshold, which left +# no band in which a generation is late but still working: a conditioned call +# measured at 8.4s worst case was retried and ultimately failed rather than +# returned, surfacing as an intermittent 503 (#106). +# +# 15s is ~1.8x the worst conditioned generation observed at max_tokens=512 +# (8.4s, eu-de, llama-3-3-70b). The SLA is unchanged and still tracked; it is +# now measured on results instead of enforced by killing them. +HARD_TIMEOUT_SECONDS = 15.0 # Three retries after the first failure → four attempts total. _RETRY_DELAYS_SECONDS: tuple[float, ...] = (1.0, 2.0, 4.0) _DEFAULT_URL = "https://us-south.ml.cloud.ibm.com" diff --git a/backend/tests/test_generation_params.py b/backend/tests/test_generation_params.py index 171dbae..28beebc 100644 --- a/backend/tests/test_generation_params.py +++ b/backend/tests/test_generation_params.py @@ -64,7 +64,7 @@ def test_generation_params_match_chat_schema(): def test_output_cap_is_declared(): """The output cap must be present — an absent key falls back to 1024.""" - assert _GENERATION_PARAMS.get("max_tokens") == 320 + assert _GENERATION_PARAMS.get("max_tokens") == 512 assert "max_new_tokens" not in _GENERATION_PARAMS diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index f35fcd3..7fe3cd7 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -179,9 +179,15 @@ export function slugifyAuthorName(name: string): string { /** * POST /api/generate — run vanilla + AutorIA generation in parallel on the backend. * - * Uses AbortController with a 10 s timeout so a hanging Watsonx call resolves + * Uses AbortController with a 20 s timeout so a hanging Watsonx call resolves * to a timeout state rather than an infinite spinner (design-system §1 / §7.4). * + * 20 s, not the 10 s this used to be, so the client outlives the server: the + * backend allows a single Watsonx attempt 15 s (HARD_TIMEOUT_SECONDS) and the + * worst conditioned generation measured at max_tokens=512 took 8.4 s. A 10 s + * abort here would cancel requests the backend was about to answer, and the + * user would see a timeout for a generation that actually succeeded. + * * Return contract: * 200 → resolves with GenerateResponse * AbortError → throws with name "AbortError" (timeout path) @@ -192,7 +198,7 @@ export async function generateText( prompt: string, ): Promise { const controller = new AbortController(); - const timerId = setTimeout(() => controller.abort(), 10_000); + const timerId = setTimeout(() => controller.abort(), 20_000); try { const body: GenerateRequest = { author_id: authorId, prompt };