From 7582d09cf8931cb555278019b8737bf5fe7da199 Mon Sep 17 00:00:00 2001 From: Eman Cickusic Date: Sat, 11 Jul 2026 22:40:54 +0200 Subject: [PATCH] Raise Gemini output token caps so thinking can't truncate JSON gemini-2.5-flash counts internal thinking tokens against max_output_tokens. The 700/512 caps let thinking consume the whole budget on complex labels, truncating the structured JSON mid-string and returning HTTP 502 (model_response_unparseable) from /analyze; /chat had the same latent risk. Raise defaults to 4096 (analysis) and 2048 (chat) so thinking + answer both fit. Env-var overrides still work; README updated. Verified live on Cloud Run: a 15-ingredient product that previously 502'd now returns NOVA 4 with correct markers/allergens, and a complex /chat question returns a scoped answer. Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/README.md | 4 ++-- backend/main.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/README.md b/backend/README.md index 419bec0..9f4f001 100644 --- a/backend/README.md +++ b/backend/README.md @@ -113,8 +113,8 @@ On model/auth/timeout failure the service returns HTTP 502 with | `GCP_LOCATION` | `us-east1` | Vertex AI region | | `GEMINI_MODEL` | `gemini-2.5-flash` | Model id | | `GEMINI_TIMEOUT_MS` | `30000` | Per-request model timeout (ms) | -| `GEMINI_MAX_OUTPUT_TOKENS` | `700` | Analysis output cap for the single structured call | -| `GEMINI_CHAT_MAX_OUTPUT_TOKENS` | `512` | Result-chat output cap | +| `GEMINI_MAX_OUTPUT_TOKENS` | `4096` | Analysis output cap (shared by gemini-2.5-flash thinking + JSON answer) | +| `GEMINI_CHAT_MAX_OUTPUT_TOKENS` | `2048` | Result-chat output cap (shared by thinking + answer) | | `ENABLE_OPENAPI_DOCS` | `false` | Enables `/docs`, `/redoc`, and `/openapi.json` only for local/dev use | | `CORS_ALLOWED_ORIGINS` | empty | Comma-separated browser origins allowed to call the API; empty disables CORS middleware | | `PORT` | `8080` | Server port (set by Cloud Run) | diff --git a/backend/main.py b/backend/main.py index 0431660..c0a16e0 100644 --- a/backend/main.py +++ b/backend/main.py @@ -33,8 +33,12 @@ GCP_LOCATION = os.getenv("GCP_LOCATION", "us-east1") GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.5-flash") REQUEST_TIMEOUT_MS = int(os.getenv("GEMINI_TIMEOUT_MS", "30000")) -FULL_ANALYSIS_MAX_OUTPUT_TOKENS = int(os.getenv("GEMINI_MAX_OUTPUT_TOKENS", "700")) -CHAT_MAX_OUTPUT_TOKENS = int(os.getenv("GEMINI_CHAT_MAX_OUTPUT_TOKENS", "512")) +# ponytail: gemini-2.5-flash "thinking" tokens share this output budget. 700/512 let thinking +# eat the whole cap on complex labels, truncating the JSON to garbage -> 502 model_response_unparseable. +# Raised to comfortably fit thinking + answer for real OCR inputs. If a pathological label still +# truncates, bound thinking with types.ThinkingConfig(thinking_budget=N) on the generate_content calls. +FULL_ANALYSIS_MAX_OUTPUT_TOKENS = int(os.getenv("GEMINI_MAX_OUTPUT_TOKENS", "4096")) +CHAT_MAX_OUTPUT_TOKENS = int(os.getenv("GEMINI_CHAT_MAX_OUTPUT_TOKENS", "2048")) MAX_INGREDIENT_CHARS = 20_000 MAX_CHAT_QUESTION_CHARS = 1_000 MAX_CHAT_HISTORY_MESSAGES = 12