diff --git a/docs/serving.md b/docs/serving.md index 5fb0bde843..4137089815 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -108,6 +108,9 @@ The endpoint supports: - `temperature`, `top_p`, presence/frequency penalties, and signed integer `seed`; - the compatible `top_k` (`0..20`) and `min_p` (`0..1`) sampler extensions; - up to four non-empty stop strings, applied to both reasoning and answer output; +- the boolean `ignore_eos` extension: `true` suppresses the checkpoint's own stop tokens so + generation runs to the requested token budget, while caller-supplied stop strings and stop token + ids still apply; omitted or `false` keeps them; - `n:1`, text-only `modalities`, and `response_format: {"type":"text"}`; - non-streaming responses and server-sent event streams; - `stream_options.include_usage`; diff --git a/src/serve/openai_chat_request.cpp b/src/serve/openai_chat_request.cpp index 2a7611c1e7..537638f8f9 100644 --- a/src/serve/openai_chat_request.cpp +++ b/src/serve/openai_chat_request.cpp @@ -753,6 +753,7 @@ void parse_parallel_tool_calls(const Json& body, const GenerationRequest& output } void parse_stop(const Json& body, GenerationRequest& output) { + output.ignore_eos = get_bool(body, "ignore_eos", false); if (!body.contains("stop") || body.at("stop").is_null()) { return; } output.stop_strings_apply_to_reasoning = true; const Json& stop = body.at("stop"); diff --git a/src/serve/request.h b/src/serve/request.h index cf87d3621f..fa7368bfd5 100644 --- a/src/serve/request.h +++ b/src/serve/request.h @@ -178,6 +178,10 @@ struct GenerationRequest { ToolChoice tool_choice; std::vector stop_strings; bool stop_strings_apply_to_reasoning = false; + // Benchmark/serving extension shared with vLLM, SGLang and llama.cpp: suppress the + // checkpoint's default stop tokens so generation runs to the requested token budget. + // Caller-supplied stop tokens and stop strings still apply. + bool ignore_eos = false; int max_tokens = 0; // resolved budget; zero means immediate output limit std::optional enable_thinking; // unset => use the server default std::optional thinking_budget; diff --git a/src/serve/translate.cpp b/src/serve/translate.cpp index c2effe323b..14279ce978 100644 --- a/src/serve/translate.cpp +++ b/src/serve/translate.cpp @@ -306,6 +306,7 @@ ninfer::RequestOptions to_request_options(const GenerationRequest& request, options.output.raw = false; options.output.preserve_special_tokens = request.uses_tools() || request.has_tool_history(); options.output.tool_name_max_length = static_cast(request.tool_name_max_length); + options.stop.include_model_defaults = !request.ignore_eos; options.stop.strings.reserve(request.stop_strings.size() * (request.stop_strings_apply_to_reasoning ? 2U : 1U)); for (const std::string& stop : request.stop_strings) { diff --git a/tests/test_openai_schema.cpp b/tests/test_openai_schema.cpp index c4378233f4..395a93a8e2 100644 --- a/tests/test_openai_schema.cpp +++ b/tests/test_openai_schema.cpp @@ -580,6 +580,25 @@ int test_stops_and_ranges() { failures += check(api_error([&] { (void)parse(body); }).param == "stop", "empty stop string rejected"); + body = base_request(); + failures += check(options(parse(body).generation).stop.include_model_defaults, + "an omitted ignore_eos keeps the checkpoint's own stop tokens"); + body["ignore_eos"] = false; + failures += check(options(parse(body).generation).stop.include_model_defaults, + "ignore_eos false keeps the checkpoint's own stop tokens"); + body["ignore_eos"] = true; + failures += check(parse(body).generation.ignore_eos && + !options(parse(body).generation).stop.include_model_defaults, + "ignore_eos suppresses the checkpoint's own stop tokens"); + body["stop"] = Json::array({"A"}); + failures += check(options(parse(body).generation).stop.strings.size() == 2 && + !options(parse(body).generation).stop.include_model_defaults, + "ignore_eos leaves caller stop strings in place"); + body.erase("stop"); + body["ignore_eos"] = "true"; + failures += check(api_error([&] { (void)parse(body); }).param == "ignore_eos", + "a non-boolean ignore_eos is rejected"); + body = base_request(); body["top_k"] = 21; const GenerationRequest invalid_top_k = parse(body).generation;