Skip to content

UPSTREAM PR #21216: common : simplify autoparser tagged parser rules #1335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
loci-dev wants to merge 9 commits into
base: main
Choose a base branch
from
Open

UPSTREAM PR #21216: common : simplify autoparser tagged parser rules #1335

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
870e5d5
common : simplify autoparser tagged parser rules
aldehir Mar 31, 2026
d417cd6
cont : remove upper limit on optional args
aldehir Apr 1, 2026
508af90
cont : revert changes to parsing at the end
aldehir Apr 1, 2026
fd78b13
cont : undo arbitrary ordering of optional args
aldehir Apr 1, 2026
e6871d2
cont : fix uninitialized required parameters
aldehir Apr 1, 2026
f285719
revert to simplify merge
aldehir Apr 6, 2026
271ca72
merge
aldehir Apr 6, 2026
a8e5523
re-apply patches
aldehir Apr 6, 2026
b24d87f
restore flexible optional arg ordering tests
aldehir Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 14 additions & 36 deletions common/chat-auto-parser-generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,58 +332,36 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
const auto & inputs = ctx.inputs;
bool force_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;

auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix));

common_peg_parser tool_choice = p.choice();

foreach_function(inputs.tools, [&](const json & tool) {
const auto & func = tool.at("function");
std::string name = func.at("name");
const auto & params = func.contains("parameters") ? func.at("parameters") : json::object();
auto params = func.contains("parameters") ? func.at("parameters") : json::object();
const auto & properties = params.contains("properties") ? params.at("properties") : json::object();

std::set<std::string> required;
if (params.contains("required")) {
params.at("required").get_to(required);
}

auto schema_info = common_schema_info();
schema_info.resolve_refs(params);

// Build parser for each argument, separating required and optional
std::vector<common_peg_parser> required_parsers;
std::vector<common_peg_parser> optional_parsers;
for (const auto & [param_name, param_schema] : properties.items()) {
bool is_required = required.find(param_name) != required.end();
std::string type = "object";
if (param_schema.contains("type")) {
const auto & type_obj = param_schema.at("type");
if (type_obj.is_string()) {
type_obj.get_to(type);
} else if (type_obj.is_array()) {
// Handle nullable types like ["string", "null"]
for (const auto & t : type_obj) {
if (t.is_string() && t.get<std::string>() != "null") {
type = t.get<std::string>();
break;
}
}
} else if (type_obj.is_object()) {
if (type_obj.contains("type") && type_obj.at("type").is_string()) {
type_obj.at("type").get_to(type);
}
}
}
// Infer string type from enum values when type is unspecified
if (type == "object" && param_schema.contains("enum")) {
const auto & enum_vals = param_schema.at("enum");
if (enum_vals.is_array()) {
for (const auto & v : enum_vals) {
if (v.is_string()) {
type = "string";
break;
}
}
}
}
bool is_required = required.find(param_name) != required.end();

auto arg =
p.tool_arg(p.tool_arg_open(arguments.name_prefix + p.tool_arg_name(p.literal(param_name)) +
arguments.name_suffix) +
arguments.value_prefix +
(type == "string" ?
p.tool_arg_string_value(p.schema(p.until(arguments.value_suffix),
(schema_info.resolves_to_string(param_schema) ?
p.tool_arg_string_value(p.schema(until_suffix,
"tool-" + name + "-arg-" + param_name + "-schema",
param_schema, true)) :
p.tool_arg_json_value(p.schema(
Expand Down Expand Up @@ -414,7 +392,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte
for (const auto & opt : optional_parsers) {
any_opt |= opt;
}
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, (int) optional_parsers.size());
args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1);
}

if (!arguments.start.empty()) {
Expand Down
Loading