From c2c02ba547add3d382a39452e4025542cd69cc43 Mon Sep 17 00:00:00 2001 From: Justin Mclean Date: Sat, 29 Aug 2026 02:22:28 +0000 Subject: [PATCH] Make reasoning_effort configurable per stage qwen3.8-27b defaults to xhigh and produces reasoning blocks that dwarf the answer -- a live run's transcript shows the model re-deriving the check's own constraints (section ordering, the 120-character minimums, which issue numbers are citable) at length before doing any work. There was no way to ask for less: the payload was model + messages + tools and nothing else. reasoning_effort resolves through the same chain as the model -- [harness.profiles.NAME.] -> [harness.profiles.NAME] -> [harness.] -> [harness] -- because it is a per-stage decision for the same reason the model is. decompose and judge are single bounded calls where deliberation is cheap; execute runs a tool loop and pays the cost on every turn. Sent only when configured, so an endpoint that rejects unknown keys sees no change by default. Validated at load against a known set, and for every profile rather than only the selected one: an unrecognised value would otherwise be forwarded verbatim, ignored by the server, and leave a run that looks configured and is not. Measurement caveat worth recording: probing an LM Studio endpoint with a short prompt showed no difference between xhigh, medium and low (119/141/ 129 reasoning tokens). That probe was too easy to make the model think at any setting, so it is evidence about the probe, not about the parameter. --- src/jumar/cli.py | 1 + src/jumar/config.py | 48 +++++++++++++++++++++++++++++++- src/jumar/decompose.py | 1 + src/jumar/execute.py | 1 + src/jumar/models.py | 1 + src/jumar/openai_agent.py | 4 +++ tests/test_config.py | 44 ++++++++++++++++++++++++++++++ tests/test_openai_agent.py | 56 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 155 insertions(+), 1 deletion(-) diff --git a/src/jumar/cli.py b/src/jumar/cli.py index d9928da..feac634 100644 --- a/src/jumar/cli.py +++ b/src/jumar/cli.py @@ -669,6 +669,7 @@ def run_item( invoked_as=judge_resolved.agent, base_url=judge_resolved.base_url, api_key_env=judge_resolved.api_key_env, + reasoning_effort=judge_resolved.reasoning_effort, commands_allow=config.commands.allow, commands_deny=config.commands.deny, ) diff --git a/src/jumar/config.py b/src/jumar/config.py index 60e0b2f..5eabe5f 100644 --- a/src/jumar/config.py +++ b/src/jumar/config.py @@ -166,8 +166,17 @@ def _system_timezone() -> str: # so that a typo in jumar.toml does not silently fail to apply the override. _VALID_HARNESS_STAGES: frozenset[str] = frozenset({"decompose", "execute", "judge"}) +# Values accepted for `reasoning_effort`. Sent verbatim to the chat-completions +# endpoint, which passes it to the model's chat template; a model that does not +# understand the key ignores it. Validated rather than free-form so a typo is a +# startup error instead of a silently ignored setting that looks applied. +# "none" is included because several thinking models spell "off" that way. +_VALID_REASONING_EFFORTS: frozenset[str] = frozenset({"none", "low", "medium", "high", "xhigh"}) + # Scalar keys valid at [harness], inside a stage table, and inside a profile. -_HARNESS_SCALAR_KEYS: frozenset[str] = frozenset({"agent", "model", "base_url", "api_key_env"}) +_HARNESS_SCALAR_KEYS: frozenset[str] = frozenset( + {"agent", "model", "base_url", "api_key_env", "reasoning_effort"} +) # Sub-table under [harness] holding named alternative harnesses: # [harness.profiles.] with the same shape as [harness] itself. One is @@ -206,6 +215,11 @@ class HarnessConfig: none, e.g. a local LM Studio server). Every subprocess harness ignores both. + ``reasoning_effort`` is forwarded to an in-process harness's chat- + completions request when set. It is per-stage for the same reason the + model is: decompose and judge are single bounded calls, while execute + runs a tool loop where the cost of deliberation is paid on every turn. + ``None`` for a per-stage field means "inherit the top-level value". A ``HarnessConfig`` is always fully resolved by the time it reaches this @@ -218,19 +232,23 @@ class HarnessConfig: model: str = "sonnet" base_url: str | None = None api_key_env: str | None = None + reasoning_effort: str | None = None # Per-stage overrides — None inherits the top-level value. decompose_agent: str | None = None decompose_model: str | None = None decompose_base_url: str | None = None decompose_api_key_env: str | None = None + decompose_reasoning_effort: str | None = None execute_agent: str | None = None execute_model: str | None = None execute_base_url: str | None = None execute_api_key_env: str | None = None + execute_reasoning_effort: str | None = None judge_agent: str | None = None judge_model: str | None = None judge_base_url: str | None = None judge_api_key_env: str | None = None + judge_reasoning_effort: str | None = None def for_stage(self, stage: str) -> HarnessConfig: """Return the resolved ``HarnessConfig`` for *stage*. @@ -247,11 +265,13 @@ def for_stage(self, stage: str) -> HarnessConfig: m: str | None = getattr(self, f"{stage}_model") b: str | None = getattr(self, f"{stage}_base_url") k: str | None = getattr(self, f"{stage}_api_key_env") + r: str | None = getattr(self, f"{stage}_reasoning_effort") return HarnessConfig( agent=a if a is not None else self.agent, model=m if m is not None else self.model, base_url=b if b is not None else self.base_url, api_key_env=k if k is not None else self.api_key_env, + reasoning_effort=r if r is not None else self.reasoning_effort, ) @@ -506,6 +526,25 @@ def _first(*values: str | None) -> str | None: ) profile_raw = cast(dict[str, Any], profiles_raw[harness_profile]) + def _validate_reasoning_efforts(resolved: HarnessConfig, label: str) -> None: + """Reject an unrecognised reasoning_effort at load, not at request time. + + A bad value would otherwise be forwarded verbatim and silently ignored + by the endpoint, leaving a run that looks configured and is not. Every + profile is checked, not just the selected one, so a typo in an unused + profile still fails the next load rather than the next scheduled run + that selects it. + """ + stage_attrs = (f"{s}_reasoning_effort" for s in sorted(_VALID_HARNESS_STAGES)) + for attr in ("reasoning_effort", *stage_attrs): + value = getattr(resolved, attr) + if value is not None and value not in _VALID_REASONING_EFFORTS: + where = label if attr == "reasoning_effort" else f"{label}.{attr.split('_')[0]}" + raise ConfigError( + f"Invalid reasoning_effort {value!r} under [{where}]. " + f"Valid values: {sorted(_VALID_REASONING_EFFORTS)}." + ) + # Resolution order, highest first: # [harness.profiles..] → [harness.profiles.] # → [harness.] → [harness] @@ -540,16 +579,23 @@ def _build(profile_table: dict[str, Any], profile_name: str | None) -> HarnessCo api_key_env=_first( _scalar(profile_table, "api_key_env"), _scalar(harness_raw, "api_key_env") ), + reasoning_effort=_first( + _scalar(profile_table, "reasoning_effort"), + _scalar(harness_raw, "reasoning_effort"), + ), **stage_kwargs, ) harness = _build(profile_raw, harness_profile) + _validate_reasoning_efforts(harness, "harness") # Every profile is resolved, not just the selected one, so an item can # name one with @harness= after load without re-reading the file. harness_profiles: dict[str, HarnessConfig] = { name: _build(cast(dict[str, Any], profiles_raw[name]), name) for name in sorted(profiles_raw) } + for _name, _resolved in harness_profiles.items(): + _validate_reasoning_efforts(_resolved, f"harness.{_HARNESS_PROFILES_KEY}.{_name}") commands_raw: dict[str, Any] = raw.get("commands", {}) commands = CommandPolicy( diff --git a/src/jumar/decompose.py b/src/jumar/decompose.py index 609c9bd..40f9529 100644 --- a/src/jumar/decompose.py +++ b/src/jumar/decompose.py @@ -437,6 +437,7 @@ def decompose( invoked_as=resolved.agent, base_url=resolved.base_url, api_key_env=resolved.api_key_env, + reasoning_effort=resolved.reasoning_effort, commands_allow=config.commands.allow, commands_deny=config.commands.deny, ) diff --git a/src/jumar/execute.py b/src/jumar/execute.py index 0d322a2..8bcc1c8 100644 --- a/src/jumar/execute.py +++ b/src/jumar/execute.py @@ -140,6 +140,7 @@ def execute( invoked_as=resolved.agent, base_url=resolved.base_url, api_key_env=resolved.api_key_env, + reasoning_effort=resolved.reasoning_effort, commands_allow=config.commands.allow, commands_deny=config.commands.deny, ) diff --git a/src/jumar/models.py b/src/jumar/models.py index 07ad8a3..da1dd33 100644 --- a/src/jumar/models.py +++ b/src/jumar/models.py @@ -203,6 +203,7 @@ class HarnessInfo: invoked_as: str base_url: str | None = None api_key_env: str | None = None + reasoning_effort: str | None = None commands_allow: tuple[str, ...] = () commands_deny: tuple[str, ...] = () diff --git a/src/jumar/openai_agent.py b/src/jumar/openai_agent.py index 1e99d42..85fd650 100644 --- a/src/jumar/openai_agent.py +++ b/src/jumar/openai_agent.py @@ -385,6 +385,10 @@ def _rate() -> dict[str, Any]: ) payload: dict[str, Any] = {"model": harness.model, "messages": messages} + # Sent only when configured, so an endpoint that rejects unknown keys + # is unaffected by the default. + if harness.reasoning_effort: + payload["reasoning_effort"] = harness.reasoning_effort if allow_tools: payload["tools"] = list(_TOOLS) diff --git a/tests/test_config.py b/tests/test_config.py index d418d65..1525142 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -992,3 +992,47 @@ def test_custom_deny_list() -> None: cfg = Config(commands=CommandPolicy(allow=("python3", "make", "git"), deny=("make",))) assert is_allowed(["make", "check"], cfg) is False assert is_allowed(["python3", "test.py"], cfg) is True + + +def test_reasoning_effort_resolves_per_stage(tmp_path: Path) -> None: + """A stage override wins; other stages inherit the top-level value.""" + cfg_path = tmp_path / "jumar.toml" + cfg_path.write_text( + "[jumar.harness]\n" + 'agent = "openai"\n' + 'model = "m"\n' + 'reasoning_effort = "low"\n' + "[jumar.harness.execute]\n" + 'reasoning_effort = "medium"\n' + ) + config = load_config(tmp_path) + assert config.harness.for_stage("decompose").reasoning_effort == "low" + assert config.harness.for_stage("judge").reasoning_effort == "low" + assert config.harness.for_stage("execute").reasoning_effort == "medium" + + +def test_reasoning_effort_defaults_to_none(tmp_path: Path) -> None: + cfg_path = tmp_path / "jumar.toml" + cfg_path.write_text('[jumar.harness]\nagent = "openai"\nmodel = "m"\n') + config = load_config(tmp_path) + assert config.harness.for_stage("execute").reasoning_effort is None + + +def test_invalid_reasoning_effort_is_a_startup_error(tmp_path: Path) -> None: + cfg_path = tmp_path / "jumar.toml" + cfg_path.write_text('[jumar.harness]\nagent = "openai"\nreasoning_effort = "lwo"\n') + with pytest.raises(ConfigError, match="Invalid reasoning_effort"): + load_config(tmp_path) + + +def test_invalid_reasoning_effort_in_an_unselected_profile_still_fails(tmp_path: Path) -> None: + """Every profile is validated, not only the one selected for this run.""" + cfg_path = tmp_path / "jumar.toml" + cfg_path.write_text( + "[jumar.harness]\n" + 'agent = "openai"\n' + "[jumar.harness.profiles.heavy]\n" + 'reasoning_effort = "enormous"\n' + ) + with pytest.raises(ConfigError, match="Invalid reasoning_effort"): + load_config(tmp_path) diff --git a/tests/test_openai_agent.py b/tests/test_openai_agent.py index a61f31b..ab7a52e 100644 --- a/tests/test_openai_agent.py +++ b/tests/test_openai_agent.py @@ -21,6 +21,7 @@ import time import urllib.error from collections.abc import Callable, Iterator +from dataclasses import replace from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from typing import Any @@ -907,3 +908,58 @@ def test_malformed_usage_values_are_ignored(tmp_path: Path, chat_server: Any) -> assert result.exit_status == 0 assert result.prompt_tokens == 0 assert result.completion_tokens == 12 + + +# --------------------------------------------------------------------------- +# reasoning_effort +# --------------------------------------------------------------------------- + + +def test_reasoning_effort_is_sent_when_configured(tmp_path: Path, chat_server: Any) -> None: + base_url, handler = chat_server([_message("All done.")]) + harness = replace(_harness(base_url), reasoning_effort="low") + result = run_openai_agent( + "do the thing", + cwd=tmp_path, + capabilities=_ALL_CAPS, + timeout_s=30, + harness=harness, + ) + assert result.exit_status == 0 + assert handler.received[0]["reasoning_effort"] == "low" + + +def test_reasoning_effort_is_omitted_when_unset(tmp_path: Path, chat_server: Any) -> None: + """An endpoint that rejects unknown keys must not see the key by default.""" + base_url, handler = chat_server([_message("All done.")]) + result = run_openai_agent( + "do the thing", + cwd=tmp_path, + capabilities=_ALL_CAPS, + timeout_s=30, + harness=_harness(base_url), + ) + assert result.exit_status == 0 + assert "reasoning_effort" not in handler.received[0] + + +def test_reasoning_effort_is_sent_on_every_turn_of_the_loop( + tmp_path: Path, chat_server: Any +) -> None: + (tmp_path / "foo.txt").write_text("hello world") + base_url, handler = chat_server( + [ + _message("", [_tool_call("1", "read_file", {"path": "foo.txt"})]), + _message("done."), + ] + ) + result = run_openai_agent( + "read foo.txt", + cwd=tmp_path, + capabilities=_ALL_CAPS, + timeout_s=30, + harness=replace(_harness(base_url), reasoning_effort="medium"), + ) + assert result.exit_status == 0 + assert len(handler.received) == 2 + assert all(r["reasoning_effort"] == "medium" for r in handler.received)