From 70beb3f069353240fa198715d4064f9e20f16bb8 Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:57:57 +0100 Subject: [PATCH 1/6] fix: decouple web search from Codex search tool --- src/litellm_codex_models/mapping.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/litellm_codex_models/mapping.py b/src/litellm_codex_models/mapping.py index 4f290bd..8470143 100644 --- a/src/litellm_codex_models/mapping.py +++ b/src/litellm_codex_models/mapping.py @@ -10,7 +10,7 @@ REASONING_DESCRIPTIONS = { "none": "No reasoning effort", - "minimal": "Minimal reasoning", + "minimal": "Minimal reasoning effort", "low": "Fast responses with lighter reasoning", "medium": "Balances speed and reasoning depth for everyday tasks", "high": "Greater reasoning depth for complex problems", @@ -195,10 +195,6 @@ def _overlay_exact(entry: dict[str, Any], row: dict[str, Any], provenance: dict[ else: notes.append("LiteLLM confirms parallel_tool_calls transport parameter") - if info.get("supports_web_search") is False and entry.get("supports_search_tool") is True: - entry["supports_search_tool"] = False - provenance["supports_search_tool"] = "codex:exact-template downgraded by LiteLLM supports_web_search=false" - _restrict_exact_reasoning(entry, row, provenance) max_input = info.get("max_input_tokens") @@ -278,6 +274,7 @@ def _build_foreign( "display_name": "derived: LiteLLM model_name", "description": "derived: canonical model identity", "model_messages": "codex:version-matched-fallback-prompt", + "supports_search_tool": "conservative: foreign tool-search/deferred discovery disabled", }) max_input = info.get("max_input_tokens") @@ -314,10 +311,12 @@ def _build_foreign( entry["supports_parallel_tool_calls"] = info.get("supports_function_calling") is True provenance["supports_parallel_tool_calls"] = "LiteLLM parallel_tool_calls + explicit supports_function_calling" - # Intentionally do not turn on Codex search/tool-mode/multi-agent behavior for - # a foreign model solely because LiteLLM advertises a similarly named feature. + # Web-search evidence is intentionally independent from Codex + # `supports_search_tool`, which controls tool-search/deferred discovery. if info.get("supports_web_search") is True: - notes.append("LiteLLM advertises web search; Codex supports_search_tool remains disabled for foreign model") + notes.append( + "LiteLLM advertises web search; evidence is audit-only and does not control Codex supports_search_tool" + ) _apply_foreign_schema_guard(entry, provenance, model_info_schema, codex_catalog) From a8567f574acf9f94bb442fae9dc1d49300b5f79f Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:58:59 +0100 Subject: [PATCH 2/6] fix: keep web search evidence out of search-tool provenance --- src/litellm_codex_models/group_mapping.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/litellm_codex_models/group_mapping.py b/src/litellm_codex_models/group_mapping.py index b008158..a0e8ff1 100644 --- a/src/litellm_codex_models/group_mapping.py +++ b/src/litellm_codex_models/group_mapping.py @@ -362,15 +362,6 @@ def _repair_exact_override_provenance( source = f"config:model_overrides.{model_name}.supports_audio_input" provenance["input_modalities"] = _exact_retained_source(prepared, source) - if override.supports_web_search is not None and "supports_search_tool" in entry: - source = f"config:model_overrides.{model_name}.supports_web_search" - if override.supports_web_search is True and entry.get("supports_search_tool") is True: - provenance["supports_search_tool"] = _exact_retained_source(prepared, source) - elif provenance.get("supports_search_tool") == ( - "codex:exact-template downgraded by LiteLLM supports_web_search=false" - ): - provenance["supports_search_tool"] = source - if override.supported_openai_params is not None: source = f"config:model_overrides.{model_name}.supported_openai_params" params = set(override.supported_openai_params) @@ -465,12 +456,6 @@ def _repair_foreign_override_provenance( + ", ".join(parallel_sources) ) - if override.supports_web_search is not None and "supports_search_tool" in model.entry: - model.provenance["supports_search_tool"] = ( - f"conservative: foreign web search remains disabled despite " - f"config:model_overrides.{model_name}.supports_web_search" - ) - def generate_prepared_model( prepared: PreparedModelGroup, From c4dcd1592e1c6264e286dc5f5a202891959a13b1 Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 12:59:25 +0100 Subject: [PATCH 3/6] test: lock web-search and search-tool semantics --- tests/test_web_search_semantics.py | 112 +++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/test_web_search_semantics.py diff --git a/tests/test_web_search_semantics.py b/tests/test_web_search_semantics.py new file mode 100644 index 0000000..210010c --- /dev/null +++ b/tests/test_web_search_semantics.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from litellm_codex_models.config import ModelOverride +from litellm_codex_models.group_mapping import generate_prepared_catalog, prepare_model_groups +from litellm_codex_models.mapping import generate_model +from litellm_codex_models.schema import ModelInfoSchema + + +FIXTURE = Path(__file__).parent / "fixtures" / "codex-models.json" +CATALOG = json.loads(FIXTURE.read_text(encoding="utf-8")) +INDEX = {model["slug"]: model for model in CATALOG["models"]} +FALLBACK_PROMPT = "You are a generic Codex coding agent." +SCHEMA = ModelInfoSchema(fields=frozenset(), required_fields=frozenset()) + + +def row(name: str, model: str, **info): + return { + "model_name": name, + "litellm_params": {"model": model, "base_model": model}, + "model_info": {"mode": "chat", **info}, + } + + +@pytest.mark.parametrize("supports_web_search", [True, False, None]) +def test_exact_search_tool_is_codex_owned_regardless_of_litellm_web_search( + supports_web_search: bool | None, +): + generated = generate_model( + row( + "gpt-5.6-sol", + "openai/gpt-5.6-sol", + supports_web_search=supports_web_search, + ), + INDEX, + ) + + assert generated.entry["supports_search_tool"] is True + assert generated.provenance["supports_search_tool"] == ( + "codex:exact-template:gpt-5.6-sol" + ) + + +@pytest.mark.parametrize( + ("reported_value", "override_value", "expected_state"), + [ + (False, True, "guaranteed"), + (True, False, "denied"), + ], +) +def test_exact_web_search_override_is_evidence_only_for_codex_search_tool( + reported_value: bool, + override_value: bool, + expected_state: str, +): + source = row( + "gpt-5.6-sol", + "openai/gpt-5.6-sol", + supports_web_search=reported_value, + ) + prepared = prepare_model_groups( + [[source]], + INDEX, + {"gpt-5.6-sol": ModelOverride(supports_web_search=override_value)}, + ) + + _generated, explanations = generate_prepared_catalog(prepared, CATALOG) + model = explanations["gpt-5.6-sol"] + + assert model.entry["supports_search_tool"] is True + assert model.provenance["supports_search_tool"] == ( + "codex:exact-template:gpt-5.6-sol" + ) + assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == ( + expected_state + ) + assert "supports_web_search" in model.group_evidence["configured_overrides"] + + +def test_foreign_search_tool_is_conservative_tool_discovery_not_web_search_policy(): + source = row( + "foreign-model", + "vendor/model", + max_input_tokens=200_000, + supports_web_search=False, + ) + prepared = prepare_model_groups( + [[source]], + INDEX, + {"foreign-model": ModelOverride(supports_web_search=True)}, + ) + + _generated, explanations = generate_prepared_catalog( + prepared, + CATALOG, + fallback_prompt=FALLBACK_PROMPT, + model_info_schema=SCHEMA, + ) + model = explanations["foreign-model"] + + assert model.entry["supports_search_tool"] is False + assert model.provenance["supports_search_tool"] == ( + "conservative: foreign tool-search/deferred discovery disabled" + ) + assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == ( + "guaranteed" + ) + assert "supports_web_search" in model.group_evidence["configured_overrides"] From 3707adfdd02181db7a7084e990642c2cb7027536 Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:00:15 +0100 Subject: [PATCH 4/6] test: align foreign search-tool provenance --- tests/test_model_overrides.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_model_overrides.py b/tests/test_model_overrides.py index 94fee79..a37c0c9 100644 --- a/tests/test_model_overrides.py +++ b/tests/test_model_overrides.py @@ -219,7 +219,7 @@ def test_foreign_context_override_removes_multi_deployment_synthesis_blocker(): assert audit["effective"] == {"state": "known", "value": 123_456} -def test_foreign_web_search_remains_disabled_even_when_override_is_true(): +def test_foreign_web_search_override_is_evidence_only_for_search_tool(): source = row( "foreign-model", "vendor/model", @@ -240,7 +240,9 @@ def test_foreign_web_search_remains_disabled_even_when_override_is_true(): assert model.entry["supports_search_tool"] is False assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == "guaranteed" - assert "foreign web search remains disabled" in model.provenance["supports_search_tool"] + assert model.provenance["supports_search_tool"] == ( + "conservative: foreign tool-search/deferred discovery disabled" + ) def test_supported_parameter_override_can_enable_foreign_verbosity_without_donor_leakage(): From ad885587c5b3174f4a38883b023ab9b37e48cee7 Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:00:37 +0100 Subject: [PATCH 5/6] test: focus search semantics regressions --- tests/test_web_search_semantics.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/tests/test_web_search_semantics.py b/tests/test_web_search_semantics.py index 210010c..6c2fe29 100644 --- a/tests/test_web_search_semantics.py +++ b/tests/test_web_search_semantics.py @@ -8,14 +8,11 @@ from litellm_codex_models.config import ModelOverride from litellm_codex_models.group_mapping import generate_prepared_catalog, prepare_model_groups from litellm_codex_models.mapping import generate_model -from litellm_codex_models.schema import ModelInfoSchema FIXTURE = Path(__file__).parent / "fixtures" / "codex-models.json" CATALOG = json.loads(FIXTURE.read_text(encoding="utf-8")) INDEX = {model["slug"]: model for model in CATALOG["models"]} -FALLBACK_PROMPT = "You are a generic Codex coding agent." -SCHEMA = ModelInfoSchema(fields=frozenset(), required_fields=frozenset()) def row(name: str, model: str, **info): @@ -79,34 +76,3 @@ def test_exact_web_search_override_is_evidence_only_for_codex_search_tool( expected_state ) assert "supports_web_search" in model.group_evidence["configured_overrides"] - - -def test_foreign_search_tool_is_conservative_tool_discovery_not_web_search_policy(): - source = row( - "foreign-model", - "vendor/model", - max_input_tokens=200_000, - supports_web_search=False, - ) - prepared = prepare_model_groups( - [[source]], - INDEX, - {"foreign-model": ModelOverride(supports_web_search=True)}, - ) - - _generated, explanations = generate_prepared_catalog( - prepared, - CATALOG, - fallback_prompt=FALLBACK_PROMPT, - model_info_schema=SCHEMA, - ) - model = explanations["foreign-model"] - - assert model.entry["supports_search_tool"] is False - assert model.provenance["supports_search_tool"] == ( - "conservative: foreign tool-search/deferred discovery disabled" - ) - assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == ( - "guaranteed" - ) - assert "supports_web_search" in model.group_evidence["configured_overrides"] From 77179a28b85415bb77fd5e7d1ec1838649256ede Mon Sep 17 00:00:00 2001 From: Avi Back To Black <54722547+AviBackToBlack@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:01:09 +0100 Subject: [PATCH 6/6] docs: align aggregation search semantics --- docs/model-group-aggregation.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/model-group-aggregation.md b/docs/model-group-aggregation.md index fc5972c..f58cf8c 100644 --- a/docs/model-group-aggregation.md +++ b/docs/model-group-aggregation.md @@ -12,7 +12,7 @@ Aggregation is therefore field-specific, conservative, deterministic, and order- This design does not: -- enable foreign-model web search; +- automatically enable foreign hosted web search from LiteLLM capability evidence; - change exact Codex template ownership of Codex-native metadata; - infer capabilities from provider names or neighboring models; - rely on LiteLLM `/model_group/info` as a safety contract; @@ -119,11 +119,16 @@ For exact templates, a known deployment list may explicitly prove absence of a t ## Web search -Foreign web search stays disabled in v0.3 aggregation. +LiteLLM `supports_web_search` remains model-group capability evidence only. It does not own Codex `supports_search_tool`, which controls tool-search/deferred-discovery behavior rather than hosted web search. -LiteLLM can filter web-search deployments at request time, but missing `supports_web_search` currently behaves permissively upstream and registry metadata is incomplete. That is not sufficient evidence for Codex search-tool wire compatibility or a mandatory safe routing guarantee. +Therefore: -Exact Codex templates may retain their Codex-native search support unless any deployment explicitly reports `supports_web_search == false`, in which case the group is downgraded. +- exact Codex templates preserve the version-matched template's `supports_search_tool` value regardless of aggregated `supports_web_search` evidence; +- a configured or aggregated `supports_web_search=false` must not downgrade `supports_search_tool`; +- foreign models keep the conservative `supports_search_tool=false` choice as a tool-search/deferred-discovery decision, not as a hosted-web-search suppression guarantee; +- hosted web-search enablement remains deferred until the provider/runtime boundary and exact Codex/LiteLLM wire compatibility are proven. + +See `docs/foreign-web-search.md` for the dedicated compatibility contract and source-backed revalidation rules. ## Provenance and explanations @@ -155,4 +160,4 @@ The first v0.3 code PR should stay below the production mapping path: 4. in a later PR, connect grouped selection to mapping and extend `explain`/provenance; 5. enable duplicate model groups only after those integration tests pass. -This sequencing makes the safety contract testable without silently changing current v0.2 behavior. \ No newline at end of file +This sequencing makes the safety contract testable without silently changing current v0.2 behavior.