diff --git a/docs/model-group-aggregation.md b/docs/model-group-aggregation.md index fc5972c..f58cf8c 100644 --- a/docs/model-group-aggregation.md +++ b/docs/model-group-aggregation.md @@ -12,7 +12,7 @@ Aggregation is therefore field-specific, conservative, deterministic, and order- This design does not: -- enable foreign-model web search; +- automatically enable foreign hosted web search from LiteLLM capability evidence; - change exact Codex template ownership of Codex-native metadata; - infer capabilities from provider names or neighboring models; - rely on LiteLLM `/model_group/info` as a safety contract; @@ -119,11 +119,16 @@ For exact templates, a known deployment list may explicitly prove absence of a t ## Web search -Foreign web search stays disabled in v0.3 aggregation. +LiteLLM `supports_web_search` remains model-group capability evidence only. It does not own Codex `supports_search_tool`, which controls tool-search/deferred-discovery behavior rather than hosted web search. -LiteLLM can filter web-search deployments at request time, but missing `supports_web_search` currently behaves permissively upstream and registry metadata is incomplete. That is not sufficient evidence for Codex search-tool wire compatibility or a mandatory safe routing guarantee. +Therefore: -Exact Codex templates may retain their Codex-native search support unless any deployment explicitly reports `supports_web_search == false`, in which case the group is downgraded. +- exact Codex templates preserve the version-matched template's `supports_search_tool` value regardless of aggregated `supports_web_search` evidence; +- a configured or aggregated `supports_web_search=false` must not downgrade `supports_search_tool`; +- foreign models keep the conservative `supports_search_tool=false` choice as a tool-search/deferred-discovery decision, not as a hosted-web-search suppression guarantee; +- hosted web-search enablement remains deferred until the provider/runtime boundary and exact Codex/LiteLLM wire compatibility are proven. + +See `docs/foreign-web-search.md` for the dedicated compatibility contract and source-backed revalidation rules. ## Provenance and explanations @@ -155,4 +160,4 @@ The first v0.3 code PR should stay below the production mapping path: 4. in a later PR, connect grouped selection to mapping and extend `explain`/provenance; 5. enable duplicate model groups only after those integration tests pass. -This sequencing makes the safety contract testable without silently changing current v0.2 behavior. \ No newline at end of file +This sequencing makes the safety contract testable without silently changing current v0.2 behavior. diff --git a/src/litellm_codex_models/group_mapping.py b/src/litellm_codex_models/group_mapping.py index b008158..a0e8ff1 100644 --- a/src/litellm_codex_models/group_mapping.py +++ b/src/litellm_codex_models/group_mapping.py @@ -362,15 +362,6 @@ def _repair_exact_override_provenance( source = f"config:model_overrides.{model_name}.supports_audio_input" provenance["input_modalities"] = _exact_retained_source(prepared, source) - if override.supports_web_search is not None and "supports_search_tool" in entry: - source = f"config:model_overrides.{model_name}.supports_web_search" - if override.supports_web_search is True and entry.get("supports_search_tool") is True: - provenance["supports_search_tool"] = _exact_retained_source(prepared, source) - elif provenance.get("supports_search_tool") == ( - "codex:exact-template downgraded by LiteLLM supports_web_search=false" - ): - provenance["supports_search_tool"] = source - if override.supported_openai_params is not None: source = f"config:model_overrides.{model_name}.supported_openai_params" params = set(override.supported_openai_params) @@ -465,12 +456,6 @@ def _repair_foreign_override_provenance( + ", ".join(parallel_sources) ) - if override.supports_web_search is not None and "supports_search_tool" in model.entry: - model.provenance["supports_search_tool"] = ( - f"conservative: foreign web search remains disabled despite " - f"config:model_overrides.{model_name}.supports_web_search" - ) - def generate_prepared_model( prepared: PreparedModelGroup, diff --git a/src/litellm_codex_models/mapping.py b/src/litellm_codex_models/mapping.py index 4f290bd..8470143 100644 --- a/src/litellm_codex_models/mapping.py +++ b/src/litellm_codex_models/mapping.py @@ -10,7 +10,7 @@ REASONING_DESCRIPTIONS = { "none": "No reasoning effort", - "minimal": "Minimal reasoning", + "minimal": "Minimal reasoning effort", "low": "Fast responses with lighter reasoning", "medium": "Balances speed and reasoning depth for everyday tasks", "high": "Greater reasoning depth for complex problems", @@ -195,10 +195,6 @@ def _overlay_exact(entry: dict[str, Any], row: dict[str, Any], provenance: dict[ else: notes.append("LiteLLM confirms parallel_tool_calls transport parameter") - if info.get("supports_web_search") is False and entry.get("supports_search_tool") is True: - entry["supports_search_tool"] = False - provenance["supports_search_tool"] = "codex:exact-template downgraded by LiteLLM supports_web_search=false" - _restrict_exact_reasoning(entry, row, provenance) max_input = info.get("max_input_tokens") @@ -278,6 +274,7 @@ def _build_foreign( "display_name": "derived: LiteLLM model_name", "description": "derived: canonical model identity", "model_messages": "codex:version-matched-fallback-prompt", + "supports_search_tool": "conservative: foreign tool-search/deferred discovery disabled", }) max_input = info.get("max_input_tokens") @@ -314,10 +311,12 @@ def _build_foreign( entry["supports_parallel_tool_calls"] = info.get("supports_function_calling") is True provenance["supports_parallel_tool_calls"] = "LiteLLM parallel_tool_calls + explicit supports_function_calling" - # Intentionally do not turn on Codex search/tool-mode/multi-agent behavior for - # a foreign model solely because LiteLLM advertises a similarly named feature. + # Web-search evidence is intentionally independent from Codex + # `supports_search_tool`, which controls tool-search/deferred discovery. if info.get("supports_web_search") is True: - notes.append("LiteLLM advertises web search; Codex supports_search_tool remains disabled for foreign model") + notes.append( + "LiteLLM advertises web search; evidence is audit-only and does not control Codex supports_search_tool" + ) _apply_foreign_schema_guard(entry, provenance, model_info_schema, codex_catalog) diff --git a/tests/test_model_overrides.py b/tests/test_model_overrides.py index 94fee79..a37c0c9 100644 --- a/tests/test_model_overrides.py +++ b/tests/test_model_overrides.py @@ -219,7 +219,7 @@ def test_foreign_context_override_removes_multi_deployment_synthesis_blocker(): assert audit["effective"] == {"state": "known", "value": 123_456} -def test_foreign_web_search_remains_disabled_even_when_override_is_true(): +def test_foreign_web_search_override_is_evidence_only_for_search_tool(): source = row( "foreign-model", "vendor/model", @@ -240,7 +240,9 @@ def test_foreign_web_search_remains_disabled_even_when_override_is_true(): assert model.entry["supports_search_tool"] is False assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == "guaranteed" - assert "foreign web search remains disabled" in model.provenance["supports_search_tool"] + assert model.provenance["supports_search_tool"] == ( + "conservative: foreign tool-search/deferred discovery disabled" + ) def test_supported_parameter_override_can_enable_foreign_verbosity_without_donor_leakage(): diff --git a/tests/test_web_search_semantics.py b/tests/test_web_search_semantics.py new file mode 100644 index 0000000..6c2fe29 --- /dev/null +++ b/tests/test_web_search_semantics.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from litellm_codex_models.config import ModelOverride +from litellm_codex_models.group_mapping import generate_prepared_catalog, prepare_model_groups +from litellm_codex_models.mapping import generate_model + + +FIXTURE = Path(__file__).parent / "fixtures" / "codex-models.json" +CATALOG = json.loads(FIXTURE.read_text(encoding="utf-8")) +INDEX = {model["slug"]: model for model in CATALOG["models"]} + + +def row(name: str, model: str, **info): + return { + "model_name": name, + "litellm_params": {"model": model, "base_model": model}, + "model_info": {"mode": "chat", **info}, + } + + +@pytest.mark.parametrize("supports_web_search", [True, False, None]) +def test_exact_search_tool_is_codex_owned_regardless_of_litellm_web_search( + supports_web_search: bool | None, +): + generated = generate_model( + row( + "gpt-5.6-sol", + "openai/gpt-5.6-sol", + supports_web_search=supports_web_search, + ), + INDEX, + ) + + assert generated.entry["supports_search_tool"] is True + assert generated.provenance["supports_search_tool"] == ( + "codex:exact-template:gpt-5.6-sol" + ) + + +@pytest.mark.parametrize( + ("reported_value", "override_value", "expected_state"), + [ + (False, True, "guaranteed"), + (True, False, "denied"), + ], +) +def test_exact_web_search_override_is_evidence_only_for_codex_search_tool( + reported_value: bool, + override_value: bool, + expected_state: str, +): + source = row( + "gpt-5.6-sol", + "openai/gpt-5.6-sol", + supports_web_search=reported_value, + ) + prepared = prepare_model_groups( + [[source]], + INDEX, + {"gpt-5.6-sol": ModelOverride(supports_web_search=override_value)}, + ) + + _generated, explanations = generate_prepared_catalog(prepared, CATALOG) + model = explanations["gpt-5.6-sol"] + + assert model.entry["supports_search_tool"] is True + assert model.provenance["supports_search_tool"] == ( + "codex:exact-template:gpt-5.6-sol" + ) + assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == ( + expected_state + ) + assert "supports_web_search" in model.group_evidence["configured_overrides"]