Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions docs/model-group-aggregation.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Aggregation is therefore field-specific, conservative, deterministic, and order-

This design does not:

- enable foreign-model web search;
- automatically enable foreign hosted web search from LiteLLM capability evidence;
- change exact Codex template ownership of Codex-native metadata;
- infer capabilities from provider names or neighboring models;
- rely on LiteLLM `/model_group/info` as a safety contract;
Expand Down Expand Up @@ -119,11 +119,16 @@ For exact templates, a known deployment list may explicitly prove absence of a t

## Web search

Foreign web search stays disabled in v0.3 aggregation.
LiteLLM `supports_web_search` remains model-group capability evidence only. It does not own Codex `supports_search_tool`, which controls tool-search/deferred-discovery behavior rather than hosted web search.

LiteLLM can filter web-search deployments at request time, but missing `supports_web_search` currently behaves permissively upstream and registry metadata is incomplete. That is not sufficient evidence for Codex search-tool wire compatibility or a mandatory safe routing guarantee.
Therefore:

Exact Codex templates may retain their Codex-native search support unless any deployment explicitly reports `supports_web_search == false`, in which case the group is downgraded.
- exact Codex templates preserve the version-matched template's `supports_search_tool` value regardless of aggregated `supports_web_search` evidence;
- a configured or aggregated `supports_web_search=false` must not downgrade `supports_search_tool`;
- foreign models keep the conservative `supports_search_tool=false` choice as a tool-search/deferred-discovery decision, not as a hosted-web-search suppression guarantee;
- hosted web-search enablement remains deferred until the provider/runtime boundary and exact Codex/LiteLLM wire compatibility are proven.

See `docs/foreign-web-search.md` for the dedicated compatibility contract and source-backed revalidation rules.

## Provenance and explanations

Expand Down Expand Up @@ -155,4 +160,4 @@ The first v0.3 code PR should stay below the production mapping path:
4. in a later PR, connect grouped selection to mapping and extend `explain`/provenance;
5. enable duplicate model groups only after those integration tests pass.

This sequencing makes the safety contract testable without silently changing current v0.2 behavior.
This sequencing makes the safety contract testable without silently changing current v0.2 behavior.
15 changes: 0 additions & 15 deletions src/litellm_codex_models/group_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,15 +362,6 @@ def _repair_exact_override_provenance(
source = f"config:model_overrides.{model_name}.supports_audio_input"
provenance["input_modalities"] = _exact_retained_source(prepared, source)

if override.supports_web_search is not None and "supports_search_tool" in entry:
source = f"config:model_overrides.{model_name}.supports_web_search"
if override.supports_web_search is True and entry.get("supports_search_tool") is True:
provenance["supports_search_tool"] = _exact_retained_source(prepared, source)
elif provenance.get("supports_search_tool") == (
"codex:exact-template downgraded by LiteLLM supports_web_search=false"
):
provenance["supports_search_tool"] = source

if override.supported_openai_params is not None:
source = f"config:model_overrides.{model_name}.supported_openai_params"
params = set(override.supported_openai_params)
Expand Down Expand Up @@ -465,12 +456,6 @@ def _repair_foreign_override_provenance(
+ ", ".join(parallel_sources)
)

if override.supports_web_search is not None and "supports_search_tool" in model.entry:
model.provenance["supports_search_tool"] = (
f"conservative: foreign web search remains disabled despite "
f"config:model_overrides.{model_name}.supports_web_search"
)


def generate_prepared_model(
prepared: PreparedModelGroup,
Expand Down
15 changes: 7 additions & 8 deletions src/litellm_codex_models/mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

REASONING_DESCRIPTIONS = {
"none": "No reasoning effort",
"minimal": "Minimal reasoning",
"minimal": "Minimal reasoning effort",
"low": "Fast responses with lighter reasoning",
"medium": "Balances speed and reasoning depth for everyday tasks",
"high": "Greater reasoning depth for complex problems",
Expand Down Expand Up @@ -195,10 +195,6 @@ def _overlay_exact(entry: dict[str, Any], row: dict[str, Any], provenance: dict[
else:
notes.append("LiteLLM confirms parallel_tool_calls transport parameter")

if info.get("supports_web_search") is False and entry.get("supports_search_tool") is True:
entry["supports_search_tool"] = False
provenance["supports_search_tool"] = "codex:exact-template downgraded by LiteLLM supports_web_search=false"

_restrict_exact_reasoning(entry, row, provenance)

max_input = info.get("max_input_tokens")
Expand Down Expand Up @@ -278,6 +274,7 @@ def _build_foreign(
"display_name": "derived: LiteLLM model_name",
"description": "derived: canonical model identity",
"model_messages": "codex:version-matched-fallback-prompt",
"supports_search_tool": "conservative: foreign tool-search/deferred discovery disabled",
})

max_input = info.get("max_input_tokens")
Expand Down Expand Up @@ -314,10 +311,12 @@ def _build_foreign(
entry["supports_parallel_tool_calls"] = info.get("supports_function_calling") is True
provenance["supports_parallel_tool_calls"] = "LiteLLM parallel_tool_calls + explicit supports_function_calling"

# Intentionally do not turn on Codex search/tool-mode/multi-agent behavior for
# a foreign model solely because LiteLLM advertises a similarly named feature.
# Web-search evidence is intentionally independent from Codex
# `supports_search_tool`, which controls tool-search/deferred discovery.
if info.get("supports_web_search") is True:
notes.append("LiteLLM advertises web search; Codex supports_search_tool remains disabled for foreign model")
notes.append(
"LiteLLM advertises web search; evidence is audit-only and does not control Codex supports_search_tool"
)

_apply_foreign_schema_guard(entry, provenance, model_info_schema, codex_catalog)

Expand Down
6 changes: 4 additions & 2 deletions tests/test_model_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ def test_foreign_context_override_removes_multi_deployment_synthesis_blocker():
assert audit["effective"] == {"state": "known", "value": 123_456}


def test_foreign_web_search_remains_disabled_even_when_override_is_true():
def test_foreign_web_search_override_is_evidence_only_for_search_tool():
source = row(
"foreign-model",
"vendor/model",
Expand All @@ -240,7 +240,9 @@ def test_foreign_web_search_remains_disabled_even_when_override_is_true():

assert model.entry["supports_search_tool"] is False
assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == "guaranteed"
assert "foreign web search remains disabled" in model.provenance["supports_search_tool"]
assert model.provenance["supports_search_tool"] == (
"conservative: foreign tool-search/deferred discovery disabled"
)


def test_supported_parameter_override_can_enable_foreign_verbosity_without_donor_leakage():
Expand Down
78 changes: 78 additions & 0 deletions tests/test_web_search_semantics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
from __future__ import annotations

import json
from pathlib import Path

import pytest

from litellm_codex_models.config import ModelOverride
from litellm_codex_models.group_mapping import generate_prepared_catalog, prepare_model_groups
from litellm_codex_models.mapping import generate_model


FIXTURE = Path(__file__).parent / "fixtures" / "codex-models.json"
CATALOG = json.loads(FIXTURE.read_text(encoding="utf-8"))
INDEX = {model["slug"]: model for model in CATALOG["models"]}


def row(name: str, model: str, **info):
return {
"model_name": name,
"litellm_params": {"model": model, "base_model": model},
"model_info": {"mode": "chat", **info},
}


@pytest.mark.parametrize("supports_web_search", [True, False, None])
def test_exact_search_tool_is_codex_owned_regardless_of_litellm_web_search(
supports_web_search: bool | None,
):
generated = generate_model(
row(
"gpt-5.6-sol",
"openai/gpt-5.6-sol",
supports_web_search=supports_web_search,
),
INDEX,
)

assert generated.entry["supports_search_tool"] is True
assert generated.provenance["supports_search_tool"] == (
"codex:exact-template:gpt-5.6-sol"
)


@pytest.mark.parametrize(
("reported_value", "override_value", "expected_state"),
[
(False, True, "guaranteed"),
(True, False, "denied"),
],
)
def test_exact_web_search_override_is_evidence_only_for_codex_search_tool(
reported_value: bool,
override_value: bool,
expected_state: str,
):
source = row(
"gpt-5.6-sol",
"openai/gpt-5.6-sol",
supports_web_search=reported_value,
)
prepared = prepare_model_groups(
[[source]],
INDEX,
{"gpt-5.6-sol": ModelOverride(supports_web_search=override_value)},
)

_generated, explanations = generate_prepared_catalog(prepared, CATALOG)
model = explanations["gpt-5.6-sol"]

assert model.entry["supports_search_tool"] is True
assert model.provenance["supports_search_tool"] == (
"codex:exact-template:gpt-5.6-sol"
)
assert model.group_evidence["capabilities"]["supports_web_search"]["state"] == (
expected_state
)
assert "supports_web_search" in model.group_evidence["configured_overrides"]