diff --git a/amplifier_app_cli/main.py b/amplifier_app_cli/main.py index 7bbd3725..d9a27f65 100644 --- a/amplifier_app_cli/main.py +++ b/amplifier_app_cli/main.py @@ -50,6 +50,10 @@ from .dedicated_tty_input import close_dedicated_tty_input, get_dedicated_tty_input from .effective_config import get_effective_config_summary from .key_manager import KeyManager +from .provider_diagnostics import DEFAULT_TIMEOUT_S as _PROVIDER_DIAGNOSTIC_TIMEOUT_S +from .provider_diagnostics import format_model_line +from .provider_diagnostics import invoke_list_models +from .provider_diagnostics import test_provider_connectivity from .session_runner import SessionConfig, create_initialized_session from .session_store import SessionStore from .stdout_offload import patch_stdout_offloaded as patch_stdout @@ -468,7 +472,8 @@ class CommandProcessor: "action": "handle_provider", "description": ( "(experimental) Show/pin the conversation-scope provider: " - "/provider (status) | /provider use | /provider auto" + "/provider (status) | /provider use | /provider auto | " + "/provider test | /provider models " ), }, } @@ -1502,6 +1507,153 @@ def _render_provider_status(self, pin: Any) -> str: return "\n".join(lines) + # === /provider test | /provider models: read-only diagnostics === + # + # Unlike 'use'/'auto', these two are NEVER gated on the + # 'conversation.provider_pin' capability -- they answer "why can't I + # pin/why is nothing answering", which is exactly the question that + # matters most when the pin capability is absent or refusing. Gating + # them on that same capability would remove the diagnostic exactly + # when it's needed. + # + # They also intentionally query THIS SESSION'S MOUNTED providers + # (coordinator.get("providers")), never settings.yaml -- that's the + # entire reason to run them mid-conversation instead of dropping to a + # shell for `amplifier provider test`/`amplifier provider models`. If + # the mounted set and settings.yaml disagree, the session's live + # reality is what the user needs to see. + + def _unknown_provider_message(self, name: str, mounted: dict[str, Any]) -> str: + """Refusal text for a name that isn't currently mounted. + + Names what IS mounted rather than a bare failure -- matches the + wording ConversationProviderPin.pin() already uses for the same + situation (see the module docstring's WHY: this is the message a + failed pin sends the user here to investigate). + + Deliberately NOT tagged (experimental) or dim -- matches the + capability-absent and unmounted-provider refusals from 'use': the + refusal is the whole message, tagging or dimming would dilute it. + """ + available = ", ".join(sorted(mounted)) if mounted else "(none)" + return ( + f"\u2717 provider {name!r} is not mounted in this session. " + f"Mounted providers: {available}" + ) + + def _resolve_diagnostic_targets( + self, name: str + ) -> tuple[dict[str, Any], str | None]: + """Resolve which mounted provider(s) a diagnostic subcommand + should target: all of them (no name given), or exactly one. + + Returns ``(targets, error)``. ``error`` is set (and ``targets`` + empty) only when a name was given but isn't mounted. + """ + mounted: dict[str, Any] = self.session.coordinator.get("providers") or {} + if not name: + return mounted, None + if name not in mounted: + return {}, self._unknown_provider_message(name, mounted) + return {name: mounted[name]}, None + + async def _handle_provider_test(self, name: str) -> str: + """`/provider test [name]` -- connectivity check against this + session's mounted providers. No name tests all of them, + concurrently (see class docstring above for why this is never + gated on the pin capability, and always live-session-scoped). + + Reuses provider_diagnostics.test_provider_connectivity so "ok" + means exactly what `amplifier provider test` means (list_models() + succeeds) -- the two surfaces cannot silently disagree. + """ + targets, error = self._resolve_diagnostic_targets(name) + if error: + return error + if not targets: + return "Provider test (experimental):\n (no providers mounted in this session)" + + # Network I/O in an interactive REPL: tell the user before the + # (possibly multi-second, possibly multi-provider) wait rather + # than freezing silently. Run all targets concurrently -- and each + # is individually timeout-bounded (see test_provider_connectivity) + # -- so one slow/hung provider can't stall the others or the + # session. + plural = "" if len(targets) == 1 else "s" + console.print(self._dim(f"Testing {len(targets)} provider{plural}...")) + + results = await asyncio.gather( + *(test_provider_connectivity(n, p) for n, p in sorted(targets.items())) + ) + + lines = ["Provider test (experimental):"] + for r in results: + mark = "\u2713" if r.ok else "\u2717" + lines.append(f" {mark} {r.name:<24} {r.elapsed_s:>5.1f}s {r.detail}") + return "\n".join(lines) + + async def _handle_provider_models(self, name: str) -> str: + """`/provider models [name]` -- list the models a mounted, LIVE + provider actually offers right now. No name given means "the + provider actually answering this conversation" (pinned if + pinned, else the priority winner) -- mirroring the CLI's own + "uses current provider" default, translated to session terms. + + Never gated on the pin capability -- see class docstring above. + """ + # Shares _resolve_diagnostic_targets with /provider test: a named, + # unmounted provider is refused (naming what IS mounted) the same + # way in both -- including when nothing at all is mounted, where + # the unknown-name refusal still fires rather than being masked by + # a generic "nothing mounted" message. + targets, error = self._resolve_diagnostic_targets(name) + if error: + return error + if not targets: + return "Provider models (experimental):\n (no providers mounted in this session)" + + if name: + target_name = name + else: + # No name: use whichever provider is actually answering right + # now -- pinned if pinned, else the priority winner -- so + # "/provider models" with no argument means "the provider + # currently in play", mirroring the CLI's own "uses current + # provider" default in session terms. + pin = self.session.coordinator.get_capability("conversation.provider_pin") + pinned_name: str | None = None + if pin is not None: + try: + pinned_name = pin.current() + except Exception: + pinned_name = None + if pinned_name and pinned_name in targets: + target_name = pinned_name + else: + target_name = min( + targets, + key=lambda n: self._provider_priority_for_display(targets[n]), + ) + target_provider = targets[target_name] + + console.print(self._dim(f"Fetching models for '{target_name}'...")) + + try: + models = await asyncio.wait_for( + invoke_list_models(target_provider), _PROVIDER_DIAGNOSTIC_TIMEOUT_S + ) + except TimeoutError: + return f"\u2717 {target_name}: timed out after {_PROVIDER_DIAGNOSTIC_TIMEOUT_S:.0f}s" + except Exception as e: + return f"\u2717 {target_name}: {type(e).__name__}: {e}" + + if not models: + return f"Provider models (experimental):\n {target_name}: (no models reported)" + + lines = [f"Models for '{target_name}' (experimental):"] + lines.extend(f" {format_model_line(model)}" for model in models) + return "\n".join(lines) + async def _handle_provider(self, args: str) -> str: """Handle /provider: status (no args), 'use ' to pin, or 'auto' to unpin. See REQUIRED BEHAVIORS in the task spec this @@ -1526,6 +1678,20 @@ async def _handle_provider(self, args: str) -> str: if not subcmd: return self._render_provider_status(pin) + # 'test' and 'models' are read-only diagnostics over THIS SESSION'S + # mounted providers -- deliberately never gated on `pin` (see the + # block comment above _handle_provider_test/_handle_provider_models): + # they're the answer to "why isn't pinning working", so gating them + # on the same capability that might be missing/refusing would + # remove the diagnostic exactly when it's needed. + if subcmd == "test": + name = parts[1].strip() if len(parts) > 1 else "" + return await self._handle_provider_test(name) + + if subcmd == "models": + name = parts[1].strip() if len(parts) > 1 else "" + return await self._handle_provider_models(name) + if subcmd in ("use", "auto") and pin is None: return self._provider_pin_unavailable_message() @@ -1572,7 +1738,8 @@ async def _handle_provider(self, args: str) -> str: return ( f"Unknown /provider subcommand: {subcmd!r}. " - f"Usage: /provider | /provider use | /provider auto" + f"Usage: /provider | /provider use | /provider auto | " + f"/provider test | /provider models " ) async def _rename_session(self, new_name: str) -> str: diff --git a/amplifier_app_cli/provider_diagnostics.py b/amplifier_app_cli/provider_diagnostics.py new file mode 100644 index 00000000..a2d230b1 --- /dev/null +++ b/amplifier_app_cli/provider_diagnostics.py @@ -0,0 +1,114 @@ +"""Shared, live-instance provider diagnostics primitives. + +These are the reusable mechanics behind "does this provider answer" and +"what models does it offer" -- the SAME questions ``amplifier provider +test``/``amplifier provider models`` answer for disk-config providers (via +``provider_loader.get_provider_models``, which instantiates a throwaway +provider from settings.yaml), and that the in-session ``/provider +test``/``/provider models`` slash commands answer for this session's +already-mounted, LIVE provider instances. + +The two surfaces intentionally source providers differently: + +- CLI (``amplifier provider ...``): reads settings.yaml, instantiates a + disposable provider object for the single call, and is responsible for + closing it afterward. +- In-session (``/provider ...``): reads ``coordinator.get("providers")`` -- + the actual mounted objects still answering this conversation -- and must + NEVER close them; they are not disposable, they keep running after the + diagnostic completes. + +What must not diverge between the two is the actual definition of +"connectivity is OK" (list_models() succeeds) and how a possibly-async +``list_models()`` is invoked. That mechanic lives here, once, and both +``provider_loader.get_provider_models`` and the in-session slash commands +call through it. +""" + +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from amplifier_core import ModelInfo # pyright: ignore[reportAttributeAccessIssue] + + +async def invoke_list_models(provider: Any) -> list["ModelInfo"]: + """Call ``list_models()`` on an already-instantiated provider. + + Awaits it if async, calls it directly if sync. Returns ``[]`` if the + provider has no ``list_models`` at all. Does NO cleanup and NO error + handling of its own -- exceptions from ``list_models()`` propagate + unchanged, and callers decide whether/how to close the instance + afterward (a throwaway CLI instance should be closed; a session-mounted + instance must not be). + """ + list_models_fn = getattr(provider, "list_models", None) + if list_models_fn is None: + return [] + if asyncio.iscoroutinefunction(list_models_fn): + return await list_models_fn() + return list_models_fn() + + +@dataclass +class ProviderTestResult: + """Outcome of a single provider connectivity check.""" + + name: str + ok: bool + elapsed_s: float + detail: str + + +DEFAULT_TIMEOUT_S = 15.0 + + +async def test_provider_connectivity( + name: str, provider: Any, timeout_s: float = DEFAULT_TIMEOUT_S +) -> ProviderTestResult: + """Connectivity check for one provider: call list_models(), time it. + + Never raises -- failures (including a timeout) are captured in the + returned result, not thrown, so a caller can run many of these + concurrently (e.g. via ``asyncio.gather``) without one slow or failing + provider aborting the batch or hanging an interactive session forever. + + This is the same definition of "connectivity is OK" that ``amplifier + provider test`` uses (list_models() succeeds), so the CLI and the + in-session diagnostic cannot silently disagree on what "ok" means. + """ + start = time.monotonic() + try: + models = await asyncio.wait_for(invoke_list_models(provider), timeout_s) + except TimeoutError: + elapsed = time.monotonic() - start + detail = f"timed out after {timeout_s:.0f}s" + return ProviderTestResult(name=name, ok=False, elapsed_s=elapsed, detail=detail) + except Exception as e: + elapsed = time.monotonic() - start + detail = f"{type(e).__name__}: {e}" + return ProviderTestResult(name=name, ok=False, elapsed_s=elapsed, detail=detail) + + elapsed = time.monotonic() - start + count = len(models) + detail = f"{count} model{'s' if count != 1 else ''} available" + return ProviderTestResult(name=name, ok=True, elapsed_s=elapsed, detail=detail) + + +def format_model_line(model: "ModelInfo") -> str: + """Render one ``ModelInfo`` as a compact, single-line summary. + + Used by the in-session ``/provider models`` slash command, which + returns a plain string (unlike the CLI's Rich ``Table``) -- kept here + so the fields shown (id, context window, max output, capabilities) + match what ``amplifier provider models`` already surfaces. + """ + context = f"{model.context_window:,}" if model.context_window else "-" + max_out = f"{model.max_output_tokens:,}" if model.max_output_tokens else "-" + caps = ", ".join(model.capabilities) if model.capabilities else "-" + return f"{model.id:<28} context={context:<10} max_out={max_out:<8} caps={caps}" diff --git a/amplifier_app_cli/provider_loader.py b/amplifier_app_cli/provider_loader.py index ce6a6b39..3f47c14c 100644 --- a/amplifier_app_cli/provider_loader.py +++ b/amplifier_app_cli/provider_loader.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING from typing import Any +from .provider_diagnostics import invoke_list_models + if TYPE_CHECKING: from amplifier_core import ModelInfo # pyright: ignore[reportAttributeAccessIssue] @@ -168,7 +170,14 @@ def get_provider_models( logger.debug(f"Provider '{provider_id}' does not have list_models()") return [] - # Call list_models (may be sync or async) + # Call list_models (may be sync or async) via the shared invocation + # primitive in provider_diagnostics -- this is the same "call + # list_models, async-aware" mechanic the in-session /provider + # test|models slash commands use against already-mounted providers, so + # the two surfaces can't quietly diverge on how a provider is asked for + # its models. This function still owns instantiation (above) and + # cleanup (below) itself, since only it knows this provider instance is + # disposable -- a mounted session provider must never be closed. # Let exceptions propagate - auth errors, API errors, connection errors # should be shown to the user, not silently swallowed list_models_fn = provider.list_models @@ -176,7 +185,7 @@ def get_provider_models( async def _list_and_cleanup(): try: - return await list_models_fn() + return await invoke_list_models(provider) finally: if hasattr(provider, "close") and callable(provider.close): try: diff --git a/tests/test_provider_test_and_models_command.py b/tests/test_provider_test_and_models_command.py new file mode 100644 index 00000000..e3a2e5ca --- /dev/null +++ b/tests/test_provider_test_and_models_command.py @@ -0,0 +1,402 @@ +"""Tests for the read-only `/provider test [name]` and `/provider models +[name]` diagnostic subcommands. + +Unlike `/provider use`/`/provider auto`, these are NEVER gated on the +'conversation.provider_pin' capability -- they are useful precisely when +pinning is unavailable or refusing (see amplifier_app_cli.main's +_handle_provider_test/_handle_provider_models docstrings). They also always +query THIS SESSION'S mounted providers (coordinator.get("providers")), never +settings.yaml. +""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +from helpers import _make_command_processor +from test_provider_command import _cp_with, _make_pin, _visible + + +def _make_model( + id="model-1", + display_name=None, + context_window=100_000, + max_output_tokens=4096, + capabilities=None, +): + """A ModelInfo-shaped stand-in (format_model_line only reads attributes).""" + return SimpleNamespace( + id=id, + display_name=display_name or id, + context_window=context_window, + max_output_tokens=max_output_tokens, + capabilities=capabilities if capabilities is not None else ["tools"], + ) + + +def _make_live_provider(models=None, error=None, sync=False, no_list_models=False): + """Build a mock, already-instantiated (session-mounted) provider. + + - models: list of ModelInfo-shaped objects returned by list_models() + - error: if set, list_models() raises this exception + - sync: if True, list_models is a plain sync callable (not async) + - no_list_models: if True, the provider has no list_models attribute at all + """ + provider = MagicMock() + if no_list_models: + del provider.list_models + return provider + + if sync: + if error is not None: + provider.list_models = MagicMock(side_effect=error) + else: + provider.list_models = MagicMock(return_value=models or []) + else: + if error is not None: + provider.list_models = AsyncMock(side_effect=error) + else: + provider.list_models = AsyncMock(return_value=models or []) + + # A session-mounted provider must never be closed by a diagnostic -- + # assert on this via close.assert_not_called()/not_awaited() in tests + # that care. + provider.close = AsyncMock() + return provider + + +# ============================================================ +# /provider test +# ============================================================ + + +class TestProviderTestNoName: + @pytest.mark.asyncio + async def test_no_providers_mounted(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("test") + assert "no providers mounted" in result + + @pytest.mark.asyncio + async def test_tests_all_mounted_providers(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[_make_model("m1"), _make_model("m2")] + ), + "openai-fast": _make_live_provider(models=[_make_model("m3")]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "anthropic-fable" in result + assert "openai-fast" in result + assert "2 models available" in result + assert "1 model available" in result + assert "\u2713" in result # success checkmark for both + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin_capability(self): + """This is the whole point of the feature: it must work when + pinning is unavailable -- that's exactly when it's needed.""" + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "not registered" not in result + assert "\u2713" in result + assert "anthropic-fable" in result + + @pytest.mark.asyncio + async def test_reports_failure_without_raising(self): + providers = { + "broken-provider": _make_live_provider(error=RuntimeError("bad api key")), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "\u2717" in result + assert "broken-provider" in result + assert "bad api key" in result + + @pytest.mark.asyncio + async def test_mixed_success_and_failure(self): + providers = { + "good": _make_live_provider(models=[_make_model()]), + "bad": _make_live_provider(error=ValueError("no credentials")), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + lines = result.splitlines() + good_line = next(line for line in lines if line.split()[1] == "good") + bad_line = next(line for line in lines if line.split()[1] == "bad") + assert "\u2713" in good_line + assert "\u2717" in bad_line + assert "no credentials" in bad_line + + @pytest.mark.asyncio + async def test_never_closes_mounted_provider(self): + """The mounted instance must keep answering the conversation after + the diagnostic runs -- closing it would break subsequent turns.""" + provider = _make_live_provider(models=[_make_model()]) + cp = _cp_with( + pin=_make_pin(available=["anthropic-fable"]), + providers={"anthropic-fable": provider}, + ) + await cp._handle_provider("test") + provider.close.assert_not_called() + provider.close.assert_not_awaited() + + @pytest.mark.asyncio + async def test_result_is_tagged_experimental(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test") + assert "(experimental)" in result + + @pytest.mark.asyncio + async def test_sync_list_models_supported(self): + providers = { + "sync-provider": _make_live_provider(models=[_make_model()], sync=True) + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "\u2713" in result + assert "1 model available" in result + + +class TestProviderTestWithName: + @pytest.mark.asyncio + async def test_tests_only_named_provider(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider( + error=RuntimeError("should not be called") + ), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test anthropic-fable")) + assert "anthropic-fable" in result + assert "openai-fast" not in result + + @pytest.mark.asyncio + async def test_unknown_name_names_whats_available(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider(models=[_make_model()]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "\u2717" in result + assert "nonexistent" in result + assert "not mounted" in result + assert "anthropic-fable" in result + assert "openai-fast" in result + + @pytest.mark.asyncio + async def test_unknown_name_error_is_not_tagged_experimental(self): + """Matches the existing /provider use convention: the refusal is + the whole message, tagging it would dilute it.""" + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "(experimental)" not in result + + @pytest.mark.asyncio + async def test_unknown_name_when_nothing_mounted_says_none(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("test nonexistent") + assert "(none)" in result + + @pytest.mark.asyncio + async def test_unknown_name_never_gated_on_missing_pin(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=None, providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "not mounted" in result + assert "not registered" not in result + + +# ============================================================ +# /provider models +# ============================================================ + + +class TestProviderModelsNoName: + @pytest.mark.asyncio + async def test_no_providers_mounted(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("models") + assert "no providers mounted" in result + + @pytest.mark.asyncio + async def test_uses_pinned_provider_when_pinned(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[ + _make_model( + "claude-x", context_window=200_000, max_output_tokens=8192 + ) + ] + ), + "openai-fast": _make_live_provider(models=[_make_model("gpt-x")]), + } + pin = _make_pin(available=list(providers), current="anthropic-fable") + cp = _cp_with(pin=pin, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "claude-x" in result + assert "gpt-x" not in result + assert "context=200,000" in result + + @pytest.mark.asyncio + async def test_uses_priority_winner_when_unpinned(self): + providers = { + "low-priority": _make_live_provider(models=[_make_model("low-model")]), + "high-priority": _make_live_provider(models=[_make_model("high-model")]), + } + providers["low-priority"].priority = 5 + providers["high-priority"].priority = 1 + pin = _make_pin(available=list(providers), current=None) + cp = _cp_with(pin=pin, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "high-model" in result + assert "low-model" not in result + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin_capability(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model("claude-x")]) + } + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "not registered" not in result + assert "claude-x" in result + + @pytest.mark.asyncio + async def test_empty_model_list_reported_not_swallowed(self): + providers = {"anthropic-fable": _make_live_provider(models=[])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models") + assert "no models reported" in result + + @pytest.mark.asyncio + async def test_never_closes_mounted_provider(self): + provider = _make_live_provider(models=[_make_model()]) + cp = _cp_with( + pin=_make_pin(available=["anthropic-fable"]), + providers={"anthropic-fable": provider}, + ) + await cp._handle_provider("models") + provider.close.assert_not_called() + provider.close.assert_not_awaited() + + +class TestProviderModelsWithName: + @pytest.mark.asyncio + async def test_lists_models_for_named_provider(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[ + _make_model( + "claude-x", + context_window=200_000, + max_output_tokens=8192, + capabilities=["tools", "vision"], + ) + ] + ), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("models anthropic-fable")) + assert "claude-x" in result + assert "context=200,000" in result + assert "max_out=8,192" in result + assert "tools, vision" in result + + @pytest.mark.asyncio + async def test_unknown_name_names_whats_available(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider(models=[_make_model()]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models nonexistent") + assert "\u2717" in result + assert "nonexistent" in result + assert "not mounted" in result + assert "anthropic-fable" in result + assert "openai-fast" in result + + @pytest.mark.asyncio + async def test_unknown_name_error_is_not_tagged_experimental(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models nonexistent") + assert "(experimental)" not in result + + @pytest.mark.asyncio + async def test_unknown_name_when_nothing_mounted_says_none(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("models nonexistent") + assert "(none)" in result + + @pytest.mark.asyncio + async def test_reports_failure_without_raising(self): + providers = { + "broken": _make_live_provider(error=RuntimeError("connection refused")) + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models broken") + assert "\u2717" in result + assert "connection refused" in result + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model("claude-x")]) + } + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("models anthropic-fable")) + assert "not registered" not in result + assert "claude-x" in result + + @pytest.mark.asyncio + async def test_sync_list_models_supported(self): + providers = { + "sync-provider": _make_live_provider(models=[_make_model("m1")], sync=True), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("models sync-provider")) + assert "m1" in result + + +# ============================================================ +# Registration / discoverability +# ============================================================ + + +class TestRegistrationAndUsage: + @pytest.mark.asyncio + async def test_help_output_mentions_test_and_models(self): + cp = _make_command_processor() + help_text = cp._format_help() + provider_line = next( + line for line in help_text.splitlines() if line.startswith(" /provider") + ) + assert "/provider test" in provider_line + assert "/provider models" in provider_line + + def test_commands_dict_description_mentions_test_and_models(self): + from amplifier_app_cli.main import CommandProcessor + + description = CommandProcessor.COMMANDS["/provider"]["description"] + assert "/provider test" in description + assert "/provider models" in description + + @pytest.mark.asyncio + async def test_unknown_subcommand_usage_mentions_test_and_models(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("frobnicate") + assert "/provider test" in result + assert "/provider models" in result