From 8a1e1abf1f7b552620835e15a17cac71d4aab0b7 Mon Sep 17 00:00:00 2001 From: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:56:12 -0700 Subject: [PATCH] feat: add /provider test and /provider models slash commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two new read-only diagnostic subcommands to the in-session /provider slash command, closing a diagnostic dead end when provider pins fail. BEFORE: /provider supported exactly three forms—status, use , auto. When a pin failed with ✗ cannot pin conversation provider 'fable': it is not mounted in this session, the user had no in-session way to find out WHY. Bad API key? Wrong model name? They had to leave the session and run amplifier provider test / amplifier provider models in a shell. AFTER: Two new read-only diagnostic forms, callable mid-conversation: • /provider test [name] — connectivity check against the live mounted providers. No name tests all, concurrently, bounded at 15s per provider. • /provider models [name] — lists models a provider actually offers. No name uses the active provider. FILES • amplifier_app_cli/provider_diagnostics.py (new, 114 lines) Shared primitives: invoke_list_models(), ProviderTestResult, test_provider_connectivity(), format_model_line(). Async-aware; 15s timeout per provider; single source of truth for connectivity semantics. • amplifier_app_cli/provider_loader.py (+13/-4) get_provider_models() now delegates to invoke_list_models() rather than keeping its own inline implementation. One less independent implementation of the same mechanic. • amplifier_app_cli/main.py (+171/-4) Slash command dispatch wiring, two new handlers (_handle_provider_test, _handle_provider_models), usage string generation, error messages. Updated /help description; unknown-subcommand error lists all five forms. • tests/test_provider_test_and_models_command.py (new, 402 lines, 29 tests) Unit and integration coverage for both commands, all error paths. THREE DESIGN DECISIONS 1. MOUNTED PROVIDER REALITY, NOT DISK CONFIG These operate on THIS SESSION'S MOUNTED PROVIDERS, not settings.yaml. The existing amplifier provider test/models CLI commands read config from disk and instantiate a throwaway provider. The slash versions read coordinator.get("providers") — the live mounted instances. That is the entire reason to run them mid-conversation instead of in a shell: if the mounted set and the on-disk config disagree, the session's reality is what the user needs. The session's mounted instances are never closed after testing (asserted in tests); only the CLI's throwaway instances are. 2. NOT GATED ON CONVERSATION.PROVIDER_PIN CAPABILITY /provider use and /provider auto require pin capability and refuse loudly without it. These two are pure read-only diagnostics and are useful precisely WHEN pinning is unavailable or failing — gating them would remove the diagnostic exactly when it is needed most. Dispatched before the capability gate; tested with pin=None. 3. SHARED LOGIC AT THE RIGHT SEAM The existing click commands in commands/provider.py were left untouched — they are disk-config-driven and their tests mock at that boundary. What is genuinely shared is the lower mechanic ("call list_models() on a provider object, async-aware") and the definition of "connectivity is OK", both of which now live once in provider_diagnostics.py. So the two surfaces cannot drift on what "OK" means. CONCURRENCY & UX /provider test with no name runs all targets via asyncio.gather, each wrapped in asyncio.wait_for(..., 15s) so one hung provider cannot freeze the REPL. Progress line prints before await. Wall clock tracks the slowest provider (~1.1s cold, ~0.5s warm), not the sum. Markup safety: [name] in usage strings was parsed as Rich markup and silently vanished. Switched to , matching /provider use style. EVIDENCE • 29 new tests; 1413 passed / 1 skipped / 13 deselected / 1 xfailed Same pass count as baseline (verified re-run against original stash). ruff clean; pyright error count identical to baseline (none in new code). • Verified end-to-end in live TUI (build 2026.08.14-25e9944), 8 checks: ✓ /help lists all five forms ✓ /provider models anthropic-sonnet — real network, 3 models, aligned ✓ /provider models (no name) — defaults to priority winner, byte-identical ✓ /provider test anthropic-sonnet — ✓ 0.1s, singular grammar ✓ /provider test (all four) — 4/4 ✓, 3+3+37+31 models across vendors ✓ Concurrency confirmed — wall clock ≠ sum(individual) ✓ /provider test nope — clean refusal, no traceback ✓ Pin regression intact: pin → pinned footer → unpin → priority winner OUT OF SCOPE add, edit, remove, install, manage were NOT added. They mutate persistent config, a different concern from "who is answering me right now", and mixing them in would blur the line: amplifier provider changes what is available tomorrow; /provider changes who is answering now. Closes: #267, #268, #269, #271 (extends /provider feature) Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- amplifier_app_cli/main.py | 171 +++++++- amplifier_app_cli/provider_diagnostics.py | 114 +++++ amplifier_app_cli/provider_loader.py | 13 +- .../test_provider_test_and_models_command.py | 402 ++++++++++++++++++ 4 files changed, 696 insertions(+), 4 deletions(-) create mode 100644 amplifier_app_cli/provider_diagnostics.py create mode 100644 tests/test_provider_test_and_models_command.py diff --git a/amplifier_app_cli/main.py b/amplifier_app_cli/main.py index 7bbd3725..d9a27f65 100644 --- a/amplifier_app_cli/main.py +++ b/amplifier_app_cli/main.py @@ -50,6 +50,10 @@ from .dedicated_tty_input import close_dedicated_tty_input, get_dedicated_tty_input from .effective_config import get_effective_config_summary from .key_manager import KeyManager +from .provider_diagnostics import DEFAULT_TIMEOUT_S as _PROVIDER_DIAGNOSTIC_TIMEOUT_S +from .provider_diagnostics import format_model_line +from .provider_diagnostics import invoke_list_models +from .provider_diagnostics import test_provider_connectivity from .session_runner import SessionConfig, create_initialized_session from .session_store import SessionStore from .stdout_offload import patch_stdout_offloaded as patch_stdout @@ -468,7 +472,8 @@ class CommandProcessor: "action": "handle_provider", "description": ( "(experimental) Show/pin the conversation-scope provider: " - "/provider (status) | /provider use | /provider auto" + "/provider (status) | /provider use | /provider auto | " + "/provider test | /provider models " ), }, } @@ -1502,6 +1507,153 @@ def _render_provider_status(self, pin: Any) -> str: return "\n".join(lines) + # === /provider test | /provider models: read-only diagnostics === + # + # Unlike 'use'/'auto', these two are NEVER gated on the + # 'conversation.provider_pin' capability -- they answer "why can't I + # pin/why is nothing answering", which is exactly the question that + # matters most when the pin capability is absent or refusing. Gating + # them on that same capability would remove the diagnostic exactly + # when it's needed. + # + # They also intentionally query THIS SESSION'S MOUNTED providers + # (coordinator.get("providers")), never settings.yaml -- that's the + # entire reason to run them mid-conversation instead of dropping to a + # shell for `amplifier provider test`/`amplifier provider models`. If + # the mounted set and settings.yaml disagree, the session's live + # reality is what the user needs to see. + + def _unknown_provider_message(self, name: str, mounted: dict[str, Any]) -> str: + """Refusal text for a name that isn't currently mounted. + + Names what IS mounted rather than a bare failure -- matches the + wording ConversationProviderPin.pin() already uses for the same + situation (see the module docstring's WHY: this is the message a + failed pin sends the user here to investigate). + + Deliberately NOT tagged (experimental) or dim -- matches the + capability-absent and unmounted-provider refusals from 'use': the + refusal is the whole message, tagging or dimming would dilute it. + """ + available = ", ".join(sorted(mounted)) if mounted else "(none)" + return ( + f"\u2717 provider {name!r} is not mounted in this session. " + f"Mounted providers: {available}" + ) + + def _resolve_diagnostic_targets( + self, name: str + ) -> tuple[dict[str, Any], str | None]: + """Resolve which mounted provider(s) a diagnostic subcommand + should target: all of them (no name given), or exactly one. + + Returns ``(targets, error)``. ``error`` is set (and ``targets`` + empty) only when a name was given but isn't mounted. + """ + mounted: dict[str, Any] = self.session.coordinator.get("providers") or {} + if not name: + return mounted, None + if name not in mounted: + return {}, self._unknown_provider_message(name, mounted) + return {name: mounted[name]}, None + + async def _handle_provider_test(self, name: str) -> str: + """`/provider test [name]` -- connectivity check against this + session's mounted providers. No name tests all of them, + concurrently (see class docstring above for why this is never + gated on the pin capability, and always live-session-scoped). + + Reuses provider_diagnostics.test_provider_connectivity so "ok" + means exactly what `amplifier provider test` means (list_models() + succeeds) -- the two surfaces cannot silently disagree. + """ + targets, error = self._resolve_diagnostic_targets(name) + if error: + return error + if not targets: + return "Provider test (experimental):\n (no providers mounted in this session)" + + # Network I/O in an interactive REPL: tell the user before the + # (possibly multi-second, possibly multi-provider) wait rather + # than freezing silently. Run all targets concurrently -- and each + # is individually timeout-bounded (see test_provider_connectivity) + # -- so one slow/hung provider can't stall the others or the + # session. + plural = "" if len(targets) == 1 else "s" + console.print(self._dim(f"Testing {len(targets)} provider{plural}...")) + + results = await asyncio.gather( + *(test_provider_connectivity(n, p) for n, p in sorted(targets.items())) + ) + + lines = ["Provider test (experimental):"] + for r in results: + mark = "\u2713" if r.ok else "\u2717" + lines.append(f" {mark} {r.name:<24} {r.elapsed_s:>5.1f}s {r.detail}") + return "\n".join(lines) + + async def _handle_provider_models(self, name: str) -> str: + """`/provider models [name]` -- list the models a mounted, LIVE + provider actually offers right now. No name given means "the + provider actually answering this conversation" (pinned if + pinned, else the priority winner) -- mirroring the CLI's own + "uses current provider" default, translated to session terms. + + Never gated on the pin capability -- see class docstring above. + """ + # Shares _resolve_diagnostic_targets with /provider test: a named, + # unmounted provider is refused (naming what IS mounted) the same + # way in both -- including when nothing at all is mounted, where + # the unknown-name refusal still fires rather than being masked by + # a generic "nothing mounted" message. + targets, error = self._resolve_diagnostic_targets(name) + if error: + return error + if not targets: + return "Provider models (experimental):\n (no providers mounted in this session)" + + if name: + target_name = name + else: + # No name: use whichever provider is actually answering right + # now -- pinned if pinned, else the priority winner -- so + # "/provider models" with no argument means "the provider + # currently in play", mirroring the CLI's own "uses current + # provider" default in session terms. + pin = self.session.coordinator.get_capability("conversation.provider_pin") + pinned_name: str | None = None + if pin is not None: + try: + pinned_name = pin.current() + except Exception: + pinned_name = None + if pinned_name and pinned_name in targets: + target_name = pinned_name + else: + target_name = min( + targets, + key=lambda n: self._provider_priority_for_display(targets[n]), + ) + target_provider = targets[target_name] + + console.print(self._dim(f"Fetching models for '{target_name}'...")) + + try: + models = await asyncio.wait_for( + invoke_list_models(target_provider), _PROVIDER_DIAGNOSTIC_TIMEOUT_S + ) + except TimeoutError: + return f"\u2717 {target_name}: timed out after {_PROVIDER_DIAGNOSTIC_TIMEOUT_S:.0f}s" + except Exception as e: + return f"\u2717 {target_name}: {type(e).__name__}: {e}" + + if not models: + return f"Provider models (experimental):\n {target_name}: (no models reported)" + + lines = [f"Models for '{target_name}' (experimental):"] + lines.extend(f" {format_model_line(model)}" for model in models) + return "\n".join(lines) + async def _handle_provider(self, args: str) -> str: """Handle /provider: status (no args), 'use ' to pin, or 'auto' to unpin. See REQUIRED BEHAVIORS in the task spec this @@ -1526,6 +1678,20 @@ async def _handle_provider(self, args: str) -> str: if not subcmd: return self._render_provider_status(pin) + # 'test' and 'models' are read-only diagnostics over THIS SESSION'S + # mounted providers -- deliberately never gated on `pin` (see the + # block comment above _handle_provider_test/_handle_provider_models): + # they're the answer to "why isn't pinning working", so gating them + # on the same capability that might be missing/refusing would + # remove the diagnostic exactly when it's needed. + if subcmd == "test": + name = parts[1].strip() if len(parts) > 1 else "" + return await self._handle_provider_test(name) + + if subcmd == "models": + name = parts[1].strip() if len(parts) > 1 else "" + return await self._handle_provider_models(name) + if subcmd in ("use", "auto") and pin is None: return self._provider_pin_unavailable_message() @@ -1572,7 +1738,8 @@ async def _handle_provider(self, args: str) -> str: return ( f"Unknown /provider subcommand: {subcmd!r}. " - f"Usage: /provider | /provider use | /provider auto" + f"Usage: /provider | /provider use | /provider auto | " + f"/provider test | /provider models " ) async def _rename_session(self, new_name: str) -> str: diff --git a/amplifier_app_cli/provider_diagnostics.py b/amplifier_app_cli/provider_diagnostics.py new file mode 100644 index 00000000..a2d230b1 --- /dev/null +++ b/amplifier_app_cli/provider_diagnostics.py @@ -0,0 +1,114 @@ +"""Shared, live-instance provider diagnostics primitives. + +These are the reusable mechanics behind "does this provider answer" and +"what models does it offer" -- the SAME questions ``amplifier provider +test``/``amplifier provider models`` answer for disk-config providers (via +``provider_loader.get_provider_models``, which instantiates a throwaway +provider from settings.yaml), and that the in-session ``/provider +test``/``/provider models`` slash commands answer for this session's +already-mounted, LIVE provider instances. + +The two surfaces intentionally source providers differently: + +- CLI (``amplifier provider ...``): reads settings.yaml, instantiates a + disposable provider object for the single call, and is responsible for + closing it afterward. +- In-session (``/provider ...``): reads ``coordinator.get("providers")`` -- + the actual mounted objects still answering this conversation -- and must + NEVER close them; they are not disposable, they keep running after the + diagnostic completes. + +What must not diverge between the two is the actual definition of +"connectivity is OK" (list_models() succeeds) and how a possibly-async +``list_models()`` is invoked. That mechanic lives here, once, and both +``provider_loader.get_provider_models`` and the in-session slash commands +call through it. +""" + +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING +from typing import Any + +if TYPE_CHECKING: + from amplifier_core import ModelInfo # pyright: ignore[reportAttributeAccessIssue] + + +async def invoke_list_models(provider: Any) -> list["ModelInfo"]: + """Call ``list_models()`` on an already-instantiated provider. + + Awaits it if async, calls it directly if sync. Returns ``[]`` if the + provider has no ``list_models`` at all. Does NO cleanup and NO error + handling of its own -- exceptions from ``list_models()`` propagate + unchanged, and callers decide whether/how to close the instance + afterward (a throwaway CLI instance should be closed; a session-mounted + instance must not be). + """ + list_models_fn = getattr(provider, "list_models", None) + if list_models_fn is None: + return [] + if asyncio.iscoroutinefunction(list_models_fn): + return await list_models_fn() + return list_models_fn() + + +@dataclass +class ProviderTestResult: + """Outcome of a single provider connectivity check.""" + + name: str + ok: bool + elapsed_s: float + detail: str + + +DEFAULT_TIMEOUT_S = 15.0 + + +async def test_provider_connectivity( + name: str, provider: Any, timeout_s: float = DEFAULT_TIMEOUT_S +) -> ProviderTestResult: + """Connectivity check for one provider: call list_models(), time it. + + Never raises -- failures (including a timeout) are captured in the + returned result, not thrown, so a caller can run many of these + concurrently (e.g. via ``asyncio.gather``) without one slow or failing + provider aborting the batch or hanging an interactive session forever. + + This is the same definition of "connectivity is OK" that ``amplifier + provider test`` uses (list_models() succeeds), so the CLI and the + in-session diagnostic cannot silently disagree on what "ok" means. + """ + start = time.monotonic() + try: + models = await asyncio.wait_for(invoke_list_models(provider), timeout_s) + except TimeoutError: + elapsed = time.monotonic() - start + detail = f"timed out after {timeout_s:.0f}s" + return ProviderTestResult(name=name, ok=False, elapsed_s=elapsed, detail=detail) + except Exception as e: + elapsed = time.monotonic() - start + detail = f"{type(e).__name__}: {e}" + return ProviderTestResult(name=name, ok=False, elapsed_s=elapsed, detail=detail) + + elapsed = time.monotonic() - start + count = len(models) + detail = f"{count} model{'s' if count != 1 else ''} available" + return ProviderTestResult(name=name, ok=True, elapsed_s=elapsed, detail=detail) + + +def format_model_line(model: "ModelInfo") -> str: + """Render one ``ModelInfo`` as a compact, single-line summary. + + Used by the in-session ``/provider models`` slash command, which + returns a plain string (unlike the CLI's Rich ``Table``) -- kept here + so the fields shown (id, context window, max output, capabilities) + match what ``amplifier provider models`` already surfaces. + """ + context = f"{model.context_window:,}" if model.context_window else "-" + max_out = f"{model.max_output_tokens:,}" if model.max_output_tokens else "-" + caps = ", ".join(model.capabilities) if model.capabilities else "-" + return f"{model.id:<28} context={context:<10} max_out={max_out:<8} caps={caps}" diff --git a/amplifier_app_cli/provider_loader.py b/amplifier_app_cli/provider_loader.py index ce6a6b39..3f47c14c 100644 --- a/amplifier_app_cli/provider_loader.py +++ b/amplifier_app_cli/provider_loader.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING from typing import Any +from .provider_diagnostics import invoke_list_models + if TYPE_CHECKING: from amplifier_core import ModelInfo # pyright: ignore[reportAttributeAccessIssue] @@ -168,7 +170,14 @@ def get_provider_models( logger.debug(f"Provider '{provider_id}' does not have list_models()") return [] - # Call list_models (may be sync or async) + # Call list_models (may be sync or async) via the shared invocation + # primitive in provider_diagnostics -- this is the same "call + # list_models, async-aware" mechanic the in-session /provider + # test|models slash commands use against already-mounted providers, so + # the two surfaces can't quietly diverge on how a provider is asked for + # its models. This function still owns instantiation (above) and + # cleanup (below) itself, since only it knows this provider instance is + # disposable -- a mounted session provider must never be closed. # Let exceptions propagate - auth errors, API errors, connection errors # should be shown to the user, not silently swallowed list_models_fn = provider.list_models @@ -176,7 +185,7 @@ def get_provider_models( async def _list_and_cleanup(): try: - return await list_models_fn() + return await invoke_list_models(provider) finally: if hasattr(provider, "close") and callable(provider.close): try: diff --git a/tests/test_provider_test_and_models_command.py b/tests/test_provider_test_and_models_command.py new file mode 100644 index 00000000..e3a2e5ca --- /dev/null +++ b/tests/test_provider_test_and_models_command.py @@ -0,0 +1,402 @@ +"""Tests for the read-only `/provider test [name]` and `/provider models +[name]` diagnostic subcommands. + +Unlike `/provider use`/`/provider auto`, these are NEVER gated on the +'conversation.provider_pin' capability -- they are useful precisely when +pinning is unavailable or refusing (see amplifier_app_cli.main's +_handle_provider_test/_handle_provider_models docstrings). They also always +query THIS SESSION'S mounted providers (coordinator.get("providers")), never +settings.yaml. +""" + +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +from helpers import _make_command_processor +from test_provider_command import _cp_with, _make_pin, _visible + + +def _make_model( + id="model-1", + display_name=None, + context_window=100_000, + max_output_tokens=4096, + capabilities=None, +): + """A ModelInfo-shaped stand-in (format_model_line only reads attributes).""" + return SimpleNamespace( + id=id, + display_name=display_name or id, + context_window=context_window, + max_output_tokens=max_output_tokens, + capabilities=capabilities if capabilities is not None else ["tools"], + ) + + +def _make_live_provider(models=None, error=None, sync=False, no_list_models=False): + """Build a mock, already-instantiated (session-mounted) provider. + + - models: list of ModelInfo-shaped objects returned by list_models() + - error: if set, list_models() raises this exception + - sync: if True, list_models is a plain sync callable (not async) + - no_list_models: if True, the provider has no list_models attribute at all + """ + provider = MagicMock() + if no_list_models: + del provider.list_models + return provider + + if sync: + if error is not None: + provider.list_models = MagicMock(side_effect=error) + else: + provider.list_models = MagicMock(return_value=models or []) + else: + if error is not None: + provider.list_models = AsyncMock(side_effect=error) + else: + provider.list_models = AsyncMock(return_value=models or []) + + # A session-mounted provider must never be closed by a diagnostic -- + # assert on this via close.assert_not_called()/not_awaited() in tests + # that care. + provider.close = AsyncMock() + return provider + + +# ============================================================ +# /provider test +# ============================================================ + + +class TestProviderTestNoName: + @pytest.mark.asyncio + async def test_no_providers_mounted(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("test") + assert "no providers mounted" in result + + @pytest.mark.asyncio + async def test_tests_all_mounted_providers(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[_make_model("m1"), _make_model("m2")] + ), + "openai-fast": _make_live_provider(models=[_make_model("m3")]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "anthropic-fable" in result + assert "openai-fast" in result + assert "2 models available" in result + assert "1 model available" in result + assert "\u2713" in result # success checkmark for both + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin_capability(self): + """This is the whole point of the feature: it must work when + pinning is unavailable -- that's exactly when it's needed.""" + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "not registered" not in result + assert "\u2713" in result + assert "anthropic-fable" in result + + @pytest.mark.asyncio + async def test_reports_failure_without_raising(self): + providers = { + "broken-provider": _make_live_provider(error=RuntimeError("bad api key")), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "\u2717" in result + assert "broken-provider" in result + assert "bad api key" in result + + @pytest.mark.asyncio + async def test_mixed_success_and_failure(self): + providers = { + "good": _make_live_provider(models=[_make_model()]), + "bad": _make_live_provider(error=ValueError("no credentials")), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + lines = result.splitlines() + good_line = next(line for line in lines if line.split()[1] == "good") + bad_line = next(line for line in lines if line.split()[1] == "bad") + assert "\u2713" in good_line + assert "\u2717" in bad_line + assert "no credentials" in bad_line + + @pytest.mark.asyncio + async def test_never_closes_mounted_provider(self): + """The mounted instance must keep answering the conversation after + the diagnostic runs -- closing it would break subsequent turns.""" + provider = _make_live_provider(models=[_make_model()]) + cp = _cp_with( + pin=_make_pin(available=["anthropic-fable"]), + providers={"anthropic-fable": provider}, + ) + await cp._handle_provider("test") + provider.close.assert_not_called() + provider.close.assert_not_awaited() + + @pytest.mark.asyncio + async def test_result_is_tagged_experimental(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test") + assert "(experimental)" in result + + @pytest.mark.asyncio + async def test_sync_list_models_supported(self): + providers = { + "sync-provider": _make_live_provider(models=[_make_model()], sync=True) + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test")) + assert "\u2713" in result + assert "1 model available" in result + + +class TestProviderTestWithName: + @pytest.mark.asyncio + async def test_tests_only_named_provider(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider( + error=RuntimeError("should not be called") + ), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("test anthropic-fable")) + assert "anthropic-fable" in result + assert "openai-fast" not in result + + @pytest.mark.asyncio + async def test_unknown_name_names_whats_available(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider(models=[_make_model()]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "\u2717" in result + assert "nonexistent" in result + assert "not mounted" in result + assert "anthropic-fable" in result + assert "openai-fast" in result + + @pytest.mark.asyncio + async def test_unknown_name_error_is_not_tagged_experimental(self): + """Matches the existing /provider use convention: the refusal is + the whole message, tagging it would dilute it.""" + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "(experimental)" not in result + + @pytest.mark.asyncio + async def test_unknown_name_when_nothing_mounted_says_none(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("test nonexistent") + assert "(none)" in result + + @pytest.mark.asyncio + async def test_unknown_name_never_gated_on_missing_pin(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=None, providers=providers) + result = await cp._handle_provider("test nonexistent") + assert "not mounted" in result + assert "not registered" not in result + + +# ============================================================ +# /provider models +# ============================================================ + + +class TestProviderModelsNoName: + @pytest.mark.asyncio + async def test_no_providers_mounted(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("models") + assert "no providers mounted" in result + + @pytest.mark.asyncio + async def test_uses_pinned_provider_when_pinned(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[ + _make_model( + "claude-x", context_window=200_000, max_output_tokens=8192 + ) + ] + ), + "openai-fast": _make_live_provider(models=[_make_model("gpt-x")]), + } + pin = _make_pin(available=list(providers), current="anthropic-fable") + cp = _cp_with(pin=pin, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "claude-x" in result + assert "gpt-x" not in result + assert "context=200,000" in result + + @pytest.mark.asyncio + async def test_uses_priority_winner_when_unpinned(self): + providers = { + "low-priority": _make_live_provider(models=[_make_model("low-model")]), + "high-priority": _make_live_provider(models=[_make_model("high-model")]), + } + providers["low-priority"].priority = 5 + providers["high-priority"].priority = 1 + pin = _make_pin(available=list(providers), current=None) + cp = _cp_with(pin=pin, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "high-model" in result + assert "low-model" not in result + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin_capability(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model("claude-x")]) + } + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("models")) + assert "not registered" not in result + assert "claude-x" in result + + @pytest.mark.asyncio + async def test_empty_model_list_reported_not_swallowed(self): + providers = {"anthropic-fable": _make_live_provider(models=[])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models") + assert "no models reported" in result + + @pytest.mark.asyncio + async def test_never_closes_mounted_provider(self): + provider = _make_live_provider(models=[_make_model()]) + cp = _cp_with( + pin=_make_pin(available=["anthropic-fable"]), + providers={"anthropic-fable": provider}, + ) + await cp._handle_provider("models") + provider.close.assert_not_called() + provider.close.assert_not_awaited() + + +class TestProviderModelsWithName: + @pytest.mark.asyncio + async def test_lists_models_for_named_provider(self): + providers = { + "anthropic-fable": _make_live_provider( + models=[ + _make_model( + "claude-x", + context_window=200_000, + max_output_tokens=8192, + capabilities=["tools", "vision"], + ) + ] + ), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("models anthropic-fable")) + assert "claude-x" in result + assert "context=200,000" in result + assert "max_out=8,192" in result + assert "tools, vision" in result + + @pytest.mark.asyncio + async def test_unknown_name_names_whats_available(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model()]), + "openai-fast": _make_live_provider(models=[_make_model()]), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models nonexistent") + assert "\u2717" in result + assert "nonexistent" in result + assert "not mounted" in result + assert "anthropic-fable" in result + assert "openai-fast" in result + + @pytest.mark.asyncio + async def test_unknown_name_error_is_not_tagged_experimental(self): + providers = {"anthropic-fable": _make_live_provider(models=[_make_model()])} + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models nonexistent") + assert "(experimental)" not in result + + @pytest.mark.asyncio + async def test_unknown_name_when_nothing_mounted_says_none(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("models nonexistent") + assert "(none)" in result + + @pytest.mark.asyncio + async def test_reports_failure_without_raising(self): + providers = { + "broken": _make_live_provider(error=RuntimeError("connection refused")) + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = await cp._handle_provider("models broken") + assert "\u2717" in result + assert "connection refused" in result + + @pytest.mark.asyncio + async def test_never_gated_on_missing_pin(self): + providers = { + "anthropic-fable": _make_live_provider(models=[_make_model("claude-x")]) + } + cp = _cp_with(pin=None, providers=providers) + result = _visible(await cp._handle_provider("models anthropic-fable")) + assert "not registered" not in result + assert "claude-x" in result + + @pytest.mark.asyncio + async def test_sync_list_models_supported(self): + providers = { + "sync-provider": _make_live_provider(models=[_make_model("m1")], sync=True), + } + cp = _cp_with(pin=_make_pin(available=list(providers)), providers=providers) + result = _visible(await cp._handle_provider("models sync-provider")) + assert "m1" in result + + +# ============================================================ +# Registration / discoverability +# ============================================================ + + +class TestRegistrationAndUsage: + @pytest.mark.asyncio + async def test_help_output_mentions_test_and_models(self): + cp = _make_command_processor() + help_text = cp._format_help() + provider_line = next( + line for line in help_text.splitlines() if line.startswith(" /provider") + ) + assert "/provider test" in provider_line + assert "/provider models" in provider_line + + def test_commands_dict_description_mentions_test_and_models(self): + from amplifier_app_cli.main import CommandProcessor + + description = CommandProcessor.COMMANDS["/provider"]["description"] + assert "/provider test" in description + assert "/provider models" in description + + @pytest.mark.asyncio + async def test_unknown_subcommand_usage_mentions_test_and_models(self): + cp = _cp_with(pin=_make_pin(), providers={}) + result = await cp._handle_provider("frobnicate") + assert "/provider test" in result + assert "/provider models" in result