diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f3684d9..45fa0e5 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,13 +1,15 @@ { "name": "system-one", + "description": "System One skills for structured AI decisions and local response validation.", + "owner": { + "name": "Rodrigo Albe" + }, "plugins": [ { "name": "system-one", - "version": "1.0.0", - "description": "Structured choices, probabilities, and scores with local response validation.", - "skills": [ - "skills/system-one" - ] + "source": "./", + "version": "1.1.0", + "description": "Structured choices, probabilities, and scores with local response validation." } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index ee2fea3..53dd187 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,8 +1,10 @@ { "name": "system-one", - "version": "1.0.0", + "version": "1.1.0", "description": "Structured AI decisions with multiple providers and local response validation.", - "author": "Rodrigo Albe", + "author": { + "name": "Rodrigo Albe" + }, "license": "MIT", "repository": "https://github.com/RodrigoAlbe/system-one" } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82bc2b9..0060334 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,3 +32,26 @@ jobs: pytest - name: Build distributions run: python -m build + - name: Verify wheel installation and CLI entry point + run: python scripts/check_wheel_install.py dist + + plugin: + runs-on: ubuntu-latest + env: + CLAUDE_CONFIG_DIR: ${{ runner.temp }}/system-one-claude + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install pinned Claude Code validator + run: npm install --global @anthropic-ai/claude-code@2.1.240 + - name: Validate marketplace + run: claude plugin validate --strict .claude-plugin/marketplace.json + - name: Validate plugin + run: claude plugin validate --strict .claude-plugin/plugin.json + - name: Install plugin from the local marketplace + run: | + claude plugin marketplace add "$GITHUB_WORKSPACE" --scope user + claude plugin install system-one@system-one --scope user + claude plugin details system-one diff --git a/MANIFEST.in b/MANIFEST.in index cb5a095..a737ac3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ include tests/benchmark.py +include scripts/check_wheel_install.py diff --git a/README.md b/README.md index a1d3e49..38e9797 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,21 @@ that a classification is correct. ## Installation -From a checkout: +Install from GitHub (requires Git and Python 3.9+): ```bash -pip install -e . +python -m pip install "git+https://github.com/RodrigoAlbe/system-one.git" ``` -For the published package (check its version before relying on changes on `main`): +There is currently no public PyPI release. The GitHub command installs `main`; +append `@` to the URL to pin a particular revision. + +For development, install from a checkout: ```bash -pip install system-one-native +git clone https://github.com/RodrigoAlbe/system-one.git +cd system-one +python -m pip install -e . ``` Agent skill: @@ -31,7 +36,15 @@ Claude Code plugin: ```bash claude plugin marketplace add RodrigoAlbe/system-one -claude plugin install system-one +claude plugin install system-one@system-one +``` + +The skill/plugin installs agent instructions. Install the Python library separately +when you want to execute those examples. Maintainers can validate both manifests: + +```bash +claude plugin validate --strict .claude-plugin/marketplace.json +claude plugin validate --strict .claude-plugin/plugin.json ``` ## Quickstart @@ -52,7 +65,7 @@ response = client.evaluate( ) print(response.answers["department"].value) print(response.answers["urgent"].value) # Model-estimated number in [0, 1] -print(response.metrics.output_tokens) +print(response.metrics.output_tokens) # None if the provider omitted usage print(response.metrics.estimated_cost_usd) # None: unknown, not zero ``` @@ -166,6 +179,14 @@ successful-request latency p50/p95 (nearest-rank p95). Latency includes retries. Token totals cover successful responses only; failed requests may consume tokens. Cost remains unknown. No competitor or savings figures are fabricated. +Token fields are `None` (JSON `null`) when missing or null in a provider response; +an explicit zero remains zero. Each `successful_*_tokens` total is only populated +when every successful response reported that field. Otherwise it is null. +`reported_*_tokens` sums the available measurements and is null when none exist. +`token_usage_coverage` reports measured/missing response counts and the fraction +covered per field, using successful responses as the denominator. With no successful +responses, totals and coverage fractions are null. + A labeled dataset is a JSON array. Each case needs a unique `id`, `state`, `questions`, and an `expected` label for every question: @@ -195,6 +216,8 @@ is still needed before claiming real-world quality or calibrated confidence. - Logprobs are disabled by default and no longer rewrite values or confidence. - Unsupported diagnostic logprob requests fail before network access. - `estimated_cost_usd` is now optional and returns `None` when unknown. +- `input_tokens`, `output_tokens`, and `total_tokens` are optional too; missing + usage is no longer recorded as zero. Benchmark totals include coverage metadata. - Empty/duplicate options and invalid question definitions are rejected locally. - Multi-position logprob extraction requires an explicit token position. @@ -204,8 +227,21 @@ is still needed before claiming real-world quality or calibrated confidence. system-one noul "Customer requests a refund" "Is the customer requesting a refund?" system-one choice "Server CPU at 99%" "Action" Scale Restart Ignore system-one score "Database disk at 92%" "Severity" Low Medium High Critical +system-one --provider ollama --model qwen2.5:7b --json choice "Payment failed" "Department" Billing Support ``` +`choice` and `score` require instructions followed by at least one explicit +option/level. They never supply placeholder options. `noul` accepts optional +instructions and rejects extra options. Run `system-one --help` for usage. +`--provider`, `--model`, and `--json` can appear before or after positional arguments. +Credentials come from the provider environment variables described above. + +With `--json`, stdout contains `answers` (the single question ID is `q`) and +`metrics`; it excludes raw provider data. Successful runs exit 0. Invalid command +arguments exit 2 with argparse diagnostics on stderr. Evaluation/configuration +failures exit 1 with a short error on stderr (a JSON `error` object when `--json` +is set), leaving stdout empty. + ## License MIT. diff --git a/scripts/check_wheel_install.py b/scripts/check_wheel_install.py new file mode 100644 index 0000000..b4b77b1 --- /dev/null +++ b/scripts/check_wheel_install.py @@ -0,0 +1,56 @@ +"""Install a built wheel in an isolated environment and check its CLI entry point.""" + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import venv + + +def main(): + wheels = list(Path(sys.argv[1]).resolve().glob("*.whl")) + if len(wheels) != 1: + raise SystemExit("Expected exactly one wheel") + # Keep temporary artifacts inside the build directory on all platforms. + with tempfile.TemporaryDirectory( + prefix="install-check-", dir=wheels[0].parent + ) as tmp: + root = Path(tmp) + venv.EnvBuilder(with_pip=True).create(root / "venv") + executable_dir = root / "venv" / ("Scripts" if os.name == "nt" else "bin") + python = executable_dir / ("python.exe" if os.name == "nt" else "python") + cli = executable_dir / ("system-one.exe" if os.name == "nt" else "system-one") + env = dict(os.environ) + env.pop("PYTHONPATH", None) + subprocess.run( + [str(python), "-m", "pip", "install", str(wheels[0])], + cwd=root, + env=env, + check=True, + ) + help_result = subprocess.run( + [str(cli), "--help"], + cwd=root, + env=env, + capture_output=True, + text=True, + check=True, + ) + for option in ("--provider", "--model", "--json"): + assert option in help_result.stdout, help_result.stdout + invalid = subprocess.run( + [str(cli), "choice", "state", "instructions"], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + assert invalid.returncode == 2, invalid + assert "explicit option/level" in invalid.stderr, invalid.stderr + assert invalid.stdout == "", invalid.stdout + print("Wheel installation and CLI smoke checks passed") + + +if __name__ == "__main__": + main() diff --git a/src/system_one/cli.py b/src/system_one/cli.py index 303b9ac..b2db07f 100644 --- a/src/system_one/cli.py +++ b/src/system_one/cli.py @@ -1,50 +1,94 @@ -""" -Command-line interface for System One decisions. -""" +"""Command-line interface for validated System One decisions.""" from __future__ import annotations + +import argparse +import json import sys +from dataclasses import asdict + from .client import SystemOneClient +from .errors import InvalidResponseError, ProviderError +from .primitives import Choice, Noul, Score -def main(): - if len(sys.argv) < 3: - print( - "Usage: system-one [instructions] [options...]" - ) - print("Examples:") - print( - " system-one noul 'User reported payout failure' 'Is this an urgent production bug?'" - ) - print( - " system-one choice 'Payment gateway 500 error' 'Department' Backend DevOps Support" +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="system-one", + description="Evaluate a structured decision. Choice/score require explicit options/levels.", + ) + parser.add_argument("type", choices=("noul", "choice", "score")) + parser.add_argument("state", help="Text to evaluate") + parser.add_argument( + "instructions", nargs="?", default="Evaluate the provided state" + ) + parser.add_argument( + "options", nargs="*", help="Allowed choices or ordered score levels" + ) + parser.add_argument("--provider", choices=("gemini", "groq", "openai", "ollama")) + parser.add_argument("--model", help="Override the provider's default model") + parser.add_argument( + "--json", + action="store_true", + dest="json_output", + help="Print answers and metrics as JSON; raw provider data is omitted", + ) + args = parser.parse_intermixed_args(argv) + if args.type in ("choice", "score") and not args.options: + parser.error( + f"{args.type} requires instructions and at least one explicit option/level" ) + if args.type == "noul" and args.options: + parser.error("noul does not accept options/levels") + try: + if args.type == "choice": + question = Choice(options=args.options, instructions=args.instructions) + elif args.type == "score": + question = Score(levels=args.options, instructions=args.instructions) + else: + question = Noul(instructions=args.instructions) + except ValueError as exc: + parser.error(str(exc)) + + try: + client = SystemOneClient(provider=args.provider, model=args.model) + result = client.evaluate(args.state, {"q": question}) + except (InvalidResponseError, ProviderError, ValueError) as exc: + if args.json_output: + print( + json.dumps( + {"error": {"type": type(exc).__name__, "message": str(exc)}}, + ensure_ascii=False, + ), + file=sys.stderr, + ) + else: + print(f"system-one: {exc}", file=sys.stderr) + return 1 + + if args.json_output: print( - " system-one score 'Critical outage detected' 'Severity' Low Medium High Critical" + json.dumps( + { + "answers": { + key: asdict(answer) for key, answer in result.answers.items() + }, + "metrics": asdict(result.metrics), + }, + ensure_ascii=False, + allow_nan=False, + ) ) - sys.exit(1) - - q_type = sys.argv[1].lower() - state = sys.argv[2] - instr = sys.argv[3] if len(sys.argv) > 3 else "Evaluate the provided state" - - client = SystemOneClient() - - if q_type == "noul": - prob = client.noul(state, instr) - print(f"Probability (True/Yes): {prob:.2f}") - elif q_type == "choice": - options = sys.argv[4:] if len(sys.argv) > 4 else ["Option_A", "Option_B"] - selected = client.choice(state, instr, options) - print(f"Selected: {selected}") - elif q_type == "score": - levels = sys.argv[4:] if len(sys.argv) > 4 else ["Low", "Medium", "High"] - lvl = client.score(state, instr, levels) - print(f"Score Level: {lvl}") else: - print(f"Unknown question type: {q_type}") - sys.exit(1) + value = result.answers["q"].value + if args.type == "noul": + print(f"Probability (True/Yes): {value:.2f}") + elif args.type == "choice": + print(f"Selected: {value}") + else: + print(f"Score Level: {value}") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/src/system_one/primitives.py b/src/system_one/primitives.py index a622e50..0cc7126 100644 --- a/src/system_one/primitives.py +++ b/src/system_one/primitives.py @@ -87,9 +87,9 @@ class EvaluationMetrics: """Telemetry and cost metrics for the evaluation.""" latency_ms: float - input_tokens: int - output_tokens: int - total_tokens: int + input_tokens: Optional[int] + output_tokens: Optional[int] + total_tokens: Optional[int] estimated_cost_usd: Optional[float] provider: str model: str diff --git a/src/system_one/providers.py b/src/system_one/providers.py index c2ee2f2..05cb11b 100644 --- a/src/system_one/providers.py +++ b/src/system_one/providers.py @@ -105,7 +105,7 @@ def extract_content(provider, data): for part in parts if not part.get("thought") and "text" in part ) - usage = data.get("usageMetadata") or {} + usage = data.get("usageMetadata") fields = ("promptTokenCount", "candidatesTokenCount", "totalTokenCount") else: choice = data["choices"][0] @@ -118,13 +118,20 @@ def extract_content(provider, data): if choice.get("finish_reason") not in (None, "stop"): raise IncompleteResponseError("Provider did not finish a text response") content = message["content"] - usage = data.get("usage") or {} + usage = data.get("usage") fields = ("prompt_tokens", "completion_tokens", "total_tokens") + if usage is None: + usage = {} if not isinstance(usage, dict): raise InvalidResponseError("Token usage must be an object") - counts = tuple(usage.get(field, 0) for field in fields) - if any(type(count) is not int or count < 0 for count in counts): - raise InvalidResponseError("Token counts must be non-negative integers") + counts = tuple(usage.get(field) for field in fields) + if any( + count is not None and (type(count) is not int or count < 0) + for count in counts + ): + raise InvalidResponseError( + "Token counts must be non-negative integers or null" + ) return content, counts except (KeyError, IndexError, TypeError, AttributeError) as exc: raise InvalidResponseError("Malformed provider response envelope") from exc diff --git a/tests/benchmark.py b/tests/benchmark.py index fffb9db..d6fe45b 100644 --- a/tests/benchmark.py +++ b/tests/benchmark.py @@ -176,7 +176,8 @@ def run_benchmark(client, cases=None, repeats=1): ).hexdigest() records, latencies, brier = [], [], [] correct, labeled, attempted_labels, valid, invalid, failures = 0, 0, 0, 0, 0, 0 - input_tokens, output_tokens, total_tokens = 0, 0, 0 + token_fields = ("input_tokens", "output_tokens", "total_tokens") + reported_tokens = {field: [] for field in token_fields} for repeat in range(repeats): for case in cases: record = {"case_id": case["id"], "repeat": repeat + 1} @@ -191,9 +192,10 @@ def run_benchmark(client, cases=None, repeats=1): else: valid += 1 latencies.append(result.metrics.latency_ms) - input_tokens += result.metrics.input_tokens - output_tokens += result.metrics.output_tokens - total_tokens += result.metrics.total_tokens + for field in token_fields: + value = getattr(result.metrics, field) + if value is not None: + reported_tokens[field].append(value) record.update( status="ok", metrics=asdict(result.metrics), @@ -234,9 +236,25 @@ def run_benchmark(client, cases=None, repeats=1): "successful_latency_p95_ms": ordered[math.ceil(0.95 * len(ordered)) - 1] if ordered else None, - "successful_input_tokens": input_tokens, - "successful_output_tokens": output_tokens, - "successful_total_tokens": total_tokens, + # Complete totals remain unknown if even one successful response omitted a field. + **{ + f"successful_{field}": sum(values) + if valid and len(values) == valid + else None + for field, values in reported_tokens.items() + }, + **{ + f"reported_{field}": sum(values) if values else None + for field, values in reported_tokens.items() + }, + "token_usage_coverage": { + field: { + "reported_responses": len(values), + "missing_responses": valid - len(values), + "fraction": len(values) / valid if valid else None, + } + for field, values in reported_tokens.items() + }, "estimated_cost_usd": None, "labeled_answers": labeled, "attempted_labeled_answers": attempted_labels, diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index e23b755..2904012 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -88,6 +88,9 @@ def test_all_failed_run_has_no_fabricated_metrics(): assert report["noul_brier_score"] is None assert report["accuracy_on_valid_answers"] is None assert report["correct_over_attempted_labels"] == 0 + assert report["successful_total_tokens"] is None + assert report["reported_total_tokens"] is None + assert report["token_usage_coverage"]["total_tokens"]["fraction"] is None def test_unlabeled_smoke_cases_do_not_claim_accuracy(): @@ -124,3 +127,37 @@ def test_invalid_datasets_rejected(tmp_path, data): path.write_text(json.dumps(data), encoding="utf-8") with pytest.raises(ValueError): load_cases(path) + + +def test_partial_usage_does_not_undercount_as_a_complete_total(): + first, second = result(1), result(2) + second.metrics.input_tokens = None + second.metrics.total_tokens = None + report = run_benchmark(client([first, second]), cases(), repeats=2) + assert report["successful_input_tokens"] is None + assert report["successful_total_tokens"] is None + assert report["reported_input_tokens"] == 10 + assert report["reported_total_tokens"] == 15 + assert report["successful_output_tokens"] == 10 + assert report["token_usage_coverage"]["input_tokens"] == { + "reported_responses": 1, + "missing_responses": 1, + "fraction": 0.5, + } + assert report["token_usage_coverage"]["output_tokens"]["fraction"] == 1 + assert report["records"][1]["metrics"]["input_tokens"] is None + + +@pytest.mark.parametrize("value", [None, 0]) +def test_unknown_usage_and_measured_zero_are_not_conflated(value): + response = result(1) + response.metrics.input_tokens = value + response.metrics.output_tokens = value + response.metrics.total_tokens = value + report = run_benchmark(client([response]), cases()) + assert report["successful_total_tokens"] == value + assert report["reported_total_tokens"] == value + assert report["token_usage_coverage"]["total_tokens"]["fraction"] == ( + 0 if value is None else 1 + ) + json.dumps(report, allow_nan=False) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..bbbfd5e --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,161 @@ +import json +from unittest.mock import Mock + +import pytest + +from system_one import ( + EvaluationMetrics, + EvaluationResponse, + QuestionResult, + InvalidResponseError, + ProviderError, +) +from system_one.cli import main + + +@pytest.fixture +def factory(monkeypatch): + factory = Mock() + factory.return_value.evaluate.return_value = EvaluationResponse( + {"q": QuestionResult("q", "choice", "Billing", 0.8)}, + EvaluationMetrics(1, None, 2, None, None, "ollama", "test-model"), + {"private": "raw provider details"}, + ) + monkeypatch.setattr("system_one.cli.SystemOneClient", factory) + return factory + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["unknown", "state"], + ["choice", "state"], + ["choice", "state", "instructions"], + ["score", "state", "instructions"], + ["noul", "state", "instructions", "unexpected"], + ["choice", "state", "instructions", "A", "A"], + ["score", "state", "instructions", ""], + ["noul", "state", ""], + ["noul", "state", "--provider", "unknown"], + ], +) +def test_invalid_arguments_fail_before_client_creation(factory, capsys, argv): + with pytest.raises(SystemExit) as exc: + main(argv) + assert exc.value.code == 2 + factory.assert_not_called() + output = capsys.readouterr() + assert output.out == "" + assert "error:" in output.err + + +@pytest.mark.parametrize( + "argv", + [ + [ + "--provider", + "ollama", + "--model", + "custom", + "--json", + "choice", + "state", + "Route", + "Billing", + "Support", + ], + [ + "choice", + "state", + "Route", + "Billing", + "Support", + "--provider", + "ollama", + "--model", + "custom", + "--json", + ], + [ + "choice", + "state", + "--provider", + "ollama", + "Route", + "Billing", + "--model", + "custom", + "Support", + "--json", + ], + ], +) +def test_flags_and_json_output(factory, capsys, argv): + assert main(argv) == 0 + factory.assert_called_once_with(provider="ollama", model="custom") + state, questions = factory.return_value.evaluate.call_args.args + assert state == "state" + assert questions["q"].options == ["Billing", "Support"] + assert questions["q"].instructions == "Route" + output = capsys.readouterr() + data = json.loads(output.out) + assert data["answers"]["q"]["value"] == "Billing" + assert data["metrics"]["input_tokens"] is None + assert "raw_response" not in data + assert "private" not in output.out + assert output.err == "" + + +@pytest.mark.parametrize( + "kind,values,value,label", + [ + ("choice", ["Billing", "Support"], "Billing", "Selected: Billing"), + ("score", ["Low", "High"], "High", "Score Level: High"), + ("noul", [], 0.8, "Probability (True/Yes): 0.80"), + ], +) +def test_human_output(factory, capsys, kind, values, value, label): + factory.return_value.evaluate.return_value.answers["q"].value = value + assert main([kind, "state", "instructions", *values]) == 0 + assert capsys.readouterr().out.strip() == label + + +def test_noul_keeps_optional_instructions(factory): + factory.return_value.evaluate.return_value.answers["q"].value = 0.5 + assert main(["noul", "state"]) == 0 + assert ( + factory.return_value.evaluate.call_args.args[1]["q"].instructions + == "Evaluate the provided state" + ) + + +@pytest.mark.parametrize( + "error", + [ + InvalidResponseError("Invalid answer"), + ProviderError("Unavailable"), + ValueError("Missing API key"), + ], +) +@pytest.mark.parametrize("json_output", [False, True]) +def test_runtime_errors_have_nonzero_exit_and_stderr( + factory, capsys, error, json_output +): + factory.return_value.evaluate.side_effect = error + assert main(["noul", "state", *(["--json"] if json_output else [])]) == 1 + output = capsys.readouterr() + assert output.out == "" + if json_output: + assert json.loads(output.err)["error"]["type"] == type(error).__name__ + else: + assert str(error) in output.err + assert "Traceback" not in output.err + + +def test_help_is_offline(factory, capsys): + with pytest.raises(SystemExit) as exc: + main(["--help"]) + assert exc.value.code == 0 + factory.assert_not_called() + assert "--provider" in capsys.readouterr().out diff --git a/tests/test_usage.py b/tests/test_usage.py new file mode 100644 index 0000000..db39dd7 --- /dev/null +++ b/tests/test_usage.py @@ -0,0 +1,76 @@ +import pytest + +from system_one import SystemOneClient, Noul, InvalidResponseError + + +def parse_usage(provider, usage, omitted=False): + content = '{"q":{"probability":0.5,"confidence":0.5}}' + if provider == "gemini": + data = { + "candidates": [ + {"finishReason": "STOP", "content": {"parts": [{"text": content}]}} + ] + } + key = "usageMetadata" + else: + data = {"choices": [{"finish_reason": "stop", "message": {"content": content}}]} + key = "usage" + if not omitted: + data[key] = usage + return ( + SystemOneClient(provider=provider, api_key="test") + ._parse_response(data, {"q": Noul("Q")}, 1) + .metrics + ) + + +@pytest.mark.parametrize("provider", ["gemini", "openai", "groq", "ollama"]) +@pytest.mark.parametrize("usage,omitted", [(None, True), (None, False), ({}, False)]) +def test_unknown_usage_stays_unknown(provider, usage, omitted): + metrics = parse_usage(provider, usage, omitted) + assert (metrics.input_tokens, metrics.output_tokens, metrics.total_tokens) == ( + None, + None, + None, + ) + + +@pytest.mark.parametrize( + "provider,fields", + [ + ("gemini", ("promptTokenCount", "candidatesTokenCount", "totalTokenCount")), + ("openai", ("prompt_tokens", "completion_tokens", "total_tokens")), + ], +) +def test_partial_and_zero_usage_are_distinct(provider, fields): + metrics = parse_usage(provider, {fields[0]: 0, fields[1]: None}) + assert (metrics.input_tokens, metrics.output_tokens, metrics.total_tokens) == ( + 0, + None, + None, + ) + metrics = parse_usage(provider, dict(zip(fields, [0, 0, 0]))) + assert (metrics.input_tokens, metrics.output_tokens, metrics.total_tokens) == ( + 0, + 0, + 0, + ) + metrics = parse_usage(provider, {fields[0]: 10, fields[1]: 5}) + assert ( + metrics.total_tokens is None + ) # Never infer totals that may include reasoning tokens. + + +@pytest.mark.parametrize( + "provider,field", [("gemini", "promptTokenCount"), ("openai", "prompt_tokens")] +) +@pytest.mark.parametrize("value", [True, -1, 1.5, "10", [], float("nan")]) +def test_invalid_reported_counts_rejected(provider, field, value): + with pytest.raises(InvalidResponseError): + parse_usage(provider, {field: value}) + + +@pytest.mark.parametrize("usage", [[], False, 0, ""]) +def test_invalid_usage_container_is_not_treated_as_missing(usage): + with pytest.raises(InvalidResponseError): + parse_usage("openai", usage)