Fix distribution, CLI validation, and unknown token metrics

.claude-plugin/marketplace.json

@@ -1,13 +1,15 @@
 {
   "name": "system-one",
+  "description": "System One skills for structured AI decisions and local response validation.",
+  "owner": {
+    "name": "Rodrigo Albe"
+  },
   "plugins": [
     {
       "name": "system-one",
-      "version": "1.0.0",
-      "description": "Structured choices, probabilities, and scores with local response validation.",
-      "skills": [
-        "skills/system-one"
-      ]
+      "source": "./",
+      "version": "1.1.0",
+      "description": "Structured choices, probabilities, and scores with local response validation."
     }
   ]
 }

.claude-plugin/plugin.json

@@ -1,8 +1,10 @@
 {
   "name": "system-one",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Structured AI decisions with multiple providers and local response validation.",
-  "author": "Rodrigo Albe",
+  "author": {
+    "name": "Rodrigo Albe"
+  },
   "license": "MIT",
   "repository": "https://github.com/RodrigoAlbe/system-one"
 }

.github/workflows/ci.yml

@@ -32,3 +32,26 @@ jobs:
         pytest
     - name: Build distributions
       run: python -m build
+    - name: Verify wheel installation and CLI entry point
+      run: python scripts/check_wheel_install.py dist
+
+  plugin:
+    runs-on: ubuntu-latest
+    env:
+      CLAUDE_CONFIG_DIR: ${{ runner.temp }}/system-one-claude
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: "22"
+    - name: Install pinned Claude Code validator
+      run: npm install --global @anthropic-ai/claude-code@2.1.240
+    - name: Validate marketplace
+      run: claude plugin validate --strict .claude-plugin/marketplace.json
+    - name: Validate plugin
+      run: claude plugin validate --strict .claude-plugin/plugin.json
+    - name: Install plugin from the local marketplace
+      run: |
+        claude plugin marketplace add "$GITHUB_WORKSPACE" --scope user
+        claude plugin install system-one@system-one --scope user
+        claude plugin details system-one

MANIFEST.in

@@ -1 +1,2 @@
 include tests/benchmark.py
+include scripts/check_wheel_install.py

README.md

@@ -9,16 +9,21 @@ that a classification is correct.
 
 ## Installation
 
-From a checkout:
+Install from GitHub (requires Git and Python 3.9+):
 
 ```bash
-pip install -e .
+python -m pip install "git+https://github.com/RodrigoAlbe/system-one.git"
 ```
 
-For the published package (check its version before relying on changes on `main`):
+There is currently no public PyPI release. The GitHub command installs `main`;
+append `@<commit-sha>` to the URL to pin a particular revision.
+
+For development, install from a checkout:
 
 ```bash
-pip install system-one-native
+git clone https://github.com/RodrigoAlbe/system-one.git
+cd system-one
+python -m pip install -e .
 ```
 
 Agent skill:
@@ -31,7 +36,15 @@ Claude Code plugin:
 
 ```bash
 claude plugin marketplace add RodrigoAlbe/system-one
-claude plugin install system-one
+claude plugin install system-one@system-one
+```
+
+The skill/plugin installs agent instructions. Install the Python library separately
+when you want to execute those examples. Maintainers can validate both manifests:
+
+```bash
+claude plugin validate --strict .claude-plugin/marketplace.json
+claude plugin validate --strict .claude-plugin/plugin.json
 ```
 
 ## Quickstart
@@ -52,7 +65,7 @@ response = client.evaluate(
 )
 print(response.answers["department"].value)
 print(response.answers["urgent"].value)  # Model-estimated number in [0, 1]
-print(response.metrics.output_tokens)
+print(response.metrics.output_tokens)  # None if the provider omitted usage
 print(response.metrics.estimated_cost_usd)  # None: unknown, not zero
 ```
 
@@ -166,6 +179,14 @@ successful-request latency p50/p95 (nearest-rank p95). Latency includes retries.
 Token totals cover successful responses only; failed requests may consume tokens.
 Cost remains unknown. No competitor or savings figures are fabricated.
 
+Token fields are `None` (JSON `null`) when missing or null in a provider response;
+an explicit zero remains zero. Each `successful_*_tokens` total is only populated
+when every successful response reported that field. Otherwise it is null.
+`reported_*_tokens` sums the available measurements and is null when none exist.
+`token_usage_coverage` reports measured/missing response counts and the fraction
+covered per field, using successful responses as the denominator. With no successful
+responses, totals and coverage fractions are null.
+
 A labeled dataset is a JSON array. Each case needs a unique `id`, `state`,
 `questions`, and an `expected` label for every question:
 
@@ -195,6 +216,8 @@ is still needed before claiming real-world quality or calibrated confidence.
 - Logprobs are disabled by default and no longer rewrite values or confidence.
 - Unsupported diagnostic logprob requests fail before network access.
 - `estimated_cost_usd` is now optional and returns `None` when unknown.
+- `input_tokens`, `output_tokens`, and `total_tokens` are optional too; missing
+  usage is no longer recorded as zero. Benchmark totals include coverage metadata.
 - Empty/duplicate options and invalid question definitions are rejected locally.
 - Multi-position logprob extraction requires an explicit token position.
 
@@ -204,8 +227,21 @@ is still needed before claiming real-world quality or calibrated confidence.
 system-one noul "Customer requests a refund" "Is the customer requesting a refund?"
 system-one choice "Server CPU at 99%" "Action" Scale Restart Ignore
 system-one score "Database disk at 92%" "Severity" Low Medium High Critical
+system-one --provider ollama --model qwen2.5:7b --json choice "Payment failed" "Department" Billing Support
 ```
 
+`choice` and `score` require instructions followed by at least one explicit
+option/level. They never supply placeholder options. `noul` accepts optional
+instructions and rejects extra options. Run `system-one --help` for usage.
+`--provider`, `--model`, and `--json` can appear before or after positional arguments.
+Credentials come from the provider environment variables described above.
+
+With `--json`, stdout contains `answers` (the single question ID is `q`) and
+`metrics`; it excludes raw provider data. Successful runs exit 0. Invalid command
+arguments exit 2 with argparse diagnostics on stderr. Evaluation/configuration
+failures exit 1 with a short error on stderr (a JSON `error` object when `--json`
+is set), leaving stdout empty.
+
 ## License
 
 MIT.

scripts/check_wheel_install.py

@@ -0,0 +1,56 @@
+"""Install a built wheel in an isolated environment and check its CLI entry point."""
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import venv
+
+
+def main():
+    wheels = list(Path(sys.argv[1]).resolve().glob("*.whl"))
+    if len(wheels) != 1:
+        raise SystemExit("Expected exactly one wheel")
+    # Keep temporary artifacts inside the build directory on all platforms.
+    with tempfile.TemporaryDirectory(
+        prefix="install-check-", dir=wheels[0].parent
+    ) as tmp:
+        root = Path(tmp)
+        venv.EnvBuilder(with_pip=True).create(root / "venv")
+        executable_dir = root / "venv" / ("Scripts" if os.name == "nt" else "bin")
+        python = executable_dir / ("python.exe" if os.name == "nt" else "python")
+        cli = executable_dir / ("system-one.exe" if os.name == "nt" else "system-one")
+        env = dict(os.environ)
+        env.pop("PYTHONPATH", None)
+        subprocess.run(
+            [str(python), "-m", "pip", "install", str(wheels[0])],
+            cwd=root,
+            env=env,
+            check=True,
+        )
+        help_result = subprocess.run(
+            [str(cli), "--help"],
+            cwd=root,
+            env=env,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        for option in ("--provider", "--model", "--json"):
+            assert option in help_result.stdout, help_result.stdout
+        invalid = subprocess.run(
+            [str(cli), "choice", "state", "instructions"],
+            cwd=root,
+            env=env,
+            capture_output=True,
+            text=True,
+        )
+        assert invalid.returncode == 2, invalid
+        assert "explicit option/level" in invalid.stderr, invalid.stderr
+        assert invalid.stdout == "", invalid.stdout
+        print("Wheel installation and CLI smoke checks passed")
+
+
+if __name__ == "__main__":
+    main()

src/system_one/cli.py

@@ -1,50 +1,94 @@
-"""
-Command-line interface for System One decisions.
-"""
+"""Command-line interface for validated System One decisions."""
 
 from __future__ import annotations
+
+import argparse
+import json
 import sys
+from dataclasses import asdict
+
 from .client import SystemOneClient
+from .errors import InvalidResponseError, ProviderError
+from .primitives import Choice, Noul, Score
 
 
-def main():
-    if len(sys.argv) < 3:
-        print(
-            "Usage: system-one <type: noul|choice|score> <state/text> [instructions] [options...]"
-        )
-        print("Examples:")
-        print(
-            "  system-one noul 'User reported payout failure' 'Is this an urgent production bug?'"
-        )
-        print(
-            "  system-one choice 'Payment gateway 500 error' 'Department' Backend DevOps Support"
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="system-one",
+        description="Evaluate a structured decision. Choice/score require explicit options/levels.",
+    )
+    parser.add_argument("type", choices=("noul", "choice", "score"))
+    parser.add_argument("state", help="Text to evaluate")
+    parser.add_argument(
+        "instructions", nargs="?", default="Evaluate the provided state"
+    )
+    parser.add_argument(
+        "options", nargs="*", help="Allowed choices or ordered score levels"
+    )
+    parser.add_argument("--provider", choices=("gemini", "groq", "openai", "ollama"))
+    parser.add_argument("--model", help="Override the provider's default model")
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        dest="json_output",
+        help="Print answers and metrics as JSON; raw provider data is omitted",
+    )
+    args = parser.parse_intermixed_args(argv)
+    if args.type in ("choice", "score") and not args.options:
+        parser.error(
+            f"{args.type} requires instructions and at least one explicit option/level"
         )
+    if args.type == "noul" and args.options:
+        parser.error("noul does not accept options/levels")
+    try:
+        if args.type == "choice":
+            question = Choice(options=args.options, instructions=args.instructions)
+        elif args.type == "score":
+            question = Score(levels=args.options, instructions=args.instructions)
+        else:
+            question = Noul(instructions=args.instructions)
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    try:
+        client = SystemOneClient(provider=args.provider, model=args.model)
+        result = client.evaluate(args.state, {"q": question})
+    except (InvalidResponseError, ProviderError, ValueError) as exc:
+        if args.json_output:
+            print(
+                json.dumps(
+                    {"error": {"type": type(exc).__name__, "message": str(exc)}},
+                    ensure_ascii=False,
+                ),
+                file=sys.stderr,
+            )
+        else:
+            print(f"system-one: {exc}", file=sys.stderr)
+        return 1
+
+    if args.json_output:
         print(
-            "  system-one score 'Critical outage detected' 'Severity' Low Medium High Critical"
+            json.dumps(
+                {
+                    "answers": {
+                        key: asdict(answer) for key, answer in result.answers.items()
+                    },
+                    "metrics": asdict(result.metrics),
+                },
+                ensure_ascii=False,
+                allow_nan=False,
+            )
         )
-        sys.exit(1)
-
-    q_type = sys.argv[1].lower()
-    state = sys.argv[2]
-    instr = sys.argv[3] if len(sys.argv) > 3 else "Evaluate the provided state"
-
-    client = SystemOneClient()
-
-    if q_type == "noul":
-        prob = client.noul(state, instr)
-        print(f"Probability (True/Yes): {prob:.2f}")
-    elif q_type == "choice":
-        options = sys.argv[4:] if len(sys.argv) > 4 else ["Option_A", "Option_B"]
-        selected = client.choice(state, instr, options)
-        print(f"Selected: {selected}")
-    elif q_type == "score":
-        levels = sys.argv[4:] if len(sys.argv) > 4 else ["Low", "Medium", "High"]
-        lvl = client.score(state, instr, levels)
-        print(f"Score Level: {lvl}")
     else:
-        print(f"Unknown question type: {q_type}")
-        sys.exit(1)
+        value = result.answers["q"].value
+        if args.type == "noul":
+            print(f"Probability (True/Yes): {value:.2f}")
+        elif args.type == "choice":
+            print(f"Selected: {value}")
+        else:
+            print(f"Score Level: {value}")
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    raise SystemExit(main())

src/system_one/primitives.py

@@ -87,9 +87,9 @@ class EvaluationMetrics:
     """Telemetry and cost metrics for the evaluation."""
 
     latency_ms: float
-    input_tokens: int
-    output_tokens: int
-    total_tokens: int
+    input_tokens: Optional[int]
+    output_tokens: Optional[int]
+    total_tokens: Optional[int]
     estimated_cost_usd: Optional[float]
     provider: str
     model: str

src/system_one/providers.py

@@ -105,7 +105,7 @@ def extract_content(provider, data):
                 for part in parts
                 if not part.get("thought") and "text" in part
             )
-            usage = data.get("usageMetadata") or {}
+            usage = data.get("usageMetadata")
             fields = ("promptTokenCount", "candidatesTokenCount", "totalTokenCount")
         else:
             choice = data["choices"][0]
@@ -118,13 +118,20 @@ def extract_content(provider, data):
             if choice.get("finish_reason") not in (None, "stop"):
                 raise IncompleteResponseError("Provider did not finish a text response")
             content = message["content"]
-            usage = data.get("usage") or {}
+            usage = data.get("usage")
             fields = ("prompt_tokens", "completion_tokens", "total_tokens")
+        if usage is None:
+            usage = {}
         if not isinstance(usage, dict):
             raise InvalidResponseError("Token usage must be an object")
-        counts = tuple(usage.get(field, 0) for field in fields)
-        if any(type(count) is not int or count < 0 for count in counts):
-            raise InvalidResponseError("Token counts must be non-negative integers")
+        counts = tuple(usage.get(field) for field in fields)
+        if any(
+            count is not None and (type(count) is not int or count < 0)
+            for count in counts
+        ):
+            raise InvalidResponseError(
+                "Token counts must be non-negative integers or null"
+            )
         return content, counts
     except (KeyError, IndexError, TypeError, AttributeError) as exc:
         raise InvalidResponseError("Malformed provider response envelope") from exc