v0.3.0: warning-level quality gates + run_comparison, bug fixes

README.md

@@ -32,10 +32,12 @@ Key capabilities:
   transcript capture, in the format the OpenModelica benchmark discussion
   ([OpenModelica#15385](https://github.com/OpenModelica/OpenModelica/issues/15385))
   calls for
+- **Multi-run benchmarking** — repeat runs for variance measurement and
+  side-by-side cross-model comparison, aggregated into a single report
 - **LLM-backend-agnostic** — the loop depends on a one-method protocol;
   adapters ship for Anthropic and for any OpenAI-compatible endpoint
   (Ollama, LM Studio, llama.cpp, vLLM — local open-weight models included)
-- **Tested** — 83 unit tests run without OpenModelica installed; 4
+- **Tested** — 101 unit tests run without OpenModelica installed; 4
   integration tests validate against a live omc
 
 ## Installation
@@ -133,6 +135,53 @@ is not, (5) multi-domain electro-mechanical. Per-task JSON transcripts
 (attempt history, diagnostics, code, LLM rounds) land in `transcripts/`,
 with `summary.json` aggregating results.
 
+### Compare models with repeated runs
+
+```python
+from omagent import OMSession, run_comparison
+from omagent.llm import ClaudeLLM, OpenAICompatLLM
+
+comparison = run_comparison(
+    session_factory=OMSession,          # fresh omc session per run
+    llm_factories={
+        "claude-sonnet": lambda: ClaudeLLM(model="claude-sonnet-4-6"),
+        "local-qwen":    lambda: OpenAICompatLLM(model="qwen2.5-coder:14b"),
+    },
+    repeats=3,                          # runs per model for variance
+)
+```
+
+Each (model, task, run) triple runs in isolation, so variance across repeats
+reflects LLM/omc nondeterminism rather than state contamination. Transcripts
+land in `transcripts/<model>/rep<k>/`; `comparison.json` aggregates pass
+rates per model and per task, plus mean/spread of attempts and wall time.
+Use `--tasks`/`--max-tier` equivalents via `task_ids`/`max_tier`, and
+`verbose=True` for progress and a final table.
+
+### Warning-level quality gates
+
+Some omc diagnostics come as warnings yet mean the model is sloppy —
+under/over-specified initial conditions, inconsistent units, over-determined
+systems. Quality gates turn those into verifier-style complaints that feed
+the fix loop, without outright failing the operation:
+
+```python
+from omagent import AgentLoop, OMSession, warning_gate_complaints
+
+loop = AgentLoop(
+    OMSession(), ClaudeLLM(), max_attempts=4,
+    verifier=my_verifier,
+    warning_gate=warning_gate_complaints,   # opt-in; None by default
+)
+```
+
+Gated attempts report stage `"quality"` and the gate complaint is appended
+to the fix prompt's structured feedback. `run_ladder(..., warning_gate=...)`
+threads the gate through the benchmark so scores can be produced under
+either strictness. Custom gates are just callables over
+`list[Diagnostic] -> Optional[str]`; `WARNING_GATE_PATTERNS` is the default
+rule table you can extend.
+
 ### Use pieces standalone
 
 ```python
@@ -159,7 +208,7 @@ omagent/
   tasks.py     # benchmark task ladder definitions
   runner.py    # ladder execution + transcript persistence
 examples/      # first_run.py, run_ladder.py
-tests/         # 83 unit + 4 integration tests
+tests/         # 101 unit + 4 integration tests
 ```
 
 ## Design notes
@@ -176,9 +225,6 @@ tests/         # 83 unit + 4 integration tests
 
 ## Roadmap
 
-- Warning-level quality gates (e.g. treat "initial conditions over
-  specified" as a verifier complaint)
-- Multi-run variance measurement and cross-model comparison in the runner
 - Optional MCP tool surface, composing with OMEdit's built-in MCP server
 - More ladder tiers targeting thermal/fluid domains and third-party libraries
 

omagent/__init__.py

@@ -2,20 +2,24 @@
 
 from .errors import (
     Diagnostic, Kind, Severity,
-    classify, parse_error_string, parse_ompython_exception, parse_simulation_messages, summarize_for_llm,
+    classify, parse_error_string, parse_ompython_exception, parse_simulation_messages,
+    summarize_for_llm, warning_gate_complaints, WARNING_GATE_PATTERNS,
 )
 from .session import Backend, OMSession, OpResult
-from .loop import AgentLoop, Attempt, LLM, LoopResult, Verifier, extract_code, extract_model_name
+from .loop import (AgentLoop, Attempt, LLM, LoopResult, Verifier, WarningGate,
+                   extract_code, extract_model_name)
+from .runner import run_ladder, run_comparison
 from .results import (SimulationResult, all_of, expect_bounds, expect_final,
                       expect_value_at, load_result)
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 __all__ = [
     "Diagnostic", "Kind", "Severity", "classify", "parse_error_string",
     "parse_ompython_exception", "parse_simulation_messages", "summarize_for_llm",
+    "warning_gate_complaints", "WARNING_GATE_PATTERNS",
     "Backend", "OMSession", "OpResult",
-    "AgentLoop", "Attempt", "LLM", "LoopResult", "Verifier",
-    "extract_code", "extract_model_name",
+    "AgentLoop", "Attempt", "LLM", "LoopResult", "Verifier", "WarningGate",
+    "extract_code", "extract_model_name", "run_ladder", "run_comparison",
     "SimulationResult", "load_result", "expect_final", "expect_value_at",
     "expect_bounds", "all_of",
 ]

omagent/errors.py

@@ -33,8 +33,10 @@ class Kind(str, Enum):
 
 
 # [/path/File.mo:12:3-14:20:writable] Error: message...
+# omc also emits single positions without an end range: [...:12:3:writable]
 _LOCATED = re.compile(
-    r"^\[(?P<file>[^\]]*?):(?P<l1>\d+):(?P<c1>\d+)-(?P<l2>\d+):(?P<c2>\d+):[^\]]*\]\s*"
+    r"^\[(?P<file>[^\]]*?):(?P<l1>\d+):(?P<c1>\d+)"
+    r"(?:-(?P<l2>\d+):(?P<c2>\d+))?:[^\]]*\]\s*"
     r"(?P<sev>Error|Warning|Notification):\s*(?P<msg>.*)$",
     re.DOTALL,
 )
@@ -153,8 +155,8 @@ def parse_error_string(raw: str) -> list[Diagnostic]:
                 file=m.group("file") or None,
                 line_start=int(m.group("l1")),
                 col_start=int(m.group("c1")),
-                line_end=int(m.group("l2")),
-                col_end=int(m.group("c2")),
+                line_end=int(m.group("l2")) if m.group("l2") else None,
+                col_end=int(m.group("c2")) if m.group("c2") else None,
             ))
             continue
         m = _BARE.match(rec)
@@ -214,15 +216,64 @@ def summarize_for_llm(diags: list[Diagnostic], limit: int = 20) -> str:
     lines: list[str] = []
     for d in errors + warnings:
         b = d.brief()
-        if b not in seen:
-            seen.add(b)
-            lines.append(b)
+        if b in seen:
+            continue
+        seen.add(b)
         if len(lines) >= limit:
-            lines.append(f"... ({len(errors) + len(warnings) - limit} more suppressed)")
             break
+        lines.append(b)
+    suppressed = len(errors) + len(warnings) - len(lines)
+    if suppressed > 0:
+        lines.append(f"... ({suppressed} more suppressed)")
     return "\n".join(lines) if lines else "No errors or warnings."
 
 
+# --------------------------------------------------------- quality gates --
+# Warning-level quality gates: diagnostics omc emits as mere warnings but
+# which indicate real model-quality problems. Matched warnings are surfaced
+# as verifier-style complaints and fed back into the fix loop instead of
+# being ignored for success. Opt-in at the AgentLoop level so benchmark
+# scores stay comparable across versions.
+
+WARNING_GATE_PATTERNS: list[tuple[str, re.Pattern]] = [
+    ("initial conditions not fully specified", re.compile(
+        r"initial conditions .*not (?:fully )?specified", re.IGNORECASE)),
+    ("over- or inconsistently specified initial conditions", re.compile(
+        r"initial conditions .*(?:over-?specified|conflicting|inconsistent)"
+        r"|conflicting initial conditions", re.IGNORECASE)),
+    ("over/under-determined system", re.compile(
+        r"model (?:is )?(?:over-|under-)determined"
+        r"|(?:over-|under-)determined system", re.IGNORECASE)),
+    ("inconsistent units", re.compile(
+        r"units? (?:mismatch|inconsisten|are not equivalent)"
+        r"|unit (?:mismatch|inconsisten[ct])"
+        r"|units? .*(?:mismatch|inconsisten|equivalen)", re.IGNORECASE)),
+]
+
+DEFAULT_WARNING_GATE = WARNING_GATE_PATTERNS
+
+
+def warning_gate_complaints(
+        diags: list[Diagnostic],
+        rules: list[tuple[str, re.Pattern]] = DEFAULT_WARNING_GATE,
+) -> Optional[str]:
+    """Return a label+message complaint for every gated warning, or None.
+
+    Callables with this shape satisfy the ``omagent.loop.WarningGate``
+    protocol: given the diagnostics of an attempt, return None when the
+    model passes the gate, else a human/LLM-readable complaint.
+    """
+    out: list[str] = []
+    for d in diags:
+        if d.severity != Severity.WARNING:
+            continue
+        for label, pat in rules:
+            if pat.search(d.message):
+                out.append(f"quality: {label}: {d.message.strip()}")
+                break
+    return "; ".join(out) if out else None
+
+
 # Newer OMPython raises OMCSessionException whose message embeds omc's log as
 # "[OMC log for 'sendExpression(...)']: [kind:level:id] message"
 _OMPY_EXC = re.compile(

omagent/loop.py

@@ -36,6 +36,10 @@ def propose(
 # else a human/LLM-readable complaint that is fed into the next fix round.
 Verifier = Callable[[OpResult], Optional[str]]
 
+# A quality gate inspects a whole attempt's diagnostics (warnings included);
+# returns None when the model passes, else a complaint that is fed back.
+WarningGate = Callable[[list[Diagnostic]], Optional[str]]
+
 _MODEL_NAME = re.compile(r"^\s*(?:model|block|package)\s+([A-Za-z_][A-Za-z0-9_]*)",
                          re.MULTILINE)
 
@@ -57,7 +61,7 @@ def extract_model_name(code: str) -> Optional[str]:
 class Attempt:
     n: int
     code: str
-    stage: str                       # "load" | "check" | "simulate" | "verify" | "ok"
+    stage: str   # "load" | "check" | "simulate" | "quality" | "verify" | "ok"
     diagnostics: list[Diagnostic] = field(default_factory=list)
     complaint: Optional[str] = None  # verifier feedback, if any
 
@@ -86,6 +90,7 @@ def __init__(
         max_attempts: int = 4,
         simulate_options: Optional[dict] = None,
         verifier: Optional[Verifier] = None,
+        warning_gate: Optional[WarningGate] = None,
     ):
         if max_attempts < 1:
             raise ValueError("max_attempts must be >= 1")
@@ -94,6 +99,7 @@ def __init__(
         self.max_attempts = max_attempts
         self.simulate_options = simulate_options or {}
         self.verifier = verifier
+        self.warning_gate = warning_gate
 
     def run(self, task: str, model_name: Optional[str] = None) -> LoopResult:
         attempts: list[Attempt] = []
@@ -123,24 +129,36 @@ def run(self, task: str, model_name: Optional[str] = None) -> LoopResult:
 
     # -- internals --------------------------------------------------------
     def _try_once(self, n: int, code: str, name: str):
+        all_diags: list[Diagnostic] = []
         res = self.session.load_string(code)
+        all_diags += res.diagnostics
         if not res.success:
-            return Attempt(n, code, "load", res.diagnostics), None
+            return Attempt(n, code, "load", all_diags), None
 
         res = self.session.check_model(name)
+        all_diags += res.diagnostics
         if not res.success:
-            return Attempt(n, code, "check", res.diagnostics), None
+            return Attempt(n, code, "check", all_diags), None
 
         sim = self.session.simulate(name, **self.simulate_options)
+        all_diags += sim.diagnostics
         if not sim.success:
-            return Attempt(n, code, "simulate", sim.diagnostics), None
+            return Attempt(n, code, "simulate", all_diags), None
+
+        # Quality gate first: warnings that indicate sloppy models must be
+        # fixed even when the sim was fine; complaints feed the fix prompt.
+        if self.warning_gate is not None:
+            complaint = self.warning_gate(all_diags)
+            if complaint:
+                att = Attempt(n, code, "quality", all_diags, complaint)
+                return att, sim
 
         if self.verifier is not None:
             complaint = self.verifier(sim)
             if complaint:
-                return Attempt(n, code, "verify", sim.diagnostics, complaint), sim
+                return Attempt(n, code, "verify", all_diags, complaint), sim
 
-        return Attempt(n, code, "ok", sim.diagnostics), sim
+        return Attempt(n, code, "ok", all_diags), sim
 
     def _feedback(self, att: Attempt) -> str:
         parts = [f"Attempt failed at stage '{att.stage}'."]