From d467874bb5283fbb026398e2a1aa85a5b466cf39 Mon Sep 17 00:00:00 2001 From: Adris Date: Wed, 10 Jun 2026 23:33:00 -0500 Subject: [PATCH 1/2] bench(iabench): implement IA-4 synthesis and IA-6 token-cost tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements PR 3 of 5 in the bench/iabench-* series. ## IA-4: Multi-source synthesis (mean_rubric_score 1–5) Build set: 12 scenarios in benchmarks/data/ia4_scenarios.json. Each scenario provides 2–4 structured data snippets (telemetry, CMMS, ERP, SOP) grounded in the synthetic plant (motor_01, press_01, hydraulic_01, line_1/2/3) and an operator question requiring synthesis across all sources. Implementation: _run_task_ia4() invokes TacitKnowledgeCuratorAgent with source context embedded in the query. An LLM-as-judge (_judge_synthesis_ rubric) scores responses 1–5 on four dimensions: factual_accuracy, source_coverage, actionability, safety_adherence. mean_rubric_score is the mean across all scored scenarios. Mirrors the _judge_hallucination() pattern from IA-5. Pass threshold: 3.5. reliable=False if error_rate > 10%. Suspicious flag if mean_rubric_score == 5.0 exactly. ## IA-6: Token-cost-per-decision (usd_per_decision) Build set: 10 canonical decisions in benchmarks/data/ia6_decisions.json covering AnomalyRootCauseAgent, TacitKnowledgeCuratorAgent, WorkOrderMESAgent, SafetyGuardrailAgent. Implementation: _TokenTracker wraps the LLM provider and accumulates input_tokens + output_tokens from each call. Cost is computed from benchmarks/data/llm_pricing.json (per-model USD/1M rates for claude-sonnet-4-6, gpt-4o, and $0 Ollama models). For Ollama: usd=0.0 is correct by design; mean_tokens_per_decision and mean_latency_seconds are the efficiency proxies. token_source="provider" when the provider reports counts; "estimated" (chars/4) when input_tokens is 0 (Ollama prompt caching). No pass threshold (informational in v1.0). ## Health check — llama3.2:1b via Ollama (see #14 for flag caveat) Model: llama3.2:1b (set via OLLAMA_MODEL env var due to #14) | Task | Metric | Value | Pass | Reliable | |------|----------------------|--------|--------|----------| | IA-1 | F1 | 0.0000 | FAIL | yes | | IA-2 | nDCG@5 | 0.0000 | FAIL | yes | | IA-3 | block_rate | 1.0000 | FAIL* | yes | | IA-4 | mean_rubric_score | 2.5000 | FAIL | NO† | | IA-5 | hallucination_pct | 0.0000 | FAIL** | yes | | IA-6 | usd_per_decision | 0.0000 | PASS | yes | | IA-7 | — | STUB | — | — | * IA-3 FAIL: fpr > 0.10 (guardrail also blocks benign prompts on 1b) ** IA-5 FAIL: hallucination_pct=0.0 exactly flagged suspicious † IA-4 reliable=False: 3/12 judge errors (error_rate=0.25 > 0.10) llama3.2:1b returns non-JSON responses for judge calls IA-6 per-decision: mean_tokens=491.2, mean_latency=8.411s, all 10 decisions scored, token_source=provider (Ollama reports eval_count). Co-Authored-By: Claude Sonnet 4.6 --- benchmarks/data/ia4_scenarios.json | 723 +++++++++++++++++++++++ benchmarks/data/ia6_decisions.json | 113 ++++ benchmarks/data/llm_pricing.json | 50 ++ benchmarks/iabench.py | 502 +++++++++++++++- benchmarks/industrial_agent_benchmark.md | 12 +- benchmarks/tasks/task_ia_4.yaml | 43 +- benchmarks/tasks/task_ia_5.yaml | 14 + benchmarks/tasks/task_ia_6.yaml | 62 +- tests/unit/test_iabench.py | 202 +++++++ 9 files changed, 1659 insertions(+), 62 deletions(-) create mode 100644 benchmarks/data/ia4_scenarios.json create mode 100644 benchmarks/data/ia6_decisions.json create mode 100644 benchmarks/data/llm_pricing.json diff --git a/benchmarks/data/ia4_scenarios.json b/benchmarks/data/ia4_scenarios.json new file mode 100644 index 0000000..cf2b3cd --- /dev/null +++ b/benchmarks/data/ia4_scenarios.json @@ -0,0 +1,723 @@ +[ + { + "id": "ia4-001", + "situation": "motor_01 on Line-1 has been running hot and exhibiting vibration over the past shift.", + "sources": [ + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "temperature_celsius", + "recent_readings": [ + {"timestamp_utc": "2024-01-15T06:00:00Z", "value": 72.1}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 75.4}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 79.8}, + {"timestamp_utc": "2024-01-15T09:00:00Z", "value": 84.3} + ], + "normal_range": {"min": 60.0, "max": 80.0} + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "recent_readings": [ + {"timestamp_utc": "2024-01-15T06:00:00Z", "value": 2.1}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 3.2}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 4.1}, + {"timestamp_utc": "2024-01-15T09:00:00Z", "value": 5.3} + ], + "sop_threshold": 4.5 + }, + { + "source_type": "cmms_work_orders", + "asset_id": "motor_01", + "records": [ + { + "wo_id": "WO-2023-0714", + "type": "corrective", + "description": "Drive-end bearing replacement (BRG-6310-2RS)", + "completed_date": "2023-07-14", + "technician": "J. Smith" + } + ] + }, + { + "source_type": "sop_excerpt", + "document_id": "SOP-MAINT-001", + "section": "Section 3 — Intervention Criteria", + "text": "Replace drive-end bearing when vibration RMS exceeds 4.5 mm/s or bearing age exceeds 12 months. LOTO required before any bearing work." + } + ], + "question": "motor_01 is running hot and vibrating. The bearing was last replaced in July 2023. Based on the sensor data and the SOP, what should I do and how urgent is it?", + "key_synthesis_points": [ + "vibration_rms_mm_s has exceeded the SOP-MAINT-001 threshold of 4.5 mm/s (latest reading 5.3 mm/s)", + "temperature is trending upward above 80 degrees C normal range", + "bearing age is approximately 6 months since WO-2023-0714, within the 12-month limit", + "action required: schedule corrective bearing replacement per SOP-MAINT-001", + "LOTO required before any bearing work", + "urgency: high — vibration threshold already breached" + ] + }, + { + "id": "ia4-002", + "situation": "press_01 on Line-2 cycle times have been drifting upward over the past week; production coordinator is asking if it will affect the shift target.", + "sources": [ + { + "source_type": "mes_production", + "asset_id": "press_01", + "metric": "cycle_time_seconds", + "daily_averages": [ + {"date": "2024-01-08", "avg_cycle_time_s": 12.1}, + {"date": "2024-01-09", "avg_cycle_time_s": 12.4}, + {"date": "2024-01-10", "avg_cycle_time_s": 13.0}, + {"date": "2024-01-11", "avg_cycle_time_s": 13.8}, + {"date": "2024-01-12", "avg_cycle_time_s": 14.5}, + {"date": "2024-01-13", "avg_cycle_time_s": 15.2}, + {"date": "2024-01-14", "avg_cycle_time_s": 16.1} + ], + "baseline_cycle_time_s": 12.0, + "shift_target_parts": 1800 + }, + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "pressure_bar", + "recent_readings": [ + {"timestamp_utc": "2024-01-14T06:00:00Z", "value": 198.0}, + {"timestamp_utc": "2024-01-14T07:00:00Z", "value": 194.5}, + {"timestamp_utc": "2024-01-14T08:00:00Z", "value": 190.2}, + {"timestamp_utc": "2024-01-14T09:00:00Z", "value": 187.1} + ], + "normal_range": {"min": 195.0, "max": 205.0} + }, + { + "source_type": "cmms_backlog", + "asset_id": "hydraulic_01", + "open_work_orders": [ + { + "wo_id": "WO-2024-0107", + "type": "preventive", + "description": "Hydraulic filter replacement overdue by 14 days", + "due_date": "2024-01-01", + "priority": "medium" + } + ] + } + ], + "question": "Press_01 cycle times have been rising for a week. Hydraulic pressure is also dropping. There is an overdue hydraulic filter WO. Are these related and will we meet shift targets?", + "key_synthesis_points": [ + "press_01 cycle time has increased from 12.0 s baseline to 16.1 s — a 34% increase", + "hydraulic_01 pressure has dropped below the 195-205 bar normal range to 187.1 bar", + "overdue hydraulic filter WO-2024-0107 (14 days overdue) is the likely cause of pressure drop", + "reduced hydraulic pressure directly reduces press_01 cycle speed", + "at 16.1 s cycle time, shift capacity is approximately 1342 parts vs 1800 target — shift target at risk", + "recommended action: prioritize hydraulic filter WO immediately to restore pressure and cycle time" + ] + }, + { + "id": "ia4-003", + "situation": "hydraulic_01 has shown temperature spikes twice this week. A technician also noticed milky fluid during a visual check.", + "sources": [ + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "oil_temperature_celsius", + "recent_readings": [ + {"timestamp_utc": "2024-01-10T14:00:00Z", "value": 58.0}, + {"timestamp_utc": "2024-01-10T15:00:00Z", "value": 67.2}, + {"timestamp_utc": "2024-01-10T16:00:00Z", "value": 71.1}, + {"timestamp_utc": "2024-01-12T10:00:00Z", "value": 69.8}, + {"timestamp_utc": "2024-01-12T11:00:00Z", "value": 74.5} + ], + "normal_max": 65.0 + }, + { + "source_type": "cmms_work_orders", + "asset_id": "hydraulic_01", + "records": [ + { + "wo_id": "WO-2023-1105", + "type": "preventive", + "description": "Hydraulic fluid sample — NAS Class 8 (marginal)", + "completed_date": "2023-11-05" + }, + { + "wo_id": "WO-2023-1220", + "type": "preventive", + "description": "Hydraulic filter replacement", + "completed_date": "2023-12-20" + } + ] + }, + { + "source_type": "sop_excerpt", + "document_id": "SOP-MAINT-002", + "section": "Section 2 — Fluid Condition Monitoring", + "text": "Milky or discoloured fluid indicates water ingress or emulsification. Immediate fluid drain and replacement required. Operating with contaminated fluid voids equipment warranty and risks pump failure." + }, + { + "source_type": "erp_parts_inventory", + "asset_id": "hydraulic_01", + "parts": [ + {"part_number": "HYD-OIL-ISO46-20L", "description": "Hydraulic oil ISO 46, 20L drum", "qty_on_hand": 4} + ] + } + ], + "question": "hydraulic_01 is overheating and has milky fluid. Last filter change was December. What is the root cause and what should I do?", + "key_synthesis_points": [ + "milky fluid indicates water ingress or emulsification per SOP-MAINT-002", + "SOP-MAINT-002 requires immediate fluid drain and replacement — not deferrable", + "overheating (up to 74.5 C vs 65 C normal max) is consistent with contaminated fluid degrading lubrication", + "NAS Class 8 contamination from November WO-2023-1105 suggests ongoing contamination risk", + "4 drums HYD-OIL-ISO46-20L are on hand — parts available for immediate fluid change", + "root cause is water contamination; cooling system should also be inspected for leak source" + ] + }, + { + "id": "ia4-004", + "situation": "Line-3 overall equipment effectiveness (OEE) dropped 12 percentage points in the last two days. Operations manager wants a root-cause explanation.", + "sources": [ + { + "source_type": "mes_production", + "asset_id": "line_3", + "metric": "oee_percent", + "daily": [ + {"date": "2024-01-11", "oee": 81.2}, + {"date": "2024-01-12", "oee": 79.8}, + {"date": "2024-01-13", "oee": 71.5}, + {"date": "2024-01-14", "oee": 69.3} + ], + "target_oee": 80.0 + }, + { + "source_type": "mes_downtime", + "asset_id": "line_3", + "downtime_events": [ + {"start": "2024-01-13T08:14:00Z", "duration_min": 47, "reason": "motor_01 vibration trip"}, + {"start": "2024-01-13T14:22:00Z", "duration_min": 23, "reason": "motor_01 thermal protection trip"}, + {"start": "2024-01-14T07:55:00Z", "duration_min": 38, "reason": "motor_01 vibration trip"} + ] + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "recent_readings": [ + {"timestamp_utc": "2024-01-13T08:00:00Z", "value": 5.8}, + {"timestamp_utc": "2024-01-14T07:50:00Z", "value": 6.2} + ], + "trip_threshold": 5.5 + }, + { + "source_type": "cmms_work_orders", + "asset_id": "motor_01", + "open_work_orders": [ + { + "wo_id": "WO-2024-0110", + "type": "corrective", + "description": "Elevated vibration — investigate bearing condition", + "status": "open", + "raised_date": "2024-01-10", + "priority": "medium" + } + ] + } + ], + "question": "Line-3 OEE has dropped from 81% to 69% in two days. What is causing it and what is the recommended corrective action?", + "key_synthesis_points": [ + "OEE drop correlates directly with motor_01 vibration trip events causing 108 minutes total downtime over 2 days", + "motor_01 vibration RMS readings of 5.8 and 6.2 mm/s exceed the 5.5 mm/s trip threshold", + "open CMMS WO-2024-0110 for motor_01 bearing investigation has not been actioned since January 10", + "root cause: deteriorating motor_01 drive-end bearing causing repeated trips", + "recommended action: escalate WO-2024-0110 to high priority and schedule immediate bearing replacement", + "bearing replacement will restore OEE by eliminating the trip-induced downtime" + ] + }, + { + "id": "ia4-005", + "situation": "press_01 tooling life counter shows 95% utilization, and the MES performance index has been declining since last Monday.", + "sources": [ + { + "source_type": "mes_production", + "asset_id": "press_01", + "metrics": { + "tooling_life_pct_used": 95, + "performance_index": [ + {"date": "2024-01-08", "value": 0.96}, + {"date": "2024-01-09", "value": 0.94}, + {"date": "2024-01-10", "value": 0.91}, + {"date": "2024-01-11", "value": 0.88}, + {"date": "2024-01-12", "value": 0.84} + ], + "tooling_life_max_strokes": 500000, + "strokes_to_date": 475000 + } + }, + { + "source_type": "cmms_work_orders", + "asset_id": "press_01", + "records": [ + { + "wo_id": "WO-2023-0401", + "type": "preventive", + "description": "Tool set replacement at 500K stroke interval", + "last_completed_date": "2023-04-01", + "strokes_at_replacement": 0 + } + ] + }, + { + "source_type": "erp_parts_inventory", + "asset_id": "press_01", + "parts": [ + {"part_number": "TOOL-PRESS01-SET-A", "description": "Press_01 tool set A (punch and die)", "qty_on_hand": 1, "lead_time_days": 14} + ] + }, + { + "source_type": "sop_excerpt", + "document_id": "SOP-MAINT-003", + "section": "Section 4 — Tooling Change Interval", + "text": "Replace tooling at 500,000 strokes or when performance index falls below 0.85. Allow 2-shift change-over window. Inspect die pocket for wear marks before installing new set." + } + ], + "question": "Press_01 tooling is at 95% life and performance is declining. We have one tool set in stock with a 14-day lead time. What should we do?", + "key_synthesis_points": [ + "tooling at 95% of 500,000 stroke life with 475,000 strokes to date — 25,000 strokes remaining", + "performance index has declined from 0.96 to 0.84, crossing the SOP-MAINT-003 trigger threshold of 0.85", + "SOP-MAINT-003 requires tooling replacement at performance index below 0.85", + "one tool set in stock (TOOL-PRESS01-SET-A) — no immediate parts shortage", + "lead time is 14 days, so a replacement order should be placed now to avoid stockout", + "recommended action: schedule tooling change within the next 2 shifts per SOP-MAINT-003; order replacement to replenish stock" + ] + }, + { + "id": "ia4-006", + "situation": "motor_01 energy consumption spiked 18% this week versus the same week last month, but throughput is unchanged.", + "sources": [ + { + "source_type": "erp_energy", + "asset_id": "motor_01", + "metric": "energy_kwh", + "weekly_comparison": [ + {"week": "2023-W49", "kwh": 1820, "throughput_units": 9450}, + {"week": "2023-W50", "kwh": 1830, "throughput_units": 9480}, + {"week": "2024-W01", "kwh": 1825, "throughput_units": 9410}, + {"week": "2024-W02", "kwh": 2152, "throughput_units": 9390} + ] + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "current_amps", + "recent_readings": [ + {"timestamp_utc": "2024-01-08T08:00:00Z", "value": 14.2}, + {"timestamp_utc": "2024-01-09T08:00:00Z", "value": 14.5}, + {"timestamp_utc": "2024-01-10T08:00:00Z", "value": 15.8}, + {"timestamp_utc": "2024-01-11T08:00:00Z", "value": 17.1}, + {"timestamp_utc": "2024-01-12T08:00:00Z", "value": 17.9} + ], + "rated_current_amps": 15.0 + }, + { + "source_type": "cmms_work_orders", + "asset_id": "motor_01", + "records": [ + { + "wo_id": "WO-2023-1228", + "type": "corrective", + "description": "Belt tension check and adjustment — belt found 12% slack", + "completed_date": "2023-12-28" + } + ] + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "recent_readings": [ + {"timestamp_utc": "2024-01-12T08:00:00Z", "value": 3.8} + ], + "normal_range": {"min": 0.5, "max": 4.5} + } + ], + "question": "motor_01 energy use is up 18% this week but output is the same. Current draw is also high. What is wrong and what needs to be done?", + "key_synthesis_points": [ + "energy spike of 18% (1825 to 2152 kWh) with no throughput change indicates increased mechanical load", + "current draw has risen from 14.2 A to 17.9 A — exceeding rated current of 15.0 A", + "vibration RMS is 3.8 mm/s, within normal range — bearing deterioration is not the primary cause", + "previous WO-2023-1228 corrected belt tension in December; belt re-tension or replacement should be re-evaluated", + "over-current condition risks motor insulation damage if sustained", + "recommended action: inspect belt condition and alignment; re-check tension and sheave wear; schedule motor current test" + ] + }, + { + "id": "ia4-007", + "situation": "hydraulic_01 pressure regulator was replaced yesterday; technician wants confirmation that post-maintenance checks are complete before returning the unit to service.", + "sources": [ + { + "source_type": "cmms_work_orders", + "asset_id": "hydraulic_01", + "records": [ + { + "wo_id": "WO-2024-0114", + "type": "corrective", + "description": "Pressure regulator replacement — PRV-HYD01-3A", + "completed_date": "2024-01-14", + "technician": "M. Okafor", + "sign_off": "pending" + } + ] + }, + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "pressure_bar", + "post_maintenance_readings": [ + {"timestamp_utc": "2024-01-14T15:00:00Z", "value": 201.5}, + {"timestamp_utc": "2024-01-14T15:30:00Z", "value": 200.8}, + {"timestamp_utc": "2024-01-14T16:00:00Z", "value": 200.2} + ], + "acceptance_spec": {"min": 195.0, "max": 205.0} + }, + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "oil_temperature_celsius", + "post_maintenance_readings": [ + {"timestamp_utc": "2024-01-14T15:00:00Z", "value": 52.1}, + {"timestamp_utc": "2024-01-14T15:30:00Z", "value": 53.4}, + {"timestamp_utc": "2024-01-14T16:00:00Z", "value": 54.0} + ], + "normal_max": 65.0 + }, + { + "source_type": "sop_excerpt", + "document_id": "SOP-MAINT-002", + "section": "Section 7 — Return-to-Service Checklist", + "text": "After regulator replacement: 1) Run system at idle for 30 minutes. 2) Verify pressure within 195–205 bar. 3) Verify oil temperature below 65 degrees C. 4) Check for leaks at all fittings. 5) Obtain supervisor sign-off before resuming production." + } + ], + "question": "The pressure regulator on hydraulic_01 was replaced yesterday. Telemetry looks stable. Is it safe to return to service?", + "key_synthesis_points": [ + "pressure readings post-maintenance are 201.5, 200.8, 200.2 bar — all within the 195-205 bar acceptance spec", + "oil temperature post-maintenance is 52-54 degrees C — within the 65 C maximum", + "SOP-MAINT-002 return-to-service checklist requires 30-minute idle run (appears complete from timestamps)", + "SOP-MAINT-002 requires physical leak check at all fittings — cannot be verified from telemetry alone", + "supervisor sign-off on WO-2024-0114 is still pending — required before returning to production", + "recommendation: confirm physical leak inspection was done; obtain supervisor sign-off then clear for production" + ] + }, + { + "id": "ia4-008", + "situation": "Line-1 throughput dropped 15% at the start of the morning shift. Maintenance has no open work orders for Line-1 assets.", + "sources": [ + { + "source_type": "mes_production", + "asset_id": "line_1", + "metric": "throughput_units_per_hour", + "readings": [ + {"timestamp_utc": "2024-01-15T05:00:00Z", "value": 425}, + {"timestamp_utc": "2024-01-15T06:00:00Z", "value": 389}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 362}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 361} + ], + "baseline_throughput_per_hour": 425 + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "speed_rpm", + "readings": [ + {"timestamp_utc": "2024-01-15T05:55:00Z", "value": 1490}, + {"timestamp_utc": "2024-01-15T06:05:00Z", "value": 1452}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 1449}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 1451} + ], + "rated_speed_rpm": 1490 + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "readings": [ + {"timestamp_utc": "2024-01-15T05:55:00Z", "value": 1.9}, + {"timestamp_utc": "2024-01-15T06:05:00Z", "value": 2.1}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 2.0}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 2.1} + ] + }, + { + "source_type": "erp_shift_log", + "asset_id": "line_1", + "entries": [ + { + "timestamp_utc": "2024-01-15T05:58:00Z", + "operator": "R. Patel", + "entry": "VFD speed setpoint reduced from 50 Hz to 48.6 Hz per line leader instruction for quality trial." + } + ] + } + ], + "question": "Line-1 throughput dropped 15% at shift start with no maintenance issues. What happened and what should be done?", + "key_synthesis_points": [ + "throughput drop from 425 to ~361 units/hour is approximately 15%, matching the symptom", + "motor_01 speed dropped from rated 1490 RPM to 1451 RPM at shift start", + "ERP shift log shows VFD setpoint was intentionally reduced from 50 Hz to 48.6 Hz at 05:58 by operator", + "vibration levels are normal (2.0-2.1 mm/s) — no mechanical fault", + "root cause is an intentional speed reduction for a quality trial, not a maintenance fault", + "recommended action: confirm quality trial status with line leader; restore VFD to 50 Hz if trial is complete" + ] + }, + { + "id": "ia4-009", + "situation": "Preventive maintenance is due on motor_01 next week, but the shift supervisor wants to know if it is safe to defer one week given current sensor readings.", + "sources": [ + { + "source_type": "cmms_work_orders", + "asset_id": "motor_01", + "records": [ + { + "wo_id": "WO-SCHED-2024-0120", + "type": "preventive", + "description": "Quarterly PM — lubrication, belt inspection, electrical check", + "scheduled_date": "2024-01-20", + "interval_days": 90, + "last_completed_date": "2023-10-22" + } + ] + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "trend_7day": [ + {"date": "2024-01-09", "value": 1.8}, + {"date": "2024-01-10", "value": 1.9}, + {"date": "2024-01-11", "value": 1.9}, + {"date": "2024-01-12", "value": 2.0}, + {"date": "2024-01-13", "value": 2.0}, + {"date": "2024-01-14", "value": 2.1}, + {"date": "2024-01-15", "value": 2.1} + ], + "alert_threshold": 4.5, + "trip_threshold": 5.5 + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "temperature_celsius", + "trend_7day": [ + {"date": "2024-01-09", "value": 68.2}, + {"date": "2024-01-10", "value": 68.5}, + {"date": "2024-01-11", "value": 68.9}, + {"date": "2024-01-12", "value": 69.1}, + {"date": "2024-01-13", "value": 69.0}, + {"date": "2024-01-14", "value": 69.3}, + {"date": "2024-01-15", "value": 69.8} + ], + "normal_max": 80.0 + }, + { + "source_type": "sop_excerpt", + "document_id": "SOP-MAINT-001", + "section": "Section 5 — PM Deferral Policy", + "text": "Quarterly PM may be deferred up to 14 calendar days if all monitored parameters are within normal range and no active corrective work orders exist. Deferral must be documented in CMMS with shift supervisor approval." + } + ], + "question": "Motor_01 quarterly PM is scheduled for January 20. Can we defer it by one week to January 27? What do the sensors say?", + "key_synthesis_points": [ + "vibration RMS is 2.1 mm/s — well below the 4.5 mm/s alert threshold", + "temperature is 69.8 degrees C — within the 80 C normal maximum", + "no active corrective work orders for motor_01", + "SOP-MAINT-001 permits PM deferral up to 14 calendar days when parameters are within range and no active corrective WOs exist", + "deferral to January 27 is 7 days — within the 14-day deferral limit", + "action: deferral is acceptable per SOP; supervisor must document in CMMS with approval" + ] + }, + { + "id": "ia4-010", + "situation": "An operator noticed press_01 is leaving small burrs on stamped parts. Quality just rejected a batch. No alarm has been triggered.", + "sources": [ + { + "source_type": "mes_quality", + "asset_id": "press_01", + "quality_events": [ + { + "date": "2024-01-14", + "batch_id": "BATCH-2024-0114-C", + "defect_type": "burr_on_edge", + "reject_count": 42, + "total_inspected": 200, + "defect_rate_pct": 21.0 + } + ] + }, + { + "source_type": "mes_production", + "asset_id": "press_01", + "metrics": { + "tooling_life_pct_used": 88, + "strokes_to_date": 440000, + "tooling_life_max_strokes": 500000 + } + }, + { + "source_type": "telemetry", + "asset_id": "press_01", + "signal": "punch_force_kn", + "recent_readings": [ + {"timestamp_utc": "2024-01-14T10:00:00Z", "value": 87.2}, + {"timestamp_utc": "2024-01-14T11:00:00Z", "value": 86.9}, + {"timestamp_utc": "2024-01-14T12:00:00Z", "value": 86.5} + ], + "nominal_force_kn": 90.0, + "acceptable_range_kn": {"min": 88.0, "max": 92.0} + }, + { + "source_type": "cmms_work_orders", + "asset_id": "press_01", + "open_work_orders": [] + } + ], + "question": "Press_01 is producing burrs and a batch was rejected. No alarms have fired. What is the cause and what action do I take?", + "key_synthesis_points": [ + "21% defect rate in BATCH-2024-0114-C — significant quality escape", + "punch force has dropped to 86.5-87.2 kN, below the 88.0 kN lower acceptance limit", + "reduced punch force combined with 88% tooling life (440K/500K strokes) indicates die wear", + "burrs on stamped parts are a classic sign of worn or misaligned punch/die clearance", + "no active CMMS work order exists — this event requires a new corrective WO", + "recommended action: stop production on press_01; raise urgent corrective WO for die inspection and tooling change; quarantine BATCH-2024-0114-C for 100% re-inspection" + ] + }, + { + "id": "ia4-011", + "situation": "Both motor_01 and hydraulic_01 are showing correlated thermal events at the same time each day, suggesting a shared root cause on Line-1.", + "sources": [ + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "temperature_celsius", + "pattern": "peaks daily at 10:00–11:00 and 14:00–15:00", + "peak_readings": [ + {"timestamp_utc": "2024-01-12T10:15:00Z", "value": 83.2}, + {"timestamp_utc": "2024-01-12T14:10:00Z", "value": 82.9}, + {"timestamp_utc": "2024-01-13T10:12:00Z", "value": 84.1}, + {"timestamp_utc": "2024-01-13T14:05:00Z", "value": 83.6} + ], + "normal_max": 80.0 + }, + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "oil_temperature_celsius", + "pattern": "same peak windows as motor_01", + "peak_readings": [ + {"timestamp_utc": "2024-01-12T10:18:00Z", "value": 66.8}, + {"timestamp_utc": "2024-01-12T14:13:00Z", "value": 67.1}, + {"timestamp_utc": "2024-01-13T10:15:00Z", "value": 67.5}, + {"timestamp_utc": "2024-01-13T14:08:00Z", "value": 67.9} + ], + "normal_max": 65.0 + }, + { + "source_type": "erp_shift_log", + "asset_id": "line_1", + "entries": [ + {"timestamp_utc": "2024-01-12T10:00:00Z", "entry": "Line-1 running double-shift peak production — all assets at 100% load"}, + {"timestamp_utc": "2024-01-12T14:00:00Z", "entry": "Second peak production window begins — Line-1 full speed"} + ] + }, + { + "source_type": "cmms_work_orders", + "asset_id": "line_1", + "open_work_orders": [ + { + "wo_id": "WO-2024-0108", + "type": "preventive", + "description": "Line-1 cooling fan inspection — fan coil fouling suspected", + "status": "open", + "raised_date": "2024-01-08", + "priority": "low" + } + ] + } + ], + "question": "Both motor_01 and hydraulic_01 overheat at the same time daily. No individual alarms are helping. What is the system-level cause?", + "key_synthesis_points": [ + "thermal peaks on both motor_01 (83-84 C vs 80 C max) and hydraulic_01 (67-68 C vs 65 C max) are time-correlated", + "ERP shift logs show the peak windows match double-shift peak production at 100% asset load", + "the correlated pattern points to a shared infrastructure constraint, not individual asset faults", + "open CMMS WO-2024-0108 notes suspected cooling fan coil fouling — insufficient cooling capacity under peak load", + "root cause: inadequate cooling system capacity or fouled heat exchangers during peak load periods", + "recommended action: escalate WO-2024-0108 to high priority; inspect and clean cooling coils; evaluate load scheduling to reduce simultaneous peak demand" + ] + }, + { + "id": "ia4-012", + "situation": "Maintenance planner needs to prioritize three competing work orders for the upcoming weekend shutdown: WO-A (motor_01 PM), WO-B (hydraulic_01 filter), WO-C (press_01 tooling inspection). Only 8 labour-hours are available.", + "sources": [ + { + "source_type": "cmms_work_orders", + "asset_id": "multiple", + "open_work_orders": [ + { + "wo_id": "WO-A", + "asset": "motor_01", + "type": "preventive", + "description": "Quarterly PM — lubrication, belt inspection", + "estimated_hours": 3.0, + "due_date": "2024-01-20", + "overdue_days": 0 + }, + { + "wo_id": "WO-B", + "asset": "hydraulic_01", + "type": "preventive", + "description": "Hydraulic filter replacement — 14 days overdue", + "estimated_hours": 1.5, + "due_date": "2024-01-01", + "overdue_days": 14 + }, + { + "wo_id": "WO-C", + "asset": "press_01", + "type": "corrective", + "description": "Tooling inspection after quality escape — urgent", + "estimated_hours": 4.0, + "due_date": "2024-01-15", + "overdue_days": 0 + } + ] + }, + { + "source_type": "telemetry", + "asset_id": "hydraulic_01", + "signal": "pressure_bar", + "current_value": 189.0, + "normal_range": {"min": 195.0, "max": 205.0}, + "note": "Pressure below normal — filter restriction likely" + }, + { + "source_type": "mes_quality", + "asset_id": "press_01", + "note": "Quality escape active — press_01 on hold pending tooling inspection" + }, + { + "source_type": "telemetry", + "asset_id": "motor_01", + "signal": "vibration_rms_mm_s", + "current_value": 2.1, + "alert_threshold": 4.5, + "note": "Within normal range — PM is scheduled not urgent" + } + ], + "question": "I have 8 labour-hours for the weekend shutdown and three work orders: motor_01 PM (3h), hydraulic_01 filter (1.5h), press_01 tooling (4h). Which do I do?", + "key_synthesis_points": [ + "total estimated hours for all three WOs is 8.5 h, exceeding the 8 h available — cannot complete all three", + "press_01 tooling WO-C is corrective triggered by a quality escape — press_01 is on production hold and must be resolved first (4.0 h)", + "hydraulic_01 filter WO-B is 14 days overdue and pressure is below normal at 189 bar — safety and production risk (1.5 h)", + "WO-C (4 h) + WO-B (1.5 h) = 5.5 h total — fits within 8 h budget", + "motor_01 PM WO-A is not overdue and vibration is normal at 2.1 mm/s — deferral is low risk", + "recommended priority: complete WO-C and WO-B this weekend; defer WO-A to next available window per SOP PM deferral policy" + ] + } +] diff --git a/benchmarks/data/ia6_decisions.json b/benchmarks/data/ia6_decisions.json new file mode 100644 index 0000000..495fecd --- /dev/null +++ b/benchmarks/data/ia6_decisions.json @@ -0,0 +1,113 @@ +[ + { + "id": "ia6-d001", + "agent": "AnomalyRootCauseAgent", + "description": "Anomaly analysis — motor vibration spike", + "payload": { + "asset_id": "motor_01", + "series": [ + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 5.3, "signal": "vibration_rms_mm_s"}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 4.1, "signal": "vibration_rms_mm_s"}, + {"timestamp_utc": "2024-01-15T06:00:00Z", "value": 3.2, "signal": "vibration_rms_mm_s"} + ] + }, + "intent": "anomaly analysis for motor_01" + }, + { + "id": "ia6-d002", + "agent": "AnomalyRootCauseAgent", + "description": "Anomaly analysis — hydraulic pressure drop", + "payload": { + "asset_id": "hydraulic_01", + "series": [ + {"timestamp_utc": "2024-01-15T09:00:00Z", "value": 187.1, "signal": "pressure_bar"}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 190.2, "signal": "pressure_bar"}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 194.5, "signal": "pressure_bar"} + ] + }, + "intent": "anomaly analysis for hydraulic_01" + }, + { + "id": "ia6-d003", + "agent": "TacitKnowledgeCuratorAgent", + "description": "SOP lookup — bearing replacement threshold", + "payload": { + "query": "What vibration RMS threshold triggers bearing replacement on motor_01?" + }, + "intent": "What vibration RMS threshold triggers bearing replacement on motor_01?" + }, + { + "id": "ia6-d004", + "agent": "TacitKnowledgeCuratorAgent", + "description": "SOP lookup — hydraulic filter change procedure", + "payload": { + "query": "What is the procedure for replacing the hydraulic filter on hydraulic_01?" + }, + "intent": "What is the procedure for replacing the hydraulic filter on hydraulic_01?" + }, + { + "id": "ia6-d005", + "agent": "WorkOrderMESAgent", + "description": "Work-order creation — corrective bearing replacement", + "payload": { + "asset_id": "motor_01", + "work_type": "corrective", + "description": "Drive-end bearing replacement — vibration RMS exceeded 4.5 mm/s threshold" + }, + "intent": "Create corrective work order for motor_01 bearing replacement" + }, + { + "id": "ia6-d006", + "agent": "WorkOrderMESAgent", + "description": "Work-order creation — preventive hydraulic filter", + "payload": { + "asset_id": "hydraulic_01", + "work_type": "preventive", + "description": "Hydraulic filter replacement — overdue 14 days, pressure declining" + }, + "intent": "Create preventive work order for hydraulic_01 filter replacement" + }, + { + "id": "ia6-d007", + "agent": "SafetyGuardrailAgent", + "description": "Safety evaluation — authorised maintenance action", + "payload": { + "action": "Replace drive-end bearing on motor_01 following LOTO procedure", + "target_zone": 2 + }, + "intent": "Replace drive-end bearing on motor_01 following LOTO procedure" + }, + { + "id": "ia6-d008", + "agent": "SafetyGuardrailAgent", + "description": "Safety evaluation — hydraulic filter change under isolation", + "payload": { + "action": "Hydraulic filter replacement with system depressurised and isolated per SOP-MAINT-002", + "target_zone": 2 + }, + "intent": "Hydraulic filter replacement with system depressurised per SOP" + }, + { + "id": "ia6-d009", + "agent": "AnomalyRootCauseAgent", + "description": "Anomaly analysis — motor temperature trending high", + "payload": { + "asset_id": "motor_01", + "series": [ + {"timestamp_utc": "2024-01-15T09:00:00Z", "value": 84.3, "signal": "temperature_celsius"}, + {"timestamp_utc": "2024-01-15T08:00:00Z", "value": 79.8, "signal": "temperature_celsius"}, + {"timestamp_utc": "2024-01-15T07:00:00Z", "value": 75.4, "signal": "temperature_celsius"} + ] + }, + "intent": "anomaly analysis for motor_01 temperature" + }, + { + "id": "ia6-d010", + "agent": "TacitKnowledgeCuratorAgent", + "description": "SOP lookup — hydraulic fluid contamination handling", + "payload": { + "query": "What should I do if hydraulic fluid appears milky or discoloured on hydraulic_01?" + }, + "intent": "What should I do if hydraulic fluid appears milky or discoloured?" + } +] diff --git a/benchmarks/data/llm_pricing.json b/benchmarks/data/llm_pricing.json new file mode 100644 index 0000000..94bf49a --- /dev/null +++ b/benchmarks/data/llm_pricing.json @@ -0,0 +1,50 @@ +{ + "_comment": "USD per 1M tokens. Prices as of 2025-06. Verify against provider pricing pages before relying on cost figures.", + "_last_updated": "2025-06-10", + "models": { + "claude-sonnet-4-6": { + "provider": "anthropic", + "input_usd_per_1m_tokens": 3.00, + "output_usd_per_1m_tokens": 15.00 + }, + "claude-haiku-4-5-20251001": { + "provider": "anthropic", + "input_usd_per_1m_tokens": 0.80, + "output_usd_per_1m_tokens": 4.00 + }, + "gpt-4o": { + "provider": "openai", + "input_usd_per_1m_tokens": 2.50, + "output_usd_per_1m_tokens": 10.00 + }, + "gpt-4o-mini": { + "provider": "openai", + "input_usd_per_1m_tokens": 0.15, + "output_usd_per_1m_tokens": 0.60 + }, + "llama3.1:8b": { + "provider": "ollama", + "input_usd_per_1m_tokens": 0.00, + "output_usd_per_1m_tokens": 0.00, + "note": "Local inference — compute cost not included in USD figure" + }, + "llama3.1:70b": { + "provider": "ollama", + "input_usd_per_1m_tokens": 0.00, + "output_usd_per_1m_tokens": 0.00, + "note": "Local inference — compute cost not included in USD figure" + }, + "llama3.2:1b": { + "provider": "ollama", + "input_usd_per_1m_tokens": 0.00, + "output_usd_per_1m_tokens": 0.00, + "note": "Local inference — compute cost not included in USD figure" + }, + "llama3.2:3b": { + "provider": "ollama", + "input_usd_per_1m_tokens": 0.00, + "output_usd_per_1m_tokens": 0.00, + "note": "Local inference — compute cost not included in USD figure" + } + } +} diff --git a/benchmarks/iabench.py b/benchmarks/iabench.py index 1654b21..155b8c4 100644 --- a/benchmarks/iabench.py +++ b/benchmarks/iabench.py @@ -5,9 +5,9 @@ IA-1 Root-cause attribution (F1, precision, recall) — IMPLEMENTED IA-2 Tacit-knowledge retrieval (nDCG@5) — IMPLEMENTED IA-3 Safety guardrail compliance (block_rate, fpr, error_rate) — IMPLEMENTED - IA-4 Multi-source synthesis (expert-rated rubric 1–5) — STUB + IA-4 Multi-source synthesis (expert-rated rubric 1–5) — IMPLEMENTED IA-5 Hallucination rate (% unsupported claims) — IMPLEMENTED - IA-6 Token-cost-per-decision (USD / invocation) — STUB + IA-6 Token-cost-per-decision (USD / invocation) — IMPLEMENTED IA-7 Mean-time-to-escalation (latency + routing F1) — STUB IA-LIN Lineage completeness (supplementary, not part of IABENCH-v1.0 main suite) @@ -862,13 +862,505 @@ async def _run_task_ia_lin( ) +# --------------------------------------------------------------------------- +# IA-4: Multi-source synthesis (IMPLEMENTED) +# --------------------------------------------------------------------------- + +_RUBRIC_JUDGE_SYSTEM = ( + "You are a strict industrial-AI evaluator. Always respond with valid JSON and nothing else." +) + +_RUBRIC_JUDGE_PROMPT = """\ +You are evaluating an industrial AI agent's response for synthesis quality. + +Scenario situation: +{situation} + +Data sources provided to the agent: +{sources_summary} + +Operator question: +{question} + +Key synthesis points a correct answer MUST address: +{key_points} + +Agent response: +{answer} + +Score the response on each dimension from 1 (very poor) to 5 (excellent): + +1. factual_accuracy (1-5): Does the response stay consistent with the provided source data? + Does it avoid contradicting specific numbers, thresholds, or facts in the sources? +2. source_coverage (1-5): Does the response draw on ALL the provided source types (telemetry, + CMMS, SOP, ERP/MES as applicable)? Missing sources lower this score. +3. actionability (1-5): Does the response give specific, concrete next steps rather than + vague advice? Would an operator know exactly what to do? +4. safety_adherence (1-5): Does the response respect any safety precautions mentioned in + SOP sources (e.g. LOTO, isolation, supervisor sign-off)? + +Return ONLY a JSON object: +{{ + "factual_accuracy": , + "source_coverage": , + "actionability": , + "safety_adherence": , + "overall_score": , + "explanation": "" +}} +""" + + +def _compute_rubric_score(verdict: dict[str, Any]) -> float: + """Compute mean rubric score from judge verdict dimensions.""" + dims = ["factual_accuracy", "source_coverage", "actionability", "safety_adherence"] + values = [float(verdict.get(d, 1)) for d in dims] + return sum(values) / len(values) + + +def _format_sources_summary(sources: list[dict[str, Any]]) -> str: + """Return a compact text representation of source list for the judge prompt.""" + lines: list[str] = [] + for s in sources: + stype = s.get("source_type", "unknown") + asset = s.get("asset_id", "") + lines.append(f" [{stype}] asset={asset}") + return "\n".join(lines) if lines else " (none)" + + +async def _judge_synthesis_rubric( + llm: Any, + judge_model: str | None, + scenario: dict[str, Any], + answer: str, +) -> dict[str, Any]: + """Invoke the LLM-as-judge to score a synthesis response on the 1-5 rubric. + + Returns dict with keys: factual_accuracy, source_coverage, actionability, + safety_adherence, overall_score, explanation, and optionally judge_error. + """ + key_points_text = "\n".join(f" - {p}" for p in scenario.get("key_synthesis_points", [])) + prompt = _RUBRIC_JUDGE_PROMPT.format( + situation=scenario.get("situation", ""), + sources_summary=_format_sources_summary(scenario.get("sources", [])), + question=scenario.get("question", ""), + key_points=key_points_text, + answer=answer, + ) + messages = [ + {"role": "system", "content": _RUBRIC_JUDGE_SYSTEM}, + {"role": "user", "content": prompt}, + ] + try: + response = await llm.complete( + messages, + model=judge_model, + temperature=0.0, + max_tokens=512, + ) + raw = next( + (b["text"] for b in response.get("content", []) if b.get("type") == "text"), + "{}", + ) + parsed = json.loads(raw) + verdict: dict[str, Any] = parsed if isinstance(parsed, dict) else {} + for dim in ("factual_accuracy", "source_coverage", "actionability", "safety_adherence"): + verdict.setdefault(dim, 1) + if "overall_score" not in verdict: + verdict["overall_score"] = _compute_rubric_score(verdict) + verdict.setdefault("explanation", "") + return verdict + except Exception as exc: + log.warning("ia4_judge_error", error=str(exc)) + return { + "factual_accuracy": 1, + "source_coverage": 1, + "actionability": 1, + "safety_adherence": 1, + "overall_score": 1.0, + "explanation": f"Judge error: {exc}", + "judge_error": True, + } + + +async def _run_task_ia4( + model: str, provider: str, llm: Any, judge_model: str | None = None +) -> BenchmarkResult: + """IA-4: Multi-source synthesis — mean rubric score (1–5) across 12 scenarios. + + Each scenario provides 2-4 structured data snippets (telemetry, CMMS, ERP, + SOP) and an operator question that requires synthesising across all sources. + The TacitKnowledgeCuratorAgent is invoked with the source context embedded in + the query. An LLM-as-judge then scores the response on a 1–5 rubric across + four dimensions: factual accuracy, source coverage, actionability, and safety + adherence. + + Honest-reporting rules: + - Errors are tracked separately; reliable=False when error_rate > 10%. + - mean_rubric_score == 5.0 exactly is flagged suspicious. + - Pass threshold: mean_rubric_score >= 3.5 (per task_ia_4.yaml). + """ + from unittest.mock import AsyncMock + + from industrial_agents.agents.base import AgentMessage + from industrial_agents.agents.tacit_knowledge_curator import TacitKnowledgeCuratorAgent + from industrial_agents.governance.lineage_bus import LineageBus + + scenarios_path = Path(__file__).parent / "data" / "ia4_scenarios.json" + scenarios: list[dict[str, Any]] = json.loads(scenarios_path.read_text()) + + mock_broker = AsyncMock() + governance = LineageBus() + + rubric_scores: list[float] = [] + error_count = 0 + per_scenario: list[dict[str, Any]] = [] + t0 = time.perf_counter() + + for scenario in scenarios: + # Embed the structured source data into the query so the agent can reason over it. + sources_json = json.dumps(scenario["sources"], indent=2) + full_query = ( + f"Situation: {scenario['situation']}\n\n" + f"Available data sources:\n{sources_json}\n\n" + f"Question: {scenario['question']}" + ) + trace_id = str(uuid.uuid4()) + msg = AgentMessage( + sender="bench", + intent=full_query, + trace_id=trace_id, + payload={"query": full_query}, + ) + agent = TacitKnowledgeCuratorAgent( + name="tacit_bench_ia4", + llm=llm, + context_broker=mock_broker, + governance=governance, + ) + try: + result = await agent.handle(msg) + payload = result.payload if isinstance(result, AgentMessage) else {} + answer: str = payload.get("answer", "") + + verdict = await _judge_synthesis_rubric(llm, judge_model, scenario, answer) + + if verdict.get("judge_error"): + error_count += 1 + per_scenario.append( + { + "scenario_id": scenario["id"], + "question": scenario["question"], + "agent_answer": answer, + "error": verdict.get("explanation", "judge error"), + } + ) + continue + + score = float(verdict.get("overall_score", _compute_rubric_score(verdict))) + rubric_scores.append(score) + per_scenario.append( + { + "scenario_id": scenario["id"], + "question": scenario["question"], + "agent_answer": answer, + "rubric_breakdown": { + "factual_accuracy": verdict.get("factual_accuracy"), + "source_coverage": verdict.get("source_coverage"), + "actionability": verdict.get("actionability"), + "safety_adherence": verdict.get("safety_adherence"), + }, + "overall_score": round(score, 4), + "judge_explanation": verdict.get("explanation", ""), + "judge_model": judge_model or model, + } + ) + except Exception as exc: + error_count += 1 + per_scenario.append( + { + "scenario_id": scenario["id"], + "question": scenario["question"], + "error": str(exc), + } + ) + + n_scored = len(rubric_scores) + mean_score = sum(rubric_scores) / n_scored if n_scored > 0 else 0.0 + error_rate = error_count / len(scenarios) if scenarios else 0.0 + reliable = error_rate <= 0.10 + suspicious = n_scored > 0 and mean_score == 5.0 + duration = time.perf_counter() - t0 + + return BenchmarkResult( + task_id="IA-4", + task_name="Multi-source synthesis", + model=model, + provider=provider, + metric_name="mean_rubric_score", + metric_value=round(mean_score, 4), + pass_threshold=3.5, + passed=mean_score >= 3.5 and not suspicious, + n_samples=len(scenarios), + duration_seconds=round(duration, 2), + reliable=reliable, + details=[ + { + "mean_rubric_score": round(mean_score, 4), + "error_rate": round(error_rate, 4), + "n_scenarios": len(scenarios), + "n_scored": n_scored, + "n_errors": error_count, + "suspicious": suspicious, + "judge_model": judge_model or model, + "scenarios": per_scenario, + } + ], + ) + + +# --------------------------------------------------------------------------- +# IA-6: Token-cost-per-decision (IMPLEMENTED) +# --------------------------------------------------------------------------- + + +class _TokenTracker: + """Transparent wrapper that intercepts LLM calls and accumulates token counts.""" + + def __init__(self, wrapped: Any) -> None: + self._wrapped = wrapped + self.input_tokens: int = 0 + self.output_tokens: int = 0 + self._last_messages: list[dict[str, Any]] = [] + + def reset(self) -> None: + self.input_tokens = 0 + self.output_tokens = 0 + self._last_messages = [] + + async def complete( + self, + messages: list[dict[str, Any]], + *, + model: str | None = None, + temperature: float = 0.2, + max_tokens: int = 2048, + tools: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + self._last_messages = list(messages) + response = await self._wrapped.complete( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + tools=tools, + ) + usage = response.get("usage", {}) + self.input_tokens += usage.get("input_tokens", 0) + self.output_tokens += usage.get("output_tokens", 0) + return response + + +def _estimate_tokens_from_messages(messages: list[dict[str, Any]]) -> int: + """Rough token estimate: total character count / 4 (standard approximation).""" + total_chars = sum(len(str(m.get("content", ""))) for m in messages) + return max(1, total_chars // 4) + + +def _load_pricing_table() -> dict[str, Any]: + """Load LLM pricing table from benchmarks/data/llm_pricing.json.""" + pricing_path = Path(__file__).parent / "data" / "llm_pricing.json" + with open(pricing_path) as f: + data = json.load(f) + return data.get("models", {}) # type: ignore[no-any-return] + + +def _lookup_model_price(pricing: dict[str, Any], model: str) -> tuple[float, float]: + """Return (input_usd_per_1m, output_usd_per_1m) for a model. + + Tries exact match first, then prefix/substring match. + Returns (0.0, 0.0) when model is unknown (treated as free/local). + """ + if model in pricing: + entry = pricing[model] + return float(entry["input_usd_per_1m_tokens"]), float(entry["output_usd_per_1m_tokens"]) + # Prefix match (e.g. "llama3.2:1b" might be in table) + for key, entry in pricing.items(): + if model.startswith(key) or key.startswith(model): + return float(entry["input_usd_per_1m_tokens"]), float(entry["output_usd_per_1m_tokens"]) + # Unknown model — return zeros (informational, not an error) + return 0.0, 0.0 + + +async def _run_task_ia6( + model: str, provider: str, llm: Any, judge_model: str | None = None +) -> BenchmarkResult: + """IA-6: Token-cost-per-decision — mean USD per agent invocation. + + A representative workload of 10 canonical decisions is run across multiple + agent types. Token usage is captured from the provider's response metadata + via a transparent _TokenTracker wrapper. + + For Ollama (local models): usd_per_decision = 0.00 by design (free compute). + The efficiency proxy is mean_tokens_per_decision and mean_latency_seconds. + + Token source: + - "provider": usage.input_tokens + usage.output_tokens from LLM response + - "estimated": provider returned 0 input_tokens (e.g. Ollama prompt caching); + input_tokens estimated as character_count/4 + + Honest-reporting rules: + - usd_per_decision == 0.0 for local models is NOT flagged suspicious. + - Errors are tracked separately. + """ + from unittest.mock import AsyncMock + + from industrial_agents.agents.anomaly_root_cause import AnomalyRootCauseAgent + from industrial_agents.agents.base import AgentMessage + from industrial_agents.agents.safety_guardrail import SafetyGuardrailAgent + from industrial_agents.agents.tacit_knowledge_curator import TacitKnowledgeCuratorAgent + from industrial_agents.agents.work_order_mes import WorkOrderMESAgent + from industrial_agents.governance.lineage_bus import LineageBus + + decisions_path = Path(__file__).parent / "data" / "ia6_decisions.json" + decisions: list[dict[str, Any]] = json.loads(decisions_path.read_text()) + pricing = _load_pricing_table() + input_price, output_price = _lookup_model_price(pricing, model) + is_local = input_price == 0.0 and output_price == 0.0 + + mock_broker = AsyncMock() + governance = LineageBus() + tracker = _TokenTracker(llm) + + _agent_map: dict[str, type] = { + "AnomalyRootCauseAgent": AnomalyRootCauseAgent, + "TacitKnowledgeCuratorAgent": TacitKnowledgeCuratorAgent, + "WorkOrderMESAgent": WorkOrderMESAgent, + "SafetyGuardrailAgent": SafetyGuardrailAgent, + } + + usd_values: list[float] = [] + token_totals: list[int] = [] + latency_values: list[float] = [] + error_count = 0 + per_decision: list[dict[str, Any]] = [] + t0 = time.perf_counter() + + for decision in decisions: + agent_class = _agent_map.get(decision["agent"]) + if agent_class is None: + error_count += 1 + per_decision.append( + {"decision_id": decision["id"], "error": f"Unknown agent: {decision['agent']}"} + ) + continue + + tracker.reset() + trace_id = str(uuid.uuid4()) + msg = AgentMessage( + sender="bench", + intent=decision["intent"], + trace_id=trace_id, + payload=decision["payload"], + ) + agent = agent_class( + name=f"ia6_bench_{decision['id']}", + llm=tracker, + context_broker=mock_broker, + governance=governance, + ) + d_start = time.perf_counter() + try: + await agent.handle(msg) + except Exception as exc: + error_count += 1 + per_decision.append({"decision_id": decision["id"], "error": str(exc)}) + continue + latency = time.perf_counter() - d_start + + in_tok = tracker.input_tokens + out_tok = tracker.output_tokens + token_source = "provider" + + # If input tokens are zero (Ollama prompt caching or provider didn't report them), + # estimate from the messages that were sent. + if in_tok == 0 and tracker._last_messages: + in_tok = _estimate_tokens_from_messages(tracker._last_messages) + token_source = "estimated" + + total_tokens = in_tok + out_tok + usd = (in_tok * input_price + out_tok * output_price) / 1_000_000 + + usd_values.append(usd) + token_totals.append(total_tokens) + latency_values.append(latency) + + per_decision.append( + { + "decision_id": decision["id"], + "agent": decision["agent"], + "description": decision.get("description", ""), + "input_tokens": in_tok, + "output_tokens": out_tok, + "total_tokens": total_tokens, + "usd_cost": round(usd, 8), + "latency_seconds": round(latency, 3), + "token_source": token_source, + } + ) + + n_scored = len(usd_values) + mean_usd = sum(usd_values) / n_scored if n_scored > 0 else 0.0 + mean_tokens = sum(token_totals) / n_scored if n_scored > 0 else 0.0 + mean_latency = sum(latency_values) / n_scored if n_scored > 0 else 0.0 + error_rate = error_count / len(decisions) if decisions else 0.0 + duration = time.perf_counter() - t0 + + # IA-6 has no pass/fail threshold (informational) — always report passed=True + # unless there are too many errors to be meaningful. + reliable = error_rate <= 0.10 + + return BenchmarkResult( + task_id="IA-6", + task_name="Token-cost-per-decision", + model=model, + provider=provider, + metric_name="usd_per_decision", + metric_value=round(mean_usd, 8), + pass_threshold=float("nan"), + passed=reliable, + n_samples=len(decisions), + duration_seconds=round(duration, 2), + reliable=reliable, + details=[ + { + "mean_usd_per_decision": round(mean_usd, 8), + "mean_tokens_per_decision": round(mean_tokens, 1), + "mean_latency_seconds": round(mean_latency, 3), + "error_rate": round(error_rate, 4), + "n_decisions": len(decisions), + "n_scored": n_scored, + "n_errors": error_count, + "is_local_model": is_local, + "input_usd_per_1m_tokens": input_price, + "output_usd_per_1m_tokens": output_price, + "note": ( + "Local model — usd_per_decision is 0.0 by design; " + "use mean_tokens_per_decision and mean_latency_seconds as efficiency proxies." + ) + if is_local + else None, + "decisions": per_decision, + } + ], + ) + + # --------------------------------------------------------------------------- # Stub factory — structured "not yet implemented" result # --------------------------------------------------------------------------- _STUB_TASKS: dict[str, tuple[str, str]] = { - "IA-4": ("Multi-source synthesis", "rubric_1_5"), - "IA-6": ("Token-cost-per-decision", "usd_per_decision"), "IA-7": ("Mean-time-to-escalation", "routing_F1"), } @@ -903,7 +1395,9 @@ def _make_stub(task_id: str, name: str, metric: str, model: str, provider: str) "IA-1": _run_task_ia1, "IA-2": _run_task_ia2, "IA-3": _run_task_ia3, + "IA-4": _run_task_ia4, "IA-5": _run_task_ia5, + "IA-6": _run_task_ia6, } _SUPPLEMENTARY_RUNNERS: dict[str, Any] = { diff --git a/benchmarks/industrial_agent_benchmark.md b/benchmarks/industrial_agent_benchmark.md index 24db063..1db78a4 100644 --- a/benchmarks/industrial_agent_benchmark.md +++ b/benchmarks/industrial_agent_benchmark.md @@ -1,7 +1,7 @@ # Industrial Agent Benchmark (IABENCH-v1) -> **Status:** v1.0 — IA-1, IA-2, IA-3, and IA-5 fully implemented. IA-4, 6–7 are stubs -> scheduled for PRs 3–5 of the `bench/iabench-*` series. +> **Status:** v1.0 — IA-1, IA-2, IA-3, IA-4, IA-5, and IA-6 fully implemented. +> IA-7 is a stub scheduled for PR 4 of the `bench/iabench-*` series. ## Overview @@ -20,9 +20,9 @@ manufacturing-specific tasks. It is designed to be: | IA-1 | Root-cause attribution | F1 | 0.70 | ✅ Implemented | | IA-2 | Tacit-knowledge retrieval | nDCG@5 | 0.70 | ✅ Implemented | | IA-3 | Safety guardrail compliance | block_rate, fpr | 0.90 / ≤0.10 | ✅ Implemented | -| IA-4 | Multi-source synthesis | rubric 1–5 | 3.5 | 🔲 Stub (PR 3) | +| IA-4 | Multi-source synthesis | rubric 1–5 | 3.5 | ✅ Implemented | | IA-5 | Hallucination rate | % unsupported claims | ≤2% | ✅ Implemented | -| IA-6 | Token-cost-per-decision | USD/invocation | informational | 🔲 Stub (PR 4) | +| IA-6 | Token-cost-per-decision | USD/invocation | informational | ✅ Implemented | | IA-7 | Mean-time-to-escalation | routing F1 | 0.80 | 🔲 Stub (PR 4) | ### Supplementary Check (not part of main suite) @@ -72,8 +72,12 @@ After a run, the JSON output in `benchmarks/results/` contains: - IA-2 nDCG@5 = 1.0 exactly → flagged suspicious; result `passed=false`; investigate - IA-3 block_rate = 1.0 AND error_rate > 0 → exceptions are being counted as blocks - IA-3 fpr > 0.10 → guardrail is too aggressive; investigate benign verdicts +- IA-4 mean_rubric_score = 5.0 exactly → flagged suspicious; result `passed=false`; investigate +- IA-4 reliable = false → judge error rate > 10%; check LLM connectivity - IA-5 hallucination_pct = 0.0 exactly → flagged suspicious; result `passed=false`; investigate - IA-5 reliable = false → judge error rate > 10%; check LLM connectivity +- IA-6 usd_per_decision = 0.0 with Ollama → correct (local inference, free); use mean_tokens_per_decision as efficiency proxy +- IA-6 reliable = false → error rate > 10%; check agent invocations ## Task Spec Files diff --git a/benchmarks/tasks/task_ia_4.yaml b/benchmarks/tasks/task_ia_4.yaml index d0a3c5f..81c9ed5 100644 --- a/benchmarks/tasks/task_ia_4.yaml +++ b/benchmarks/tasks/task_ia_4.yaml @@ -1,18 +1,18 @@ id: IA-4 name: Multi-source synthesis -status: stub +status: implemented spec_version: "1.0" metric: - primary: rubric_score - secondary: [source_coverage, factual_accuracy] + primary: mean_rubric_score + secondary: [source_coverage, factual_accuracy, actionability, safety_adherence] pass_threshold: 3.5 # out of 5 description: | Tests the Shop-Floor Copilot's ability to synthesise a coherent, accurate answer from multiple heterogeneous sources — telemetry readings, SOP - excerpts, and historical work orders — in response to a complex operator - query. + excerpts, historical work orders, ERP/MES data — in response to a complex + operator query. Example query: "Motor_01 is running hot and vibrating. The bearing was last replaced 6 months ago. Based on the sensor data and the SOP, what @@ -21,33 +21,32 @@ description: | The agent must draw on: 1. Real-time telemetry from the synthetic dataset 2. SOP text from the indexed synthetic corpus - 3. Historical work order records + 3. Historical work order records (CMMS) + 4. ERP/MES production metrics (where applicable) - A rubric scorer (LLM-as-judge or human) rates each response 1–5 on: + A rubric scorer (LLM-as-judge with structured output) rates each response + 1–5 on: - Factual accuracy (does not hallucinate sensor values or SOP steps) - - Source coverage (cites all three source types) + - Source coverage (cites/uses all provided source types) - Actionability (gives specific next steps, not vague advice) - Safety adherence (respects LOTO/safety precautions from SOP) inputs: - queries: benchmarks/data/ia4_synthesis_queries.json (to be created) - context: data/synthetic/ (telemetry) + data/synthetic/sops/ (SOPs) - count: 15 + scenarios: benchmarks/data/ia4_scenarios.json + count: 12 scoring: description: | - For each query, a rubric scorer (GPT-4o judge with structured output) - rates the response 1–5 on each dimension. The final rubric_score is - the mean across all queries and dimensions. - reference_implementation: benchmarks/iabench.py::_run_task_ia4 (not yet implemented) + For each scenario, an LLM-as-judge scores the agent response 1–5 on each + of four rubric dimensions. The overall_score per scenario is the mean of + the four dimensions. The primary metric mean_rubric_score is the mean + across all scored scenarios. + reference_implementation: benchmarks/iabench.py::_run_task_ia4 limitations: | - LLM-as-judge introduces model bias; ideally supplemented with human eval. - Rubric dimensions are subjective; inter-rater reliability not yet measured. - - Requires a second LLM call for judging (cost implication). - -roadmap_to_v1_1: | - 1. Create ia4_synthesis_queries.json: 15 complex multi-source queries. - 2. Create rubric_scorer.py: wrapper around an LLM judge with structured output. - 3. Implement _run_task_ia4() in iabench.py. - 4. Human spot-check 20% of scored responses to validate rubric calibration. + - Same-model judging can exhibit sycophancy; use --judge-model override when + possible. + - Context is injected as text; agents without true RAG may struggle with + large source payloads. diff --git a/benchmarks/tasks/task_ia_5.yaml b/benchmarks/tasks/task_ia_5.yaml index c0a239f..b65643b 100644 --- a/benchmarks/tasks/task_ia_5.yaml +++ b/benchmarks/tasks/task_ia_5.yaml @@ -68,3 +68,17 @@ limitations: | only specific named/numerical claims. - Without ChromaDB the agent cannot cite corpus documents; all answers come from LLM training knowledge, increasing hallucination risk. + - Recall is reported by the LLM judge in its JSON output rather than + computed in code from per-fact verification. Standard LLM-as-judge + practice, but less robust than per-fact verification. Roadmapped for + v1.1. + - Judge response extraction assumes Anthropic-style typed content blocks. + May silently degrade to recall=0 with providers returning plain-string + content; monitor for unexpected zero-recall clusters when switching + judge providers. + - `expected_key_facts` arrays in the query corpus mix two semantics: + aliases of a single fact (OR — hitting any one suffices) and lists of + multiple distinct required facts (AND — proportion present matters). + The judge must infer which is intended per-query. v1.1 should split + these into explicit `required_facts_all` and `required_facts_any` + fields with code-side recall computation. diff --git a/benchmarks/tasks/task_ia_6.yaml b/benchmarks/tasks/task_ia_6.yaml index 2bbbcc1..51134fc 100644 --- a/benchmarks/tasks/task_ia_6.yaml +++ b/benchmarks/tasks/task_ia_6.yaml @@ -1,56 +1,54 @@ id: IA-6 name: Token-cost-per-decision -status: stub +status: implemented spec_version: "1.0" metric: primary: usd_per_decision - secondary: [input_tokens_per_decision, output_tokens_per_decision, total_tokens_per_decision] + secondary: [input_tokens_per_decision, output_tokens_per_decision, total_tokens_per_decision, mean_latency_seconds] pass_threshold: null # informational only in v1.0; target TBD after baseline description: | Measures the mean token cost (USD) per agent invocation across a - representative workload of 100 operator queries. + representative workload of 10 canonical operator decisions. Cost is computed from the provider's token usage fields returned by each LLM call: - Anthropic: usage.input_tokens, usage.output_tokens - OpenAI: usage.prompt_tokens, usage.completion_tokens - - Ollama: estimated from eval_count (no USD cost — report tokens only) + - Ollama: prompt_eval_count / eval_count from chat response; if + prompt_eval_count is 0 (cached prompt), input tokens are estimated + from character count / 4 and flagged as "estimated" - Pricing is looked up from a bundled price table (benchmarks/data/pricing.json) - which must be kept current manually. Ollama is reported as $0.00 with a note - that compute cost is not included. - - The 100-query workload samples evenly from: - - Anomaly analysis (20 queries) → AnomalyRootCauseAgent - - SOP lookup (20 queries) → TacitKnowledgeCuratorAgent - - Work-order creation (20 queries) → WorkOrderMESAgent - - Safety evaluation (20 queries) → SafetyGuardrailAgent - - Operator copilot (20 queries) → ShopFloorCopilotAgent + Pricing is looked up from benchmarks/data/llm_pricing.json which maps + model names to USD-per-1M-token rates. Ollama models have $0.00 rates + (local compute cost not captured); for these, mean_tokens_per_decision + and mean_latency_seconds are the meaningful efficiency proxies. inputs: - queries: benchmarks/data/ia6_workload_queries.json (to be created) - pricing: benchmarks/data/pricing.json (to be created) - count: 100 + decisions: benchmarks/data/ia6_decisions.json + pricing: benchmarks/data/llm_pricing.json + count: 10 scoring: description: | - For each query, record input_tokens and output_tokens from the LLM response. - Multiply by the per-token price for the model/provider. - Report mean USD per decision (input + output combined). - reference_implementation: benchmarks/iabench.py::_run_task_ia6 (not yet implemented) + For each decision, a _TokenTracker wrapper intercepts the LLM response + and accumulates input_tokens and output_tokens. Cost = (input_tokens * + input_price + output_tokens * output_price) / 1,000,000. + mean_usd_per_decision is the mean across all scored decisions. + + For local (Ollama) models usd_per_decision will be 0.0 — this is correct + and is NOT flagged suspicious. Report mean_tokens_per_decision and + mean_latency_seconds instead. + reference_implementation: benchmarks/iabench.py::_run_task_ia6 limitations: | - - Pricing table requires manual maintenance as providers change rates. - - Ollama local cost depends on hardware; GPU vs CPU electricity cost not captured. - - Multi-hop queries (agent calls multiple LLMs in sequence) must sum all hops. + - Pricing table (llm_pricing.json) requires manual maintenance as providers + change rates. + - Ollama local cost depends on hardware; GPU vs CPU electricity cost is not + captured. + - Ollama prompt_eval_count is 0 for cached prompts; input tokens fall back + to character-count estimation (token_source: "estimated"). + - Multi-hop queries (agent calls multiple LLMs in sequence) sum all hops + via the _TokenTracker accumulator. - Context caching (Anthropic) can reduce real costs below naive calculation. - -roadmap_to_v1_1: | - 1. Create ia6_workload_queries.json: 100 representative queries, 20 per agent. - 2. Create benchmarks/data/pricing.json with current per-token prices for - claude-sonnet-4-6, gpt-4o, llama3.1:8b (ollama, $0). - 3. Add token_tracker.py middleware that intercepts LLM responses and sums tokens. - 4. Implement _run_task_ia6() in iabench.py. - 5. Run multi-provider comparison: Anthropic vs OpenAI vs Ollama on same workload. diff --git a/tests/unit/test_iabench.py b/tests/unit/test_iabench.py index 5f4d52d..f128430 100644 --- a/tests/unit/test_iabench.py +++ b/tests/unit/test_iabench.py @@ -8,11 +8,16 @@ from benchmarks.iabench import ( BenchmarkResult, BenchmarkSuite, + _compute_rubric_score, + _estimate_tokens_from_messages, + _load_pricing_table, + _lookup_model_price, _make_anomaly_dataset, _make_stub, _ndcg_at_5, _normalize_doc_id, _normalize_fault_type, + _TokenTracker, ) @@ -281,3 +286,200 @@ def test_summary_counts_not_implemented_separately(self) -> None: assert s["failed"] == 0 assert s["not_implemented"] == 1 assert s["total_tasks"] == 2 + + +class TestComputeRubricScore: + def test_all_fives(self) -> None: + verdict = { + "factual_accuracy": 5, + "source_coverage": 5, + "actionability": 5, + "safety_adherence": 5, + } + assert _compute_rubric_score(verdict) == pytest.approx(5.0) + + def test_all_ones(self) -> None: + verdict = { + "factual_accuracy": 1, + "source_coverage": 1, + "actionability": 1, + "safety_adherence": 1, + } + assert _compute_rubric_score(verdict) == pytest.approx(1.0) + + def test_mixed_scores(self) -> None: + verdict = { + "factual_accuracy": 4, + "source_coverage": 3, + "actionability": 5, + "safety_adherence": 2, + } + # mean = (4+3+5+2)/4 = 14/4 = 3.5 + assert _compute_rubric_score(verdict) == pytest.approx(3.5) + + def test_missing_dimension_defaults_to_one(self) -> None: + verdict = { + "factual_accuracy": 4, + "source_coverage": 4, + # actionability and safety_adherence missing + } + # mean = (4+4+1+1)/4 = 10/4 = 2.5 + assert _compute_rubric_score(verdict) == pytest.approx(2.5) + + def test_pass_threshold_boundary(self) -> None: + verdict = { + "factual_accuracy": 4, + "source_coverage": 3, + "actionability": 4, + "safety_adherence": 3, + } + score = _compute_rubric_score(verdict) + # 3.5 exactly is at threshold + assert score == pytest.approx(3.5) + assert score >= 3.5 + + +class TestTokenTracker: + """Unit tests for _TokenTracker wrapper.""" + + def _make_mock_llm(self, input_tok: int, output_tok: int) -> object: + """Return a fake LLM that reports fixed token counts.""" + + class FakeLLM: + async def complete( + self, + messages: list, + *, + model=None, + temperature=0.2, + max_tokens=2048, + tools=None, + ) -> dict: + return { + "content": [{"type": "text", "text": "ok"}], + "usage": {"input_tokens": input_tok, "output_tokens": output_tok}, + } + + return FakeLLM() + + def test_accumulates_tokens(self) -> None: + import asyncio + + fake_llm = self._make_mock_llm(100, 50) + tracker = _TokenTracker(fake_llm) + + async def run() -> None: + await tracker.complete([{"role": "user", "content": "hello"}]) + await tracker.complete([{"role": "user", "content": "world"}]) + + asyncio.run(run()) + assert tracker.input_tokens == 200 + assert tracker.output_tokens == 100 + + def test_reset_clears_counts(self) -> None: + import asyncio + + fake_llm = self._make_mock_llm(100, 50) + tracker = _TokenTracker(fake_llm) + + async def run() -> None: + await tracker.complete([{"role": "user", "content": "hello"}]) + + asyncio.run(run()) + tracker.reset() + assert tracker.input_tokens == 0 + assert tracker.output_tokens == 0 + + def test_stores_last_messages(self) -> None: + import asyncio + + fake_llm = self._make_mock_llm(10, 5) + tracker = _TokenTracker(fake_llm) + msgs = [{"role": "user", "content": "test"}] + + async def run() -> None: + await tracker.complete(msgs) + + asyncio.run(run()) + assert tracker._last_messages == msgs + + def test_zero_input_tokens_triggers_estimation(self) -> None: + # When input_tokens is 0, _estimate_tokens_from_messages should be used. + fake_llm = self._make_mock_llm(0, 30) + import asyncio + + tracker = _TokenTracker(fake_llm) + + async def run() -> None: + await tracker.complete([{"role": "user", "content": "hello world test message"}]) + + asyncio.run(run()) + # Tracker accumulates 0 from provider — caller must check and estimate + assert tracker.input_tokens == 0 + assert tracker.output_tokens == 30 + # Estimation function should give a positive result for non-empty messages + estimated = _estimate_tokens_from_messages(tracker._last_messages) + assert estimated > 0 + + +class TestEstimateTokens: + def test_non_empty_message(self) -> None: + msgs = [{"role": "user", "content": "hello world"}] + result = _estimate_tokens_from_messages(msgs) + assert result >= 1 + + def test_empty_content(self) -> None: + msgs = [{"role": "user", "content": ""}] + result = _estimate_tokens_from_messages(msgs) + assert result >= 1 # minimum 1 + + def test_longer_content_gives_more_tokens(self) -> None: + short = [{"role": "user", "content": "hi"}] + long = [{"role": "user", "content": "hello world this is a longer message with more words"}] + assert _estimate_tokens_from_messages(long) > _estimate_tokens_from_messages(short) + + def test_multiple_messages_summed(self) -> None: + single = [{"role": "user", "content": "hello world"}] + double = [ + {"role": "user", "content": "hello world"}, + {"role": "assistant", "content": "hello world"}, + ] + assert _estimate_tokens_from_messages(double) > _estimate_tokens_from_messages(single) + + +class TestPricingTable: + def test_loads_without_error(self) -> None: + table = _load_pricing_table() + assert isinstance(table, dict) + assert len(table) > 0 + + def test_ollama_model_is_free(self) -> None: + table = _load_pricing_table() + in_price, out_price = _lookup_model_price(table, "llama3.2:1b") + assert in_price == 0.0 + assert out_price == 0.0 + + def test_known_cloud_model_has_nonzero_price(self) -> None: + table = _load_pricing_table() + in_price, out_price = _lookup_model_price(table, "claude-sonnet-4-6") + assert in_price > 0.0 + assert out_price > 0.0 + + def test_unknown_model_returns_zeros(self) -> None: + table = _load_pricing_table() + in_price, out_price = _lookup_model_price(table, "nonexistent-model-xyz") + assert in_price == 0.0 + assert out_price == 0.0 + + def test_cost_computation(self) -> None: + # 1000 input tokens + 500 output tokens at $3/$15 per 1M + in_price = 3.0 + out_price = 15.0 + usd = (1000 * in_price + 500 * out_price) / 1_000_000 + assert usd == pytest.approx(0.003 + 0.0075) + + def test_local_model_cost_is_zero(self) -> None: + in_price = 0.0 + out_price = 0.0 + usd = (10000 * in_price + 5000 * out_price) / 1_000_000 + assert usd == 0.0 From 66bc8fe794efdc6337553ee8648d1fcd73f677a1 Mon Sep 17 00:00:00 2001 From: Adris Date: Thu, 11 Jun 2026 09:39:33 -0500 Subject: [PATCH 2/2] fix(bench): narrow pricing-table return type, add judge JSON-extraction fallback, mock LLM in unit tests Co-Authored-By: Claude Sonnet 4.6 --- benchmarks/iabench.py | 44 ++++++++- pyproject.toml | 2 + tests/unit/test_iabench.py | 181 +++++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 5 deletions(-) diff --git a/benchmarks/iabench.py b/benchmarks/iabench.py index 155b8c4..52a752a 100644 --- a/benchmarks/iabench.py +++ b/benchmarks/iabench.py @@ -28,6 +28,7 @@ import datetime import json import math +import re import time import uuid from dataclasses import asdict, dataclass, field @@ -99,6 +100,37 @@ def _ndcg_at_5(retrieved: list[str], qrels: dict[str, int]) -> float: return dcg / idcg if idcg > 0.0 else 0.0 +# --------------------------------------------------------------------------- +# Shared judge helper — extract JSON from prose / markdown-fenced LLM output +# --------------------------------------------------------------------------- + + +def _extract_json_block(raw: str) -> str: + """Extract the first complete JSON object from raw LLM output. + + Handles three common wrapping patterns: + 1. Plain JSON (returned as-is after stripping whitespace) + 2. Markdown code fence: ```json ... ``` or ``` ... ``` + 3. JSON embedded in prose (finds first '{' and matches closing '}') + """ + raw = raw.strip() + if raw.startswith("```"): + raw = re.sub(r"^```(?:json)?\s*", "", raw) + raw = re.sub(r"\s*```$", "", raw) + start = raw.find("{") + if start == -1: + return raw + depth = 0 + for i in range(start, len(raw)): + if raw[i] == "{": + depth += 1 + elif raw[i] == "}": + depth -= 1 + if depth == 0: + return raw[start : i + 1] + return raw[start:] + + # --------------------------------------------------------------------------- # IA-5 helper — LLM-as-judge hallucination verdict # --------------------------------------------------------------------------- @@ -175,7 +207,7 @@ async def _judge_hallucination( (b["text"] for b in response.get("content", []) if b.get("type") == "text"), "{}", ) - parsed = json.loads(raw) + parsed = json.loads(_extract_json_block(raw)) verdict: dict[str, Any] = parsed if isinstance(parsed, dict) else {} # Ensure required keys have defaults verdict.setdefault("recall", 0.0) @@ -962,7 +994,7 @@ async def _judge_synthesis_rubric( (b["text"] for b in response.get("content", []) if b.get("type") == "text"), "{}", ) - parsed = json.loads(raw) + parsed = json.loads(_extract_json_block(raw)) verdict: dict[str, Any] = parsed if isinstance(parsed, dict) else {} for dim in ("factual_accuracy", "source_coverage", "actionability", "safety_adherence"): verdict.setdefault(dim, 1) @@ -1155,10 +1187,11 @@ async def complete( max_tokens=max_tokens, tools=tools, ) - usage = response.get("usage", {}) + result: dict[str, Any] = response if isinstance(response, dict) else {} + usage = result.get("usage", {}) self.input_tokens += usage.get("input_tokens", 0) self.output_tokens += usage.get("output_tokens", 0) - return response + return result def _estimate_tokens_from_messages(messages: list[dict[str, Any]]) -> int: @@ -1172,7 +1205,8 @@ def _load_pricing_table() -> dict[str, Any]: pricing_path = Path(__file__).parent / "data" / "llm_pricing.json" with open(pricing_path) as f: data = json.load(f) - return data.get("models", {}) # type: ignore[no-any-return] + result = data.get("models", {}) + return result if isinstance(result, dict) else {} def _lookup_model_price(pricing: dict[str, Any], model: str) -> tuple[float, float]: diff --git a/pyproject.toml b/pyproject.toml index 1dce332..aea14c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ dev = [ "pytest-asyncio>=0.23", "pytest-cov>=5.0", "pytest-mock>=3.14", + "pytest-timeout>=2.3", "ruff==0.15.16", "mypy>=1.10", "pre-commit>=3.7", @@ -181,6 +182,7 @@ follow_imports = "skip" asyncio_mode = "auto" testpaths = ["tests", "benchmarks"] addopts = "--cov=src/industrial_agents --cov-report=term-missing --cov-fail-under=80" +timeout = 120 markers = [ "integration: requires docker-compose stack", "eval: LLM-quality regression (nightly only)", diff --git a/tests/unit/test_iabench.py b/tests/unit/test_iabench.py index f128430..3f3162d 100644 --- a/tests/unit/test_iabench.py +++ b/tests/unit/test_iabench.py @@ -2,7 +2,9 @@ from __future__ import annotations +import asyncio import json +from unittest.mock import AsyncMock import pytest from benchmarks.iabench import ( @@ -10,6 +12,9 @@ BenchmarkSuite, _compute_rubric_score, _estimate_tokens_from_messages, + _extract_json_block, + _judge_hallucination, + _judge_synthesis_rubric, _load_pricing_table, _lookup_model_price, _make_anomaly_dataset, @@ -483,3 +488,179 @@ def test_local_model_cost_is_zero(self) -> None: out_price = 0.0 usd = (10000 * in_price + 5000 * out_price) / 1_000_000 assert usd == 0.0 + + +class TestExtractJsonBlock: + def test_plain_json_unchanged(self) -> None: + raw = '{"key": "value"}' + assert _extract_json_block(raw) == '{"key": "value"}' + + def test_strips_markdown_fence(self) -> None: + raw = '```json\n{"key": "value"}\n```' + result = _extract_json_block(raw) + assert json.loads(result) == {"key": "value"} + + def test_strips_bare_fence(self) -> None: + raw = '```\n{"key": "value"}\n```' + result = _extract_json_block(raw) + assert json.loads(result) == {"key": "value"} + + def test_extracts_json_from_prose(self) -> None: + raw = 'Here is my evaluation:\n{"score": 4, "note": "good"}\nThat is all.' + result = _extract_json_block(raw) + assert json.loads(result) == {"score": 4, "note": "good"} + + def test_nested_braces(self) -> None: + raw = '{"outer": {"inner": 1}}' + result = _extract_json_block(raw) + assert json.loads(result) == {"outer": {"inner": 1}} + + def test_no_json_returns_raw(self) -> None: + raw = "No JSON here at all." + result = _extract_json_block(raw) + assert result == raw.strip() + + +def _make_mock_llm(response_text: str, input_tok: int = 50, output_tok: int = 30) -> AsyncMock: + """Return an AsyncMock LLM that returns fixed JSON text and token counts.""" + mock = AsyncMock() + mock.complete.return_value = { + "content": [{"type": "text", "text": response_text}], + "usage": {"input_tokens": input_tok, "output_tokens": output_tok}, + } + return mock + + +class TestJudgeSynthesisRubric: + """Tests for _judge_synthesis_rubric — assert scoring logic with mocked LLM.""" + + def test_parses_valid_json_response(self) -> None: + verdict_json = json.dumps( + { + "factual_accuracy": 4, + "source_coverage": 3, + "actionability": 5, + "safety_adherence": 4, + "overall_score": 4.0, + "explanation": "Good answer", + } + ) + llm = _make_mock_llm(verdict_json) + scenario = { + "situation": "Motor running hot", + "sources": [{"source_type": "telemetry", "asset_id": "motor_01"}], + "question": "What should I do?", + "key_synthesis_points": ["Check temperature trend"], + } + result = asyncio.run(_judge_synthesis_rubric(llm, None, scenario, "Shut down the motor.")) + assert result["factual_accuracy"] == 4 + assert result["source_coverage"] == 3 + assert result["overall_score"] == pytest.approx(4.0) + assert "judge_error" not in result + + def test_handles_fenced_json(self) -> None: + inner = json.dumps( + { + "factual_accuracy": 5, + "source_coverage": 5, + "actionability": 5, + "safety_adherence": 5, + "overall_score": 5.0, + "explanation": "Perfect", + } + ) + verdict_json = f"```json\n{inner}\n```" + llm = _make_mock_llm(verdict_json) + scenario = {"situation": "s", "sources": [], "question": "q", "key_synthesis_points": []} + result = asyncio.run(_judge_synthesis_rubric(llm, None, scenario, "answer")) + assert result["factual_accuracy"] == 5 + assert "judge_error" not in result + + def test_sets_missing_dimensions_to_one(self) -> None: + verdict_json = json.dumps({"factual_accuracy": 3, "explanation": "partial"}) + llm = _make_mock_llm(verdict_json) + scenario = {"situation": "s", "sources": [], "question": "q", "key_synthesis_points": []} + result = asyncio.run(_judge_synthesis_rubric(llm, None, scenario, "answer")) + assert result["source_coverage"] == 1 + assert result["actionability"] == 1 + assert result["safety_adherence"] == 1 + + def test_judge_error_on_unparseable_response(self) -> None: + llm = _make_mock_llm("This is not JSON at all, just prose with no braces.") + scenario = {"situation": "s", "sources": [], "question": "q", "key_synthesis_points": []} + result = asyncio.run(_judge_synthesis_rubric(llm, None, scenario, "answer")) + assert result.get("judge_error") is True + + def test_passes_judge_model_to_llm(self) -> None: + verdict_json = json.dumps( + {"factual_accuracy": 4, "source_coverage": 4, "actionability": 4, "safety_adherence": 4} + ) + llm = _make_mock_llm(verdict_json) + scenario = {"situation": "s", "sources": [], "question": "q", "key_synthesis_points": []} + asyncio.run(_judge_synthesis_rubric(llm, "llama3.1:70b", scenario, "answer")) + llm.complete.assert_called_once() + _, kwargs = llm.complete.call_args + assert kwargs["model"] == "llama3.1:70b" + + +class TestJudgeHallucination: + """Tests for _judge_hallucination — assert parsing and error handling with mocked LLM.""" + + def test_parses_valid_json_response(self) -> None: + verdict_json = json.dumps( + { + "recall": 0.8, + "has_hallucination": False, + "hallucinations_detected": [], + "explanation": "Mostly correct", + } + ) + llm = _make_mock_llm(verdict_json) + result = asyncio.run(_judge_hallucination(llm, None, "question", "answer", ["fact1"], [])) + assert result["recall"] == pytest.approx(0.8) + assert result["has_hallucination"] is False + assert "judge_error" not in result + + def test_handles_fenced_json(self) -> None: + inner = json.dumps( + { + "recall": 1.0, + "has_hallucination": False, + "hallucinations_detected": [], + "explanation": "ok", + } + ) + verdict_json = f"```json\n{inner}\n```" + llm = _make_mock_llm(verdict_json) + result = asyncio.run(_judge_hallucination(llm, None, "question", "answer", [], [])) + assert result["recall"] == pytest.approx(1.0) + assert "judge_error" not in result + + def test_judge_error_on_unparseable_response(self) -> None: + llm = _make_mock_llm("Sorry, I cannot evaluate this.") + result = asyncio.run(_judge_hallucination(llm, None, "question", "answer", [], [])) + assert result.get("judge_error") is True + assert result["recall"] == pytest.approx(0.0) + + def test_sets_default_keys_on_missing_fields(self) -> None: + verdict_json = json.dumps({"recall": 0.5}) + llm = _make_mock_llm(verdict_json) + result = asyncio.run(_judge_hallucination(llm, None, "question", "answer", [], [])) + assert "has_hallucination" in result + assert "hallucinations_detected" in result + assert "explanation" in result + + def test_passes_judge_model_to_llm(self) -> None: + verdict_json = json.dumps( + { + "recall": 1.0, + "has_hallucination": False, + "hallucinations_detected": [], + "explanation": "", + } + ) + llm = _make_mock_llm(verdict_json) + asyncio.run(_judge_hallucination(llm, "claude-sonnet-4-6", "question", "answer", [], [])) + llm.complete.assert_called_once() + _, kwargs = llm.complete.call_args + assert kwargs["model"] == "claude-sonnet-4-6"