From dab896451dd47378084d63b96736fea664515b93 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:35:42 +0800 Subject: [PATCH 1/7] eval: carry real OMP RPC implementation into current main (backend/scripts/live_eval/agent_executor.py) --- backend/scripts/live_eval/agent_executor.py | 708 ++++++++++++++------ 1 file changed, 491 insertions(+), 217 deletions(-) diff --git a/backend/scripts/live_eval/agent_executor.py b/backend/scripts/live_eval/agent_executor.py index 5340699f..18d6e878 100644 --- a/backend/scripts/live_eval/agent_executor.py +++ b/backend/scripts/live_eval/agent_executor.py @@ -1,263 +1,537 @@ -""" -Real Agent Executor for OfferU E2E eval. +"""Real OMP Agent executor for OfferU live eval. + +This module drives OMP through its documented RPC protocol. It does not choose +OfferU Skills or Operations on behalf of the model. -Launches a real Agent session (OMP/Claude Code/Codex) and lets the model -decide which OfferU CLI operations to call. This is the true Agent-native eval. +The harness owns only: +- process/session lifecycle; +- isolated environment wiring; +- model/thinking selection; +- event capture; +- fail-closed local bash policy; +- timeout/abort. -Unlike scripted_cli_executor.py which hardcodes the operation sequence, -this executor: -1. Gives the Agent a natural language task -2. Lets it discover Skills via `manifest` -3. Lets it select and call Operations via `run` -4. Captures all model-issued tool calls for verification +The model owns: +- Skill discovery; +- Operation selection; +- CLI tool calls; +- interpreting intermediate results. -Usage: - python -m scripts.live_eval.agent_executor --case PR01 --model swe-2 +OfferU still owns business authorization: a side-effect Operation creates a +Proposal and the Agent must leave that Proposal for the human to review. """ from __future__ import annotations import argparse +import asyncio +import contextlib import json import os -import subprocess +import shutil import sys import time +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any -BACKEND_DIR = Path(__file__).resolve().parents[2] -EVAL_DIR = Path(r"H:\tmp\offeru\private-eval") -RUN_DIR = Path(r"H:\tmp\offeru\live-eval-runs\agent") +PROJECT_ROOT = Path(__file__).resolve().parents[3] +BACKEND_DIR = PROJECT_ROOT / "backend" +DEFAULT_EVAL_DIR = Path(os.environ.get("OFFERU_PRIVATE_EVAL_ROOT") or r"H:\tmp\offeru\private-eval") +DEFAULT_RUN_ROOT = Path( + os.environ.get("OFFERU_LIVE_EVAL_AGENT_ROOT") + or r"H:\tmp\offeru\live-eval-runs\agent" +) +DEFAULT_OMP_MODEL = os.environ.get("OFFERU_LIVE_EVAL_OMP_MODEL") or "avabbbb/devin/swe-2" +DEFAULT_OMP_THINKING = os.environ.get("OFFERU_LIVE_EVAL_OMP_THINKING") or "xhigh" + +_TERMINAL_AGENT_END = "agent_end" +_RPC_READY_TYPES = {"ready", "rpc_ready"} + + +@dataclass(slots=True) +class OmpRpcExecution: + ok: bool + model_requested: str + thinking_requested: str + model_observed: str = "" + thinking_observed: str = "" + session_id: str = "" + elapsed_s: float = 0.0 + final_text: str = "" + events: list[dict[str, Any]] = field(default_factory=list) + tool_calls: list[dict[str, Any]] = field(default_factory=list) + state_before: dict[str, Any] = field(default_factory=dict) + state_after: dict[str, Any] = field(default_factory=dict) + stderr: str = "" + error: str = "" + aborted: bool = False + + def to_dict(self) -> dict[str, Any]: + return asdict(self) -def _build_agent_prompt(case: dict[str, Any]) -> str: - """Build the prompt for the Agent.""" - user_turns = case.get("user_turns", []) - task = user_turns[0] if user_turns else "No task specified" - - return f"""You are an AI assistant helping with a job application task. +def _venv_scripts_dir() -> Path | None: + candidates = ( + BACKEND_DIR / ".venv312" / ("Scripts" if os.name == "nt" else "bin"), + BACKEND_DIR / ".venv" / ("Scripts" if os.name == "nt" else "bin"), + ) + return next((path for path in candidates if path.is_dir()), None) + + +def _agent_environment(eval_db: Path, run_dir: Path) -> dict[str, str]: + """Build an OMP environment without WorkBuddy desktop-session contamination.""" -## Task -{task} + env = os.environ.copy() + for key in tuple(env): + upper = key.upper() + if upper.startswith("CODEBUDDY_GATEWAY_") or upper.startswith("CODEBUDDY_CONVERSATION_"): + env.pop(key, None) + if upper in {"CODEBUDDY_HOST", "CODEBUDDY_PROJECT_DIR"}: + env.pop(key, None) + + env["DATABASE_URL"] = f"sqlite+aiosqlite:///{eval_db.resolve().as_posix()}" + env["OFFERU_DATA_DIR"] = str((run_dir / "offeru-data").resolve()) + env["PYTHONPATH"] = str(BACKEND_DIR.resolve()) + env["PYTHONIOENCODING"] = "utf-8" + env["PYTHONUNBUFFERED"] = "1" + env["NO_COLOR"] = "1" -## Environment -- Working directory: {BACKEND_DIR} -- Database: {EVAL_DIR}/eval.db -- Frontend: http://127.0.0.1:7410 -- Backend: http://127.0.0.1:8766 + venv_bin = _venv_scripts_dir() + if venv_bin is not None: + env["PATH"] = str(venv_bin) + os.pathsep + env.get("PATH", "") + return env -## Available Tools -You have access to a `bash` tool. Use it to run OfferU CLI commands: -```bash -python -m app.cli doctor --pretty # Check system health -python -m app.cli manifest --pretty # List available skills -python -m app.cli manifest --skill --pretty # Show skill details -python -m app.cli ops --pretty # List all operations -python -m app.cli schema --pretty # Show operation schema -python -m app.cli run --args '{{"key":"val"}}' # Run operation -python -m app.cli confirm # Confirm proposal -``` +def _write_omp_eval_config(run_dir: Path) -> Path: + """Create a least-privilege OMP overlay for Agent-native eval. + + OMP itself may auto-approve normal tool execution, but this eval only permits + bash calls that target OfferU's CLI. Human business approval remains inside + OfferU Proposal/HITL and the CLI confirm command is explicitly denied. + """ + + path = run_dir / "omp-eval.yml" + path.write_text( + """tools: + approvalMode: always-ask + approval: + read: allow + grep: allow + glob: allow +bash: + patterns: + - match: "*app.cli confirm*" + approval: deny + - match: "python* -m app.cli *" + approval: allow +""", + encoding="utf-8", + ) + return path + + +def _offeru_agent_prompt(user_prompt: str) -> str: + """Wrap only integration/safety context; do not leak an expected Operation path.""" + + return f"""Use the OfferU integration provided by this repository to complete the user's career task. + +User request: +{user_prompt} + +Acceptance constraints: +- This is an Agent-native acceptance run. You decide which Skill and OfferU Operations are appropriate. +- Read and follow the generated OfferU Skill available in the repository. +- The process environment already points OfferU CLI at the isolated eval database. +- Run OfferU CLI from the repository root as: python -m app.cli ... +- Do not edit project files or use raw database writes/HTTP as a shortcut. +- Do not call app.cli confirm. Side-effect Operations must remain pending for the human to review in OfferU. +- Do not claim success unless the CLI result actually supports it. +- If information is missing, say so rather than inventing it. + +Do not follow a pre-scripted tool sequence. Solve the user's request using the live Skill/capability contract. +""" + -## Rules -1. Discover skills first via `manifest` -2. Read operation schemas before calling -3. For mutations, check if confirmation required -4. Report what you did and the result +def _extract_response_data(frame: dict[str, Any]) -> dict[str, Any]: + for key in ("data", "result", "state"): + value = frame.get(key) + if isinstance(value, dict): + return value + return {} -Start by checking system health and listing available skills.""" +def _extract_identity(state: dict[str, Any]) -> tuple[str, str, str]: + """Best-effort identity extraction while retaining full raw state in artifacts.""" -def _check_agent_available() -> dict[str, Any]: - """Check if any Agent runtime is available.""" - # Check for Claude Code + session_id = str( + state.get("sessionId") + or state.get("session_id") + or state.get("session") + or "" + ) + thinking = str( + state.get("thinkingLevel") + or state.get("thinking_level") + or state.get("thinking") + or "" + ) + model_value = state.get("model") or state.get("currentModel") or state.get("current_model") + if isinstance(model_value, dict): + provider = str(model_value.get("provider") or model_value.get("providerId") or "") + model_id = str(model_value.get("id") or model_value.get("modelId") or "") + observed = f"{provider}/{model_id}".strip("/") + else: + observed = str(model_value or "") + return observed, thinking, session_id + + +def _tool_input(frame: dict[str, Any]) -> dict[str, Any]: + value = frame.get("args") + if not isinstance(value, dict): + value = frame.get("input") + return value if isinstance(value, dict) else {} + + +def _record_tool_start(frame: dict[str, Any]) -> dict[str, Any]: + tool_name = str(frame.get("toolName") or frame.get("tool_name") or "") + payload = _tool_input(frame) + command = str(payload.get("command") or "") + rendered = command if tool_name == "bash" else json.dumps(payload, ensure_ascii=False) + return { + "tool": tool_name, + "input": rendered, + "tool_call_id": str(frame.get("toolCallId") or frame.get("tool_call_id") or ""), + "source": "model", + } + + +async def _send_frame(proc: asyncio.subprocess.Process, payload: dict[str, Any]) -> None: + if proc.stdin is None: + raise RuntimeError("OMP RPC stdin is unavailable") + proc.stdin.write((json.dumps(payload, ensure_ascii=False) + "\n").encode("utf-8")) + await proc.stdin.drain() + + +async def _read_stderr(stream: asyncio.StreamReader | None, sink: list[str]) -> None: + if stream is None: + return + async for raw in stream: + sink.append(raw.decode("utf-8", errors="replace")) + + +async def _read_frame( + proc: asyncio.subprocess.Process, + *, + deadline: float, +) -> dict[str, Any]: + if proc.stdout is None: + raise RuntimeError("OMP RPC stdout is unavailable") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("OMP RPC deadline exceeded") + raw = await asyncio.wait_for(proc.stdout.readline(), timeout=remaining) + if not raw: + raise RuntimeError(f"OMP RPC exited before completion (code={proc.returncode})") try: - proc = subprocess.run( - ["claude", "--version"], - capture_output=True, - text=True, - timeout=10, + value = json.loads(raw.decode("utf-8", errors="replace")) + except json.JSONDecodeError as exc: + raise RuntimeError(f"OMP RPC emitted non-JSON stdout: {raw[:200]!r}") from exc + if not isinstance(value, dict): + raise RuntimeError("OMP RPC frame must be a JSON object") + return value + + +async def _request( + proc: asyncio.subprocess.Process, + events: list[dict[str, Any]], + payload: dict[str, Any], + *, + deadline: float, +) -> dict[str, Any]: + request_id = str(payload["id"]) + await _send_frame(proc, payload) + while True: + frame = await _read_frame(proc, deadline=deadline) + events.append(frame) + if frame.get("type") == "response" and str(frame.get("id") or "") == request_id: + if frame.get("success") is False: + raise RuntimeError(str(frame.get("error") or f"RPC command {request_id} failed")) + return frame + + +async def run_omp_rpc_agent( + user_prompt: str, + *, + eval_db: Path, + run_dir: Path, + model: str = DEFAULT_OMP_MODEL, + thinking: str = DEFAULT_OMP_THINKING, + timeout: int = 600, + omp_bin: str | None = None, +) -> OmpRpcExecution: + """Run one real OMP Agent turn and capture model-issued tool events.""" + + run_dir.mkdir(parents=True, exist_ok=True) + resolved_omp = omp_bin or shutil.which("omp") or "" + if not resolved_omp: + return OmpRpcExecution( + ok=False, + model_requested=model, + thinking_requested=thinking, + error="OMP executable not found on PATH", ) - if proc.returncode == 0: - return {"available": True, "provider": "claude", "version": proc.stdout.strip()} - except Exception: - pass - - # Check for Codex - try: - proc = subprocess.run( - ["codex", "--version"], - capture_output=True, - text=True, - timeout=10, + if not eval_db.is_file(): + return OmpRpcExecution( + ok=False, + model_requested=model, + thinking_requested=thinking, + error=f"Eval database does not exist: {eval_db}", ) - if proc.returncode == 0: - return {"available": True, "provider": "codex", "version": proc.stdout.strip()} - except Exception: - pass - - # Check for OMP + + config_path = _write_omp_eval_config(run_dir) + env = _agent_environment(eval_db, run_dir) + command = [ + resolved_omp, + "--mode", + "rpc", + "--no-session", + "--model", + model, + "--thinking", + thinking, + "--config", + str(config_path), + "--tools", + "read,bash,grep,glob", + ] + + started = time.perf_counter() + proc = await asyncio.create_subprocess_exec( + *command, + cwd=str(PROJECT_ROOT), + env=env, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stderr_chunks: list[str] = [] + stderr_task = asyncio.create_task(_read_stderr(proc.stderr, stderr_chunks)) + events: list[dict[str, Any]] = [] + tool_calls: list[dict[str, Any]] = [] + final_parts: list[str] = [] + state_before: dict[str, Any] = {} + state_after: dict[str, Any] = {} + aborted = False + deadline = time.monotonic() + timeout + try: - proc = subprocess.run( - ["omp", "--version"], - capture_output=True, - text=True, - timeout=10, + # OMP emits a ready frame before command processing. Older builds may + # begin with a notice; retain every frame until ready/response-capable. + while True: + frame = await _read_frame(proc, deadline=deadline) + events.append(frame) + if str(frame.get("type") or "") in _RPC_READY_TYPES: + break + if frame.get("type") == "extension_error": + raise RuntimeError(str(frame.get("error") or "OMP extension startup error")) + + before = await _request( + proc, + events, + {"id": "state-before", "type": "get_state"}, + deadline=deadline, ) - if proc.returncode == 0: - return {"available": True, "provider": "omp", "version": proc.stdout.strip()} - except Exception: - pass - - return {"available": False, "error": "No Agent runtime found"} - - -def _run_agent_session(case: dict[str, Any], model: str, timeout: int) -> dict[str, Any]: - """Run a real Agent session.""" - - # Check Agent availability - agent_status = _check_agent_available() - if not agent_status["available"]: - return { - "ok": False, - "error": "No Agent runtime available", - "details": agent_status, - "fallback": "Use scripted_cli_executor.py for deterministic testing", - } - - provider = agent_status["provider"] - - # Build prompt - prompt = _build_agent_prompt(case) - - # Create run directory - run_id = f"agent_{case['case_id']}_{int(time.time())}" - run_dir = RUN_DIR / run_id - run_dir.mkdir(parents=True, exist_ok=True) - - # Write prompt - prompt_file = run_dir / "prompt.txt" - prompt_file.write_text(prompt, encoding="utf-8") - - # Launch Agent session based on provider - if provider == "claude": - result = _run_claude_session(prompt, model, timeout, run_dir) - elif provider == "codex": - result = _run_codex_session(prompt, model, timeout, run_dir) - elif provider == "omp": - result = _run_omp_session(prompt, model, timeout, run_dir) + state_before = _extract_response_data(before) + + await _send_frame( + proc, + { + "id": "prompt-1", + "type": "prompt", + "message": _offeru_agent_prompt(user_prompt), + }, + ) + + prompt_accepted = False + terminal = False + while not terminal: + frame = await _read_frame(proc, deadline=deadline) + events.append(frame) + frame_type = str(frame.get("type") or "") + + if frame_type == "response" and frame.get("id") == "prompt-1": + if frame.get("success") is False: + raise RuntimeError(str(frame.get("error") or "OMP prompt rejected")) + prompt_accepted = True + continue + + if frame_type == "prompt_result" and frame.get("id") == "prompt-1": + if frame.get("agentInvoked") is False: + raise RuntimeError("OMP prompt resolved locally without invoking the Agent") + continue + + if frame_type == "tool_execution_start": + tool_calls.append(_record_tool_start(frame)) + continue + + if frame_type == "tool_execution_end": + tool_calls.append( + { + "tool": "result", + "input": "", + "tool_call_id": str( + frame.get("toolCallId") or frame.get("tool_call_id") or "" + ), + "source": "runtime", + "is_error": bool(frame.get("isError")), + } + ) + continue + + if frame_type == "message_update": + update = frame.get("assistantMessageEvent") + if isinstance(update, dict) and update.get("type") == "text_delta": + final_parts.append(str(update.get("delta") or "")) + continue + + if frame_type == "extension_ui_request": + # Eval is non-interactive at the OMP layer. Business approval is + # handled in OfferU UI, not by satisfying arbitrary OMP prompts. + request_id = str(frame.get("id") or "") + if request_id: + await _send_frame( + proc, + { + "type": "extension_ui_response", + "id": request_id, + "cancelled": True, + }, + ) + continue + + if frame_type == _TERMINAL_AGENT_END and frame.get("isTerminal") is not False: + terminal = True + + if not prompt_accepted: + raise RuntimeError("OMP Agent ended before prompt acknowledgement") + + after = await _request( + proc, + events, + {"id": "state-after", "type": "get_state"}, + deadline=deadline, + ) + state_after = _extract_response_data(after) + + except (TimeoutError, asyncio.TimeoutError): + aborted = True + with contextlib.suppress(Exception): + await _send_frame(proc, {"id": "abort-timeout", "type": "abort"}) + await asyncio.sleep(0.2) + if proc.returncode is None: + proc.kill() + error = f"OMP Agent timeout after {timeout}s" + except Exception as exc: # noqa: BLE001 - harness must turn failures into artifacts + error = f"{type(exc).__name__}: {exc}" + if proc.returncode is None: + proc.kill() else: - result = {"ok": False, "error": f"Unknown provider: {provider}"} - - result["run_id"] = run_id - result["case_id"] = case["case_id"] - result["provider"] = provider - - # Save trace - trace_file = run_dir / "trace.json" - trace_file.write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") - - return result + error = "" + finally: + if proc.stdin is not None and not proc.stdin.is_closing(): + proc.stdin.close() + with contextlib.suppress(Exception): + await asyncio.wait_for(proc.wait(), timeout=5) + if proc.returncode is None: + proc.kill() + with contextlib.suppress(Exception): + await proc.wait() + with contextlib.suppress(Exception): + await asyncio.wait_for(stderr_task, timeout=2) + observed_state = state_after or state_before + model_observed, thinking_observed, session_id = _extract_identity(observed_state) + result = OmpRpcExecution( + ok=not error and bool(events), + model_requested=model, + thinking_requested=thinking, + model_observed=model_observed, + thinking_observed=thinking_observed, + session_id=session_id, + elapsed_s=round(time.perf_counter() - started, 2), + final_text="".join(final_parts).strip(), + events=events, + tool_calls=tool_calls, + state_before=state_before, + state_after=state_after, + stderr="".join(stderr_chunks)[-8000:], + error=error, + aborted=aborted, + ) -def _run_claude_session(prompt: str, model: str, timeout: int, run_dir: Path) -> dict[str, Any]: - """Run a Claude Code session.""" - env = os.environ.copy() - env["DATABASE_URL"] = f"sqlite+aiosqlite:///{EVAL_DIR}/eval.db" - env["PYTHONIOENCODING"] = "utf-8" - env["PYTHONPATH"] = str(BACKEND_DIR) - - proc = subprocess.run( - ["claude", "--print", "--verbose", "--output-format", "stream-json", "--model", model, prompt], - capture_output=True, - text=True, - timeout=timeout, - env=env, - cwd=str(BACKEND_DIR), + (run_dir / "omp-rpc-events.ndjson").write_text( + "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in events), + encoding="utf-8", ) - - return { - "ok": proc.returncode == 0, - "stdout": proc.stdout[:5000], - "stderr": proc.stderr[:2000], - "returncode": proc.returncode, - } + (run_dir / "agent-execution.json").write_text( + json.dumps(result.to_dict(), ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return result -def _run_codex_session(prompt: str, model: str, timeout: int, run_dir: Path) -> dict[str, Any]: - """Run a Codex session.""" - env = os.environ.copy() - env["DATABASE_URL"] = f"sqlite+aiosqlite:///{EVAL_DIR}/eval.db" - env["PYTHONIOENCODING"] = "utf-8" - env["PYTHONPATH"] = str(BACKEND_DIR) - - proc = subprocess.run( - ["codex", "exec", "--model", model, prompt], - capture_output=True, - text=True, - timeout=timeout, - env=env, - cwd=str(BACKEND_DIR), - ) - - return { - "ok": proc.returncode == 0, - "stdout": proc.stdout[:5000], - "stderr": proc.stderr[:2000], - "returncode": proc.returncode, - } +def _load_case_prompt(cases_file: Path, case_id: str) -> str: + payload = json.loads(cases_file.read_text(encoding="utf-8")) + items = payload.get("cases") if isinstance(payload, dict) else [] + for item in items or []: + if isinstance(item, dict) and str(item.get("case_id") or "") == case_id: + turns = item.get("user_turns") if isinstance(item.get("user_turns"), list) else [] + if turns: + return str(turns[0]) + raise ValueError(f"Case not found or has no user turn: {case_id}") -def _run_omp_session(prompt: str, model: str, timeout: int, run_dir: Path) -> dict[str, Any]: - """Run an OMP session.""" - env = os.environ.copy() - env["DATABASE_URL"] = f"sqlite+aiosqlite:///{EVAL_DIR}/eval.db" - env["PYTHONIOENCODING"] = "utf-8" - env["PYTHONPATH"] = str(BACKEND_DIR) - - proc = subprocess.run( - ["omp", "exec", "--model", model, prompt], - capture_output=True, - text=True, - timeout=timeout, - env=env, - cwd=str(BACKEND_DIR), +async def _main_async(args: argparse.Namespace) -> int: + if args.prompt: + prompt = args.prompt + else: + prompt = _load_case_prompt(Path(args.cases_file), args.case) + + run_id = f"omp-{args.case or 'prompt'}-{int(time.time())}" + run_dir = Path(args.output_root) / run_id + result = await run_omp_rpc_agent( + prompt, + eval_db=Path(args.db), + run_dir=run_dir, + model=args.model, + thinking=args.thinking, + timeout=args.timeout, + omp_bin=args.omp_bin or None, ) - - return { - "ok": proc.returncode == 0, - "stdout": proc.stdout[:5000], - "stderr": proc.stderr[:2000], - "returncode": proc.returncode, - } + print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2)) + return 0 if result.ok else 1 def main() -> int: - parser = argparse.ArgumentParser(description="Real Agent executor for live eval") - parser.add_argument("--case", required=True, help="Eval case ID") - parser.add_argument("--model", default="swe-2", help="Model to use") - parser.add_argument("--timeout", type=int, default=300, help="Timeout in seconds") - parser.add_argument("--cases-file", default=str(EVAL_DIR / "private_resume_opt_6.json")) + parser = argparse.ArgumentParser(description="Real OMP RPC Agent executor for OfferU eval") + parser.add_argument("--case", default="", help="Eval case id") + parser.add_argument("--prompt", default="", help="Natural-language task; overrides --case") + parser.add_argument( + "--cases-file", + default=str(DEFAULT_EVAL_DIR / "private_resume_opt_6.json"), + ) + parser.add_argument("--db", default=str(DEFAULT_EVAL_DIR / "eval.db")) + parser.add_argument("--model", default=DEFAULT_OMP_MODEL) + parser.add_argument("--thinking", default=DEFAULT_OMP_THINKING) + parser.add_argument("--timeout", type=int, default=600) + parser.add_argument("--omp-bin", default="") + parser.add_argument("--output-root", default=str(DEFAULT_RUN_ROOT)) args = parser.parse_args() - - # Load case - cases_file = Path(args.cases_file) - if not cases_file.exists(): - print(json.dumps({"ok": False, "error": f"Cases file not found: {cases_file}"})) - return 1 - - cases = json.loads(cases_file.read_text(encoding="utf-8")) - case = next((c for c in cases["cases"] if c["case_id"] == args.case), None) - if not case: - print(json.dumps({"ok": False, "error": f"Case not found: {args.case}"})) - return 1 - - # Run Agent session - result = _run_agent_session(case, args.model, args.timeout) - print(json.dumps(result, indent=2, ensure_ascii=False)) - return 0 if result.get("ok") else 1 + if not args.prompt and not args.case: + parser.error("provide --prompt or --case") + return asyncio.run(_main_async(args)) if __name__ == "__main__": - sys.exit(main()) + raise SystemExit(main()) From e4775d42e7a3e5e4ae0d9b78f17a0ecfbf1b6983 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:35:47 +0800 Subject: [PATCH 2/7] eval: carry real OMP RPC implementation into current main (backend/scripts/live_eval/runner.py) --- backend/scripts/live_eval/runner.py | 205 +++++++++++++++++----------- 1 file changed, 128 insertions(+), 77 deletions(-) diff --git a/backend/scripts/live_eval/runner.py b/backend/scripts/live_eval/runner.py index d00f6186..67be30ac 100644 --- a/backend/scripts/live_eval/runner.py +++ b/backend/scripts/live_eval/runner.py @@ -17,6 +17,7 @@ import io import json import os +import shutil import sqlite3 import subprocess import sys @@ -63,6 +64,7 @@ ) from scripts.live_eval.skill_route import load_skill_route_cases # type: ignore[import-not-found] from scripts.live_eval.private_suite import load_private_real_user_cases # type: ignore[import-not-found] + from scripts.live_eval.agent_executor import run_omp_rpc_agent # type: ignore[import-not-found] else: from .cases import ( BENCHMARK_VERSION, @@ -90,6 +92,7 @@ ) from .skill_route import load_skill_route_cases from .private_suite import load_private_real_user_cases + from .agent_executor import run_omp_rpc_agent PROJECT_ROOT = Path(__file__).resolve().parents[3] BACKEND_DIR = PROJECT_ROOT / "backend" @@ -195,6 +198,26 @@ def _runtime_version() -> str: ).strip() else "unknown" +def _omp_runtime_version() -> str: + executable = shutil.which("omp") + if not executable: + return "unavailable" + try: + proc = subprocess.run( + [executable, "--version"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=20, + ) + except (OSError, subprocess.TimeoutExpired): + return "unavailable" + text = ((proc.stdout or "") + (proc.stderr or "")).strip() + return text.splitlines()[0][:80] if text else "unknown" + + def _mutable_hashes() -> dict[str, str]: base = BACKEND_DIR / "scripts" / "live_eval" return { @@ -208,6 +231,8 @@ def _mutable_hashes() -> dict[str, str]: "private_suite": _sha16(base / "private_suite.py"), "skill_route": _sha16(base / "skill_route.py"), "human_grading": _sha16(base / "human_grading.py"), + "agent_executor": _sha16(base / "agent_executor.py"), + "scripted_cli_executor": _sha16(base / "scripted_cli_executor.py"), "skill_registry": _sha16(BACKEND_DIR / "app" / "services" / "agent_skill_registry.py"), "offeru_skill": _sha16(PROJECT_ROOT / ".agents" / "skills" / "offeru" / "SKILL.md"), } @@ -223,6 +248,8 @@ def _freeze_metadata( case_count: int, private_case_file: Path | None = None, runtime: str = "codebuddy", + model: str = "", + thinking: str = "", ) -> dict[str, Any]: return { "benchmark_version": BENCHMARK_VERSION, @@ -232,11 +259,19 @@ def _freeze_metadata( "component_hashes": _mutable_hashes(), "seed_path": source_db.name, "runtime": runtime, - "runtime_version": (_runtime_version() if runtime == "codebuddy" else "omp"), - # 模型身份在此协议下不可核验：诚实记 UNVERIFIED，不写 harness-provided - # 冒充已证明。若外部执行器在结果里回报模型，记为 model_reported。 + "runtime_version": ( + _runtime_version() if runtime == "codebuddy" else _omp_runtime_version() + ), + # Freeze 只记录请求条件；真实模型身份在每个 OMP RPC trial 的 + # get_state / event artifacts 中单独记录，不能把 selector 冒充成已验证身份。 + "model_requested": model if runtime == "omp" else "", + "thinking_requested": thinking if runtime == "omp" else "", "model": "UNVERIFIED", - "model_source": "not_observable_in_handoff_protocol", + "model_source": ( + "trial_rpc_state_required" + if runtime == "omp" + else "not_observable_in_subprocess_stream" + ), "mode": mode, "suite": suite, "repeat": repeat, @@ -503,71 +538,65 @@ async def _run_harness_once(prompt: str, *, eval_db: Path, timeout: int) -> Trac trace.provider_failure = classify_provider_failure(trace) return trace async def _run_harness_omp( - prompt: str, *, eval_db: Path, timeout: int, case_dir: Path, round_index: int + prompt: str, + *, + eval_db: Path, + timeout: int, + case_dir: Path, + round_index: int, + model: str, + thinking: str, ) -> Trace: - """omp (swe-2) executor backend — **External Executor Handoff 协议**，非托管 Runtime。 - - codebuddy is spawned as a subprocess; an omp agent is not — it runs inside - the omp harness, outside this process. So in ``--runtime omp`` mode the - runner writes a request file the external omp orchestrator picks up, and - blocks until the agent writes ``omp_result.json`` back into the case dir. - - 协议现在携带 ``request_id``：每次请求生成唯一 id，结果必须回显同一 id - 才被接受——防止跨轮次/跨 case 消费陈旧结果文件（E4/E6）。 - 结果文件仍非原子写，故额外要求 ``request_id`` 匹配 + JSON 可解析； - 半个文件不会被当成有效返回（E5）。 - - Request: ``/omp_request.json`` — {request_id, prompt, eval_db, timeout, round} - Result: ``/omp_result.json`` — {request_id, tool_calls, final_text, is_error, model?} - - ``tool_calls`` items carry the literal CLI command in ``input``. 注意：这些 - 是自报文本，grader 只用作 trajectory 诊断；真实执行证据看 audit.json。 - """ - request_path = case_dir / "omp_request.json" - result_path = case_dir / "omp_result.json" - # 清掉上一轮/上一题的遗留结果，避免消费陈旧文件。 - with contextlib.suppress(OSError): - result_path.unlink() - request_id = f"{case_dir.name}-r{round_index}-{uuid.uuid4().hex[:12]}" - write_json(request_path, { - "request_id": request_id, - "round": round_index, - "prompt": prompt, - "eval_db": str(eval_db), - "timeout_seconds": timeout, - "runtime": "omp", - "protocol": "external-executor-handoff.v1", - }) - started = time.perf_counter() - deadline = started + timeout - trace = Trace() - while time.perf_counter() < deadline: - if result_path.is_file(): - try: - payload = json.loads(result_path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - payload = {} - # 结果必须回显当前 request_id 才算本次应答；不匹配说明是陈旧/迟到文件。 - if payload.get("request_id") != request_id: - await asyncio.sleep(0.5) - continue - trace.tool_calls = [ - {"tool": "Bash", "input": str(call.get("input") or "")} - for call in (payload.get("tool_calls") or []) - ] - trace.final_text = str(payload.get("final_text") or "") - trace.is_error = bool(payload.get("is_error")) - trace.note = str(payload.get("note") or "omp runtime") - # 模型身份：执行器可自报 model，但仍是 unverified —— 单独记录。 - if payload.get("model"): - trace.note = f"{trace.note} | model_reported={payload.get('model')}" - break - await asyncio.sleep(0.5) - else: - trace.note = f"omp executor timeout after {timeout}s (no omp_result.json)" - trace.is_error = True - trace.elapsed_s = round(time.perf_counter() - started, 1) + """Run a real OMP Agent over RPC; never synthesize its Operation choices.""" + + rpc_dir = case_dir / f"omp-rpc-round-{round_index:02d}" + result = await run_omp_rpc_agent( + prompt, + eval_db=eval_db, + run_dir=rpc_dir, + model=model, + thinking=thinking, + timeout=timeout, + ) + trace = Trace( + events=list(result.events), + tool_calls=list(result.tool_calls), + final_text=result.final_text, + elapsed_s=result.elapsed_s, + is_error=not result.ok, + note=( + f"omp-rpc-v2 model_requested={result.model_requested} " + f"model_observed={result.model_observed or 'UNVERIFIED'} " + f"thinking_requested={result.thinking_requested} " + f"thinking_observed={result.thinking_observed or 'UNVERIFIED'} " + f"session_id={result.session_id or 'UNVERIFIED'}" + ), + ) + if result.error: + trace.events.append( + { + "type": "error", + "is_error": True, + "source": "omp_rpc_harness", + "error": result.error, + } + ) + if not trace.final_text: + trace.final_text = result.error trace.provider_failure = classify_provider_failure(trace) + write_json( + rpc_dir / "identity.json", + { + "runtime": "omp", + "protocol": "rpc", + "model_requested": result.model_requested, + "model_observed": result.model_observed or None, + "thinking_requested": result.thinking_requested, + "thinking_observed": result.thinking_observed or None, + "session_id": result.session_id or None, + "identity_verified": bool(result.model_observed and result.session_id), + }, + ) return trace # ---------------------------------------------------------------- case run @@ -582,6 +611,8 @@ async def run_case_once( mode: str = "real-user", discovery_mode: str = "progressive", runtime: str = "codebuddy", + model: str = "", + thinking: str = "", ) -> tuple[Trace, dict[str, Any]]: """跑一次 case；返回 (合并 trace, verdict dict)。 @@ -638,18 +669,22 @@ async def run_case_once( "human_rating_required": case.human_rating_required, "tags": list(case.tags), }) - # runtime.json 记录的是**实际**执行通道：codebuddy 是子进程（有 node/script/ - # 工具白名单），omp 是外部执行器交接协议（没有这些字段，标 handoff）。 + # runtime.json 记录**实际**执行通道：codebuddy 是 stream-json 子进程； + # omp 由 runner 直接托管 RPC 进程，并在每轮 identity.json 记录观察到的模型/会话身份。 write_json(case_dir / "runtime.json", { "harness": runtime, - "protocol": ( - "external-executor-handoff.v1" if runtime == "omp" else "subprocess-stdout-events" - ), + "protocol": "omp-rpc-v2" if runtime == "omp" else "subprocess-stdout-events", "node": str(NODE_EXE) if runtime == "codebuddy" else None, "script": str(CODEBUDDY_SCRIPT) if runtime == "codebuddy" else None, - "tools": HARNESS_TOOLS if runtime == "codebuddy" else None, + "tools": HARNESS_TOOLS if runtime == "codebuddy" else "read,bash,grep,glob", "allowed_tools": HARNESS_ALLOWED_TOOLS if runtime == "codebuddy" else None, - "disallowed_tools": HARNESS_DISALLOWED_TOOLS if runtime == "codebuddy" else None, + "disallowed_tools": ( + HARNESS_DISALLOWED_TOOLS + if runtime == "codebuddy" + else "app.cli confirm (denied by OMP eval config)" + ), + "model_requested": model if runtime == "omp" else None, + "thinking_requested": thinking if runtime == "omp" else None, "timeout_seconds": timeout, }) @@ -681,6 +716,8 @@ async def run_case_once( timeout=timeout, case_dir=case_dir, round_index=turn_index, + model=model, + thinking=thinking, ) if runtime == "omp" else _run_harness_once( @@ -1064,6 +1101,8 @@ async def main_async(args: argparse.Namespace) -> int: case_count=len(selected), private_case_file=private_case_file, runtime=args.runtime, + model=args.omp_model, + thinking=args.omp_thinking, ) write_json(run_dir / "freeze.json", freeze) print(f"[live-eval] run dir: {run_dir}") @@ -1092,6 +1131,8 @@ async def main_async(args: argparse.Namespace) -> int: mode=args.mode, discovery_mode=args.discovery_mode, runtime=args.runtime, + model=args.omp_model, + thinking=args.omp_thinking, ) except Exception as exc: # noqa: BLE001 - Eval 必须记录失败而不是崩掉整个 run verdict = { @@ -1168,9 +1209,19 @@ def main(argv: list[str] | None = None) -> int: help="Eval-only Skill discovery condition for ablation.") parser.add_argument("--timeout", type=int, default=600, help="Per-round harness timeout (s).") parser.add_argument("--runtime", default="codebuddy", choices=("codebuddy", "omp"), - help="Agent executor. codebuddy spawns the CLI harness; omp writes an " - "omp_request.json per case and waits for an external omp agent to write " - "omp_result.json (the omp harness spawns its own agents, not this process).") + help="Agent executor. codebuddy uses stream-json; omp launches a real " + "OMP session through --mode rpc and captures model-issued tool events.") + parser.add_argument( + "--omp-model", + default=os.environ.get("OFFERU_LIVE_EVAL_OMP_MODEL") or "avabbbb/devin/swe-2", + help="OMP model selector used with --runtime omp.", + ) + parser.add_argument( + "--omp-thinking", + default=os.environ.get("OFFERU_LIVE_EVAL_OMP_THINKING") or "xhigh", + choices=("off", "minimal", "low", "medium", "high", "xhigh", "max"), + help="OMP thinking level used with --runtime omp.", + ) parser.add_argument("--max-cases", type=int, default=0, help="Limit number of cases (0 = all).") parser.add_argument("--source-db", default=str(DEFAULT_SOURCE_DB), help="Template database to clone.") parser.add_argument("--output-root", default=str(DEFAULT_RUN_ROOT), help="Run artifact root.") From af73786acfb8eaf847a3f0b5d61fae23419eca31 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:35:50 +0800 Subject: [PATCH 3/7] eval: carry real OMP RPC implementation into current main (docs/evals/E2E-EVAL-REPORT.md) --- docs/evals/E2E-EVAL-REPORT.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/evals/E2E-EVAL-REPORT.md b/docs/evals/E2E-EVAL-REPORT.md index 31e6de34..7a09fc36 100644 --- a/docs/evals/E2E-EVAL-REPORT.md +++ b/docs/evals/E2E-EVAL-REPORT.md @@ -140,15 +140,19 @@ prepare_resume_optimization → fact_gates_passed → proposal_ready → user_co ## Next Steps for Real Agent E2E -To validate the **Agent-native product experience**, we need: +The repository now contains a real OMP RPC harness in `agent_executor.py`, and +`runner.py --runtime omp` launches it directly rather than waiting for a scripted +handoff file. -1. **Real OMP/SWE-2 session**: Launch actual Agent runtime, not scripted CLI calls -2. **Model-issued tool calls**: Verify Agent decides which Operations to call -3. **LLM-driven optimization**: Use real API key for content rewriting -4. **Quality scoring**: LLM judge evaluates proposal against ground truth -5. **Visible HITL**: User watches frontend while Agent works in background -6. **Rejection flow**: Test "decline" path, not just "accept" -7. **Multi-turn interaction**: Agent responds to user feedback, not just one-shot +The remaining acceptance work is runtime evidence, not more scripted plumbing: + +1. **Run real OMP/SWE-2**: execute the RPC harness against the private eval snapshot +2. **Verify model identity**: compare requested selector with RPC `get_state` +3. **Verify model-issued tools**: require `tool_execution_start` events for OfferU CLI +4. **Corroborate outcome**: bind tool events to OperationAuditLog / Proposal / DB state +5. **Visible HITL**: user reviews/accepts/rejects in normal OfferU frontend +6. **Rejection + continuation**: verify Agent observes the human decision correctly +7. **Multi-turn / pass^3**: repeat from fresh isolated state --- @@ -171,4 +175,4 @@ This deterministic pipeline smoke test confirms the OfferU resume optimization * However, this is **not an Agent E2E test**. To validate the true Agent-native experience — where SWE-2 reasons about user goals, discovers Skills, selects Operations, and calls CLI tools autonomously — a separate eval with real OMP session and model-issued tool calls is required. -**Status**: ✅ Deterministic pipeline validated | ⚠️ Agent E2E not tested +**Status**: ✅ Deterministic pipeline validated | 🧪 Real OMP RPC harness implemented | ⚠️ `AGENT_NATIVE_E2E = NOT_RUN` until a live model trace + trusted outcome are captured From 61e20ef105aaf206e68a6316d01a8da7b4b76a51 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:35:55 +0800 Subject: [PATCH 4/7] eval: carry real OMP RPC implementation into current main (docs/evals/REAL-AGENT-E2E.md) --- docs/evals/REAL-AGENT-E2E.md | 381 +++++++++++++++++++++++------------ 1 file changed, 252 insertions(+), 129 deletions(-) diff --git a/docs/evals/REAL-AGENT-E2E.md b/docs/evals/REAL-AGENT-E2E.md index 66f0fb5c..807a5754 100644 --- a/docs/evals/REAL-AGENT-E2E.md +++ b/docs/evals/REAL-AGENT-E2E.md @@ -1,189 +1,312 @@ -> **HISTORICAL EVAL GUIDE.** This file predates the current Live Eval / real OMP RPC path and may mention manual API-key/runtime setup that is no longer the default product direction. Use `docs/evals/LIVE_EVAL.md`, `STATUS.md`, and PR #16 for current external-Agent validation. +# Real OMP Agent E2E Test Guide + +This guide describes OfferU's **real Agent-native** acceptance path. + +It is intentionally different from: + +- Playwright frontend regression; +- deterministic CLI workflow smoke; +- fixture/replay pipeline tests. + +The canonical Agent path is: + +```text +natural-language user goal + ↓ +real OMP session / selected model + ↓ +model-issued tool calls + ↓ +OfferU Skill → CLI / Bridge + ↓ +Operation Registry + ↓ +Proposal/HITL for protected actions + ↓ +human approval in OfferU + ↓ +Career Truth +``` -# Real Agent E2E Test Guide +## Prerequisites -This document describes how to run a **real Agent E2E test** for OfferU's resume optimization feature. +1. OMP is installed and authenticated. +2. The requested model resolves in `omp --list-models`. +3. A private/isolated eval database exists. +4. OfferU's generated Skill exists at `.agents/skills/offeru/SKILL.md`. +5. For visible HITL acceptance, the user separately runs OfferU frontend/backend against the same authorized test data. -## Prerequisites +The runner does **not** require Playwright to drive the Agent. -1. **Agent runtime installed**: - - Claude Code: `npm install -g @anthropic-ai/claude-code` - - Codex CLI: `npm install -g @openai/codex-cli` - - OMP: `npm install -g oh-my-pi` +## Recommended invocation -2. **API key configured**: - - Claude Code: `claude config set apiKey ` - - Codex: `codex login` - - OMP: `omp config set apiKey ` +From `backend/`: -3. **Backend running**: `cd backend && .venv312/Scripts/python.exe -m uvicorn app.main:app --host 127.0.0.1 --port 8766` +```powershell +python scripts/live_eval/runner.py ^ + --runtime omp ^ + --omp-model avabbbb/devin/swe-2 ^ + --omp-thinking xhigh ^ + --resume-opt-file H:\tmp\offeru\private-eval\private_resume_opt_6.json ^ + --source-db H:\tmp\offeru\private-eval\eval.db ^ + --case PR01 ^ + --mode real-user +``` + +The runner now launches OMP through its documented RPC protocol: -4. **Frontend running**: `cd frontend && npm run dev` (port 7410) +```text +omp --mode rpc --no-session --model --thinking +``` -5. **Eval DB**: `H:\tmp\offeru\private-eval\eval.db` +It captures OMP session events including: -## Running the Real Agent E2E Test +- `agent_start / agent_end`; +- `message_update`; +- `tool_execution_start`; +- `tool_execution_end`; +- `get_state` identity snapshots. -### Option 1: Claude Code +The Python runner supplies environment/isolation and captures evidence. It does **not** choose OfferU Operations. -```bash -cd H:\WorkSpace_For_VsCode\Python\OFFERU\backend +## What the model receives -claude --print --verbose --output-format stream-json --model sonnet " -You are an AI assistant helping with a job application task. +The Agent receives the real user's natural-language task plus integration/safety context. -## Task -帮我看看这个字节 AIGC 产品经理岗位值不值得投，如果值得，帮我准备针对这个岗位的简历。 +The prompt must not contain an expected Operation sequence such as: -## Environment -- Working directory: H:\WorkSpace_For_VsCode\Python\OFFERU\backend -- Database: H:\tmp\offeru\private-eval\eval.db -- Frontend: http://127.0.0.1:7410 -- Backend: http://127.0.0.1:8766 +```text +first get_profile +then get_job +then prepare_resume_optimization +``` -## Available Tools -Use bash tool to run OfferU CLI commands: +The model is expected to use the generated OfferU Skill and live manifest/schema contract to decide what it needs. -python -m app.cli doctor --pretty # Check system health -python -m app.cli manifest --pretty # List available skills -python -m app.cli manifest --skill --pretty # Show skill details -python -m app.cli ops --pretty # List all operations -python -m app.cli schema --pretty # Show operation schema -python -m app.cli run --args '{\"key\":\"val\"}' # Run operation -python -m app.cli confirm # Confirm proposal +## OMP tool boundary -## Rules -1. Discover skills first via manifest -2. Read operation schemas before calling -3. For mutations, check if confirmation required -4. Report what you did and the result +The eval launches OMP with a narrow tool set: -Start by checking system health and listing available skills. -" +```text +read,bash,grep,glob ``` -### Option 2: Codex CLI +The per-run OMP config is fail-closed for bash: -```bash -cd H:\WorkSpace_For_VsCode\Python\OFFERU\backend +- `python -m app.cli ...` is allowed; +- `app.cli confirm` is explicitly denied; +- other exec-tier bash commands require an interactive approval that the headless eval does not provide. -codex exec --model o3 " -You are an AI assistant helping with a job application task. -... -" -``` +This OMP permission layer protects the local eval process. + +It does **not** replace OfferU's business authorization layer. -### Option 3: OMP +OfferU side-effect Operations still create Proposal/HITL state. -```bash -cd H:\WorkSpace_For_VsCode\Python\OFFERU\backend +## Human confirmation -omp exec --model swe-2 " -You are an AI assistant helping with a job application task. -... -" +The Coding Agent must never confirm its own OfferU Proposal. + +Correct: + +```text +Agent selects protected Operation + ↓ +OfferU creates waiting_confirmation Proposal + ↓ +Agent stops / reports the pending decision + ↓ +human reviews in OfferU frontend + ↓ +human accepts/rejects + ↓ +Agent may continue from the resulting state ``` -## What to Verify +Incorrect: -### 1. Agent Autonomy +```text +Agent → python -m app.cli confirm ... +``` -The Agent should **autonomously**: -- Discover available skills via `manifest` -- Read operation schemas via `schema` -- Call operations via `run` -- Handle confirmations via `confirm` +The eval OMP config explicitly denies this command. -**NOT**: Pre-scripted operation sequence +## What to verify -### 2. Model-Issued Tool Calls +### 1. Real runtime identity -Verify the Agent actually issued tool calls: +Each OMP trial writes an identity artifact containing: +```text +runtime +protocol +model_requested +model_observed +thinking_requested +thinking_observed +session_id +identity_verified ``` -Agent: I'll check the system health first. -→ bash: python -m app.cli doctor --pretty +Do not call the model verified merely because `--omp-model` requested it. -Agent: I see the pre_application_decision skill is available. +If OMP's RPC state does not expose a usable model/session identity, report it as unverified. -→ bash: python -m app.cli manifest --skill pre_application_decision --pretty +### 2. Model-issued tool calls -Agent: Let me get the user profile. +A valid Agent trace contains OMP `tool_execution_start` events. -→ bash: python -m app.cli run get_profile --args '{}' +For example: + +```json +{ + "type": "tool_execution_start", + "toolName": "bash", + "args": { + "command": "python -m app.cli manifest --pretty" + } +} ``` -### 3. Frontend Updates +The evaluator records these as `source=model`. + +A Python script inventing the same command does not count. + +### 3. Trusted OfferU execution + +A model-issued shell command is trajectory evidence. + +It is not by itself proof that the OfferU Operation executed. + +The grader must corroborate relevant business execution through OfferU-controlled evidence such as: -While Agent works, open browser: -- `http://127.0.0.1:7410` — Today page -- `http://127.0.0.1:7410/#/jobs` — Jobs list -- `http://127.0.0.1:7410/#/jobs/1` — Job detail +- `OperationAuditLog`; +- persisted Proposal/AgentRun; +- CareerTask events; +- final DB/artifact state. -You should see real-time updates as Agent calls operations. +### 4. Useful outcome -### 4. HITL Flow +The Agent must produce a business result the user can inspect. -When Agent creates a proposal: -1. Frontend shows "需要你的确认" -2. You click "接受" or "拒绝" -3. Agent continues based on your choice +Examples: -## Success Criteria +- grounded role recommendation; +- Role Intelligence artifact; +- reviewable Resume Proposal; +- visible pending decision. -✅ Agent autonomously discovers and calls correct operations -✅ No hardcoded operation sequence -✅ Frontend updates in real-time -✅ HITL confirmation works -✅ Agent reports results clearly -✅ Full trace logged for audit +"Agent said it completed the task" is not an outcome. -## Failure Indicators +### 5. Frontend/HITL -❌ Agent doesn't call any OfferU operations -❌ Agent calls wrong operations (e.g., skips skill discovery) -❌ Frontend doesn't update -❌ Agent can't handle confirmation flow -❌ Agent doesn't report results +For the human-facing Golden Path, the user keeps the normal OfferU frontend open themselves. -## Comparison: Scripted vs Real Agent +The Agent operates via tools. -| Scripted CLI Executor | Real Agent E2E | -|-----------------------|----------------| -| `if "resume" in prompt: call prepare_resume_optimization()` | Agent reasons about task, discovers skills, selects operations | -| Fixed sequence | Model decides sequence | -| No LLM involved | LLM makes decisions | -| Workflow test | Agent test | +The human uses the frontend to: -## Next Steps +- understand progress; +- inspect Proposal diff; +- edit where supported; +- accept/reject; +- inspect resulting Career Truth. -1. **Install Agent runtime**: Choose Claude Code, Codex, or OMP -2. **Configure API key**: Set up authentication -3. **Run real session**: Launch Agent with natural language task -4. **Verify trace**: Check that Agent actually called operations -5. **Observe frontend**: Watch for real-time updates -6. **Test HITL**: Confirm proposal acceptance/rejection flow +This does not require Playwright or `headless=false`. + +Playwright remains a separate frontend-regression surface. ## Artifacts -- **Trace files**: `H:\tmp\offeru\live-eval-runs\agent\\trace.json` -- **Prompt files**: `H:\tmp\offeru\live-eval-runs\agent\\prompt.txt` -- **Screenshots**: `H:\tmp\offeru\eval-screenshots\` +For each OMP round the runner writes: + +```text +/omp-rpc-round-NN/ + omp-rpc-events.ndjson + agent-execution.json + identity.json + omp-eval.yml +``` + +The normal Live Eval artifacts still include: + +- database before/after snapshots; +- `audit.json`; +- `operations.json`; +- `proposals.json`; +- `events.ndjson`; +- deterministic grader/verdict output. + +## Scripted executor + +`scripted_cli_executor.py` remains useful for deterministic smoke coverage. + +It must never be reported as a model run. + +Correct status: + +```text +DETERMINISTIC_PIPELINE_SMOKE = PASS +``` + +Not: + +```text +OMP_AGENT_E2E = PASS +``` + +## Minimum PASS conditions + +A trial can report `AGENT_NATIVE_E2E = PASS` only when: + +1. OMP is actually launched. +2. A model is requested and runtime identity is recorded honestly. +3. The prompt does not leak the expected Operation path. +4. At least one meaningful OfferU command is emitted through a model-issued tool event. +5. OfferU trusted execution evidence corroborates the relevant Operation/outcome. +6. No Agent self-confirm occurs. +7. Protected mutation stays pending for a human. +8. The final outcome is visible/usable. +9. The isolated trial data is the data actually used by CLI calls. +10. No false-success claim is emitted. + +## Reliability + +After one valid Golden Path run, repeat it from fresh isolated state. + +Preferred acceptance: + +```text +same task +same data snapshot +same model +same thinking level +3 independent trials +→ pass^3 +``` + +One successful run out of three is instability, not support. + +## OMP protocol reference + +OMP RPC is an NDJSON stdio protocol. It supports: + +- `prompt`; +- `abort`; +- `get_state`; +- `set_model`; +- `set_thinking_level`; +- streamed Agent events including tool execution. + +Canonical upstream reference: + +- https://github.com/can1357/oh-my-pi/blob/main/docs/rpc.md +- https://github.com/can1357/oh-my-pi/blob/main/docs/bash-tool-runtime.md -## Troubleshooting +## Final distinction -### Agent can't connect to backend -- Check backend is running: `curl http://127.0.0.1:8766/api/health` -- Check database path: `H:\tmp\offeru\private-eval\eval.db` -- Check CORS: `env | grep CORS` +The target is not "watch a browser click around." -### Agent doesn't call operations -- Check prompt includes tool instructions -- Check Agent has `bash` tool enabled -- Check CLI commands are correct +The target is: -### Frontend doesn't update -- Check frontend is running: `curl http://127.0.0.1:7410` -- Check hash routing: `http://127.0.0.1:7410/#/jobs/1` -- Check database is same: `DATABASE_URL=sqlite+aiosqlite:///H:/tmp/offeru/private-eval/eval.db` +> A real OMP model decides what to do through governed OfferU tools, while the human understands and approves the resulting state through OfferU's normal frontend. From 6aaeda13e199aa845eeb2447288af7e634e25cb4 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:36:06 +0800 Subject: [PATCH 5/7] test(eval): restore real OMP RPC executor coverage --- .../evals/test_omp_rpc_agent_executor.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 backend/tests/evals/test_omp_rpc_agent_executor.py diff --git a/backend/tests/evals/test_omp_rpc_agent_executor.py b/backend/tests/evals/test_omp_rpc_agent_executor.py new file mode 100644 index 00000000..433ed1f3 --- /dev/null +++ b/backend/tests/evals/test_omp_rpc_agent_executor.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.live_eval.agent_executor import ( + _extract_identity, + _offeru_agent_prompt, + _record_tool_start, + _write_omp_eval_config, +) + + +def test_extract_identity_from_rpc_state() -> None: + model, thinking, session_id = _extract_identity( + { + "sessionId": "sess_123", + "thinkingLevel": "xhigh", + "model": {"provider": "avabbbb", "modelId": "devin/swe-2"}, + } + ) + + assert model == "avabbbb/devin/swe-2" + assert thinking == "xhigh" + assert session_id == "sess_123" + + +def test_tool_execution_start_is_marked_model_issued() -> None: + call = _record_tool_start( + { + "type": "tool_execution_start", + "toolName": "bash", + "toolCallId": "tool_1", + "args": {"command": "python -m app.cli manifest --pretty"}, + } + ) + + assert call == { + "tool": "bash", + "input": "python -m app.cli manifest --pretty", + "tool_call_id": "tool_1", + "source": "model", + } + + +def test_agent_prompt_does_not_prescribe_business_operation_sequence() -> None: + prompt = _offeru_agent_prompt("帮我判断这个岗位值不值得投。") + + assert "get_profile" not in prompt + assert "get_job" not in prompt + assert "prepare_resume_optimization" not in prompt + assert "Do not follow a pre-scripted tool sequence" in prompt + assert "app.cli confirm" in prompt + + +def test_eval_config_denies_cli_confirm_before_allowing_cli(tmp_path: Path) -> None: + path = _write_omp_eval_config(tmp_path) + text = path.read_text(encoding="utf-8") + + deny = text.index('match: "*app.cli confirm*"') + allow = text.index('match: "python* -m app.cli *"') + + assert deny < allow + assert "approvalMode: always-ask" in text + assert "approval: deny" in text From 1dc2aca956b43d705572800a35a35db3c2b44de2 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:36:50 +0800 Subject: [PATCH 6/7] docs(eval): distill real Agent-native acceptance into Live Eval --- docs/evals/LIVE_EVAL.md | 59 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/docs/evals/LIVE_EVAL.md b/docs/evals/LIVE_EVAL.md index e112f48c..f8072248 100644 --- a/docs/evals/LIVE_EVAL.md +++ b/docs/evals/LIVE_EVAL.md @@ -4,14 +4,63 @@ > 「在固定 OfferU Career World 的 20 个真实求职任务上，这个 Runtime 的 pass@1 是多少、 > 连续多次成功率是多少、有没有越权、失败到底属于模型 / Harness / Provider / 产品代码」。 -- 被测对象：**外部 Coding Agent**（WorkBuddy / Codex / Claude / Pi），它通过 OfferU Operation Registry 干活 -- 判分依据：**数据库最终状态 + 工具轨迹**，不看 Agent 自称完成 -- 关键前提：模型能力由外部 Harness 自带，因此 **Eval 不需要单独的 LLM 凭据** +- 被测对象：**外部 Coding Agent**（优先真实 OMP RPC；其它 Harness 按同一证据合同接入），它通过 OfferU Skill / CLI / Bridge → Operation Registry 干活 +- 判分依据：**可信 OfferU 执行证据 + 数据库最终状态 + 模型工具轨迹**，不看 Agent 自称完成 +- 关键前提：模型能力由外部 Harness 自带，因此 **Eval 不需要再伪造一套“Agent”控制流** Reference implementation: **https://github.com/luyishui/OfferU** --- +## 验收类型必须分开 + +报告里不得把下列四类验证混成一个“E2E PASS”： + +| 类型 | 能证明什么 | 不能证明什么 | +| --- | --- | --- | +| `FRONTEND_PLAYWRIGHT_FLOW` | UI、路由、表单、可见 HITL 回归 | Coding Agent 推理/选 Skill/选 Operation | +| `DETERMINISTIC_PIPELINE_SMOKE` | CLI、Registry、已知流程、持久化 plumbing | 模型自主决策 | +| `AGENT_NATIVE_E2E` | 真实 Harness + 模型自主发现/调用 OfferU 能力 | 不等同于发布就绪 | +| Computer Use | 截图/鼠标/键盘型 Agent | OfferU canonical Coding Agent 集成 | + +OfferU 的 canonical Agent 路径是： + +```text +natural-language goal +→ real Agent session +→ OfferU Skill discovery +→ model-issued tool call +→ CLI / Bridge +→ Operation Registry +→ Proposal/HITL when protected +→ human decision +→ Career Truth +→ Agent observes the resulting state +``` + +Playwright 只能单独证明前端回归；scripted executor 只能单独证明 deterministic smoke。二者都不能冒充 Agent-native acceptance。 + +### `AGENT_NATIVE_E2E = PASS` 的最小门槛 + +只有以下条件全部成立才能使用这个标签： + +1. 真实 Agent Harness/session 被实际启动； +2. requested / observed model、thinking、session identity 被诚实记录，无法核实时标记 unverified； +3. 用户 prompt 不泄露预期 Operation 顺序； +4. Skill / capability / Operation 选择由模型完成； +5. 至少一个有意义的 OfferU CLI/Bridge 调用来自 **model-issued tool event**； +6. 对应业务执行有 OfferU 自己的可信证据（OperationAuditLog / Proposal / AgentRun / DB outcome），不能只信 shell 文本； +7. protected mutation 产生 Proposal/HITL，Agent 不得自行 `confirm`； +8. 最终存在用户可检查的业务结果； +9. trial 使用明确授权且隔离的数据； +10. cancellation / late result 不能污染后续 trial 或 Career Truth； +11. 多轮切换 Job 时上下文不串线； +12. 不把 provider failure、grader/harness bug 冒充模型能力结论。 + +首次通过后至少做 fresh-state **pass^3**，再讨论稳定支持。 + +--- + ## 5 分钟上手 ```bash @@ -137,8 +186,8 @@ Regression 与 Capability 分开），才升级为 **`OfferU-EvolveBench v1`** ## 安全边界（不可绕过） 1. 每个 case 单独克隆隔离库副本（SQLite online backup），**绝不碰真实库**。 -2. 被测 Agent 只能调用 `Read / Grep / Bash`，且 Bash 仅允许 app.cli 的只读子命令。 -3. 外部不可逆动作（提交申请、发信）**默认禁止**，任何测试只在 sandbox 内验证。 +2. 被测 Agent 只开放完成 Eval 所需的窄工具面；OMP RPC 默认使用 `read / grep / glob / bash`，bash 只允许 OfferU CLI 边界内的命令。**`app.cli confirm` 必须显式 deny**；业务 mutation 只能推进到 OfferU Proposal/HITL，不能由 Agent 自批。 +3. 外部不可逆动作（提交申请、发信）**默认禁止**；任何写能力只在 sandbox/隔离数据上验证 Proposal、审计、状态机与人工确认边界。 4. Provider 层失败（401 / 424 / 429 / timeout）单独归类 `BLOCKED`，不计入 Agent 能力。 5. 产物不写任何凭据。 From 7a7f2af84b90f610c861ffdc9f7aa5e868b65100 Mon Sep 17 00:00:00 2001 From: avabbbb <89175608+avabbbb@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:37:17 +0800 Subject: [PATCH 7/7] docs(eval): document trusted execution grading boundary --- docs/evals/GRADING.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/evals/GRADING.md b/docs/evals/GRADING.md index 1457880a..bee5f3b3 100644 --- a/docs/evals/GRADING.md +++ b/docs/evals/GRADING.md @@ -2,12 +2,39 @@ 判分原则（GOAL §2.1–§2.3、§11、§12）： -- **Outcome > Agent self-report**：只看数据库最终状态与工具轨迹。 +- **Outcome > Agent self-report**：只看可信执行证据、数据库最终状态与模型工具轨迹，不看 Agent 自称完成。 - **Grade Outcome, not Tool Path**：除非路径本身是安全要求，否则允许多条合法路径。 +- **Trusted execution > command-shaped text**：模型“请求了什么”和 OfferU“实际执行了什么”必须分开。 - **Deterministic First**：能用代码判断的不用 LLM judge。 --- +## 可信执行证据 + +Live Eval 把 trajectory 和 execution 分开： + +```text +model-issued tool event + ↓ +requested Operation / CLI command + ↓ +OfferU Operation Registry + ↓ +OperationAuditLog / Proposal / DB outcome + ↓ +executed Operation +``` + +- `Trace.operations_used` 只用于 trajectory 诊断：它来自 Harness/tool-call 文本，不能单独证明执行。 +- 判 `read_at_least_one_operation`、self-confirm、业务写入等执行事实时，以 `OperationAuditLog` 和持久化 outcome 为准。 +- `echo "python -m app.cli run get_profile"`、Agent 最终答复、伪造 JSON 都不能构成 execution evidence。 +- requested model/tool 与 observed/executed evidence 应分别落盘；无法核实 model identity 时必须标记 unverified。 +- 未知 Outcome Criterion 必须 fail-closed 为 `INVALID / grader_bug`，不能静默跳过。 + +这条边界用于防止“脚本长得像 Agent”“文本长得像命令”被误判为真实自主执行。 + +--- + ## Outcome Success Criteria 决定 PASS / FAIL **PASS 只看 Outcome 是否完成，与 Agent 走了哪条工具路径无关。**