diff --git a/action.yml b/action.yml index fe1a5f826a..83639d7e6f 100644 --- a/action.yml +++ b/action.yml @@ -409,6 +409,60 @@ runs: "${STATUS_FLAGS[@]+"${STATUS_FLAGS[@]}"}" \ "${MINT_FLAGS[@]+"${MINT_FLAGS[@]}"}" + # Eval measurements (fail-open): score run-telemetry.jsonl with the agents + # measurement manifest. Same job as fullsend run; never fails the agent. + # Writes eval-measurements.jsonl when at least one new score row is produced + # (tool-agnostic artifact); missing telemetry/manifest skips with no file. + # Manifest trust: prefer a local override from the PR base SHA (same + # trusted tip reusable-* workflows check out for kill-switch config). + # Path prefix comes from inputs.fullsend-dir (install layout: per-repo + # `.fullsend/` or per-org workspace root); file bytes come from the + # trusted ref — never from a PR-head working tree. When no trusted + # override exists, fetch SHA-pinned agents@v0 (no --fullsend-dir). + - name: Eval measurements + if: always() && inputs.agent != '__install_only__' + continue-on-error: true + shell: bash + env: + AGENT: ${{ inputs.agent }} + FULLSEND_DIR: ${{ inputs.fullsend-dir }} + # For GetRef of agents@v0 (SHA pin). Not sent to raw.githubusercontent.com. + GH_TOKEN: ${{ inputs.github_token }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + MEASURE_ARGS=(--agent "${AGENT}" --output-dir "${GITHUB_WORKSPACE}/output") + TRUSTED_REF="${PR_BASE_SHA:-}" + if [[ -z "${TRUSTED_REF}" ]]; then + TRUSTED_REF="${GITHUB_SHA:-}" + fi + FULLSEND_DIR="${FULLSEND_DIR:-.fullsend}" + MEASURE_REL="" + case "${FULLSEND_DIR}" in + "${GITHUB_WORKSPACE}"/*) + MEASURE_REL="${FULLSEND_DIR#"${GITHUB_WORKSPACE}"/}" + ;; + /*) + MEASURE_REL="" + ;; + *) + MEASURE_REL="${FULLSEND_DIR}" + ;; + esac + MEASURE_REL="${MEASURE_REL#./}" + MEASURE_REL="${MEASURE_REL%/}" + if [[ -n "${TRUSTED_REF}" && -n "${MEASURE_REL}" ]] \ + && git cat-file -e "${TRUSTED_REF}:${MEASURE_REL}/eval/measurements/${AGENT}.yaml" 2>/dev/null; then + MEASURE_FILE="${GITHUB_WORKSPACE}/output/.fullsend-measure-${AGENT}.yaml" + mkdir -p "${GITHUB_WORKSPACE}/output" + git show "${TRUSTED_REF}:${MEASURE_REL}/eval/measurements/${AGENT}.yaml" > "${MEASURE_FILE}" + MEASURE_ARGS+=(--registry "${MEASURE_FILE}") + fi + # Binary resolves SHA-pinned agents@v0 when --registry is unset + # (allowlist/hash/audit) and scores only platform run-telemetry.jsonl + # at the top of each runDir. + fullsend eval-measure "${MEASURE_ARGS[@]}" + - name: Upload fullsend artifacts if: always() && inputs.agent != '__install_only__' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e8ee68efac..33d711735a 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -260,6 +260,7 @@ export default defineConfig({ { text: "Standalone Mint", link: "/guides/infrastructure/standalone-mint" }, { text: "Private Repositories", link: "/guides/infrastructure/private-repositories" }, { text: "Tracing Reference", link: "/guides/infrastructure/distributed-tracing" }, + { text: "Eval Measurements", link: "/guides/infrastructure/eval-measurements" }, { text: "Advanced Setup", link: "/guides/infrastructure/advanced-setup" }, { text: "Layered Config Reference", diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index 1a8fba2f29..c004e115b1 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -156,6 +156,14 @@ artifact. OTLP export also changed from post-hoc directory upload to live span export via the OTel SDK's batch processor. The core decision (three-level opt-in, OTel-native, W3C propagation) is unchanged. +**2026-08-10 — Eval measurements ([ADR 0087](0087-eval-measurements-online-trace-scoring.md)):** +online scoring of wild-run traces writes `eval-measurements.jsonl` +beside telemetry when at least one new score is produced (tool-agnostic). Distinct from functional eval fixtures +([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). + +> **Planned:** portable remote score export follows the same OTLP +> configuration as this ADR — no vendor score adapters in core. + **2026-08-18 — Remove duplicate token/cost from root span (3278b059):** `gen_ai.request.model` and `gen_ai.usage.*` token attributes moved to agent spans only; the root span keeps `fullsend.cost_usd` and `fullsend.tool_calls` diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md new file mode 100644 index 0000000000..ceff7a86e6 --- /dev/null +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -0,0 +1,151 @@ +--- +title: "87. Eval measurements as online trace scoring with portable export" +status: Accepted +relates_to: + - operational-observability + - testing-agents +topics: + - observability + - evaluation + - opentelemetry +--- + +# 87. Eval measurements as online trace scoring with portable export + +Date: 2026-08-10 + +## Status + +Accepted + +## Context + +Agent runs already emit OpenTelemetry traces as `run-telemetry.jsonl`, with +optional live OTLP export when `OTEL_EXPORTER_OTLP_*` is set +([ADR 0050](0050-distributed-tracing-instrumentation.md)). Separately, +[ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md) owns the +**functional** eval harness: curated fixtures / scenarios in +`fullsend-ai/agents` `eval//` that gate agent PRs. Those fixtures do +not score wild production runs. + +Operators also need an **online / trend** layer on wild traces (completeness +first; quality signals later). Fullsend must stay **backend-agnostic**: orgs +already choose Phoenix, MLflow, Jaeger, or another OTLP collector for traces. +Baking a single product’s Assessments/Quality API into the core CLI or managed +workflows would force a tool decision on every install. + +Adjacent telemetry work (not competing with this score path): + +- **Level 3 content capture** ([ADR 0050](0050-distributed-tracing-instrumentation.md); + activation draft closed without merge in + [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)): first ship + reads Level 1/2 metadata in `run-telemetry.jsonl` (fitness foundation). + Content-aware scorers on prompt/completion bodies are the intended next + layer once Level 3 is implemented. Measure CLI is host-side after sandbox + exit. +- **Span status from run outcome** + ([#5944](https://github.com/fullsend-ai/fullsend/pull/5944), merged): + OTLP Status (and `fullsend.transcript_error`) become the reliable + success/failure signal. EM-001 only checks that `exit_code` is **present** + (fitness). Outcome scorers must key on Status, not `exit_code == 0`. +- **Observer / lessons → fixtures** (draft closed without merge in + [#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): narrative + analysis and golden-set promotion remain a sibling idea. This ADR is + same-job deterministic scoring on traces. +- **Harness snapshot / forge join keys** + ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524), open): + sibling artifact for harness fingerprint and forge/CI pointers beside + telemetry. Complementary join/identity layer; primary run facts belong on + the OTEL trace (Level 1), while measurements stay a derived sibling file. + +## Options + +1. **Local JSONL only** — portable offline artifact; no remote scores from + fullsend itself. +2. **Backend-native APIs in core** (e.g. one vendor’s Assessments API) — + couples every managed workflow to that product’s auth and schema. +3. **Local JSONL + same OTLP path as agent traces for remote** — scores travel + with the endpoint/headers orgs already configure for ADR 0050; no second + vendor stack in core. + +## Decision + +Introduce **eval measurements**: deterministic scorers that read +`run-telemetry.jsonl` after `fullsend run` in the **same** managed job +(`fullsend eval-measure` in `action.yml`), **fail-open**. Functional eval +scenarios remain ADR 0051 / `eval//`; measurements never block +delivery. + +In plain terms: eval measurements are the concept of scoring traces. +[OTEL primary facts](../glossary.md#otel-primary-facts) are what happened +on the run (the OTEL trace / `run-telemetry.jsonl`). +[OTEL derived products](../glossary.md#otel-derived-products) are scores +computed from that trace (`eval-measurements.jsonl`). Measurements never +rewrite primary facts, and they are [fail-open](../glossary.md#fail-open). + +Scores land in a tool-agnostic `eval-measurements.jsonl` (plus a +small idempotency ledger) next to `run-telemetry.jsonl` whenever at least +one new measurement row is produced (including `label: skip`). Remote score export +will use the same `OTEL_EXPORTER_OTLP_*` configuration as ADR 0050 — no +vendor-specific score adapters in core. `fullsend` owns the parser, scorers, +CLI, and GHA step; `fullsend-ai/agents` owns per-agent measurement manifests +(`eval/measurements/.yaml`) that declare which scorers to enable. +Stock-agent defaults resolve from `agents@v0` at runtime; local files are for +override, opt-out, or custom agents only. Activation is **two-step**: merge +measurement manifests into `fullsend-ai/agents` **and** cut a `v0.x.y` release +that re-points the floating `v0` tag. Merging alone does not activate managed +jobs. Tracking: [#6384](https://github.com/fullsend-ai/fullsend/issues/6384). +Until that release lands, GHA/GitLab `eval-measure` wiring is provisional +(clean skip when the remote manifest is missing). Local `FULLSEND_DIR` +manifests are exercised in unit tests today. + +The first scorer is `trace_fitness` (catalog id `em-001`) — span-tree and +attribute fitness so later scorers can trust the trace. EM-001 reads +OpenTelemetry GenAI attribute names (`gen_ai.*` constants in +`internal/evalmeasure`). `gen_ai.system` was renamed to `gen_ai.provider.name` +in semconv v1.37.0; `modelOK` accepts either so `em-001@1` survives the +emitter migration. Other upstream renames remain an `em-001` version bump. +Pre-script-skipped runs, runs with no `agent` span (never reached an +iteration), and runs where agent spans flushed but the root `run` span never +ended (hard kill / timeout) record `label: skip` and are excluded from +pass/(pass+fail). + +### Versioning (per measurement, not platform “v1”) + +There is no product-wide “eval measurements v1” switch. “First ship” just +means only one scorer is enabled yet. Each manifest entry carries: + +| Field | Meaning | +|---|---| +| `id` | Stable catalog id (`em-001`). New measurement concept → new id. | +| `scorer` | Go dispatch name (`trace_fitness`). | +| `version` | Integer **contract** version of that measurement’s checks / pass rule. | + +Scores and the idempotency ledger key on `id@version` (e.g. `em-001@1`). +Bump `version` when pass/fail semantics change so trends do not mix eras. +Add a check that does not change the pass definition → same version is fine. +Entirely new signal → new `em-NNN` (and usually a new `scorer` string). + +## Consequences + +- Every measured run produces a reviewable, backend-agnostic score file beside + telemetry; missing manifests skip cleanly and measure failure never fails + the agent job. GitHub Actions is the first-ship managed path (uploads + `output/`). GitLab CI calls the same fail-open `eval-measure` CLI under + `$CI_PROJECT_DIR/output` with `artifacts: when: always`. Stock manifests + fetch from public `agents@v0` even without `GH_TOKEN` (rate-limited); a + token is recommended on shared runners. +- Core stays tool-agnostic: no product-specific score env vars in managed + workflows; remote scores follow OTEL when that path lands. +- Functional scenarios (gate) and eval measurements (trend) stay separate; + retro can recommend either a manifest scorer or a scenario fixture. +- Level 1/2 metadata scorers (EM-001) are the foundation; Level 3 content + capture expands what scorers *can* assert (quality / LLM-judge style) once + implemented — it does not replace this same-job path. +- Per-measurement versioning (`id@version`) lets pass/fail semantics evolve + without mixing trend eras. +- Pre-script skipped runs (`fullsend.prescript.skipped=true` on the root span), + runs with no `agent` span (never reached an iteration), and runs where agent + spans flushed but the root `run` span never ended (hard kill / timeout) are + excluded from EM-001: the scorer writes `label: skip` instead of failing a run + that never produced a full telemetry contract. diff --git a/docs/architecture.md b/docs/architecture.md index 197d6ebce4..4e89f1422b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -319,11 +319,15 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). +- Eval measurements: the concept of scoring traces ([fail-open](glossary.md#fail-open)). [OTEL primary facts](glossary.md#otel-primary-facts) stay on the run trace (`run-telemetry.jsonl`); [OTEL derived products](glossary.md#otel-derived-products) are the scores (`eval-measurements.jsonl`) ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). + + > **Planned:** portable remote score export via the same OTLP configuration as agent traces ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). Not yet implemented. **Open questions:** - What signals matter most — cost, latency, token usage, action logs, decision traces, or something else? - ~~How do we balance detailed tracing (useful for debugging) with the volume of data agents will produce?~~ Decided in [ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md): instrument all lifecycle steps comprehensively; volume is managed by backends not by suppressing data at the source. +- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry when at least one new score row is produced (including `label: skip`); portable remote export uses the same OTLP config as traces (planned). The JSONL is absent (not empty) when telemetry/manifest is missing, no traces match, or every candidate is already in the ledger. - What is the retention and access model for agent logs? Who can see what? (JSONL trace access model decided in [ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md); retention policy and broader log access remain open.) - How does observability interact with the security requirement that "every action is logged, attributable, and reviewable"? (See [security-threat-model.md](problems/security-threat-model.md).) - Is there a real-time monitoring requirement (agent is stuck, agent is behaving anomalously), or is observability primarily forensic? diff --git a/docs/cli/README.md b/docs/cli/README.md index 09b70ed236..d049d7d99a 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -27,6 +27,7 @@ Download the latest binary from [GitHub Releases](https://github.com/fullsend-ai | `fullsend run` | Execute an agent locally in a sandbox. See [running agents locally](../guides/user/running-agents-locally.md). | | `fullsend lock [agent-name]` | Pin remote dependencies to `lock.yaml` | | `fullsend scan` | Run security scanners on agent input/output | +| `fullsend eval-measure` | Score wild-run traces into `eval-measurements.jsonl`. See [Eval measurements](../guides/infrastructure/eval-measurements.md). | ## Global flags diff --git a/docs/glossary.md b/docs/glossary.md index 29bdc5541e..aa59337b72 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -88,13 +88,13 @@ See [autonomy-spectrum.md](problems/autonomy-spectrum.md) and [agent-architectur ### Eval Measurement -A score, judge, or metric applied to agent (or agent-chain) behavior — for example cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are what you score. The same measurement can be applied to curated [eval scenarios](#eval-scenario) or to live ("wild") production traffic, at agent scope or across the platform chain. Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). -See [testing-agents.md](problems/testing-agents.md) and [Observability](#observability). +The concept of **scoring traces** — a score, judge, or metric applied to agent (or agent-chain) behavior. Example signals: cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are an [OTEL derived product](#otel-derived-products) computed from [OTEL primary facts](#otel-primary-facts). Online / trend scoring of wild-run traces is decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) (`fullsend eval-measure`, `eval-measurements.jsonl`) and is [fail-open](#fail-open). The same measurement *idea* can also be applied to curated [eval scenarios](#eval-scenario), but those PR-gate fixtures are a separate path ([ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md)). Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). +See [Eval Measurements](guides/infrastructure/eval-measurements.md), [testing-agents.md](problems/testing-agents.md), and [Observability](#observability). ### Eval Scenario -A fixed, reproducible test case — a concrete input with an expected outcome that you re-run when an agent changes. Example: triage is presented with an issue asking to add a cheeseburger to the README and is expected to reject and close it. Scenarios are maintained like tests: if intentional agent behavior changes, update the scenario expectations. They answer "did this change make the agent better or worse on known cases?" and can later grow by promoting interesting production cases from telemetry into the curated set. Distinct from [eval measurements](#eval-measurement) (the scores/judges applied to a scenario or to wild traffic). Prefer this term over the bare word "evals." -See [testing-agents.md](problems/testing-agents.md) (golden-set evaluation). +A fixed, reproducible test case — a concrete input with an expected outcome that you re-run when an agent changes. Example: triage is presented with an issue asking to add a cheeseburger to the README and is expected to reject and close it. Scenarios are maintained like tests: if intentional agent behavior changes, update the scenario expectations. They answer "did this change make the agent better or worse on known cases?" and can later grow by promoting interesting production cases from telemetry into the curated set. Distinct from [eval measurements](#eval-measurement) (online/trend scores on wild traces, or judges applied to a scenario). Prefer this term over the bare word "evals." Also called a *functional eval fixture* in agent CI. +See [ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md) and [testing-agents.md](problems/testing-agents.md) (golden-set evaluation). ### Evergreen @@ -102,6 +102,11 @@ A workflow concept where a repository automatically stays up-to-date with depend ## F +### Fail-Open + +When a step's error or a `fail` score must not fail the surrounding job or block delivery. Eval measurements are fail-open: a missing manifest, a scorer `fail`/`skip` label, or a measure-step IO error never fails the agent run. Contrast with fail-closed gates (auth, kill switch) where an error must stop the run. In scripts, fail-open is acceptable for non-critical steps (logging, metrics) and dangerous for gates. +See [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md), [Eval Measurements](guides/infrastructure/eval-measurements.md), and [Shell scripting](contributing/shell-scripting.md). + ### Flapping When agents enter a cycle of conflicting feedback that prevents convergence. Example: the security review agent rejects what the code agent produces to satisfy the correctness review agent, and vice versa, creating an oscillating loop. Flapping is a primary trigger for [escalation](#escalation) — after a configurable number of cycles, the system stops and routes to humans. @@ -147,6 +152,16 @@ See [security-threat-model.md](problems/security-threat-model.md). The logging, tracing, and audit layer for agent actions. Every agent action must be attributable, traceable, and reviewable — both for debugging failures and for security auditability. In practice, this includes capturing agent JSONL logs (including "thinking" traces), converting them to human-readable format, and uploading them as artifacts. Observability is a cross-cutting concern that touches every other component. See [architecture.md](architecture.md). +### OTEL Derived Products + +Values **computed from** a run's OpenTelemetry trace after the fact — scores, fitness checks, later quality signals. They are not a second copy of what happened. First-ship example: `eval-measurements.jsonl` from `fullsend eval-measure` ([eval measurements](#eval-measurement) are the concept of scoring traces). Derived products sit beside telemetry as sibling files; they never replace [OTEL primary facts](#otel-primary-facts). +See [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) and [Eval Measurements](guides/infrastructure/eval-measurements.md). + +### OTEL Primary Facts + +What **actually happened** on an agent run, recorded as OpenTelemetry (OTEL) spans. The local source of truth is `run-telemetry.jsonl`; when `OTEL_EXPORTER_OTLP_*` is set, the same spans also export live over OTLP ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). Agent identity, work item, tokens, cost, span tree, and `exit_code` belong here. Sibling files (including [eval measurements](#eval-measurement)) must not become a second source of run truth. +See [Distributed Tracing](guides/infrastructure/distributed-tracing.md) and [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md). + ## P ### Policy Store diff --git a/docs/guides/README.md b/docs/guides/README.md index 63d1f15e2e..6394cd97ba 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -27,6 +27,7 @@ Advanced guides for platform operators who deploy and manage the GCP-side infras - [Infrastructure reference](infrastructure/infrastructure-reference.md) — Token mint, WIF, and secrets deployment details - [Enabling fullsend on private repositories](infrastructure/private-repositories.md) — Additional guardrails and configuration for private repos - [Tracing reference](infrastructure/distributed-tracing.md) — Telemetry levels, environment variables, span hierarchy, and attributes +- [Eval measurements](infrastructure/eval-measurements.md) — Online trace scoring with `eval-measurements.jsonl` and measurement manifests ## User guides diff --git a/docs/guides/dev/cli-internals.md b/docs/guides/dev/cli-internals.md index 2fd7312631..d68d17e2d5 100644 --- a/docs/guides/dev/cli-internals.md +++ b/docs/guides/dev/cli-internals.md @@ -127,6 +127,14 @@ fullsend │ ├── --head-sha # Expected PR HEAD SHA (skips review if HEAD moved) │ └── --dry-run # Print what would be posted without API calls ├── post-comment # Post issue/PR comments to GitHub (deprecated) +├── eval-measure # Score wild-run traces (eval measurements) +│ ├── --telemetry # Path to run-telemetry.jsonl (or --output-dir) +│ ├── --output-dir # CI output base or runDir (managed-job form) +│ ├── --registry # Agents measurement manifest YAML (or --agent) +│ ├── --agent # Agent name for manifest resolution (managed-job form) +│ ├── --fullsend-dir # .fullsend dir (local manifest override + fetch cache) +│ ├── --offline # Reject network fetches (local manifest only) +│ └── --out-dir # Output dir (default: telemetry directory) └── reconcile-status # Finalize orphaned status comments ├── --repo # Repository in owner/repo format ├── --number # Issue/PR number diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index b780596cab..985905cfcf 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -108,7 +108,7 @@ and are recognized by LLM-aware backends for GenAI dashboards. |-----------|---------|------------| | `gen_ai.operation.name` | `invoke_agent` | `run`, `agent` (`create_agent` on `sandbox_create`) | | `gen_ai.agent.name` | `triage` | `run`, `agent` | -| `gen_ai.system` | `anthropic` | `agent` (the model vendor, from the runtime) | +| `gen_ai.system` / `gen_ai.provider.name` | `anthropic` | `agent` (model vendor; `system` deprecated in OTel GenAI semconv v1.37 — EM-001 accepts either) | | `gen_ai.request.model` | `claude-opus-4-6` | `agent` (resolved model) | | `gen_ai.usage.input_tokens` / `output_tokens` / `cache_*_input_tokens` | `109938` | `agent` | @@ -244,7 +244,26 @@ Any variable and secret names work here; the values reach the exporter as-is. Consult your backend's documentation for the endpoint URL and authentication mechanism. +## Eval measurements + +After each managed agent run, `fullsend eval-measure` scores +`run-telemetry.jsonl` in the same job (fail-open). Scores land in +`eval-measurements.jsonl` beside telemetry when at least one new score is +produced (tool-agnostic artifact). Portable +remote export will reuse the same `OTEL_EXPORTER_OTLP_*` configuration as +agent traces when implemented. + +Today's scorers (starting with EM-001) read the Level 1/2 **metadata** +contract of `run-telemetry.jsonl` — span tree and attributes, not prompt or +completion bodies. That foundation is intentional: fitness scores must trust +the trace before quality scores can. **Planned:** content-aware scorers that +consume Level 3 prompt/completion capture once Level 3 is implemented — +that is where the real quality signal lives. See +[Eval Measurements](./eval-measurements.md) and +[ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). + ## See also - [How To Emit Traces](../user/how-to-emit-traces.md): step-by-step setup guide - [Tracing Development Guide](../dev/tracing.md): implementation details for contributors +- [Eval Measurements](./eval-measurements.md): online scoring of wild-run traces diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md new file mode 100644 index 0000000000..1efd751386 --- /dev/null +++ b/docs/guides/infrastructure/eval-measurements.md @@ -0,0 +1,260 @@ +# Eval Measurements + +Eval measurements score agent runs from their **OpenTelemetry traces** for +trends over time. They are **not** functional evals (PR-gate fixtures under +`fullsend-ai/agents` `eval//`). + +**Decided:** [ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). +Telemetry baseline: [Distributed Tracing](./distributed-tracing.md) +([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md)). + +## Prerequisites + +- A repository with fullsend installed and producing `run-telemetry.jsonl` + (see [Distributed Tracing](./distributed-tracing.md)). +- A measurement manifest for the agent (stock agents get one from + `fullsend-ai/agents@v0`; custom agents need a local YAML under + `${FULLSEND_DIR}/eval/measurements/`). + +## Architecture (read this first) + +Eval measurements are the concept of scoring traces. +[OTEL primary facts](../../glossary.md#otel-primary-facts) are what happened +on the run (`run-telemetry.jsonl`). +[OTEL derived products](../../glossary.md#otel-derived-products) are scores +computed from that trace (`eval-measurements.jsonl`). The step is +[fail-open](../../glossary.md#fail-open): it never blocks delivery. + +Fullsend does not pick an observability product for scores. The portable +contract is a local JSONL artifact next to telemetry; remote export reuses +the same OpenTelemetry (`OTEL_EXPORTER_OTLP_*`) configuration as agent +traces when implemented. + +OTLP (OpenTelemetry Protocol) is the wire format that carries spans and +scores to any compatible backend — Phoenix, MLflow, Jaeger, etc. + +```text +fullsend run + └─ always writes output//run-telemetry.jsonl + └─ if OTEL_EXPORTER_OTLP_* set → live OTLP export of agent spans + (any compatible backend — ADR 0050) + +fullsend eval-measure (same GHA job, fail-open, after run) + └─ writes output//eval-measurements.jsonl when at least one + new score is produced (+ eval-measure-ledger.txt for idempotency) +``` + +> **Planned:** portable remote score export via the same `OTEL_EXPORTER_OTLP_*` +> path as agent traces. Not yet implemented. + +| Artifact | When | Purpose | +|---|---|---| +| `run-telemetry.jsonl` | Every run | OTLP JSON TracesData lines (local source of truth for spans) | +| `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …). On `label: skip`, `value` is unused (serialized as `0`; ignore it). | +| Remote agent spans | OTEL configured | Same spans the local file holds | +| Remote scores *(planned)* | OTEL configured | Scores on the OTLP path — any OTLP backend | + +Orgs choose Phoenix, MLflow, Jaeger, or another collector independently. +Fullsend does not forward vendor-specific score credentials in managed +workflows. + +## Measurements vs functional evals + +| | Functional evals | Eval measurements | +|---|---|---| +| **Repo path** | `agents/eval//` | `agents/eval/measurements/.yaml` | +| **When** | PR / CI fixture gates | After each managed agent run | +| **Input** | Case fixtures + judge harness | `run-telemetry.jsonl` only | +| **Blocks delivery?** | Yes (when wired as a check) | Never (fail-open) | + +## Ownership + +| Concern | Repo | +|---|---| +| Parser, scorer **implementations**, CLI, GHA post-step | `fullsend-ai/fullsend` | +| Default manifests for stock agents (which scorers / ids) | `fullsend-ai/agents` | +| Overrides, opt-out, BYOA agent manifests | Consumer repo (`FULLSEND_DIR`) | + +Defaults for stock agents live next to those agents in `fullsend-ai/agents`. +Managed jobs fetch them from `agents@v0` when no local file exists — users do +**not** duplicate YAML into every install. Put a file under +`${FULLSEND_DIR}/eval/measurements/` only to change policy or score a custom +agent. + +**Activation is two-step:** merge manifests (e.g. +[fullsend-ai/agents#722](https://github.com/fullsend-ai/agents/pull/722)) +**and** cut a `v0.x.y` release that re-points the floating `v0` tag +([#6384](https://github.com/fullsend-ai/fullsend/issues/6384)). Until that +release lands, managed GHA/GitLab `eval-measure` steps stay provisional +(missing remote manifest → clean skip). Local `FULLSEND_DIR` overrides work +today and are covered by CLI tests. + +Scorer *code* stays in fullsend: the measure CLI is the released engine that +understands `run-telemetry.jsonl`. Agents ships **policy** (manifests), not Go. +EM-001 (`trace_fitness`) evaluates fullsend’s telemetry contract across agents; +stock agents opt in via manifests here / in `agents`. + +| Change | Where | +|---|---| +| New Go scorer or new declarative `assert:` primitive | `fullsend` PR | +| New measurement `id` / enable / thresholds for a stock agent (existing scorer) | `agents` PR | +| Org-specific policy for stock or custom agents | Local override in the consumer repo | + +**Planned (not in first ship):** declarative checks in the manifest (attribute +exists, ratio/threshold bands) so most agent-specific policy is YAML-only. +Until then, agent-specific math still lands as a named Go scorer in fullsend, +enabled only for the agents that list it. + +First scorer: **`trace_fitness`** (catalog id `em-001`) — span tree + expected +attributes so later scorers can trust the trace. + +Manifest shape (first ship — enablement only): + +```yaml +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +``` + +Illustrative **logic-as-config** (future declarative engine — not wired yet). +Attribute names in the example match attrs fullsend emits on the **`run`** +span today; they are **not** a contract for the declarative surface: + +```yaml +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + - id: em-010 + scorer: declarative + version: 1 + where: + span: run + checks: + - name: turn_token_ratio + assert: ratio_lte + numerator: gen_ai.usage.output_tokens + denominator: fullsend.num_turns + max: 8000 +``` + +### Versioning + +Not a platform “v1.” Each entry versions its own contract: + +- **`id`** — stable catalog id (`em-001`). New concept → new id. +- **`scorer`** — which Go scorer to run (`trace_fitness`). +- **`version`** — bump when pass/fail semantics change; scores store + `em-001@1`. Ledger is idempotent per `(trace_id, name, id@version)`. + +The EM-001 `exit` check only requires that `exit_code` is **present** on the +run span (instrumentation fitness). It does **not** treat `exit_code == 0` as +success. After [#5944](https://github.com/fullsend-ai/fullsend/pull/5944), +run/agent **OTLP Status** (and `fullsend.transcript_error`) are the +success/failure signal for outcome scorers. + +Pre-script **skipped** runs set `fullsend.prescript.skipped=true` on the root +span and never create a sandbox. EM-001 records `label: skip` for those traces +instead of failing the span-tree / model / usage checks. Runs with no `agent` +span (never reached an iteration — e.g. sandbox/provider failure) and runs +where agent spans flushed but the root `run` span never ended (hard kill / +timeout) are also `label: skip`, so pass/(pass+fail) measures the telemetry +contract rather than runner health. An unknown `scorer:` string (for example a +newer `agents@v0` manifest this binary does not implement yet) also writes +`label: skip`, not `fail`. Trend pass-rate as `pass / (pass + fail)` and drop +`skip`. + +## Adjacent telemetry work + +| Topic | Relationship to measurements | +|---|---| +| Level 3 content capture ([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md); activation draft closed without merge in [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)) | First ship scores Level 1/2 metadata fitness. **Planned:** content-aware scorers on Level 3 prompt/completion bodies once L3 is implemented — that is the real quality signal. Measure CLI is host-side after the sandbox exits. | +| [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome *(merged)* | Unblocks outcome scorers keyed on Status, not raw exit alone. | +| Semantic observability / observer / lessons (draft closed without merge in [#2423](https://github.com/fullsend-ai/fullsend/pull/2423)) | Observer + lessons → fixtures remains a sibling idea; measurements are the online score path. | +| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys *(open)* | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | + +## Same-job timing + +```text +GitHub Actions job +├── fullsend run +├── fullsend eval-measure # reads output//run-telemetry.jsonl; never fails the job +└── upload-artifact # includes both JSONL files under output/ + +GitLab CI agent job +├── fullsend run # --output-dir $CI_PROJECT_DIR/output +├── fullsend eval-measure # always (even if run failed); || true +└── artifacts: output/ # when: always (parity with GHA upload of output/) +``` + +Add `output/` to the consuming repo's `.gitignore` so local GitLab-checkout +runs do not stage telemetry accidentally. The GitLab per-repo scaffold embeds +a recommended `.gitignore` fragment (asserted in tests) but does **not** +install it as a root file — that would overwrite an existing consumer ignore +list. When `--output-dir` sits inside `--target-repo` (GitLab layout), +`fullsend run` omits that top-level directory from the sandbox tarball and +`.git/info/exclude`; sibling layouts (GitHub Actions) are unchanged. + +## Manifest resolution in CI + +`fullsend eval-measure` resolves the measurement manifest for the agent: + +1. Explicit `--registry ` when the managed job materializes a **trusted** + local override (GitLab: `git show ${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/…`; + GHA: `git show` of `pull_request.base.sha` or `GITHUB_SHA`), else +2. SHA-pinned `eval/measurements/${AGENT}.yaml` from public `fullsend-ai/agents` + (same `v0` → commit SHA, allowlist, hash, and fetch audit as harness + fallback — not a floating `raw.githubusercontent.com/.../v0/...` curl). + GitHub Actions injects `GH_TOKEN` for that `GetRef`. GitLab CI has no + GitHub token by default; because `agents` is public, `GetRef` still runs + unauthenticated (~60 req/hr per IP). On busy shared runners, export + `GH_TOKEN` / `GITHUB_TOKEN` to avoid rate-limit skips. + +Managed GHA/GitLab scaffolds deliberately **do not** pass `--fullsend-dir` into +`eval-measure`: that flag would prefer `${FULLSEND_DIR}/eval/measurements/` +from the checked-out MR/PR working tree and let an author change which +already-shipped scorers run (or their `id@version`) for that job's trend — +unlike kill-switch/role config, which already reads the default/base tip. +Local/dev invocations may still pass `--fullsend-dir` when you intentionally +want the working-tree override. + +Step 2 is how stock-agent defaults reach every install. Step 1 is org override +on the **default/base branch** only (or an explicit `--registry` path). + +Platform telemetry is `run-telemetry.jsonl` at the top of the host run +directory (`agent---` under the CI output base). Nested +`iteration-N/output/run-telemetry.jsonl` copies and leftover sibling runDirs +are ignored. + +Missing manifest or telemetry → log and exit `0` (skip). `--registry` and +`--telemetry` remain for local/debug use. + +## CLI + +```bash +fullsend eval-measure \ + --agent review \ + --fullsend-dir "${FULLSEND_DIR}" \ + --output-dir path/to/output +``` + +- `--agent` + `--output-dir` is the managed-job form. `--registry` / + `--telemetry` remain for pointing at explicit files. +- `--offline` rejects the remote `agents@v0` fetch (local FULLSEND_DIR + manifest only), matching `fullsend run --offline`. +- Exit `0` when a score is `fail` — scores are data. +- Exit `0` when telemetry or the manifest is missing (skip). + +## Implementation note + +Today the measure CLI writes local `eval-measurements.jsonl` whenever at +least one new measurement row is appended (including `label: skip`). No +file is written when telemetry/manifest is missing, no traces match, or +every candidate row is already in the ledger. + +> **Planned:** portable OTLP score export (same `OTEL_*` as traces) is the +> ADR 0087 remote contract and is not wired yet. Until it lands, consume the +> JSONL artifact (or your own pipeline) for remote dashboards. diff --git a/docs/problems/operational-observability.md b/docs/problems/operational-observability.md index cacd55639b..92a491ff6e 100644 --- a/docs/problems/operational-observability.md +++ b/docs/problems/operational-observability.md @@ -191,8 +191,8 @@ This works for early experimentation when the volume is low and the operators ar - ~~How do we keep the triggering event and the agent run linked in the GitHub Actions UI for debugging?~~ Decided in [ADR 0041](../ADRs/0041-synchronous-workflow-call-event-dispatch.md) (synchronous `workflow_call` dispatch for the event path). - How should trace access be controlled? (JSONL trace exposure decided in [ADR 0021](../ADRs/0021-jsonl-reasoning-trace-exposure.md): owner-scoped storage with credential scanning as defense-in-depth. Broader question of balancing security and transparency for non-JSONL observability data remains open.) - What retention policy applies to traces? Indefinite retention supports audit requirements but increases storage cost and data sensitivity exposure. Time-bounded retention (e.g., 90 days) limits exposure but may lose traces needed for incident investigation. -- How do we measure "is the system getting better"? What metrics constitute a meaningful quality signal for an autonomous software factory? Merge revert rate? Human override rate? Time-to-review? Cost per decision? Some composite score? The choice of metric shapes what gets optimized. -- At what scale does a dedicated LLM observability platform justify its operational overhead (Postgres, ClickHouse, Redis, S3 for something like Langfuse)? Is there a threshold of agent activity below which structured logging suffices? +- How do we measure "is the system getting better"? What metrics constitute a meaningful quality signal for an autonomous software factory? Merge revert rate? Human override rate? Time-to-review? Cost per decision? Some composite score? The choice of metric shapes what gets optimized. First-ship trend scores (trace fitness on wild runs, local `eval-measurements.jsonl`) are [ADR 0087](../ADRs/0087-eval-measurements-online-trace-scoring.md); richer quality signals remain open. +- At what scale does a dedicated LLM observability platform justify its operational overhead (Postgres, ClickHouse, Redis, S3 for something like Langfuse)? Is there a threshold of agent activity below which structured logging suffices? Score files are local JSONL ([ADR 0087](../ADRs/0087-eval-measurements-online-trace-scoring.md)); remote scores reuse `OTEL_EXPORTER_OTLP_*` when implemented. Platform choice remains open. - ~~How do we handle the bootstrapping problem — the factory needs observability to improve, but building the observability infrastructure is itself work that competes with building the factory?~~ Decided in [ADR 0050](../ADRs/0050-distributed-tracing-instrumentation.md): zero-configuration baseline (local JSONL + summary files) eliminates infrastructure requirements for initial observability; OTLP export adds backends when the org is ready. - Should observability data feed back into agent instructions automatically (e.g., auto-adjusting prompts when false positive rates exceed a threshold), or should it only inform human-driven instruction changes? Automatic feedback creates the risk of instruction oscillation; human-only feedback is slower but more controlled. - How do we build community dashboards that are useful to contributors with different levels of technical depth — from "is the agent doing a good job on my repo" to "show me the trace of this specific review"? diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go new file mode 100644 index 0000000000..e3db77ed4f --- /dev/null +++ b/internal/cli/evalmeasure.go @@ -0,0 +1,263 @@ +package cli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/fullsend-ai/fullsend/internal/config" + "github.com/fullsend-ai/fullsend/internal/evalmeasure" + "github.com/fullsend-ai/fullsend/internal/fetch" + "github.com/fullsend-ai/fullsend/internal/forge" + gh "github.com/fullsend-ai/fullsend/internal/forge/github" + "github.com/fullsend-ai/fullsend/internal/harness" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +func newEvalMeasureCmd() *cobra.Command { + var ( + telemetryPath string + registryPath string + outDir string + outputDir string + agent string + fullsendDir string + offline bool + ) + + cmd := &cobra.Command{ + Use: "eval-measure", + Short: "Score agent run traces with eval measurements", + Long: `Parse run-telemetry.jsonl, score with an agents measurement manifest, +and write eval-measurements.jsonl beside the telemetry artifact. + +The binary resolves the manifest (local FULLSEND_DIR override, else a +SHA-pinned fetch from fullsend-ai/agents — same pin, allowlist, hash, and +audit as harness fallback). Platform telemetry is the file at the top of +each run directory; nested iteration-N/output/ copies are ignored. + +Remote backends are not selected by fullsend: scores are a portable local +JSONL artifact. When portable OTLP score export lands, it will reuse the +same OTEL_EXPORTER_OTLP_* configuration as agent traces (ADR 0050 / 0087). + +Exit 0 when scores fail — measurements are data, not gates. Non-zero only +on hard IO/parse errors. Missing telemetry or manifest is a skip (exit 0).`, + RunE: func(cmd *cobra.Command, args []string) error { + printer := ui.New(cmd.OutOrStdout()) + printer.Header("Eval Measure") + + results, skipped, err := runEvalMeasure(cmd.Context(), printer, evalMeasureOpts{ + telemetryPath: telemetryPath, + registryPath: registryPath, + outDir: outDir, + outputDir: outputDir, + agent: agent, + fullsendDir: fullsendDir, + offline: offline, + }) + if err != nil { + printMeasurementResults(printer, results, false) + return err + } + if skipped { + return nil + } + printMeasurementResults(printer, results, true) + return nil + }, + } + + cmd.Flags().StringVar(&telemetryPath, "telemetry", "", "path to run-telemetry.jsonl (or use --output-dir)") + cmd.Flags().StringVar(®istryPath, "registry", "", "path to agents measurement manifest YAML (or use --agent)") + cmd.Flags().StringVar(&outDir, "out-dir", "", "directory for eval-measurements.jsonl (default: telemetry directory)") + cmd.Flags().StringVar(&outputDir, "output-dir", "", "CI output base or runDir; scores only top-of-runDir telemetry") + cmd.Flags().StringVar(&agent, "agent", "", "agent name for manifest resolution") + cmd.Flags().StringVar(&fullsendDir, "fullsend-dir", "", "path to the .fullsend directory (local manifest override + fetch cache)") + cmd.Flags().BoolVar(&offline, "offline", false, "reject network fetches; only use a local FULLSEND_DIR measurement manifest") + return cmd +} + +type evalMeasureOpts struct { + telemetryPath string + registryPath string + outDir string + outputDir string + agent string + fullsendDir string + offline bool +} + +func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOpts) ([]evalmeasure.EvaluationResult, bool, error) { + telemPaths, err := resolveEvalMeasureTelemetry(opts) + if err != nil { + return nil, false, err + } + if len(telemPaths) == 0 { + printer.StepInfo("No platform run-telemetry.jsonl at the top of a run directory; skipping eval measurements") + return nil, true, nil + } + + registry, err := resolveEvalMeasureRegistry(ctx, printer, opts) + if err != nil { + return nil, false, err + } + if registry == "" { + printer.StepInfo("No eval measurement manifest; skipping") + return nil, true, nil + } + + var all []evalmeasure.EvaluationResult + for _, p := range telemPaths { + results, stats, err := evalmeasure.MeasureAndExport(ctx, p, registry, opts.outDir) + if stats.Incomplete != "" { + printer.StepWarn(fmt.Sprintf("%s: telemetry parse incomplete (%s); scored available traces", p, stats.Incomplete)) + } + if stats.SkippedLines > 0 { + printer.StepWarn(fmt.Sprintf("%s: skipped %d unreadable of %d telemetry line(s)", p, stats.SkippedLines, stats.NonEmptyLines)) + } + if stats.SkippedSpans > 0 { + printer.StepWarn(fmt.Sprintf("%s: skipped %d unreadable span(s) inside otherwise-valid telemetry line(s)", p, stats.SkippedSpans)) + } + if err != nil { + return append(all, results...), false, err + } + all = append(all, results...) + } + return all, false, nil +} + +func resolveEvalMeasureTelemetry(opts evalMeasureOpts) ([]string, error) { + if opts.telemetryPath != "" { + return []string{opts.telemetryPath}, nil + } + if opts.outputDir != "" { + agent := "" + if opts.agent != "" { + a, err := sanitizeMeasurementAgentName(opts.agent) + if err != nil { + return nil, err + } + agent = a + } + return evalmeasure.FindPlatformTelemetry(opts.outputDir, agent) + } + return nil, fmt.Errorf("either --telemetry or --output-dir is required") +} + +func resolveEvalMeasureRegistry(ctx context.Context, printer *ui.Printer, opts evalMeasureOpts) (string, error) { + if opts.registryPath != "" { + return opts.registryPath, nil + } + if opts.agent == "" { + return "", fmt.Errorf("either --registry or --agent is required") + } + agent, err := sanitizeMeasurementAgentName(opts.agent) + if err != nil { + return "", err + } + // --fullsend-dir prefers a working-tree override. Managed CI scaffolds + // must not pass the MR/PR checkout here (trend-poisoning); they pass + // --registry from the default/base tip or omit both and fetch agents@v0. + if opts.fullsendDir != "" { + local, err := localMeasurementManifest(opts.fullsendDir, agent) + if err != nil { + return "", err + } + if st, err := os.Stat(local); err == nil && !st.IsDir() { + printer.StepInfo("Using local measurement manifest " + local) + return local, nil + } + } + composeOpts, client := evalMeasureFetchContext(opts.fullsendDir, opts.offline, printer) + path, ok := tryAgentsRepoMeasurementManifest(ctx, agent, client, composeOpts, printer) + if !ok { + return "", nil + } + return path, nil +} + +func sanitizeMeasurementAgentName(agent string) (string, error) { + a := strings.ToLower(strings.TrimSpace(agent)) + if a == "" || strings.ContainsAny(a, `/\`) || strings.Contains(a, "..") { + return "", fmt.Errorf("invalid --agent name %q", agent) + } + return a, nil +} + +func localMeasurementManifest(fullsendDir, agent string) (string, error) { + rel := filepath.Join("eval", "measurements", agent+".yaml") + resolved := filepath.Clean(filepath.Join(fullsendDir, rel)) + if r, err := filepath.Rel(fullsendDir, resolved); err != nil || !filepath.IsLocal(r) { + return "", fmt.Errorf("agent name %q escapes fullsend directory", agent) + } + return resolved, nil +} + +func evalMeasureFetchContext(fullsendDir string, offline bool, printer *ui.Printer) (harness.ComposeOpts, forge.Client) { + workspace := fullsendDir + if workspace == "" { + // Prefer a per-user cache dir over shared os.TempDir() (sticky, + // multi-user /tmp races and predictable paths). Fall back to a + // process-private temp dir if UserCacheDir is unavailable. + if cache, err := os.UserCacheDir(); err == nil && cache != "" { + workspace = filepath.Join(cache, "fullsend", "eval-measure") + } else { + workspace = filepath.Join(os.TempDir(), fmt.Sprintf("fullsend-evalmeasure-%d", os.Getpid())) + } + if err := os.MkdirAll(workspace, 0o700); err != nil && printer != nil { + printer.StepWarn("Could not create eval-measure cache dir: " + err.Error()) + } + } + abs, err := filepath.Abs(workspace) + if err != nil { + abs = workspace + } + orgAllowlist := config.DefaultAllowedRemoteResources() + if fullsendDir != "" && printer != nil { + if orgCfg := tryLoadOrgConfig(filepath.Join(abs, "config.yaml"), printer); orgCfg != nil { + orgAllowlist = orgCfg.AllowedResources() + } + } + token, err := resolveToken() + if (err != nil || token == "") && printer != nil && !offline { + printer.StepWarn("No GH_TOKEN/GITHUB_TOKEN; agents@v0 GetRef runs unauthenticated (public repo, ~60 req/hr per IP). Prefer a token on shared runners; local FULLSEND_DIR override skips the fetch.") + } + policy := fetch.DefaultPolicy + if offline { + policy.Offline = true + } + return harness.ComposeOpts{ + WorkspaceRoot: abs, + FetchPolicy: policy, + AuditLogPath: filepath.Join(abs, ".fullsend-cache", "fetch-audit.jsonl"), + OrgAllowlist: orgAllowlist, + GitToken: token, + }, gh.New(token) +} + +func printMeasurementResults(printer *ui.Printer, results []evalmeasure.EvaluationResult, wroteOK bool) { + if len(results) == 0 { + if wroteOK { + printer.StepDone("No new measurements (already scored or no matching traces)") + } + return + } + for _, r := range results { + line := fmt.Sprintf("%s %s=%.2f (%s) %s", r.Version, r.Name, r.Value, r.Label, r.Explanation) + switch r.Label { + case evalmeasure.LabelPass: + printer.StepDone(line) + case evalmeasure.LabelSkip: + printer.StepInfo(line) + default: + printer.StepWarn(line) + } + } + if wroteOK { + printer.StepDone(fmt.Sprintf("Wrote %d measurement(s)", len(results))) + } +} diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go new file mode 100644 index 0000000000..b3e1321a9e --- /dev/null +++ b/internal/cli/evalmeasure_test.go @@ -0,0 +1,444 @@ +package cli + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/fullsend-ai/fullsend/internal/evalmeasure" + "github.com/fullsend-ai/fullsend/internal/telemetry" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { + out := t.TempDir() + telemetryPath := filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl") + registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telemetryPath, + "--registry", registry, + "--out-dir", out, + }) + err := cmd.Execute() + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(out, "eval-measurements.jsonl")) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) + assert.Contains(t, buf.String(), "Wrote 1 measurement(s)") +} + +func TestEvalMeasureCmd_HasOfflineFlag(t *testing.T) { + cmd := newEvalMeasureCmd() + f := cmd.Flags().Lookup("offline") + require.NotNil(t, f) + assert.Equal(t, "false", f.DefValue) +} + +func TestRootCommand_HasEvalMeasureSubcommand(t *testing.T) { + cmd := newRootCmd() + found := false + for _, sub := range cmd.Commands() { + if sub.Use == "eval-measure" { + found = true + break + } + } + assert.True(t, found, "expected eval-measure subcommand") +} + +func TestEvalMeasureCmd_MissingRequiredFlags(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"eval-measure"}) + err := cmd.Execute() + require.Error(t, err) +} + +func TestEvalMeasureCmd_OutputDirIgnoresNestedTelemetry(t *testing.T) { + fsDir := t.TempDir() + outBase := t.TempDir() + runDir := filepath.Join(outBase, "agent-triage-1-1") + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(runDir, telemetry.TelemetryFile), good, 0o644)) + // Agent-planted copy: valid JSONL with a different trace id would produce + // a second row if scored. Nested path must be ignored. + planted := bytes.ReplaceAll(good, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("ffffffffffffffffffffffffffffffff")) + require.NoError(t, os.WriteFile(filepath.Join(nested, telemetry.TelemetryFile), planted, 0o644)) + + reg, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml")) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), reg, 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--fullsend-dir", fsDir, + "--output-dir", outBase, + }) + require.NoError(t, cmd.Execute()) + + b, err := os.ReadFile(filepath.Join(runDir, evalmeasure.MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"trace_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`) + assert.NotContains(t, string(b), "ffffffffffffffffffffffffffffffff") +} + +// TestEvalMeasureCmd_LocalFullsendDirManifestProducesJSONL locks the +// resolved-manifest path used before agents@v0 carries stock YAML: a local +// FULLSEND_DIR eval/measurements/.yaml must produce eval-measurements.jsonl. +func TestEvalMeasureCmd_LocalFullsendDirManifestProducesJSONL(t *testing.T) { + fsDir := t.TempDir() + outBase := t.TempDir() + runDir := filepath.Join(outBase, "agent-triage-2-2") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(runDir, telemetry.TelemetryFile), good, 0o644)) + + reg, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml")) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), reg, 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--fullsend-dir", fsDir, + "--output-dir", outBase, + "--offline", + }) + require.NoError(t, cmd.Execute()) + + b, err := os.ReadFile(filepath.Join(runDir, evalmeasure.MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) + assert.Contains(t, buf.String(), "Wrote") +} + +func writeTwoTraceTelemetry(t *testing.T) string { + t.Helper() + src, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + src = bytes.TrimSpace(src) + second := bytes.ReplaceAll(src, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")) + p := filepath.Join(t.TempDir(), "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(p, append(append(src, '\n'), second...), 0o644)) + return p +} + +func TestRunEvalMeasure_ErrorIncludesPartialResults(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t) + registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") + ctx := evalmeasure.WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, evalmeasure.MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + var buf bytes.Buffer + results, skipped, err := runEvalMeasure(ctx, ui.New(&buf), evalMeasureOpts{ + telemetryPath: telem, + registryPath: registry, + outDir: out, + }) + require.Error(t, err) + assert.False(t, skipped) + require.Len(t, results, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) + + printMeasurementResults(ui.New(&buf), results, false) + assert.Contains(t, buf.String(), "trace_fitness") + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_ErrorPrintsPartialFromFailingFile(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t) + ctx := evalmeasure.WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, evalmeasure.MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + + cmd := newRootCmd() + cmd.SetContext(ctx) + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, buf.String(), "trace_fitness") + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_ErrorDoesNotPrintWrote(t *testing.T) { + out := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(out, []byte("x"), 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl"), + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + err := cmd.Execute() + require.Error(t, err) + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_SkipWhenNoTelemetry(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--output-dir", t.TempDir(), + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "skipping eval measurements") + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_WarnsOnCorruptTelemetryLine(t *testing.T) { + out := t.TempDir() + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + telem := filepath.Join(out, "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(telem, append(append([]byte{}, good...), []byte("\nnot-json\n")...), 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "skipped 1 unreadable of 2 telemetry line") +} + +func TestEvalMeasureCmd_WarnsOnIncompleteParse(t *testing.T) { + out := t.TempDir() + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + telem := filepath.Join(out, "run-telemetry.jsonl") + f, err := os.Create(telem) + require.NoError(t, err) + _, err = f.Write(append(bytes.TrimSpace(good), '\n')) + require.NoError(t, err) + huge := make([]byte, 11*1024*1024) + for i := range huge { + huge[i] = 'a' + } + _, err = f.Write(huge) + require.NoError(t, err) + _, err = f.Write([]byte("\n")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "telemetry parse incomplete") + assert.Contains(t, buf.String(), "scored available traces") + assert.Contains(t, buf.String(), "Wrote") +} + +func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { + b, err := os.ReadFile(filepath.Join("..", "..", "action.yml")) + require.NoError(t, err) + s := string(b) + assert.NotContains(t, s, "raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements") + assert.NotContains(t, s, `find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl`) + idx := strings.Index(s, "name: Eval measurements") + require.Greater(t, idx, 0) + step := s[idx:] + if end := strings.Index(step, "name: Upload fullsend artifacts"); end > 0 { + step = step[:end] + } + assert.Contains(t, step, "eval-measure") + assert.Contains(t, step, "--output-dir") + assert.Contains(t, step, "--agent") + assert.Contains(t, step, "continue-on-error: true") + assert.Contains(t, step, "if: always()") + assert.Contains(t, step, "GH_TOKEN:") + assert.Contains(t, step, "PR_BASE_SHA:") + assert.Contains(t, step, "--registry") + assert.Contains(t, step, "FULLSEND_DIR:") + assert.Contains(t, step, "MEASURE_REL") + assert.NotContains(t, step, "--fullsend-dir") + assert.NotContains(t, step, "curl ") + assert.NotContains(t, step, "Authorization: Bearer") +} + +func TestPlatformTelemetryFileMatchesRecorder(t *testing.T) { + assert.Equal(t, telemetry.TelemetryFile, evalmeasure.PlatformTelemetryFile) +} + +func TestEvalMeasureFetchContext(t *testing.T) { + printer := ui.New(io.Discard) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + + opts, client := evalMeasureFetchContext("", false, printer) + require.NotNil(t, client) + assert.NotEmpty(t, opts.OrgAllowlist) + assert.NotEmpty(t, opts.WorkspaceRoot) + assert.False(t, opts.FetchPolicy.Offline) + assert.NotEqual(t, filepath.Clean(os.TempDir()), filepath.Clean(opts.WorkspaceRoot), + "empty --fullsend-dir must not use shared os.TempDir() as WorkspaceRoot") + if cache, err := os.UserCacheDir(); err == nil && cache != "" { + assert.Equal(t, filepath.Join(cache, "fullsend", "eval-measure"), opts.WorkspaceRoot) + } + + dir := t.TempDir() + opts2, _ := evalMeasureFetchContext(dir, true, printer) + assert.Contains(t, opts2.AuditLogPath, ".fullsend-cache") + abs, err := filepath.Abs(dir) + require.NoError(t, err) + assert.Equal(t, abs, opts2.WorkspaceRoot) + assert.True(t, opts2.FetchPolicy.Offline) +} + +func TestResolveEvalMeasureRegistry_LocalOverride(t *testing.T) { + fsDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + local := filepath.Join(fsDir, "eval", "measurements", "triage.yaml") + require.NoError(t, os.WriteFile(local, []byte("agent: triage\n"), 0o644)) + got, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "triage", + fullsendDir: fsDir, + }) + require.NoError(t, err) + assert.Equal(t, local, got) +} + +func TestResolveEvalMeasureRegistry_UnknownAgentSkipsRemote(t *testing.T) { + got, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "not-a-stock-agent", + fullsendDir: t.TempDir(), + }) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestResolveEvalMeasureRegistry_RequiresAgentOrRegistry(t *testing.T) { + _, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--registry or --agent") +} + +func TestSanitizeMeasurementAgentName(t *testing.T) { + got, err := sanitizeMeasurementAgentName(" Triage ") + require.NoError(t, err) + assert.Equal(t, "triage", got) + + for _, bad := range []string{"", " ", "../evil", "foo/bar", `foo\bar`, "a..b"} { + _, err := sanitizeMeasurementAgentName(bad) + require.Error(t, err, "agent %q", bad) + } +} + +func TestLocalMeasurementManifestStaysUnderFullsendDir(t *testing.T) { + fsDir := t.TempDir() + got, err := localMeasurementManifest(fsDir, "triage") + require.NoError(t, err) + assert.Equal(t, filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), got) +} + +func TestResolveEvalMeasureRegistry_RejectsPathAgent(t *testing.T) { + _, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "../../../tmp/evil", + fullsendDir: t.TempDir(), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --agent") +} + +func TestEvalMeasureCmd_TelemetryWithoutRegistry(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl"), + }) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "--registry or --agent") +} + +func TestPrintMeasurementResults_SkipAndNoWroteOnError(t *testing.T) { + var buf bytes.Buffer + p := ui.New(&buf) + printMeasurementResults(p, []evalmeasure.EvaluationResult{{ + Name: "trace_fitness", + Version: "em-001@1", + Label: evalmeasure.LabelSkip, + }}, true) + assert.Contains(t, buf.String(), "skip") + assert.Contains(t, buf.String(), "Wrote 1 measurement(s)") + + buf.Reset() + printMeasurementResults(p, []evalmeasure.EvaluationResult{{ + Name: "trace_fitness", + Version: "em-001@1", + Label: evalmeasure.LabelFail, + }}, false) + assert.NotContains(t, buf.String(), "Wrote") +} diff --git a/internal/cli/root.go b/internal/cli/root.go index faf9f1976a..6b9a567cc0 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -55,6 +55,7 @@ func newRootCmd() *cobra.Command { cmd.AddCommand(newPostCommentCmd()) cmd.AddCommand(newReconcileStatusCmd()) cmd.AddCommand(newPollCmd()) + cmd.AddCommand(newEvalMeasureCmd()) return cmd } diff --git a/internal/cli/run.go b/internal/cli/run.go index d84d32851d..3e4f7eb053 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -27,6 +27,7 @@ import ( "github.com/fullsend-ai/fullsend/internal/binary" "github.com/fullsend-ai/fullsend/internal/config" "github.com/fullsend-ai/fullsend/internal/envfile" + "github.com/fullsend-ai/fullsend/internal/evalmeasure" "github.com/fullsend-ai/fullsend/internal/fetch" "github.com/fullsend-ai/fullsend/internal/fetchsvc" "github.com/fullsend-ai/fullsend/internal/forge" @@ -101,9 +102,10 @@ var defaultAgentsRepoKnownAgents = map[string]bool{ // runtime tokens). Tests that override it affect both paths. var statusMintToken = mintclient.MintToken -// agentWorkingDirExcludes lists directory patterns that agents may create -// during execution but must never commit. These are added to -// .git/info/exclude before the agent runs so git ignores them entirely. +// agentWorkingDirExcludes lists fullsend-reserved directory patterns that +// must stay out of commits. They are appended to .git/info/exclude before +// the agent runs. Host run output (often named output/) is excluded only +// when it actually sits inside --target-repo — see outputDirExcludeRel. var agentWorkingDirExcludes = []string{ ".agentready/", ".fullsend-workspace/", @@ -897,7 +899,9 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep workItemID := resolveWorkItemID() // 3. Create run directory and initialise tracer. - sandboxName := fmt.Sprintf("agent-%s-%d-%d", agentName, os.Getpid(), time.Now().Unix()) + // Lowercase the agent segment so eval-measure --agent (also lowercased) + // matches the host runDir even when the CLI arg was mixed-case. + sandboxName := fmt.Sprintf("agent-%s-%d-%d", strings.ToLower(agentName), os.Getpid(), time.Now().Unix()) if outputBase == "" { outputBase = filepath.Join(os.TempDir(), "fullsend") } @@ -1239,9 +1243,19 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep printer.StepDone(fmt.Sprintf("Sandbox bootstrapped (%.1fs)", time.Since(bootstrapStart).Seconds())) // 8. Make project code available (copy repo root into a named subdirectory). + // When --output-dir sits inside --target-repo (GitLab CI layout), omit that + // top-level directory from the tarball and git exclude so host telemetry + // is not uploaded or committed. GHA keeps output as a sibling, so Rel + // fails IsLocal and nothing is excluded. copyStart := time.Now() printer.StepStart("Copying project code into sandbox") - if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir); err != nil { + var uploadExcludes []string + var gitExtraExcludes []string + if rel, ok := outputDirExcludeRel(hostRepositoryDir, outputBase); ok { + uploadExcludes = append(uploadExcludes, rel) + gitExtraExcludes = append(gitExtraExcludes, rel+"/") + } + if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir, uploadExcludes...); err != nil { printer.StepFail("Failed to copy project code") return fmt.Errorf("copying project code: %w", err) } @@ -1283,7 +1297,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // Agents may create working directories (e.g. .agentready/) during // execution. These must never appear in commits. Adding them to // .git/info/exclude ensures git status/add ignores them entirely. - if err := excludeAgentWorkingDirs(sandboxName, remoteRepositoryDir, printer); err != nil { + if err := excludeAgentWorkingDirs(sandboxName, remoteRepositoryDir, gitExtraExcludes, printer); err != nil { printer.StepWarn("Could not exclude agent working dirs: " + err.Error()) } @@ -2401,7 +2415,15 @@ func resolveWorkItemID() string { if prNum := strings.TrimSpace(os.Getenv("PR_NUMBER")); prNum != "" { return prNum } - return "unknown" + // GitHub retro: reusable-retro.yml sets ORIGINATING_URL (PR/issue HTML URL). + // GitLab agent jobs export GITLAB_ISSUE_URL (issue or MR) when IID is known. + if v := strings.TrimSpace(os.Getenv("ORIGINATING_URL")); v != "" { + return v + } + if v := strings.TrimSpace(os.Getenv("GITLAB_ISSUE_URL")); v != "" { + return v + } + return evalmeasure.UnknownSentinel } // telemetryExitCode maps the run's final state to the exit code recorded on @@ -3028,13 +3050,43 @@ func relOrAbs(base, path string) string { return rel } +// outputDirExcludeRel returns the top-level directory name to omit from the +// sandbox upload when outputBase is inside hostRepositoryDir. Only single- +// segment relative paths are returned so nested names like build/output are +// not handled by dropping an entire parent tree (GitLab uses top-level +// output/). Returns ok=false when output is a sibling of the checkout +// (GitHub Actions layout), multi-segment, or otherwise outside the repo. +func outputDirExcludeRel(hostRepositoryDir, outputBase string) (string, bool) { + if hostRepositoryDir == "" || outputBase == "" { + return "", false + } + absRepo, err := filepath.Abs(hostRepositoryDir) + if err != nil { + return "", false + } + absOut, err := filepath.Abs(outputBase) + if err != nil { + return "", false + } + rel, err := filepath.Rel(absRepo, absOut) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return "", false + } + if strings.ContainsRune(rel, os.PathSeparator) { + return "", false + } + return rel, true +} + // excludeAgentWorkingDirs adds agent working directory patterns to // .git/info/exclude so they are invisible to git status and git add. -func excludeAgentWorkingDirs(sandboxName, repoDir string, printer *ui.Printer) error { +// extra holds layout-specific patterns (e.g. host output/ when nested). +func excludeAgentWorkingDirs(sandboxName, repoDir string, extra []string, printer *ui.Printer) error { var lines []string for _, pattern := range agentWorkingDirExcludes { lines = append(lines, pattern) } + lines = append(lines, extra...) if len(lines) == 0 { return nil } @@ -3813,11 +3865,37 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo if !defaultAgentsRepoKnownAgents[normalizedName] { return "", nil, false } - if composeOpts.FetchPolicy.Offline { + path, dep, ok := fetchPinnedAgentsRepoFile(ctx, "harness/"+normalizedName+".yaml", forgeClient, composeOpts, printer, "agent "+agentName) + if !ok { return "", nil, false } + return path, []harness.Dependency{dep}, true +} + +// tryAgentsRepoMeasurementManifest SHA-pins eval/measurements/.yaml +// from fullsend-ai/agents (same pin, allowlist, hash, and audit as harness +// fallback). Missing manifests (HTTP 404) skip; network errors warn. +func tryAgentsRepoMeasurementManifest(ctx context.Context, agentName string, forgeClient forge.Client, composeOpts harness.ComposeOpts, printer *ui.Printer) (string, bool) { + normalizedName := strings.ToLower(agentName) + if !defaultAgentsRepoKnownAgents[normalizedName] { + return "", false + } + path, _, ok := fetchPinnedAgentsRepoFile(ctx, "eval/measurements/"+normalizedName+".yaml", forgeClient, composeOpts, printer, "eval measurement manifest for "+agentName) + return path, ok +} + +// fetchPinnedAgentsRepoFile resolves tags/DefaultUpstreamRef to a commit SHA +// and fetches relPath from fullsend-ai/agents. All errors are non-fatal. +func fetchPinnedAgentsRepoFile(ctx context.Context, relPath string, forgeClient forge.Client, composeOpts harness.ComposeOpts, printer *ui.Printer, noun string) (string, harness.Dependency, bool) { + var none harness.Dependency + if strings.Contains(relPath, "..") || strings.HasPrefix(relPath, "/") { + return "", none, false + } + if composeOpts.FetchPolicy.Offline { + return "", none, false + } if forgeClient == nil { - return "", nil, false + return "", none, false } allowlist := composeOpts.OrgAllowlist @@ -3826,30 +3904,34 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo tagSHA, err := forgeClient.GetRef(ctx, defaultAgentsRepoOwner, defaultAgentsRepoName, tagRef) if err != nil { printer.StepWarn(fmt.Sprintf("Could not resolve %s/%s@%s: %v", defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef, err)) - return "", nil, false + return "", none, false } if !commitSHAPattern.MatchString(tagSHA) { printer.StepWarn(fmt.Sprintf("Invalid SHA from %s/%s@%s: %q", defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef, tagSHA)) - return "", nil, false + return "", none, false } - rawURL := defaultAgentsRepoURLPrefix + tagSHA + "/harness/" + normalizedName + ".yaml" + rawURL := defaultAgentsRepoURLPrefix + tagSHA + "/" + relPath if harness.MatchingAllowedPrefixInList(rawURL, allowlist) == "" { - printer.StepWarn(fmt.Sprintf("Agents repo fallback skipped for %s: URL not in allowed_remote_resources", agentName)) - return "", nil, false + printer.StepWarn(fmt.Sprintf("Agents repo fallback skipped for %s: URL not in allowed_remote_resources", noun)) + return "", none, false } shortSHA := tagSHA if len(shortSHA) > 12 { shortSHA = shortSHA[:12] } - printer.StepStart(fmt.Sprintf("Fetching agent %s from %s/%s@%s", agentName, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) + printer.StepStart(fmt.Sprintf("Fetching %s from %s/%s@%s", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) content, err := fetch.FetchURL(ctx, rawURL, composeOpts.FetchPolicy) if err != nil { - printer.StepWarn(fmt.Sprintf("Failed to fetch agent %s from agents repo: %v", agentName, err)) - return "", nil, false + if isFetchHTTPStatus(err, http.StatusNotFound) { + printer.StepInfo(fmt.Sprintf("No %s at %s/%s@%s (HTTP 404); skipping", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) + } else { + printer.StepWarn(fmt.Sprintf("Failed to fetch %s from agents repo: %v", noun, err)) + } + return "", none, false } // Content is fetched once and used directly — no self-referential hash @@ -3860,13 +3942,13 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo if err := fetch.CachePut(composeOpts.WorkspaceRoot, rawURL, content); err != nil { printer.StepWarn(fmt.Sprintf("Failed to cache agents repo content: %v", err)) - return "", nil, false + return "", none, false } cachePath, err := fetch.CachePath(composeOpts.WorkspaceRoot, contentHash) if err != nil { - printer.StepWarn(fmt.Sprintf("Failed to resolve cache path for agent %s: %v", agentName, err)) - return "", nil, false + printer.StepWarn(fmt.Sprintf("Failed to resolve cache path for %s: %v", noun, err)) + return "", none, false } localPath := filepath.Join(cachePath, "content") @@ -3893,8 +3975,13 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo Type: "file", } - printer.StepDone(fmt.Sprintf("Agent %s resolved from %s/%s@%s", agentName, defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef)) - return localPath, []harness.Dependency{dep}, true + printer.StepDone(fmt.Sprintf("%s resolved from %s/%s@%s", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef)) + return localPath, dep, true +} + +func isFetchHTTPStatus(err error, code int) bool { + var httpErr fetch.HTTPStatusError + return errors.As(err, &httpErr) && httpErr.Status == code } // containedLocalPath resolves a relative source path against baseDir and diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index b6e5424dd2..fba9cf0a60 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1354,6 +1354,133 @@ func TestTryAgentsRepoFallback_SuccessPath(t *testing.T) { assert.NotEmpty(t, deps[0].SHA256) } +func TestTryAgentsRepoMeasurementManifest_Success(t *testing.T) { + manifest := []byte("agent: triage\nmeasurements:\n - id: em-001\n scorer: trace_fitness\n version: 1\n") + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedPath := "/" + fakeSHA + "/eval/measurements/triage.yaml" + if r.URL.Path == expectedPath { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifest) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + printer := ui.New(io.Discard) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + path, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + require.True(t, ok) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, manifest, got) + assert.Contains(t, path, "content") +} + +func TestTryAgentsRepoMeasurementManifest_HTTP404(t *testing.T) { + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + var buf bytes.Buffer + printer := ui.New(&buf) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + assert.False(t, ok) + assert.Contains(t, buf.String(), "HTTP 404") + assert.NotContains(t, buf.String(), "Failed to fetch") +} + +func TestTryAgentsRepoMeasurementManifest_NetworkFailure(t *testing.T) { + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusBadGateway) + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + var buf bytes.Buffer + printer := ui.New(&buf) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + assert.False(t, ok) + assert.Contains(t, buf.String(), "Failed to fetch") + assert.NotContains(t, buf.String(), "HTTP 404") +} + +func TestTryAgentsRepoMeasurementManifest_UnknownAgent(t *testing.T) { + fakeClient := forge.NewFakeClient() + printer := ui.New(io.Discard) + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "custom-agent", fakeClient, harness.ComposeOpts{}, printer) + assert.False(t, ok) +} + +func TestIsFetchHTTPStatus(t *testing.T) { + assert.True(t, isFetchHTTPStatus(fetch.HTTPStatusError{Status: 404}, 404)) + assert.True(t, isFetchHTTPStatus(fmt.Errorf("wrap: %w", fetch.HTTPStatusError{Status: 404}), 404)) + assert.False(t, isFetchHTTPStatus(fetch.HTTPStatusError{Status: 500}, 404)) + assert.False(t, isFetchHTTPStatus(fmt.Errorf("fetch: request failed: connection refused"), 404)) + assert.False(t, isFetchHTTPStatus(nil, 404)) +} + func TestTryAgentsRepoFallback_AuditLog(t *testing.T) { harnessContent := []byte("agent: agents/triage.md\nrole: test\n") fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" @@ -2071,6 +2198,7 @@ func TestValidateLinuxBinary_AcceptsHostBinary(t *testing.T) { func TestAgentWorkingDirExcludes_ContainsKnownPatterns(t *testing.T) { // Verify the exclusion list contains the known agent working directories. + // Host output/ is layout-scoped via outputDirExcludeRel, not this list. expected := []string{".agentready/", ".fullsend-workspace/"} for _, pattern := range expected { found := false @@ -2082,6 +2210,29 @@ func TestAgentWorkingDirExcludes_ContainsKnownPatterns(t *testing.T) { } assert.True(t, found, "agentWorkingDirExcludes should contain %q", pattern) } + for _, exclude := range agentWorkingDirExcludes { + assert.NotEqual(t, "output/", exclude, "output/ must not be hardcoded in agentWorkingDirExcludes") + } +} + +func TestOutputDirExcludeRel(t *testing.T) { + t.Parallel() + repo := t.TempDir() + nested := filepath.Join(repo, "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + rel, ok := outputDirExcludeRel(repo, nested) + assert.True(t, ok) + assert.Equal(t, "output", rel) + + sibling := filepath.Join(filepath.Dir(repo), "output-sibling") + _, ok = outputDirExcludeRel(repo, sibling) + assert.False(t, ok, "sibling output must not be excluded") + + deep := filepath.Join(repo, "build", "output") + require.NoError(t, os.MkdirAll(deep, 0o755)) + _, ok = outputDirExcludeRel(repo, deep) + assert.False(t, ok, "multi-segment Rel must not exclude a whole parent tree") } func TestAgentWorkingDirExcludes_NotEmpty(t *testing.T) { diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index 18280ea09e..07833f450d 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -17,6 +17,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" + "github.com/fullsend-ai/fullsend/internal/evalmeasure" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" "github.com/fullsend-ai/fullsend/internal/security" ) @@ -39,14 +40,16 @@ func TestSecurityTraceID_ShellSafe(t *testing.T) { func TestResolveWorkItemID(t *testing.T) { cases := []struct { - name string - issueKey string - repoFull string - issueNumber string - issueURL string - prURL string - prNumber string - want string + name string + issueKey string + repoFull string + issueNumber string + issueURL string + prURL string + prNumber string + originatingURL string + gitlabIssueURL string + want string }{ { name: "ISSUE_KEY wins over everything", @@ -102,7 +105,23 @@ func TestResolveWorkItemID(t *testing.T) { }, { name: "unknown when nothing is set", - want: "unknown", + want: evalmeasure.UnknownSentinel, + }, + { + name: "ORIGINATING_URL for retro", + originatingURL: "https://github.com/octo/repo/issues/99", + want: "https://github.com/octo/repo/issues/99", + }, + { + name: "GITLAB_ISSUE_URL for GitLab agent jobs", + gitlabIssueURL: "https://gitlab.example/group/proj/-/issues/12", + want: "https://gitlab.example/group/proj/-/issues/12", + }, + { + name: "ORIGINATING_URL beats GITLAB_ISSUE_URL when both set", + originatingURL: "https://github.com/octo/repo/issues/99", + gitlabIssueURL: "https://gitlab.example/group/proj/-/issues/12", + want: "https://github.com/octo/repo/issues/99", }, } for _, tc := range cases { @@ -113,6 +132,8 @@ func TestResolveWorkItemID(t *testing.T) { t.Setenv("GITHUB_ISSUE_URL", tc.issueURL) t.Setenv("GITHUB_PR_URL", tc.prURL) t.Setenv("PR_NUMBER", tc.prNumber) + t.Setenv("ORIGINATING_URL", tc.originatingURL) + t.Setenv("GITLAB_ISSUE_URL", tc.gitlabIssueURL) assert.Equal(t, tc.want, resolveWorkItemID()) }) } diff --git a/internal/evalmeasure/export_local.go b/internal/evalmeasure/export_local.go new file mode 100644 index 0000000000..1ded235fb7 --- /dev/null +++ b/internal/evalmeasure/export_local.go @@ -0,0 +1,83 @@ +package evalmeasure + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + MeasurementsFile = "eval-measurements.jsonl" + LedgerFile = "eval-measure-ledger.txt" +) + +// AppendMeasurements writes one NDJSON EvaluationResult per line. +func AppendMeasurements(path string, results []EvaluationResult) (retErr error) { + if len(results) == 0 { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer func() { + if cerr := f.Close(); retErr == nil { + retErr = cerr + } + }() + enc := json.NewEncoder(f) + for _, r := range results { + if err := enc.Encode(r); err != nil { + return err + } + } + return nil +} + +func ledgerKey(traceID, evalName, version string) string { + return traceID + "|" + evalName + "|" + version +} + +// AlreadyScored reports whether the ledger contains this measurement. +func AlreadyScored(ledgerPath, traceID, evalName, version string) (bool, error) { + f, err := os.Open(ledgerPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + defer f.Close() + want := ledgerKey(traceID, evalName, version) + sc := bufio.NewScanner(f) + for sc.Scan() { + if strings.TrimSpace(sc.Text()) == want { + return true, nil + } + } + return false, sc.Err() +} + +// RecordScored appends a ledger entry. +func RecordScored(ledgerPath, traceID, evalName, version string) (retErr error) { + if err := os.MkdirAll(filepath.Dir(ledgerPath), 0o755); err != nil { + return err + } + f, err := os.OpenFile(ledgerPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer func() { + if cerr := f.Close(); retErr == nil { + retErr = cerr + } + }() + _, err = fmt.Fprintln(f, ledgerKey(traceID, evalName, version)) + return err +} diff --git a/internal/evalmeasure/export_local_test.go b/internal/evalmeasure/export_local_test.go new file mode 100644 index 0000000000..be1f062e69 --- /dev/null +++ b/internal/evalmeasure/export_local_test.go @@ -0,0 +1,67 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAppendMeasurements_EmptySlice(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "out.jsonl") + require.NoError(t, AppendMeasurements(path, nil)) + _, err := os.Stat(path) + assert.True(t, os.IsNotExist(err), "file should not be created for empty slice") +} + +func TestAppendMeasurements_CreatesParentDirs(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "sub", "deep", "out.jsonl") + err := AppendMeasurements(path, []EvaluationResult{{Name: "test", Value: 1}}) + require.NoError(t, err) + b, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"test"`) +} + +func TestAlreadyScored_NonExistentLedger(t *testing.T) { + t.Parallel() + done, err := AlreadyScored("/nonexistent/ledger.jsonl", "trace1", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done) +} + +func TestRecordScored_ThenAlreadyScored(t *testing.T) { + t.Parallel() + dir := t.TempDir() + ledger := filepath.Join(dir, "ledger.jsonl") + + done, err := AlreadyScored(ledger, "trace1", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done) + + require.NoError(t, RecordScored(ledger, "trace1", "eval1", "v1")) + + done, err = AlreadyScored(ledger, "trace1", "eval1", "v1") + require.NoError(t, err) + assert.True(t, done) + + done, err = AlreadyScored(ledger, "trace2", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done, "different trace should not match") +} + +func TestRecordScored_CreatesParentDirs(t *testing.T) { + t.Parallel() + dir := t.TempDir() + ledger := filepath.Join(dir, "sub", "ledger.jsonl") + require.NoError(t, RecordScored(ledger, "trace1", "eval1", "v1")) + b, err := os.ReadFile(ledger) + require.NoError(t, err) + assert.Contains(t, string(b), "trace1|eval1|v1") +} diff --git a/internal/evalmeasure/find.go b/internal/evalmeasure/find.go new file mode 100644 index 0000000000..2ff007f9c9 --- /dev/null +++ b/internal/evalmeasure/find.go @@ -0,0 +1,96 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// PlatformTelemetryFile is the host recorder's JSONL at the top of runDir. +// It matches internal/telemetry.TelemetryFile. Nested copies under +// iteration-N/output/ are agent-writable and must not be scored. +const PlatformTelemetryFile = "run-telemetry.jsonl" + +// hostRunDirPattern matches agent--- from fullsend run. +// name is lowercased when the sandbox is created; charset matches run +// (ToLower only), so pid/unix must be the trailing numeric pair. +var hostRunDirPattern = regexp.MustCompile(`^agent-(.+)-([0-9]+)-([0-9]+)$`) + +// FindPlatformTelemetry returns run-telemetry.jsonl files that sit at the +// top of outputDir itself (when outputDir is a runDir) or at the top of +// a host-created child runDir (when outputDir is the CI output base). +// +// Host runDirs are named agent--- (see fullsend run). +// When agent is non-empty, only children matching that exact shape for +// the lowercased agent name are considered (so agent-code does not match +// agent-code-review-…). If several match, only the newest platform file +// is scored — leftover sibling directories from a previous job are ignored. +// +// Matching child runDirs outrank a root-level run-telemetry.jsonl under +// outputDir so an agent-planted file at the CI output base cannot displace +// the real host recorder. If any matching child runDir exists (even +// without a platform file yet), the root file is ignored. Nested +// iteration-N/output/ copies are never walked. +func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { + wantAgent := strings.ToLower(agent) + paths, sawMatch, err := findChildPlatformTelemetry(outputDir, wantAgent) + if err != nil { + return nil, err + } + if sawMatch { + return paths, nil + } + + direct := filepath.Join(outputDir, PlatformTelemetryFile) + if st, err := os.Stat(direct); err == nil && !st.IsDir() { + // outputDir is a runDir (or has only a root-level file and no + // matching child): score only the platform file at the top. + return []string{direct}, nil + } + return nil, nil +} + +// findChildPlatformTelemetry looks for agent--- children. +// sawMatch is true when at least one directory matched the pattern (and +// agent filter), whether or not PlatformTelemetryFile was present. +func findChildPlatformTelemetry(outputDir, wantAgent string) (paths []string, sawMatch bool, err error) { + entries, err := os.ReadDir(outputDir) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, err + } + var bestPath string + var bestMod time.Time + foundFile := false + for _, e := range entries { + if !e.IsDir() { + continue + } + m := hostRunDirPattern.FindStringSubmatch(e.Name()) + if m == nil { + continue + } + if wantAgent != "" && m[1] != wantAgent { + continue + } + sawMatch = true + p := filepath.Join(outputDir, e.Name(), PlatformTelemetryFile) + st, err := os.Stat(p) + if err != nil || st.IsDir() { + continue + } + if !foundFile || st.ModTime().After(bestMod) { + bestPath = p + bestMod = st.ModTime() + foundFile = true + } + } + if !foundFile { + return nil, sawMatch, nil + } + return []string{bestPath}, sawMatch, nil +} diff --git a/internal/evalmeasure/find_test.go b/internal/evalmeasure/find_test.go new file mode 100644 index 0000000000..d9ceeb9bfb --- /dev/null +++ b/internal/evalmeasure/find_test.go @@ -0,0 +1,157 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindPlatformTelemetry_IgnoresNestedIterationCopy(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + runDir := filepath.Join(outputDir, "agent-review-3311-1") + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + platform := filepath.Join(runDir, PlatformTelemetryFile) + planted := filepath.Join(nested, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_RunDirIgnoresImmediateChild(t *testing.T) { + t.Parallel() + runDir := t.TempDir() + child := filepath.Join(runDir, "planted-child") + require.NoError(t, os.MkdirAll(child, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(child, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(runDir, "") + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_RunDirDirect(t *testing.T) { + t.Parallel() + runDir := t.TempDir() + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(runDir, "") + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_MissingDir(t *testing.T) { + t.Parallel() + got, err := FindPlatformTelemetry(filepath.Join(t.TempDir(), "nope"), "") + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestFindPlatformTelemetry_EmptyWhenOnlyNested(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + nested := filepath.Join(outputDir, "agent-x", "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestFindPlatformTelemetry_IgnoresSiblingLeftoverRunDir(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + leftover := filepath.Join(outputDir, "agent-triage-1-1") + current := filepath.Join(outputDir, "agent-review-9-9") + require.NoError(t, os.MkdirAll(leftover, 0o755)) + require.NoError(t, os.MkdirAll(current, 0o755)) + leftFile := filepath.Join(leftover, PlatformTelemetryFile) + curFile := filepath.Join(current, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(leftFile, []byte("old\n"), 0o644)) + require.NoError(t, os.WriteFile(curFile, []byte("new\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + require.Equal(t, []string{curFile}, got) + + gotAllNewest, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + require.Len(t, gotAllNewest, 1) +} + +func TestFindPlatformTelemetry_DoesNotMatchLongerAgentPrefix(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + codeReview := filepath.Join(outputDir, "agent-code-review-1-1") + require.NoError(t, os.MkdirAll(codeReview, 0o755)) + f := filepath.Join(codeReview, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(f, []byte("x\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "code") + require.NoError(t, err) + assert.Empty(t, got, "agent-code must not match agent-code-review-*") + + gotReview, err := FindPlatformTelemetry(outputDir, "code-review") + require.NoError(t, err) + require.Equal(t, []string{f}, gotReview) +} + +func TestFindPlatformTelemetry_MatchesUnderscorePrefixedAgent(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + runDir := filepath.Join(outputDir, "agent-_helper-7-9") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + f := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(f, []byte("x\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "_helper") + require.NoError(t, err) + require.Equal(t, []string{f}, got) +} + +func TestFindPlatformTelemetry_PrefersRunDirOverPlantedRoot(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + planted := filepath.Join(outputDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + + runDir := filepath.Join(outputDir, "agent-review-3-4") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + require.Equal(t, []string{platform}, got) + + gotAll, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + require.Equal(t, []string{platform}, gotAll) +} + +func TestFindPlatformTelemetry_EmptyMatchingRunDirIgnoresPlantedRoot(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + planted := filepath.Join(outputDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(outputDir, "agent-review-5-6"), 0o755)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + assert.Empty(t, got, "matching empty runDir must not fall back to planted root") +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go new file mode 100644 index 0000000000..cad9393d22 --- /dev/null +++ b/internal/evalmeasure/fitness.go @@ -0,0 +1,262 @@ +package evalmeasure + +import ( + "strings" +) + +const ( + ScorerFitness = "trace_fitness" + LabelPass = "pass" + LabelFail = "fail" + LabelSkip = "skip" + + // UnknownSentinel is the CLI fallback for missing work-item / agent + // identity (resolveWorkItemID). Scorers must treat it as absent. + UnknownSentinel = "unknown" +) + +// ScoreFitness implements EM-001 Trace Fitness against a single trace. +func ScoreFitness(tr Trace) EvaluationResult { + return ScoreFitnessNamed(tr, ScorerFitness, "em-001@1") +} + +// ScoreFitnessNamed is ScoreFitness with explicit evaluation name/version. +func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { + if evalName == "" { + evalName = ScorerFitness + } + if version == "" { + version = "em-001@1" + } + + run, hasRun := tr.SpanByName("run") + if hasRun { + if skipped, ok := run.AttrBool("fullsend.prescript.skipped"); ok && skipped { + spanID := run.SpanID + workItem, _ := run.AttrString(AttrFullsendWorkItemID) + reason := "pre-script skipped run; excluded from trace_fitness" + if r, ok := run.AttrString("fullsend.prescript.skip_reason"); ok && r != "" { + reason = reason + ": " + r + } + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: reason, + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: tr.AgentName(), + Version: version, + } + } + } + agents := tr.SpansByName("agent") + // No agent span: run never reached an iteration (sandbox/provider/image + // failure). Exclude from pass/(pass+fail) so EM-001 trends measure the + // telemetry contract, not runner health. + if len(agents) == 0 { + spanID := "" + workItem := "" + if hasRun { + spanID = run.SpanID + workItem, _ = run.AttrString(AttrFullsendWorkItemID) + } + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: "no agent span; run never reached an iteration; excluded from trace_fitness", + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: tr.AgentName(), + Version: version, + } + } + // Agent spans flushed but root run never ended (SIGKILL / OOM / job + // timeout). Same exclusion: do not score incomplete telemetry as fail. + if !hasRun { + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: "root run span missing; run terminated before flush; excluded from trace_fitness", + TraceID: tr.TraceID, + SpanID: "", + WorkItemID: "", + Agent: tr.AgentName(), + Version: version, + } + } + _, hasSandbox := tr.SpanByName("sandbox_create") + costOK, costMissing := costToolsTurnsDetail(run, agents) + + type check struct { + id string + pass bool + } + checks := []check{ + // len(agents) >= 1 is guaranteed by the early-return guard above. + {"span_tree", hasRun && hasSandbox}, + {"identity", identityOK(run, agents)}, + // UnknownSentinel is the CLI fallback when no issue- or PR-shaped env is set. + // Review jobs that have PR_NUMBER / GITHUB_PR_URL should not hit this + // after #5622; a remaining unknown is a real fitness fail. + {"work_item", workItemOK(run)}, + {"operation", attrNonEmpty(run, AttrGenAIOperationName)}, + {"model", modelOK(run, agents)}, + {"usage", usageOK(run, agents)}, + {"cost_tools_turns", costOK}, + {"exit", hasExit(run)}, + } + + passed := 0 + var failed []string + var details []string + for _, c := range checks { + if c.pass { + passed++ + details = append(details, c.id+"=pass") + } else { + failed = append(failed, c.id) + if c.id == "cost_tools_turns" && len(costMissing) > 0 { + details = append(details, c.id+"=fail["+strings.Join(costMissing, ",")+"]") + } else { + details = append(details, c.id+"=fail") + } + } + } + total := len(checks) + value := float64(passed) / float64(total) + label := LabelFail + if value == 1.0 { + label = LabelPass + } + expl := strings.Join(details, ", ") + if len(failed) > 0 { + expl = "missing: " + strings.Join(failed, ", ") + "; " + expl + } + + spanID := "" + workItem := "" + agent := tr.AgentName() + if hasRun { + spanID = run.SpanID + workItem, _ = run.AttrString(AttrFullsendWorkItemID) + } + + return EvaluationResult{ + Name: evalName, + Label: label, + Explanation: expl, + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: agent, + Version: version, + Value: value, + } +} + +func identityOK(run Span, agents []Span) bool { + fa, okFA := run.AttrString(AttrFullsendAgent) + if !okFA || fa == "" || fa == UnknownSentinel { + return false + } + if ga, ok := run.AttrString(AttrGenAIAgentName); ok && ga == fa { + return true + } + for _, a := range agents { + if ga, ok := a.AttrString(AttrGenAIAgentName); ok && ga == fa { + return true + } + } + return false +} + +func workItemOK(run Span) bool { + v, ok := run.AttrString(AttrFullsendWorkItemID) + return ok && v != "" && v != UnknownSentinel +} + +func attrNonEmpty(s Span, key string) bool { + v, ok := s.AttrString(key) + return ok && v != "" +} + +func modelOK(run Span, agents []Span) bool { + hasModel := attrNonEmpty(run, AttrGenAIRequestModel) + if !hasModel { + for _, a := range agents { + if attrNonEmpty(a, AttrGenAIRequestModel) { + hasModel = true + break + } + } + } + hasSystem := false + for _, a := range agents { + if attrNonEmpty(a, AttrGenAIProviderName) || attrNonEmpty(a, AttrGenAISystem) { + hasSystem = true + break + } + } + return hasModel && hasSystem +} + +func usageOK(run Span, agents []Span) bool { + if _, ok := run.AttrInt(AttrGenAIUsageInputTokens); ok { + if _, ok := run.AttrInt(AttrGenAIUsageOutputTokens); ok { + return true + } + } + for _, a := range agents { + _, inOK := a.AttrInt(AttrGenAIUsageInputTokens) + _, outOK := a.AttrInt(AttrGenAIUsageOutputTokens) + if inOK && outOK { + return true + } + } + return false +} + +func costToolsTurnsDetail(run Span, agents []Span) (bool, []string) { + hasCost := false + hasTools := false + if _, ok := run.AttrFloat("fullsend.cost_usd"); ok { + hasCost = true + } + if _, ok := run.AttrInt("fullsend.tool_calls"); ok { + hasTools = true + } + for _, a := range agents { + if !hasCost { + if _, ok := a.AttrFloat("fullsend.cost_usd"); ok { + hasCost = true + } + } + if !hasTools { + if _, ok := a.AttrInt("fullsend.tool_calls"); ok { + hasTools = true + } + } + } + var missing []string + if !hasCost { + missing = append(missing, "cost") + } + if !hasTools { + missing = append(missing, "tool_calls") + } + if iters, ok := run.AttrInt("fullsend.iterations"); ok && iters > 0 { + if _, ok := run.AttrInt("fullsend.num_turns"); !ok { + missing = append(missing, "num_turns") + } + } + return len(missing) == 0, missing +} + +func hasExit(run Span) bool { + // Fitness: attribute present. Do not treat exit_code==0 as success — + // after #5944, OTLP Status / transcript_error carry outcome. + _, ok := run.AttrInt("exit_code") + return ok +} diff --git a/internal/evalmeasure/parse.go b/internal/evalmeasure/parse.go new file mode 100644 index 0000000000..4ec713f079 --- /dev/null +++ b/internal/evalmeasure/parse.go @@ -0,0 +1,211 @@ +package evalmeasure + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strconv" +) + +type otlpTracesData struct { + ResourceSpans []otlpResourceSpans `json:"resourceSpans"` +} + +type otlpResourceSpans struct { + ScopeSpans []otlpScopeSpans `json:"scopeSpans"` +} + +type otlpScopeSpans struct { + Spans []otlpSpan `json:"spans"` +} + +type otlpSpan struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentSpanID string `json:"parentSpanId"` + Name string `json:"name"` + StartTimeUnixNano string `json:"startTimeUnixNano"` + EndTimeUnixNano string `json:"endTimeUnixNano"` + Attributes []otlpKeyValue `json:"attributes"` + Status *otlpStatus `json:"status"` +} + +type otlpStatus struct { + Code int `json:"code"` +} + +type otlpKeyValue struct { + Key string `json:"key"` + Value map[string]any `json:"value"` +} + +// ParseStats counts telemetry JSONL lines and unusable spans so operators +// can tell "no traces" from "file present but partially unreadable". +type ParseStats struct { + NonEmptyLines int + SkippedLines int // whole JSONL lines that failed to unmarshal + SkippedSpans int // spans inside a valid line that failed convertSpan + // Incomplete is set when the scanner stopped early (e.g. bufio.ErrTooLong) + // but some traces were still recovered. Callers should warn; MeasureAndExport + // still scores those traces and returns a nil error. + Incomplete string +} + +// ParseTelemetryFile reads OTLP JSON TracesData lines from run-telemetry.jsonl +// and merges spans by trace id. Truncated or corrupt lines, and spans that +// fail conversion, are skipped (fail-open). SkippedLines counts whole-line +// failures; SkippedSpans counts per-span convert failures. An oversized-line +// scanner error returns traces gathered so far alongside the error so the +// caller can still score them. +func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { + var stats ParseStats + f, err := os.Open(path) + if err != nil { + return nil, stats, err + } + defer f.Close() + + byID := make(map[string]*Trace) + var order []string + + sc := bufio.NewScanner(f) + // Spans can be large; raise buffer. + buf := make([]byte, 0, 64*1024) + sc.Buffer(buf, 10*1024*1024) + + for sc.Scan() { + line := sc.Bytes() + if len(line) == 0 { + continue + } + stats.NonEmptyLines++ + var doc otlpTracesData + if err := json.Unmarshal(line, &doc); err != nil { + stats.SkippedLines++ + continue + } + for _, rs := range doc.ResourceSpans { + for _, ss := range rs.ScopeSpans { + for _, raw := range ss.Spans { + sp, err := convertSpan(raw) + if err != nil { + stats.SkippedSpans++ + continue + } + tr, ok := byID[sp.TraceID] + if !ok { + tr = &Trace{TraceID: sp.TraceID} + byID[sp.TraceID] = tr + order = append(order, sp.TraceID) + } + tr.Spans = append(tr.Spans, sp) + } + } + } + } + if err := sc.Err(); err != nil { + // Oversized line (bufio.ErrTooLong) or other scanner error: keep + // traces already parsed and surface the error to the caller. + out := make([]Trace, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, stats, err + } + + out := make([]Trace, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, stats, nil +} + +func convertSpan(raw otlpSpan) (Span, error) { + start, err := parseUint(raw.StartTimeUnixNano) + if err != nil { + return Span{}, fmt.Errorf("startTimeUnixNano: %w", err) + } + end, err := parseUint(raw.EndTimeUnixNano) + if err != nil { + return Span{}, fmt.Errorf("endTimeUnixNano: %w", err) + } + attrs := make(map[string]any, len(raw.Attributes)) + for _, kv := range raw.Attributes { + if kv.Key == "" { + continue + } + if v, ok := decodeAny(kv.Value); ok { + attrs[kv.Key] = v + } + } + status := 0 + if raw.Status != nil { + status = raw.Status.Code + } + return Span{ + TraceID: raw.TraceID, + SpanID: raw.SpanID, + ParentSpanID: raw.ParentSpanID, + Name: raw.Name, + StartUnixNano: start, + EndUnixNano: end, + StatusCode: status, + Attrs: attrs, + }, nil +} + +func parseUint(s string) (uint64, error) { + if s == "" { + return 0, nil + } + return strconv.ParseUint(s, 10, 64) +} + +// decodeAny extracts scalar OTLP attribute values. arrayValue and kvlistValue +// are intentionally unsupported — fitness scoring uses only scalar attributes. +func decodeAny(m map[string]any) (any, bool) { + if m == nil { + return nil, false + } + if v, ok := m["stringValue"]; ok { + return fmt.Sprint(v), true + } + if v, ok := m["boolValue"]; ok { + switch t := v.(type) { + case bool: + return t, true + default: + return nil, false + } + } + if v, ok := m["doubleValue"]; ok { + switch t := v.(type) { + case float64: + return t, true + case string: + n, err := strconv.ParseFloat(t, 64) + if err != nil { + return nil, false + } + return n, true + default: + return nil, false + } + } + if v, ok := m["intValue"]; ok { + switch t := v.(type) { + case string: + n, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return nil, false + } + return n, true + case float64: + return int64(t), true + default: + return nil, false + } + } + return nil, false +} diff --git a/internal/evalmeasure/parse_roundtrip_test.go b/internal/evalmeasure/parse_roundtrip_test.go new file mode 100644 index 0000000000..fa3cd81710 --- /dev/null +++ b/internal/evalmeasure/parse_roundtrip_test.go @@ -0,0 +1,69 @@ +package evalmeasure + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + + "github.com/fullsend-ai/fullsend/internal/telemetry" +) + +// TestParseTelemetryFile_RoundTripFromExporter binds the evalmeasure reader +// to the real file exporter: if the writer ever switches encoding (protojson +// enums, base64 ids, etc.), this fails instead of silently scoring nothing. +func TestParseTelemetryFile_RoundTripFromExporter(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + + dir := t.TempDir() + tracer, cleanup := telemetry.Setup(dir, "roundtrip-test") + + ctx := context.Background() + ctx, run := tracer.Start(ctx, "run") + run.SetAttributes( + attribute.String("fullsend.agent", "triage"), + attribute.String("fullsend.work_item_id", "acme/demo#1"), + attribute.String("gen_ai.operation.name", "invoke_agent"), + attribute.Int64("exit_code", 0), + attribute.Int64("fullsend.num_turns", 1), + attribute.Float64("fullsend.cost_usd", 0.01), + attribute.Int64("fullsend.tool_calls", 0), + attribute.Int64("fullsend.iterations", 1), + attribute.String("gen_ai.request.model", "test-model"), + attribute.Int64("gen_ai.usage.input_tokens", 1), + attribute.Int64("gen_ai.usage.output_tokens", 1), + ) + _, sandbox := tracer.Start(ctx, "sandbox_create") + sandbox.End() + _, agent := tracer.Start(ctx, "agent") + agent.SetAttributes( + attribute.String("gen_ai.agent.name", "triage"), + attribute.String("gen_ai.system", "anthropic"), + attribute.String("gen_ai.request.model", "test-model"), + attribute.Int64("gen_ai.usage.input_tokens", 1), + attribute.Int64("gen_ai.usage.output_tokens", 1), + attribute.Float64("fullsend.cost_usd", 0.01), + attribute.Int64("fullsend.tool_calls", 0), + ) + agent.End() + run.End() + cleanup(ctx) + + path := filepath.Join(dir, telemetry.TelemetryFile) + _, err := os.Stat(path) + require.NoError(t, err) + + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err) + assert.Equal(t, 0, stats.SkippedLines, "exporter JSONL must unmarshal") + assert.Equal(t, 0, stats.SkippedSpans, "exporter spans must convert") + require.NotEmpty(t, traces) + + r := ScoreFitness(traces[0]) + assert.Equal(t, LabelPass, r.Label, "round-tripped EM-001 fixture should pass: %s", r.Explanation) +} diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go new file mode 100644 index 0000000000..5ec316d68a --- /dev/null +++ b/internal/evalmeasure/parse_test.go @@ -0,0 +1,95 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseTelemetryFile_Complete(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + require.Len(t, traces, 1) + tr := traces[0] + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", tr.TraceID) + run, ok := tr.SpanByName("run") + require.True(t, ok) + got, ok := run.AttrString("fullsend.agent") + require.True(t, ok) + assert.Equal(t, "triage", got) + cost, ok := run.AttrFloat("fullsend.cost_usd") + require.True(t, ok) + assert.InDelta(t, 0.54, cost, 1e-9) + assert.Len(t, tr.SpansByName("agent"), 1) + assert.InDelta(t, 6.0, run.DurationSeconds(), 1e-9) +} + +func TestParseTelemetryFile_MergesLinesSameTrace(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "split.jsonl")) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Len(t, traces[0].Spans, 3) + _, ok := traces[0].SpanByName("sandbox_create") + assert.True(t, ok) +} + +func TestParseTelemetryFile_InvalidLineSkipped(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.jsonl") + require.NoError(t, os.WriteFile(path, []byte("not-json\n"), 0o644)) + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err, "a truncated/corrupt line must not fail the whole file") + assert.Empty(t, traces) + assert.Equal(t, 1, stats.NonEmptyLines) + assert.Equal(t, 1, stats.SkippedLines) + assert.Equal(t, 0, stats.SkippedSpans) +} + +func TestParseTelemetryFile_TruncatedLineDoesNotDiscardFile(t *testing.T) { + t.Parallel() + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + path := filepath.Join(dir, "mixed.jsonl") + body := string(good) + if body != "" && body[len(body)-1] != '\n' { + body += "\n" + } + body += "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":\"truncated\n" + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", traces[0].TraceID) + assert.GreaterOrEqual(t, stats.SkippedLines, 1) + assert.Greater(t, stats.NonEmptyLines, stats.SkippedLines) +} + +func TestParseTelemetryFile_MissingFile(t *testing.T) { + t.Parallel() + _, _, err := ParseTelemetryFile(filepath.Join(t.TempDir(), "missing.jsonl")) + require.Error(t, err) +} + +func TestParseTelemetryFile_BadSpanSkippedKeepsGood(t *testing.T) { + t.Parallel() + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + path := filepath.Join(dir, "mixed.jsonl") + // Valid JSON with a non-numeric startTimeUnixNano on one span. + bad := `{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"cccccccccccccccccccccccccccccccc","spanId":"9999999999999999","name":"run","startTimeUnixNano":"not-a-number","endTimeUnixNano":"2","attributes":[]}]}]}]}` + "\n" + require.NoError(t, os.WriteFile(path, append(append([]byte{}, good...), []byte(bad)...), 0o644)) + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", traces[0].TraceID) + assert.Equal(t, 0, stats.SkippedLines) + assert.GreaterOrEqual(t, stats.SkippedSpans, 1) +} diff --git a/internal/evalmeasure/registry.go b/internal/evalmeasure/registry.go new file mode 100644 index 0000000000..3d34f02f12 --- /dev/null +++ b/internal/evalmeasure/registry.go @@ -0,0 +1,94 @@ +package evalmeasure + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// Registry is an agent-specific measurement manifest (owned by agents repo). +type Registry struct { + Agent string `yaml:"agent"` + Measurements []MeasurementSpec `yaml:"measurements"` +} + +// MeasurementSpec selects a framework scorer. +type MeasurementSpec struct { + ID string `yaml:"id"` + Scorer string `yaml:"scorer"` + Name string `yaml:"name"` // optional display override; default = Scorer + Version int `yaml:"version"` +} + +// LoadRegistry loads a measurement manifest YAML file. +func LoadRegistry(path string) (Registry, error) { + b, err := os.ReadFile(path) + if err != nil { + return Registry{}, err + } + var reg Registry + if err := yaml.Unmarshal(b, ®); err != nil { + return Registry{}, err + } + if reg.Agent == "" { + return Registry{}, fmt.Errorf("manifest %s: agent is required", path) + } + for i, m := range reg.Measurements { + if m.ID == "" { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].id is required", path, i) + } + if m.Scorer == "" { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].scorer is required", path, i) + } + if m.Version <= 0 { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].version must be >= 1", path, i) + } + if strings.ContainsAny(m.ID, "|\n") || strings.ContainsAny(m.Scorer, "|\n") || strings.ContainsAny(m.Name, "|\n") { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].id, .scorer, and .name must not contain pipe or newline", path, i) + } + } + return reg, nil +} + +func (m MeasurementSpec) versionString() string { + return fmt.Sprintf("%s@%d", m.ID, m.Version) +} + +func (m MeasurementSpec) evalName() string { + if m.Name != "" { + return m.Name + } + return m.Scorer +} + +// ScoreTrace runs enabled measurements for traces matching the manifest agent. +// An empty or UnknownSentinel agent name still scores so identity=fail is +// recorded (EM-001 exists to catch missing identity). A different, known +// agent name is skipped. Unknown scorer strings write a skip row rather +// than silent-no-op or a fail that mixes into pass-rate (a newer agents@v0 +// manifest can name a scorer this binary does not implement yet). +func ScoreTrace(tr Trace, reg Registry) []EvaluationResult { + name := tr.AgentName() + if name != "" && name != UnknownSentinel && !strings.EqualFold(name, reg.Agent) { + return nil + } + var out []EvaluationResult + for _, m := range reg.Measurements { + switch m.Scorer { + case ScorerFitness: + out = append(out, ScoreFitnessNamed(tr, m.evalName(), m.versionString())) + default: + out = append(out, EvaluationResult{ + Name: m.evalName(), + Label: LabelSkip, + Explanation: "unknown scorer: " + m.Scorer, + TraceID: tr.TraceID, + Agent: name, + Version: m.versionString(), + }) + } + } + return out +} diff --git a/internal/evalmeasure/registry_test.go b/internal/evalmeasure/registry_test.go new file mode 100644 index 0000000000..75600459ac --- /dev/null +++ b/internal/evalmeasure/registry_test.go @@ -0,0 +1,86 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadRegistry_Valid(t *testing.T) { + t.Parallel() + reg, err := LoadRegistry(filepath.Join("testdata", "sample-registry.yaml")) + require.NoError(t, err) + assert.Equal(t, "triage", reg.Agent) + require.Len(t, reg.Measurements, 1) + assert.Equal(t, "em-001", reg.Measurements[0].ID) + assert.Equal(t, "trace_fitness", reg.Measurements[0].Scorer) + assert.Equal(t, 1, reg.Measurements[0].Version) +} + +func TestLoadRegistry_MissingAgent(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("measurements:\n - id: em-001\n scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "agent is required") +} + +func TestLoadRegistry_MissingID(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "id is required") +} + +func TestLoadRegistry_MissingScorer(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: em-001\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "scorer is required") +} + +func TestLoadRegistry_ZeroVersion(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: em-001\n scorer: trace_fitness\n version: 0\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "version must be >= 1") +} + +func TestLoadRegistry_PipeInID(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: \"em|001\"\n scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain pipe or newline") +} + +func TestLoadRegistry_InvalidYAML(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("{{invalid yaml"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) +} + +func TestLoadRegistry_FileNotFound(t *testing.T) { + t.Parallel() + _, err := LoadRegistry("/nonexistent/path.yaml") + require.Error(t, err) +} diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go new file mode 100644 index 0000000000..490c8213a8 --- /dev/null +++ b/internal/evalmeasure/run.go @@ -0,0 +1,83 @@ +package evalmeasure + +import ( + "context" + "fmt" + "path/filepath" +) + +type persistHookKey struct{} + +// WithPersistHook runs fn after each successful RecordScored. Tests use it +// to fail a later persist; production callers pass a plain context. +func WithPersistHook(ctx context.Context, fn func()) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, persistHookKey{}, fn) +} + +// MeasureFile parses telemetry, scores with the manifest, and writes local +// eval-measurements.jsonl. Idempotent per ledger. +func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { + r, _, err := MeasureAndExport(context.Background(), telemetryPath, registryPath, outDir) + return r, err +} + +// MeasureAndExport is MeasureFile with an explicit context (reserved for +// future portable OTLP score export on the same OTEL_* path as ADR 0050). +func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, ParseStats, error) { + var stats ParseStats + if err := ctx.Err(); err != nil { + return nil, stats, err + } + if outDir == "" { + outDir = filepath.Dir(telemetryPath) + } + reg, err := LoadRegistry(registryPath) + if err != nil { + return nil, stats, fmt.Errorf("load registry: %w", err) + } + traces, stats, parseErr := ParseTelemetryFile(telemetryPath) + if parseErr != nil && len(traces) == 0 { + return nil, stats, fmt.Errorf("parse telemetry: %w", parseErr) + } + if parseErr != nil { + // Oversized/corrupt tail after good lines: score what we have and + // surface the damage via stats (CLI warns; still exit 0). + stats.Incomplete = parseErr.Error() + } + + ledgerPath := filepath.Join(outDir, LedgerFile) + measPath := filepath.Join(outDir, MeasurementsFile) + var all []EvaluationResult + hook, _ := ctx.Value(persistHookKey{}).(func()) + + for _, tr := range traces { + results := ScoreTrace(tr, reg) + for _, r := range results { + done, err := AlreadyScored(ledgerPath, r.TraceID, r.Name, r.Version) + if err != nil { + return all, stats, fmt.Errorf("check ledger: %w", err) + } + if done { + continue + } + if err := AppendMeasurements(measPath, []EvaluationResult{r}); err != nil { + return all, stats, fmt.Errorf("append measurements: %w", err) + } + // Only count rows that landed in eval-measurements.jsonl so + // CLI stdout matches disk on a later ledger/write error. + all = append(all, r) + if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { + return all, stats, fmt.Errorf("record scored: %w", err) + } + if hook != nil { + hook() + } + } + } + // Partial parse with traces already scored is success: scores are data. + // stats.Incomplete (if set) lets the CLI warn without failing the job. + return all, stats, nil +} diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go new file mode 100644 index 0000000000..7e3566aa86 --- /dev/null +++ b/internal/evalmeasure/run_test.go @@ -0,0 +1,198 @@ +package evalmeasure + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadRegistryAndScoreTrace(t *testing.T) { + t.Parallel() + reg, err := LoadRegistry(filepath.Join("testdata", "sample-registry.yaml")) + require.NoError(t, err) + assert.Equal(t, "triage", reg.Agent) + + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 1) + assert.Equal(t, "trace_fitness", results[0].Name) + assert.Equal(t, "em-001@1", results[0].Version) + assert.Equal(t, "pass", results[0].Label) +} + +func TestMeasureFile_Idempotent(t *testing.T) { + out := t.TempDir() + telemetry := filepath.Join("testdata", "complete.jsonl") + registry := filepath.Join("testdata", "sample-registry.yaml") + + first, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, first, 1) + + second, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + assert.Empty(t, second) + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) +} + +func TestMeasureFile_AppendBeforeLedger(t *testing.T) { + out := t.TempDir() + telemetry := filepath.Join("testdata", "complete.jsonl") + registry := filepath.Join("testdata", "sample-registry.yaml") + ledgerPath := filepath.Join(out, LedgerFile) + + first, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, first, 1) + + require.NoError(t, os.Remove(ledgerPath)) + + second, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, second, 1, "retry after missing ledger should re-score") + + lines, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Equal(t, 2, bytes.Count(lines, []byte("\n")), "missing ledger may duplicate JSONL lines; consumers dedupe on trace_id+version") +} + +func TestMeasureFile_BadRegistry(t *testing.T) { + _, err := MeasureFile( + filepath.Join("testdata", "complete.jsonl"), + filepath.Join(t.TempDir(), "missing.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} + +func TestMeasureFile_BadTelemetry(t *testing.T) { + _, err := MeasureFile( + filepath.Join(t.TempDir(), "missing.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} + +func TestMeasureAndExport_CancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, _, err := MeasureAndExport( + ctx, + filepath.Join("testdata", "complete.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} + +func TestWithPersistHook_NilContext(t *testing.T) { + ctx := WithPersistHook(nil, func() {}) + require.NotNil(t, ctx) + _, ok := ctx.Value(persistHookKey{}).(func()) + assert.True(t, ok) +} + +func writeTwoTraceTelemetry(t *testing.T, completePath string) string { + t.Helper() + src, err := os.ReadFile(completePath) + require.NoError(t, err) + src = bytes.TrimSpace(src) + second := bytes.ReplaceAll(src, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")) + p := filepath.Join(t.TempDir(), "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(p, append(append(src, '\n'), second...), 0o644)) + return p +} + +func TestMeasureAndExport_KeepsFirstWhenSecondPersistFails(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t, filepath.Join("testdata", "complete.jsonl")) + ctx := WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + results, _, err := MeasureAndExport(ctx, telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.Error(t, err) + require.Len(t, results, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) + assert.Contains(t, err.Error(), "append measurements") +} + +func TestMeasureAndExport_ScoresPartialFileDespiteParseError(t *testing.T) { + // Oversized line after a good line: ParseTelemetryFile keeps the good + // trace and returns sc.Err(); MeasureAndExport still scores it and + // treats the partial parse as success (scores are data). + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + telem := filepath.Join(dir, "run-telemetry.jsonl") + f, err := os.Create(telem) + require.NoError(t, err) + _, err = f.Write(append(bytes.TrimSpace(good), '\n')) + require.NoError(t, err) + huge := make([]byte, 11*1024*1024) + for i := range huge { + huge[i] = 'a' + } + _, err = f.Write(huge) + require.NoError(t, err) + _, err = f.Write([]byte("\n")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + out := t.TempDir() + results, stats, err := MeasureAndExport(context.Background(), telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelPass, results[0].Label) + assert.Greater(t, stats.NonEmptyLines, 0) + assert.NotEmpty(t, stats.Incomplete, "oversized-line parse must set Incomplete for CLI warn") +} + +func TestMeasureFile_PrescriptSkippedRecordsSkip(t *testing.T) { + out := t.TempDir() + results, err := MeasureFile( + filepath.Join("testdata", "prescript-skipped.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + out, + ) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelSkip, results[0].Label) + assert.NotEqual(t, LabelFail, results[0].Label) + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"label":"skip"`) + assert.NotContains(t, string(b), `"label":"fail"`) +} + +func TestMeasureFile_EmptyIdentityPersistsFailRow(t *testing.T) { + dir := t.TempDir() + telem := filepath.Join(dir, "run-telemetry.jsonl") + // Minimal OTLP line: run span with no agent identity. + line := `{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"1111111111111111","name":"run","startTimeUnixNano":"1","endTimeUnixNano":"2","attributes":[{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}}]},{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"2222222222222222","name":"sandbox_create","startTimeUnixNano":"1","endTimeUnixNano":"2"},{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"3333333333333333","name":"agent","startTimeUnixNano":"1","endTimeUnixNano":"2","attributes":[{"key":"gen_ai.system","value":{"stringValue":"anthropic"}}]}]}]}]}` + "\n" + require.NoError(t, os.WriteFile(telem, []byte(line), 0o644)) + + out := t.TempDir() + results, err := MeasureFile(telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelFail, results[0].Label) + assert.Contains(t, results[0].Explanation, "identity=fail") + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"label":"fail"`) + assert.Contains(t, string(b), "identity=fail") +} diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go new file mode 100644 index 0000000000..54cae036f3 --- /dev/null +++ b/internal/evalmeasure/score_test.go @@ -0,0 +1,285 @@ +package evalmeasure + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScoreFitness_CompletePass(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, ScorerFitness, r.Name) + assert.Equal(t, "em-001@1", r.Version) + assert.Equal(t, "pass", r.Label) + assert.Equal(t, 1.0, r.Value) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", r.TraceID) + assert.Contains(t, r.Explanation, "span_tree=pass") + assert.Contains(t, r.Explanation, "cost_tools_turns=pass") + assert.Contains(t, r.Explanation, "exit=pass") + assert.NotContains(t, r.Explanation, "=fail") +} + +func TestScoreFitness_MissingCostFails(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "fail", r.Label) + assert.Less(t, r.Value, 1.0) + assert.Contains(t, r.Explanation, "cost_tools_turns") + assert.Contains(t, r.Explanation, "cost_tools_turns=fail") +} + +func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "review-unknown-workitem.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "review", r.Agent) + assert.Equal(t, "fail", r.Label) + assert.Contains(t, r.Explanation, "identity=pass") + assert.Contains(t, r.Explanation, "work_item=fail") + assert.Contains(t, r.Explanation, "missing: work_item") +} + +func TestScoreFitness_PrescriptSkippedExcluded(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "prescript-skipped.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "triage", r.Agent) + assert.Equal(t, LabelSkip, r.Label) + assert.NotEqual(t, "fail", r.Label) + assert.NotEqual(t, "pass", r.Label) + assert.Contains(t, r.Explanation, "pre-script skipped") + assert.NotContains(t, r.Explanation, "span_tree=fail") + assert.Equal(t, "cccccccccccccccccccccccccccccccc", r.TraceID) +} + +func TestScoreTrace_AgentMismatchReturnsNil(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{Agent: "code", Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}} + assert.Empty(t, ScoreTrace(traces[0], reg)) +} + +func TestScoreTrace_EmptyAgentNameRecordsIdentityFail(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "dddddddddddddddddddddddddddddddd", + Spans: []Span{ + { + Name: "run", + SpanID: "1111111111111111", + Attrs: map[string]any{ + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "anthropic"}}, + }, + } + assert.Empty(t, tr.AgentName()) + reg := Registry{ + Agent: "triage", + Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}, + } + results := ScoreTrace(tr, reg) + require.Len(t, results, 1, "empty identity must still write a row — silent drop is survivorship bias") + assert.Equal(t, LabelFail, results[0].Label) + assert.Contains(t, results[0].Explanation, "identity=fail") +} + +func TestScoreTrace_UnknownAgentSentinelRecordsIdentityFail(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": UnknownSentinel, + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent"}, + }, + } + reg := Registry{ + Agent: "review", + Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}, + } + results := ScoreTrace(tr, reg) + require.Len(t, results, 1) + assert.Contains(t, results[0].Explanation, "identity=fail") +} + +func TestScoreTrace_UnknownScorerSkipsNotSilent(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{ + Agent: "triage", + Measurements: []MeasurementSpec{ + {ID: "em-999", Scorer: "future_scorer", Version: 1}, + {ID: "em-001", Scorer: ScorerFitness, Version: 1}, + }, + } + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 2) + assert.Equal(t, "future_scorer", results[0].Name) + assert.Equal(t, LabelSkip, results[0].Label) + assert.Contains(t, results[0].Explanation, "unknown scorer") + assert.Equal(t, 0.0, results[0].Value) + assert.Equal(t, "trace_fitness", results[1].Name) +} + +func TestScoreFitness_CostToolsTurnsNamesSubcheck(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, LabelFail, r.Label) + assert.Contains(t, r.Explanation, "cost_tools_turns=fail[cost]") +} + +func TestScoreFitness_MissingTurnsNamesSubcheck(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "1", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "gen_ai.agent.name": "triage", + "gen_ai.operation.name": "invoke_agent", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + "gen_ai.request.model": "claude", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.1, + "fullsend.tool_calls": int64(1), + "fullsend.iterations": int64(1), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "anthropic", "gen_ai.agent.name": "triage"}}, + }, + } + r := ScoreFitness(tr) + assert.Contains(t, r.Explanation, "cost_tools_turns=fail[num_turns]") +} + +func TestScoreFitness_EmptyModelStringFails(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "2", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "gen_ai.agent.name": "triage", + "gen_ai.operation.name": "invoke_agent", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + "gen_ai.request.model": "", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.1, + "fullsend.tool_calls": int64(1), + "fullsend.iterations": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "", "gen_ai.agent.name": "triage"}}, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelFail, r.Label) + assert.Contains(t, r.Explanation, "model=fail") +} + +func TestScoreTrace_AgentNameEqualFold(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{Agent: "Triage", Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}} + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 1) + assert.Equal(t, LabelPass, results[0].Label) +} + +func TestScoreFitness_NoAgentSpanSkipped(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(1), + "gen_ai.operation.name": "invoke_agent", + }, + }, + {Name: "sandbox_create"}, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelSkip, r.Label) + assert.Contains(t, r.Explanation, "no agent span") +} + +func TestScoreFitness_AgentSpansWithoutRunSkipped(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "dddddddddddddddddddddddddddddddd", + Spans: []Span{ + { + Name: "agent", + Attrs: map[string]any{ + "gen_ai.system": "anthropic", + "gen_ai.agent.name": "triage", + "gen_ai.request.model": "claude", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.01, + "fullsend.tool_calls": int64(0), + "fullsend.num_turns": int64(1), + }, + }, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelSkip, r.Label) + assert.Contains(t, r.Explanation, "root run span missing") +} + +func TestScoreFitness_ProviderNameAccepted(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + tr := traces[0] + for i := range tr.Spans { + if tr.Spans[i].Name != "agent" { + continue + } + delete(tr.Spans[i].Attrs, "gen_ai.system") + tr.Spans[i].Attrs["gen_ai.provider.name"] = "anthropic" + } + r := ScoreFitness(tr) + assert.Equal(t, LabelPass, r.Label) +} diff --git a/internal/evalmeasure/testdata/README.md b/internal/evalmeasure/testdata/README.md new file mode 100644 index 0000000000..a49d56f033 --- /dev/null +++ b/internal/evalmeasure/testdata/README.md @@ -0,0 +1,6 @@ +# evalmeasure testdata + +Synthetic OTLP JSONL fixtures and a sample registry for unit tests only. + +Production registries live in `fullsend-ai/agents` at +`eval/measurements/.yaml`. diff --git a/internal/evalmeasure/testdata/complete.jsonl b/internal/evalmeasure/testdata/complete.jsonl new file mode 100644 index 0000000000..8e96a8c202 --- /dev/null +++ b/internal/evalmeasure/testdata/complete.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/missing-cost.jsonl b/internal/evalmeasure/testdata/missing-cost.jsonl new file mode 100644 index 0000000000..fb84a58319 --- /dev/null +++ b/internal/evalmeasure/testdata/missing-cost.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/prescript-skipped.jsonl b/internal/evalmeasure/testdata/prescript-skipped.jsonl new file mode 100644 index 0000000000..6477768774 --- /dev/null +++ b/internal/evalmeasure/testdata/prescript-skipped.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"cccccccccccccccccccccccccccccccc","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"2000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"fullsend.prescript.skipped","value":{"boolValue":true}},{"key":"fullsend.prescript.skip_reason","value":{"stringValue":"already handled"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/review-unknown-workitem.jsonl b/internal/evalmeasure/testdata/review-unknown-workitem.jsonl new file mode 100644 index 0000000000..61678b2fae --- /dev/null +++ b/internal/evalmeasure/testdata/review-unknown-workitem.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"review"}},{"key":"gen_ai.agent.name","value":{"stringValue":"review"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"unknown"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"review"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/sample-registry.yaml b/internal/evalmeasure/testdata/sample-registry.yaml new file mode 100644 index 0000000000..ef8492df09 --- /dev/null +++ b/internal/evalmeasure/testdata/sample-registry.yaml @@ -0,0 +1,5 @@ +agent: triage +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/internal/evalmeasure/testdata/split.jsonl b/internal/evalmeasure/testdata/split.jsonl new file mode 100644 index 0000000000..434f0b0351 --- /dev/null +++ b/internal/evalmeasure/testdata/split.jsonl @@ -0,0 +1,2 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/types.go b/internal/evalmeasure/types.go new file mode 100644 index 0000000000..0512df4aec --- /dev/null +++ b/internal/evalmeasure/types.go @@ -0,0 +1,190 @@ +// Package evalmeasure scores agent run traces for online eval measurements. +package evalmeasure + +import ( + "fmt" + "strconv" +) + +// Attribute names used by EM-001. Most gen_ai.* keys follow OpenTelemetry +// GenAI semantic conventions. gen_ai.system was renamed to +// gen_ai.provider.name in semconv v1.37.0; modelOK accepts either so +// em-001@1 stays green across the emitter migration. Other upstream +// renames remain an em-001 version bump. +const ( + AttrFullsendAgent = "fullsend.agent" + AttrFullsendWorkItemID = "fullsend.work_item_id" + AttrGenAIAgentName = "gen_ai.agent.name" + AttrGenAISystem = "gen_ai.system" // deprecated; prefer AttrGenAIProviderName + AttrGenAIProviderName = "gen_ai.provider.name" + AttrGenAIRequestModel = "gen_ai.request.model" + AttrGenAIUsageInputTokens = "gen_ai.usage.input_tokens" + AttrGenAIUsageOutputTokens = "gen_ai.usage.output_tokens" + AttrGenAIOperationName = "gen_ai.operation.name" +) + +// Span is a portable view of one OTEL span from run-telemetry.jsonl. +type Span struct { + TraceID string + SpanID string + ParentSpanID string + Name string + StartUnixNano uint64 + EndUnixNano uint64 + StatusCode int // 0 unset, 1 OK, 2 ERROR (OTLP) + Attrs map[string]any +} + +// Trace groups spans that share a trace id. +type Trace struct { + TraceID string + Spans []Span +} + +// EvaluationResult is a portable measurement score (no vendor types). +type EvaluationResult struct { + Name string `json:"name"` + Label string `json:"label"` + Explanation string `json:"explanation"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + WorkItemID string `json:"work_item_id,omitempty"` + Agent string `json:"agent"` + Version string `json:"version"` + // Value is the numeric score. Skip rows leave this at 0; consumers + // must key off Label, not Value==0. Do not add omitempty: a real + // fail can also be 0.0. + Value float64 `json:"value"` +} + +// AttrString returns a string attribute. +func (s Span) AttrString(key string) (string, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return "", false + } + switch t := v.(type) { + case string: + return t, true + default: + return fmt.Sprint(t), true + } +} + +// AttrBool returns a bool attribute. +func (s Span) AttrBool(key string) (bool, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return false, false + } + switch t := v.(type) { + case bool: + return t, true + case string: + switch t { + case "true": + return true, true + case "false": + return false, true + default: + return false, false + } + default: + return false, false + } +} + +// AttrInt returns an int64 attribute. +func (s Span) AttrInt(key string) (int64, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return 0, false + } + switch t := v.(type) { + case int64: + return t, true + case int: + return int64(t), true + case float64: + return int64(t), true + case string: + n, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return 0, false + } + return n, true + default: + return 0, false + } +} + +// AttrFloat returns a float64 attribute. +func (s Span) AttrFloat(key string) (float64, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return 0, false + } + switch t := v.(type) { + case float64: + return t, true + case int64: + return float64(t), true + case int: + return float64(t), true + case string: + n, err := strconv.ParseFloat(t, 64) + if err != nil { + return 0, false + } + return n, true + default: + return 0, false + } +} + +// DurationSeconds returns end-start in seconds. +func (s Span) DurationSeconds() float64 { + if s.EndUnixNano <= s.StartUnixNano { + return 0 + } + return float64(s.EndUnixNano-s.StartUnixNano) / 1e9 +} + +// SpanByName returns the first span with the given name. +func (t Trace) SpanByName(name string) (Span, bool) { + for _, s := range t.Spans { + if s.Name == name { + return s, true + } + } + return Span{}, false +} + +// SpansByName returns all spans with the given name. +func (t Trace) SpansByName(name string) []Span { + var out []Span + for _, s := range t.Spans { + if s.Name == name { + out = append(out, s) + } + } + return out +} + +// AgentName returns the agent identity from run or agent spans. +func (t Trace) AgentName() string { + if run, ok := t.SpanByName("run"); ok { + if a, ok := run.AttrString(AttrFullsendAgent); ok && a != "" { + return a + } + if a, ok := run.AttrString(AttrGenAIAgentName); ok && a != "" { + return a + } + } + for _, a := range t.SpansByName("agent") { + if name, ok := a.AttrString(AttrGenAIAgentName); ok && name != "" { + return name + } + } + return "" +} diff --git a/internal/evalmeasure/types_test.go b/internal/evalmeasure/types_test.go new file mode 100644 index 0000000000..44bbdb3a9e --- /dev/null +++ b/internal/evalmeasure/types_test.go @@ -0,0 +1,209 @@ +package evalmeasure + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAttrString(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want string + wantOK bool + }{ + {"string value", map[string]any{"k": "v"}, "k", "v", true}, + {"int value coerced", map[string]any{"k": 42}, "k", "42", true}, + {"missing key", map[string]any{}, "k", "", false}, + {"nil value", map[string]any{"k": nil}, "k", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrString(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAttrInt(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want int64 + wantOK bool + }{ + {"int64", map[string]any{"k": int64(42)}, "k", 42, true}, + {"int", map[string]any{"k": 42}, "k", 42, true}, + {"float64", map[string]any{"k": float64(42)}, "k", 42, true}, + {"string parseable", map[string]any{"k": "42"}, "k", 42, true}, + {"string unparseable", map[string]any{"k": "abc"}, "k", 0, false}, + {"missing key", map[string]any{}, "k", 0, false}, + {"nil value", map[string]any{"k": nil}, "k", 0, false}, + {"bool not int", map[string]any{"k": true}, "k", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrInt(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAttrFloat(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want float64 + wantOK bool + }{ + {"float64", map[string]any{"k": float64(3.14)}, "k", 3.14, true}, + {"int64", map[string]any{"k": int64(42)}, "k", 42, true}, + {"int", map[string]any{"k": 42}, "k", 42, true}, + {"string parseable", map[string]any{"k": "3.14"}, "k", 3.14, true}, + {"string unparseable", map[string]any{"k": "abc"}, "k", 0, false}, + {"missing key", map[string]any{}, "k", 0, false}, + {"nil value", map[string]any{"k": nil}, "k", 0, false}, + {"bool not float", map[string]any{"k": true}, "k", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrFloat(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.InDelta(t, tt.want, got, 1e-9) + }) + } +} + +func TestDurationSeconds(t *testing.T) { + t.Parallel() + tests := []struct { + name string + start uint64 + end uint64 + want float64 + }{ + {"normal", 1000000000, 7000000000, 6.0}, + {"zero duration", 100, 100, 0}, + {"end before start", 200, 100, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{StartUnixNano: tt.start, EndUnixNano: tt.end} + assert.InDelta(t, tt.want, s.DurationSeconds(), 1e-9) + }) + } +} + +func TestSpanByName(t *testing.T) { + t.Parallel() + tr := Trace{ + Spans: []Span{ + {Name: "run"}, + {Name: "agent"}, + }, + } + s, ok := tr.SpanByName("run") + assert.True(t, ok) + assert.Equal(t, "run", s.Name) + + _, ok = tr.SpanByName("missing") + assert.False(t, ok) +} + +func TestSpansByName(t *testing.T) { + t.Parallel() + tr := Trace{ + Spans: []Span{ + {Name: "agent"}, + {Name: "run"}, + {Name: "agent"}, + }, + } + assert.Len(t, tr.SpansByName("agent"), 2) + assert.Empty(t, tr.SpansByName("missing")) +} + +func TestAgentName(t *testing.T) { + t.Parallel() + tests := []struct { + name string + spans []Span + want string + }{ + { + "from run fullsend.agent", + []Span{{Name: "run", Attrs: map[string]any{"fullsend.agent": "triage"}}}, + "triage", + }, + { + "from run gen_ai.agent.name", + []Span{{Name: "run", Attrs: map[string]any{"gen_ai.agent.name": "review"}}}, + "review", + }, + { + "from agent span", + []Span{ + {Name: "run", Attrs: map[string]any{}}, + {Name: "agent", Attrs: map[string]any{"gen_ai.agent.name": "code"}}, + }, + "code", + }, + { + "empty when no identity", + []Span{{Name: "run", Attrs: map[string]any{}}}, + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tr := Trace{Spans: tt.spans} + assert.Equal(t, tt.want, tr.AgentName()) + }) + } +} + +func TestAttrBool(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want bool + wantOK bool + }{ + {"true", map[string]any{"k": true}, "k", true, true}, + {"false", map[string]any{"k": false}, "k", false, true}, + {"string true", map[string]any{"k": "true"}, "k", true, true}, + {"string false", map[string]any{"k": "false"}, "k", false, true}, + {"string other", map[string]any{"k": "yes"}, "k", false, false}, + {"missing key", map[string]any{}, "k", false, false}, + {"nil value", map[string]any{"k": nil}, "k", false, false}, + {"int not bool", map[string]any{"k": 1}, "k", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrBool(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index 09f8b158bc..1c170bbf8c 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -87,6 +87,17 @@ var ( errTooLarge = errors.New("fetch: response body exceeds size limit") ) +// HTTPStatusError is returned when FetchURL gets a non-200 response. +type HTTPStatusError struct { + Status int +} + +func (e HTTPStatusError) Error() string { + return fmt.Sprintf("%s: got %d", errNonOK.Error(), e.Status) +} + +func (e HTTPStatusError) Unwrap() error { return errNonOK } + // FetchURL retrieves the content at rawURL subject to the given policy. // It returns the response body bytes or an error describing why the fetch // was rejected or failed. @@ -193,7 +204,7 @@ func FetchURL(ctx context.Context, rawURL string, policy FetchPolicy) ([]byte, e // 9. Only accept 200 OK. if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: got %d", errNonOK, resp.StatusCode) + return nil, HTTPStatusError{Status: resp.StatusCode} } // 10. Size limit: read one extra byte to detect overflow. diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go index 382ec8c7ff..265c42eb1c 100644 --- a/internal/fetch/fetch_test.go +++ b/internal/fetch/fetch_test.go @@ -151,6 +151,10 @@ func TestFetchURL(t *testing.T) { if !errors.Is(err, errNonOK) { t.Fatalf("expected errNonOK, got: %v", err) } + var httpErr HTTPStatusError + if !errors.As(err, &httpErr) || httpErr.Status != http.StatusNotFound { + t.Fatalf("expected HTTPStatusError 404, got: %v", err) + } }) t.Run("Success", func(t *testing.T) { diff --git a/internal/prescript/prescript.go b/internal/prescript/prescript.go index ea7276319b..0607dd6922 100644 --- a/internal/prescript/prescript.go +++ b/internal/prescript/prescript.go @@ -61,7 +61,7 @@ const EnvVar = "FULLSEND_PRESCRIPT_OUTPUT" // ExitCodeNeutral is the exit code a pre-script uses to signal "nothing to // do, skip cleanly" (issue #582). When fullsend run sees this exit code it // treats the run as skipped/neutral — no sandbox is created, no LLM is -// invoked, and the run reports a ⏭️ skipped status. The code follows the +// invoked, and the run reports a skipped status. The code follows the // CI convention for "neutral" (used by GitHub Actions and others). // // Exit 78 is complementary to the file-based skip protocol (skipped=true). diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 45888b73a1..f3d7acdba3 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -885,7 +885,7 @@ func resolvedBasename(path string) string { // same remotePath overwrite deterministically rather than merging files // from both. Do not point two different, unrelated sources at the same // remotePath expecting their content to coexist. -func UploadDir(sandboxName, localPath, remotePath string) error { +func UploadDir(sandboxName, localPath, remotePath string, excludes ...string) error { tmp, err := os.CreateTemp("", "openshell-upload-*.tar.gz") if err != nil { return fmt.Errorf("creating temp tarball: %w", err) @@ -894,7 +894,21 @@ func UploadDir(sandboxName, localPath, remotePath string) error { tmp.Close() defer os.Remove(tmpPath) - tarCmd := exec.Command("tar", "-czf", tmpPath, "-C", localPath, ".") + members, err := tarRootMembers(localPath, excludes...) + if err != nil { + return fmt.Errorf("listing %q for upload: %w", localPath, err) + } + tarArgs := []string{"-czf", tmpPath, "-C", localPath} + if len(members) == 0 { + // Everything at the root was excluded (or the dir is empty). An + // empty --files-from list yields an empty archive on GNU tar and + // bsdtar alike; do not fall back to "." (that would re-include + // excluded names). + tarArgs = append(tarArgs, "--files-from", os.DevNull) + } else { + tarArgs = append(tarArgs, members...) + } + tarCmd := exec.Command("tar", tarArgs...) // Suppress macOS AppleDouble (._*) files in the tarball. On macOS, // bsdtar generates ._* companion files for any file with extended // attributes. These corrupt .git after a sandbox round-trip. @@ -931,6 +945,39 @@ func UploadDir(sandboxName, localPath, remotePath string) error { return nil } +// tarRootMembers lists top-level archive members under localPath, omitting +// excludes. Matching is by top-level entry name only (nested path components +// in an exclude are ignored beyond the first segment), so nested directories +// like sub/output/ are never dropped — unlike tar --exclude on bsdtar, +// which matches the basename at any depth. +func tarRootMembers(localPath string, excludes ...string) ([]string, error) { + skip := make(map[string]struct{}, len(excludes)) + for _, ex := range excludes { + name := strings.Trim(ex, `/\`) + if name == "" { + continue + } + // Top-level basenames only. Rejecting nested paths avoids silently + // truncating "build/output" to "build" and dropping an entire tree. + if strings.ContainsAny(name, `/\`) { + return nil, fmt.Errorf("upload exclude %q must be a top-level name", ex) + } + skip[name] = struct{}{} + } + entries, err := os.ReadDir(localPath) + if err != nil { + return nil, err + } + members := make([]string, 0, len(entries)) + for _, e := range entries { + if _, ok := skip[e.Name()]; ok { + continue + } + members = append(members, "./"+e.Name()) + } + return members, nil +} + // Download copies a file or directory from a sandbox to the local machine. // The localPath is always treated as a directory by openshell — for single-file // downloads use DownloadFile instead. diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index 5aa7af9bc6..5b64528b20 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -673,6 +673,40 @@ func TestUploadDir_TarIncludesCopyfileDisable(t *testing.T) { assert.Equal(t, "1", strings.TrimSpace(string(data)), "COPYFILE_DISABLE should be set to 1") } +func TestUploadDir_ExcludesPatternsFromTarball(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "output", "agent-x-1-1"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "output", "agent-x-1-1", "run-telemetry.jsonl"), []byte("telem\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub", "output"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "sub", "output", "keep.txt"), []byte("nested\n"), 0o644)) + + binDir := t.TempDir() + logPath := filepath.Join(binDir, "openshell.log") + sentinelPath := filepath.Join(binDir, "uploaded.tar.gz") + fakeOpenshell(t, binDir, logPath, sentinelPath) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + require.NoError(t, UploadDir("test-sandbox", dir, "/sandbox/workspace/repo", "output/")) + + extracted := extractTar(t, sentinelPath) + _, err := os.Stat(filepath.Join(extracted, "keep.txt")) + require.NoError(t, err, "keep.txt should be in the tarball") + _, err = os.Stat(filepath.Join(extracted, "output")) + assert.True(t, os.IsNotExist(err), "root output/ must be excluded from the sandbox tarball") + got, err := os.ReadFile(filepath.Join(extracted, "sub", "output", "keep.txt")) + require.NoError(t, err, "nested sub/output/ must survive (bsdtar --exclude would drop it)") + assert.Equal(t, "nested\n", string(got)) +} + +func TestTarRootMembers_RejectsNestedExclude(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("x"), 0o644)) + _, err := tarRootMembers(dir, "build/output") + require.Error(t, err) + assert.Contains(t, err.Error(), "top-level") +} + // fakeOpenshell writes a script that logs every invocation's full argv to // logPath, and — when invoked as "sandbox upload " — // copies to sentinelPath so the test can inspect exactly what bytes diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitignore b/internal/scaffold/fullsend-repo-gitlab/.gitignore new file mode 100644 index 0000000000..734b5fe12f --- /dev/null +++ b/internal/scaffold/fullsend-repo-gitlab/.gitignore @@ -0,0 +1,6 @@ +# Recommended ignore for host runDir / eval-measure artifacts +# (GitLab CI artifacts: output/). Not installed as a root .gitignore — +# CollectGitLabPerRepoInstallFiles skips this file so it cannot overwrite +# a consumer's existing ignore list. Copy the line below into your repo +# .gitignore (or rely on fullsend run's UploadDir + .git/info/exclude). +output/ diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index a7cb90abd3..318ca85de4 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -174,14 +174,18 @@ stages: # Read config from default branch (trusted), not MR source branch. # GitHub reads config from pull_request.base.sha; this is the GitLab # equivalent. git fetch is needed because CI shallow clones only - # include the pipeline commit. + # include the pipeline commit. Save the SHA so later steps (eval + # measurement manifests) can reuse the same trusted tip even if + # FETCH_HEAD moves. CONFIG_YAML="" + DEFAULT_BRANCH_SHA="" if [ -n "${CI_DEFAULT_BRANCH:-}" ]; then if ! git fetch origin "${CI_DEFAULT_BRANCH}" --depth=1; then echo "ERROR: cannot fetch default branch — refusing to run without trusted config" exit 1 fi - CONFIG_YAML=$(git show "FETCH_HEAD:.fullsend/config.yaml" 2>/dev/null || echo "") + DEFAULT_BRANCH_SHA=$(git rev-parse FETCH_HEAD) + CONFIG_YAML=$(git show "${DEFAULT_BRANCH_SHA}:.fullsend/config.yaml" 2>/dev/null || echo "") fi # Kill switch — halt all agent dispatch when active @@ -265,15 +269,26 @@ stages: # Construct the entity URL for the harness. EVENT_TYPE values from # the poller use prefixed forms: issue_note, issue_label (→ issue # URL) and mr_note, mr_event (→ merge request URL). + # Always export GITLAB_ISSUE_URL (empty is OK for harness env + # validation). Only set a real URL when the IID is non-empty and + # not "0" — inventing …/issues/0 passes EM-001's work_item check. + GITLAB_ISSUE_URL="" case "${EVENT_TYPE:-}" in issue_*) - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID:-0}" + if [[ -n "${STATUS_IID:-}" && "${STATUS_IID}" != "0" ]]; then + GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID}" + fi ;; *) - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${CI_MERGE_REQUEST_IID:-${STATUS_IID:-0}}" + _fs_mr_iid="${CI_MERGE_REQUEST_IID:-${STATUS_IID:-}}" + if [[ -n "${_fs_mr_iid}" && "${_fs_mr_iid}" != "0" ]]; then + GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${_fs_mr_iid}" + fi + unset _fs_mr_iid export FULLSEND_NOTE_TARGET="merge_requests" ;; esac + export GITLAB_ISSUE_URL # Pre-fetch prior review for the review agent — equivalent to # pre-fetch-prior-review.sh in the GitHub scaffold. Queries the @@ -365,14 +380,52 @@ stages: # Run the agent — fullsend run resolves the harness file, reads # the image field, and creates the sandbox container via Podman. + # Capture the run status so eval-measure still runs after a failed + # agent (failed runs still write run-telemetry.jsonl). + # Eval measurements (fail-open): same CLI as GitHub Actions. Never + # fail the agent job. + # Manifest trust (mirrors kill-switch config): prefer a local override + # from the default branch tip (DEFAULT_BRANCH_SHA), else SHA-pinned + # fetch from public fullsend-ai/agents (GitHub GetRef). Never read + # .fullsend/eval/measurements/ from the MR source tree — that would + # let an MR author change which scorers run or their id@version for + # the job's trend. Do not pass --fullsend-dir to eval-measure here. + # agents is public, so GetRef works without GH_TOKEN, but unauthenticated + # calls share GitHub's ~60 req/hr per-IP limit — export GH_TOKEN / + # GITHUB_TOKEN on busy shared runners. + # Write under $CI_PROJECT_DIR so GitLab can retain artifacts (not an + # ephemeral tmp path). BREAKING CHANGE vs older scaffolds: default + # --output-dir now lives inside the project dir (artifact retention + # + top-level output/ sandbox exclude). Re-sync adopts the new layout. + mkdir -p "${CI_PROJECT_DIR}/output" + set +e fullsend run "${STAGE}" \ --fullsend-dir .fullsend \ --target-repo . \ - --output-dir /tmp/fullsend-output \ + --output-dir "${CI_PROJECT_DIR}/output" \ --forge gitlab \ --run-url "${CI_PIPELINE_URL}" \ --status-repo "${CI_PROJECT_PATH}" \ --status-number "${CI_MERGE_REQUEST_IID:-${STATUS_IID:-0}}" + RUN_STATUS=$? + set -e + + MEASURE_ARGS=(--agent "${STAGE}" --output-dir "${CI_PROJECT_DIR}/output") + if [ -n "${DEFAULT_BRANCH_SHA}" ]; then + if MEASURE_YAML=$(git show "${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/${STAGE}.yaml" 2>/dev/null); then + MEASURE_FILE="${CI_PROJECT_DIR}/output/.fullsend-measure-${STAGE}.yaml" + printf '%s\n' "${MEASURE_YAML}" > "${MEASURE_FILE}" + MEASURE_ARGS+=(--registry "${MEASURE_FILE}") + fi + fi + fullsend eval-measure "${MEASURE_ARGS[@]}" || true + + exit "${RUN_STATUS}" + artifacts: + when: always + paths: + - output/ + expire_in: 1 week resource_group: "fullsend-${STAGE}-${RESOURCE_KEY}" rules: - if: $STAGE diff --git a/internal/scaffold/installfiles.go b/internal/scaffold/installfiles.go index 391e7e735f..5e069d71d5 100644 --- a/internal/scaffold/installfiles.go +++ b/internal/scaffold/installfiles.go @@ -68,6 +68,12 @@ func CollectPerRepoInstallFiles(vendored bool, upstreamRef, upstreamTag string) // CollectGitLabPerRepoInstallFiles gathers CI template files for GitLab // per-repo installation. The embedded .fullsend/config.yaml is excluded — // callers generate a config with roles and forge field instead. +// The embedded root .gitignore is also excluded: installing it would +// overwrite a consumer's existing ignore file with a one-line fragment. +// The embed still ships for docs/tests; adopters (and the guide) add +// output/ themselves. When --output-dir sits inside --target-repo +// (GitLab layout), fullsend run omits that top-level directory from the +// sandbox tarball and .git/info/exclude via outputDirExcludeRel. // runnerTags specifies GitLab runner tags to inject into CI job definitions. // upstreamRef and upstreamTag control the version marker embedded in the // dispatch file for upgrade/status drift detection. @@ -76,7 +82,7 @@ func CollectGitLabPerRepoInstallFiles(runnerTags []string, upstreamRef, upstream versionMarker := FormatVersionMarker(upstreamRef, upstreamTag) var files InstallFiles err := WalkGitLabPerRepo(func(path string, content []byte) error { - if path == ".fullsend/config.yaml" { + if path == ".fullsend/config.yaml" || path == ".gitignore" { return nil } rendered := strings.ReplaceAll(string(content), "__RUNNER_TAGS__", tagYAML) diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index d887a482f3..4762e7a1c2 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -10,6 +10,7 @@ import ( func TestGitLabPerRepoFilesExist(t *testing.T) { expected := []string{ + ".gitignore", ".gitlab-ci.yml", ".fullsend/config.yaml", ".gitlab/ci/fullsend-dispatch.yml", @@ -31,6 +32,22 @@ func TestGitLabConfigContent(t *testing.T) { assert.Contains(t, s, "forge: gitlab") } +func TestGitLabGitignoreExcludesOutput(t *testing.T) { + content, err := GitLabPerRepoFile(".gitignore") + require.NoError(t, err) + s := string(content) + assert.Contains(t, s, "output/") +} + +func TestCollectGitLabPerRepoInstallFiles_SkipsGitignore(t *testing.T) { + files, err := CollectGitLabPerRepoInstallFiles(nil, "", "") + require.NoError(t, err) + for _, f := range files { + assert.NotEqual(t, ".gitignore", f.Path, + "install must not overwrite consumer .gitignore with the scaffold fragment") + } +} + func TestGitLabPerRepoFileNotFound(t *testing.T) { _, err := GitLabPerRepoFile("nonexistent-file.yml") assert.Error(t, err) @@ -155,6 +172,30 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "- agent") // Generic template parameterized by STAGE assert.Contains(t, s, `fullsend run "${STAGE}"`) + assert.Contains(t, s, `fullsend eval-measure`) + assert.Contains(t, s, "RUN_STATUS=$?") + assert.Contains(t, s, `exit "${RUN_STATUS}"`) + assert.Contains(t, s, "|| true") + assert.Contains(t, s, "60 req/hr") + assert.Contains(t, s, "artifacts:") + assert.Contains(t, s, "when: always") + assert.Contains(t, s, `${CI_PROJECT_DIR}/output`) + assert.NotContains(t, s, "/tmp/fullsend-output") + // Measurement override from default branch tip — never MR-tree --fullsend-dir. + assert.Contains(t, s, "DEFAULT_BRANCH_SHA") + assert.Contains(t, s, `git show "${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/${STAGE}.yaml"`) + assert.Contains(t, s, `MEASURE_ARGS+=(--registry "${MEASURE_FILE}")`) + assert.Contains(t, s, `fullsend eval-measure "${MEASURE_ARGS[@]}"`) + assert.NotContains(t, s, `eval-measure \ + --agent "${STAGE}" \ + --fullsend-dir .fullsend`) + // work_item URL must not invent …/issues/0 when IID is missing, but + // GITLAB_ISSUE_URL must still be exported (empty OK) so harness env + // validation does not reject a truly unset variable. + assert.NotContains(t, s, `/-/issues/${STATUS_IID:-0}`) + assert.Contains(t, s, `"${STATUS_IID}" != "0"`) + assert.Contains(t, s, `GITLAB_ISSUE_URL=""`) + assert.Contains(t, s, "export GITLAB_ISSUE_URL") assert.Contains(t, s, "--fullsend-dir") assert.Contains(t, s, "--target-repo") assert.Contains(t, s, "--output-dir") @@ -229,6 +270,7 @@ func TestGitLabAgentTemplateKillSwitch(t *testing.T) { assert.Contains(t, s, "kill_switch: false") // Config read from default branch (trusted), not MR source assert.Contains(t, s, "CI_DEFAULT_BRANCH") + assert.Contains(t, s, "DEFAULT_BRANCH_SHA") assert.Contains(t, s, "FETCH_HEAD") assert.Contains(t, s, "CONFIG_YAML") // Fetch failure fails the job (not silently permissive)