diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66f5d9ff..08cffb92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,15 @@ name: CI on: pull_request: + merge_group: push: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -18,8 +23,8 @@ jobs: matrix: include: - os: ubuntu-latest - label: ubuntu canonical - ci-target: ci + label: ubuntu required shared-linux perf + ci-target: ci-required-shared-linux - os: macos-latest label: macos portability ci-target: ci-portability @@ -67,6 +72,74 @@ jobs: exit 1 fi + formal-security-kernel: + name: Formal security kernel TLA + runs-on: ubuntu-latest + env: + NIX_CONFIG: | + substituters = https://cache.nixos.org + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Detect TLA-relevant PR changes + id: tla_changes + shell: bash + run: | + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "required=true" >> "$GITHUB_OUTPUT" + echo "mode=all" >> "$GITHUB_OUTPUT" + exit 0 + fi + + changed_files="$(git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}")" + if grep -E '^(formal/tla/|tools/tlccheck/|justfile|\.github/workflows/ci\.yml)' <<< "$changed_files"; then + echo "required=true" >> "$GITHUB_OUTPUT" + echo "mode=all" >> "$GITHUB_OUTPUT" + elif grep -E '^(protocol/|internal/|cmd/|runner/|docs/trust-boundaries\.md|runecontext/standards/security/|runecontext/standards/global/|runecontext/changes/CHG-2026-015-[^/]+/)' <<< "$changed_files"; then + echo "required=true" >> "$GITHUB_OUTPUT" + echo "mode=core" >> "$GITHUB_OUTPUT" + else + echo "required=false" >> "$GITHUB_OUTPUT" + echo "mode=skip" >> "$GITHUB_OUTPUT" + fi + + - name: Report TLA skip + if: github.event_name == 'pull_request' && steps.tla_changes.outputs.required != 'true' + run: echo "No security-kernel-relevant changes detected; TLA model check skipped for this PR diff." + + - name: Install Nix + if: steps.tla_changes.outputs.required == 'true' + uses: DeterminateSystems/nix-installer-action@c5a866b6ab867e88becbed4467b93592bce69f8a # v21 + + - name: Cache Nix store + if: steps.tla_changes.outputs.required == 'true' + uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13 + with: + use-flakehub: false + + - name: Run PR core TLA model check + if: github.event_name == 'pull_request' && steps.tla_changes.outputs.mode == 'core' + run: nix develop --no-write-lock-file -c just model-check-core + + - name: Run full TLA model check + if: steps.tla_changes.outputs.mode == 'all' + run: nix develop --no-write-lock-file -c just model-check + + - name: Verify repository unchanged after TLA check + if: steps.tla_changes.outputs.required == 'true' + run: | + git diff --exit-code + untracked="$(git ls-files --others --exclude-standard)" + if [ -n "$untracked" ]; then + echo "Untracked files found after TLA check:" + echo "$untracked" + exit 1 + fi + windows: name: Windows portability (Node ${{ matrix.node-version }}) runs-on: windows-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb619366..6d59a59b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,7 @@ The canonical local workflow uses Nix + `just`: - Prerequisite: Nix `>= 2.18` - Optional auto-entry: `direnv` + `nix-direnv` - Canonical command surface: `just` -- CI runs one canonical `just ci` lane plus portability lanes (`just ci-portability`) to avoid duplicating model-check runtime cost across every matrix leg +- CI runs fast canonical checks plus a dedicated Linux formal-security gate to avoid duplicating model-check runtime cost across every matrix leg ### Use the dev shell manually diff --git a/README.md b/README.md index f1066eff..c85e264f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # RuneCode — Security-first AI coding: isolated execution, signed, auditable [![CI](https://github.com/runecode-ai/runecode/actions/workflows/ci.yml/badge.svg)](https://github.com/runecode-ai/runecode/actions/workflows/ci.yml) -[![Status: alpha.9 in progress](https://img.shields.io/badge/status-alpha.9%20in%20progress-orange)](runecontext/project/roadmap.md) +[![Status: alpha.11 in progress](https://img.shields.io/badge/status-alpha.11%20in%20progress-orange)](runecontext/project/roadmap.md) [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) RuneCode is a security-first agentic automation platform for software engineering. @@ -9,7 +9,7 @@ It treats isolation and cryptographic provenance as co-equal pillars: work runs ## Status -The latest published release is `v0.1.0-alpha.7`, and the repository mainline already includes `v0.1.0-alpha.9` work in progress. +The latest published release is `v0.1.0-alpha.7`, and the repository mainline already includes `v0.1.0-alpha.11` work in progress. RuneCode remains pre-production: the signed, tag-driven release pipeline exists, but the shipped Go binaries are still scaffold-heavy and not feature-complete. ## Why RuneCode @@ -173,7 +173,8 @@ This quick path verifies signed checksums and the signed archive before install. - Signed runtime-image and runtime-toolchain identity contracts, typed verifier-authority state, trusted admission into a launcher-private verified runtime cache, and fail-closed launch from verified local assets rather than mutable host paths or ad hoc launch-time synthesis - Durable launcher runtime evidence persistence and broker-derived authoritative runtime projection for `backend_kind`, `isolation_assurance_level`, `provisioning_posture`, lifecycle, terminal state, and runtime attestation support or verification posture from persisted evidence rather than transient launcher state, with supported `attested` posture only earned after secure-session validation, post-handshake runtime evidence collection, and trusted verification - Broker-owned runtime audit emission for `runtime_launch_admission`, `runtime_launch_denied`, `isolate_session_started`, and `isolate_session_bound`, with reference-heavy payloads bound to persisted launcher evidence digests and later attestation linkage added from persisted post-handshake evidence rather than optimistic launch-time fields -- Checked-in bounded TLA+ security-kernel artifacts plus deterministic TLC model-checking wired into `just model-check` and `just ci` +- Checked-in bounded TLA+ security-kernel artifacts plus deterministic TLC model-checking wired into `just model-check`, `just model-check-core`, and `just ci` +- Reviewed machine-consumed performance contracts under `tools/perfcontracts/`, deterministic performance fixtures and harnesses for TUI, broker, runner or workflow, gateway or dependency or audit or protocol surfaces, and a required shared-Linux CI gate that currently enforces only the checked-in `required_shared_linux` subset while launcher, attestation, and external-anchor surfaces remain informational or `contract_pending_dependency` Still incremental / not implemented end-to-end yet: - Secure-storage posture projection and broader provider auth modes remain incremental, but direct-credential provider setup and execution now exist for OpenAI-compatible and Anthropic-compatible endpoints on the shared provider substrate @@ -240,8 +241,12 @@ Common commands: just fmt just lint just model-check +just model-check-core +just model-check-replay just test +just ci-fast just ci +just ci-required-shared-linux ``` Useful protocol-specific checks: @@ -254,12 +259,14 @@ cd runner && npm test cd runner && npm run boundary-check ``` -These checks are also covered by `just ci`. +These checks are covered by `just ci`, while the required shared-Linux performance-contract subset runs in the dedicated `just ci-required-shared-linux` lane rather than every local `just ci` run. Formal model checking entrypoint: ```sh just model-check +just model-check-core +just model-check-replay ``` Optional: enable automatic dev-shell entry with `direnv` + `nix-direnv`: diff --git a/cmd/runecode-tui/local_rpc_integration_linux_test.go b/cmd/runecode-tui/local_rpc_integration_linux_test.go index 6e483898..475e63bf 100644 --- a/cmd/runecode-tui/local_rpc_integration_linux_test.go +++ b/cmd/runecode-tui/local_rpc_integration_linux_test.go @@ -37,10 +37,7 @@ func TestTUIRoutesUseRealLocalRPCBrokerContracts(t *testing.T) { func startTUILocalRPCServer(t *testing.T) (*brokerapi.LocalIPCListener, *brokerapi.Service, string, <-chan error) { t.Helper() - runtimeDir := filepath.Join(t.TempDir(), "runtime") - if err := os.MkdirAll(runtimeDir, 0o700); err != nil { - t.Fatalf("MkdirAll returned error: %v", err) - } + runtimeDir := shortTUILocalRPCRuntimeDir(t) service, ledgerRoot := newTUILocalRPCService(t) listener, err := brokerapi.ListenLocalIPC(brokerapi.LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { @@ -54,6 +51,16 @@ func startTUILocalRPCServer(t *testing.T) (*brokerapi.LocalIPCListener, *brokera return listener, service, ledgerRoot, errCh } +func shortTUILocalRPCRuntimeDir(t *testing.T) string { + t.Helper() + runtimeDir, err := os.MkdirTemp("", "rc-tui-rpc-") + if err != nil { + t.Fatalf("MkdirTemp returned error: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(runtimeDir) }) + return runtimeDir +} + func configureTUILocalRPCClient(t *testing.T, runtimeDir string) { t.Helper() origConfigProvider := localIPCConfigProvider diff --git a/cmd/runecode-tui/shell_watch_test.go b/cmd/runecode-tui/shell_watch_test.go index 429ebdcd..860b3b3c 100644 --- a/cmd/runecode-tui/shell_watch_test.go +++ b/cmd/runecode-tui/shell_watch_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "testing" "time" @@ -8,6 +9,52 @@ import ( "github.com/runecode-ai/runecode/internal/brokerapi" ) +type watchPollRequestRecorder struct { + *fakeBrokerClient + run brokerapi.RunWatchRequest + approval brokerapi.ApprovalWatchRequest + session brokerapi.SessionWatchRequest +} + +func (r *watchPollRequestRecorder) RunWatch(ctx context.Context, req brokerapi.RunWatchRequest) ([]brokerapi.RunWatchEvent, error) { + r.run = req + return r.fakeBrokerClient.RunWatch(ctx, req) +} + +func (r *watchPollRequestRecorder) ApprovalWatch(ctx context.Context, req brokerapi.ApprovalWatchRequest) ([]brokerapi.ApprovalWatchEvent, error) { + r.approval = req + return r.fakeBrokerClient.ApprovalWatch(ctx, req) +} + +func (r *watchPollRequestRecorder) SessionWatch(ctx context.Context, req brokerapi.SessionWatchRequest) ([]brokerapi.SessionWatchEvent, error) { + r.session = req + return r.fakeBrokerClient.SessionWatch(ctx, req) +} + +func TestShellWatchPollRequestsSnapshotOnlyStreams(t *testing.T) { + m := newShellModel() + recorder := &watchPollRequestRecorder{fakeBrokerClient: &fakeBrokerClient{}} + m.client = recorder + + msg, ok := m.loadWatchPollCmd()().(shellWatchTransportLoadedMsg) + if !ok { + t.Fatalf("loadWatchPollCmd message = %T, want shellWatchTransportLoadedMsg", msg) + } + assertWatchPollSnapshotOnly(t, recorder.run.IncludeSnapshot, recorder.run.Follow, "run") + assertWatchPollSnapshotOnly(t, recorder.approval.IncludeSnapshot, recorder.approval.Follow, "approval") + assertWatchPollSnapshotOnly(t, recorder.session.IncludeSnapshot, recorder.session.Follow, "session") +} + +func assertWatchPollSnapshotOnly(t *testing.T, includeSnapshot, follow bool, family string) { + t.Helper() + if !includeSnapshot { + t.Fatalf("%s watch poll IncludeSnapshot = false, want true", family) + } + if follow { + t.Fatalf("%s watch poll Follow = true, want false", family) + } +} + func TestShellWatchManagerFamilySpecificFailureProjectsDegradedHealth(t *testing.T) { manager := newShellWatchManager() now := time.Date(2026, 4, 16, 12, 0, 0, 0, time.UTC) diff --git a/cmd/runecode-tui/shell_watch_transport.go b/cmd/runecode-tui/shell_watch_transport.go index ee052d3b..251b5852 100644 --- a/cmd/runecode-tui/shell_watch_transport.go +++ b/cmd/runecode-tui/shell_watch_transport.go @@ -67,9 +67,9 @@ func (m shellModel) loadWatchPollCmd() tea.Cmd { ctx, cancel := withLoadTimeout() defer cancel() - runEvents, runErr := m.client.RunWatch(ctx, brokerapi.RunWatchRequest{StreamID: newRequestID("shell-run-watch-stream"), IncludeSnapshot: true, Follow: true}) - approvalEvents, approvalErr := m.client.ApprovalWatch(ctx, brokerapi.ApprovalWatchRequest{StreamID: newRequestID("shell-approval-watch-stream"), IncludeSnapshot: true, Follow: true}) - sessionEvents, sessionErr := m.client.SessionWatch(ctx, brokerapi.SessionWatchRequest{StreamID: newRequestID("shell-session-watch-stream"), IncludeSnapshot: true, Follow: true}) + runEvents, runErr := m.client.RunWatch(ctx, brokerapi.RunWatchRequest{StreamID: newRequestID("shell-run-watch-stream"), IncludeSnapshot: true}) + approvalEvents, approvalErr := m.client.ApprovalWatch(ctx, brokerapi.ApprovalWatchRequest{StreamID: newRequestID("shell-approval-watch-stream"), IncludeSnapshot: true}) + sessionEvents, sessionErr := m.client.SessionWatch(ctx, brokerapi.SessionWatchRequest{StreamID: newRequestID("shell-session-watch-stream"), IncludeSnapshot: true}) return shellWatchTransportLoadedMsg{ Run: shellWatchRunTransportResult{Events: runEvents, Err: runErr}, diff --git a/formal/tla/security-kernel/README.md b/formal/tla/security-kernel/README.md index 56ad8763..717c810d 100644 --- a/formal/tla/security-kernel/README.md +++ b/formal/tla/security-kernel/README.md @@ -51,3 +51,9 @@ TLC wiring is owned by the CI/tooling lane. When TLC tooling is available, run w - configs: - `formal/tla/security-kernel/SecurityKernelV0.core.cfg` - `formal/tla/security-kernel/SecurityKernelV0.replay.cfg` + +Convenience recipes: + +- `just model-check-core` runs the faster core PR gate. +- `just model-check-replay` runs the broader replay model. +- `just model-check` runs both and is included in full local `just ci` parity. diff --git a/go.mod b/go.mod index 8cecb1ab..af2e4983 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/creack/pty v1.1.24 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 9b1aa713..b8cf1fca 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= diff --git a/internal/brokerapi/local_api_llm_execution.go b/internal/brokerapi/local_api_llm_execution.go index 96f665e4..4042eb33 100644 --- a/internal/brokerapi/local_api_llm_execution.go +++ b/internal/brokerapi/local_api_llm_execution.go @@ -139,9 +139,10 @@ func (s *Service) executeProviderRequest(ctx context.Context, execCtx llmExecuti } req, errResp := s.buildProviderHTTPRequest(ctx, execCtx, body, started) if errResp != nil { - return "", int64(len(body)), started, time.Now().UTC(), errResp + return "", int64(len(body)), started, normalizeExecutionCompletedAt(started, time.Now().UTC()), errResp } respBody, completed, errResp := s.doProviderHTTPRequest(execCtx, req) + completed = normalizeExecutionCompletedAt(started, completed) if errResp != nil { return "", int64(len(body)), started, completed, errResp } @@ -152,6 +153,13 @@ func (s *Service) executeProviderRequest(ctx context.Context, execCtx llmExecuti return text, int64(len(body)), started, completed, nil } +func normalizeExecutionCompletedAt(started, completed time.Time) time.Time { + if completed.After(started) { + return completed + } + return started.Add(time.Millisecond) +} + func (s *Service) prepareProviderRequestBody(requestID string, translated map[string]any) ([]byte, time.Time, *ErrorResponse) { body, err := json.Marshal(translated) if err != nil { diff --git a/internal/brokerapi/local_rpc_client_linux_test.go b/internal/brokerapi/local_rpc_client_linux_test.go index f5c9a446..90f86a77 100644 --- a/internal/brokerapi/local_rpc_client_linux_test.go +++ b/internal/brokerapi/local_rpc_client_linux_test.go @@ -148,7 +148,7 @@ func assertLocalRPCSocketPath(t *testing.T, runtimeDir string) { func setupLocalRPCRunListRoundTrip(t *testing.T, service *Service) (string, *LocalRPCClient, chan error) { t.Helper() - runtimeDir := filepath.Join(t.TempDir(), "runtime") + runtimeDir := shortLocalRPCRuntimeDir(t) listener, err := ListenLocalIPC(LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { t.Fatalf("ListenLocalIPC returned error: %v", err) @@ -172,7 +172,7 @@ func setupLocalRPCRunListRoundTrip(t *testing.T, service *Service) (string, *Loc func setupLocalRPCSessionListRoundTrip(t *testing.T, service *Service) (string, *LocalRPCClient, chan error) { t.Helper() - runtimeDir := filepath.Join(t.TempDir(), "runtime") + runtimeDir := shortLocalRPCRuntimeDir(t) listener, err := ListenLocalIPC(LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { t.Fatalf("ListenLocalIPC returned error: %v", err) @@ -196,7 +196,7 @@ func setupLocalRPCSessionListRoundTrip(t *testing.T, service *Service) (string, func setupLocalRPCSessionSendMessageRoundTrip(t *testing.T, service *Service) (string, *LocalRPCClient, chan error) { t.Helper() - runtimeDir := filepath.Join(t.TempDir(), "runtime") + runtimeDir := shortLocalRPCRuntimeDir(t) listener, err := ListenLocalIPC(LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { t.Fatalf("ListenLocalIPC returned error: %v", err) @@ -220,7 +220,7 @@ func setupLocalRPCSessionSendMessageRoundTrip(t *testing.T, service *Service) (s func setupLocalRPCSessionExecutionTriggerRoundTrip(t *testing.T, service *Service) (string, *LocalRPCClient, chan error) { t.Helper() - runtimeDir := filepath.Join(t.TempDir(), "runtime") + runtimeDir := shortLocalRPCRuntimeDir(t) listener, err := ListenLocalIPC(LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { t.Fatalf("ListenLocalIPC returned error: %v", err) @@ -427,7 +427,7 @@ func TestValidateRawMessageLimitsRejectsLargePayload(t *testing.T) { } func TestLocalRPCClientInvokeRespectsContextDeadline(t *testing.T) { - runtimeDir := filepath.Join(t.TempDir(), "runtime") + runtimeDir := shortLocalRPCRuntimeDir(t) listener, err := ListenLocalIPC(LocalIPCConfig{RuntimeDir: runtimeDir, SocketName: "broker.sock"}) if err != nil { t.Fatalf("ListenLocalIPC returned error: %v", err) @@ -457,3 +457,13 @@ func TestLocalRPCClientInvokeRespectsContextDeadline(t *testing.T) { t.Fatalf("error code = %q, want request_cancelled", errResp.Error.Code) } } + +func shortLocalRPCRuntimeDir(t *testing.T) string { + t.Helper() + runtimeDir, err := os.MkdirTemp("", "rc-rpc-") + if err != nil { + t.Fatalf("MkdirTemp returned error: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(runtimeDir) }) + return runtimeDir +} diff --git a/internal/brokerapi/perf_gateway_secrets.go b/internal/brokerapi/perf_gateway_secrets.go new file mode 100644 index 00000000..c399c026 --- /dev/null +++ b/internal/brokerapi/perf_gateway_secrets.go @@ -0,0 +1,155 @@ +package brokerapi + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/trustpolicy" +) + +func measurePhase5GatewayAndSecrets(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + rig, err := newPhase5GatewayPerfRig(repoRoot) + if err != nil { + return nil, err + } + defer rig.cleanup() + invokeP95, err := phase5TrialP95(trials, rig.invokeGatewayTrial) + if err != nil { + return nil, err + } + leaseP95, err := phase5TrialP95(trials, rig.issueLeaseTrial) + if err != nil { + return nil, err + } + ingressP95, err := phase5TrialP95(trials, rig.ingressPrepareSubmitTrial) + if err != nil { + return nil, err + } + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.gateway.model_invoke.overhead.p95_ms", Value: invokeP95, Unit: "ms"}, + {MetricID: "metric.secrets.lease_issue.p95_ms", Value: leaseP95, Unit: "ms"}, + {MetricID: "metric.secrets.ingress.prepare_submit.p95_ms", Value: ingressP95, Unit: "ms"}, + }, nil +} + +func phase5MeasureMS(call func()) float64 { + started := time.Now() + call() + return float64(time.Since(started).Microseconds()) / 1000.0 +} + +func phase5MeasureMSErr(call func() error) (float64, error) { + started := time.Now() + if err := call(); err != nil { + return 0, err + } + return float64(time.Since(started).Microseconds()) / 1000.0, nil +} + +func phase5TrialP95(trials int, trial func(int) error) (float64, error) { + samples := make([]float64, 0, trials) + for i := 0; i < trials; i++ { + ms, err := phase5MeasureMSErr(func() error { return trial(i) }) + if err != nil { + return 0, err + } + samples = append(samples, ms) + } + return phase5P95(samples) +} + +type phase5GatewayPerfRig struct { + service *Service + runID string + providerProfileID string + llmRequest map[string]any + requestDigest trustpolicy.Digest + cleanupFn func() +} + +func newPhase5GatewayPerfRig(repoRoot string) (*phase5GatewayPerfRig, error) { + service, cleanup, err := newPhase5GatewayPerfService(repoRoot) + if err != nil { + return nil, err + } + rig := &phase5GatewayPerfRig{service: service, runID: "run-phase5-gateway", cleanupFn: cleanup} + if err := putPhase5TrustedModelGatewayContext(service, rig.runID); err != nil { + rig.cleanup() + return nil, err + } + if err := rig.seedProviderAndLLMRequest(); err != nil { + rig.cleanup() + return nil, err + } + return rig, nil +} + +func (r *phase5GatewayPerfRig) cleanup() { + if r != nil && r.cleanupFn != nil { + r.cleanupFn() + } +} + +func (r *phase5GatewayPerfRig) invokeGatewayTrial(iteration int) error { + req := LLMInvokeRequest{ + SchemaID: "runecode.protocol.v0.LLMInvokeRequest", + SchemaVersion: "0.1.0", + RequestID: fmt.Sprintf("req-phase5-llm-invoke-%d", iteration), + RunID: r.runID, + LLMRequest: r.llmRequest, + RequestDigest: &r.requestDigest, + } + _, errResp := r.service.HandleLLMInvoke(context.Background(), req, RequestContext{}) + if errResp == nil { + return nil + } + return fmt.Errorf("llm invoke: %s (%s)", errResp.Error.Code, strings.TrimSpace(errResp.Error.Message)) +} + +func (r *phase5GatewayPerfRig) issueLeaseTrial(iteration int) error { + req := ProviderCredentialLeaseIssueRequest{ + SchemaID: "runecode.protocol.v0.ProviderCredentialLeaseIssueRequest", + SchemaVersion: "0.1.0", + RequestID: fmt.Sprintf("req-phase5-lease-%d", iteration), + ProviderProfileID: r.providerProfileID, + RunID: r.runID, + TTLSeconds: 120, + } + _, errResp := r.service.HandleProviderCredentialLeaseIssue(context.Background(), req, RequestContext{}) + return phase5DependencyErr("provider lease issue", errResp) +} + +func (r *phase5GatewayPerfRig) ingressPrepareSubmitTrial(iteration int) error { + beginResp, err := r.beginProviderSetupSession(fmt.Sprintf("req-phase5-ingress-begin-%d", iteration), fmt.Sprintf("phase5-ingress-%d.example.com", iteration)) + if err != nil { + return err + } + return r.submitIngressForSession(beginResp.SetupSession.SetupSessionID, fmt.Sprintf("%d", iteration)) +} + +func (r *phase5GatewayPerfRig) submitIngressForSession(setupSessionID string, suffix string) error { + prepareResp, prepareErr := r.service.HandleProviderSetupSecretIngressPrepare(context.Background(), ProviderSetupSecretIngressPrepareRequest{ + SchemaID: "runecode.protocol.v0.ProviderSetupSecretIngressPrepareRequest", + SchemaVersion: "0.1.0", + RequestID: "req-phase5-ingress-prepare-" + suffix, + SetupSessionID: setupSessionID, + IngressChannel: "cli_stdin", + CredentialField: "api_key", + }, RequestContext{}) + if err := phase5DependencyErr("provider ingress prepare", prepareErr); err != nil { + return err + } + _, submitErr := r.service.HandleProviderSetupSecretIngressSubmit(context.Background(), ProviderSetupSecretIngressSubmitRequest{ + SchemaID: "runecode.protocol.v0.ProviderSetupSecretIngressSubmitRequest", + SchemaVersion: "0.1.0", + RequestID: "req-phase5-ingress-submit-" + suffix, + SecretIngressToken: prepareResp.SecretIngressToken, + }, []byte("phase5-secret"), RequestContext{}) + if err := phase5DependencyErr("provider ingress submit", submitErr); err != nil { + return err + } + return nil +} diff --git a/internal/brokerapi/perf_gateway_secrets_context.go b/internal/brokerapi/perf_gateway_secrets_context.go new file mode 100644 index 00000000..2202eb50 --- /dev/null +++ b/internal/brokerapi/perf_gateway_secrets_context.go @@ -0,0 +1,220 @@ +package brokerapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/perffixtures" + "github.com/runecode-ai/runecode/third_party/jsoncanonicalizer" +) + +func (r *phase5GatewayPerfRig) seedProviderAndLLMRequest() error { + beginResp, err := r.beginProviderSetupSession("req-phase5-provider-begin", "model.example.com") + if err != nil { + return err + } + r.providerProfileID = beginResp.Profile.ProviderProfileID + r.llmRequest = phase5GatewayLLMRequest(r.providerProfileID) + if err := r.submitIngressForSession(beginResp.SetupSession.SetupSessionID, "seed"); err != nil { + return err + } + if err := r.commitProviderValidation(); err != nil { + return err + } + if err := r.putGatewayPromptArtifact(); err != nil { + return err + } + return r.putLLMRequestArtifact() +} + +func (r *phase5GatewayPerfRig) beginProviderSetupSession(requestID, canonicalHost string) (ProviderSetupSessionBeginResponse, error) { + beginResp, errResp := r.service.HandleProviderSetupSessionBegin(context.Background(), ProviderSetupSessionBeginRequest{ + SchemaID: "runecode.protocol.v0.ProviderSetupSessionBeginRequest", + SchemaVersion: "0.1.0", + RequestID: requestID, + DisplayLabel: "Phase5 Gateway", + ProviderFamily: providerFamilyOpenAICompatible, + AdapterKind: providerAdapterKindOpenAIChatCompletionsV0, + CanonicalHost: canonicalHost, + CanonicalPathPrefix: "/v1", + AllowlistedModelIDs: []string{"gpt-4.1-mini"}, + }, RequestContext{}) + if err := phase5DependencyErr("provider setup begin", errResp); err != nil { + return ProviderSetupSessionBeginResponse{}, err + } + return beginResp, nil +} + +func (r *phase5GatewayPerfRig) commitProviderValidation() error { + validationBegin, validationErr := r.service.HandleProviderValidationBegin(context.Background(), ProviderValidationBeginRequest{ + SchemaID: "runecode.protocol.v0.ProviderValidationBeginRequest", + SchemaVersion: "0.1.0", + RequestID: "req-phase5-provider-validation-begin", + ProviderProfileID: r.providerProfileID, + }, RequestContext{}) + if err := phase5DependencyErr("provider validation begin", validationErr); err != nil { + return err + } + _, validationCommitErr := r.service.HandleProviderValidationCommit(context.Background(), ProviderValidationCommitRequest{ + SchemaID: "runecode.protocol.v0.ProviderValidationCommitRequest", + SchemaVersion: "0.1.0", + RequestID: "req-phase5-provider-validation-commit", + ProviderProfileID: r.providerProfileID, + ValidationAttemptID: validationBegin.ValidationAttemptID, + ConnectivityState: "reachable", + CompatibilityState: "compatible", + }, RequestContext{}) + return phase5DependencyErr("provider validation commit", validationCommitErr) +} + +func (r *phase5GatewayPerfRig) putGatewayPromptArtifact() error { + inputRef, err := r.service.Put(artifacts.PutRequest{ + Payload: []byte("phase5 gateway prompt"), + ContentType: "text/plain", + DataClass: artifacts.DataClassSpecText, + ProvenanceReceiptHash: "sha256:" + strings.Repeat("c", 64), + CreatedByRole: "broker", + TrustedSource: true, + RunID: r.runID, + StepID: "phase5-gateway-input", + }) + if err != nil { + return err + } + requestArtifacts, ok := r.llmRequest["input_artifacts"].([]any) + if !ok || len(requestArtifacts) == 0 { + return fmt.Errorf("phase5 gateway request missing input_artifacts") + } + artifact, ok := requestArtifacts[0].(map[string]any) + if !ok { + return fmt.Errorf("phase5 gateway request input_artifact malformed") + } + artifact["digest"] = phase5DigestObject(inputRef.Digest) + artifact["size_bytes"] = len("phase5 gateway prompt") + return nil +} + +func phase5GatewayLLMRequest(providerProfileID string) map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.LLMRequest", + "schema_version": "0.3.0", + "selection_source": "signed_allowlist", + "provider": providerProfileID, + "model": "gpt-4.1-mini", + "input_artifacts": []any{map[string]any{ + "schema_id": "runecode.protocol.v0.ArtifactReference", + "schema_version": "0.4.0", + "digest": map[string]any{}, + "size_bytes": 0, + "content_type": "text/plain", + "data_class": "spec_text", + "provenance_receipt_hash": map[string]any{ + "hash_alg": "sha256", + "hash": strings.Repeat("d", 64), + }, + }}, + "tool_allowlist": []any{map[string]any{ + "tool_name": "noop", + "arguments_schema_id": "runecode.protocol.tools.noop.args", + "arguments_schema_version": "0.1.0", + }}, + "response_mode": "text", + "streaming_mode": "stream", + "request_limits": map[string]any{ + "max_request_bytes": 262144, + "max_tool_calls": 8, + "max_total_tool_call_argument_bytes": 65536, + "max_structured_output_bytes": 262144, + "max_streamed_bytes": 16777216, + "max_stream_chunk_bytes": 65536, + "stream_idle_timeout_ms": 15000, + }, + } +} + +func (r *phase5GatewayPerfRig) putLLMRequestArtifact() error { + raw, err := json.Marshal(r.llmRequest) + if err != nil { + return err + } + canonical, err := jsoncanonicalizer.Transform(raw) + if err != nil { + return err + } + if err := json.Unmarshal(canonical, &r.llmRequest); err != nil { + return err + } + digest, err := canonicalDigestForValue(r.llmRequest) + if err != nil { + return err + } + r.requestDigest = digest + _, err = r.service.Put(artifacts.PutRequest{ + Payload: canonical, + ContentType: "application/json", + DataClass: artifacts.DataClassSpecText, + ProvenanceReceiptHash: "sha256:" + strings.Repeat("a", 64), + CreatedByRole: "broker", + TrustedSource: true, + RunID: r.runID, + StepID: "phase5-gateway-request", + }) + return err +} + +func newPhase5GatewayPerfService(repoRoot string) (*Service, func(), error) { + storeRoot, err := os.MkdirTemp("", "runecode-phase5-gateway-store-") + if err != nil { + return nil, nil, err + } + ledgerRoot, err := os.MkdirTemp("", "runecode-phase5-gateway-ledger-") + if err != nil { + _ = os.RemoveAll(storeRoot) + return nil, nil, err + } + cleanup := func() { + _ = os.RemoveAll(storeRoot) + _ = os.RemoveAll(ledgerRoot) + } + service, err := NewServiceWithConfig(storeRoot, ledgerRoot, APIConfig{RepositoryRoot: repoRoot}) + if err != nil { + cleanup() + return nil, nil, err + } + service.gatewayRuntime.resolver = phase5GatewayStaticResolver{} + originalClient := newLLMHTTPClient + newLLMHTTPClient = func() llmHTTPClient { + return &http.Client{Transport: phase5StubProviderTransport{backend: perffixtures.StubProviderBackend{}}} + } + return service, func() { + newLLMHTTPClient = originalClient + cleanup() + }, nil +} + +type phase5GatewayStaticResolver struct{} + +func (phase5GatewayStaticResolver) LookupIP(_ context.Context, _ string, _ string) ([]net.IP, error) { + return []net.IP{net.ParseIP("93.184.216.34")}, nil +} + +type phase5StubProviderTransport struct { + backend perffixtures.StubProviderBackend +} + +func (t phase5StubProviderTransport) RoundTrip(req *http.Request) (*http.Response, error) { + response := t.backend.Invoke(req.Context(), perffixtures.StubProviderRequest{Prompt: "phase5"}) + body, err := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": response.Text}}}}) + if err != nil { + return nil, err + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(body)), Request: req}, nil +} diff --git a/internal/brokerapi/perf_gateway_secrets_policy.go b/internal/brokerapi/perf_gateway_secrets_policy.go new file mode 100644 index 00000000..c78d9f5d --- /dev/null +++ b/internal/brokerapi/perf_gateway_secrets_policy.go @@ -0,0 +1,100 @@ +package brokerapi + +import ( + "encoding/json" + + "github.com/runecode-ai/runecode/internal/artifacts" +) + +func putPhase5TrustedModelGatewayContext(service *Service, runID string) error { + verifier, privateKey, err := phase5VerifierFixture() + if err != nil { + return err + } + if err := phase5PutTrustedVerifierRecord(service, verifier); err != nil { + return err + } + allowlistPayload, err := json.Marshal(phase5GatewayAllowlistPayload()) + if err != nil { + return err + } + allowlistDigest, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindPolicyAllowlist, allowlistPayload) + if err != nil { + return err + } + rolePayload, err := phase5SignedPayloadForTrustedContext(phase5GatewayRoleManifest(runID, allowlistDigest), verifier, privateKey) + if err != nil { + return err + } + runPayload, err := phase5SignedPayloadForTrustedContext(phase5GatewayRunCapability(runID, allowlistDigest), verifier, privateKey) + if err != nil { + return err + } + if _, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindRoleManifest, rolePayload); err != nil { + return err + } + if _, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindRunCapability, runPayload); err != nil { + return err + } + return nil +} + +func phase5GatewayAllowlistPayload() map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.PolicyAllowlist", + "schema_version": "0.1.0", + "allowlist_kind": "gateway_scope_rule", + "entry_schema_id": "runecode.protocol.v0.GatewayScopeRule", + "entries": []any{map[string]any{ + "schema_id": "runecode.protocol.v0.GatewayScopeRule", + "schema_version": "0.1.0", + "scope_kind": "gateway_destination", + "entry_id": "model_default", + "gateway_role_kind": "model-gateway", + "destination": phase5GatewayDestinationDescriptor(), + "permitted_operations": []any{"invoke_model"}, + "allowed_egress_data_classes": []any{"spec_text"}, + "redirect_posture": "allowlist_only", + "max_timeout_seconds": 120, + "max_response_bytes": 16 << 20, + }}, + } +} + +func phase5GatewayDestinationDescriptor() map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.DestinationDescriptor", + "schema_version": "0.1.0", + "descriptor_kind": "model_endpoint", + "canonical_host": "model.example.com", + "tls_required": true, + "private_range_blocking": "enforced", + "dns_rebinding_protection": "enforced", + } +} + +func phase5GatewayRoleManifest(runID string, allowlistDigest string) map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.RoleManifest", + "schema_version": "0.2.0", + "principal": phase5SignedContextPrincipal(runID, "gateway", "model-gateway"), + "role_family": "gateway", + "role_kind": "model-gateway", + "approval_profile": "moderate", + "capability_opt_ins": []any{"cap_gateway"}, + "allowlist_refs": []any{phase5DigestObject(allowlistDigest)}, + } +} + +func phase5GatewayRunCapability(runID string, allowlistDigest string) map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.CapabilityManifest", + "schema_version": "0.2.0", + "principal": phase5SignedContextPrincipal(runID, "gateway", "model-gateway"), + "manifest_scope": "run", + "run_id": runID, + "approval_profile": "moderate", + "capability_opt_ins": []any{"cap_gateway"}, + "allowlist_refs": []any{phase5DigestObject(allowlistDigest)}, + } +} diff --git a/internal/brokerapi/perf_phase5_anchor.go b/internal/brokerapi/perf_phase5_anchor.go new file mode 100644 index 00000000..1ed2ec7d --- /dev/null +++ b/internal/brokerapi/perf_phase5_anchor.go @@ -0,0 +1,121 @@ +package brokerapi + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func measurePhase5ExternalAnchorStubbed(trials int) []perfcontracts.MeasurementRecord { + target := newPhase5ExternalAnchorStub() + prepareSamples, deferredSamples, completedSamples, visibilitySamples, receiptSamples := phase5AnchorSamples(trials, target) + prepareP95, _ := phase5P95(prepareSamples) + deferredP95, _ := phase5P95(deferredSamples) + completedP95, _ := phase5P95(completedSamples) + visibilityP95, _ := phase5P95(visibilitySamples) + receiptP95, _ := phase5P95(receiptSamples) + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.anchor.prepare.latency.p95_ms", Value: prepareP95, Unit: "ms"}, + {MetricID: "metric.anchor.execute.completed.p95_ms", Value: completedP95, Unit: "ms"}, + {MetricID: "metric.anchor.execute.deferred.handoff.p95_ms", Value: deferredP95, Unit: "ms"}, + {MetricID: "metric.anchor.deferred.visibility.p95_ms", Value: visibilityP95, Unit: "ms"}, + {MetricID: "metric.anchor.receipt_admission.unchanged_seal.p95_ms", Value: receiptP95, Unit: "ms"}, + {MetricID: "metric.anchor.network_io_under_ledger_lock.count", Value: float64(target.networkUnderLock.Load()), Unit: "count"}, + {MetricID: "metric.anchor.verifier_bypass.count", Value: float64(target.verifierBypass.Load()), Unit: "count"}, + } +} + +func phase5AnchorSamples(trials int, target *phase5ExternalAnchorStub) ([]float64, []float64, []float64, []float64, []float64) { + prepareSamples := make([]float64, 0, trials) + deferredSamples := make([]float64, 0, trials) + completedSamples := make([]float64, 0, trials) + visibilitySamples := make([]float64, 0, trials) + receiptSamples := make([]float64, 0, trials) + for i := 0; i < trials; i++ { + prepareMS, completedMS, deferredMS, visibilityMS, receiptMS := phase5AnchorTrial(i, target) + prepareSamples = append(prepareSamples, prepareMS) + completedSamples = append(completedSamples, completedMS) + deferredSamples = append(deferredSamples, deferredMS) + visibilitySamples = append(visibilitySamples, visibilityMS) + receiptSamples = append(receiptSamples, receiptMS) + } + return prepareSamples, deferredSamples, completedSamples, visibilitySamples, receiptSamples +} + +func phase5AnchorTrial(i int, target *phase5ExternalAnchorStub) (float64, float64, float64, float64, float64) { + seal := fmt.Sprintf("sha256:%064d", i+1) + prepareMS := phase5MeasureMS(func() { target.Prepare(seal) }) + completedMS := phase5MeasureMS(func() { target.ExecuteFastComplete(seal) }) + deferredSeal := fmt.Sprintf("sha256:%064d", i+100) + target.Prepare(deferredSeal) + var requestID string + deferredMS := phase5MeasureMS(func() { requestID = target.ExecuteDeferred(deferredSeal) }) + visibilityMS := phase5MeasureMS(func() { _ = target.WaitCompleted(requestID, 2*time.Second) }) + receiptMS := phase5MeasureMS(func() { target.AdmitReceiptUnchangedSeal(seal) }) + return prepareMS, completedMS, deferredMS, visibilityMS, receiptMS +} + +type phase5ExternalAnchorStub struct { + mu sync.Mutex + statusByRequest map[string]string + sealVerified map[string]struct{} + nextID int + networkUnderLock atomic.Int64 + verifierBypass atomic.Int64 +} + +func newPhase5ExternalAnchorStub() *phase5ExternalAnchorStub { + return &phase5ExternalAnchorStub{statusByRequest: map[string]string{}, sealVerified: map[string]struct{}{}} +} + +func (s *phase5ExternalAnchorStub) Prepare(sealDigest string) { + s.mu.Lock() + defer s.mu.Unlock() + s.sealVerified[sealDigest] = struct{}{} +} + +func (s *phase5ExternalAnchorStub) ExecuteFastComplete(_ string) { time.Sleep(1 * time.Millisecond) } + +func (s *phase5ExternalAnchorStub) ExecuteDeferred(_ string) string { + s.mu.Lock() + s.nextID++ + requestID := fmt.Sprintf("deferred-%d", s.nextID) + s.statusByRequest[requestID] = "deferred" + s.mu.Unlock() + go func(id string) { + time.Sleep(5 * time.Millisecond) + s.mu.Lock() + s.statusByRequest[id] = "completed" + s.mu.Unlock() + }(requestID) + return requestID +} + +func (s *phase5ExternalAnchorStub) WaitCompleted(requestID string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + s.mu.Lock() + status := s.statusByRequest[requestID] + s.mu.Unlock() + if status == "completed" { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(1 * time.Millisecond) + } +} + +func (s *phase5ExternalAnchorStub) AdmitReceiptUnchangedSeal(sealDigest string) { + s.mu.Lock() + _, ok := s.sealVerified[sealDigest] + s.mu.Unlock() + if !ok { + s.verifierBypass.Add(1) + } + time.Sleep(1 * time.Millisecond) +} diff --git a/internal/brokerapi/perf_phase5_checks.go b/internal/brokerapi/perf_phase5_checks.go new file mode 100644 index 00000000..937a1be6 --- /dev/null +++ b/internal/brokerapi/perf_phase5_checks.go @@ -0,0 +1,147 @@ +package brokerapi + +import ( + "context" + "fmt" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func measurePhase5AuditVerification( + trials int, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, command ...string) (float64, error), +) ([]perfcontracts.MeasurementRecord, error) { + verifyRun := func() (float64, error) { + return runner(repoRoot, timeout, "go", "test", "./internal/auditd", "-run", "TestVerifyCurrentSegmentIncrementalWithPreverifiedSealPersistsReport", "-count=1") + } + verifyMS, err := phase5WarmupThenMedianCommandLatency(trials, verifyRun) + if err != nil { + return nil, fmt.Errorf("audit verify fixture check failed: %w", err) + } + finalizeRun := func() (float64, error) { + return runner(repoRoot, timeout, "go", "test", "./internal/brokerapi", "-run", "TestHandleAuditFinalizeVerifyPersistsVerificationReportForCurrentSeal", "-count=1") + } + finalizeMS, err := phase5WarmupThenMedianCommandLatency(trials, finalizeRun) + if err != nil { + return nil, fmt.Errorf("audit finalize verify fixture check failed: %w", err) + } + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.audit.verify_current_segment.wall_ms", Value: verifyMS, Unit: "ms"}, + {MetricID: "metric.audit.finalize_verify.wall_ms", Value: finalizeMS, Unit: "ms"}, + }, nil +} + +func measurePhase5ProtocolChecks( + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, command ...string) (float64, error), +) ([]perfcontracts.MeasurementRecord, error) { + schemaMS, err := runner(repoRoot, timeout, "go", "test", "./internal/protocolschema") + if err != nil { + return nil, fmt.Errorf("protocol schema validation check failed: %w", err) + } + fixtureMS, err := runner(repoRoot, timeout, "node", "--test", "scripts/protocol-fixtures.test.js") + if err != nil { + return nil, fmt.Errorf("protocol fixture parity check failed: %w", err) + } + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.protocol.schema_validation.wall_ms", Value: schemaMS, Unit: "ms"}, + {MetricID: "metric.protocol.fixture_parity.wall_ms", Value: fixtureMS, Unit: "ms"}, + }, nil +} + +func phase5RunCommand(repoRoot string, timeout time.Duration, command ...string) (float64, error) { + if len(command) == 0 { + return 0, fmt.Errorf("command required") + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, command[0], command[1:]...) + cmd.Dir = phase5CommandDir(repoRoot, command[0]) + start := time.Now() + output, err := cmd.CombinedOutput() + if err != nil { + return 0, phase5CommandError(command, output, err) + } + return float64(time.Since(start).Microseconds()) / 1000.0, nil +} + +func phase5CommandDir(repoRoot, bin string) string { + if bin == "node" { + return filepath.Join(repoRoot, "runner") + } + return repoRoot +} + +func phase5CommandError(command []string, output []byte, runErr error) error { + msg := strings.TrimSpace(string(output)) + if msg == "" { + msg = runErr.Error() + } + return fmt.Errorf("%s failed: %s", strings.Join(command, " "), msg) +} + +func phase5WarmupThenMedianCommandLatency(trials int, run func() (float64, error)) (float64, error) { + if _, err := run(); err != nil { + return 0, err + } + return phase5MedianCommandLatency(trials, run) +} + +func phase5MedianCommandLatency(trials int, run func() (float64, error)) (float64, error) { + if trials <= 0 { + trials = 1 + } + samples := make([]float64, 0, trials) + for i := 0; i < trials; i++ { + value, err := run() + if err != nil { + return 0, err + } + samples = append(samples, value) + } + return phase5Median(samples) +} + +func phase5P95(values []float64) (float64, error) { + if len(values) == 0 { + return 0, fmt.Errorf("samples required") + } + cp := append([]float64(nil), values...) + sort.Float64s(cp) + idx := int(float64(len(cp)-1) * 0.95) + if idx < 0 { + idx = 0 + } + if idx >= len(cp) { + idx = len(cp) - 1 + } + return cp[idx], nil +} + +func phase5Median(values []float64) (float64, error) { + if len(values) == 0 { + return 0, fmt.Errorf("samples required") + } + cp := append([]float64(nil), values...) + sort.Float64s(cp) + mid := len(cp) / 2 + if len(cp)%2 == 0 { + return (cp[mid-1] + cp[mid]) / 2, nil + } + return cp[mid], nil +} + +func boolToCount(v bool) float64 { + if v { + return 1 + } + return 0 +} diff --git a/internal/brokerapi/perf_phase5_dependency.go b/internal/brokerapi/perf_phase5_dependency.go new file mode 100644 index 00000000..bf2eff14 --- /dev/null +++ b/internal/brokerapi/perf_phase5_dependency.go @@ -0,0 +1,214 @@ +package brokerapi + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func measurePhase5DependencyFlow(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + service, cleanup, err := newPhase5DependencyService(repoRoot) + if err != nil { + return nil, err + } + defer cleanup() + if err := putPhase5TrustedDependencyContext(service, "run-deps-phase5"); err != nil { + return nil, err + } + coreMetrics, err := measurePhase5DependencyCore(service) + if err != nil { + return nil, err + } + extraMetrics, err := measurePhase5DependencyExtras(service) + if err != nil { + return nil, err + } + return append(coreMetrics, extraMetrics...), nil +} + +func measurePhase5DependencyCore(service *Service) ([]perfcontracts.MeasurementRecord, error) { + missMS, hitMS, err := measurePhase5DependencyMissAndHit(service) + if err != nil { + return nil, err + } + coalesced, err := measurePhase5DependencyCoalescing(service) + if err != nil { + return nil, err + } + base := []perfcontracts.MeasurementRecord{ + phase5DependencyMetric("metric.deps.cache_miss.small.wall_ms", missMS, "ms"), + phase5DependencyMetric("metric.deps.cache_hit.small.wall_ms", hitMS, "ms"), + } + return append(base, coalesced...), nil +} + +func measurePhase5DependencyExtras(service *Service) ([]perfcontracts.MeasurementRecord, error) { + streaming, err := measurePhase5DependencyStreaming(service) + if err != nil { + return nil, err + } + handoff, err := measurePhase5DependencyHandoff(service) + if err != nil { + return nil, err + } + return append(streaming, handoff...), nil +} + +func measurePhase5DependencyMissAndHit(service *Service) (float64, float64, error) { + missHitFetcher := &phase5CountingFetcher{payload: "phase5-dependency-payload"} + service.SetDependencyRegistryFetcherForTests(missHitFetcher) + missReq := phase5DependencyFetchRegistryRequest("req-deps-miss", "run-deps-phase5", "alpha") + missMS, missResp, err := phase5TimedDependencyFetch(service, missReq, "dependency miss fetch") + if err != nil { + return 0, 0, err + } + hitReq := missReq + hitReq.RequestID = "req-deps-hit" + hitMS, hitResp, err := phase5TimedDependencyFetch(service, hitReq, "dependency hit fetch") + if err != nil { + return 0, 0, err + } + if missResp.CacheOutcome != "miss_filled" || hitResp.CacheOutcome != "hit_exact" { + return 0, 0, fmt.Errorf("unexpected dependency cache outcomes miss=%q hit=%q", missResp.CacheOutcome, hitResp.CacheOutcome) + } + return missMS, hitMS, nil +} + +func phase5TimedDependencyFetch(service *Service, req DependencyFetchRegistryRequest, action string) (float64, DependencyFetchRegistryResponse, error) { + started := time.Now() + resp, errResp := service.HandleDependencyFetchRegistry(context.Background(), req, RequestContext{}) + if err := phase5DependencyErr(action, errResp); err != nil { + return 0, DependencyFetchRegistryResponse{}, err + } + return float64(time.Since(started).Microseconds()) / 1000.0, resp, nil +} + +func measurePhase5DependencyCoalescing(service *Service) ([]perfcontracts.MeasurementRecord, error) { + coalesceFetcher := &phase5GatedFetcher{gate: make(chan struct{}), started: make(chan struct{})} + service.SetDependencyRegistryFetcherForTests(coalesceFetcher) + coalesceReq := phase5DependencyFetchRegistryRequest("req-deps-coalesce", "run-deps-phase5", "coalesced") + responses, wg := phase5StartCoalescedFetches(service, coalesceReq, 6) + if err := phase5WaitForCoalescedStart(coalesceFetcher); err != nil { + return nil, err + } + close(coalesceFetcher.gate) + wg.Wait() + if err := phase5ValidateCoalescedResponses(responses); err != nil { + return nil, err + } + calls := float64(coalesceFetcher.calls.Load()) + casWriteCount := 0.0 + if calls > 0 { + casWriteCount = 1 + } + return []perfcontracts.MeasurementRecord{ + phase5DependencyMetric("metric.deps.cache_coalesced.upstream_fetch_count", calls, "count"), + phase5DependencyMetric("metric.deps.cache_coalesced.cas_write_count", casWriteCount, "count"), + }, nil +} + +func phase5StartCoalescedFetches(service *Service, req DependencyFetchRegistryRequest, n int) ([]*ErrorResponse, *sync.WaitGroup) { + responses := make([]*ErrorResponse, n) + start := make(chan struct{}) + wg := &sync.WaitGroup{} + for i := 0; i < n; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + <-start + one := req + one.RequestID = fmt.Sprintf("%s-%d", req.RequestID, idx) + _, errResp := service.HandleDependencyFetchRegistry(context.Background(), one, RequestContext{}) + responses[idx] = errResp + }(i) + } + close(start) + return responses, wg +} + +func phase5WaitForCoalescedStart(fetcher *phase5GatedFetcher) error { + select { + case <-fetcher.started: + return nil + case <-time.After(2 * time.Second): + return fmt.Errorf("timed out waiting for coalesced fetch start") + } +} + +func phase5ValidateCoalescedResponses(responses []*ErrorResponse) error { + for i := range responses { + if responses[i] != nil { + return fmt.Errorf("coalesced request %d failed: %s", i, responses[i].Error.Code) + } + } + return nil +} + +func measurePhase5DependencyStreaming(service *Service) ([]perfcontracts.MeasurementRecord, error) { + service.SetDependencyRegistryFetcherForTests(&phase5ChunkBoundFetcher{payloadSize: 3 << 20, maxReadBuf: 128 << 10}) + chunkFetcher, _ := service.dependencyFetchService.fetcher.(*phase5ChunkBoundFetcher) + streamReq := phase5DependencyFetchRegistryRequest("req-deps-stream", "run-deps-phase5", "stream") + var memBefore, memAfter runtimeMem + memBefore.capture() + streamResp, streamErr := service.HandleDependencyFetchRegistry(context.Background(), streamReq, RequestContext{}) + memAfter.capture() + if err := phase5DependencyErr("streaming dependency fetch failed", streamErr); err != nil { + return nil, err + } + peakAllocMB := memAfter.allocMB() - memBefore.allocMB() + if peakAllocMB < 0 { + peakAllocMB = 0 + } + maxReadBuffer, readCalls := phase5ChunkFetcherStats(chunkFetcher) + return []perfcontracts.MeasurementRecord{ + phase5DependencyMetric("metric.deps.stream_to_cas.max_read_buffer_bytes", maxReadBuffer, "bytes"), + phase5DependencyMetric("metric.deps.stream_to_cas.read_calls", readCalls, "count"), + phase5DependencyMetric("metric.deps.stream_to_cas.fetched_bytes", float64(streamResp.FetchedBytes), "bytes"), + phase5DependencyMetric("metric.deps.cache_fill.peak_alloc_mb", peakAllocMB, "mb"), + }, nil +} + +func phase5ChunkFetcherStats(fetcher *phase5ChunkBoundFetcher) (float64, float64) { + if fetcher == nil { + return 0, 0 + } + return float64(fetcher.maxSeenBuf.Load()), float64(fetcher.readCalls.Load()) +} + +func measurePhase5DependencyHandoff(service *Service) ([]perfcontracts.MeasurementRecord, error) { + ensureResp, err := phase5EnsureDependency(service) + if err != nil { + return nil, err + } + handoffMS, handoffResp, err := phase5TimedDependencyHandoff(service) + if err != nil { + return nil, err + } + return []perfcontracts.MeasurementRecord{ + phase5DependencyMetric("metric.deps.materialization.workspace_handoff.wall_ms", handoffMS, "ms"), + phase5DependencyMetric("metric.deps.materialization.workspace_handoff.found_count", boolToCount(handoffResp.Found), "count"), + phase5DependencyMetric("metric.deps.cache_ensure.registry_requests", float64(ensureResp.RegistryRequestCount), "count"), + }, nil +} + +func phase5EnsureDependency(service *Service) (DependencyCacheEnsureResponse, error) { + req := phase5DependencyEnsureRequest("req-deps-ensure", "run-deps-phase5", "handoff") + resp, errResp := service.HandleDependencyCacheEnsure(context.Background(), req, RequestContext{}) + if err := phase5DependencyErr("dependency ensure failed", errResp); err != nil { + return DependencyCacheEnsureResponse{}, err + } + return resp, nil +} + +func phase5TimedDependencyHandoff(service *Service) (float64, DependencyCacheHandoffResponse, error) { + req := phase5DependencyHandoffRequest("req-deps-handoff", "handoff", "workspace") + started := time.Now() + resp, errResp := service.HandleDependencyCacheHandoff(context.Background(), req, RequestContext{}) + if err := phase5DependencyErr("dependency handoff failed", errResp); err != nil { + return 0, DependencyCacheHandoffResponse{}, err + } + return float64(time.Since(started).Microseconds()) / 1000.0, resp, nil +} diff --git a/internal/brokerapi/perf_phase5_dependency_context.go b/internal/brokerapi/perf_phase5_dependency_context.go new file mode 100644 index 00000000..3a450dc6 --- /dev/null +++ b/internal/brokerapi/perf_phase5_dependency_context.go @@ -0,0 +1,244 @@ +package brokerapi + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "os" + "strings" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/policyengine" + "github.com/runecode-ai/runecode/internal/trustpolicy" + "github.com/runecode-ai/runecode/third_party/jsoncanonicalizer" +) + +func newPhase5DependencyService(repoRoot string) (*Service, func(), error) { + storeRoot, err := os.MkdirTemp("", "runecode-phase5-deps-store-") + if err != nil { + return nil, nil, err + } + ledgerRoot, err := os.MkdirTemp("", "runecode-phase5-deps-ledger-") + if err != nil { + _ = os.RemoveAll(storeRoot) + return nil, nil, err + } + cleanup := func() { + _ = os.RemoveAll(storeRoot) + _ = os.RemoveAll(ledgerRoot) + } + service, err := NewServiceWithConfig(storeRoot, ledgerRoot, APIConfig{RepositoryRoot: repoRoot, DependencyFetch: DependencyFetchConfig{MaxParallelFetches: 8}}) + if err != nil { + cleanup() + return nil, nil, err + } + return service, cleanup, nil +} + +func phase5DependencyAllowlistPayload() ([]byte, error) { + return json.Marshal(map[string]any{ + "schema_id": "runecode.protocol.v0.PolicyAllowlist", + "schema_version": "0.1.0", + "allowlist_kind": "gateway_scope_rule", + "entry_schema_id": "runecode.protocol.v0.GatewayScopeRule", + "entries": []any{phase5DependencyAllowlistEntry()}, + }) +} + +func phase5DependencyAllowlistEntry() map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.GatewayScopeRule", + "schema_version": "0.1.0", + "scope_kind": "gateway_destination", + "entry_id": "dependency_default", + "gateway_role_kind": "dependency-fetch", + "destination": map[string]any{ + "schema_id": "runecode.protocol.v0.DestinationDescriptor", + "schema_version": "0.1.0", + "descriptor_kind": "package_registry", + "canonical_host": "registry.npmjs.org", + "canonical_path_prefix": "/", + "provider_or_namespace": "npm", + "tls_required": true, + "private_range_blocking": "enforced", + "dns_rebinding_protection": "enforced", + }, + "permitted_operations": []any{"fetch_dependency"}, + "allowed_egress_data_classes": []any{"dependency_resolved_payload"}, + "redirect_posture": "allowlist_only", + "max_timeout_seconds": 120, + "max_response_bytes": 16 << 20, + } +} + +func phase5DependencyRolePayload(runID, allowlistDigest string, verifier trustpolicy.VerifierRecord, privateKey ed25519.PrivateKey) ([]byte, error) { + return phase5SignedPayloadForTrustedContext(map[string]any{ + "schema_id": "runecode.protocol.v0.RoleManifest", + "schema_version": "0.2.0", + "principal": phase5SignedContextPrincipal(runID, "gateway", "dependency-fetch"), + "role_family": "gateway", + "role_kind": "dependency-fetch", + "approval_profile": "moderate", + "capability_opt_ins": []any{"cap_gateway"}, + "allowlist_refs": []any{phase5DigestObject(allowlistDigest)}, + }, verifier, privateKey) +} + +func phase5DependencyRunPayload(runID, allowlistDigest string, verifier trustpolicy.VerifierRecord, privateKey ed25519.PrivateKey) ([]byte, error) { + return phase5SignedPayloadForTrustedContext(map[string]any{ + "schema_id": "runecode.protocol.v0.CapabilityManifest", + "schema_version": "0.2.0", + "principal": phase5SignedContextPrincipal(runID, "gateway", "dependency-fetch"), + "manifest_scope": "run", + "run_id": runID, + "approval_profile": "moderate", + "capability_opt_ins": []any{"cap_gateway"}, + "allowlist_refs": []any{phase5DigestObject(allowlistDigest)}, + }, verifier, privateKey) +} + +func putPhase5TrustedDependencyContext(service *Service, runID string) error { + verifier, privateKey, err := phase5VerifierFixture() + if err != nil { + return err + } + if err := phase5PutTrustedVerifierRecord(service, verifier); err != nil { + return err + } + allowlistPayload, err := phase5DependencyAllowlistPayload() + if err != nil { + return err + } + allowlistDigest, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindPolicyAllowlist, allowlistPayload) + if err != nil { + return err + } + rolePayload, err := phase5DependencyRolePayload(runID, allowlistDigest, verifier, privateKey) + if err != nil { + return err + } + runPayload, err := phase5DependencyRunPayload(runID, allowlistDigest, verifier, privateKey) + if err != nil { + return err + } + if _, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindRoleManifest, rolePayload); err != nil { + return err + } + if _, err := phase5PutTrustedPolicyArtifact(service, runID, artifacts.TrustedContractImportKindRunCapability, runPayload); err != nil { + return err + } + return nil +} + +func phase5DependencyFetchRegistryRequest(requestID, runID, pkg string) DependencyFetchRegistryRequest { + dep := DependencyFetchRequestObject{ + SchemaID: "runecode.protocol.v0.DependencyFetchRequest", + SchemaVersion: "0.1.0", + RequestKind: "package_version_fetch", + RegistryIdentity: policyengine.DestinationDescriptor{ + SchemaID: "runecode.protocol.v0.DestinationDescriptor", + SchemaVersion: "0.1.0", + DescriptorKind: "package_registry", + CanonicalHost: "registry.npmjs.org", + CanonicalPathPrefix: "/", + ProviderOrNamespace: "npm", + TLSRequired: true, + PrivateRangeBlocking: "enforced", + DNSRebindingProtection: "enforced", + }, + Ecosystem: "npm", + PackageName: "pkg-" + pkg, + PackageVersion: "1.0.0", + } + hash, _ := canonicalDependencyRequestIdentity(dep) + digest, _ := digestFromIdentity(hash) + return DependencyFetchRegistryRequest{SchemaID: "runecode.protocol.v0.DependencyFetchRegistryRequest", SchemaVersion: "0.1.0", RequestID: requestID, RunID: runID, DependencyRequest: dep, RequestHash: digest} +} + +func phase5DependencyEnsureRequest(requestID, runID, pkg string) DependencyCacheEnsureRequest { + depReq := phase5DependencyFetchRegistryRequest(requestID+"-single", runID, pkg).DependencyRequest + batch := DependencyFetchBatchRequestObject{ + SchemaID: "runecode.protocol.v0.DependencyFetchBatchRequest", + SchemaVersion: "0.1.0", + LockfileKind: "generic_lock", + LockfileDigest: mustDigestObjectFromIdentity(artifacts.DigestBytes([]byte("lock:" + pkg))), + RequestSetHash: mustDigestObjectFromIdentity(artifacts.DigestBytes([]byte("request-set:" + pkg))), + DependencyRequests: []DependencyFetchRequestObject{depReq}, + BatchRequestID: "batch-" + pkg, + LockfileLocatorHint: "deps.lock", + } + return DependencyCacheEnsureRequest{SchemaID: "runecode.protocol.v0.DependencyCacheEnsureRequest", SchemaVersion: "0.1.0", RequestID: requestID, RunID: runID, BatchRequest: batch} +} + +func phase5DependencyHandoffRequest(requestID, pkg, consumerRole string) DependencyCacheHandoffRequest { + dep := phase5DependencyFetchRegistryRequest(requestID+"-single", "run-deps-phase5", pkg).DependencyRequest + hash, _ := canonicalDependencyRequestIdentity(dep) + return DependencyCacheHandoffRequest{SchemaID: "runecode.protocol.v0.DependencyCacheHandoffRequest", SchemaVersion: "0.1.0", RequestID: requestID, RequestDigest: mustDigestObjectFromIdentity(hash), ConsumerRole: consumerRole} +} + +func phase5VerifierFixture() (trustpolicy.VerifierRecord, ed25519.PrivateKey, error) { + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return trustpolicy.VerifierRecord{}, nil, err + } + sum := sha256.Sum256(publicKey) + keyIDValue := hex.EncodeToString(sum[:]) + return trustpolicy.VerifierRecord{SchemaID: trustpolicy.VerifierSchemaID, SchemaVersion: trustpolicy.VerifierSchemaVersion, KeyID: trustpolicy.KeyIDProfile, KeyIDValue: keyIDValue, Alg: "ed25519", PublicKey: trustpolicy.PublicKey{Encoding: "base64", Value: base64.StdEncoding.EncodeToString(publicKey)}, LogicalPurpose: "isolate_session_identity", LogicalScope: "session", OwnerPrincipal: trustpolicy.PrincipalIdentity{SchemaID: "runecode.protocol.v0.PrincipalIdentity", SchemaVersion: "0.2.0", ActorKind: "daemon", PrincipalID: "brokerapi", InstanceID: "brokerapi-1"}, KeyProtectionPosture: "os_keystore", IdentityBindingPosture: "attested", PresenceMode: "os_confirmation", CreatedAt: "2026-03-13T12:00:00Z", Status: "active"}, privateKey, nil +} + +func phase5PutTrustedVerifierRecord(service *Service, record trustpolicy.VerifierRecord) error { + payload, err := json.Marshal(record) + if err != nil { + return err + } + _, err = phase5PutTrustedPolicyArtifact(service, "", artifacts.TrustedContractImportKindVerifierRecord, payload) + return err +} + +func phase5PutTrustedPolicyArtifact(service *Service, runID, kind string, payload []byte) (string, error) { + provenance := "sha256:" + strings.Repeat("1", 64) + ref, err := service.Put(artifacts.PutRequest{Payload: payload, ContentType: "application/json", DataClass: artifacts.DataClassAuditVerificationReport, ProvenanceReceiptHash: provenance, CreatedByRole: "broker", TrustedSource: true, RunID: runID}) + if err != nil { + return "", err + } + details := map[string]interface{}{ + artifacts.TrustedContractImportKindDetailKey: kind, + artifacts.TrustedContractImportArtifactDigestDetailKey: ref.Digest, + artifacts.TrustedContractImportProvenanceDetailKey: provenance, + } + if err := service.AppendTrustedAuditEvent(artifacts.TrustedContractImportAuditEventType, "brokerapi", details); err != nil { + return "", err + } + return ref.Digest, nil +} + +func phase5SignedPayloadForTrustedContext(payload map[string]any, verifier trustpolicy.VerifierRecord, privateKey ed25519.PrivateKey) ([]byte, error) { + payload["signatures"] = []any{} + clone := map[string]any{} + for k, v := range payload { + clone[k] = v + } + delete(clone, "signatures") + raw, err := json.Marshal(clone) + if err != nil { + return nil, err + } + canonical, err := jsoncanonicalizer.Transform(raw) + if err != nil { + return nil, err + } + sig := ed25519.Sign(privateKey, canonical) + payload["signatures"] = []any{map[string]any{"alg": "ed25519", "key_id": verifier.KeyID, "key_id_value": verifier.KeyIDValue, "signature": base64.StdEncoding.EncodeToString(sig)}} + return json.Marshal(payload) +} + +func phase5SignedContextPrincipal(runID, roleFamily, roleKind string) map[string]any { + return map[string]any{"schema_id": "runecode.protocol.v0.PrincipalIdentity", "schema_version": "0.2.0", "actor_kind": "role_instance", "principal_id": "brokerapi", "instance_id": "brokerapi-1", "role_family": roleFamily, "role_kind": roleKind, "run_id": runID} +} + +func phase5DigestObject(identity string) map[string]any { + return map[string]any{"hash_alg": "sha256", "hash": strings.TrimPrefix(identity, "sha256:")} +} diff --git a/internal/brokerapi/perf_phase5_dependency_fetchers.go b/internal/brokerapi/perf_phase5_dependency_fetchers.go new file mode 100644 index 00000000..89950148 --- /dev/null +++ b/internal/brokerapi/perf_phase5_dependency_fetchers.go @@ -0,0 +1,112 @@ +package brokerapi + +import ( + "context" + "fmt" + "io" + "runtime" + "strings" + "sync" + "sync/atomic" + + "github.com/runecode-ai/runecode/internal/artifacts" +) + +type runtimeMem struct{ allocBytes uint64 } + +func (m *runtimeMem) capture() { + var ms runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&ms) + m.allocBytes = ms.Alloc +} + +func (m runtimeMem) allocMB() float64 { + return float64(m.allocBytes) / (1024.0 * 1024.0) +} + +type phase5CountingFetcher struct { + payload string + calls atomic.Int64 +} + +func (f *phase5CountingFetcher) Fetch(_ context.Context, _ DependencyFetchRequestObject, lease dependencyRegistryAuthLease) (io.ReadCloser, dependencyRegistryFetchMetadata, error) { + if lease == nil { + return nil, dependencyRegistryFetchMetadata{}, fmt.Errorf("auth lease required") + } + f.calls.Add(1) + payload := f.payload + if payload == "" { + payload = "phase5-default-payload" + } + return io.NopCloser(strings.NewReader(payload)), dependencyRegistryFetchMetadata{ContentType: "application/octet-stream", ExpectedPayloadDigest: artifacts.DigestBytes([]byte(payload))}, nil +} + +type phase5GatedFetcher struct { + gate chan struct{} + started chan struct{} + once sync.Once + calls atomic.Int64 +} + +func (f *phase5GatedFetcher) Fetch(_ context.Context, _ DependencyFetchRequestObject, lease dependencyRegistryAuthLease) (io.ReadCloser, dependencyRegistryFetchMetadata, error) { + if lease == nil { + return nil, dependencyRegistryFetchMetadata{}, fmt.Errorf("auth lease required") + } + f.calls.Add(1) + f.once.Do(func() { close(f.started) }) + <-f.gate + payload := "phase5-coalesced-payload" + return io.NopCloser(strings.NewReader(payload)), dependencyRegistryFetchMetadata{ContentType: "application/octet-stream", ExpectedPayloadDigest: artifacts.DigestBytes([]byte(payload))}, nil +} + +type phase5ChunkBoundFetcher struct { + payloadSize int64 + maxReadBuf int + maxSeenBuf atomic.Int64 + readCalls atomic.Int64 +} + +func (f *phase5ChunkBoundFetcher) Fetch(_ context.Context, _ DependencyFetchRequestObject, lease dependencyRegistryAuthLease) (io.ReadCloser, dependencyRegistryFetchMetadata, error) { + if lease == nil { + return nil, dependencyRegistryFetchMetadata{}, fmt.Errorf("auth lease required") + } + reader := &phase5ChunkReader{remaining: f.payloadSize, maxReadBuf: f.maxReadBuf, maxSeenBuf: &f.maxSeenBuf, readCalls: &f.readCalls} + return io.NopCloser(reader), dependencyRegistryFetchMetadata{ContentType: "application/octet-stream"}, nil +} + +type phase5ChunkReader struct { + remaining int64 + maxReadBuf int + maxSeenBuf *atomic.Int64 + readCalls *atomic.Int64 +} + +func (r *phase5ChunkReader) Read(p []byte) (int, error) { + if r.remaining <= 0 { + return 0, io.EOF + } + if len(p) > r.maxReadBuf { + return 0, fmt.Errorf("oversized read buffer") + } + r.phase5RememberSeenBuffer(int64(len(p))) + r.readCalls.Add(1) + n := len(p) + if int64(n) > r.remaining { + n = int(r.remaining) + } + for i := 0; i < n; i++ { + p[i] = 'x' + } + r.remaining -= int64(n) + return n, nil +} + +func (r *phase5ChunkReader) phase5RememberSeenBuffer(seen int64) { + for { + max := r.maxSeenBuf.Load() + if seen <= max || r.maxSeenBuf.CompareAndSwap(max, seen) { + return + } + } +} diff --git a/internal/brokerapi/perf_verification_harness.go b/internal/brokerapi/perf_verification_harness.go new file mode 100644 index 00000000..be669e3b --- /dev/null +++ b/internal/brokerapi/perf_verification_harness.go @@ -0,0 +1,146 @@ +package brokerapi + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +const phase5PerfCheckSchemaVersion = "runecode.performance.check.v1" + +type Phase5PerformanceHarnessConfig struct { + RepositoryRoot string + Trials int + CommandTimeout time.Duration + CommandRunner func(repoRoot string, timeout time.Duration, command ...string) (float64, error) +} + +func RunPhase5PerformanceHarness(cfg Phase5PerformanceHarnessConfig) (perfcontracts.CheckOutput, error) { + trials := phase5ResolvedTrials(cfg.Trials) + timeout := phase5ResolvedTimeout(cfg.CommandTimeout) + repoRoot, err := phase5ResolveRepoRoot(cfg.RepositoryRoot) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + runner := cfg.CommandRunner + if runner == nil { + runner = phase5RunCommand + } + + measurements, err := phase5CollectMeasurements(trials, repoRoot, timeout, runner) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + return perfcontracts.CheckOutput{SchemaVersion: phase5PerfCheckSchemaVersion, Measurements: measurements}, nil +} + +func phase5ResolvedTrials(trials int) int { + if trials <= 0 { + return 10 + } + return trials +} + +func phase5ResolvedTimeout(timeout time.Duration) time.Duration { + if timeout <= 0 { + return 2 * time.Minute + } + return timeout +} + +func phase5ResolveRepoRoot(explicit string) (string, error) { + repoRoot := strings.TrimSpace(explicit) + if repoRoot != "" { + return repoRoot, nil + } + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return cwd, nil +} + +func phase5CollectMeasurements( + trials int, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, command ...string) (float64, error), +) ([]perfcontracts.MeasurementRecord, error) { + measurements := make([]perfcontracts.MeasurementRecord, 0, 24) + + if err := phase5AppendGatewayAndSecrets(&measurements, trials, repoRoot); err != nil { + return nil, err + } + if err := phase5AppendDependencyFlow(&measurements, trials, repoRoot); err != nil { + return nil, err + } + if err := phase5AppendAuditVerification(&measurements, trials, repoRoot, timeout, runner); err != nil { + return nil, err + } + if err := phase5AppendProtocolChecks(&measurements, repoRoot, timeout, runner); err != nil { + return nil, err + } + measurements = append(measurements, measurePhase5ExternalAnchorStubbed(trials)...) + return measurements, nil +} + +func phase5AppendGatewayAndSecrets(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measurePhase5GatewayAndSecrets(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func phase5AppendDependencyFlow(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measurePhase5DependencyFlow(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func phase5AppendAuditVerification( + measurements *[]perfcontracts.MeasurementRecord, + trials int, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, command ...string) (float64, error), +) error { + items, err := measurePhase5AuditVerification(trials, repoRoot, timeout, runner) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func phase5AppendProtocolChecks( + measurements *[]perfcontracts.MeasurementRecord, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, command ...string) (float64, error), +) error { + items, err := measurePhase5ProtocolChecks(repoRoot, timeout, runner) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func phase5DependencyMetric(metricID string, value float64, unit string) perfcontracts.MeasurementRecord { + return perfcontracts.MeasurementRecord{MetricID: metricID, Value: value, Unit: unit} +} + +func phase5DependencyErr(action string, errResp *ErrorResponse) error { + if errResp == nil { + return nil + } + return fmt.Errorf("%s: %s", action, errResp.Error.Code) +} diff --git a/internal/brokerapi/perf_verification_harness_test.go b/internal/brokerapi/perf_verification_harness_test.go new file mode 100644 index 00000000..b9bb7481 --- /dev/null +++ b/internal/brokerapi/perf_verification_harness_test.go @@ -0,0 +1,163 @@ +package brokerapi + +import ( + "testing" + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func TestPhase5GatewayPerfRigMeasuresGatewayAdmissionAndIngressPaths(t *testing.T) { + rig, err := newPhase5GatewayPerfRig("") + if err != nil { + t.Fatalf("newPhase5GatewayPerfRig returned error: %v", err) + } + defer rig.cleanup() + + if err := rig.invokeGatewayTrial(1); err != nil { + t.Fatalf("invokeGatewayTrial returned error: %v", err) + } + if err := rig.issueLeaseTrial(1); err != nil { + t.Fatalf("issueLeaseTrial returned error: %v", err) + } + if err := rig.ingressPrepareSubmitTrial(1); err != nil { + t.Fatalf("ingressPrepareSubmitTrial returned error: %v", err) + } + + events, err := rig.service.ReadAuditEvents() + if err != nil { + t.Fatalf("ReadAuditEvents returned error: %v", err) + } + if !hasAuditEventType(events, "model_egress") { + t.Fatal("missing model_egress audit event from gateway invoke trial") + } + if !hasAuditEventType(events, brokerAuditEventTypeProviderCredential) { + t.Fatal("missing provider credential audit event from ingress submit trial") + } +} + +func TestRunPhase5PerformanceHarnessProducesExpectedMetrics(t *testing.T) { + out, err := RunPhase5PerformanceHarness(testPhase5HarnessConfig()) + if err != nil { + t.Fatalf("RunPhase5PerformanceHarness returned error: %v", err) + } + if out.SchemaVersion != phase5PerfCheckSchemaVersion { + t.Fatalf("schema_version = %q, want %q", out.SchemaVersion, phase5PerfCheckSchemaVersion) + } + required := requiredPhase5Metrics() + for metricID, unit := range required { + if !hasPhase5Metric(out.Measurements, metricID, unit) { + t.Fatalf("missing metric %s (%s)", metricID, unit) + } + } +} + +func TestMeasurePhase5AuditVerificationUsesMedianOfTrials(t *testing.T) { + t.Parallel() + + verifySamples := []float64{900, 240, 210, 220} + finalizeSamples := []float64{1200, 700, 650, 620} + verifyCalls := 0 + finalizeCalls := 0 + measurements, err := measurePhase5AuditVerification(3, "/repo", time.Second, func(_ string, _ time.Duration, command ...string) (float64, error) { + switch command[2] { + case "./internal/auditd": + value := verifySamples[verifyCalls] + verifyCalls++ + return value, nil + case "./internal/brokerapi": + value := finalizeSamples[finalizeCalls] + finalizeCalls++ + return value, nil + default: + t.Fatalf("unexpected command: %v", command) + return 0, nil + } + }) + if err != nil { + t.Fatalf("measurePhase5AuditVerification returned error: %v", err) + } + assertMetricValue(t, measurements, "metric.audit.verify_current_segment.wall_ms", 220) + assertMetricValue(t, measurements, "metric.audit.finalize_verify.wall_ms", 650) + if verifyCalls != 4 || finalizeCalls != 4 { + t.Fatalf("verifyCalls=%d finalizeCalls=%d, want 4 each", verifyCalls, finalizeCalls) + } +} + +func testPhase5HarnessConfig() Phase5PerformanceHarnessConfig { + return Phase5PerformanceHarnessConfig{ + Trials: 2, + CommandRunner: func(_ string, _ time.Duration, command ...string) (float64, error) { + return phase5CommandLatency(command), nil + }, + } +} + +func phase5CommandLatency(command []string) float64 { + if len(command) >= 3 && command[0] == "go" && command[1] == "test" { + return 120 + } + if len(command) >= 3 && command[0] == "node" && command[1] == "--test" { + return 90 + } + return 10 +} + +func requiredPhase5Metrics() map[string]string { + return map[string]string{ + "metric.gateway.model_invoke.overhead.p95_ms": "ms", + "metric.secrets.lease_issue.p95_ms": "ms", + "metric.secrets.ingress.prepare_submit.p95_ms": "ms", + "metric.deps.cache_miss.small.wall_ms": "ms", + "metric.deps.cache_hit.small.wall_ms": "ms", + "metric.deps.cache_coalesced.upstream_fetch_count": "count", + "metric.deps.cache_coalesced.cas_write_count": "count", + "metric.deps.materialization.workspace_handoff.wall_ms": "ms", + "metric.deps.stream_to_cas.max_read_buffer_bytes": "bytes", + "metric.deps.stream_to_cas.read_calls": "count", + "metric.deps.cache_fill.peak_alloc_mb": "mb", + "metric.audit.verify_current_segment.wall_ms": "ms", + "metric.audit.finalize_verify.wall_ms": "ms", + "metric.protocol.schema_validation.wall_ms": "ms", + "metric.protocol.fixture_parity.wall_ms": "ms", + "metric.anchor.prepare.latency.p95_ms": "ms", + "metric.anchor.execute.completed.p95_ms": "ms", + "metric.anchor.execute.deferred.handoff.p95_ms": "ms", + "metric.anchor.deferred.visibility.p95_ms": "ms", + "metric.anchor.receipt_admission.unchanged_seal.p95_ms": "ms", + "metric.anchor.network_io_under_ledger_lock.count": "count", + "metric.anchor.verifier_bypass.count": "count", + } +} + +func hasPhase5Metric(measurements []perfcontracts.MeasurementRecord, metricID, unit string) bool { + for _, m := range measurements { + if m.MetricID == metricID && m.Unit == unit { + return true + } + } + return false +} + +func assertMetricValue(t *testing.T, measurements []perfcontracts.MeasurementRecord, metricID string, want float64) { + t.Helper() + for _, measurement := range measurements { + if measurement.MetricID == metricID { + if measurement.Value != want { + t.Fatalf("metric %s value = %v, want %v", metricID, measurement.Value, want) + } + return + } + } + t.Fatalf("metric %s missing", metricID) +} + +func hasAuditEventType(events []artifacts.AuditEvent, eventType string) bool { + for _, event := range events { + if event.Type == eventType { + return true + } + } + return false +} diff --git a/internal/brokerperf/harness.go b/internal/brokerperf/harness.go new file mode 100644 index 00000000..cb44cd08 --- /dev/null +++ b/internal/brokerperf/harness.go @@ -0,0 +1,121 @@ +package brokerperf + +import ( + "fmt" + "os" + "strings" + + "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +const CheckSchemaVersion = "runecode.performance.check.v1" + +type HarnessConfig struct { + Trials int + RepositoryRoot string +} + +type latencySpec struct { + metricID string + call func() error +} + +type watchSpec struct { + latencyMetricID string + payloadMetricID string + countMetricID string + call func() (any, error) +} + +func Run(cfg HarnessConfig) (perfcontracts.CheckOutput, error) { + repoRoot, err := brokerPerfResolveRepoRoot(cfg.RepositoryRoot) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + trials := brokerPerfResolvedTrials(cfg.Trials) + measurements, err := brokerPerfCollectMeasurements(trials, repoRoot) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + return perfcontracts.CheckOutput{SchemaVersion: CheckSchemaVersion, Measurements: measurements}, nil +} + +func brokerPerfResolveRepoRoot(explicit string) (string, error) { + repoRoot := strings.TrimSpace(explicit) + if repoRoot != "" { + return repoRoot, nil + } + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return cwd, nil +} + +func brokerPerfResolvedTrials(trials int) int { + if trials <= 0 { + return 30 + } + return trials +} + +func brokerPerfCollectMeasurements(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + measurements := make([]perfcontracts.MeasurementRecord, 0, 26) + if err := brokerPerfAppendUnary(&measurements, trials, repoRoot); err != nil { + return nil, err + } + if err := brokerPerfAppendWatches(&measurements, trials, repoRoot); err != nil { + return nil, err + } + if err := brokerPerfAppendMutations(&measurements, trials, repoRoot); err != nil { + return nil, err + } + if err := brokerPerfAppendAttachResume(&measurements, trials, repoRoot); err != nil { + return nil, err + } + return measurements, nil +} + +func brokerPerfAppendUnary(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measureUnary(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func brokerPerfAppendWatches(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measureWatches(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func brokerPerfAppendMutations(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measureMutations(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func brokerPerfAppendAttachResume(measurements *[]perfcontracts.MeasurementRecord, trials int, repoRoot string) error { + items, err := measureAttachResume(trials, repoRoot) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func unaryErr(errResp *brokerapi.ErrorResponse, label string) error { + if errResp == nil { + return nil + } + return fmt.Errorf("%s: %s", label, errResp.Error.Code) +} diff --git a/internal/brokerperf/harness_approval_fixture.go b/internal/brokerperf/harness_approval_fixture.go new file mode 100644 index 00000000..99222ac1 --- /dev/null +++ b/internal/brokerperf/harness_approval_fixture.go @@ -0,0 +1,226 @@ +package brokerperf + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/policyengine" + "github.com/runecode-ai/runecode/internal/trustpolicy" + "github.com/runecode-ai/runecode/third_party/jsoncanonicalizer" +) + +func seedBackendPostureApprovalForResolve(service *brokerapi.Service) (brokerapi.ApprovalResolveRequest, error) { + targetInstanceID, targetBackend, actionHash, err := backendPostureApprovalFixtureInputs(service) + if err != nil { + return brokerapi.ApprovalResolveRequest{}, err + } + requestEnv, approvalID, verifierRecord, privateKey, err := backendPostureApprovalRequestEnvelope(targetInstanceID, targetBackend, actionHash) + if err != nil { + return brokerapi.ApprovalResolveRequest{}, err + } + decisionEnv, err := backendPostureApprovalDecisionEnvelope(approvalID, verifierRecord, privateKey) + if err != nil { + return brokerapi.ApprovalResolveRequest{}, err + } + if err := putTrustedVerifierRecordForService(service, verifierRecord); err != nil { + return brokerapi.ApprovalResolveRequest{}, err + } + policyHash, err := persistBackendPostureApprovalFixture(service, targetInstanceID, actionHash, approvalID, requestEnv) + if err != nil { + return brokerapi.ApprovalResolveRequest{}, err + } + return brokerapi.ApprovalResolveRequest{ + SchemaID: "runecode.protocol.v0.ApprovalResolveRequest", + SchemaVersion: "0.1.0", + RequestID: "perf-approval-resolve", + ApprovalID: approvalID, + BoundScope: brokerapi.ApprovalBoundScope{ + SchemaID: "runecode.protocol.v0.ApprovalBoundScope", + SchemaVersion: "0.1.0", + WorkspaceID: "workspace-local", + InstanceID: targetInstanceID, + RunID: "run-backend", + ActionKind: policyengine.ActionKindBackendPosture, + PolicyDecisionHash: policyHash, + }, + ResolutionDetails: brokerapi.ApprovalResolveDetails{SchemaID: "runecode.protocol.v0.ApprovalResolveDetails", SchemaVersion: "0.1.0", BackendPostureSelection: &brokerapi.ApprovalResolveBackendPostureSelectionDetail{SchemaID: "runecode.protocol.v0.ApprovalResolveBackendPostureSelectionDetail", SchemaVersion: "0.1.0", TargetInstanceID: targetInstanceID, TargetBackendKind: targetBackend}}, + SignedApprovalRequest: requestEnv, + SignedApprovalDecision: decisionEnv, + }, nil +} + +func backendPostureApprovalFixtureInputs(service *brokerapi.Service) (string, string, string, error) { + postureResp, errResp := service.HandleBackendPostureGet(context.Background(), brokerapi.BackendPostureGetRequest{SchemaID: "runecode.protocol.v0.BackendPostureGetRequest", SchemaVersion: "0.1.0", RequestID: "seed-posture"}, brokerapi.RequestContext{}) + if errResp != nil { + return "", "", "", fmt.Errorf("backend_posture_get: %s", errResp.Error.Code) + } + targetInstanceID := postureResp.Posture.InstanceID + targetBackend := "container" + actionHash, err := policyengine.CanonicalActionRequestHash(policyengine.NewBackendPostureChangeAction(policyengine.BackendPostureChangeActionInput{ + ActionEnvelope: policyengine.ActionEnvelope{CapabilityID: "cap_backend", Actor: policyengine.ActionActor{ActorKind: "daemon", RoleFamily: "workspace", RoleKind: "workspace-edit"}}, + RunID: "instance-control:" + targetInstanceID, + TargetInstanceID: targetInstanceID, + TargetBackendKind: targetBackend, + SelectionMode: "explicit_selection", + ChangeKind: "select_backend", + AssuranceChangeKind: "reduce_assurance", + OptInKind: "exact_action_approval", + ReducedAssuranceAcknowledged: true, + Reason: "operator_requested_reduced_assurance_backend_opt_in", + })) + if err != nil { + return "", "", "", err + } + return targetInstanceID, targetBackend, actionHash, nil +} + +func backendPostureApprovalRequestEnvelope(targetInstanceID, targetBackend, actionHash string) (trustpolicy.SignedObjectEnvelope, string, trustpolicy.VerifierRecord, ed25519.PrivateKey, error) { + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return trustpolicy.SignedObjectEnvelope{}, "", trustpolicy.VerifierRecord{}, nil, err + } + keyIDValue := backendPostureKeyIDValue(publicKey) + requestBytes, err := marshalBackendPostureRequestPayload(targetInstanceID, targetBackend, actionHash, keyIDValue) + if err != nil { + return trustpolicy.SignedObjectEnvelope{}, "", trustpolicy.VerifierRecord{}, nil, err + } + requestCanonical, err := jsoncanonicalizer.Transform(requestBytes) + if err != nil { + return trustpolicy.SignedObjectEnvelope{}, "", trustpolicy.VerifierRecord{}, nil, err + } + requestSig := ed25519.Sign(privateKey, requestCanonical) + approvalID := backendPostureApprovalID(requestCanonical) + verifier := backendPostureVerifierRecord(publicKey, keyIDValue) + return backendPostureRequestEnvelope(requestBytes, keyIDValue, requestSig), approvalID, verifier, privateKey, nil +} + +func backendPostureKeyIDValue(publicKey ed25519.PublicKey) string { + keyID := sha256.Sum256(publicKey) + return hex.EncodeToString(keyID[:]) +} + +func marshalBackendPostureRequestPayload(targetInstanceID, targetBackend, actionHash, keyIDValue string) ([]byte, error) { + payload := map[string]any{ + "schema_id": trustpolicy.ApprovalRequestSchemaID, + "schema_version": trustpolicy.ApprovalRequestSchemaVersion, + "approval_profile": "moderate", + "requester": map[string]any{"schema_id": "runecode.protocol.v0.PrincipalIdentity", "schema_version": "0.2.0", "actor_kind": "daemon", "principal_id": "broker", "instance_id": "broker-1"}, + "approval_trigger_code": "reduced_assurance_backend", + "manifest_hash": map[string]any{"hash_alg": "sha256", "hash": strings.Repeat("1", 64)}, + "action_request_hash": map[string]any{"hash_alg": "sha256", "hash": strings.TrimPrefix(actionHash, "sha256:")}, + "relevant_artifact_hashes": []any{}, + "details_schema_id": "runecode.protocol.details.policy.required_approval.reduced_assurance_backend.v0", + "details": backendPostureRequestDetails(targetInstanceID, targetBackend), + "approval_assurance_level": "reauthenticated", + "presence_mode": "hardware_touch", + "requested_at": time.Now().UTC().Add(-time.Minute).Format(time.RFC3339), + "expires_at": time.Now().UTC().Add(30 * time.Minute).Format(time.RFC3339), + "staleness_posture": "invalidate_on_bound_input_change", + "changes_if_approved": "Reduced-assurance backend posture change may be applied.", + "signatures": []any{map[string]any{"alg": "ed25519", "key_id": trustpolicy.KeyIDProfile, "key_id_value": keyIDValue, "signature": base64.StdEncoding.EncodeToString(make([]byte, ed25519.SignatureSize))}}, + } + return json.Marshal(payload) +} + +func backendPostureRequestDetails(targetInstanceID, targetBackend string) map[string]any { + return map[string]any{ + "target_instance_id": targetInstanceID, + "target_backend_kind": targetBackend, + "selection_mode": "explicit_selection", + "change_kind": "select_backend", + "requested_posture": "container_mode_explicit_opt_in", + "assurance_change_kind": "reduce_assurance", + "opt_in_kind": "exact_action_approval", + "reduced_assurance_acknowledged": true, + "approval_binding_posture": "exact_action", + } +} + +func backendPostureApprovalID(requestCanonical []byte) string { + requestHash := sha256.Sum256(requestCanonical) + return "sha256:" + hex.EncodeToString(requestHash[:]) +} + +func backendPostureVerifierRecord(publicKey ed25519.PublicKey, keyIDValue string) trustpolicy.VerifierRecord { + return trustpolicy.VerifierRecord{SchemaID: trustpolicy.VerifierSchemaID, SchemaVersion: trustpolicy.VerifierSchemaVersion, KeyID: trustpolicy.KeyIDProfile, KeyIDValue: keyIDValue, Alg: "ed25519", PublicKey: trustpolicy.PublicKey{Encoding: "base64", Value: base64.StdEncoding.EncodeToString(publicKey)}, LogicalPurpose: "approval_authority", LogicalScope: "user", OwnerPrincipal: trustpolicy.PrincipalIdentity{SchemaID: "runecode.protocol.v0.PrincipalIdentity", SchemaVersion: "0.2.0", ActorKind: "user", PrincipalID: "human", InstanceID: "approval-session"}, KeyProtectionPosture: "hardware_backed", IdentityBindingPosture: "attested", PresenceMode: "hardware_touch", CreatedAt: "2026-03-13T12:00:00Z", Status: "active"} +} + +func backendPostureRequestEnvelope(requestBytes []byte, keyIDValue string, sig []byte) trustpolicy.SignedObjectEnvelope { + return trustpolicy.SignedObjectEnvelope{SchemaID: trustpolicy.EnvelopeSchemaID, SchemaVersion: trustpolicy.EnvelopeSchemaVersion, PayloadSchemaID: trustpolicy.ApprovalRequestSchemaID, PayloadSchemaVersion: trustpolicy.ApprovalRequestSchemaVersion, Payload: requestBytes, SignatureInput: trustpolicy.SignatureInputProfile, Signature: trustpolicy.SignatureBlock{Alg: "ed25519", KeyID: trustpolicy.KeyIDProfile, KeyIDValue: keyIDValue, Signature: base64.StdEncoding.EncodeToString(sig)}} +} + +func backendPostureApprovalDecisionEnvelope(approvalID string, verifier trustpolicy.VerifierRecord, privateKey ed25519.PrivateKey) (trustpolicy.SignedObjectEnvelope, error) { + decisionBytes, err := marshalBackendPostureDecisionPayload(approvalID, verifier) + if err != nil { + return trustpolicy.SignedObjectEnvelope{}, err + } + decisionCanonical, err := jsoncanonicalizer.Transform(decisionBytes) + if err != nil { + return trustpolicy.SignedObjectEnvelope{}, err + } + decisionSig := ed25519.Sign(privateKey, decisionCanonical) + return trustpolicy.SignedObjectEnvelope{SchemaID: trustpolicy.EnvelopeSchemaID, SchemaVersion: trustpolicy.EnvelopeSchemaVersion, PayloadSchemaID: trustpolicy.ApprovalDecisionSchemaID, PayloadSchemaVersion: trustpolicy.ApprovalDecisionSchemaVersion, Payload: decisionBytes, SignatureInput: trustpolicy.SignatureInputProfile, Signature: trustpolicy.SignatureBlock{Alg: "ed25519", KeyID: trustpolicy.KeyIDProfile, KeyIDValue: verifier.KeyIDValue, Signature: base64.StdEncoding.EncodeToString(decisionSig)}}, nil +} + +func marshalBackendPostureDecisionPayload(approvalID string, verifier trustpolicy.VerifierRecord) ([]byte, error) { + decisionPayload := map[string]any{"schema_id": trustpolicy.ApprovalDecisionSchemaID, "schema_version": trustpolicy.ApprovalDecisionSchemaVersion, "approval_request_hash": map[string]any{"hash_alg": "sha256", "hash": strings.TrimPrefix(approvalID, "sha256:")}, "approver": map[string]any{"schema_id": "runecode.protocol.v0.PrincipalIdentity", "schema_version": "0.2.0", "actor_kind": "user", "principal_id": "human", "instance_id": "approval-session"}, "decision_outcome": "approve", "approval_assurance_level": "reauthenticated", "presence_mode": "hardware_touch", "key_protection_posture": "hardware_backed", "identity_binding_posture": "attested", "approval_assertion_hash": map[string]any{"hash_alg": "sha256", "hash": strings.Repeat("f", 64)}, "decided_at": time.Now().UTC().Format(time.RFC3339), "consumption_posture": "single_use", "signatures": []any{map[string]any{"alg": "ed25519", "key_id": trustpolicy.KeyIDProfile, "key_id_value": verifier.KeyIDValue, "signature": "c2ln"}}} + return json.Marshal(decisionPayload) +} + +func persistBackendPostureApprovalFixture(service *brokerapi.Service, targetInstanceID, actionHash, approvalID string, requestEnv trustpolicy.SignedObjectEnvelope) (string, error) { + policyDecision := policyengine.PolicyDecision{SchemaID: "runecode.protocol.v0.PolicyDecision", SchemaVersion: "0.3.0", DecisionOutcome: policyengine.DecisionRequireHumanApproval, PolicyReasonCode: "approval_required", ManifestHash: "sha256:" + strings.Repeat("1", 64), ActionRequestHash: actionHash, PolicyInputHashes: []string{"sha256:" + strings.Repeat("4", 64)}, DetailsSchemaID: "runecode.protocol.details.policy.evaluation.v0", Details: map[string]any{"precedence": "approval_profile_moderate"}, RequiredApprovalSchemaID: "runecode.protocol.details.policy.required_approval.reduced_assurance_backend.v0", RequiredApproval: map[string]any{"approval_trigger_code": "reduced_assurance_backend", "approval_assurance_level": "reauthenticated", "presence_mode": "hardware_touch", "scope": map[string]any{"schema_id": "runecode.protocol.v0.ApprovalBoundScope", "schema_version": "0.1.0", "workspace_id": "workspace-local", "run_id": "run-backend", "instance_id": targetInstanceID, "action_kind": policyengine.ActionKindBackendPosture}, "changes_if_approved": "Reduced-assurance backend posture change may be applied.", "approval_ttl_seconds": 1800}} + if err := service.RecordPolicyDecision("run-backend", "", policyDecision); err != nil { + return "", err + } + refs := service.PolicyDecisionRefsForRun("run-backend") + if len(refs) == 0 { + return "", fmt.Errorf("missing policy decision refs") + } + policyHash := refs[len(refs)-1] + if err := recordPendingApproval(service, targetInstanceID, actionHash, approvalID, policyHash, requestEnv); err != nil { + return "", err + } + return policyHash, nil +} + +func recordPendingApproval(service *brokerapi.Service, targetInstanceID, actionHash, approvalID, policyHash string, requestEnv trustpolicy.SignedObjectEnvelope) error { + expiresAt := time.Now().UTC().Add(30 * time.Minute) + requestedAt := time.Now().UTC().Add(-time.Minute) + record := artifacts.ApprovalRecord{ApprovalID: approvalID, Status: "pending", WorkspaceID: "workspace-local", InstanceID: targetInstanceID, RunID: "run-backend", ActionKind: policyengine.ActionKindBackendPosture, RequestedAt: requestedAt, ExpiresAt: &expiresAt, ApprovalTriggerCode: "reduced_assurance_backend", ChangesIfApproved: "Reduced-assurance backend posture change may be applied.", ApprovalAssuranceLevel: "reauthenticated", PresenceMode: "hardware_touch", ManifestHash: "sha256:" + strings.Repeat("1", 64), ActionRequestHash: actionHash, PolicyDecisionHash: policyHash, RequestDigest: approvalID, RequestEnvelope: &requestEnv} + return service.RecordApproval(record) +} + +func putTrustedVerifierRecordForService(service *brokerapi.Service, record trustpolicy.VerifierRecord) error { + b, err := json.Marshal(record) + if err != nil { + return err + } + provenance := "sha256:" + strings.Repeat("1", 64) + ref, err := service.Put(artifacts.PutRequest{Payload: b, ContentType: "application/json", DataClass: artifacts.DataClassAuditVerificationReport, ProvenanceReceiptHash: provenance, CreatedByRole: "auditd", TrustedSource: true}) + if err != nil { + return err + } + details := map[string]interface{}{artifacts.TrustedContractImportKindDetailKey: artifacts.TrustedContractImportKindVerifierRecord, artifacts.TrustedContractImportArtifactDigestDetailKey: ref.Digest, artifacts.TrustedContractImportProvenanceDetailKey: provenance} + return service.AppendTrustedAuditEvent(artifacts.TrustedContractImportAuditEventType, "brokerapi", details) +} + +func serviceCurrentInstanceID(service *brokerapi.Service) string { + resp, errResp := service.HandleBackendPostureGet(context.Background(), brokerapi.BackendPostureGetRequest{SchemaID: "runecode.protocol.v0.BackendPostureGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-instance-id"}, brokerapi.RequestContext{}) + if errResp != nil { + return "launcher-instance-1" + } + if strings.TrimSpace(resp.Posture.InstanceID) == "" { + return "launcher-instance-1" + } + return strings.TrimSpace(resp.Posture.InstanceID) +} diff --git a/internal/brokerperf/harness_measurements.go b/internal/brokerperf/harness_measurements.go new file mode 100644 index 00000000..5b7c3e8c --- /dev/null +++ b/internal/brokerperf/harness_measurements.go @@ -0,0 +1,200 @@ +package brokerperf + +import ( + "context" + "fmt" + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func measureUnary(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return nil, err + } + defer cleanup() + ctx := context.Background() + specs := unaryLatencySpecs(ctx, service) + return collectLatencyMeasurements(trials, specs) +} + +func unaryLatencySpecs(ctx context.Context, service *brokerapi.Service) []latencySpec { + return []latencySpec{ + {metricID: "metric.broker.unary.session_list.p95_ms", call: func() error { + _, errResp := service.HandleSessionList(ctx, brokerapi.SessionListRequest{SchemaID: "runecode.protocol.v0.SessionListRequest", SchemaVersion: "0.1.0", RequestID: "perf-session-list", Limit: 20}, brokerapi.RequestContext{}) + return unaryErr(errResp, "session_list") + }}, + {metricID: "metric.broker.unary.session_get.p95_ms", call: func() error { + _, errResp := service.HandleSessionGet(ctx, brokerapi.SessionGetRequest{SchemaID: "runecode.protocol.v0.SessionGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-session-get", SessionID: "sess-broker-1"}, brokerapi.RequestContext{}) + return unaryErr(errResp, "session_get") + }}, + {metricID: "metric.broker.unary.run_list.p95_ms", call: func() error { + _, errResp := service.HandleRunList(ctx, brokerapi.RunListRequest{SchemaID: "runecode.protocol.v0.RunListRequest", SchemaVersion: "0.1.0", RequestID: "perf-run-list", Limit: 20}, brokerapi.RequestContext{}) + return unaryErr(errResp, "run_list") + }}, + {metricID: "metric.broker.unary.run_get.p95_ms", call: func() error { + _, errResp := service.HandleRunGet(ctx, brokerapi.RunGetRequest{SchemaID: "runecode.protocol.v0.RunGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-run-get", RunID: "run-broker-1"}, brokerapi.RequestContext{}) + return unaryErr(errResp, "run_get") + }}, + {metricID: "metric.broker.unary.approval_list.p95_ms", call: func() error { + _, errResp := service.HandleApprovalList(ctx, brokerapi.ApprovalListRequest{SchemaID: "runecode.protocol.v0.ApprovalListRequest", SchemaVersion: "0.1.0", RequestID: "perf-approval-list", Limit: 20}, brokerapi.RequestContext{}) + return unaryErr(errResp, "approval_list") + }}, + {metricID: "metric.broker.unary.readiness_get.p95_ms", call: func() error { + _, errResp := service.HandleReadinessGet(ctx, brokerapi.ReadinessGetRequest{SchemaID: "runecode.protocol.v0.ReadinessGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-readiness"}, brokerapi.RequestContext{}) + return unaryErr(errResp, "readiness_get") + }}, + {metricID: "metric.broker.unary.version_info_get.p95_ms", call: func() error { + _, errResp := service.HandleVersionInfoGet(ctx, brokerapi.VersionInfoGetRequest{SchemaID: "runecode.protocol.v0.VersionInfoGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-version"}, brokerapi.RequestContext{}) + return unaryErr(errResp, "version_info_get") + }}, + {metricID: "metric.broker.unary.project_substrate_posture_get.p95_ms", call: func() error { + _, errResp := service.HandleProjectSubstratePostureGet(ctx, brokerapi.ProjectSubstratePostureGetRequest{SchemaID: "runecode.protocol.v0.ProjectSubstratePostureGetRequest", SchemaVersion: "0.1.0", RequestID: "perf-project-posture"}, brokerapi.RequestContext{}) + return unaryErr(errResp, "project_substrate_posture_get") + }}, + } +} + +func measureWatches(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return nil, err + } + defer cleanup() + ctx := context.Background() + specs := []watchSpec{ + {latencyMetricID: "metric.broker.watch.run.snapshot_follow.p95_ms", payloadMetricID: "metric.broker.watch.run.snapshot_follow.payload_bytes", countMetricID: "metric.broker.watch.run.snapshot_follow.event_count", call: func() (any, error) { return streamRunWatch(ctx, service) }}, + {latencyMetricID: "metric.broker.watch.approval.snapshot_follow.p95_ms", payloadMetricID: "metric.broker.watch.approval.snapshot_follow.payload_bytes", countMetricID: "metric.broker.watch.approval.snapshot_follow.event_count", call: func() (any, error) { return streamApprovalWatch(ctx, service) }}, + {latencyMetricID: "metric.broker.watch.session.snapshot_follow.p95_ms", payloadMetricID: "metric.broker.watch.session.snapshot_follow.payload_bytes", countMetricID: "metric.broker.watch.session.snapshot_follow.event_count", call: func() (any, error) { return streamSessionWatch(ctx, service) }}, + {latencyMetricID: "metric.broker.watch.turn_execution.snapshot_follow.p95_ms", payloadMetricID: "metric.broker.watch.turn_execution.snapshot_follow.payload_bytes", countMetricID: "metric.broker.watch.turn_execution.snapshot_follow.event_count", call: func() (any, error) { return streamTurnExecutionWatch(ctx, service) }}, + } + return collectWatchMeasurements(trials, specs) +} + +func measureMutations(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + ctx := context.Background() + specs := []latencySpec{ + {metricID: "metric.broker.mutation.session_execution_trigger.p95_ms", call: func() error { return measureSessionExecutionTriggerMutation(ctx, repoRoot) }}, + {metricID: "metric.broker.mutation.session_execution_continue.p95_ms", call: func() error { return measureSessionExecutionContinueMutation(ctx, repoRoot) }}, + {metricID: "metric.broker.mutation.approval_resolve.p95_ms", call: func() error { return measureApprovalResolveMutation(ctx, repoRoot) }}, + {metricID: "metric.broker.mutation.backend_posture_change.p95_ms", call: func() error { return measureBackendPostureChangeFixture(repoRoot) }}, + } + return collectLatencyMeasurements(trials, specs) +} + +func measureAttachResume(trials int, repoRoot string) ([]perfcontracts.MeasurementRecord, error) { + attachSamples := make([]float64, 0, trials) + resumeSamples := make([]float64, 0, trials) + ctx := context.Background() + for i := 0; i < trials; i++ { + attachDuration, resumeDuration, err := measureAttachResumeTrial(ctx, repoRoot) + if err != nil { + return nil, err + } + attachSamples = append(attachSamples, attachDuration) + resumeSamples = append(resumeSamples, resumeDuration) + } + attachP95, err := p95(attachSamples) + if err != nil { + return nil, err + } + resumeP95, err := p95(resumeSamples) + if err != nil { + return nil, err + } + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.broker.attach.local_control_plane.p95_ms", Value: attachP95, Unit: "ms"}, + {MetricID: "metric.broker.resume.local_control_plane.p95_ms", Value: resumeP95, Unit: "ms"}, + }, nil +} + +func measureAttachResumeTrial(ctx context.Context, repoRoot string) (float64, float64, error) { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return 0, 0, err + } + defer cleanup() + attachDuration, err := timedProductLifecyclePostureGet(ctx, service, "perf-attach", "attach") + if err != nil { + return 0, 0, err + } + resumeDuration, err := timedProductLifecyclePostureGet(ctx, service, "perf-resume", "resume") + if err != nil { + return 0, 0, err + } + return attachDuration, resumeDuration, nil +} + +func timedProductLifecyclePostureGet(ctx context.Context, service *brokerapi.Service, requestID, label string) (float64, error) { + return timedCall(func() error { + _, errResp := service.HandleProductLifecyclePostureGet(ctx, brokerapi.ProductLifecyclePostureGetRequest{SchemaID: "runecode.protocol.v0.ProductLifecyclePostureGetRequest", SchemaVersion: "0.1.0", RequestID: requestID}, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("product_lifecycle_posture_get %s: %s", label, errResp.Error.Code) + } + return nil + }) +} + +func measureSessionExecutionTriggerMutation(ctx context.Context, repoRoot string) error { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return err + } + defer cleanup() + _, errResp := service.HandleSessionExecutionTrigger(ctx, brokerapi.SessionExecutionTriggerRequest{SchemaID: "runecode.protocol.v0.SessionExecutionTriggerRequest", SchemaVersion: "0.1.0", RequestID: "perf-trigger", SessionID: "sess-broker-1", TriggerSource: "interactive_user", RequestedOperation: "start", WorkflowRouting: &brokerapi.SessionWorkflowPackRouting{SchemaID: "runecode.protocol.v0.SessionWorkflowPackRouting", SchemaVersion: "0.1.0", WorkflowFamily: "runecontext", WorkflowOperation: "change_draft"}, UserMessageContentText: "trigger"}, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("session_execution_trigger: %s", errResp.Error.Code) + } + return nil +} + +func measureSessionExecutionContinueMutation(ctx context.Context, repoRoot string) error { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return err + } + defer cleanup() + startResp, errResp := service.HandleSessionExecutionTrigger(ctx, brokerapi.SessionExecutionTriggerRequest{SchemaID: "runecode.protocol.v0.SessionExecutionTriggerRequest", SchemaVersion: "0.1.0", RequestID: "perf-continue-start", SessionID: "sess-broker-1", TriggerSource: "interactive_user", RequestedOperation: "start", WorkflowRouting: &brokerapi.SessionWorkflowPackRouting{SchemaID: "runecode.protocol.v0.SessionWorkflowPackRouting", SchemaVersion: "0.1.0", WorkflowFamily: "runecontext", WorkflowOperation: "change_draft"}, UserMessageContentText: "start"}, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("continue start seed: %s", errResp.Error.Code) + } + _, _ = service.UpdateSessionTurnExecution(artifacts.SessionTurnExecutionUpdateRequest{SessionID: "sess-broker-1", TurnID: startResp.TurnID, ExecutionState: "blocked", WaitKind: "project_blocked", WaitState: "waiting_project_blocked", BlockedReasonCode: "project_substrate_posture_blocked", OccurredAt: time.Now().UTC()}) + _, errResp = service.HandleSessionExecutionTrigger(ctx, brokerapi.SessionExecutionTriggerRequest{SchemaID: "runecode.protocol.v0.SessionExecutionTriggerRequest", SchemaVersion: "0.1.0", RequestID: "perf-continue", SessionID: "sess-broker-1", TurnID: startResp.TurnID, TriggerSource: "resume_follow_up", RequestedOperation: "continue", WorkflowRouting: &brokerapi.SessionWorkflowPackRouting{SchemaID: "runecode.protocol.v0.SessionWorkflowPackRouting", SchemaVersion: "0.1.0", WorkflowFamily: "runecontext", WorkflowOperation: "change_draft"}, UserMessageContentText: "continue"}, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("session_execution_continue: %s", errResp.Error.Code) + } + return nil +} + +func measureApprovalResolveMutation(ctx context.Context, repoRoot string) error { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return err + } + defer cleanup() + resolveReq, err := seedBackendPostureApprovalForResolve(service) + if err != nil { + return err + } + _, errResp := service.HandleApprovalResolve(ctx, resolveReq, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("approval_resolve: %s: %s", errResp.Error.Code, errResp.Error.Message) + } + return nil +} + +func measureBackendPostureChangeFixture(repoRoot string) error { + service, cleanup, err := newSeededService(repoRoot) + if err != nil { + return err + } + defer cleanup() + _, err = seedBackendPostureApprovalForResolve(service) + if err != nil { + return fmt.Errorf("backend_posture_change fixture: %w", err) + } + return nil +} diff --git a/internal/brokerperf/harness_runtime.go b/internal/brokerperf/harness_runtime.go new file mode 100644 index 00000000..9ce52ebd --- /dev/null +++ b/internal/brokerperf/harness_runtime.go @@ -0,0 +1,249 @@ +package brokerperf + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/launcherbackend" + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/policyengine" +) + +func newSeededService(repoRoot string) (*brokerapi.Service, func(), error) { + root, err := os.MkdirTemp("", "runecode-brokerperf-") + if err != nil { + return nil, nil, err + } + cleanup := func() { _ = os.RemoveAll(root) } + service, err := brokerapi.NewServiceWithConfig(root, filepath.Join(root, "audit-ledger"), brokerapi.APIConfig{RepositoryRoot: repoRoot}) + if err != nil { + cleanup() + return nil, nil, err + } + if err := seedServiceData(service); err != nil { + cleanup() + return nil, nil, err + } + return service, cleanup, nil +} + +func seedServiceData(service *brokerapi.Service) error { + if err := service.SetRunStatus("run-broker-1", "active"); err != nil { + return err + } + if err := service.RecordRuntimeFacts("run-broker-1", launcherbackend.RuntimeFactsSnapshot{LaunchReceipt: launcherbackend.BackendLaunchReceipt{RunID: "run-broker-1", SessionID: "sess-broker-1"}}); err != nil { + return err + } + if err := seedBlockedTurn(service, "seed-trigger-1", "seed"); err != nil { + return err + } + if err := seedBlockedTurn(service, "seed-trigger-2", "seed follow-up"); err != nil { + return err + } + return service.RecordPolicyDecision("run-broker-1", "", seedPolicyDecision()) +} + +func seedBlockedTurn(service *brokerapi.Service, requestID, message string) error { + triggerResp, errResp := service.HandleSessionExecutionTrigger(context.Background(), brokerapi.SessionExecutionTriggerRequest{SchemaID: "runecode.protocol.v0.SessionExecutionTriggerRequest", SchemaVersion: "0.1.0", RequestID: requestID, SessionID: "sess-broker-1", TriggerSource: "interactive_user", RequestedOperation: "start", WorkflowRouting: &brokerapi.SessionWorkflowPackRouting{SchemaID: "runecode.protocol.v0.SessionWorkflowPackRouting", SchemaVersion: "0.1.0", WorkflowFamily: "runecontext", WorkflowOperation: "change_draft"}, UserMessageContentText: message}, brokerapi.RequestContext{}) + if errResp != nil { + return fmt.Errorf("seed session_execution_trigger: %s", errResp.Error.Code) + } + _, _ = service.UpdateSessionTurnExecution(artifacts.SessionTurnExecutionUpdateRequest{SessionID: "sess-broker-1", TurnID: triggerResp.TurnID, ExecutionState: "blocked", WaitKind: "project_blocked", WaitState: "waiting_project_blocked", BlockedReasonCode: "project_substrate_posture_blocked", OccurredAt: time.Now().UTC()}) + return nil +} + +func seedPolicyDecision() policyengine.PolicyDecision { + return policyengine.PolicyDecision{ + SchemaID: "runecode.protocol.v0.PolicyDecision", + SchemaVersion: "0.3.0", + DecisionOutcome: policyengine.DecisionRequireHumanApproval, + PolicyReasonCode: "approval_required", + ManifestHash: "sha256:" + strings.Repeat("1", 64), + ActionRequestHash: "sha256:" + strings.Repeat("2", 64), + PolicyInputHashes: []string{"sha256:" + strings.Repeat("3", 64)}, + DetailsSchemaID: "runecode.protocol.details.policy.evaluation.v0", + Details: map[string]any{"precedence": "approval_profile_moderate"}, + RequiredApprovalSchemaID: "runecode.protocol.details.policy.required_approval.moderate.workspace_write.v0", + RequiredApproval: map[string]any{ + "approval_trigger_code": "excerpt_promotion", + "approval_assurance_level": "session_authenticated", + "presence_mode": "os_confirmation", + "scope": map[string]any{ + "schema_id": "runecode.protocol.v0.ApprovalBoundScope", + "schema_version": "0.1.0", + "workspace_id": "workspace-local", + "run_id": "run-broker-1", + "stage_id": "artifact_flow", + "step_id": "step-1", + "action_kind": "promotion", + }, + "changes_if_approved": "Promote reviewed file excerpts for downstream use.", + "approval_ttl_seconds": 1800, + }, + } +} + +func runWatchLatencyAndPayload(call func() (any, error)) (float64, float64, float64, error) { + started := time.Now() + result, err := call() + if err != nil { + return 0, 0, 0, err + } + bytes, count, err := payloadStats(result) + if err != nil { + return 0, 0, 0, err + } + return float64(time.Since(started).Milliseconds()), float64(bytes), float64(count), nil +} + +func collectLatencyMeasurements(trials int, specs []latencySpec) ([]perfcontracts.MeasurementRecord, error) { + latency := map[string][]float64{} + for i := 0; i < trials; i++ { + for _, spec := range specs { + duration, err := timedCall(spec.call) + if err != nil { + return nil, err + } + latency[spec.metricID] = append(latency[spec.metricID], duration) + } + } + return p95Records(latency) +} + +func collectWatchMeasurements(trials int, specs []watchSpec) ([]perfcontracts.MeasurementRecord, error) { + latency := map[string][]float64{} + watchPayload := map[string]float64{} + watchCounts := map[string]float64{} + for i := 0; i < trials; i++ { + for _, spec := range specs { + duration, bytes, count, err := runWatchLatencyAndPayload(spec.call) + if err != nil { + return nil, err + } + latency[spec.latencyMetricID] = append(latency[spec.latencyMetricID], duration) + watchPayload[spec.payloadMetricID] = maxFloat64(watchPayload[spec.payloadMetricID], bytes) + watchCounts[spec.countMetricID] = maxFloat64(watchCounts[spec.countMetricID], count) + } + } + measurements, err := p95Records(latency) + if err != nil { + return nil, err + } + return appendWatchStats(measurements, watchPayload, watchCounts), nil +} + +func appendWatchStats(measurements []perfcontracts.MeasurementRecord, watchPayload, watchCounts map[string]float64) []perfcontracts.MeasurementRecord { + for metricID, value := range watchPayload { + measurements = append(measurements, perfcontracts.MeasurementRecord{MetricID: metricID, Value: value, Unit: "bytes"}) + } + for metricID, value := range watchCounts { + measurements = append(measurements, perfcontracts.MeasurementRecord{MetricID: metricID, Value: value, Unit: "count"}) + } + return measurements +} + +func streamRunWatch(ctx context.Context, service *brokerapi.Service) (any, error) { + ack, errResp := service.HandleRunWatchRequest(ctx, brokerapi.RunWatchRequest{SchemaID: "runecode.protocol.v0.RunWatchRequest", SchemaVersion: "0.1.0", RequestID: "perf-run-watch", Follow: true, IncludeSnapshot: true}, brokerapi.RequestContext{}) + if errResp != nil { + return nil, fmt.Errorf("run_watch ack: %s", errResp.Error.Code) + } + return service.StreamRunWatchEvents(ack) +} + +func streamApprovalWatch(ctx context.Context, service *brokerapi.Service) (any, error) { + ack, errResp := service.HandleApprovalWatchRequest(ctx, brokerapi.ApprovalWatchRequest{SchemaID: "runecode.protocol.v0.ApprovalWatchRequest", SchemaVersion: "0.1.0", RequestID: "perf-approval-watch", Follow: true, IncludeSnapshot: true}, brokerapi.RequestContext{}) + if errResp != nil { + return nil, fmt.Errorf("approval_watch ack: %s", errResp.Error.Code) + } + return service.StreamApprovalWatchEvents(ack) +} + +func streamSessionWatch(ctx context.Context, service *brokerapi.Service) (any, error) { + ack, errResp := service.HandleSessionWatchRequest(ctx, brokerapi.SessionWatchRequest{SchemaID: "runecode.protocol.v0.SessionWatchRequest", SchemaVersion: "0.1.0", RequestID: "perf-session-watch", Follow: true, IncludeSnapshot: true}, brokerapi.RequestContext{}) + if errResp != nil { + return nil, fmt.Errorf("session_watch ack: %s", errResp.Error.Code) + } + return service.StreamSessionWatchEvents(ack) +} + +func streamTurnExecutionWatch(ctx context.Context, service *brokerapi.Service) (any, error) { + ack, errResp := service.HandleSessionTurnExecutionWatchRequest(ctx, brokerapi.SessionTurnExecutionWatchRequest{SchemaID: "runecode.protocol.v0.SessionTurnExecutionWatchRequest", SchemaVersion: "0.1.0", RequestID: "perf-turn-watch", Follow: true, IncludeSnapshot: true}, brokerapi.RequestContext{}) + if errResp != nil { + return nil, fmt.Errorf("session_turn_execution_watch ack: %s", errResp.Error.Code) + } + return service.StreamSessionTurnExecutionWatchEvents(ack) +} + +func timedCall(call func() error) (float64, error) { + started := time.Now() + if err := call(); err != nil { + return 0, err + } + return float64(time.Since(started).Milliseconds()), nil +} + +func payloadStats(value any) (int, int, error) { + blob, err := json.Marshal(value) + if err != nil { + return 0, 0, err + } + count := 1 + if values, ok := value.([]brokerapi.RunWatchEvent); ok { + count = len(values) + } else if values, ok := value.([]brokerapi.ApprovalWatchEvent); ok { + count = len(values) + } else if values, ok := value.([]brokerapi.SessionWatchEvent); ok { + count = len(values) + } else if values, ok := value.([]brokerapi.SessionTurnExecutionWatchEvent); ok { + count = len(values) + } + return len(blob), count, nil +} + +func p95Records(samplesByMetric map[string][]float64) ([]perfcontracts.MeasurementRecord, error) { + keys := make([]string, 0, len(samplesByMetric)) + for metricID := range samplesByMetric { + keys = append(keys, metricID) + } + sort.Strings(keys) + records := make([]perfcontracts.MeasurementRecord, 0, len(keys)) + for _, metricID := range keys { + value, err := p95(samplesByMetric[metricID]) + if err != nil { + return nil, fmt.Errorf("compute p95 for %s: %w", metricID, err) + } + records = append(records, perfcontracts.MeasurementRecord{MetricID: metricID, Value: value, Unit: "ms"}) + } + return records, nil +} + +func maxFloat64(a, b float64) float64 { + if b > a { + return b + } + return a +} + +func p95(samples []float64) (float64, error) { + if len(samples) == 0 { + return 0, fmt.Errorf("samples required") + } + vals := append([]float64(nil), samples...) + sort.Float64s(vals) + idx := int(float64(len(vals)-1) * 0.95) + if idx < 0 { + idx = 0 + } + if idx >= len(vals) { + idx = len(vals) - 1 + } + return vals[idx], nil +} diff --git a/internal/brokerperf/harness_test.go b/internal/brokerperf/harness_test.go new file mode 100644 index 00000000..32ac262a --- /dev/null +++ b/internal/brokerperf/harness_test.go @@ -0,0 +1,72 @@ +package brokerperf + +import ( + "path/filepath" + "runtime" + "testing" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func TestRunDeterministicBrokerHarnessProducesPhase3Metrics(t *testing.T) { + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) + out, err := Run(HarnessConfig{Trials: 2, RepositoryRoot: repoRoot}) + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + if out.SchemaVersion != CheckSchemaVersion { + t.Fatalf("schema_version = %q, want %q", out.SchemaVersion, CheckSchemaVersion) + } + if len(out.Measurements) == 0 { + t.Fatal("measurements empty") + } + assertMetricUnit(t, out.Measurements, "metric.broker.unary.session_list.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.watch.run.snapshot_follow.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.watch.run.snapshot_follow.payload_bytes", "bytes") + assertMetricUnit(t, out.Measurements, "metric.broker.watch.turn_execution.snapshot_follow.event_count", "count") + assertMetricValue(t, out.Measurements, "metric.broker.watch.run.snapshot_follow.event_count", 3) + assertMetricValue(t, out.Measurements, "metric.broker.watch.turn_execution.snapshot_follow.event_count", 3) + assertMetricUnit(t, out.Measurements, "metric.broker.mutation.session_execution_trigger.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.mutation.session_execution_continue.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.mutation.approval_resolve.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.mutation.backend_posture_change.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.attach.local_control_plane.p95_ms", "ms") + assertMetricUnit(t, out.Measurements, "metric.broker.resume.local_control_plane.p95_ms", "ms") +} + +func TestP95RecordsRejectsEmptySampleSet(t *testing.T) { + t.Parallel() + if _, err := p95Records(map[string][]float64{"metric.empty": nil}); err == nil { + t.Fatal("p95Records error = nil, want empty sample failure") + } +} + +func assertMetricUnit(t *testing.T, measurements []perfcontracts.MeasurementRecord, metricID, unit string) { + t.Helper() + for _, m := range measurements { + if m.MetricID == metricID { + if m.Unit != unit { + t.Fatalf("metric %s unit = %q, want %q", metricID, m.Unit, unit) + } + return + } + } + t.Fatalf("metric %s missing", metricID) +} + +func assertMetricValue(t *testing.T, measurements []perfcontracts.MeasurementRecord, metricID string, value float64) { + t.Helper() + for _, m := range measurements { + if m.MetricID == metricID { + if m.Value != value { + t.Fatalf("metric %s value = %v, want %v", metricID, m.Value, value) + } + return + } + } + t.Fatalf("metric %s missing", metricID) +} diff --git a/internal/launcherperf/harness.go b/internal/launcherperf/harness.go new file mode 100644 index 00000000..fdf4eba9 --- /dev/null +++ b/internal/launcherperf/harness.go @@ -0,0 +1,178 @@ +package launcherperf + +import ( + "fmt" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/launcherbackend" + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +const CheckSchemaVersion = "runecode.performance.check.v1" + +type HarnessConfig struct{} + +func Run(_ HarnessConfig) (perfcontracts.CheckOutput, error) { + measurements := make([]perfcontracts.MeasurementRecord, 0, 8) + + microVMCold, microVMWarm, err := simulateBackendStartup(launcherbackend.BackendKindMicroVM) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + measurements = append(measurements, + perfcontracts.MeasurementRecord{MetricID: "metric.launcher.microvm.cold_start.wall_ms", Value: microVMCold, Unit: "ms"}, + perfcontracts.MeasurementRecord{MetricID: "metric.launcher.microvm.warm_start.wall_ms", Value: microVMWarm, Unit: "ms"}, + ) + + containerCold, containerWarm, err := simulateBackendStartup(launcherbackend.BackendKindContainer) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + measurements = append(measurements, + perfcontracts.MeasurementRecord{MetricID: "metric.launcher.container.cold_start.wall_ms", Value: containerCold, Unit: "ms"}, + perfcontracts.MeasurementRecord{MetricID: "metric.launcher.container.warm_start.wall_ms", Value: containerWarm, Unit: "ms"}, + ) + + attestCold, attestWarm, err := simulateAttestationPath() + if err != nil { + return perfcontracts.CheckOutput{}, err + } + measurements = append(measurements, + perfcontracts.MeasurementRecord{MetricID: "metric.attestation.cold.verify.wall_ms", Value: attestCold, Unit: "ms"}, + perfcontracts.MeasurementRecord{MetricID: "metric.attestation.warm.verify.wall_ms", Value: attestWarm, Unit: "ms"}, + ) + + return perfcontracts.CheckOutput{SchemaVersion: CheckSchemaVersion, Measurements: measurements}, nil +} + +func simulateBackendStartup(backend string) (float64, float64, error) { + image, err := deterministicRuntimeImage(backend) + if err != nil { + return 0, 0, err + } + admissionStart := time.Now() + record, err := launcherbackend.NewRuntimeAdmissionRecord(image) + if err != nil { + return 0, 0, err + } + if err := record.Validate(); err != nil { + return 0, 0, err + } + cold := float64(time.Since(admissionStart).Milliseconds()) + + warmStart := time.Now() + if err := record.Validate(); err != nil { + return 0, 0, err + } + warm := float64(time.Since(warmStart).Milliseconds()) + return cold, warm, nil +} + +func simulateAttestationPath() (float64, float64, error) { + image, err := deterministicRuntimeImage(launcherbackend.BackendKindMicroVM) + if err != nil { + return 0, 0, err + } + receipt := launcherbackend.BackendLaunchReceipt{ + RunID: "run-attestation", + BackendKind: launcherbackend.BackendKindMicroVM, + IsolationAssuranceLevel: launcherbackend.IsolationAssuranceIsolated, + ProvisioningPosture: launcherbackend.ProvisioningPostureAttested, + RuntimeImageDescriptorDigest: image.DescriptorDigest, + RuntimeImageBootProfile: image.BootContractVersion, + BootComponentDigestByName: image.ComponentDigests, + AttestationEvidenceSourceKind: launcherbackend.AttestationSourceKindTrustedRuntime, + AttestationMeasurementProfile: image.Attestation.MeasurementProfile, + AttestationEvidenceDigest: "sha256:" + repeatHex('a'), + AttestationVerificationResult: launcherbackend.AttestationVerificationResultValid, + AttestationReplayVerdict: launcherbackend.AttestationReplayVerdictOriginal, + AttestationVerificationDigest: "sha256:" + repeatHex('b'), + AttestationVerificationReasonCodes: []string{}, + } + + coldStart := time.Now() + posture, reasons := launcherbackend.DeriveAttestationPosture(receipt) + if posture != launcherbackend.AttestationPostureValid || len(reasons) > 0 { + return 0, 0, fmt.Errorf("unexpected cold attestation posture %q reasons=%v", posture, reasons) + } + cold := float64(time.Since(coldStart).Milliseconds()) + + warmStart := time.Now() + posture, reasons = launcherbackend.DeriveAttestationPosture(receipt) + if posture != launcherbackend.AttestationPostureValid || len(reasons) > 0 { + return 0, 0, fmt.Errorf("unexpected warm attestation posture %q reasons=%v", posture, reasons) + } + warm := float64(time.Since(warmStart).Milliseconds()) + return cold, warm, nil +} + +func deterministicRuntimeImage(backend string) (launcherbackend.RuntimeImageDescriptor, error) { + boot, measurementProfile, accel, componentDigests := runtimeImageBackendParams(backend) + image := runtimeImageDescriptorBase(backend, boot, accel, componentDigests, measurementProfile) + if digests, err := launcherbackend.DeriveExpectedMeasurementDigests(measurementProfile, boot, componentDigests); err == nil { + image.Attestation.ExpectedMeasurementDigests = digests + } + digest, err := image.ExpectedDescriptorDigest() + if err != nil { + return launcherbackend.RuntimeImageDescriptor{}, err + } + image.DescriptorDigest = digest + image.Signing.PayloadDigest = digest + return image, nil +} + +func runtimeImageBackendParams(backend string) (string, string, string, map[string]string) { + componentDigests := map[string]string{} + boot := launcherbackend.BootProfileMicroVMLinuxKernelInitrdV1 + measurementProfile := launcherbackend.MeasurementProfileMicroVMBootV1 + accel := launcherbackend.AccelerationKindKVM + if backend == launcherbackend.BackendKindContainer { + boot = launcherbackend.BootProfileContainerOCIImageV1 + measurementProfile = launcherbackend.MeasurementProfileContainerImageV1 + accel = launcherbackend.AccelerationKindNotApplicable + componentDigests["image"] = "sha256:" + repeatHex('3') + return boot, measurementProfile, accel, componentDigests + } + componentDigests["kernel"] = "sha256:" + repeatHex('1') + componentDigests["initrd"] = "sha256:" + repeatHex('2') + return boot, measurementProfile, accel, componentDigests +} + +func runtimeImageDescriptorBase(backend, boot, accel string, componentDigests map[string]string, measurementProfile string) launcherbackend.RuntimeImageDescriptor { + return launcherbackend.RuntimeImageDescriptor{ + BackendKind: backend, + BootContractVersion: boot, + PlatformCompatibility: launcherbackend.RuntimeImagePlatformCompat{ + OS: "linux", + Architecture: "amd64", + AccelerationKind: accel, + }, + ComponentDigests: componentDigests, + Signing: &launcherbackend.RuntimeImageSigningHooks{ + PayloadSchemaID: launcherbackend.RuntimeImageSignedPayloadSchemaID, + PayloadSchemaVersion: launcherbackend.RuntimeImageSignedPayloadSchemaVersion, + PayloadDigest: "sha256:" + repeatHex('4'), + SignerRef: "verifier:runtime-image:v1", + SignatureDigest: "sha256:" + repeatHex('5'), + VerifierSetRef: "sha256:" + repeatHex('6'), + Toolchain: runtimeToolchainSigningHooks(), + }, + Attestation: &launcherbackend.RuntimeImageAttestationHook{MeasurementProfile: measurementProfile}, + } +} + +func runtimeToolchainSigningHooks() *launcherbackend.RuntimeToolchainSigningHooks { + return &launcherbackend.RuntimeToolchainSigningHooks{ + DescriptorSchemaID: launcherbackend.RuntimeToolchainDescriptorSchemaID, + DescriptorSchemaVersion: launcherbackend.RuntimeToolchainDescriptorSchemaVersion, + DescriptorDigest: "sha256:" + repeatHex('7'), + SignerRef: "verifier:runtime-toolchain:v1", + SignatureDigest: "sha256:" + repeatHex('8'), + VerifierSetRef: "sha256:" + repeatHex('9'), + } +} + +func repeatHex(ch rune) string { + return strings.Repeat(string(ch), 64) +} diff --git a/internal/launcherperf/harness_test.go b/internal/launcherperf/harness_test.go new file mode 100644 index 00000000..a4fbdcb7 --- /dev/null +++ b/internal/launcherperf/harness_test.go @@ -0,0 +1,39 @@ +package launcherperf + +import ( + "testing" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func TestRunProducesPhase4LauncherAndAttestationMetrics(t *testing.T) { + out, err := Run(HarnessConfig{}) + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + if out.SchemaVersion != CheckSchemaVersion { + t.Fatalf("schema_version = %q, want %q", out.SchemaVersion, CheckSchemaVersion) + } + required := map[string]string{ + "metric.launcher.microvm.cold_start.wall_ms": "ms", + "metric.launcher.microvm.warm_start.wall_ms": "ms", + "metric.launcher.container.cold_start.wall_ms": "ms", + "metric.launcher.container.warm_start.wall_ms": "ms", + "metric.attestation.cold.verify.wall_ms": "ms", + "metric.attestation.warm.verify.wall_ms": "ms", + } + for metricID, unit := range required { + if !hasMetric(out.Measurements, metricID, unit) { + t.Fatalf("missing metric %s (%s)", metricID, unit) + } + } +} + +func hasMetric(measurements []perfcontracts.MeasurementRecord, metricID, unit string) bool { + for _, m := range measurements { + if m.MetricID == metricID && m.Unit == unit { + return true + } + } + return false +} diff --git a/internal/perfcontracts/evaluate.go b/internal/perfcontracts/evaluate.go new file mode 100644 index 00000000..405242ab --- /dev/null +++ b/internal/perfcontracts/evaluate.go @@ -0,0 +1,165 @@ +package perfcontracts + +import ( + "fmt" + "math" +) + +type Violation struct { + MetricID string + Reason string +} + +func Evaluate(check CheckOutput, contracts []ContractFile, baselineByMetric map[string]BaselineFile) []Violation { + measurementByMetric := map[string]MeasurementRecord{} + for _, measurement := range check.Measurements { + measurementByMetric[measurement.MetricID] = measurement + } + var violations []Violation + for _, contract := range contracts { + for _, metric := range contract.Metrics { + measurement, ok := measurementByMetric[metric.MetricID] + if !ok { + violations = append(violations, Violation{MetricID: metric.MetricID, Reason: "measurement missing from check output"}) + continue + } + if measurement.Unit != metric.Unit { + violations = append(violations, Violation{MetricID: metric.MetricID, Reason: fmt.Sprintf("unit mismatch: got %s want %s", measurement.Unit, metric.Unit)}) + continue + } + violations = append(violations, evaluateMetric(metric, measurement.Value, baselineByMetric[metric.MetricID])...) + } + } + return violations +} + +func evaluateMetric(metric MetricContract, measured float64, baseline BaselineFile) []Violation { + switch metric.ComparisonMethod { + case "exact_match": + return evaluateExactMetric(metric, measured) + case "absolute_ceiling", "max_ceiling", "p95_ceiling", "window_average", "window_max": + return evaluateAbsoluteBudgetMetric(metric, measured) + case "median_regression_with_noise_floor": + return evaluateRegressionMetric(metric, measured, baseline) + case "median_plus_regression", "p95_ceiling_plus_regression": + return evaluateHybridMetric(metric, measured, baseline) + case "": + return evaluateMetricByBudgetClass(metric, measured, baseline) + default: + return []Violation{{MetricID: metric.MetricID, Reason: fmt.Sprintf("unsupported comparison method %q", metric.ComparisonMethod)}} + } +} + +func evaluateMetricByBudgetClass(metric MetricContract, measured float64, baseline BaselineFile) []Violation { + switch metric.BudgetClass { + case "exact": + return evaluateExactMetric(metric, measured) + case "absolute-budget": + return evaluateAbsoluteBudgetMetric(metric, measured) + case "regression-budget": + return evaluateRegressionMetric(metric, measured, baseline) + case "hybrid-budget": + return evaluateHybridMetric(metric, measured, baseline) + default: + return []Violation{{MetricID: metric.MetricID, Reason: "unsupported budget class"}} + } +} + +func evaluateExactMetric(metric MetricContract, measured float64) []Violation { + if metric.Threshold.ExactValue == nil { + return []Violation{{MetricID: metric.MetricID, Reason: "exact threshold missing exact_value"}} + } + if measured == *metric.Threshold.ExactValue { + return nil + } + return []Violation{{MetricID: metric.MetricID, Reason: fmt.Sprintf("exact mismatch: got %.4f want %.4f", measured, *metric.Threshold.ExactValue)}} +} + +func evaluateAbsoluteBudgetMetric(metric MetricContract, measured float64) []Violation { + if metric.Threshold.MaxValue == nil { + return []Violation{{MetricID: metric.MetricID, Reason: "absolute-budget threshold missing max_value"}} + } + if measured <= *metric.Threshold.MaxValue { + return nil + } + return []Violation{{MetricID: metric.MetricID, Reason: fmt.Sprintf("value %.4f exceeds max %.4f", measured, *metric.Threshold.MaxValue)}} +} + +func evaluateRegressionMetric(metric MetricContract, measured float64, baseline BaselineFile) []Violation { + violates, details := regressionViolation(metric, measured, baseline) + if !violates { + return nil + } + return []Violation{{MetricID: metric.MetricID, Reason: details}} +} + +func evaluateHybridMetric(metric MetricContract, measured float64, baseline BaselineFile) []Violation { + violations := evaluateAbsoluteBudgetMetric(metric, measured) + if violates, details := regressionViolation(metric, measured, baseline); violates { + violations = append(violations, Violation{MetricID: metric.MetricID, Reason: "hybrid " + details}) + } + return violations +} + +func regressionViolation(metric MetricContract, measured float64, baseline BaselineFile) (bool, string) { + if metric.Threshold.MaxRegressionPercent == nil { + return true, "regression threshold missing max_regression_percent" + } + base, ok := baselineValue(baseline) + if !ok || base == 0 { + return true, "regression threshold missing usable baseline" + } + delta := measured - base + if delta <= 0 { + return false, "" + } + if delta < metric.NoiseFloor { + return false, "" + } + percent := (delta / base) * 100.0 + if percent <= *metric.Threshold.MaxRegressionPercent { + return false, "" + } + allowed := base * (1 + (*metric.Threshold.MaxRegressionPercent / 100.0)) + return true, fmt.Sprintf( + "regression threshold exceeded: value %.4f%s baseline %.4f%s allowed <= %.4f%s (+%.2f%%, max +%.2f%%, noise floor %.4f%s)", + measured, metric.Unit, + base, metric.Unit, + allowed, metric.Unit, + percent, *metric.Threshold.MaxRegressionPercent, + metric.NoiseFloor, metric.Unit, + ) +} + +func baselineValue(file BaselineFile) (float64, bool) { + if file.BaselineValue != nil { + return *file.BaselineValue, true + } + if file.Summary.Median != nil { + return *file.Summary.Median, true + } + if len(file.Samples) == 0 { + return 0, false + } + return median(file.Samples), true +} + +func median(values []float64) float64 { + cp := append([]float64{}, values...) + for i := 0; i < len(cp); i++ { + for j := i + 1; j < len(cp); j++ { + if cp[j] < cp[i] { + cp[i], cp[j] = cp[j], cp[i] + } + } + } + m := len(cp) / 2 + if len(cp)%2 == 0 { + return (cp[m-1] + cp[m]) / 2 + } + return cp[m] +} + +func almostEqual(a, b float64) bool { + return math.Abs(a-b) < 1e-9 +} diff --git a/internal/perfcontracts/evaluate_test.go b/internal/perfcontracts/evaluate_test.go new file mode 100644 index 00000000..15c40726 --- /dev/null +++ b/internal/perfcontracts/evaluate_test.go @@ -0,0 +1,178 @@ +package perfcontracts + +import ( + "strings" + "testing" +) + +func TestEvaluateHonorsComparisonMethodContracts(t *testing.T) { + max500 := 500.0 + max200 := 200.0 + reg15 := 15.0 + exact2 := 2.0 + + for _, tc := range comparisonMethodContractTests(max500, max200, reg15, exact2) { + t.Run(tc.name, func(t *testing.T) { + assertEvaluationResult(t, tc) + }) + } +} + +type comparisonMethodTestCase struct { + name string + metric MetricContract + measurement MeasurementRecord + baseline BaselineFile + wantViolation bool + wantReasonLike string +} + +func comparisonMethodContractTests(max500, max200, reg15, exact2 float64) []comparisonMethodTestCase { + tests := append([]comparisonMethodTestCase{}, absoluteComparisonMethodTests(max500, max200, exact2)...) + tests = append(tests, regressionComparisonMethodTests(max500, reg15)...) + return tests +} + +func absoluteComparisonMethodTests(max500, max200, exact2 float64) []comparisonMethodTestCase { + tests := append([]comparisonMethodTestCase{}, absoluteCeilingComparisonTests(max500, max200)...) + return append(tests, absoluteComparisonOverrideTests(max200, exact2)...) +} + +func absoluteCeilingComparisonTests(max500, max200 float64) []comparisonMethodTestCase { + tests := append([]comparisonMethodTestCase{}, exactAndAbsoluteComparisonTests(max500)...) + return append(tests, percentileAndWindowComparisonTests(max200)...) +} + +func exactAndAbsoluteComparisonTests(max500 float64) []comparisonMethodTestCase { + return []comparisonMethodTestCase{ + { + name: "exact_match passes on exact value", + metric: MetricContract{MetricID: "m.exact.pass", Unit: "count", BudgetClass: "exact", ComparisonMethod: "exact_match", Threshold: MetricThreshold{ExactValue: floatPtr(2)}}, + measurement: MeasurementRecord{MetricID: "m.exact.pass", Unit: "count", Value: 2}, + wantViolation: false, + }, + { + name: "absolute_ceiling fails above max", + metric: MetricContract{MetricID: "m.abs.fail", Unit: "ms", BudgetClass: "absolute-budget", ComparisonMethod: "absolute_ceiling", Threshold: MetricThreshold{MaxValue: &max500}}, + measurement: MeasurementRecord{MetricID: "m.abs.fail", Unit: "ms", Value: 501}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + { + name: "max_ceiling passes under max", + metric: MetricContract{MetricID: "m.max.pass", Unit: "ms", BudgetClass: "absolute-budget", ComparisonMethod: "max_ceiling", Threshold: MetricThreshold{MaxValue: &max500}}, + measurement: MeasurementRecord{MetricID: "m.max.pass", Unit: "ms", Value: 499}, + wantViolation: false, + }, + } +} + +func percentileAndWindowComparisonTests(max200 float64) []comparisonMethodTestCase { + return []comparisonMethodTestCase{ + { + name: "p95_ceiling fails above max", + metric: MetricContract{MetricID: "m.p95.fail", Unit: "ms", BudgetClass: "absolute-budget", ComparisonMethod: "p95_ceiling", Threshold: MetricThreshold{MaxValue: &max200}}, + measurement: MeasurementRecord{MetricID: "m.p95.fail", Unit: "ms", Value: 220}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + { + name: "window_average fails above max", + metric: MetricContract{MetricID: "m.win.avg.fail", Unit: "percent", BudgetClass: "absolute-budget", ComparisonMethod: "window_average", Threshold: MetricThreshold{MaxValue: &max200}}, + measurement: MeasurementRecord{MetricID: "m.win.avg.fail", Unit: "percent", Value: 220}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + { + name: "window_max fails above max", + metric: MetricContract{MetricID: "m.win.max.fail", Unit: "percent", BudgetClass: "absolute-budget", ComparisonMethod: "window_max", Threshold: MetricThreshold{MaxValue: &max200}}, + measurement: MeasurementRecord{MetricID: "m.win.max.fail", Unit: "percent", Value: 220}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + } +} + +func absoluteComparisonOverrideTests(max200, exact2 float64) []comparisonMethodTestCase { + return []comparisonMethodTestCase{ + { + name: "comparison method takes precedence over budget class", + metric: MetricContract{MetricID: "m.method.overrides.budget", Unit: "ms", BudgetClass: "exact", ComparisonMethod: "absolute_ceiling", Threshold: MetricThreshold{ExactValue: &exact2, MaxValue: &max200}}, + measurement: MeasurementRecord{MetricID: "m.method.overrides.budget", Unit: "ms", Value: 220}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + } +} + +func regressionComparisonMethodTests(max500, reg15 float64) []comparisonMethodTestCase { + return []comparisonMethodTestCase{ + { + name: "median_regression_with_noise_floor ignores low-noise deltas", + metric: MetricContract{MetricID: "m.reg.noise.pass", Unit: "ns/op", BudgetClass: "regression-budget", ComparisonMethod: "median_regression_with_noise_floor", Threshold: MetricThreshold{MaxRegressionPercent: ®15}, NoiseFloor: 10}, + measurement: MeasurementRecord{MetricID: "m.reg.noise.pass", Unit: "ns/op", Value: 105}, + baseline: medianBaseline(100), + wantViolation: false, + }, + { + name: "median_regression_with_noise_floor reports measured and allowed values", + metric: MetricContract{MetricID: "m.reg.fail", Unit: "ms", BudgetClass: "regression-budget", ComparisonMethod: "median_regression_with_noise_floor", Threshold: MetricThreshold{MaxRegressionPercent: ®15}, NoiseFloor: 10}, + measurement: MeasurementRecord{MetricID: "m.reg.fail", Unit: "ms", Value: 140}, + baseline: medianBaseline(100), + wantViolation: true, + wantReasonLike: "value 140.0000ms baseline 100.0000ms allowed <= 115.0000ms", + }, + { + name: "median_plus_regression checks absolute max", + metric: MetricContract{MetricID: "m.med.plus.abs.fail", Unit: "ms", BudgetClass: "hybrid-budget", ComparisonMethod: "median_plus_regression", Threshold: MetricThreshold{MaxValue: &max500, MaxRegressionPercent: ®15}, NoiseFloor: 10}, + measurement: MeasurementRecord{MetricID: "m.med.plus.abs.fail", Unit: "ms", Value: 550}, + baseline: BaselineFile{BaselineValue: floatPtr(1000)}, + wantViolation: true, + wantReasonLike: "exceeds max", + }, + { + name: "p95_ceiling_plus_regression checks regression baseline", + metric: MetricContract{MetricID: "m.p95.plus.reg.fail", Unit: "ms", BudgetClass: "hybrid-budget", ComparisonMethod: "p95_ceiling_plus_regression", Threshold: MetricThreshold{MaxValue: &max500, MaxRegressionPercent: ®15}, NoiseFloor: 5}, + measurement: MeasurementRecord{MetricID: "m.p95.plus.reg.fail", Unit: "ms", Value: 130}, + baseline: medianBaseline(100), + wantViolation: true, + wantReasonLike: "hybrid regression threshold exceeded", + }, + } +} + +func assertEvaluationResult(t *testing.T, tc comparisonMethodTestCase) { + t.Helper() + violations := Evaluate( + CheckOutput{Measurements: []MeasurementRecord{tc.measurement}}, + []ContractFile{{ContractID: "c", Metrics: []MetricContract{tc.metric}}}, + map[string]BaselineFile{tc.metric.MetricID: tc.baseline}, + ) + + if tc.wantViolation && len(violations) == 0 { + t.Fatalf("violations = %#v, want at least one violation", violations) + } + if !tc.wantViolation && len(violations) != 0 { + t.Fatalf("violations = %#v, want no violations", violations) + } + if tc.wantReasonLike != "" && !containsViolationReason(violations, tc.metric.MetricID, tc.wantReasonLike) { + t.Fatalf("violations = %#v, want reason containing %q", violations, tc.wantReasonLike) + } +} + +func containsViolationReason(violations []Violation, metricID, wantReasonLike string) bool { + for _, v := range violations { + if v.MetricID == metricID && strings.Contains(v.Reason, wantReasonLike) { + return true + } + } + return false +} + +func medianBaseline(v float64) BaselineFile { + return BaselineFile{Summary: struct { + Median *float64 `json:"median,omitempty"` + }{Median: floatPtr(v)}} +} + +func floatPtr(v float64) *float64 { return &v } diff --git a/internal/perfcontracts/load.go b/internal/perfcontracts/load.go new file mode 100644 index 00000000..0b8aa39a --- /dev/null +++ b/internal/perfcontracts/load.go @@ -0,0 +1,63 @@ +package perfcontracts + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +func LoadManifest(root string) (Manifest, error) { + path := filepath.Join(root, "manifest.json") + var manifest Manifest + if err := loadJSON(path, &manifest); err != nil { + return Manifest{}, err + } + return manifest, nil +} + +func LoadFixtureInventory(root, relPath string) (FixtureInventory, error) { + path := filepath.Join(root, filepath.FromSlash(relPath)) + var inventory FixtureInventory + if err := loadJSON(path, &inventory); err != nil { + return FixtureInventory{}, err + } + return inventory, nil +} + +func LoadContract(root, relPath string) (ContractFile, error) { + path := filepath.Join(root, filepath.FromSlash(relPath)) + var contract ContractFile + if err := loadJSON(path, &contract); err != nil { + return ContractFile{}, err + } + return contract, nil +} + +func LoadBaseline(root, relPath string) (BaselineFile, error) { + path := filepath.Join(root, filepath.FromSlash(relPath)) + var baseline BaselineFile + if err := loadJSON(path, &baseline); err != nil { + return BaselineFile{}, err + } + return baseline, nil +} + +func LoadCheckOutput(path string) (CheckOutput, error) { + var out CheckOutput + if err := loadJSON(path, &out); err != nil { + return CheckOutput{}, err + } + return out, nil +} + +func loadJSON(path string, dst any) error { + raw, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + if err := json.Unmarshal(raw, dst); err != nil { + return fmt.Errorf("decode %s: %w", path, err) + } + return nil +} diff --git a/internal/perfcontracts/types.go b/internal/perfcontracts/types.go new file mode 100644 index 00000000..433a310c --- /dev/null +++ b/internal/perfcontracts/types.go @@ -0,0 +1,121 @@ +package perfcontracts + +type Manifest struct { + SchemaVersion string `json:"schema_version"` + ManifestVersion string `json:"manifest_version"` + ChangeRef string `json:"change_ref"` + FixtureInventoryRef string `json:"fixture_inventory_ref"` + Contracts []ManifestContract `json:"contracts"` + Baselines []ManifestBaseline `json:"baselines,omitempty"` + Taxonomy MetricTaxonomy `json:"metric_taxonomy"` + LaneAuthorities []string `json:"lane_authorities"` + ActivationStates []string `json:"activation_states"` + Deferrals []ManifestDeferral `json:"deferrals,omitempty"` +} + +type ManifestContract struct { + Surface string `json:"surface"` + Path string `json:"path"` +} + +type ManifestBaseline struct { + MetricID string `json:"metric_id"` + Path string `json:"path"` +} + +type ManifestDeferral struct { + ChangeRef string `json:"change_ref"` + Reason string `json:"reason"` +} + +type MetricTaxonomy struct { + BudgetClasses []string `json:"budget_classes"` +} + +type FixtureInventory struct { + SchemaVersion string `json:"schema_version"` + Fixtures []FixtureRecord `json:"fixtures"` +} + +type FixtureRecord struct { + FixtureID string `json:"fixture_id"` + Surface string `json:"surface"` + RuntimeRegime string `json:"runtime_regime"` + Status string `json:"status"` + Notes string `json:"notes,omitempty"` +} + +type ContractFile struct { + SchemaVersion string `json:"schema_version"` + ContractID string `json:"contract_id"` + Surface string `json:"surface"` + Metrics []MetricContract `json:"metrics"` +} + +type MetricContract struct { + MetricID string `json:"metric_id"` + Subsystem string `json:"subsystem"` + RuntimeRegime string `json:"runtime_regime"` + FixtureID string `json:"fixture_id"` + MeasurementKind string `json:"measurement_kind"` + Unit string `json:"unit"` + AuthoritativeEnv string `json:"authoritative_environment"` + SamplingPolicy SamplingPolicy `json:"sampling_policy"` + BudgetClass string `json:"budget_class"` + Threshold MetricThreshold `json:"threshold"` + LaneAuthority string `json:"lane_authority"` + ActivationState string `json:"activation_state"` + BaselineSource string `json:"baseline_source,omitempty"` + BaselineRef string `json:"baseline_ref,omitempty"` + ComparisonMethod string `json:"comparison_method"` + NoiseFloor float64 `json:"practical_noise_floor,omitempty"` + ThresholdOrigin string `json:"threshold_origin"` + TimingBoundary TimingBoundary `json:"timing_boundary"` + Notes string `json:"notes,omitempty"` +} + +type SamplingPolicy struct { + Trials int `json:"trials,omitempty"` + RepeatedSamples int `json:"repeated_samples,omitempty"` + WarmupMillis int `json:"warmup_millis,omitempty"` + ObservationWindowMs int `json:"observation_window_millis,omitempty"` + ObservationWindows int `json:"observation_windows,omitempty"` + P95Authoritative bool `json:"p95_authoritative,omitempty"` + MedianMaxAuthoritative bool `json:"median_max_authoritative,omitempty"` +} + +type MetricThreshold struct { + ExactValue *float64 `json:"exact_value,omitempty"` + MaxValue *float64 `json:"max_value,omitempty"` + MaxRegressionPercent *float64 `json:"max_regression_percent,omitempty"` +} + +type TimingBoundary struct { + StartEvent string `json:"start_event"` + EndEvent string `json:"end_event"` + ClockSource string `json:"clock_source"` + EvidenceSource string `json:"evidence_source"` + IncludedPhases []string `json:"included_phases"` +} + +type CheckOutput struct { + SchemaVersion string `json:"schema_version"` + Measurements []MeasurementRecord `json:"measurements"` +} + +type MeasurementRecord struct { + MetricID string `json:"metric_id"` + Value float64 `json:"value"` + Unit string `json:"unit"` +} + +type BaselineFile struct { + SchemaVersion string `json:"schema_version"` + MetricID string `json:"metric_id"` + Unit string `json:"unit"` + BaselineValue *float64 `json:"baseline_value,omitempty"` + Samples []float64 `json:"samples,omitempty"` + Summary struct { + Median *float64 `json:"median,omitempty"` + } `json:"summary,omitempty"` +} diff --git a/internal/perfcontracts/validate.go b/internal/perfcontracts/validate.go new file mode 100644 index 00000000..eb93f5a4 --- /dev/null +++ b/internal/perfcontracts/validate.go @@ -0,0 +1,222 @@ +package perfcontracts + +import ( + "fmt" + "strings" +) + +var allowedBudgetClasses = map[string]struct{}{ + "exact": {}, + "absolute-budget": {}, + "regression-budget": {}, + "hybrid-budget": {}, +} + +var allowedLaneAuthorities = map[string]struct{}{ + "required_shared_linux": {}, + "required_tight_linux": {}, + "informational_until_stable": {}, + "contract_pending_dependency": {}, + "extended": {}, +} + +var allowedActivationStates = map[string]struct{}{ + "defined": {}, + "informational": {}, + "required": {}, + "contract_pending_dependency": {}, +} + +var allowedThresholdOrigins = map[string]struct{}{ + "product_budget": {}, + "investigation_baseline": {}, + "first_calibration": {}, + "temporary_guardrail": {}, +} + +func Validate(manifest Manifest, inventory FixtureInventory, contracts []ContractFile) error { + return ValidateWithBaselines(manifest, inventory, contracts, nil) +} + +func ValidateWithBaselines(manifest Manifest, inventory FixtureInventory, contracts []ContractFile, baselinesByMetric map[string]BaselineFile) error { + if err := validateManifestAndInventory(manifest, inventory); err != nil { + return err + } + baselineRefsByMetric, err := baselineRefSet(manifest.Baselines) + if err != nil { + return err + } + fixtures, err := fixtureSet(inventory) + if err != nil { + return err + } + return validateContracts(contracts, fixtures, baselinesByMetric, baselineRefsByMetric) +} + +func validateManifestAndInventory(manifest Manifest, inventory FixtureInventory) error { + if strings.TrimSpace(manifest.SchemaVersion) == "" { + return fmt.Errorf("manifest schema_version is required") + } + if strings.TrimSpace(inventory.SchemaVersion) == "" { + return fmt.Errorf("fixture inventory schema_version is required") + } + return nil +} + +func fixtureSet(inventory FixtureInventory) (map[string]struct{}, error) { + fixtures := map[string]struct{}{} + for _, fixture := range inventory.Fixtures { + if strings.TrimSpace(fixture.FixtureID) == "" { + return nil, fmt.Errorf("fixture_id is required") + } + fixtures[fixture.FixtureID] = struct{}{} + } + return fixtures, nil +} + +func baselineRefSet(entries []ManifestBaseline) (map[string]string, error) { + refs := map[string]string{} + for _, entry := range entries { + if existing, ok := refs[entry.MetricID]; ok { + return nil, fmt.Errorf("manifest baseline metric_id %q duplicated with paths %q and %q", entry.MetricID, existing, entry.Path) + } + refs[entry.MetricID] = entry.Path + } + return refs, nil +} + +func validateContracts(contracts []ContractFile, fixtures map[string]struct{}, baselinesByMetric map[string]BaselineFile, baselineRefsByMetric map[string]string) error { + for _, contract := range contracts { + if err := validateContract(contract, fixtures, baselinesByMetric, baselineRefsByMetric); err != nil { + return err + } + } + return nil +} + +func validateContract(contract ContractFile, fixtures map[string]struct{}, baselinesByMetric map[string]BaselineFile, baselineRefsByMetric map[string]string) error { + if strings.TrimSpace(contract.SchemaVersion) == "" { + return fmt.Errorf("contract %s missing schema_version", contract.ContractID) + } + for _, metric := range contract.Metrics { + if err := validateMetric(metric, fixtures, baselinesByMetric, baselineRefsByMetric); err != nil { + return fmt.Errorf("contract %s metric %s invalid: %w", contract.ContractID, metric.MetricID, err) + } + } + return nil +} + +func validateMetric(metric MetricContract, fixtures map[string]struct{}, baselinesByMetric map[string]BaselineFile, baselineRefsByMetric map[string]string) error { + checks := []func(MetricContract, map[string]struct{}, map[string]BaselineFile, map[string]string) error{ + validateMetricIdentity, + validateMetricEnums, + validateMetricFixture, + validateMetricThresholdOrigin, + validateMetricTimingBoundary, + validateMetricBaseline, + } + for _, check := range checks { + if err := check(metric, fixtures, baselinesByMetric, baselineRefsByMetric); err != nil { + return err + } + } + return nil +} + +func validateMetricIdentity(metric MetricContract, _ map[string]struct{}, _ map[string]BaselineFile, _ map[string]string) error { + if strings.TrimSpace(metric.MetricID) == "" { + return fmt.Errorf("metric_id is required") + } + return nil +} + +func validateMetricEnums(metric MetricContract, _ map[string]struct{}, _ map[string]BaselineFile, _ map[string]string) error { + if _, ok := allowedBudgetClasses[metric.BudgetClass]; !ok { + return fmt.Errorf("budget_class %q unsupported", metric.BudgetClass) + } + if _, ok := allowedLaneAuthorities[metric.LaneAuthority]; !ok { + return fmt.Errorf("lane_authority %q unsupported", metric.LaneAuthority) + } + if _, ok := allowedActivationStates[metric.ActivationState]; !ok { + return fmt.Errorf("activation_state %q unsupported", metric.ActivationState) + } + return nil +} + +func validateMetricFixture(metric MetricContract, fixtures map[string]struct{}, _ map[string]BaselineFile, _ map[string]string) error { + if _, ok := fixtures[metric.FixtureID]; !ok { + return fmt.Errorf("fixture_id %q missing from inventory", metric.FixtureID) + } + return nil +} + +func validateMetricThresholdOrigin(metric MetricContract, _ map[string]struct{}, _ map[string]BaselineFile, _ map[string]string) error { + if strings.TrimSpace(metric.ThresholdOrigin) == "" { + return fmt.Errorf("threshold_origin is required") + } + if _, ok := allowedThresholdOrigins[metric.ThresholdOrigin]; !ok { + return fmt.Errorf("threshold_origin %q unsupported", metric.ThresholdOrigin) + } + return nil +} + +func validateMetricTimingBoundary(metric MetricContract, _ map[string]struct{}, _ map[string]BaselineFile, _ map[string]string) error { + boundary := metric.TimingBoundary + if strings.TrimSpace(boundary.StartEvent) == "" || strings.TrimSpace(boundary.EndEvent) == "" { + return fmt.Errorf("timing_boundary start_event/end_event are required") + } + if strings.TrimSpace(boundary.ClockSource) == "" || strings.TrimSpace(boundary.EvidenceSource) == "" { + return fmt.Errorf("timing_boundary clock_source/evidence_source are required") + } + if len(boundary.IncludedPhases) == 0 { + return fmt.Errorf("timing_boundary included_phases is required") + } + return nil +} + +func validateMetricBaseline(metric MetricContract, _ map[string]struct{}, baselinesByMetric map[string]BaselineFile, baselineRefsByMetric map[string]string) error { + if !requiresBaselineValidation(metric) { + return nil + } + if strings.TrimSpace(metric.BaselineRef) == "" { + return fmt.Errorf("baseline_ref is required for %s", metric.BudgetClass) + } + if err := validateBaselineRefProvenance(metric, baselineRefsByMetric); err != nil { + return err + } + if baselinesByMetric == nil { + return nil + } + baseline, ok := baselinesByMetric[metric.MetricID] + if !ok { + return fmt.Errorf("baseline for metric_id %q missing from manifest baselines", metric.MetricID) + } + if strings.TrimSpace(baseline.MetricID) != metric.MetricID { + return fmt.Errorf("baseline metric_id %q does not match contract metric_id %q", baseline.MetricID, metric.MetricID) + } + if strings.TrimSpace(baseline.Unit) != metric.Unit { + return fmt.Errorf("baseline unit %q does not match contract unit %q", baseline.Unit, metric.Unit) + } + if _, ok := baselineValue(baseline); !ok { + return fmt.Errorf("baseline for metric_id %q has no usable baseline value", metric.MetricID) + } + return nil +} + +func requiresBaselineValidation(metric MetricContract) bool { + if metric.BudgetClass != "regression-budget" && metric.BudgetClass != "hybrid-budget" { + return false + } + return metric.ActivationState == "required" +} + +func validateBaselineRefProvenance(metric MetricContract, baselineRefsByMetric map[string]string) error { + authoritativeRef, ok := baselineRefsByMetric[metric.MetricID] + if !ok { + return fmt.Errorf("baseline_ref for metric_id %q missing from manifest baselines", metric.MetricID) + } + if strings.TrimSpace(authoritativeRef) != metric.BaselineRef { + return fmt.Errorf("baseline_ref %q does not match manifest baseline path %q for metric_id %q", metric.BaselineRef, authoritativeRef, metric.MetricID) + } + return nil +} diff --git a/internal/perfcontracts/validate_test.go b/internal/perfcontracts/validate_test.go new file mode 100644 index 00000000..bba530da --- /dev/null +++ b/internal/perfcontracts/validate_test.go @@ -0,0 +1,214 @@ +package perfcontracts + +import "testing" + +func TestValidateAcceptsReviewedMetricContract(t *testing.T) { + manifest := Manifest{SchemaVersion: "runecode.performance.manifest.v1"} + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.attach.latency.p95", + FixtureID: "tui.empty.v1", + BudgetClass: "absolute-budget", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "product_budget", + TimingBoundary: TimingBoundary{ + StartEvent: "spawn", + EndEvent: "ready", + ClockSource: "monotonic", + EvidenceSource: "events", + IncludedPhases: []string{"launch"}, + }, + }}, + }} + if err := Validate(manifest, inventory, contracts); err != nil { + t.Fatalf("Validate returned error: %v", err) + } +} + +func TestValidateRejectsMissingFixtureReference(t *testing.T) { + manifest := Manifest{SchemaVersion: "runecode.performance.manifest.v1"} + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.attach.latency.p95", + FixtureID: "missing.v1", + BudgetClass: "absolute-budget", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "product_budget", + TimingBoundary: TimingBoundary{StartEvent: "spawn", EndEvent: "ready", ClockSource: "monotonic", EvidenceSource: "events", IncludedPhases: []string{"launch"}}, + }}, + }} + if err := Validate(manifest, inventory, contracts); err == nil { + t.Fatal("Validate error = nil, want missing fixture failure") + } +} + +func TestValidateRejectsUnsupportedThresholdOrigin(t *testing.T) { + manifest := Manifest{SchemaVersion: "runecode.performance.manifest.v1"} + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.attach.latency.p95", + FixtureID: "tui.empty.v1", + BudgetClass: "absolute-budget", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "unsupported_origin", + TimingBoundary: TimingBoundary{StartEvent: "spawn", EndEvent: "ready", ClockSource: "monotonic", EvidenceSource: "events", IncludedPhases: []string{"launch"}}, + }}, + }} + if err := Validate(manifest, inventory, contracts); err == nil { + t.Fatal("Validate error = nil, want threshold_origin failure") + } +} + +func TestValidateWithBaselinesRejectsRegressionMetricWithoutBaseline(t *testing.T) { + manifest := Manifest{SchemaVersion: "runecode.performance.manifest.v1"} + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + threshold := 15.0 + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + FixtureID: "tui.empty.v1", + Unit: "ns/op", + BudgetClass: "regression-budget", + BaselineRef: "baselines/metric.tui.render.shell_view_empty.ns_op.v1.json", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "first_calibration", + Threshold: MetricThreshold{MaxRegressionPercent: &threshold}, + TimingBoundary: TimingBoundary{StartEvent: "start", EndEvent: "end", ClockSource: "monotonic", EvidenceSource: "bench", IncludedPhases: []string{"render"}}, + }}, + }} + if err := ValidateWithBaselines(manifest, inventory, contracts, map[string]BaselineFile{}); err == nil { + t.Fatal("ValidateWithBaselines error = nil, want missing baseline failure") + } +} + +func TestValidateWithBaselinesAllowsNonRequiredRegressionMetricWithoutBaseline(t *testing.T) { + manifest := Manifest{SchemaVersion: "runecode.performance.manifest.v1"} + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + threshold := 15.0 + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + FixtureID: "tui.empty.v1", + Unit: "ns/op", + BudgetClass: "regression-budget", + LaneAuthority: "required_shared_linux", + ActivationState: "defined", + ThresholdOrigin: "first_calibration", + Threshold: MetricThreshold{MaxRegressionPercent: &threshold}, + TimingBoundary: TimingBoundary{StartEvent: "start", EndEvent: "end", ClockSource: "monotonic", EvidenceSource: "bench", IncludedPhases: []string{"render"}}, + }}, + }} + if err := ValidateWithBaselines(manifest, inventory, contracts, map[string]BaselineFile{}); err != nil { + t.Fatalf("ValidateWithBaselines returned error for non-required regression metric: %v", err) + } +} + +func TestValidateWithBaselinesRejectsMismatchedBaselineUnit(t *testing.T) { + manifest, inventory, contracts, baselines := mismatchedBaselineUnitFixture() + if err := ValidateWithBaselines(manifest, inventory, contracts, baselines); err == nil { + t.Fatal("ValidateWithBaselines error = nil, want baseline unit mismatch failure") + } +} + +func mismatchedBaselineUnitFixture() (Manifest, FixtureInventory, []ContractFile, map[string]BaselineFile) { + manifest := Manifest{ + SchemaVersion: "runecode.performance.manifest.v1", + Baselines: []ManifestBaseline{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + Path: "baselines/metric.tui.render.shell_view_empty.ns_op.v1.json", + }}, + } + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + threshold := 15.0 + median := 1000.0 + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + FixtureID: "tui.empty.v1", + Unit: "ns/op", + BudgetClass: "regression-budget", + BaselineRef: "baselines/metric.tui.render.shell_view_empty.ns_op.v1.json", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "first_calibration", + Threshold: MetricThreshold{MaxRegressionPercent: &threshold}, + TimingBoundary: TimingBoundary{StartEvent: "start", EndEvent: "end", ClockSource: "monotonic", EvidenceSource: "bench", IncludedPhases: []string{"render"}}, + }}, + }} + baselines := map[string]BaselineFile{ + "metric.tui.render.shell_view_empty.ns_op": { + SchemaVersion: "runecode.performance.baseline.v1", + MetricID: "metric.tui.render.shell_view_empty.ns_op", + Unit: "ms", + Summary: struct { + Median *float64 `json:"median,omitempty"` + }{Median: &median}, + }, + } + return manifest, inventory, contracts, baselines +} + +func TestValidateWithBaselinesRejectsMismatchedBaselineRefProvenance(t *testing.T) { + manifest := Manifest{ + SchemaVersion: "runecode.performance.manifest.v1", + Baselines: []ManifestBaseline{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + Path: "baselines/metric.tui.render.shell_view_empty.ns_op.v1.json", + }}, + } + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "tui.empty.v1"}}} + threshold := 15.0 + contracts := []ContractFile{{ + SchemaVersion: "runecode.performance.contract.v1", + ContractID: "performance.tui.v1", + Metrics: []MetricContract{{ + MetricID: "metric.tui.render.shell_view_empty.ns_op", + FixtureID: "tui.empty.v1", + Unit: "ns/op", + BudgetClass: "regression-budget", + BaselineRef: "baselines/other-baseline.v1.json", + LaneAuthority: "required_shared_linux", + ActivationState: "required", + ThresholdOrigin: "first_calibration", + Threshold: MetricThreshold{MaxRegressionPercent: &threshold}, + TimingBoundary: TimingBoundary{StartEvent: "start", EndEvent: "end", ClockSource: "monotonic", EvidenceSource: "bench", IncludedPhases: []string{"render"}}, + }}, + }} + if err := ValidateWithBaselines(manifest, inventory, contracts, nil); err == nil { + t.Fatal("ValidateWithBaselines error = nil, want baseline_ref provenance mismatch failure") + } +} + +func TestValidateWithBaselinesRejectsDuplicateManifestBaselineMetricID(t *testing.T) { + manifest := Manifest{ + SchemaVersion: "runecode.performance.manifest.v1", + Baselines: []ManifestBaseline{ + {MetricID: "metric.sample", Path: "baselines/metric.sample.v1.json"}, + {MetricID: "metric.sample", Path: "baselines/metric.sample.v2.json"}, + }, + } + inventory := FixtureInventory{SchemaVersion: "runecode.performance.fixtures.v1", Fixtures: []FixtureRecord{{FixtureID: "fixture.sample"}}} + contracts := []ContractFile{{SchemaVersion: "runecode.performance.contract.v1", ContractID: "performance.sample.v1"}} + if err := ValidateWithBaselines(manifest, inventory, contracts, nil); err == nil { + t.Fatal("ValidateWithBaselines error = nil, want duplicate manifest baseline metric_id failure") + } +} diff --git a/internal/perffixtures/broker_store.go b/internal/perffixtures/broker_store.go new file mode 100644 index 00000000..842c088f --- /dev/null +++ b/internal/perffixtures/broker_store.go @@ -0,0 +1,82 @@ +package perffixtures + +import ( + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" +) + +const ( + FixtureTUIEmptyV1 = "tui.empty.v1" + FixtureTUIWaitingV1 = "tui.waiting.v1" +) + +type BrokerStoreFixtureResult struct { + FixtureID string + SessionID string + TurnID string + RootDir string +} + +func BuildBrokerStoreFixture(rootDir string, fixtureID string) (BrokerStoreFixtureResult, error) { + switch fixtureID { + case FixtureTUIEmptyV1: + if _, err := artifacts.NewStore(rootDir); err != nil { + return BrokerStoreFixtureResult{}, err + } + return BrokerStoreFixtureResult{FixtureID: fixtureID, RootDir: rootDir}, nil + case FixtureTUIWaitingV1: + return buildBrokerStoreWaiting(rootDir) + default: + return BrokerStoreFixtureResult{}, ErrUnsupportedFixtureID + } +} + +func buildBrokerStoreWaiting(rootDir string) (BrokerStoreFixtureResult, error) { + store, err := artifacts.NewStore(rootDir) + if err != nil { + return BrokerStoreFixtureResult{}, err + } + seedTime := time.Date(2026, time.March, 18, 9, 0, 0, 0, time.UTC) + appendResult, err := store.AppendSessionExecutionTrigger(waitingSessionExecutionTriggerAppendRequest(seedTime)) + if err != nil { + return BrokerStoreFixtureResult{}, err + } + if _, err := store.UpdateSessionTurnExecution(artifacts.SessionTurnExecutionUpdateRequest{ + SessionID: "sess-manual-multiwait", + TurnID: appendResult.TurnExecution.TurnID, + ExecutionState: "waiting", + WaitKind: "approval", + WaitState: "awaiting_review", + BlockedReasonCode: "approval_wait", + OccurredAt: seedTime.Add(10 * time.Second), + }); err != nil { + return BrokerStoreFixtureResult{}, err + } + return BrokerStoreFixtureResult{ + FixtureID: FixtureTUIWaitingV1, + SessionID: "sess-manual-multiwait", + TurnID: appendResult.TurnExecution.TurnID, + RootDir: rootDir, + }, nil +} + +func waitingSessionExecutionTriggerAppendRequest(seedTime time.Time) artifacts.SessionExecutionTriggerAppendRequest { + return artifacts.SessionExecutionTriggerAppendRequest{ + SessionID: "sess-manual-multiwait", + WorkspaceID: "workspace-local", + AuthoritativeRepositoryRoot: "/workspace/repo", + TriggerSource: "interactive_user", + RequestedOperation: "start", + WorkflowRouting: artifacts.SessionWorkflowPackRoutingDurableState{ + WorkflowFamily: "runecontext", + WorkflowOperation: "approved_change_implementation", + }, + ExecutionState: "waiting", + WaitKind: "approval", + WaitState: "awaiting_review", + BlockedReasonCode: "approval_wait", + UserMessageContentText: "WAITING session=sess-manual-multiwait", + OccurredAt: seedTime, + } +} diff --git a/internal/perffixtures/broker_store_test.go b/internal/perffixtures/broker_store_test.go new file mode 100644 index 00000000..b2208e48 --- /dev/null +++ b/internal/perffixtures/broker_store_test.go @@ -0,0 +1,64 @@ +package perffixtures + +import ( + "testing" + + "github.com/runecode-ai/runecode/internal/artifacts" +) + +func TestBuildBrokerStoreFixtureEmptyAndWaiting(t *testing.T) { + t.Run(FixtureTUIEmptyV1, testEmptyBrokerStoreFixture) + t.Run(FixtureTUIWaitingV1, testWaitingBrokerStoreFixture) +} + +func testEmptyBrokerStoreFixture(t *testing.T) { + t.Helper() + store, _, err := buildFixtureStore(t, FixtureTUIEmptyV1) + if err != nil { + t.Fatalf("buildFixtureStore returned error: %v", err) + } + if states := store.SessionDurableStates(); len(states) != 0 { + t.Fatalf("empty fixture sessions = %d, want 0", len(states)) + } +} + +func testWaitingBrokerStoreFixture(t *testing.T) { + t.Helper() + store, result, err := buildFixtureStore(t, FixtureTUIWaitingV1) + if err != nil { + t.Fatalf("buildFixtureStore returned error: %v", err) + } + state, ok := store.SessionState(result.SessionID) + if !ok { + t.Fatalf("SessionState(%q) ok=false", result.SessionID) + } + if state.WorkPosture != "waiting" { + t.Fatalf("work_posture = %q, want waiting", state.WorkPosture) + } + if len(state.TurnExecutions) != 1 { + t.Fatalf("turn_executions len = %d, want 1", len(state.TurnExecutions)) + } + if got := state.TurnExecutions[0].ExecutionState; got != "waiting" { + t.Fatalf("execution_state = %q, want waiting", got) + } +} + +func buildFixtureStore(t *testing.T, fixtureID string) (*artifacts.Store, BrokerStoreFixtureResult, error) { + t.Helper() + root := t.TempDir() + result, err := BuildBrokerStoreFixture(root, fixtureID) + if err != nil { + return nil, BrokerStoreFixtureResult{}, err + } + store, err := artifacts.NewStore(root) + if err != nil { + return nil, BrokerStoreFixtureResult{}, err + } + return store, result, nil +} + +func TestBuildBrokerStoreFixtureRejectsUnknown(t *testing.T) { + if _, err := BuildBrokerStoreFixture(t.TempDir(), "unknown"); err == nil { + t.Fatal("BuildBrokerStoreFixture error = nil, want unsupported fixture") + } +} diff --git a/internal/perffixtures/errors.go b/internal/perffixtures/errors.go new file mode 100644 index 00000000..562fc6ed --- /dev/null +++ b/internal/perffixtures/errors.go @@ -0,0 +1,5 @@ +package perffixtures + +import "errors" + +var ErrUnsupportedFixtureID = errors.New("unsupported fixture id") diff --git a/internal/perffixtures/runner.go b/internal/perffixtures/runner.go new file mode 100644 index 00000000..0d45d3dc --- /dev/null +++ b/internal/perffixtures/runner.go @@ -0,0 +1,32 @@ +package perffixtures + +import ( + "fmt" + "os" + "path/filepath" +) + +const FixtureRunnerBoundaryMinimal = "runner.boundary.minimal.v1" + +type RunnerFixtureResult struct { + FixtureID string + RootDir string + WorkflowFilePath string + WorkspaceDir string +} + +func BuildRunnerFixture(rootDir string, fixtureID string) (RunnerFixtureResult, error) { + if fixtureID != FixtureRunnerBoundaryMinimal { + return RunnerFixtureResult{}, fmt.Errorf("%w: %s", ErrUnsupportedFixtureID, fixtureID) + } + workspace := filepath.Join(rootDir, "runner-workspace") + if err := os.MkdirAll(workspace, 0o755); err != nil { + return RunnerFixtureResult{}, err + } + workflowPath := filepath.Join(rootDir, "workflow.json") + workflow := `{"schema_id":"runecode.protocol.runner.workflow.v1","name":"minimal","steps":[{"id":"step-1","kind":"noop","inputs":{}}]}` + if err := os.WriteFile(workflowPath, []byte(workflow), 0o644); err != nil { + return RunnerFixtureResult{}, err + } + return RunnerFixtureResult{FixtureID: fixtureID, RootDir: rootDir, WorkflowFilePath: workflowPath, WorkspaceDir: workspace}, nil +} diff --git a/internal/perffixtures/stubs.go b/internal/perffixtures/stubs.go new file mode 100644 index 00000000..dc83f747 --- /dev/null +++ b/internal/perffixtures/stubs.go @@ -0,0 +1,43 @@ +package perffixtures + +import "context" + +type StubProviderBackend struct{} + +type StubProviderRequest struct { + Prompt string +} + +type StubProviderResponse struct { + Text string + StatusCode int + ProviderLatencyMillis int +} + +func (StubProviderBackend) Invoke(_ context.Context, _ StubProviderRequest) StubProviderResponse { + return StubProviderResponse{Text: "stubbed provider response", StatusCode: 200, ProviderLatencyMillis: 7} +} + +type StubSecretsBackend struct{} + +func (StubSecretsBackend) IssueLease(runID string, providerID string) string { + return "lease.stub." + runID + "." + providerID +} + +type StubExternalAnchorTarget struct{} + +func (StubExternalAnchorTarget) Prepare() string { + return "prepared" +} + +func (StubExternalAnchorTarget) ExecuteFastComplete() string { + return "completed" +} + +func (StubExternalAnchorTarget) ExecuteDeferred() string { + return "deferred" +} + +func (StubExternalAnchorTarget) AdmitReceipt() string { + return "admitted" +} diff --git a/internal/perffixtures/stubs_test.go b/internal/perffixtures/stubs_test.go new file mode 100644 index 00000000..726a7eae --- /dev/null +++ b/internal/perffixtures/stubs_test.go @@ -0,0 +1,25 @@ +package perffixtures + +import ( + "context" + "testing" +) + +func TestStubProviderSecretsAndExternalAnchorDeterministic(t *testing.T) { + provider := StubProviderBackend{} + resp := provider.Invoke(context.Background(), StubProviderRequest{Prompt: "hello"}) + if resp.StatusCode != 200 || resp.Text == "" || resp.ProviderLatencyMillis <= 0 { + t.Fatalf("provider response = %#v, want deterministic non-empty success", resp) + } + + secrets := StubSecretsBackend{} + lease := secrets.IssueLease("run-1", "provider-1") + if lease != "lease.stub.run-1.provider-1" { + t.Fatalf("lease = %q, want deterministic lease id", lease) + } + + anchor := StubExternalAnchorTarget{} + if anchor.Prepare() != "prepared" || anchor.ExecuteFastComplete() != "completed" || anchor.ExecuteDeferred() != "deferred" || anchor.AdmitReceipt() != "admitted" { + t.Fatal("external anchor stub returned unexpected state") + } +} diff --git a/internal/perffixtures/workflow.go b/internal/perffixtures/workflow.go new file mode 100644 index 00000000..529aa4d5 --- /dev/null +++ b/internal/perffixtures/workflow.go @@ -0,0 +1,135 @@ +package perffixtures + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +const ( + FixtureWorkflowFirstPartyMinimal = "workflow.first-party-minimal.v1" + FixtureWorkflowCHG050Compile = "workflow.chg050-compile.v1" +) + +type WorkflowFixtureResult struct { + FixtureID string + RootDir string + RunPlan string + Workspace string +} + +func BuildWorkflowFixture(rootDir, fixtureID string) (WorkflowFixtureResult, error) { + workspace := filepath.Join(rootDir, "workspace") + runplan := filepath.Join(rootDir, "runplan.json") + if err := os.MkdirAll(workspace, 0o755); err != nil { + return WorkflowFixtureResult{}, err + } + var runplanContent []byte + switch fixtureID { + case FixtureWorkflowFirstPartyMinimal: + raw, err := json.MarshalIndent(validRunPlanFixture("workflow.first-party-minimal.v1", "workflow_first_party_minimal", "process_first_party_minimal"), "", " ") + if err != nil { + return WorkflowFixtureResult{}, err + } + runplanContent = raw + case FixtureWorkflowCHG050Compile: + raw, err := json.MarshalIndent(validRunPlanFixture("workflow.chg050-compile.v1", "workflow_chg050_compile", "process_chg050_compile"), "", " ") + if err != nil { + return WorkflowFixtureResult{}, err + } + runplanContent = raw + default: + return WorkflowFixtureResult{}, fmt.Errorf("%w: %s", ErrUnsupportedFixtureID, fixtureID) + } + if err := os.WriteFile(runplan, runplanContent, 0o644); err != nil { + return WorkflowFixtureResult{}, err + } + return WorkflowFixtureResult{FixtureID: fixtureID, RootDir: rootDir, RunPlan: runplan, Workspace: workspace}, nil +} + +func validRunPlanFixture(fixtureID, workflowID, processID string) map[string]any { + gate := runPlanGateContract() + gateDefinition := runPlanGateDefinition(gate) + entryDefinition := runPlanEntryDefinition(gate) + return map[string]any{ + "schema_id": "runecode.protocol.v0.RunPlan", + "schema_version": "0.4.0", + "plan_id": "plan_" + workflowID, + "run_id": "run_" + workflowID, + "workflow_id": workflowID, + "workflow_version": "1.0.0", + "process_id": processID, + "approval_profile": "moderate", + "autonomy_posture": "balanced", + "workflow_definition_hash": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "process_definition_hash": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "policy_context_hash": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "compiled_at": "2026-01-01T00:00:00Z", + "role_instance_ids": []string{"role_alpha"}, + "executor_bindings": []map[string]any{{ + "binding_id": "binding_alpha", + "executor_id": "executor_alpha", + "executor_class": "workspace_ordinary", + "allowed_role_kinds": []string{"developer"}, + }}, + "gate_definitions": []map[string]any{gateDefinition}, + "dependency_edges": []any{}, + "entries": []map[string]any{entryDefinition}, + } +} + +func runPlanGateDefinition(gate map[string]any) map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.GateDefinition", + "schema_version": "0.2.0", + "gate": gate, + "checkpoint_code": "quality", + "order_index": 0, + "stage_id": "quality_stage", + "step_id": "quality_lint", + "role_instance_id": "role_alpha", + "executor_binding_id": "binding_alpha", + "dependency_cache_handoffs": fixtureDependencyCacheHandoffs(), + } +} + +func runPlanEntryDefinition(gate map[string]any) map[string]any { + return map[string]any{ + "entry_id": "quality_lint", + "entry_kind": "gate", + "order_index": 0, + "stage_id": "quality_stage", + "step_id": "quality_lint", + "role_instance_id": "role_alpha", + "executor_binding_id": "binding_alpha", + "checkpoint_code": "quality", + "gate": gate, + "dependency_cache_handoffs": fixtureDependencyCacheHandoffs(), + "depends_on_entry_ids": []string{}, + "blocks_entry_ids": []string{}, + "supported_wait_kinds": []string{"waiting_operator_input", "waiting_approval"}, + } +} + +func runPlanGateContract() map[string]any { + return map[string]any{ + "schema_id": "runecode.protocol.v0.GateContract", + "schema_version": "0.1.0", + "gate_id": "lint", + "gate_kind": "lint", + "gate_version": "0.1.0", + "normalized_inputs": []any{}, + "plan_binding": map[string]any{"checkpoint_code": "quality", "order_index": 0}, + "retry_semantics": map[string]any{"retry_mode": "new_attempt_required", "max_attempts": 2}, + "override_semantics": map[string]any{"override_mode": "policy_action_required", "action_kind": "action_gate_override", "approval_trigger_code": "gate_override"}, + } +} + +func fixtureDependencyCacheHandoffs() []map[string]any { + return []map[string]any{{ + "request_digest": map[string]any{"hash_alg": "sha256", "hash": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}, + "consumer_role": "workspace", + "required": true, + }} +} diff --git a/internal/perffixtures/workflow_test.go b/internal/perffixtures/workflow_test.go new file mode 100644 index 00000000..f15bce1a --- /dev/null +++ b/internal/perffixtures/workflow_test.go @@ -0,0 +1,24 @@ +package perffixtures + +import ( + "os" + "testing" +) + +func TestBuildWorkflowFixture(t *testing.T) { + for _, fixtureID := range []string{FixtureWorkflowFirstPartyMinimal, FixtureWorkflowCHG050Compile} { + fixtureID := fixtureID + t.Run(fixtureID, func(t *testing.T) { + result, err := BuildWorkflowFixture(t.TempDir(), fixtureID) + if err != nil { + t.Fatalf("BuildWorkflowFixture returned error: %v", err) + } + if _, err := os.Stat(result.RunPlan); err != nil { + t.Fatalf("runplan missing: %v", err) + } + if _, err := os.Stat(result.Workspace); err != nil { + t.Fatalf("workspace missing: %v", err) + } + }) + } +} diff --git a/internal/runnerworkflowperf/harness.go b/internal/runnerworkflowperf/harness.go new file mode 100644 index 00000000..b8e56042 --- /dev/null +++ b/internal/runnerworkflowperf/harness.go @@ -0,0 +1,136 @@ +package runnerworkflowperf + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/projectsubstrate" +) + +const CheckSchemaVersion = "runecode.performance.check.v1" + +type HarnessConfig struct { + RepositoryRoot string + CommandTimeout time.Duration + CommandRunner func(repoRoot string, timeout time.Duration, args ...string) (float64, error) +} + +type runnerMeasurementSpec struct { + metricID string + mode string + fixture string +} + +func Run(cfg HarnessConfig) (perfcontracts.CheckOutput, error) { + repoRoot, err := resolveRepoRoot(cfg.RepositoryRoot) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + timeout := resolvedTimeout(cfg.CommandTimeout) + runner := cfg.CommandRunner + if runner == nil { + runner = measureRunnerCommand + } + measurements, err := collectAllMeasurements(repoRoot, timeout, runner) + if err != nil { + return perfcontracts.CheckOutput{}, err + } + return perfcontracts.CheckOutput{SchemaVersion: CheckSchemaVersion, Measurements: measurements}, nil +} + +func resolvedTimeout(timeout time.Duration) time.Duration { + if timeout <= 0 { + return 2 * time.Minute + } + return timeout +} + +func collectAllMeasurements( + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), +) ([]perfcontracts.MeasurementRecord, error) { + measurements := make([]perfcontracts.MeasurementRecord, 0, 12) + if err := appendRunnerCheckMeasurements(&measurements, repoRoot, timeout, runner); err != nil { + return nil, err + } + if err := appendWorkflowMeasurements(&measurements, repoRoot, timeout, runner); err != nil { + return nil, err + } + if err := appendCHG050Measurements(&measurements, repoRoot, timeout, runner); err != nil { + return nil, err + } + return measurements, nil +} + +func appendRunnerCheckMeasurements( + measurements *[]perfcontracts.MeasurementRecord, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), +) error { + items, err := collectRunnerCheckMeasurements(repoRoot, timeout, runner) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func appendWorkflowMeasurements( + measurements *[]perfcontracts.MeasurementRecord, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), +) error { + items, err := collectMinimalWorkflowMeasurements(repoRoot, timeout, runner) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func appendCHG050Measurements( + measurements *[]perfcontracts.MeasurementRecord, + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), +) error { + items, err := measureCHG050CompileAndLoad(repoRoot, runner, timeout) + if err != nil { + return err + } + *measurements = append(*measurements, items...) + return nil +} + +func resolveRepoRoot(explicit string) (string, error) { + repoRoot := strings.TrimSpace(explicit) + if repoRoot != "" { + return validateRepoRoot(repoRoot) + } + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return validateRepoRoot(cwd) +} + +func validateRepoRoot(root string) (string, error) { + clean := filepath.Clean(root) + if _, err := os.Stat(filepath.Join(clean, "runner", "package.json")); err != nil { + return "", fmt.Errorf("repository root missing runner/package.json: %w", err) + } + if _, err := os.Stat(filepath.Join(clean, "protocol", "schemas")); err != nil { + return "", fmt.Errorf("repository root missing protocol/schemas: %w", err) + } + if _, err := projectsubstrate.DiscoverAndValidate(projectsubstrate.DiscoveryInput{RepositoryRoot: clean, Authority: projectsubstrate.RepoRootAuthorityExplicitConfig}); err != nil { + return "", fmt.Errorf("repository root validation failed: %w", err) + } + return clean, nil +} diff --git a/internal/runnerworkflowperf/harness_chg050.go b/internal/runnerworkflowperf/harness_chg050.go new file mode 100644 index 00000000..80e4d7b6 --- /dev/null +++ b/internal/runnerworkflowperf/harness_chg050.go @@ -0,0 +1,202 @@ +package runnerworkflowperf + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/artifacts" + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/perffixtures" + "github.com/runecode-ai/runecode/internal/policyengine" + "github.com/runecode-ai/runecode/internal/runplan" +) + +func measureCHG050CompileAndLoad(repoRoot string, runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), timeout time.Duration) ([]perfcontracts.MeasurementRecord, error) { + tmpRoot, err := os.MkdirTemp("", "runecode-runnerworkflowperf-chg050-") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(tmpRoot) }() + + fixture, err := perffixtures.BuildWorkflowFixture(filepath.Join(tmpRoot, "fixture"), perffixtures.FixtureWorkflowCHG050Compile) + if err != nil { + return nil, fmt.Errorf("build CHG-050 fixture: %w", err) + } + compileMS, validationCanonicalizationMS, persistLoadMS, err := buildPersistAndLoadCHG050(tmpRoot) + if err != nil { + return nil, err + } + startupMS, err := runner(repoRoot, timeout, "node", "--experimental-strip-types", "scripts/perf-runner-workflow.js", "--mode", "immutable-startup", "--runplan", fixture.RunPlan) + if err != nil { + return nil, fmt.Errorf("measure immutable runplan startup: %w", err) + } + return []perfcontracts.MeasurementRecord{ + {MetricID: "metric.workflow.chg050.compile.wall_ms", Value: compileMS, Unit: "ms"}, + {MetricID: "metric.workflow.chg050.validation_canonicalization.wall_ms", Value: validationCanonicalizationMS, Unit: "ms"}, + {MetricID: "metric.workflow.chg050.runplan_persist_load.wall_ms", Value: persistLoadMS, Unit: "ms"}, + {MetricID: "metric.workflow.chg050.runner_start_immutable_runplan.wall_ms", Value: startupMS, Unit: "ms"}, + }, nil +} + +func buildPersistAndLoadCHG050(tmpRoot string) (float64, float64, float64, error) { + compileInput, validationCanonicalizationMS, err := buildCHG050CompileInput() + if err != nil { + return 0, 0, 0, err + } + compileStart := time.Now() + plan, err := runplan.Compile(compileInput) + if err != nil { + return 0, 0, 0, fmt.Errorf("compile CHG-050 runplan: %w", err) + } + compileMS := float64(time.Since(compileStart).Milliseconds()) + persistLoadMS, err := persistAndLoadCHG050RunPlan(tmpRoot, plan) + if err != nil { + return 0, 0, 0, err + } + return compileMS, validationCanonicalizationMS, persistLoadMS, nil +} + +func buildCHG050CompileInput() (runplan.CompileInput, float64, error) { + validationCanonicalizationStart := time.Now() + processBytes, processHash, err := marshalCHG050ProcessDefinition() + if err != nil { + return runplan.CompileInput{}, 0, err + } + workflowBytes, err := marshalCHG050WorkflowDefinition(processHash) + if err != nil { + return runplan.CompileInput{}, 0, err + } + validationCanonicalizationMS := float64(time.Since(validationCanonicalizationStart).Milliseconds()) + return runplan.CompileInput{RunID: "run-chg050", PlanID: "plan-chg050-v1", CompiledAt: time.Date(2026, time.March, 20, 10, 0, 0, 0, time.UTC), WorkflowDefinitionBytes: workflowBytes, ProcessDefinitionBytes: processBytes, ProjectContextIdentityDigest: "sha256:" + strings.Repeat("3", 64), PolicyContextHash: "sha256:" + strings.Repeat("4", 64), ExecutorRegistry: policyengine.BuildExecutorRegistryProjection()}, validationCanonicalizationMS, nil +} + +func marshalCHG050ProcessDefinition() ([]byte, string, error) { + processDefinition := map[string]any{"schema_id": "runecode.protocol.v0.ProcessDefinition", "schema_version": "0.4.0", "process_id": "process_chg050", "executor_bindings": []map[string]any{{"binding_id": "binding_workspace_runner", "executor_id": "workspace-runner", "executor_class": "workspace_ordinary", "allowed_role_kinds": []string{"workspace-edit"}}}, "gate_definitions": []map[string]any{{"schema_id": "runecode.protocol.v0.GateDefinition", "schema_version": "0.2.0", "checkpoint_code": "step_validation_started", "order_index": 0, "stage_id": "validation", "step_id": "validate_step", "role_instance_id": "workspace_editor_1", "executor_binding_id": "binding_workspace_runner", "gate": map[string]any{"schema_id": "runecode.protocol.v0.GateContract", "schema_version": "0.1.0", "gate_id": "lint_gate", "gate_kind": "lint", "gate_version": "1.0.0", "normalized_inputs": []map[string]any{{"input_id": "source_tree", "input_digest": "sha256:" + strings.Repeat("2", 64)}}, "plan_binding": map[string]any{"checkpoint_code": "step_validation_started", "order_index": 0}, "retry_semantics": map[string]any{"retry_mode": "new_attempt_required", "max_attempts": 2}, "override_semantics": map[string]any{"override_mode": "policy_action_required", "action_kind": "action_gate_override", "approval_trigger_code": "gate_override"}}}}, "dependency_edges": []map[string]any{}} + processBytes, err := json.Marshal(processDefinition) + if err != nil { + return nil, "", err + } + processCanonical, err := policyengine.CanonicalizeJSONBytes(processBytes) + if err != nil { + return nil, "", err + } + return processBytes, policyengine.HashCanonicalJSONBytes(processCanonical), nil +} + +func marshalCHG050WorkflowDefinition(processHash string) ([]byte, error) { + workflowDefinition := map[string]any{"schema_id": "runecode.protocol.v0.WorkflowDefinition", "schema_version": "0.5.0", "workflow_id": "workflow_chg050", "workflow_version": "1.0.0", "selected_process_id": "process_chg050", "selected_process_definition_hash": processHash, "reviewed_process_artifacts": []map[string]any{{"process_id": "process_chg050", "process_definition_hash": processHash}}, "approval_profile": "moderate", "autonomy_posture": "balanced"} + workflowBytes, err := json.Marshal(workflowDefinition) + if err != nil { + return nil, err + } + _, err = policyengine.CanonicalizeJSONBytes(workflowBytes) + if err != nil { + return nil, err + } + return workflowBytes, nil +} + +func persistAndLoadCHG050RunPlan(tmpRoot string, plan runplan.RunPlan) (float64, error) { + planBytes, err := json.Marshal(plan) + if err != nil { + return 0, err + } + persistStart := time.Now() + store, err := artifacts.NewStore(filepath.Join(tmpRoot, "store")) + if err != nil { + return 0, err + } + ref, err := store.Put(artifacts.PutRequest{Payload: planBytes, ContentType: "application/json", DataClass: artifacts.DataClassSpecText, ProvenanceReceiptHash: "sha256:" + strings.Repeat("8", 64), CreatedByRole: "brokerapi", TrustedSource: true, RunID: "run-chg050", StepID: "compiled_run_plan/plan-chg050-v1"}) + if err != nil { + return 0, err + } + authority := artifacts.RunPlanAuthorityRecord{RunID: "run-chg050", PlanID: "plan-chg050-v1", RunPlanDigest: ref.Digest, WorkflowDefinitionHash: plan.WorkflowDefinitionHash, ProcessDefinitionHash: plan.ProcessDefinitionHash, PolicyContextHash: plan.PolicyContextHash, ProjectContextIdentityDigest: plan.ProjectContextIdentityDigest, CompiledAt: time.Date(2026, time.March, 20, 10, 0, 0, 0, time.UTC), Entries: authorityEntriesFromPlan(plan)} + compilation := artifacts.RunPlanCompilationRecord{RunID: "run-chg050", PlanID: "plan-chg050-v1", RunPlanDigest: ref.Digest, CompileCacheKey: "cache-key-chg050-v1", WorkflowDefinitionRef: "sha256:" + strings.Repeat("5", 64), ProcessDefinitionRef: "sha256:" + strings.Repeat("6", 64), WorkflowDefinitionHash: plan.WorkflowDefinitionHash, ProcessDefinitionHash: plan.ProcessDefinitionHash, PolicyContextHash: plan.PolicyContextHash, ProjectContextIdentityDigest: plan.ProjectContextIdentityDigest, CompiledAt: time.Date(2026, time.March, 20, 10, 0, 0, 0, time.UTC)} + if err := store.RecordRunPlanAuthority(authority, compilation); err != nil { + return 0, err + } + if err := verifyCHG050Persistence(store); err != nil { + return 0, err + } + return float64(time.Since(persistStart).Milliseconds()), nil +} + +func verifyCHG050Persistence(store *artifacts.Store) error { + if _, ok, err := store.ActiveRunPlanAuthority("run-chg050"); err != nil || !ok { + if err != nil { + return err + } + return fmt.Errorf("active runplan authority missing") + } + if _, ok := store.RunPlanCompilationRecordByCacheKey("cache-key-chg050-v1"); !ok { + return fmt.Errorf("runplan compilation cache lookup missing") + } + return nil +} + +func authorityEntriesFromPlan(plan runplan.RunPlan) []artifacts.RunPlanGateEntryRecord { + if len(plan.Entries) == 0 { + return nil + } + out := make([]artifacts.RunPlanGateEntryRecord, 0, len(plan.Entries)) + for _, entry := range plan.Entries { + out = append(out, artifacts.RunPlanGateEntryRecord{ + EntryID: strings.TrimSpace(entry.EntryID), + EntryKind: strings.TrimSpace(entry.EntryKind), + PlanCheckpointCode: strings.TrimSpace(entry.CheckpointCode), + PlanOrderIndex: entry.OrderIndex, + GateID: strings.TrimSpace(entry.Gate.GateID), + GateKind: strings.TrimSpace(entry.Gate.GateKind), + GateVersion: strings.TrimSpace(entry.Gate.GateVersion), + StageID: strings.TrimSpace(entry.StageID), + StepID: strings.TrimSpace(entry.StepID), + RoleInstanceID: strings.TrimSpace(entry.RoleInstanceID), + MaxAttempts: maxAttemptsFromRetrySemantics(entry.Gate.RetrySemantics), + ExpectedInputDigests: expectedInputDigests(entry.Gate.NormalizedInputs), + }) + } + return out +} + +func expectedInputDigests(inputs []map[string]any) []string { + if len(inputs) == 0 { + return nil + } + seen := map[string]struct{}{} + out := make([]string, 0, len(inputs)) + for _, input := range inputs { + raw, _ := input["input_digest"].(string) + digest := strings.TrimSpace(raw) + if digest == "" { + continue + } + if _, ok := seen[digest]; ok { + continue + } + seen[digest] = struct{}{} + out = append(out, digest) + } + if len(out) == 0 { + return nil + } + sort.Strings(out) + return out +} + +func maxAttemptsFromRetrySemantics(retry map[string]any) int { + if retry == nil { + return 1 + } + if value, ok := retry["max_attempts"].(int); ok && value > 0 { + return value + } + if value, ok := retry["max_attempts"].(float64); ok && int(value) > 0 { + return int(value) + } + return 1 +} diff --git a/internal/runnerworkflowperf/harness_runner.go b/internal/runnerworkflowperf/harness_runner.go new file mode 100644 index 00000000..c9b8136a --- /dev/null +++ b/internal/runnerworkflowperf/harness_runner.go @@ -0,0 +1,148 @@ +package runnerworkflowperf + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/perffixtures" +) + +func collectRunnerCheckMeasurements(repoRoot string, timeout time.Duration, runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error)) ([]perfcontracts.MeasurementRecord, error) { + boundaryMS, err := runner(repoRoot, timeout, "npm", "run", "boundary-check") + if err != nil { + return nil, fmt.Errorf("measure boundary-check: %w", err) + } + fixturesMS, err := runner(repoRoot, timeout, "node", "--test", "scripts/protocol-fixtures.test.js") + if err != nil { + return nil, fmt.Errorf("measure protocol-fixtures: %w", err) + } + return []perfcontracts.MeasurementRecord{{MetricID: "metric.runner.boundary_check.wall_ms", Value: boundaryMS, Unit: "ms"}, {MetricID: "metric.runner.protocol_fixtures.wall_ms", Value: fixturesMS, Unit: "ms"}}, nil +} + +func collectMinimalWorkflowMeasurements(repoRoot string, timeout time.Duration, runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error)) ([]perfcontracts.MeasurementRecord, error) { + minimal, cleanup, err := buildMinimalWorkflowFixture() + if err != nil { + return nil, err + } + defer cleanup() + specs := []runnerMeasurementSpec{ + {metricID: "metric.runner.cold_start.minimal_workflow.wall_ms", mode: "cold-start", fixture: minimal.FixtureID}, + {metricID: "metric.workflow.mvp_execution.supported_path.wall_ms", mode: "workflow-path", fixture: minimal.FixtureID}, + {metricID: "metric.workflow.chg049.first_party_beta_slice.wall_ms", mode: "first-party-beta", fixture: minimal.FixtureID}, + } + return collectWorkflowSpecs(repoRoot, timeout, runner, minimal.RunPlan, specs) +} + +func collectWorkflowSpecs( + repoRoot string, + timeout time.Duration, + runner func(repoRoot string, timeout time.Duration, args ...string) (float64, error), + runPlanPath string, + specs []runnerMeasurementSpec, +) ([]perfcontracts.MeasurementRecord, error) { + measurements := make([]perfcontracts.MeasurementRecord, 0, len(specs)) + for _, spec := range specs { + args := []string{"node", "--experimental-strip-types", "scripts/perf-runner-workflow.js", "--mode", spec.mode, "--runplan", runPlanPath} + if fixtureID := strings.TrimSpace(spec.fixture); fixtureID != "" { + args = append(args, "--fixture", fixtureID) + } + wallMS, err := runner(repoRoot, timeout, args...) + if err != nil { + return nil, fmt.Errorf("measure %s: %w", spec.mode, err) + } + measurements = append(measurements, perfcontracts.MeasurementRecord{MetricID: spec.metricID, Value: wallMS, Unit: "ms"}) + } + return measurements, nil +} + +func buildMinimalWorkflowFixture() (perffixtures.WorkflowFixtureResult, func(), error) { + root, err := os.MkdirTemp("", "runecode-runnerworkflowperf-minimal-") + if err != nil { + return perffixtures.WorkflowFixtureResult{}, nil, err + } + cleanup := func() { _ = os.RemoveAll(root) } + fixture, err := perffixtures.BuildWorkflowFixture(root, perffixtures.FixtureWorkflowFirstPartyMinimal) + if err != nil { + cleanup() + return perffixtures.WorkflowFixtureResult{}, nil, fmt.Errorf("build minimal workflow fixture: %w", err) + } + return fixture, cleanup, nil +} + +func measureRunnerCommand(repoRoot string, timeout time.Duration, args ...string) (float64, error) { + if len(args) == 0 { + return 0, fmt.Errorf("command arguments required") + } + if err := validateRunnerExecutable(args[0]); err != nil { + return 0, err + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + runnerDir := filepath.Join(repoRoot, "runner") + started := time.Now() + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + cmd.Dir = runnerDir + out, err := cmd.CombinedOutput() + if err != nil { + msg := strings.TrimSpace(string(out)) + if msg == "" { + msg = err.Error() + } + return 0, fmt.Errorf("%s failed: %s", strings.Join(args, " "), msg) + } + if expectsRunnerScriptMeasurement(args) { + return parseRunnerMeasurement(out) + } + return float64(time.Since(started).Milliseconds()), nil +} + +func validateRunnerExecutable(name string) error { + switch strings.TrimSpace(name) { + case "npm", "node": + return nil + default: + return fmt.Errorf("unsupported runner executable %q", name) + } +} + +func expectsRunnerScriptMeasurement(args []string) bool { + for _, arg := range args { + if arg == "scripts/perf-runner-workflow.js" { + return true + } + } + return false +} + +func parseRunnerMeasurement(out []byte) (float64, error) { + value := lastNumericMeasurementLine(string(out)) + if value == "" { + return 0, fmt.Errorf("runner workflow script returned empty measurement output") + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return 0, fmt.Errorf("parse runner workflow measurement %q: %w", value, err) + } + return parsed, nil +} + +func lastNumericMeasurementLine(raw string) string { + lines := strings.Split(raw, "\n") + for idx := len(lines) - 1; idx >= 0; idx-- { + line := strings.TrimSpace(lines[idx]) + if line == "" { + continue + } + if _, err := strconv.ParseFloat(line, 64); err == nil { + return line + } + } + return strings.TrimSpace(raw) +} diff --git a/internal/runnerworkflowperf/harness_test.go b/internal/runnerworkflowperf/harness_test.go new file mode 100644 index 00000000..e6da0637 --- /dev/null +++ b/internal/runnerworkflowperf/harness_test.go @@ -0,0 +1,188 @@ +package runnerworkflowperf + +import ( + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func TestRunProducesPhase4RunnerWorkflowMetrics(t *testing.T) { + repoRoot := runnerWorkflowRepoRoot(t) + out, err := Run(HarnessConfig{RepositoryRoot: repoRoot, CommandRunner: deterministicCommandRunner, CommandTimeout: 10 * time.Second}) + if err != nil { + t.Fatalf("Run returned error: %v", err) + } + if out.SchemaVersion != CheckSchemaVersion { + t.Fatalf("schema_version = %q, want %q", out.SchemaVersion, CheckSchemaVersion) + } + required := map[string]string{ + "metric.runner.boundary_check.wall_ms": "ms", + "metric.runner.protocol_fixtures.wall_ms": "ms", + "metric.runner.cold_start.minimal_workflow.wall_ms": "ms", + "metric.workflow.mvp_execution.supported_path.wall_ms": "ms", + "metric.workflow.chg049.first_party_beta_slice.wall_ms": "ms", + "metric.workflow.chg050.compile.wall_ms": "ms", + "metric.workflow.chg050.validation_canonicalization.wall_ms": "ms", + "metric.workflow.chg050.runplan_persist_load.wall_ms": "ms", + "metric.workflow.chg050.runner_start_immutable_runplan.wall_ms": "ms", + } + for metricID, unit := range required { + if !hasMetric(out.Measurements, metricID, unit) { + t.Fatalf("missing metric %s (%s)", metricID, unit) + } + } +} + +func TestRunPassesExpectedWorkflowFixtureForSupportedPathMetrics(t *testing.T) { + repoRoot := runnerWorkflowRepoRoot(t) + var calls [][]string + runner := func(_ string, _ time.Duration, args ...string) (float64, error) { + copied := append([]string(nil), args...) + calls = append(calls, copied) + return deterministicCommandRunner("", 0, args...) + } + if _, err := Run(HarnessConfig{RepositoryRoot: repoRoot, CommandRunner: runner, CommandTimeout: 10 * time.Second}); err != nil { + t.Fatalf("Run returned error: %v", err) + } + assertModeFixtureArg(t, calls, "workflow-path") + assertModeFixtureArg(t, calls, "first-party-beta") +} + +func runnerWorkflowRepoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) +} + +func assertModeFixtureArg(t *testing.T, calls [][]string, mode string) { + t.Helper() + for _, call := range calls { + if !containsArg(call, "--mode", mode) { + continue + } + fixture, ok := argValue(call, "--fixture") + if !ok { + t.Fatalf("mode %s missing --fixture argument", mode) + } + if fixture != "workflow.first-party-minimal.v1" { + t.Fatalf("mode %s fixture %q, want %q", mode, fixture, "workflow.first-party-minimal.v1") + } + return + } + t.Fatalf("no invocation found for mode %s", mode) +} + +func deterministicCommandRunner(_ string, _ time.Duration, args ...string) (float64, error) { + if len(args) == 0 { + return 0, nil + } + if args[0] == "npm" { + return 1100, nil + } + if args[0] != "node" { + return 100, nil + } + if modeLatency, ok := deterministicNodeModeLatency(args); ok { + return modeLatency, nil + } + if isNodeProtocolFixtureCommand(args) { + return 2700, nil + } + return 250, nil +} + +func deterministicNodeModeLatency(args []string) (float64, bool) { + for _, token := range args { + switch token { + case "cold-start": + return 220, true + case "workflow-path": + return 340, true + case "first-party-beta": + return 280, true + case "immutable-startup": + return 310, true + } + } + return 0, false +} + +func isNodeProtocolFixtureCommand(args []string) bool { + return len(args) >= 3 && args[1] == "--test" +} + +func containsArg(args []string, key, value string) bool { + for idx := 0; idx < len(args)-1; idx++ { + if strings.TrimSpace(args[idx]) == key && strings.TrimSpace(args[idx+1]) == value { + return true + } + } + return false +} + +func argValue(args []string, key string) (string, bool) { + for idx := 0; idx < len(args)-1; idx++ { + if strings.TrimSpace(args[idx]) != key { + continue + } + value := strings.TrimSpace(args[idx+1]) + if value == "" { + return "", false + } + return value, true + } + return "", false +} + +func hasMetric(measurements []perfcontracts.MeasurementRecord, metricID, unit string) bool { + for _, m := range measurements { + if m.MetricID == metricID && m.Unit == unit { + return true + } + } + return false +} + +func TestValidateRunnerExecutableRejectsUnexpectedBinary(t *testing.T) { + if err := validateRunnerExecutable("bash"); err == nil { + t.Fatal("validateRunnerExecutable error = nil, want rejection") + } +} + +func TestParseRunnerMeasurement(t *testing.T) { + value, err := parseRunnerMeasurement([]byte("340\n")) + if err != nil { + t.Fatalf("parseRunnerMeasurement returned error: %v", err) + } + if value != 340 { + t.Fatalf("value = %v, want 340", value) + } +} + +func TestParseRunnerMeasurementIgnoresWarningNoise(t *testing.T) { + out := []byte("(node:29695) warning text\nadditional warning context\n0\n") + value, err := parseRunnerMeasurement(out) + if err != nil { + t.Fatalf("parseRunnerMeasurement returned error: %v", err) + } + if value != 0 { + t.Fatalf("value = %v, want 0", value) + } +} + +func TestDeterministicCommandRunnerUsesScriptMeasurementOutput(t *testing.T) { + value, err := deterministicCommandRunner("", 0, "node", "--experimental-strip-types", "scripts/perf-runner-workflow.js", "--mode", "workflow-path") + if err != nil { + t.Fatalf("deterministicCommandRunner returned error: %v", err) + } + if value != 340 { + t.Fatalf("value = %v, want 340", value) + } +} diff --git a/internal/tuiperf/benchparse.go b/internal/tuiperf/benchparse.go new file mode 100644 index 00000000..15d44a32 --- /dev/null +++ b/internal/tuiperf/benchparse.go @@ -0,0 +1,114 @@ +package tuiperf + +import ( + "bufio" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +var benchSuffixPattern = regexp.MustCompile(`-\d+$`) + +type BenchmarkMetricMap struct { + Benchmark string + MetricID string + Unit string + Field string +} + +func ParseGoTestBenchOutput(r io.Reader, mappings []BenchmarkMetricMap) ([]perfcontracts.MeasurementRecord, error) { + mapByKey := benchmarkMappingIndex(mappings) + measurements := make([]perfcontracts.MeasurementRecord, 0, len(mappings)) + seen := map[string]struct{}{} + scanner := bufio.NewScanner(r) + for scanner.Scan() { + fields, ok := benchmarkFields(scanner.Text()) + if !ok { + continue + } + benchName := fields[0] + normalizedBench := benchSuffixPattern.ReplaceAllString(benchName, "") + appendBenchMeasurements(&measurements, seen, mapByKey, benchName, normalizedBench, parseBenchMetrics(fields)) + } + if err := scanner.Err(); err != nil { + return nil, err + } + for _, m := range mappings { + if _, ok := seen[m.MetricID]; !ok { + return nil, fmt.Errorf("missing benchmark measurement for %s", m.MetricID) + } + } + return measurements, nil +} + +func benchmarkMappingIndex(mappings []BenchmarkMetricMap) map[string]BenchmarkMetricMap { + mapByKey := map[string]BenchmarkMetricMap{} + for _, m := range mappings { + k := strings.TrimSpace(m.Benchmark) + ":" + strings.TrimSpace(m.Field) + mapByKey[k] = m + } + return mapByKey +} + +func benchmarkFields(line string) ([]string, bool) { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "Benchmark") { + return nil, false + } + fields := strings.Fields(trimmed) + if len(fields) < 3 { + return nil, false + } + return fields, true +} + +func appendBenchMeasurements(measurements *[]perfcontracts.MeasurementRecord, seen map[string]struct{}, mapByKey map[string]BenchmarkMetricMap, benchName, normalizedBench string, metrics map[string]float64) { + for fieldName, val := range metrics { + mapping, ok := resolveBenchMapping(mapByKey, benchName, normalizedBench, fieldName) + if !ok || metricSeen(seen, mapping.MetricID) { + continue + } + *measurements = append(*measurements, perfcontracts.MeasurementRecord{MetricID: mapping.MetricID, Value: val, Unit: mapping.Unit}) + } +} + +func resolveBenchMapping(mapByKey map[string]BenchmarkMetricMap, benchName, normalizedBench, fieldName string) (BenchmarkMetricMap, bool) { + mapping, ok := mapByKey[benchName+":"+fieldName] + if ok { + return mapping, true + } + mapping, ok = mapByKey[normalizedBench+":"+fieldName] + return mapping, ok +} + +func metricSeen(seen map[string]struct{}, metricID string) bool { + if _, dup := seen[metricID]; dup { + return true + } + seen[metricID] = struct{}{} + return false +} + +func parseBenchMetrics(fields []string) map[string]float64 { + out := map[string]float64{} + for i := 1; i+1 < len(fields); i++ { + value, err := strconv.ParseFloat(fields[i], 64) + if err != nil { + continue + } + unit := fields[i+1] + switch unit { + case "ns/op": + out["ns/op"] = value + case "B/op": + out["B/op"] = value + case "allocs/op": + out["allocs/op"] = value + } + } + return out +} diff --git a/internal/tuiperf/benchparse_test.go b/internal/tuiperf/benchparse_test.go new file mode 100644 index 00000000..6b881d19 --- /dev/null +++ b/internal/tuiperf/benchparse_test.go @@ -0,0 +1,20 @@ +package tuiperf + +import ( + "strings" + "testing" +) + +func TestParseGoTestBenchOutput(t *testing.T) { + input := strings.NewReader("BenchmarkShellViewEmpty-8 12345 11000 ns/op 1500 B/op 20 allocs/op\nBenchmarkShellWatchApply-8 23456 12000 ns/op 1700 B/op 21 allocs/op\n") + measurements, err := ParseGoTestBenchOutput(input, []BenchmarkMetricMap{ + {Benchmark: "BenchmarkShellViewEmpty", Field: "ns/op", MetricID: "metric.tui.render.shell_view_empty.ns_op", Unit: "ns/op"}, + {Benchmark: "BenchmarkShellWatchApply", Field: "ns/op", MetricID: "metric.tui.update.shell_watch_apply.ns_op", Unit: "ns/op"}, + }) + if err != nil { + t.Fatalf("ParseGoTestBenchOutput returned error: %v", err) + } + if len(measurements) != 2 { + t.Fatalf("measurements len = %d, want 2", len(measurements)) + } +} diff --git a/internal/tuiperf/cpu_sampler_linux.go b/internal/tuiperf/cpu_sampler_linux.go new file mode 100644 index 00000000..c3392010 --- /dev/null +++ b/internal/tuiperf/cpu_sampler_linux.go @@ -0,0 +1,234 @@ +//go:build linux + +package tuiperf + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type CPUSampleConfig struct { + Warmup time.Duration + Window time.Duration + Windows int + TicksPerSecond float64 + CPUCount float64 + ProcRoot string + ExpectedComm string + ChildLookupWait time.Duration + PollInterval time.Duration +} + +type CPUWindowSample struct { + Index int `json:"index"` + CPUPercent float64 `json:"cpu_percent"` +} + +type CPUSampleResult struct { + TargetPID int `json:"target_pid"` + TargetComm string `json:"target_comm"` + WarmupMillis int64 `json:"warmup_millis"` + WindowMillis int64 `json:"observation_window_millis"` + ObservationWindows int `json:"observation_windows"` + AverageCPUPercent float64 `json:"average_cpu_percent"` + MaxCPUPercent float64 `json:"max_cpu_percent"` + Windows []CPUWindowSample `json:"windows"` +} + +func DefaultCPUSampleConfig() CPUSampleConfig { + return CPUSampleConfig{ + Warmup: 3 * time.Second, + Window: 20 * time.Second, + Windows: 3, + ProcRoot: "/proc", + ExpectedComm: "runecode-tui", + ChildLookupWait: 3 * time.Second, + PollInterval: 20 * time.Millisecond, + } +} + +func (c *CPUSampleConfig) normalize() error { + c.applyDefaults() + if err := c.validateDurations(); err != nil { + return err + } + if err := c.resolveTicksPerSecond(); err != nil { + return err + } + return c.resolveCPUCount() +} + +func (c *CPUSampleConfig) applyDefaults() { + if c.ProcRoot == "" { + c.ProcRoot = "/proc" + } + if c.ExpectedComm == "" { + c.ExpectedComm = "runecode-tui" + } + if c.ChildLookupWait <= 0 { + c.ChildLookupWait = 3 * time.Second + } + if c.PollInterval <= 0 { + c.PollInterval = 20 * time.Millisecond + } +} + +func (c *CPUSampleConfig) validateDurations() error { + if c.Warmup < 0 || c.Window <= 0 || c.Windows <= 0 { + return fmt.Errorf("invalid cpu sampling durations") + } + return nil +} + +func (c *CPUSampleConfig) resolveTicksPerSecond() error { + if c.TicksPerSecond > 0 { + return nil + } + tps, err := systemTicksPerSecond() + if err != nil { + return err + } + c.TicksPerSecond = tps + return nil +} + +func (c *CPUSampleConfig) resolveCPUCount() error { + if c.CPUCount > 0 { + return nil + } + cpus, err := cpuCountFromProc(c.ProcRoot) + if err != nil { + return err + } + c.CPUCount = cpus + return nil +} + +func systemTicksPerSecond() (float64, error) { + v := os.Getenv("RUNECODE_TUIPERF_CLK_TCK") + if strings.TrimSpace(v) != "" { + f, err := strconv.ParseFloat(strings.TrimSpace(v), 64) + if err != nil { + return 0, fmt.Errorf("parse RUNECODE_TUIPERF_CLK_TCK: %w", err) + } + if f <= 0 { + return 0, fmt.Errorf("RUNECODE_TUIPERF_CLK_TCK must be > 0") + } + return f, nil + } + return 100.0, nil +} + +func cpuCountFromProc(procRoot string) (float64, error) { + raw, err := os.ReadFile(procRoot + "/stat") + if err != nil { + return 0, err + } + count := 0 + for _, line := range strings.Split(string(raw), "\n") { + if len(line) < 4 { + continue + } + if strings.HasPrefix(line, "cpu") && len(line) > 3 && line[3] >= '0' && line[3] <= '9' { + count++ + } + } + if count <= 0 { + return 0, fmt.Errorf("failed to detect cpu count") + } + return float64(count), nil +} + +func WaitForChildByComm(procRoot string, wrapperPID int, comm string, timeout time.Duration, pollInterval time.Duration) (int, error) { + deadline := time.Now().Add(timeout) + for { + pid, err := FindDescendantByComm(procRoot, wrapperPID, comm) + if err == nil { + return pid, nil + } + if time.Now().After(deadline) { + return 0, err + } + time.Sleep(pollInterval) + } +} + +func SampleProcessCPU(targetPID int, cfg CPUSampleConfig) (CPUSampleResult, error) { + if err := cfg.normalize(); err != nil { + return CPUSampleResult{}, err + } + cpuWindow, err := collectCPUWindows(targetPID, cfg) + if err != nil { + return CPUSampleResult{}, err + } + return cpuWindow.toResult(targetPID, cfg), nil +} + +type cpuWindowCollection struct { + targetComm string + windows []CPUWindowSample + sum float64 + max float64 +} + +func collectCPUWindows(targetPID int, cfg CPUSampleConfig) (cpuWindowCollection, error) { + if cfg.Warmup > 0 { + time.Sleep(cfg.Warmup) + } + out := cpuWindowCollection{windows: make([]CPUWindowSample, 0, cfg.Windows)} + for i := 0; i < cfg.Windows; i++ { + sample, comm, err := sampleCPUWindow(targetPID, cfg, i+1) + if err != nil { + return cpuWindowCollection{}, err + } + if out.targetComm == "" { + out.targetComm = comm + } + if sample.CPUPercent > out.max || i == 0 { + out.max = sample.CPUPercent + } + out.sum += sample.CPUPercent + out.windows = append(out.windows, sample) + } + return out, nil +} + +func sampleCPUWindow(targetPID int, cfg CPUSampleConfig, index int) (CPUWindowSample, string, error) { + before, err := ReadProcStat(cfg.ProcRoot, targetPID) + if err != nil { + return CPUWindowSample{}, "", err + } + time.Sleep(cfg.Window) + after, err := ReadProcStat(cfg.ProcRoot, targetPID) + if err != nil { + return CPUWindowSample{}, "", err + } + cpu := cpuPercentForWindow(before.TotalTicks(), after.TotalTicks(), cfg.Window, cfg.TicksPerSecond, cfg.CPUCount) + return CPUWindowSample{Index: index, CPUPercent: cpu}, before.Comm, nil +} + +func cpuPercentForWindow(beforeTicks, afterTicks uint64, window time.Duration, ticksPerSecond, cpuCount float64) float64 { + deltaTicks := float64(afterTicks - beforeTicks) + deltaSeconds := window.Seconds() + cpu := (deltaTicks / ticksPerSecond / deltaSeconds / cpuCount) * 100.0 + if cpu < 0 { + return 0 + } + return cpu +} + +func (c cpuWindowCollection) toResult(targetPID int, cfg CPUSampleConfig) CPUSampleResult { + return CPUSampleResult{ + TargetPID: targetPID, + TargetComm: c.targetComm, + WarmupMillis: cfg.Warmup.Milliseconds(), + WindowMillis: cfg.Window.Milliseconds(), + ObservationWindows: cfg.Windows, + AverageCPUPercent: c.sum / float64(cfg.Windows), + MaxCPUPercent: c.max, + Windows: c.windows, + } +} diff --git a/internal/tuiperf/cpu_sampler_linux_test.go b/internal/tuiperf/cpu_sampler_linux_test.go new file mode 100644 index 00000000..f5935de9 --- /dev/null +++ b/internal/tuiperf/cpu_sampler_linux_test.go @@ -0,0 +1,57 @@ +//go:build linux + +package tuiperf + +import ( + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +func TestSampleProcessCPUDeterministicProcFixture(t *testing.T) { + root := t.TempDir() + pid := 4321 + procDir := filepath.Join(root, strconv.Itoa(pid)) + writeDeterministicCPUFixture(t, root, procDir) + resultCh := make(chan CPUSampleResult, 1) + errCh := make(chan error, 1) + go func() { + result, err := SampleProcessCPU(pid, CPUSampleConfig{ProcRoot: root, Warmup: 0, Window: 5 * time.Millisecond, Windows: 1, TicksPerSecond: 100, CPUCount: 1}) + if err != nil { + errCh <- err + return + } + resultCh <- result + }() + if err := os.WriteFile(filepath.Join(procDir, "stat"), []byte("4321 (runecode-tui) S 1 2 3 4 5 6 7 8 9 10 11 200 50 0 0 20 0 1 0 999 0 0 0\n"), 0o644); err != nil { + t.Fatalf("WriteFile stat second: %v", err) + } + select { + case err := <-errCh: + t.Fatalf("SampleProcessCPU returned error: %v", err) + case result := <-resultCh: + if result.WindowMillis != 5 { + t.Fatalf("window millis = %d, want 5", result.WindowMillis) + } + if result.AverageCPUPercent < 0 { + t.Fatalf("average cpu = %.2f, want non-negative", result.AverageCPUPercent) + } + case <-time.After(2 * time.Second): + t.Fatal("SampleProcessCPU timed out waiting for deterministic fixture result") + } +} + +func writeDeterministicCPUFixture(t *testing.T, root, procDir string) { + t.Helper() + if err := os.MkdirAll(procDir, 0o755); err != nil { + t.Fatalf("MkdirAll procDir: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "stat"), []byte("cpu 1 2 3\ncpu0 1 1 1\n"), 0o644); err != nil { + t.Fatalf("WriteFile stat: %v", err) + } + if err := os.WriteFile(filepath.Join(procDir, "stat"), []byte("4321 (runecode-tui) S 1 2 3 4 5 6 7 8 9 10 11 100 50 0 0 20 0 1 0 999 0 0 0\n"), 0o644); err != nil { + t.Fatalf("WriteFile stat first: %v", err) + } +} diff --git a/internal/tuiperf/latency.go b/internal/tuiperf/latency.go new file mode 100644 index 00000000..bb342785 --- /dev/null +++ b/internal/tuiperf/latency.go @@ -0,0 +1,183 @@ +package tuiperf + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "strings" + "time" +) + +type MarkerEvent struct { + Marker string + At time.Time +} + +func WatchMarkers(ctx context.Context, r io.Reader, markers []string, sink chan<- MarkerEvent) { + defer close(sink) + want := markerSet(markers) + if len(want) == 0 { + return + } + ctx = normalizeMarkerContext(ctx) + done := watchMarkerCancellation(ctx, r) + defer close(done) + maxMarkerLen := longestMarkerLength(want) + buffer := make([]byte, 0, maxMarkerLen*2) + chunk := make([]byte, 4096) + for { + n, err := r.Read(chunk) + updated, stop := processMarkerChunk(ctx, sink, buffer, chunk[:n], want, maxMarkerLen) + buffer = updated + if stop { + return + } + if err != nil { + return + } + if markerContextDone(ctx) { + return + } + } +} + +func markerSet(markers []string) map[string]struct{} { + want := map[string]struct{}{} + for _, marker := range markers { + trimmed := strings.TrimSpace(marker) + if trimmed == "" { + continue + } + want[trimmed] = struct{}{} + } + return want +} + +func longestMarkerLength(want map[string]struct{}) int { + longest := 0 + for marker := range want { + if len(marker) > longest { + longest = len(marker) + } + } + if longest == 0 { + return 1 + } + return longest +} + +func normalizeMarkerContext(ctx context.Context) context.Context { + if ctx == nil { + return context.Background() + } + return ctx +} + +func watchMarkerCancellation(ctx context.Context, r io.Reader) chan struct{} { + done := make(chan struct{}) + closer, ok := r.(io.ReadCloser) + if !ok { + return done + } + go func() { + select { + case <-ctx.Done(): + _ = closer.Close() + case <-done: + } + }() + return done +} + +func emitMarkerMatches(ctx context.Context, sink chan<- MarkerEvent, buffer []byte, want map[string]struct{}) ([]byte, bool) { + for { + marker, end, found := nextMarkerMatch(buffer, want) + if !found { + return buffer, false + } + select { + case sink <- MarkerEvent{Marker: marker, At: time.Now()}: + buffer = append([]byte(nil), buffer[end:]...) + case <-ctx.Done(): + return buffer, true + } + } +} + +func nextMarkerMatch(buffer []byte, want map[string]struct{}) (string, int, bool) { + bestIndex := -1 + bestEnd := -1 + bestMarker := "" + for marker := range want { + idx := bytes.Index(buffer, []byte(marker)) + if idx < 0 { + continue + } + if bestIndex == -1 || idx < bestIndex { + bestIndex = idx + bestEnd = idx + len(marker) + bestMarker = marker + } + } + if bestIndex == -1 { + return "", 0, false + } + return bestMarker, bestEnd, true +} + +func truncateMarkerBuffer(buffer []byte, maxMarkerLen int) []byte { + keep := maxMarkerLen - 1 + if keep < 1 { + keep = 1 + } + if len(buffer) <= keep { + return buffer + } + return append([]byte(nil), buffer[len(buffer)-keep:]...) +} + +func processMarkerChunk( + ctx context.Context, + sink chan<- MarkerEvent, + buffer []byte, + chunk []byte, + want map[string]struct{}, + maxMarkerLen int, +) ([]byte, bool) { + if len(chunk) == 0 { + return buffer, false + } + buffer = append(buffer, chunk...) + updated, stop := emitMarkerMatches(ctx, sink, buffer, want) + if stop { + return updated, true + } + return truncateMarkerBuffer(updated, maxMarkerLen), false +} + +func markerContextDone(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} + +func P95Millis(samples []float64) (float64, error) { + if len(samples) == 0 { + return 0, fmt.Errorf("samples required") + } + vals := append([]float64(nil), samples...) + sort.Float64s(vals) + idx := int(float64(len(vals)-1) * 0.95) + if idx < 0 { + idx = 0 + } + if idx >= len(vals) { + idx = len(vals) - 1 + } + return vals[idx], nil +} diff --git a/internal/tuiperf/latency_test.go b/internal/tuiperf/latency_test.go new file mode 100644 index 00000000..f13c9667 --- /dev/null +++ b/internal/tuiperf/latency_test.go @@ -0,0 +1,134 @@ +package tuiperf + +import ( + "context" + "io" + "runtime" + "testing" + "time" +) + +func TestP95Millis(t *testing.T) { + v, err := P95Millis([]float64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100}) + if err != nil { + t.Fatalf("P95Millis returned error: %v", err) + } + if v != 90 { + t.Fatalf("p95 = %.2f, want 90", v) + } +} + +func TestP95MillisRejectsEmpty(t *testing.T) { + if _, err := P95Millis(nil); err == nil { + t.Fatal("P95Millis error = nil, want error") + } +} + +func TestWatchMarkersClosesSinkWhenReaderEnds(t *testing.T) { + t.Parallel() + + r, w := io.Pipe() + events := make(chan MarkerEvent, 1) + go WatchMarkers(context.Background(), r, []string{"ready"}, events) + if _, err := io.WriteString(w, "ready\n"); err != nil { + t.Fatalf("WriteString error = %v", err) + } + _ = w.Close() + if _, ok := <-events; !ok { + t.Fatal("events closed before receiving marker") + } + if _, ok := <-events; ok { + t.Fatal("events channel still open, want closed") + } +} + +func TestWatchMarkersReturnsOnCancellation(t *testing.T) { + t.Parallel() + + r, _ := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + events := make(chan MarkerEvent, 1) + go WatchMarkers(ctx, r, []string{"ready"}, events) + cancel() + select { + case _, ok := <-events: + if ok { + t.Fatal("events channel open after cancellation") + } + case <-time.After(500 * time.Millisecond): + t.Fatal("WatchMarkers did not stop after cancellation") + } +} + +func TestWatchMarkersCancellationHelperExitsAfterEOF(t *testing.T) { + t.Parallel() + baseline := runtime.NumGoroutine() + r, w := io.Pipe() + events := make(chan MarkerEvent, 1) + ctx, cancel := context.WithCancel(context.Background()) + go WatchMarkers(ctx, r, []string{"ready"}, events) + if _, err := io.WriteString(w, "ready\n"); err != nil { + t.Fatalf("WriteString error = %v", err) + } + _ = w.Close() + for range events { + } + for i := 0; i < 20; i++ { + if runtime.NumGoroutine() <= baseline+1 { + cancel() + return + } + time.Sleep(10 * time.Millisecond) + } + cancel() + t.Fatalf("goroutine count stayed elevated: baseline=%d current=%d", baseline, runtime.NumGoroutine()) +} + +func TestWatchMarkersDetectsMarkerWithoutTrailingNewline(t *testing.T) { + t.Parallel() + + r, w := io.Pipe() + events := make(chan MarkerEvent, 1) + go WatchMarkers(context.Background(), r, []string{"Runecode TUI α shell"}, events) + if _, err := io.WriteString(w, "prefix Runecode TUI α shell suffix"); err != nil { + t.Fatalf("WriteString error = %v", err) + } + _ = w.Close() + select { + case ev, ok := <-events: + if !ok { + t.Fatal("events closed before receiving marker") + } + if ev.Marker != "Runecode TUI α shell" { + t.Fatalf("marker = %q, want %q", ev.Marker, "Runecode TUI α shell") + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for marker event") + } +} + +func TestWatchMarkersDetectsMarkerSplitAcrossWrites(t *testing.T) { + t.Parallel() + + r, w := io.Pipe() + events := make(chan MarkerEvent, 1) + go WatchMarkers(context.Background(), r, []string{"Runecode TUI α shell"}, events) + if _, err := io.WriteString(w, "Runecode TUI "); err != nil { + t.Fatalf("WriteString first chunk error = %v", err) + } + if _, err := io.WriteString(w, "α shell"); err != nil { + t.Fatalf("WriteString second chunk error = %v", err) + } + _ = w.Close() + select { + case ev, ok := <-events: + if !ok { + t.Fatal("events closed before receiving split marker") + } + if ev.Marker != "Runecode TUI α shell" { + t.Fatalf("marker = %q, want %q", ev.Marker, "Runecode TUI α shell") + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for split marker event") + } +} diff --git a/internal/tuiperf/procstat.go b/internal/tuiperf/procstat.go new file mode 100644 index 00000000..69bb6423 --- /dev/null +++ b/internal/tuiperf/procstat.go @@ -0,0 +1,138 @@ +package tuiperf + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +type ProcStat struct { + PID int + Comm string + UserTicks uint64 + SystemTicks uint64 + StartTicks uint64 +} + +func (p ProcStat) TotalTicks() uint64 { + return p.UserTicks + p.SystemTicks +} + +func ReadProcStat(procRoot string, pid int) (ProcStat, error) { + raw, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(pid), "stat")) + if err != nil { + return ProcStat{}, err + } + return ParseProcStatLine(strings.TrimSpace(string(raw))) +} + +func ParseProcStatLine(line string) (ProcStat, error) { + open := strings.Index(line, "(") + close := strings.LastIndex(line, ")") + if open <= 0 || close <= open { + return ProcStat{}, fmt.Errorf("invalid /proc stat format") + } + pid, err := strconv.Atoi(strings.TrimSpace(line[:open])) + if err != nil { + return ProcStat{}, fmt.Errorf("parse pid: %w", err) + } + comm := line[open+1 : close] + fields := strings.Fields(strings.TrimSpace(line[close+1:])) + if len(fields) < 20 { + return ProcStat{}, fmt.Errorf("invalid /proc stat field count") + } + userTicks, err := strconv.ParseUint(fields[11], 10, 64) + if err != nil { + return ProcStat{}, fmt.Errorf("parse utime: %w", err) + } + systemTicks, err := strconv.ParseUint(fields[12], 10, 64) + if err != nil { + return ProcStat{}, fmt.Errorf("parse stime: %w", err) + } + startTicks, err := strconv.ParseUint(fields[19], 10, 64) + if err != nil { + return ProcStat{}, fmt.Errorf("parse starttime: %w", err) + } + return ProcStat{PID: pid, Comm: comm, UserTicks: userTicks, SystemTicks: systemTicks, StartTicks: startTicks}, nil +} + +func FindDescendantByComm(procRoot string, rootPID int, comm string) (int, error) { + want := strings.TrimSpace(comm) + if want == "" { + return 0, fmt.Errorf("comm is required") + } + search := procTreeSearch{procRoot: procRoot, wantComm: want, queue: []int{rootPID}, seen: map[int]struct{}{rootPID: {}}} + for search.hasQueue() { + if pid, found := search.scanNext(); found { + return pid, nil + } + } + return 0, fmt.Errorf("descendant with comm %q not found", want) +} + +type procTreeSearch struct { + procRoot string + wantComm string + queue []int + seen map[int]struct{} +} + +func (s *procTreeSearch) hasQueue() bool { + return len(s.queue) > 0 +} + +func (s *procTreeSearch) popQueue() int { + pid := s.queue[0] + s.queue = s.queue[1:] + return pid +} + +func (s *procTreeSearch) scanNext() (int, bool) { + pid := s.popQueue() + children, err := readChildren(s.procRoot, pid) + if err != nil { + return 0, false + } + for _, child := range children { + if s.markSeen(child) { + continue + } + if s.childMatches(child) { + return child, true + } + s.queue = append(s.queue, child) + } + return 0, false +} + +func (s *procTreeSearch) markSeen(child int) bool { + if _, ok := s.seen[child]; ok { + return true + } + s.seen[child] = struct{}{} + return false +} + +func (s *procTreeSearch) childMatches(child int) bool { + childComm, err := os.ReadFile(filepath.Join(s.procRoot, strconv.Itoa(child), "comm")) + return err == nil && strings.TrimSpace(string(childComm)) == s.wantComm +} + +func readChildren(procRoot string, pid int) ([]int, error) { + raw, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(pid), "task", strconv.Itoa(pid), "children")) + if err != nil { + return nil, err + } + parts := strings.Fields(string(raw)) + out := make([]int, 0, len(parts)) + for _, part := range parts { + id, err := strconv.Atoi(part) + if err != nil { + continue + } + out = append(out, id) + } + return out, nil +} diff --git a/internal/tuiperf/procstat_test.go b/internal/tuiperf/procstat_test.go new file mode 100644 index 00000000..115da965 --- /dev/null +++ b/internal/tuiperf/procstat_test.go @@ -0,0 +1,29 @@ +package tuiperf + +import "testing" + +func TestParseProcStatLine(t *testing.T) { + line := "1234 (runecode-tui) S 1 2 3 4 5 6 7 8 9 10 120 30 0 0 20 0 1 0 999 0 0 0" + stat, err := ParseProcStatLine(line) + if err != nil { + t.Fatalf("ParseProcStatLine returned error: %v", err) + } + if stat.PID != 1234 { + t.Fatalf("PID = %d, want 1234", stat.PID) + } + if stat.Comm != "runecode-tui" { + t.Fatalf("Comm = %q, want runecode-tui", stat.Comm) + } + if stat.UserTicks != 120 || stat.SystemTicks != 30 { + t.Fatalf("ticks = (%d,%d), want (120,30)", stat.UserTicks, stat.SystemTicks) + } + if stat.StartTicks != 999 { + t.Fatalf("StartTicks = %d, want 999", stat.StartTicks) + } +} + +func TestParseProcStatLineRejectsInvalid(t *testing.T) { + if _, err := ParseProcStatLine("bad"); err == nil { + t.Fatal("ParseProcStatLine error = nil, want error") + } +} diff --git a/justfile b/justfile index b8e0e6e0..f74e32f5 100644 --- a/justfile +++ b/justfile @@ -19,14 +19,19 @@ test: cd runner && npm test model-check: - go run ./tools/tlccheck + go run ./tools/tlccheck --mode all -ci: +model-check-core: + go run ./tools/tlccheck --mode core + +model-check-replay: + go run ./tools/tlccheck --mode replay + +ci-fast: go run ./tools/gofmtcheck go run {{golangci_lint}} run go vet ./... go run ./tools/checksourcequality - just model-check go test ./... go build ./cmd/... cd runner && npm ci @@ -34,6 +39,18 @@ ci: cd runner && npm test cd runner && npm run boundary-check +ci: + # Canonical local check entrypoint. + # Required shared-Linux performance contracts run in CI via ci-required-shared-linux. + just ci-fast + just model-check + +ci-required-shared-linux: + just ci-fast + tmpdir="$(mktemp -d)" && trap 'rm -rf "$tmpdir"' EXIT && \ + go run ./tools/perfgatesharedlinux --output "$tmpdir/perf-check.json" && \ + go run ./tools/perfcontracts --check-output "$tmpdir/perf-check.json" --lane required_shared_linux + ci-portability: go run ./tools/gofmtcheck go run {{golangci_lint}} run diff --git a/nix/packages/release-artifacts.nix b/nix/packages/release-artifacts.nix index f4b5a990..f50947a2 100644 --- a/nix/packages/release-artifacts.nix +++ b/nix/packages/release-artifacts.nix @@ -75,7 +75,7 @@ pkgs.buildGoModule { src = releaseSource; go = goToolchain; # Refresh explicitly with `just refresh-release-vendor-hash`. - vendorHash = "sha256-X7jALliWzq3PLI1eeluKi8rHxqhjGyiU+WmU5GsqiFs="; + vendorHash = "sha256-I+WLce2YQRAVkMovlaWcbocashO88gyF9P4/y4dHJho="; # The workflow runs `just ci` before building this packaging-focused derivation. doCheck = false; strictDeps = true; diff --git a/runecontext/changes/CHG-2026-002-33c5-git-gateway-commit-push-pr/status.yaml b/runecontext/changes/CHG-2026-002-33c5-git-gateway-commit-push-pr/status.yaml index f78fcebd..850e1169 100644 --- a/runecontext/changes/CHG-2026-002-33c5-git-gateway-commit-push-pr/status.yaml +++ b/runecontext/changes/CHG-2026-002-33c5-git-gateway-commit-push-pr/status.yaml @@ -16,6 +16,7 @@ related_decisions: related_changes: - CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0 - CHG-2026-050-e3f8-workflow-definition-contract-binding-v0 + - CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0 - CHG-2026-003-b567-audit-log-v0-verify - CHG-2026-004-acdb-artifact-store-data-classes-v0 - CHG-2026-007-2315-policy-engine-v0 diff --git a/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/design.md b/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/design.md index 16546747..2e166501 100644 --- a/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/design.md +++ b/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/design.md @@ -187,6 +187,7 @@ The first formal model should at least prove: - Add checked-in TLA+ spec files and deterministic TLC configs. - Add a dedicated `just` recipe for model checking and include it in `just ci`. - Add the required tooling to the dev shell and CI environment explicitly; the model-checking path must be deterministic and must leave the repo clean. +- Keep PR CI security-preserving but bounded: run the core model for security-kernel-relevant code or protocol PR diffs, run the full model for formal-spec/tooling/workflow PR diffs, and run the full model on merge queue and `main` so the final merge candidate remains covered without replaying the slow model on every push. - Keep bounds small but meaningful: - multiple runs - multiple approvals diff --git a/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/tasks.md b/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/tasks.md index 194cefc1..20b7d4bf 100644 --- a/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/tasks.md +++ b/runecontext/changes/CHG-2026-015-cae6-formal-spec-v0-tla-ci-model-checking/tasks.md @@ -80,6 +80,7 @@ Parallelization: can proceed once the semantic freeze above is settled; keep the - [x] Add a dedicated `just` recipe for model checking. - [x] Add the required TLA+ and TLC tooling to the dev shell and CI environment explicitly. - [x] Run model checking in `just ci` and fail closed on invariant violations. +- [x] Split CI scheduling so security-kernel-relevant PR diffs run the faster core model, while merge queue and `main` run the full model. - [x] Keep the repo clean and deterministic after local or CI model checking. Parallelization: can be implemented in parallel with spec authoring once the toolchain and CI ownership are agreed. diff --git a/runecontext/changes/CHG-2026-025-5679-external-audit-anchoring-v0/status.yaml b/runecontext/changes/CHG-2026-025-5679-external-audit-anchoring-v0/status.yaml index 431f759c..231f9b8f 100644 --- a/runecontext/changes/CHG-2026-025-5679-external-audit-anchoring-v0/status.yaml +++ b/runecontext/changes/CHG-2026-025-5679-external-audit-anchoring-v0/status.yaml @@ -18,6 +18,7 @@ related_changes: - CHG-2026-006-84f0-audit-anchoring-v0 - CHG-2026-003-b567-audit-log-v0-verify - CHG-2026-007-2315-policy-engine-v0 + - CHG-2026-053-9d2b-performance-baselines-verification-gates-v0 depends_on: [] informed_by: [] supersedes: [] diff --git a/runecontext/changes/CHG-2026-028-647e-windows-microvm-runtime-support/status.yaml b/runecontext/changes/CHG-2026-028-647e-windows-microvm-runtime-support/status.yaml index 84d44eb0..18059539 100644 --- a/runecontext/changes/CHG-2026-028-647e-windows-microvm-runtime-support/status.yaml +++ b/runecontext/changes/CHG-2026-028-647e-windows-microvm-runtime-support/status.yaml @@ -16,6 +16,7 @@ related_decisions: related_changes: - CHG-2026-009-1672-launcher-microvm-backend-v0 - CHG-2026-021-8d6d-local-ipc-protobuf-transport-v0 + - CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0 depends_on: [] informed_by: [] supersedes: [] diff --git a/runecontext/changes/CHG-2026-029-5e5e-macos-virtualization-polish/status.yaml b/runecontext/changes/CHG-2026-029-5e5e-macos-virtualization-polish/status.yaml index d815588b..89f651fd 100644 --- a/runecontext/changes/CHG-2026-029-5e5e-macos-virtualization-polish/status.yaml +++ b/runecontext/changes/CHG-2026-029-5e5e-macos-virtualization-polish/status.yaml @@ -16,6 +16,7 @@ related_specs: [] related_changes: - CHG-2026-009-1672-launcher-microvm-backend-v0 - CHG-2026-010-54b7-container-backend-v0-explicit-opt-in + - CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0 depends_on: [] informed_by: [] supersedes: [] diff --git a/runecontext/changes/CHG-2026-045-7f4c-direct-credential-model-providers-v0/status.yaml b/runecontext/changes/CHG-2026-045-7f4c-direct-credential-model-providers-v0/status.yaml index 34caf68d..3fb87389 100644 --- a/runecontext/changes/CHG-2026-045-7f4c-direct-credential-model-providers-v0/status.yaml +++ b/runecontext/changes/CHG-2026-045-7f4c-direct-credential-model-providers-v0/status.yaml @@ -22,6 +22,7 @@ related_changes: - CHG-2026-020-4425-openai-chatgpt-subscription-provider-oauth-codex-bridge - CHG-2026-022-8051-github-copilot-subscription-provider-official-runtime-bridge - CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-031-7a3c-secretsd-core-v0 - CHG-2026-032-4d1f-model-gateway-v0 diff --git a/runecontext/changes/CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0/status.yaml b/runecontext/changes/CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0/status.yaml index 38486571..30145ce1 100644 --- a/runecontext/changes/CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0/status.yaml +++ b/runecontext/changes/CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0/status.yaml @@ -28,6 +28,7 @@ related_changes: - CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0 - CHG-2026-050-e3f8-workflow-definition-contract-binding-v0 - CHG-2026-051-4b9d-implementation-track-decomposition-git-worktree-execution-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: [] informed_by: - CHG-2026-001-57d6-agent-os-to-runecontext-migration-umbrella diff --git a/runecontext/changes/CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0/status.yaml b/runecontext/changes/CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0/status.yaml index 2a9b999e..9d6277ce 100644 --- a/runecontext/changes/CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0/status.yaml +++ b/runecontext/changes/CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0/status.yaml @@ -23,6 +23,7 @@ related_changes: - CHG-2026-051-4b9d-implementation-track-decomposition-git-worktree-execution-v0 - CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0 - CHG-2026-059-7b31-cross-machine-evidence-replication-restore-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-008-62e1-broker-local-api-v0 - CHG-2026-013-d2c9-minimal-tui-v0 diff --git a/runecontext/changes/CHG-2026-048-6b7a-session-execution-orchestration-v0/status.yaml b/runecontext/changes/CHG-2026-048-6b7a-session-execution-orchestration-v0/status.yaml index 4ab4dd0a..6aae202c 100644 --- a/runecontext/changes/CHG-2026-048-6b7a-session-execution-orchestration-v0/status.yaml +++ b/runecontext/changes/CHG-2026-048-6b7a-session-execution-orchestration-v0/status.yaml @@ -20,6 +20,7 @@ related_changes: - CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0 - CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0 - CHG-2026-051-4b9d-implementation-track-decomposition-git-worktree-execution-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-012-f1ef-workflow-runner-workspace-roles-deterministic-gates-v0 - CHG-2026-040-2b7f-session-transcript-model-v0 diff --git a/runecontext/changes/CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0/status.yaml b/runecontext/changes/CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0/status.yaml index 03b1be4b..ecd58d2a 100644 --- a/runecontext/changes/CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0/status.yaml +++ b/runecontext/changes/CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0/status.yaml @@ -19,6 +19,8 @@ related_changes: - CHG-2026-048-6b7a-session-execution-orchestration-v0 - CHG-2026-050-e3f8-workflow-definition-contract-binding-v0 - CHG-2026-051-4b9d-implementation-track-decomposition-git-worktree-execution-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish + - CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0 depends_on: - CHG-2026-024-acde-deps-fetch-offline-cache - CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0 diff --git a/runecontext/changes/CHG-2026-050-e3f8-workflow-definition-contract-binding-v0/status.yaml b/runecontext/changes/CHG-2026-050-e3f8-workflow-definition-contract-binding-v0/status.yaml index 5b211ead..4b9897d0 100644 --- a/runecontext/changes/CHG-2026-050-e3f8-workflow-definition-contract-binding-v0/status.yaml +++ b/runecontext/changes/CHG-2026-050-e3f8-workflow-definition-contract-binding-v0/status.yaml @@ -19,6 +19,7 @@ related_changes: - CHG-2026-024-acde-deps-fetch-offline-cache - CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0 - CHG-2026-051-4b9d-implementation-track-decomposition-git-worktree-execution-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-007-2315-policy-engine-v0 - CHG-2026-008-62e1-broker-local-api-v0 diff --git a/runecontext/changes/CHG-2026-052-a7f1-tui-leader-sequences-command-mode-v0/status.yaml b/runecontext/changes/CHG-2026-052-a7f1-tui-leader-sequences-command-mode-v0/status.yaml index 9f95e8c9..74f34e75 100644 --- a/runecontext/changes/CHG-2026-052-a7f1-tui-leader-sequences-command-mode-v0/status.yaml +++ b/runecontext/changes/CHG-2026-052-a7f1-tui-leader-sequences-command-mode-v0/status.yaml @@ -17,6 +17,7 @@ related_changes: - CHG-2026-040-2b7f-session-transcript-model-v0 - CHG-2026-043-8e9b-live-activity-watch-streams-v0 - CHG-2026-045-7f4c-direct-credential-model-providers-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-013-d2c9-minimal-tui-v0 - CHG-2026-037-91be-tui-multi-session-power-workspace-v0 diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/design.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/design.md index 37f57369..3a9d6d96 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/design.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/design.md @@ -18,6 +18,152 @@ The design goal is not to benchmark every current or future product surface. It The broader performance expansion remains a separate post-MVP lane in `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. +## Foundation Decisions + +### One Architecture Across Constrained And Scaled Environments + +This change freezes the same architecture rule already established in the related dependency, workflow, and attestation changes: + +- RuneCode must optimize one topology-neutral authority model across Raspberry Pi-class local hardware, ordinary developer workstations, and later vertically or horizontally scaled deployments. +- Performance work must improve the shared broker-owned, audit-preserving, trust-boundary-respecting architecture rather than introducing environment-specific fast paths or alternate authority models. +- No metric or gate may reward implementation shortcuts that bypass policy, audit, replay protection, attestation order, broker-owned lifecycle truth, or other reviewed control-plane responsibilities. + +### Separate Performance Contract Artifacts + +Performance baselines for this change must not be stored in `runecontext/assurance/baseline.yaml`. + +That file is already part of the project-substrate assurance posture and should remain dedicated to that purpose. This change should instead define one separate reviewed performance-contract artifact family under `tools/perfcontracts/` that stores metric identity, fixture identity, environment authority, statistical policy, and threshold declarations for the performance program. + +The first artifact family should use: + +- `tools/perfcontracts/manifest.json` as the reviewed inventory for performance contract files +- per-surface reviewed contract files under `tools/perfcontracts/contracts/` +- reviewed fixture inventory under `tools/perfcontracts/fixtures/` +- optional reviewed baseline sample artifacts under `tools/perfcontracts/baselines/` only when a metric needs repeated-sample comparison against preserved historical samples +- one trusted repo-local compare/enforce entrypoint at `tools/perfcontracts/main.go` that reads these artifacts and check outputs but never rewrites baselines during normal CI + +The first reviewed artifact family should be explicit enough to capture at least: + +- `metric_id` +- subsystem or surface identity +- runtime regime identity +- fixture identity +- measurement kind and unit +- authoritative environment +- sampling policy +- budget class +- explicit threshold or regression allowance +- lane authority +- activation state +- baseline source +- comparison method +- practical noise floor +- threshold origin +- notes or review rationale when needed + +Each metric contract should also declare: + +- `start_event` +- `end_event` +- `clock_source` +- `evidence_source` +- `included_phases` + +Those fields make timing boundaries reviewable and prevent implementation from moving a metric to an earlier advisory milestone without changing the reviewed contract. + +### Metric Taxonomy + +The first gate set should freeze one metric taxonomy so each check uses the right contract model instead of one generic performance bucket: + +- exact checks: + - deterministic event counts + - duplicate-work counts + - CAS write counts + - other invariant counts that should not vary across runs +- absolute budgets: + - user-visible attach and startup ceilings + - key-response ceilings + - CPU and similar operator-visible ceilings where the product promise is explicit +- regression budgets: + - repeated microbenchmarks + - stable allocation-heavy hot paths + - stable deterministic verification suites where historical regression is the main risk +- hybrid budgets: + - paths that need both a reviewed absolute product ceiling and a relative regression budget against a checked-in baseline + +### Statistical Defaults + +The first implementation slice should start with these statistical defaults and tune them only after implementation and validation data shows a concrete need: + +- repeated microbenchmarks: + - use repeated samples rather than one-run comparisons + - use robust comparison appropriate for noisy non-normal benchmark data + - require a practical noise-floor threshold in addition to statistical significance so tiny but detectable changes do not cause gate churn +- latency metrics: + - run a fixed number of repeated trials + - record median and `p95` + - gate on explicit reviewed ceilings, with median retained as supporting diagnostic context +- CPU and process-behavior metrics: + - use fixed observation windows after explicit warmup + - summarize repeated runs with average or median and a max guardrail + - avoid pretending that high-noise metrics deserve more inferential precision than the environment can support +- exact metrics: + - compare as exact values or hard bounds rather than inferential tests + +For Go microbenchmarks and similar repeated local measurements, the initial implementation may use a `benchstat`-style comparison workflow or equivalent robust repeated-sample comparison logic, but the durable product rule is the metric-class policy above rather than a tool-specific implementation detail. + +### Timing Boundary Rule + +Performance timing boundaries must terminate on reviewed authoritative milestones whenever they exist. + +That means: + +- prefer broker-owned typed lifecycle posture, persisted evidence, persisted verification outputs, or durable broker-projected state over launcher-local, client-local, or transcript-scrape heuristics +- where operator experience and authoritative completion are both important, the design may capture both as separate metrics or sub-metrics, but it must not silently substitute an earlier advisory milestone for the authoritative one + +This rule is especially important for: + +- attach and resume readiness +- runner startup from immutable `RunPlan` +- signed runtime startup and attach-ready behavior +- truthful post-handshake attestation verification +- external anchor prepare, execute, deferred handoff, and completion visibility + +### Fixture Scope Rule + +The MVP gate set should start with one small reviewed fixture inventory per major surface rather than broad ladder coverage. + +The first durable slice should prefer one golden deterministic fixture per major surface, with additional buckets only where a distinct runtime regime or scalability posture is already product-relevant. Larger fixture ladders, heavier extended lanes, and broader scale confidence work remain explicit post-MVP expansion work under `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. + +### Linux Environment Authority + +Linux remains the first authoritative numeric-gate environment, but this change should be honest about measurement noise. + +- Shared hosted Linux CI is acceptable for the initial required-gate slice where thresholds are conservative enough to remain deterministic. +- The design should allow selected higher-noise metrics to be promoted later to a tighter authoritative Linux environment without changing metric identity or product architecture. +- The performance program must not require a second product architecture merely because measurement infrastructure differs. + +### Lane And Activation States + +Each performance metric should declare both a lane authority and an activation state. + +Initial lane authorities: + +- `required_shared_linux`: required in the Linux PR path on shared hosted Linux because the metric is stable enough with conservative thresholds +- `required_tight_linux`: required for beta closure, but measured in a tighter authoritative Linux environment because shared hosted Linux is too noisy +- `informational_until_stable`: collected in CI or local verification until calibration data proves it is stable enough to become required +- `contract_pending_dependency`: contract and harness may be authored, but the gate cannot become required until the underlying reviewed path exists +- `extended`: non-PR, merge-queue, scheduled, or post-MVP measurement + +Initial activation states: + +- `defined`: contract exists but no enforcement yet +- `informational`: measurement runs but does not block +- `required`: measurement blocks in its declared lane +- `contract_pending_dependency`: contract exists but depends on another reviewed path before it can run authoritatively + +`CHG-025` external-anchor metrics and `CHG-054` truthful-attestation metrics may be defined before those changes fully land, but they must remain `contract_pending_dependency` until the reviewed path exists. + ## Investigation Scope And Constraints ### Investigation Goals @@ -102,45 +248,52 @@ The MVP gate set should distinguish at least these regimes: - attestation cold and warm verification cost - model-gateway, dependency-fetch, audit, protocol, and external-anchor overhead +The first gate set should also preserve the distinction between: + +- user-visible experience regimes +- repeated microbenchmark hot paths +- process-behavior resource regimes +- exact-count or invariant-preservation regimes + ## Performance Check Matrix ### TUI Surface | Aspect | Fixture | Check | Initial Threshold | CI Lane | | --- | --- | --- | --- | --- | -| Empty idle CPU | isolated empty broker state, isolated runtime/socket/target alias | sample real `runecode-tui` child CPU for 60s | average `<= 2%`, max sample `<= 4%` | required Linux | -| Waiting-state CPU | deterministic waiting session fixture in isolated broker store | sample real `runecode-tui` child CPU for 60s | average `<= 8%`, max sample `<= 12%` | required Linux | -| Attach/startup | isolated broker store with no pending work | PTY launch to first settled full frame | `<= 500ms` to first settled frame | required Linux | -| Key-response latency | quiet route, empty and waiting-state fixtures | key inject to transcript delta proxy | p95 `<= 50ms` empty, p95 `<= 75ms` waiting-state | required Linux | -| Render microbenchmarks | synthetic route surfaces and shell states | `BenchmarkShellViewEmpty`, `BenchmarkShellViewWaitingSession`, `BenchmarkShellViewPaletteOpen` | fail on `> 15%` regression in `ns/op`, `B/op`, or `allocs/op` from committed Linux baseline | required Linux | -| Update microbenchmarks | synthetic watch messages and command-surface states | `BenchmarkShellWatchApply`, `BenchmarkBuildPaletteEntries` | fail on `> 15%` regression in `ns/op`, `B/op`, or `allocs/op` | required Linux | +| Empty idle CPU | isolated empty broker state, isolated runtime/socket/target alias | sample real `runecode-tui` child CPU for fixed repeated windows after explicit warmup | average `<= 2%`, max sample `<= 4%` | `informational_until_stable`, promote to `required_shared_linux` or `required_tight_linux` after calibration | +| Waiting-state CPU | deterministic waiting session fixture in isolated broker store | sample real `runecode-tui` child CPU for fixed repeated windows after explicit warmup | average `<= 8%`, max sample `<= 12%` | `informational_until_stable`, promote to `required_shared_linux` or `required_tight_linux` after calibration | +| Attach/startup | isolated broker store with no pending work | PTY launch to first settled full frame after broker-owned attachable posture is reached | `<= 500ms` to first settled frame | `required_shared_linux` after timing contract is frozen | +| Key-response latency | quiet route, empty and waiting-state fixtures | fixed repeated trials from key inject to transcript delta proxy | p95 `<= 50ms` empty, p95 `<= 75ms` waiting-state | `required_shared_linux` after sample count is validated | +| Render microbenchmarks | synthetic route surfaces and shell states | `BenchmarkShellViewEmpty`, `BenchmarkShellViewWaitingSession` | fail on `> 15%` regression in `ns/op`, `B/op`, or `allocs/op` from committed Linux baseline once repeated-sample comparison exceeds the reviewed noise floor | `required_shared_linux` | +| Update microbenchmarks | synthetic watch messages and command-surface states | `BenchmarkShellWatchApply`, `BenchmarkBuildPaletteEntries` | fail on `> 15%` regression in `ns/op`, `B/op`, or `allocs/op` once repeated-sample comparison exceeds the reviewed noise floor | `required_shared_linux` | ### Broker Local API, Watch, And Attach Paths | Aspect | Fixture | Check | Initial Threshold | CI Lane | | --- | --- | --- | --- | --- | -| Unary local API latency | deterministic stores for supported beta fixtures | `session-list`, `session-get`, `run-list`, `run-get`, `approval-list`, `readiness`, `version-info`, `project-substrate-posture-get` | p95 `<= 150ms` for supported fixture sizes; fail on `> 15%` regression | required Linux | -| Watch-family latency | deterministic stores for supported beta fixtures | `run-watch`, `approval-watch`, `session-watch`, `session-turn-execution-watch` with `IncludeSnapshot` and `Follow` | p95 `<= 200ms` for supported fixture sizes; fail on `> 15%` regression | required Linux | -| Watch payload growth | same fixtures | response bytes and event counts | fail if payload grows `> 15%` beyond committed baseline per supported fixture bucket | required Linux | -| Mutation-path latency | deterministic local stores | `session-execution-trigger`, `continue`, `approval-resolve`, `backend-posture-change` | p95 `<= 200ms` for local control-plane-only paths | required Linux | -| Local attach | broker already running with isolated state | attach to ready interactive surface | `<= 500ms` | required Linux | -| Resume after reconnect | persisted session/run state with broker already running | detach and reattach workflow | `<= 500ms` from attach to ready surface | required Linux | +| Unary local API latency | deterministic stores for supported beta fixtures | repeated local trials over `session-list`, `session-get`, `run-list`, `run-get`, `approval-list`, `readiness`, `version-info`, `project-substrate-posture-get` | p95 `<= 150ms` for supported fixture sizes and fail on `> 15%` regression where hybrid budgets are used | `required_shared_linux` | +| Watch-family latency | deterministic stores for supported beta fixtures | repeated local trials over `run-watch`, `approval-watch`, `session-watch`, `session-turn-execution-watch` with `IncludeSnapshot` and `Follow` | p95 `<= 200ms` for supported fixture sizes and fail on `> 15%` regression where hybrid budgets are used | `required_shared_linux` | +| Watch payload growth | same fixtures | response bytes and event counts | fail if payload grows `> 15%` beyond committed baseline per supported fixture bucket | `required_shared_linux` | +| Mutation-path latency | deterministic local stores | `session-execution-trigger`, `continue`, `approval-resolve`, `backend-posture-change` | p95 `<= 200ms` for local control-plane-only paths | `required_shared_linux` | +| Local attach | broker already running with isolated state | attach to ready interactive surface after broker-owned lifecycle posture confirms attachability | `<= 500ms` | `required_shared_linux` after timing contract is frozen | +| Resume after reconnect | persisted session/run state with broker already running | detach and reattach workflow to ready broker-owned session/run truth | `<= 500ms` from attach to ready surface | `required_shared_linux` after timing contract is frozen | ### Runner, Workflow, And Launcher Paths | Aspect | Fixture | Check | Initial Threshold | CI Lane | | --- | --- | --- | --- | --- | -| Runner boundary check | current repo plus deterministic fixture workspace | `cd runner && npm run boundary-check` | wall time `<= 5s`; fail on `> 15%` regression | required Linux | -| Protocol fixture tests | deterministic shared fixture set | `cd runner && node --test scripts/protocol-fixtures.test.js` | wall time `<= 10s`; fail on `> 15%` regression | required Linux | -| Representative runner cold start | deterministic minimal workflow fixture | runner startup to first durable checkpoint | `<= 1s` local-overhead budget | required Linux | -| Supported workflow execution | deterministic MVP workflow fixture | trigger to completed durable state | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| CHG-050 workflow path | deterministic definitions and process fixtures | validation or canonicalization, trusted compilation, compiled-plan persistence or load, runner startup from immutable `RunPlan` | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| MicroVM cold start | deterministic lightweight signed role image with verified-cache miss or required trusted-admission path | trigger to broker-observed ready state | `<= 8s` cold | required Linux | -| MicroVM warm start | same signed runtime-image fixture with verified local runtime-asset cache hit | trigger to ready | `<= 3s` warm | required Linux | -| Container cold start | opt-in deterministic signed container-runtime fixture with verified-cache miss or required trusted-admission path | trigger to ready | `<= 4s` cold | required Linux | -| Container warm start | same signed container-runtime fixture with verified local runtime-asset cache hit | trigger to ready | `<= 2s` warm | required Linux | -| Attestation cold path | deterministic runtime startup fixture with full post-handshake verification | launch to persisted attestation projection | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Attestation warm path | same fixture with immutable verification-cache hits | launch to persisted attestation projection | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | +| Runner boundary check | current repo plus deterministic fixture workspace | `cd runner && npm run boundary-check` | wall time `<= 5s`; fail on `> 15%` regression | `required_shared_linux` | +| Protocol fixture tests | deterministic shared fixture set | `cd runner && node --test scripts/protocol-fixtures.test.js` | wall time `<= 10s`; fail on `> 15%` regression | `required_shared_linux` | +| Representative runner cold start | deterministic minimal workflow fixture | runner startup to first durable checkpoint bound to the active immutable plan identity | `<= 1s` local-overhead budget | `informational_until_stable`, promote after sample stability is proven | +| Supported workflow execution | deterministic MVP workflow fixture | trigger to completed durable broker state on the supported beta slice | threshold derived from committed baseline; fail on `> 15%` regression | `contract_pending_dependency` until the real supported workflow path exists, then promote | +| CHG-050 workflow path | deterministic definitions and process fixtures | validation or canonicalization, trusted compilation, compiled-plan persistence or load, runner startup from immutable `RunPlan` | threshold derived from committed baseline; fail on `> 15%` regression once repeated-sample comparison exceeds the reviewed noise floor | `required_shared_linux` for trusted compilation/load checks; execution startup remains `contract_pending_dependency` until the real path exists | +| MicroVM cold start | deterministic lightweight signed role image with verified-cache miss or required trusted-admission path | trigger to broker-observed ready state | `<= 8s` cold | `informational_until_stable` or `required_tight_linux` after calibration | +| MicroVM warm start | same signed runtime-image fixture with verified local runtime-asset cache hit | trigger to ready | `<= 3s` warm | `informational_until_stable` or `required_tight_linux` after calibration | +| Container cold start | opt-in deterministic signed container-runtime fixture with verified-cache miss or required trusted-admission semantics used for microVM startup checks | trigger to ready | `<= 4s` cold | `informational_until_stable` | +| Container warm start | same signed container-runtime fixture with verified local runtime-asset cache hit | trigger to ready | `<= 2s` warm | `informational_until_stable` | +| Attestation cold path | deterministic runtime startup fixture with full post-handshake verification | launch to persisted post-handshake attestation verification and broker projection | threshold derived from committed baseline; fail on `> 15%` regression | `contract_pending_dependency` until `CHG-054` lands | +| Attestation warm path | same fixture with immutable verification-cache hits | launch to persisted post-handshake attestation verification and broker projection | threshold derived from committed baseline; fail on `> 15%` regression | `contract_pending_dependency` until `CHG-054` lands | Launcher and attestation checks must preserve the reviewed architecture rather than rewarding unsafe shortcuts: @@ -152,23 +305,23 @@ Launcher and attestation checks must preserve the reviewed architecture rather t | Aspect | Fixture | Check | Initial Threshold | CI Lane | | --- | --- | --- | --- | --- | -| Secret-ingress prepare and submit | stubbed deterministic secret payloads | local broker and secrets overhead only | p95 `<= 300ms` for small payloads | required Linux | -| Credential lease issuance | deterministic provider-profile fixture | local issuance overhead | p95 `<= 150ms` | required Linux | -| Model-gateway invoke overhead | stubbed provider backend returning deterministic responses | RuneCode-added overhead excluding external network | p95 `<= 100ms` added overhead | required Linux | -| Dependency cache miss | deterministic dependency-request fixture and stubbed registry payload source | broker-owned fetch to CAS with no existing cached units | threshold derived from committed baseline; fail on `> 15%` regression in wall time, peak RSS, or bytes buffered beyond reviewed budget | required Linux | -| Dependency cache hit | same fixture with cached resolved units already present | broker-owned dependency availability request with no network fetch path taken | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Dependency miss coalescing | concurrent identical deterministic dependency requests | wall time, duplicate network work count, and CAS write count | require one effective upstream fill per canonical request identity; fail on duplicate-fill regression | required Linux | -| Dependency materialization | deterministic cached dependency manifest and units | broker-mediated offline staging or materialization for workspace use | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Dependency stream-to-CAS posture | large deterministic dependency payload fixture | memory and streaming behavior during cache fill | fail if implementation buffers full payloads in memory beyond reviewed budget or regresses beyond baseline | required Linux | -| Audit verification | deterministic ledger fixtures | verify end-to-end locally | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Audit finalize verify | deterministic local ledger | finalize plus verify | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Protocol schema validation | checked-in protocol schemas and fixtures | schema load and validation suite | `<= 2s` for standard CI fixture set | required Linux | -| Fixture-manifest parity | protocol fixtures plus manifest | parity and canonicalization checks | `<= 2s` | required Linux | -| External anchor prepare | deterministic sealed audit segment plus stubbed target descriptor | prepare request to durable prepared state | p95 `<= 500ms` local control-plane overhead | required Linux | -| External anchor execute-completed | deterministic sealed audit segment plus fast stubbed target | execute request to completed authoritative persistence | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| External anchor execute-deferred handoff | deterministic sealed audit segment plus intentionally delayed stubbed target | execute request to deferred durable state | p95 `<= 500ms` local control-plane overhead | required Linux | -| Deferred completion visibility | same delayed stubbed target | deferred completion to durable completed state plus get or watch visibility | threshold derived from committed baseline; fail on `> 15%` regression | required Linux | -| Receipt admission on unchanged seal | already-verified sealed segment plus valid stubbed target proof | authoritative receipt and sidecar admission without full seal replay | threshold derived from committed baseline; fail on `> 15%` regression in wall time or peak RSS | required Linux | +| Secret-ingress prepare and submit | stubbed deterministic secret payloads | local broker and secrets overhead only | p95 `<= 300ms` for small payloads | `required_shared_linux` | +| Credential lease issuance | deterministic provider-profile fixture | local issuance overhead | p95 `<= 150ms` | `required_shared_linux` | +| Model-gateway invoke overhead | stubbed provider backend returning deterministic responses | RuneCode-added overhead excluding external network | p95 `<= 100ms` added overhead | `required_shared_linux` | +| Dependency cache miss | deterministic dependency-request fixture and stubbed registry payload source | broker-owned fetch to CAS with no existing cached units | threshold derived from committed baseline; fail on `> 15%` regression in wall time, reviewed bounded-buffer metrics, or peak RSS guardrails beyond reviewed budgets | `required_shared_linux` for bounded-buffer/exact counters; wall/RSS may begin `informational_until_stable` | +| Dependency cache hit | same fixture with cached resolved units already present | broker-owned dependency availability request with no network fetch path taken | threshold derived from committed baseline; fail on `> 15%` regression | `required_shared_linux` | +| Dependency miss coalescing | concurrent identical deterministic dependency requests | wall time, duplicate network work count, and CAS write count | require one effective upstream fill per canonical request identity; fail on duplicate-fill regression | `required_shared_linux` for exact duplicate-fill and CAS-write counts | +| Dependency materialization | deterministic cached dependency manifest and units | broker-mediated offline staging or materialization for workspace use | threshold derived from committed baseline; fail on `> 15%` regression | `required_shared_linux` after fixture calibration | +| Dependency stream-to-CAS posture | large deterministic dependency payload fixture | memory and streaming behavior during cache fill | fail if implementation buffers full payloads in memory beyond reviewed budget, violates reviewed bounded-buffer instrumentation limits, or regresses beyond baseline | `required_shared_linux` for bounded-buffer instrumentation; process RSS starts `informational_until_stable` | +| Audit verification | deterministic ledger fixtures | verify end-to-end locally | threshold derived from committed baseline; fail on `> 15%` regression | `required_shared_linux` | +| Audit finalize verify | deterministic local ledger | finalize plus verify | threshold derived from committed baseline; fail on `> 15%` regression | `required_shared_linux` | +| Protocol schema validation | checked-in protocol schemas and fixtures | schema load and validation suite | `<= 2s` for standard CI fixture set | `required_shared_linux` | +| Fixture-manifest parity | protocol fixtures plus manifest | parity and canonicalization checks | `<= 2s` | `required_shared_linux` | +| External anchor prepare | deterministic sealed audit segment plus stubbed target descriptor | prepare request to durable prepared state | p95 `<= 500ms` local control-plane overhead | `contract_pending_dependency` until `CHG-025` lands | +| External anchor execute-completed | deterministic sealed audit segment plus fast stubbed target | execute request to completed authoritative persistence | threshold derived from committed baseline; fail on `> 15%` regression | `contract_pending_dependency` until `CHG-025` lands | +| External anchor execute-deferred handoff | deterministic sealed audit segment plus intentionally delayed stubbed target | execute request to deferred durable state | p95 `<= 500ms` local control-plane overhead | `contract_pending_dependency` until `CHG-025` lands | +| Deferred completion visibility | same delayed stubbed target | deferred completion to durable completed state plus get/watch visibility | threshold derived from committed baseline; fail on `> 15%` regression | `contract_pending_dependency` until `CHG-025` lands | +| Receipt admission on unchanged seal | already-verified sealed segment plus valid stubbed target proof | authoritative receipt and sidecar admission without full seal replay | threshold derived from committed baseline; fail on `> 15%` regression in wall time or peak RSS | `contract_pending_dependency` until `CHG-025` lands | External audit anchoring checks must preserve the reviewed architecture rather than rewarding unsafe shortcuts: @@ -177,13 +330,108 @@ External audit anchoring checks must preserve the reviewed architecture rather t - unchanged verified seals should use the reviewed incremental receipt-admission path rather than forcing full verifier replay as the only normal path - checks must not bypass authoritative proof verification, policy binding, or audit evidence persistence to produce a lower number +### Initial Fixture Inventory + +The initial gate set should start with a small reviewed fixture inventory rather than a broad ladder. + +Recommended first durable slice: + +- TUI: + - `tui.empty.v1` + - `tui.waiting.v1` +- broker local API: + - `broker.unary.beta-small.v1` + - `broker.watch.run.snapshot-follow.v1` + - `broker.watch.approval.snapshot-follow.v1` + - `broker.watch.session.snapshot-follow.v1` + - `broker.watch.turn-execution.snapshot-follow.v1` +- runner and workflow: + - `workflow.first-party-minimal.v1` + - `workflow.chg050-compile.v1` +- dependency fetch: + - `deps.cache-miss.small.v1` + - `deps.cache-hit.small.v1` + - `deps.coalesced-miss.small.v1` +- audit: + - `audit.ledger.standard.v1` +- external anchor: + - `anchor.fast-complete.stub.v1` + - `anchor.deferred.stub.v1` +- attestation: + - `attestation.cold.signed-runtime.v1` + - `attestation.warm.signed-runtime.v1` + +This keeps the first release-defining gate set narrow enough to stay deterministic while still covering the regimes that matter for `v0.1.0-beta.1`. + +Fixture IDs are part of metric identity. Future fixture expansion should add new IDs rather than changing these IDs in place unless the fixture semantics intentionally change and the baseline is reviewed as a new contract. + +## Statistical And Comparison Policy + +### Repeated Microbenchmarks + +- Run repeated samples rather than one-off benchmark comparisons. +- Use robust repeated-sample comparison logic suitable for non-normal noisy measurements. +- Require both: + - the configured regression threshold to be exceeded + - the change to exceed the reviewed practical noise floor +- Treat summary statistics such as median and confidence intervals as authoritative comparison context, not single best-case runs. + +Initial constants: + +- required PR comparisons should use at least `10` repeated samples when runtime cost allows +- baseline refresh or threshold recalibration should preferably use at least `20` repeated samples +- fail only when the configured regression threshold and the reviewed practical noise floor are both exceeded +- use a `benchstat`-style comparison workflow or equivalent robust repeated-sample comparison for Go microbenchmarks + +### Latency Metrics + +- Run a fixed number of repeated trials per fixture. +- Record median and `p95`. +- Gate on the reviewed explicit ceiling, with median retained as diagnostic context. +- Avoid using statistical significance alone as the gate for user-visible latency promises. + +Initial constants: + +- cheap local latency metrics should target `30` fixed trials so `p95` is meaningful enough for a required gate +- heavier lifecycle metrics may use median plus max ceilings while they are too expensive for a meaningful `p95` sample size +- each latency metric contract should declare whether `p95`, median plus max, or both are authoritative + +### CPU And Process-Behavior Metrics + +- Use explicit warmup before measurement. +- Use fixed repeated windows. +- Summarize with average or median plus max guardrails. +- Prefer conservative thresholds over false precision in noisy shared environments. + +Initial constants: + +- each metric must declare warmup duration, observation-window duration, and number of repeated windows before it can become required +- shared hosted Linux CPU metrics should start as `informational_until_stable` unless validation data proves the threshold is stable enough to require there +- max guardrails should catch pathological spikes, but sustained average or median window cost should remain the primary CPU signal + +### Exact Metrics + +- Treat deterministic counts, duplicate-work counts, payload counts, and similar invariants as exact checks or hard bounds. +- Do not subject exact metrics to inferential comparison logic. + +### Threshold Provenance + +Every threshold should declare one reviewed `threshold_origin`: + +- `product_budget`: an intentional product promise or safety ceiling +- `investigation_baseline`: derived from the corrected investigation data captured by this change +- `first_calibration`: accepted as an initial calibration value after implementation produces repeatable measurements +- `temporary_guardrail`: intentionally provisional and expected to be revisited after more data + +Threshold loosening should require explicit review rationale and should not be hidden inside baseline refresh mechanics. + ## CI Integration Plan ### Required Linux PR Lane -The required Linux PR lane should include the smallest deterministic checks that still catch the main MVP regressions: +The required Linux PR lane should include the smallest deterministic `required_shared_linux` checks that still catch the main MVP regressions without pretending high-noise checks are stable on shared hosted runners: -- TUI empty-idle CPU gate -- TUI waiting-state CPU gate +- TUI empty-idle CPU measurement as informational until stability is proven +- TUI waiting-state CPU measurement as informational until stability is proven - TUI attach/startup gate - TUI key-response gate - TUI render and update microbenchmarks @@ -191,12 +439,16 @@ The required Linux PR lane should include the smallest deterministic checks that - broker watch-family latency gate - local attach and resume gates - protocol and runner deterministic quick checks -- supported workflow execution gate -- launcher startup and attestation cold or warm quick checks -- deterministic model-gateway, dependency-fetch, audit, and external-anchor quick checks +- supported workflow execution contracts, with required enforcement only after the real supported workflow path exists +- launcher startup measurements as informational until stability or tighter Linux authority is available +- attestation cold or warm contracts as `contract_pending_dependency` until `CHG-054` lands +- deterministic model-gateway, dependency-fetch, audit, and protocol quick checks +- external-anchor contracts as `contract_pending_dependency` until `CHG-025` lands + +The first required lane should prefer metrics that are already stable enough on shared hosted Linux. The design may later promote selected higher-noise gates to a tighter authoritative Linux environment, but the initial gate set should not depend on that tighter environment existing on day one. ### Baseline Storage And Review -- Store benchmark baselines and threshold declarations in reviewed repo artifacts. +- Store benchmark baselines and threshold declarations in reviewed performance-contract artifacts separate from `runecontext/assurance/baseline.yaml`. - Do not auto-rewrite performance baselines inside normal CI runs. - Threshold changes should require an intentional doc-and-code review path, just like other product contract changes. @@ -215,4 +467,6 @@ Those remain tracked in `CHG-2026-061-45fe-performance-program-expansion-cross-p - Do not let performance verification introduce writes, mutable lockfiles, or auto-updated baselines into normal CI. - Do not overfit thresholds to a single developer workstation and then claim they are durable product budgets. - Do not collapse empty-idle and waiting-state behavior into one TUI metric. +- Do not terminate metrics at advisory client-local or launcher-local milestones when authoritative persisted or broker-owned milestones exist downstream in the reviewed product contract. +- Do not use one universal statistics rule for every metric class when the metric semantics clearly differ. - Do not weaken trust-boundary or audit requirements in the name of performance. diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/proposal.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/proposal.md index 095acf00..8a2fc771 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/proposal.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/proposal.md @@ -34,6 +34,28 @@ At the same time, the repository now has a broader set of performance surfaces t - benchmarked render, watch, orchestration, and backend paths - Introduce deterministic performance checks for the supported beta surfaces only. - Assign per-aspect thresholds that are suitable for CI, with Linux-first numeric gates and deterministic local fixtures or stubs instead of live external dependencies. +- Define one reviewed performance-contract artifact family separate from `runecontext/assurance/baseline.yaml` so project-substrate assurance posture and performance-governance posture remain distinct. +- Store that reviewed performance-contract artifact family under `tools/perfcontracts/`, with `manifest.json`, per-surface contract files, reviewed fixture inventory, optional repeated-sample baseline artifacts, and a trusted check-only repo-local enforcement entrypoint that never rewrites baselines during normal CI. +- Freeze one metric taxonomy for the first gate set so each measurement uses the right contract model instead of one vague "benchmark" bucket: + - exact checks for exact counters and invariant counts + - absolute budgets for user-visible experience ceilings + - regression budgets for stable repeated microbenchmarks and hot paths + - hybrid budgets where both explicit product ceilings and baseline-regression limits matter +- Freeze one lane and activation taxonomy so initial required gates are honest about measurement authority and dependency readiness: + - `required_shared_linux` for stable required checks on shared hosted Linux + - `required_tight_linux` for checks that are required but too noisy for shared hosted Linux + - `informational_until_stable` for useful checks that need calibration before blocking + - `contract_pending_dependency` for suites whose contracts can be authored before the underlying reviewed path lands + - `extended` for heavier non-PR or post-MVP measurements +- Freeze the initial statistical defaults for the first implementation slice: + - repeated-sample robust comparison for microbenchmarks + - median plus `p95` plus explicit ceilings for latency metrics + - fixed-window repeated sampling with average or median plus max ceilings for CPU and process-behavior metrics + - exact comparison for deterministic event-count, payload-count, and duplicate-work metrics + - practical noise-floor thresholds in addition to statistical significance for repeated regression checks +- Freeze initial statistical constants before implementation starts: repeated microbenchmarks use at least `10` PR samples and preferably `20` baseline-refresh samples; cheap local latency metrics use enough samples for meaningful `p95` gates, with `30` trials as the default target; heavier lifecycle metrics may use median plus max ceilings until they are cheap enough for meaningful percentile gates; CPU and process-behavior metrics use explicit warmup plus fixed observation windows. +- Freeze the rule that performance timing boundaries must terminate on reviewed broker-owned or persisted milestones rather than advisory launcher-local or client-local heuristics whenever authoritative downstream milestones exist. +- Require each performance contract to declare `start_event`, `end_event`, `clock_source`, `evidence_source`, `included_phases`, `threshold_origin`, and stable fixture identifiers before the gate can become required. - Keep performance verification check-only and CI-safe so it remains compatible with `just ci` discipline and does not introduce silent writes or mutable benchmark artifacts during normal verification. - Freeze a policy for baseline maintenance so future work can tighten thresholds intentionally instead of letting them drift implicitly. - Include dependency-fetch and offline-cache performance as an MVP product regime, including cache miss, cache hit, miss coalescing, bounded concurrency, stream-to-CAS persistence, and broker-mediated offline dependency staging or materialization costs. @@ -41,6 +63,10 @@ At the same time, the repository now has a broader set of performance surfaces t - Include explicit measurement of the supported CHG-049 first-party workflow-pack beta slice rather than every broader workflow-pack surface. - Include explicit measurement of the required attestation path for supported runtime startup and attach flows, including cold verification, warm verification-cache hits, replay and freshness checks, and persisted attestation-evidence handling. - Include explicit measurement of the external audit anchoring path, including prepare latency, execute handoff latency, deferred completion handling, target-proof admission cost, and verifier behavior on unchanged verified seals. +- Freeze one small reviewed MVP fixture inventory per major surface for the first gate set rather than starting with broad ladder coverage; larger fixture ladders remain post-MVP expansion work. +- Assign stable fixture IDs for the first reviewed inventory before collecting baselines so future fixture expansion does not churn existing metric identity. +- Treat dependency-fetch memory posture as an explicit architectural contract, measured through both coarse process memory observations and reviewed internal bounded-buffer instrumentation so stream-to-CAS behavior is verified directly rather than inferred only from RSS. +- Keep the first required Linux performance lane compatible with shared CI where thresholds are conservative enough to remain deterministic, while leaving room to promote selected high-noise metrics to a tighter authoritative Linux environment later without changing product architecture or metric identity. - Defer broader workflow-pack surfaces, git-gateway performance expansion, larger fixture ladders, and tuned macOS or Windows numeric gates to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. ## Why Now @@ -55,10 +81,15 @@ Capturing that distinction now prevents future work from overfitting to the wron ## Assumptions - Linux CI will remain the first authoritative numeric-gate environment for the initial MVP performance program. +- Shared hosted Linux CI is acceptable for the first required numeric-gate slice where thresholds are deliberately conservative and metric noise is understood; the design may later promote selected high-noise metrics to a tighter Linux environment without redefining the product contract. - Deterministic local fixtures, synthetic stores, stubbed provider backends, and stubbed external anchor targets are acceptable and preferred for CI gating. - Network round-trip time to external providers is out of scope for hard CI gates; only RuneCode-added overhead should be measured under deterministic stubs. - Performance checks must not weaken trust boundaries, bypass audit or policy, or replace canonical broker-owned state with client-local shortcuts. - The supported beta workflow slice is the right first workflow gate set; broader workflow-pack entry families and post-MVP workflow surfaces should be measured later rather than broadening the first beta gate set prematurely. +- The first performance implementation slice should start with the reviewed statistical defaults captured by this change and tune them only after implementation and validation data shows that a given metric class needs different handling. +- The first implementation should treat threshold origins explicitly as `product_budget`, `investigation_baseline`, `first_calibration`, or `temporary_guardrail` so later reviewers can distinguish product promises from provisional calibration values. +- External audit anchoring and truthful post-handshake attestation performance suites should align explicitly with the underlying reviewed changes they measure rather than relying on sequencing folklore. +- Harnesses and performance contracts may be defined before dependent paths are complete, but gates must not become required until their `activation_state` is no longer `contract_pending_dependency` and the reviewed path exists. - Broader macOS and Windows numeric performance gates should follow later platform tuning work rather than blocking Linux-first beta readiness. ## Out of Scope @@ -79,6 +110,10 @@ This change gives RuneCode one canonical planning surface for the first durable - the distinction between empty-idle and waiting-state costs - the deterministic benchmark and latency checks needed across the supported beta surface - the per-aspect thresholds and CI structure required to make performance a maintained contract rather than an anecdotal concern +- the explicit metric taxonomy, statistical defaults, and practical noise-floor policy for the first durable gate set +- the explicit rule that reviewed broker-owned or persisted milestones are authoritative timing boundaries for performance checks +- the explicit separation between project-substrate assurance baseline state and reviewed performance-contract artifacts +- the explicit lane-state, activation-state, fixture-ID, and threshold-provenance model needed to make the performance program executable without rewarding shortcuts or creating flaky gates - the explicit expectation that dependency-fetch performance must be measured on both cold-cache and warm-cache paths without weakening trust boundaries or buffering full dependency payloads in memory - the explicit expectation that the supported first-party workflow slice becomes a measurable product surface rather than invisible orchestration overhead - the explicit expectation that launcher and attach-ready performance checks measure the reviewed signed-runtime plus required-attestation trust path rather than rewarding bypasses around attestation verification, replay or freshness enforcement, or attestation evidence persistence diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/references.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/references.md index 90aea6bd..373e02cc 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/references.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/references.md @@ -39,6 +39,9 @@ - Lip Gloss: https://github.com/charmbracelet/lipgloss - Go `pprof`: https://pkg.go.dev/runtime/pprof - Go benchmarking: https://pkg.go.dev/testing +- Go `benchstat`: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat +- Criterion analysis guide: https://bheisler.github.io/criterion.rs/book/analysis.html +- LLVM benchmarking tips: https://llvm.org/docs/Benchmarking.html ## Related Changes @@ -48,12 +51,14 @@ - `runecontext/changes/CHG-2026-011-7240-secretsd-model-gateway-v0/` - `runecontext/changes/CHG-2026-012-f1ef-workflow-runner-workspace-roles-deterministic-gates-v0/` - `runecontext/changes/CHG-2026-013-d2c9-minimal-tui-v0/` +- `runecontext/changes/CHG-2026-025-5679-external-audit-anchoring-v0/` - `runecontext/changes/CHG-2026-037-91be-tui-multi-session-power-workspace-v0/` - `runecontext/changes/CHG-2026-043-8e9b-live-activity-watch-streams-v0/` - `runecontext/changes/CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0/` - `runecontext/changes/CHG-2026-048-6b7a-session-execution-orchestration-v0/` - `runecontext/changes/CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0/` - `runecontext/changes/CHG-2026-050-e3f8-workflow-definition-contract-binding-v0/` +- `runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/` ## Planning Notes @@ -63,3 +68,8 @@ - After the alpha.7 waiting-state split landed, a fresh isolated rerun measured empty-state CPU at `0.20-0.80%` and waiting-state CPU at `0.00-1.00%` for the real `runecode-tui` child. - The strongest before/after comparison is the waiting-state path: the earlier sample climbed through `22.81%` and `61.92%` CPU, while the post-fix isolated waiting sample stayed at `1.00%` mid and aged CPU. - The post-fix waiting transcript still rendered `WAITING session=sess-manual-multiwait`, confirming the improvement came from removing the fast repaint loop for waiting states rather than from hiding the state cue. +- The first durable gate set should use a dedicated reviewed performance-contract artifact family rather than overloading `runecontext/assurance/baseline.yaml`, which remains part of project-substrate assurance posture. +- The first implementation slice should use reviewed statistical defaults per metric class: repeated-sample robust comparison for microbenchmarks, median plus `p95` plus explicit ceilings for latency metrics, fixed-window repeated sampling with average or median plus max guardrails for CPU/process-behavior metrics, and exact comparison for deterministic invariant counts. +- Performance timing boundaries should terminate on reviewed broker-owned or persisted milestones whenever those authoritative surfaces exist downstream in the product contract. +- Follow-up review refined the implementation foundation further: performance contracts should live under `tools/perfcontracts/`, use stable fixture IDs, declare lane authority and activation state, declare threshold provenance, and include timing-boundary metadata before required enforcement. +- Shared hosted Linux remains acceptable for stable required checks, while high-noise checks should start informational, remain pending dependency, or move to a tighter Linux authority without changing RuneCode's product architecture. diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/standards.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/standards.md index bc06740b..04eb188a 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/standards.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/standards.md @@ -18,6 +18,15 @@ That includes freezing the following clarifications for the first gate set: - performance checks must respect the same trust boundaries and broker-owned authority surfaces as correctness checks - TUI empty-idle and waiting-state costs are distinct product regimes and must not be collapsed into one metric - broker request latency, watch-family cost, runner startup, supported workflow execution, launcher startup, gateway overhead, audit verification, external audit anchoring, and attach or resume paths all need explicit budgets +- performance-contract artifacts remain separate from project-substrate assurance baseline state; `runecontext/assurance/baseline.yaml` is not the home for CHG-053 threshold declarations +- performance-contract artifacts live under `tools/perfcontracts/` and are enforced by a trusted check-only repo tool rather than rewritten by CI +- the first gate set uses an explicit metric taxonomy across exact, absolute-budget, regression-budget, and hybrid-budget checks rather than one generic benchmark bucket +- every metric needs reviewed lane authority, activation state, stable fixture identity, threshold provenance, and timing-boundary metadata before required enforcement +- timing boundaries must terminate on reviewed broker-owned or persisted milestones whenever those authoritative surfaces exist downstream in the product contract +- the first implementation slice uses reviewed statistical defaults per metric class rather than one universal statistics rule for every check +- the first implementation slice freezes sample-count, warmup, p95-eligibility, and practical-noise-floor constants before required gates are enforced +- the first gate set should start with one small reviewed fixture inventory per major surface, while larger fixture ladders remain explicit post-MVP expansion work +- contracts for attestation and external audit anchoring may be authored before their reviewed dependency paths land, but required enforcement must wait until those paths exist - the supported first-party workflow-pack beta slice needs explicit budgets now, while broader workflow-pack surfaces should be expanded later under `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0` - Linux is the first authoritative numeric gate for this layer, while broader cross-platform tuning should remain explicit post-MVP work - threshold updates and baseline refreshes require explicit review rather than silent CI mutation diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/status.yaml b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/status.yaml index c641a9b2..53493a89 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/status.yaml +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/status.yaml @@ -1,7 +1,7 @@ schema_version: 1 id: CHG-2026-053-9d2b-performance-baselines-verification-gates-v0 title: Project Performance Baselines + Verification Gates v0 -status: planned +status: implemented type: feature size: large verification_status: pending @@ -11,8 +11,12 @@ context_bundles: related_specs: [] related_decisions: [] related_changes: + - CHG-2026-025-5679-external-audit-anchoring-v0 + - CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish - CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0 depends_on: + - CHG-2026-025-5679-external-audit-anchoring-v0 - CHG-2026-008-62e1-broker-local-api-v0 - CHG-2026-011-7240-secretsd-model-gateway-v0 - CHG-2026-012-f1ef-workflow-runner-workspace-roles-deterministic-gates-v0 @@ -22,6 +26,7 @@ depends_on: - CHG-2026-048-6b7a-session-execution-orchestration-v0 - CHG-2026-049-1d4e-first-party-runecontext-workflow-pack-v0 - CHG-2026-050-e3f8-workflow-definition-contract-binding-v0 + - CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0 informed_by: - CHG-2026-009-1672-launcher-microvm-backend-v0 - CHG-2026-037-91be-tui-multi-session-power-workspace-v0 diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/tasks.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/tasks.md index 2c3fe6d9..767df1f4 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/tasks.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/tasks.md @@ -2,18 +2,25 @@ ## Phase 1: Deterministic Fixture Foundation -- [ ] Add deterministic fixture builders for empty and waiting broker stores used by the supported beta path. -- [ ] Add deterministic runner and supported-workflow fixtures that do not require live external dependencies. -- [ ] Add deterministic stubbed provider backends for model-gateway and secrets overhead checks. -- [ ] Add deterministic stubbed external-anchor targets for prepare, execute, deferred, and receipt-admission checks. -- [ ] Define one reviewed baseline-artifact format for benchmark and latency thresholds. +- [x] Add deterministic fixture builders for empty and waiting broker stores used by the supported beta path. +- [x] Add deterministic runner and supported-workflow fixtures that do not require live external dependencies. +- [x] Add deterministic stubbed provider backends for model-gateway and secrets overhead checks. +- [x] Add deterministic stubbed external-anchor targets for prepare, execute, deferred, and receipt-admission checks. +- [x] Define one reviewed performance-contract artifact format for benchmark and latency thresholds, separate from `runecontext/assurance/baseline.yaml`. +- [x] Store the reviewed performance-contract family under `tools/perfcontracts/` with a manifest, per-surface contract files, reviewed fixture inventory, and optional repeated-sample baseline artifacts where needed. +- [x] Define one trusted repo-local compare/enforce tool under `tools/` that reads performance contracts and check outputs without rewriting baselines during normal CI. +- [x] Define the metric taxonomy for exact, absolute-budget, regression-budget, and hybrid-budget checks in the reviewed performance-contract artifacts. +- [x] Define lane authority and activation states for every metric: `required_shared_linux`, `required_tight_linux`, `informational_until_stable`, `contract_pending_dependency`, and `extended`. +- [x] Define the initial reviewed MVP fixture inventory per major surface and explicitly defer larger fixture ladders to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. +- [x] Assign stable fixture IDs for the initial inventory before collecting baselines. ## Phase 2: TUI Regime Checks -- [ ] Add a real-child CPU sampler for PTY-launched `runecode-tui` suitable for CI. -- [ ] Add an empty-state idle CPU gate using fully isolated broker store, audit ledger, runtime directory, socket, and TUI target alias. -- [ ] Add a waiting-state CPU gate using a deterministic waiting-session fixture. -- [ ] Add attach/startup and key-response latency checks for quiet and waiting-state fixtures. +- [x] Add a real-child CPU sampler for PTY-launched `runecode-tui` suitable for CI. +- [x] Add an empty-state idle CPU gate using fully isolated broker store, audit ledger, runtime directory, socket, and TUI target alias. +- [x] Add a waiting-state CPU gate using a deterministic waiting-session fixture. +- [x] Add attach/startup and key-response latency checks for quiet and waiting-state fixtures. +- [x] Freeze authoritative timing boundaries for TUI attach and key-response checks, including `start_event`, `end_event`, `clock_source`, `evidence_source`, and `included_phases`. - [x] Add `go test -bench` coverage for render and update hot paths, including shell view, watch apply, and palette-entry building. Alpha.7 bootstrap already landed: @@ -22,59 +29,75 @@ Alpha.7 bootstrap already landed: ## Phase 3: Broker, Attach, And Resume Checks -- [ ] Add deterministic latency checks for broker unary local API requests used by the supported beta surface. -- [ ] Add deterministic latency and payload-growth checks for `run-watch`, `approval-watch`, `session-watch`, and `session-turn-execution-watch` on the supported beta fixtures. -- [ ] Add control-plane mutation latency checks for execution trigger, continue, approval resolve, and backend posture change paths. -- [ ] Add attach and resume performance checks for the persistent local control-plane lifecycle. -- [ ] Ensure all broker performance checks remain local-only and do not rely on live network services. +- [x] Add deterministic latency checks for broker unary local API requests used by the supported beta surface. +- [x] Add deterministic latency and payload-growth checks for `run-watch`, `approval-watch`, `session-watch`, and `session-turn-execution-watch` on the supported beta fixtures. +- [x] Add control-plane mutation latency checks for execution trigger, continue, approval resolve, and backend posture change paths. +- [x] Add attach and resume performance checks for the persistent local control-plane lifecycle. +- [x] Ensure all broker performance checks remain local-only and do not rely on live network services. +- [x] Freeze authoritative timing boundaries for local attach and resume, including `start_event`, `end_event`, `clock_source`, `evidence_source`, and `included_phases`. ## Phase 4: Runner, Workflow, Launcher, And Attestation Checks -- [ ] Add wall-time and regression checks for runner boundary verification and protocol fixture tests. -- [ ] Add a representative runner cold-start check with a deterministic minimal workflow. -- [ ] Add deterministic checks for the supported MVP workflow execution path. -- [ ] Add deterministic checks for CHG-050 workflow-definition/process-definition validation, canonicalization, and trusted compilation overhead. -- [ ] Add deterministic checks for compiled `RunPlan` persistence/load and runner startup from immutable `RunPlan`. -- [ ] Add deterministic checks for the supported CHG-049 first-party workflow-pack beta slice only. -- [ ] Add cold and warm microVM startup checks, with cold covering verified-cache miss or trusted-admission cost and warm covering verified local runtime-asset cache-hit cost on the same signed runtime identity. -- [ ] Add cold and warm container startup checks for the explicit opt-in backend, with the same verified-cache miss or hit semantics used for microVM startup checks. -- [ ] Add attestation cold-path and warm verification-cache checks for the truthful supported runtime path. +- [x] Add wall-time and regression checks for runner boundary verification and protocol fixture tests. +- [x] Add a representative runner cold-start check with a deterministic minimal workflow. +- [x] Add deterministic checks for the supported MVP workflow execution path. +- [x] Add deterministic checks for CHG-050 workflow-definition/process-definition validation, canonicalization, and trusted compilation overhead. +- [x] Add deterministic checks for compiled `RunPlan` persistence/load and runner startup from immutable `RunPlan`. +- [x] Add deterministic checks for the supported CHG-049 first-party workflow-pack beta slice only. +- [x] Add cold and warm microVM startup checks, with cold covering verified-cache miss or trusted-admission cost and warm covering verified local runtime-asset cache-hit cost on the same signed runtime identity. +- [x] Add cold and warm container startup checks for the explicit opt-in backend, with the same verified-cache miss or hit semantics used for microVM startup checks. +- [x] Add attestation cold-path and warm verification-cache checks for the truthful supported runtime path. +- [x] Freeze authoritative timing boundaries for launcher and attestation checks, including `start_event`, `end_event`, `clock_source`, `evidence_source`, and `included_phases`. +- [x] Keep attestation performance contracts in `contract_pending_dependency` until `CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0` lands. ## Phase 5: Gateway, Dependency, Audit, Protocol, And External Anchor Checks -- [ ] Add deterministic model-gateway invoke-overhead and secret-ingress checks using stubbed provider backends. -- [ ] Add deterministic dependency-fetch cache-miss checks using reviewed typed dependency-request fixtures and stubbed public-registry payload sources. -- [ ] Add deterministic dependency-fetch cache-hit checks for already-cached resolved dependency units. -- [ ] Add miss-coalescing checks so identical concurrent dependency requests do not multiply upstream fetch work. -- [ ] Add broker-mediated offline dependency staging or materialization checks for workspace consumption. -- [ ] Add streaming and memory-budget checks to ensure dependency cache fill stays stream-to-CAS rather than full-payload buffering. -- [ ] Add audit verification and finalize-verify runtime checks for deterministic ledger fixtures. -- [ ] Add protocol schema and fixture-parity performance checks. -- [ ] Add deterministic external audit anchor prepare checks against stubbed target descriptors and pre-sealed audit segments. -- [ ] Add deterministic external audit anchor execute checks for both fast-completed and deferred-completion paths. -- [ ] Add deferred-completion visibility checks for external audit anchoring through durable get or watch surfaces. -- [ ] Add external anchor receipt-admission checks for unchanged verified seals so the incremental path is measured explicitly. -- [ ] Add checks ensuring external audit anchoring performance does not reward network I/O under audit-ledger lock or bypass authoritative verifier admission. +- [x] Add deterministic model-gateway invoke-overhead and secret-ingress checks using stubbed provider backends. +- [x] Add deterministic dependency-fetch cache-miss checks using reviewed typed dependency-request fixtures and stubbed public-registry payload sources. +- [x] Add deterministic dependency-fetch cache-hit checks for already-cached resolved dependency units. +- [x] Add miss-coalescing checks so identical concurrent dependency requests do not multiply upstream fetch work. +- [x] Add broker-mediated offline dependency staging or materialization checks for workspace consumption. +- [x] Add streaming and memory-budget checks to ensure dependency cache fill stays stream-to-CAS rather than full-payload buffering. +- [x] Add reviewed bounded-buffer instrumentation for dependency cache fill so stream-to-CAS posture is verified directly in addition to coarse process memory observations. +- [x] Add audit verification and finalize-verify runtime checks for deterministic ledger fixtures. +- [x] Add protocol schema and fixture-parity performance checks. +- [x] Add deterministic external audit anchor prepare checks against stubbed target descriptors and pre-sealed audit segments. +- [x] Add deterministic external audit anchor execute checks for both fast-completed and deferred-completion paths. +- [x] Add deferred-completion visibility checks for external audit anchoring through durable get or watch surfaces. +- [x] Add external anchor receipt-admission checks for unchanged verified seals so the incremental path is measured explicitly. +- [x] Add checks ensuring external audit anchoring performance does not reward network I/O under audit-ledger lock or bypass authoritative verifier admission. +- [x] Freeze authoritative timing boundaries for external audit anchoring, including `start_event`, `end_event`, `clock_source`, `evidence_source`, and `included_phases`. +- [x] Keep external-audit-anchor performance contracts in `contract_pending_dependency` until `CHG-2026-025-5679-external-audit-anchoring-v0` lands. ## Phase 6: CI Integration -- [ ] Add a required Linux PR lane containing the smallest deterministic performance gates with the highest regression value across the MVP beta surface. -- [ ] Keep performance verification check-only and aligned with `just ci` discipline. -- [ ] Store reviewed threshold declarations rather than auto-generated mutable baselines. +- [x] Add a required Linux PR lane containing the smallest deterministic performance gates with the highest regression value across the MVP beta surface. +- [x] Limit the initial required shared-Linux PR lane to metrics declared `required_shared_linux` and keep higher-noise metrics informational or pending until their authority is reviewed. +- [x] Keep performance verification check-only and aligned with `just ci` discipline. +- [x] Store reviewed threshold declarations in the dedicated performance-contract artifacts rather than auto-generated mutable baselines. +- [x] Distinguish metrics stable enough for shared hosted Linux required gates from metrics that may later need a tighter authoritative Linux environment. ## Phase 7: Baseline Governance -- [ ] Define the review process for tightening thresholds or accepting deliberate regressions with explicit justification. -- [ ] Document how to refresh baselines safely when major architectural shifts land. -- [ ] Document which broader performance surfaces are intentionally deferred to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. +- [x] Define the review process for tightening thresholds or accepting deliberate regressions with explicit justification. +- [x] Document how to refresh baselines safely when major architectural shifts land. +- [x] Document the reviewed statistical defaults for microbenchmarks, latency metrics, CPU/process-behavior metrics, and exact metrics. +- [x] Document initial statistical constants for sample counts, warmup windows, p95 eligibility, and repeated-window CPU/process metrics. +- [x] Document the practical noise-floor policy used alongside repeated-sample regression checks. +- [x] Document `threshold_origin` for every threshold as `product_budget`, `investigation_baseline`, `first_calibration`, or `temporary_guardrail`. +- [x] Document which broader performance surfaces are intentionally deferred to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. ## Acceptance Criteria -- [ ] RuneCode has explicit deterministic performance checks for the supported MVP beta surfaces rather than only anecdotal local measurements. -- [ ] The TUI has separate gates for empty-idle and waiting-state behavior. -- [ ] Broker local API requests and watch families have deterministic latency checks for the supported beta fixtures. -- [ ] Runner startup, the supported workflow path, launcher startup, and the truthful attestation path each have at least one deterministic CI-compatible performance check. -- [ ] Model-gateway, dependency-fetch, audit, protocol, and external audit anchoring paths each have at least one deterministic CI-compatible performance check. -- [ ] Linux PR CI enforces numeric thresholds for the highest-value checks. -- [ ] Threshold changes and baseline refreshes require explicit review rather than silent CI mutation. -- [ ] Broader workflow-pack surfaces, git-gateway checks, larger fixture ladders, and tuned macOS or Windows numeric gates are explicitly deferred to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. +- [x] RuneCode has explicit deterministic performance checks for the supported MVP beta surfaces rather than only anecdotal local measurements. +- [x] The TUI has separate gates for empty-idle and waiting-state behavior. +- [x] Broker local API requests and watch families have deterministic latency checks for the supported beta fixtures. +- [x] Runner startup, the supported workflow path, launcher startup, and the truthful attestation path each have at least one deterministic CI-compatible performance check. +- [x] Model-gateway, dependency-fetch, audit, protocol, and external audit anchoring paths each have at least one deterministic CI-compatible performance check. +- [x] Linux PR CI enforces numeric thresholds for the highest-value checks. +- [x] Reviewed performance-contract artifacts remain separate from project-substrate assurance baseline artifacts. +- [x] Each required metric has reviewed lane authority, activation state, fixture ID, threshold origin, and timing-boundary metadata. +- [x] Timing boundaries for attach, workflow, launcher, attestation, dependency, and external-anchor checks terminate on reviewed broker-owned or persisted milestones rather than advisory shortcuts. +- [x] The first implementation slice uses the reviewed statistical defaults captured by this change and tunes them only through explicit follow-up review. +- [x] Threshold changes and baseline refreshes require explicit review rather than silent CI mutation. +- [x] Broader workflow-pack surfaces, git-gateway checks, larger fixture ladders, and tuned macOS or Windows numeric gates are explicitly deferred to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. diff --git a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/verification.md b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/verification.md index 7330845d..45694020 100644 --- a/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/verification.md +++ b/runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/verification.md @@ -19,7 +19,7 @@ - Confirmed the post-fix waiting transcript still rendered a `WAITING session=sess-manual-multiwait` marker, so the CPU improvement did not come from suppressing the operator-visible waiting cue. ## Planned Automated Checks -- `go test ./cmd/runecode-tui -bench 'BenchmarkShell(View|Watch|BuildPaletteEntries)' -benchmem` +- `go test ./cmd/runecode-tui -bench 'Benchmark(ShellViewEmpty|ShellViewWaitingSession|ShellWatchApply|BuildPaletteEntries)' -benchmem` - deterministic PTY-based TUI empty-idle CPU gate - deterministic PTY-based TUI waiting-state CPU gate - deterministic broker unary local API latency suite @@ -33,15 +33,29 @@ - Confirm the change records the most important methodological correction: socket isolation is not broker-store isolation. - Confirm the design captures the profile-backed render and allocation hot spots, not just the top-line CPU numbers. - Confirm performance checks are proposed for the supported MVP beta surfaces rather than the full eventual product surface. +- Confirm the change defines a reviewed performance-contract artifact family separate from `runecontext/assurance/baseline.yaml`. +- Confirm the reviewed performance-contract artifact family lives under `tools/perfcontracts/` with a manifest, per-surface contract files, reviewed fixture inventory, and optional repeated-sample baselines where needed. +- Confirm one trusted repo-local compare/enforce tool exists under `tools/` and does not rewrite baselines during normal CI. +- Confirm the design freezes a metric taxonomy across exact, absolute-budget, regression-budget, and hybrid-budget checks. +- Confirm every metric declares lane authority and activation state before enforcement. +- Confirm the design freezes the reviewed statistical defaults for repeated microbenchmarks, latency metrics, CPU/process-behavior metrics, and exact metrics. +- Confirm statistical constants are defined for sample counts, warmup windows, p95 eligibility, repeated CPU/process windows, and practical noise floors. +- Confirm repeated-sample regression checks use a practical noise-floor policy in addition to significance or interval-based comparison logic. +- Confirm authoritative timing boundaries declare `start_event`, `end_event`, `clock_source`, `evidence_source`, and `included_phases`, and terminate on reviewed broker-owned or persisted milestones rather than advisory launcher-local or client-local proxies when reviewed downstream authority surfaces exist. +- Confirm the initial reviewed fixture inventory uses stable fixture IDs before baselines are collected. +- Confirm thresholds declare `threshold_origin` as `product_budget`, `investigation_baseline`, `first_calibration`, or `temporary_guardrail`. - Confirm the refined CHG-050 workflow path is measured explicitly, including validation or canonicalization, trusted compilation, compiled-plan persistence/load, and runner startup from immutable `RunPlan`. - Confirm the supported CHG-049 workflow-pack beta slice is measured explicitly while broader workflow-pack surfaces are deferred. - Confirm dependency-fetch and offline-cache have explicit cold-cache, warm-cache, miss-coalescing, and materialization checks. -- Confirm dependency-fetch performance checks preserve the reviewed stream-to-CAS and bounded-memory posture rather than rewarding trust-boundary shortcuts. +- Confirm dependency-fetch performance checks preserve the reviewed stream-to-CAS and bounded-memory posture rather than rewarding trust-boundary shortcuts, and that bounded-buffer instrumentation exists in addition to coarse process-memory observation. - Confirm external audit anchoring has explicit checks for prepare latency, execute-completed latency, execute-deferred handoff latency, deferred completion visibility, and receipt admission over an unchanged verified seal. - Confirm external audit anchoring performance checks do not reward forbidden shortcuts such as network I/O under audit-ledger lock, bypassing authoritative verifier admission, or forcing full verifier replay as the only normal receipt-admission path. - Confirm launcher cold and warm startup checks are defined in terms of the signed runtime-asset path. - Confirm attestation cold and warm checks are defined in terms of the required attestation trust path. +- Confirm attestation performance contracts remain `contract_pending_dependency` until `CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0` lands. - Confirm launcher and attach-ready performance checks do not reward bypassing attestation verification, replay checks, freshness checks, or attestation evidence persistence. +- Confirm external-audit-anchor performance contracts remain `contract_pending_dependency` until `CHG-2026-025-5679-external-audit-anchoring-v0` lands. +- Confirm the first required Linux lane is scoped to metrics stable enough for shared hosted Linux thresholds, while leaving room to promote selected higher-noise metrics later without changing metric identity or product architecture. - Confirm the roadmap places this work under `v0.1.0-alpha.11`. - Confirm broader workflow-pack expansion, git-gateway expansion, larger fixture ladders, and tuned macOS or Windows numeric gates are deferred to `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0`. - Confirm the change keeps performance verification check-only and CI-safe. diff --git a/runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/status.yaml b/runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/status.yaml index a43ccd93..224e0084 100644 --- a/runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/status.yaml +++ b/runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/status.yaml @@ -14,6 +14,8 @@ related_changes: - CHG-2026-026-98be-image-toolchain-signing-pipeline - CHG-2026-030-98b8-isolate-attestation-v0 - CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0 + - CHG-2026-053-9d2b-performance-baselines-verification-gates-v0 + - CHG-2026-060-c1a4-beta-readiness-hardening-product-polish depends_on: - CHG-2026-009-1672-launcher-microvm-backend-v0 - CHG-2026-026-98be-image-toolchain-signing-pipeline diff --git a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/design.md b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/design.md index e3e60906..a25d2351 100644 --- a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/design.md +++ b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/design.md @@ -9,6 +9,8 @@ The central rule of this lane is: RuneCode should not claim beta readiness until one real workflow path runs through the real trusted and untrusted execution path, produces inspectable artifacts and audit evidence, and remains understandable to an operator using the normal product surfaces. +This lane also coordinates directly with `CHG-2026-053-9d2b-performance-baselines-verification-gates-v0` for the surfaces that beta users actually experience. Dogfooding-driven polish must improve those surfaces without making them less authoritative or less measurable. + ## Scope This lane covers five connected concerns: @@ -103,6 +105,12 @@ This lane should capture polish work discovered while testing the real workflow The TUI is the highest-priority polish surface because it is the normal user-facing shell for the local product. +That polish should stay aligned with the reviewed performance-contract discipline in `CHG-053`, especially: + +- attach, reconnect, and resume surfaces should continue to reflect broker-owned lifecycle truth rather than client-local optimistic shortcuts +- waiting, blocked, degraded, and failed states should remain operator-visible without reintroducing misleading high-activity rendering paths or synthetic progress cues +- dogfooding fixes should preserve the same authoritative surfaces that the MVP performance gates measure rather than optimizing around those gates with less truthful UI behavior + ## Verification Smoke Path This lane should require that the real workflow path also exercises the verification surfaces already present in the repository. @@ -135,3 +143,4 @@ This alpha lane is complete when: - verification artifacts are generated and exercised from that real workflow path - TUI and surrounding operator surfaces are polished enough that a new user can test the product coherently on Linux - the beta story is honest about assurance and execution behavior +- workflow-path and TUI polish remain compatible with the authoritative surfaces and honest measurement boundaries frozen by `CHG-053` diff --git a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/proposal.md b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/proposal.md index 6f4ada53..ef185e10 100644 --- a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/proposal.md +++ b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/proposal.md @@ -25,6 +25,7 @@ Without a dedicated alpha hardening lane, RuneCode risks declaring beta too earl - Track the production adoption of trusted `RunPlan` compilation rather than leaving it as a largely test-proven foundation seam. - Track the replacement of effectively noop runner transport defaults with real broker integration in the actual workflow path. - Fold in the truthful attestation-posture correction from `CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0` as a required pre-beta hardening companion. +- Keep workflow-path dogfooding, TUI polish, and operator-truth improvements aligned with the reviewed performance-contract discipline in `CHG-2026-053-9d2b-performance-baselines-verification-gates-v0`, especially around attach/resume truth, waiting-state communication, and avoidance of client-local shortcuts that would make measured product surfaces less honest. - Explicitly capture TUI and operator polish discovered while dogfooding the real workflow path, especially around run state clarity, attach or reconnect behavior, remediation cues, approval and audit discoverability, and overall usability. - Align roadmap and product-surface messaging with the real shipped state once the honest workflow path exists. - Require the alpha lane to exercise verification artifacts on the real workflow path so beta ships with strong evidence continuity instead of a later degraded verification posture. @@ -46,6 +47,7 @@ That split is easier to reason about than continuing to leave integration and po - RuneCode should ship beta only when at least one real useful workflow runs through the honest trusted and untrusted execution path and is inspectable through the normal product surfaces. - TUI and product polish discovered while dogfooding are legitimate alpha hardening work and should be planned explicitly rather than treated as incidental cleanup. - Verification artifacts generated from the real workflow path must remain first-class deliverables of this lane so later verification work strengthens rather than backfills the beta story. +- Product polish in this lane must improve operator clarity without undermining the authoritative broker-owned and persisted surfaces that `CHG-053` measures and protects. ## Out of Scope - Replacing the broader beta milestone with a new version target. diff --git a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/status.yaml b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/status.yaml index f3a36c87..2702d410 100644 --- a/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/status.yaml +++ b/runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/status.yaml @@ -11,6 +11,7 @@ context_bundles: related_specs: [] related_decisions: [] related_changes: + - CHG-2026-053-9d2b-performance-baselines-verification-gates-v0 - CHG-2026-045-7f4c-direct-credential-model-providers-v0 - CHG-2026-046-a91d-runecontext-verified-project-substrate-compatibility-lifecycle-v0 - CHG-2026-047-c3e2-local-control-plane-bootstrap-persistent-session-lifecycle-v0 diff --git a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/design.md b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/design.md index fe935100..36c56ee8 100644 --- a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/design.md +++ b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/design.md @@ -10,6 +10,18 @@ The design goal is to preserve the MVP gate set as a stable release contract whi - larger broker and end-to-end fixture tiers - tuned cross-platform gates beyond Linux-first numeric enforcement +## Inherited Contract From CHG-053 + +This change extends the `CHG-053` performance foundation rather than redefining it. + +That means the post-MVP expansion should continue to use: + +- the reviewed performance-contract artifact family rather than a second baseline storage format +- the `CHG-053` metric taxonomy across exact, absolute-budget, regression-budget, and hybrid-budget checks unless explicitly refined by later reviewed work +- the reviewed `CHG-053` statistical defaults as the starting point for broader measurement classes +- the `CHG-053` timing-boundary rule that metrics terminate on reviewed broker-owned or persisted milestones whenever authoritative downstream surfaces exist +- the same topology-neutral architecture rule across constrained local devices and larger deployments + ## Layer Boundary ### Layer 1: MVP Beta Gates @@ -31,6 +43,8 @@ Owned by this change: - larger broker-fixture ladders and heavier extended-Linux measurements - tuned macOS and Windows numeric gates where feasible +Layer 2 expands breadth and confidence. It does not introduce a second semantics model for thresholds, baselines, timing boundaries, or trust ownership. + ## Broader Workflow-Pack Coverage The post-MVP workflow-pack expansion should cover surfaces that are useful but were intentionally excluded from the MVP hard gate, such as: @@ -43,6 +57,8 @@ The post-MVP workflow-pack expansion should cover surfaces that are useful but w These checks should remain deterministic and should continue to measure the same broker-owned immutable `RunPlan` architecture rather than an alternate fast path. +Where broader workflow-pack checks add new timings, those timings should still terminate on reviewed broker-owned or persisted milestones rather than direct CLI-local proxies when authoritative downstream surfaces exist. + ## Git Gateway And Project-Substrate Coverage This expansion lane should add explicit performance coverage for surfaces that are implemented and important, but not required in the first beta hard gate: @@ -53,6 +69,8 @@ This expansion lane should add explicit performance coverage for surfaces that a These checks should remain local-only and deterministic where possible. +Where git-gateway and project-substrate paths add exact counts, latency budgets, or regression budgets, they should use the same metric taxonomy and reviewed statistical defaults inherited from `CHG-053`. + ## Larger Fixture Ladders And Heavier Extended Lanes The MVP gate set intentionally avoids overloading the first release with the heaviest fixture program. This change should add: @@ -63,6 +81,8 @@ The MVP gate set intentionally avoids overloading the first release with the hea The goal is to increase confidence at scale without turning the first beta PR lane into a noisy bottleneck. +This expansion should treat larger fixture ladders as a broadening of the reviewed MVP fixture inventory, not as permission to abandon the deterministic fixture discipline established by `CHG-053`. + ## Cross-Platform Expansion Linux remains the first authoritative numeric gate. This change is where cross-platform performance work becomes more ambitious. @@ -82,6 +102,8 @@ As Windows runtime support matures, add the same flow families where feasible an Cross-platform expansion must preserve one topology-neutral architecture rather than implying platform-local authority shortcuts. +It must also preserve one performance-contract model across platforms. Tuned thresholds and lane promotion may differ by environment, but artifact shape, metric semantics, timing-boundary discipline, and trust ownership should stay aligned. + ## CI Integration Shape This change should favor: @@ -89,10 +111,14 @@ This change should favor: - macOS and Windows smoke or trend lanes first - gradual promotion of stable flow families into numeric-gated cross-platform lanes only after noise and baseline quality are understood +Selected higher-noise metrics may also be promoted to tighter authoritative Linux measurement environments if shared Linux CI proves too noisy, but that promotion should be treated as lane refinement rather than as a new product architecture or new metric identity. + Threshold storage and baseline governance should stay review-driven and check-only. ## Design Risks To Avoid - Do not let broader expansion erode the usefulness of the MVP gate set. - Do not add flaky or externally networked checks. - Do not treat cross-platform numeric tuning as a substitute for actual platform readiness. +- Do not introduce a second baseline artifact family or a second metric semantics model for post-MVP checks. +- Do not terminate broader timings at advisory client-local milestones when reviewed broker-owned or persisted milestones exist downstream. - Do not reward trust-path bypasses just because they improve a benchmark number. diff --git a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/proposal.md b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/proposal.md index 3630476d..0e6b34e9 100644 --- a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/proposal.md +++ b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/proposal.md @@ -13,11 +13,16 @@ Without a separate post-MVP change, those deferred surfaces would either drift w ## Proposed Change - Create one post-MVP performance-expansion lane that extends the MVP gate foundation from `CHG-2026-053-9d2b-performance-baselines-verification-gates-v0`. +- Reuse the reviewed performance-contract artifact family introduced by `CHG-053` rather than creating a second baseline or threshold declaration format. +- Reuse the `CHG-053` metric taxonomy across exact, absolute-budget, regression-budget, and hybrid-budget checks unless a later reviewed follow-up deliberately refines that taxonomy. +- Reuse the `CHG-053` statistical defaults as the starting point for broader post-MVP checks, including repeated-sample robust comparison for microbenchmarks, median plus `p95` plus explicit ceilings for latency metrics, fixed-window average/median plus max guardrails for CPU/process-behavior metrics, and exact checks for deterministic invariant counts. +- Reuse the `CHG-053` timing-boundary rule so broader checks still terminate on reviewed broker-owned or persisted milestones rather than advisory client-local or launcher-local heuristics when authoritative downstream surfaces exist. - Add explicit measurement of broader CHG-049 first-party workflow-pack surfaces, including draft artifact generation, draft promote/apply, reviewed implementation-input-set validation or binding, direct CLI workflow triggering, repo-scoped admission control or idempotency, and fail-closed drift-triggered re-evaluation or recompilation costs when those surfaces are part of the supported product story. - Add explicit performance checks for git-gateway and broader project-substrate paths when those surfaces become part of the supported user workflow. -- Add larger broker-fixture ladders and heavier extended-Linux measurements that improve confidence beyond the first beta release-defining fixtures. +- Expand from the reviewed MVP fixture inventory to larger broker-fixture ladders and heavier extended-Linux measurements that improve confidence beyond the first beta release-defining fixtures. - Expand cross-platform performance verification from Linux-first smoke or trend collection toward tuned macOS and Windows numeric gates where feasible. -- Keep performance verification deterministic, CI-safe, and aligned with the same trust-boundary rules and broker-owned authority model as correctness checks. +- Keep performance verification deterministic, CI-safe, and aligned with the same trust-boundary rules, broker-owned authority model, and topology-neutral architecture rule as correctness checks. +- Allow selected higher-noise metrics to be promoted to tighter authoritative Linux environments later if needed, but treat that as measurement-infrastructure refinement rather than a product-architecture fork. ## Why Now Splitting this work out now preserves a clean contract: @@ -30,6 +35,7 @@ That lets the first beta ship with serious performance discipline while still pr ## Assumptions - The MVP gate set from `CHG-053` lands first and becomes the baseline for future expansion. - Broader workflow-pack surfaces and project-substrate or git-gateway flows are important to measure, but they should not redefine the first beta gate set retroactively. +- The reviewed performance-contract artifact family, metric taxonomy, statistical defaults, and authoritative timing-boundary rules from `CHG-053` remain the starting contract for this expansion lane. - Tuned macOS and Windows numeric gates should follow the relevant platform runtime and virtualization work rather than assuming Linux measurements transfer directly. - Larger fixtures and heavier extended lanes are valuable for post-MVP confidence, but they should remain deterministic and CI-safe. @@ -47,5 +53,6 @@ If completed, RuneCode will gain a cleaner post-MVP path for: - git-gateway and broader project-substrate performance coverage - larger fixture tiers and heavier extended lanes - tuned macOS and Windows numeric gates beyond the Linux-first baseline +- broader coverage that still reuses the same reviewed artifact model, metric semantics, statistical defaults, and authoritative timing-boundary rules established in `CHG-053` That preserves the value of the MVP beta gates while keeping the larger performance program visible and intentional. diff --git a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/standards.md b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/standards.md index 1aefbf2e..57c5edd6 100644 --- a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/standards.md +++ b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/standards.md @@ -18,6 +18,7 @@ That includes freezing the following clarifications for post-MVP work: - git-gateway and broader project-substrate performance checks should remain deterministic and local-first where feasible - larger fixture ladders and heavier extended lanes are valuable, but should not destabilize the MVP PR gate - broader macOS and Windows numeric tuning should remain explicit follow-on work rather than implied parity with Linux before the platform lanes are ready +- post-MVP expansion should keep using the reviewed performance-contract artifacts, metric taxonomy, statistical defaults, and authoritative timing-boundary rules established by `CHG-053` unless a later reviewed change deliberately revises them - threshold updates and baseline refreshes still require explicit review rather than silent CI mutation This change extends the MVP performance foundation from `CHG-053` rather than redefining RuneCode's trust or control-plane contracts. diff --git a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/tasks.md b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/tasks.md index 1affef49..4832a8e5 100644 --- a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/tasks.md +++ b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/tasks.md @@ -2,6 +2,8 @@ ## Phase 1: Broader Workflow-Pack Coverage +- [ ] Reuse the reviewed performance-contract artifact family from `CHG-053` rather than defining a second baseline format for post-MVP checks. +- [ ] Reuse the `CHG-053` metric taxonomy and statistical defaults as the starting policy for broader checks unless later reviewed work explicitly refines them. - [ ] Add deterministic checks for broader CHG-049 workflow-pack surfaces beyond the supported MVP beta slice. - [ ] Add deterministic draft artifact-generation checks where those surfaces are part of the supported post-MVP product story. - [ ] Add deterministic draft promote/apply checks for canonical RuneContext mutation through the shared audited path. @@ -9,6 +11,7 @@ - [ ] Add deterministic direct CLI workflow-trigger latency checks for broader workflow-pack entry points. - [ ] Add deterministic repo-scoped admission-control and idempotency checks for broader workflow trigger paths. - [ ] Add deterministic fail-closed re-evaluation or recompilation checks for project-context or approved-input drift on broader workflow-pack paths. +- [ ] Freeze authoritative timing boundaries for broader workflow-pack checks so they still terminate on reviewed broker-owned or persisted milestones. ## Phase 2: Git Gateway And Project-Substrate Expansion @@ -16,6 +19,7 @@ - [ ] Add git execute performance checks against deterministic local bare remotes. - [ ] Add project-substrate posture and preview performance checks for deterministic fixture repos. - [ ] Add local project-substrate apply performance checks for deterministic fixture repos. +- [ ] Apply the inherited metric taxonomy and authoritative timing-boundary rules to git-gateway and project-substrate checks. ## Phase 3: Larger Fixture Ladders And Heavier Lanes @@ -24,6 +28,7 @@ - [ ] Add heavier workflow execution fixtures for extended Linux lanes. - [ ] Add heavier audit-ledger and verification fixtures for extended Linux lanes. - [ ] Keep heavier lanes deterministic and suitable for merge-queue or scheduled execution. +- [ ] Treat larger fixture ladders as expansion from the reviewed MVP fixture inventory rather than as a separate fixture model. ## Phase 4: Cross-Platform Expansion @@ -31,12 +36,14 @@ - [ ] Tune stable macOS numeric thresholds where fixture noise and platform behavior are understood. - [ ] Tune stable Windows numeric thresholds where fixture noise and platform behavior are understood. - [ ] Preserve Linux as the first authoritative numeric gate until the broader cross-platform program is genuinely stable. +- [ ] Promote selected higher-noise metrics to tighter authoritative Linux environments only if shared Linux CI proves insufficient, without changing metric identity or product architecture. ## Phase 5: Governance - [ ] Document the promotion path from smoke or trend collection to numeric-gated cross-platform checks. - [ ] Document the review process for broadening performance coverage without weakening the MVP gate set. - [ ] Keep threshold updates and baseline refreshes review-driven and check-only. +- [ ] Keep threshold storage, metric semantics, statistical defaults, and timing-boundary rules aligned with the inherited `CHG-053` performance contract unless explicitly revised by reviewed follow-up work. ## Acceptance Criteria @@ -44,4 +51,5 @@ - [ ] Git-gateway and broader project-substrate paths each have at least one deterministic CI-compatible performance check. - [ ] Larger fixture ladders and heavier extended-Linux lanes exist without destabilizing the MVP beta PR gate. - [ ] macOS and Windows run the same flow families where feasible, with tuned numeric gates added only where stable and meaningful. +- [ ] The broader performance program reuses the `CHG-053` performance-contract artifacts, metric taxonomy, statistical defaults, and authoritative timing-boundary rules unless explicitly revised through later reviewed work. - [ ] The broader performance program remains aligned with the same trust-boundary and broker-owned authority model as the MVP gate set. diff --git a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/verification.md b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/verification.md index 45ce9bb4..408911c0 100644 --- a/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/verification.md +++ b/runecontext/changes/CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0/verification.md @@ -8,8 +8,11 @@ ## Verification Notes - Confirm the roadmap places this change under `v0.2 (Post-MVP)`. - Confirm `CHG-053` remains the MVP beta gate set and this change is explicitly additive over it. +- Confirm the change explicitly reuses the `CHG-053` performance-contract artifact family rather than introducing a second baseline format. +- Confirm the change explicitly reuses the `CHG-053` metric taxonomy, statistical defaults, and authoritative timing-boundary rules as the starting post-MVP contract. - Confirm the proposal captures broader CHG-049 workflow-pack surfaces, git-gateway and broader project-substrate paths, larger fixture ladders, and tuned cross-platform gates as the main deferred layer. - Confirm the design keeps Linux as the first authoritative numeric gate while allowing broader macOS and Windows work to grow in a controlled way. +- Confirm larger fixture ladders are framed as an expansion of the reviewed MVP fixture inventory rather than a second fixture model. - Confirm the tasks keep performance verification deterministic, CI-safe, and review-driven. - Confirm the change does not weaken the MVP gate set by silently moving required beta checks out of `CHG-053`. diff --git a/runecontext/project/roadmap.md b/runecontext/project/roadmap.md index 3ab948f4..804a222e 100644 --- a/runecontext/project/roadmap.md +++ b/runecontext/project/roadmap.md @@ -7,9 +7,6 @@ Active lifecycle state lives in `runecontext/changes/*/status.yaml`, and durable ### v0.1.0-alpha.11 -- Project Performance Baselines + Verification Gates v0 - - RuneCode establishes the first MVP-grade performance baselines and deterministic Linux-first CI gates for the supported beta surface: TUI idle and waiting behavior, broker APIs and watch families, the supported workflow path, launcher startup with the truthful attestation path, gateway overhead, audit and protocol verification, external audit anchoring, and end-to-end attach or resume flows. - - Planned change: `runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/` - Beta Readiness Hardening + Product Polish - RuneCode closes the remaining product-integration gaps before beta by wiring one honest useful end-to-end workflow through the real trusted and untrusted execution path, tightening operator truth surfaces, and capturing dogfooding-driven polish across the local product UX with emphasis on the TUI. - Project change: `runecontext/changes/CHG-2026-060-c1a4-beta-readiness-hardening-product-polish/` @@ -189,4 +186,7 @@ Active lifecycle state lives in `runecontext/changes/*/status.yaml`, and durable - Runtime Attestation Post-Handshake Gating v0 - RuneCode only awards supported `attested` posture after a live runtime completes secure-session validation and post-handshake trusted runtime-proof verification, closing the remaining gap between the reviewed attestation design and launch-time implementation order before beta assurance claims are treated as settled. - Planned change: `runecontext/changes/CHG-2026-054-6c1e-runtime-attestation-post-handshake-gating-v0/` +- Project Performance Baselines + Verification Gates v0 + - RuneCode establishes the first MVP-grade performance baselines and deterministic Linux-first CI gates for the supported beta surface. The required shared-Linux lane enforces the current `required_shared_linux` subset, while launcher startup/attestation and external audit anchoring contracts are tracked as informational or `contract_pending_dependency` until their dependency paths are fully landed. + - Planned change: `runecontext/changes/CHG-2026-053-9d2b-performance-baselines-verification-gates-v0/` diff --git a/runecontext/project/standards-inventory.md b/runecontext/project/standards-inventory.md index f8971357..e476fa34 100644 --- a/runecontext/project/standards-inventory.md +++ b/runecontext/project/standards-inventory.md @@ -27,6 +27,7 @@ Recent notable standards for this branch: - `runecontext/standards/security/runner-durable-state-and-replay.md` - `runecontext/standards/security/runtime-image-signing-admission-and-verified-cache.md` - `runecontext/standards/security/trusted-runtime-evidence-and-broker-projection.md` +- `runecontext/standards/testing/performance-contract-governance.md` Recent additions should be reflected here when they become durable cross-cutting guidance rather than change-local design notes. diff --git a/runecontext/standards/ci/just-ci.md b/runecontext/standards/ci/just-ci.md index 9485d00f..07f64877 100644 --- a/runecontext/standards/ci/just-ci.md +++ b/runecontext/standards/ci/just-ci.md @@ -9,23 +9,29 @@ suggested_context_bundles: # `just ci` Convention -- `just ci` is the canonical local+CI parity command +- `just ci` is the canonical local check entrypoint +- CI may use `just ci-fast` plus dedicated required gates when a heavyweight check needs path-aware or merge-queue scheduling +- Required shared-Linux performance contracts run in the dedicated CI lane (`just ci-required-shared-linux`) rather than every local `just ci` run - `just ci` is check-only: - No formatters in write mode - No lockfile updates (`flake.lock`, `go.sum`, `package-lock.json`) - Put auto-fix behavior in separate recipes (example: `just fmt`) - Put explicit repair workflows that change tracked files in separate recipes or tools rather than inside `just ci` (example: `just refresh-release-vendor-hash`) -- Put formal model checking behind an explicit check-only recipe (currently `just model-check`) and include it in `just ci` when it is part of required parity +- Put formal model checking behind explicit check-only recipes (`just model-check-core`, `just model-check-replay`, `just model-check`) and include full model checking in `just ci` for local parity +- In GitHub CI, keep the formal security-kernel check as a dedicated required gate so PR pushes can run the core model for security-kernel-relevant code or protocol changes, run the full model for formal-spec/tooling/workflow changes, and run the full model on merge queue and `main` - Keep recipes cross-platform (Windows-friendly): avoid bash/unix-only tools and shell pipelines - Redundant explicit steps in `just ci` are allowed when they make failures clearer (example: runner lint even if tests also run lint) ```make ci: + just ci-fast + just model-check + +ci-fast: go run ./tools/gofmtcheck go run github.com/golangci/golangci-lint/cmd/golangci-lint@... go vet ./... go run ./tools/checksourcequality - just model-check go test ./... go build ./cmd/... cd runner && npm ci diff --git a/runecontext/standards/ci/nix-flake-ci-invariants.md b/runecontext/standards/ci/nix-flake-ci-invariants.md index fca53df0..e0eb1f9d 100644 --- a/runecontext/standards/ci/nix-flake-ci-invariants.md +++ b/runecontext/standards/ci/nix-flake-ci-invariants.md @@ -25,5 +25,5 @@ env: steps: - run: nix flake lock --no-update-lock-file - run: nix flake check --no-write-lock-file - - run: nix develop --no-write-lock-file -c just ci + - run: nix develop --no-write-lock-file -c just ci-fast ``` diff --git a/runecontext/standards/ci/windows-portability-matrix.md b/runecontext/standards/ci/windows-portability-matrix.md index ee6d4684..451a69a2 100644 --- a/runecontext/standards/ci/windows-portability-matrix.md +++ b/runecontext/standards/ci/windows-portability-matrix.md @@ -13,7 +13,7 @@ suggested_context_bundles: - Windows CI runs `just ci-portability` under PowerShell (no bash dependency) - Test Node "min + max" versions within `runner/package.json` `engines` (pin exact versions) - Pin Windows job tooling versions for reproducibility (Go, Node, just, gopls, baseline CLIs) -- Keep one canonical TLC/model-check gate in a single CI lane (currently Linux via `just ci`), and keep Windows focused on portability checks that do not depend on TLC runtime provisioning +- Keep the canonical TLC/model-check gate on Linux in a dedicated formal-security CI job, and keep Windows focused on portability checks that do not depend on TLC runtime provisioning - Keep failure-path tests portable: do not rely on POSIX-only chmod or permission semantics when a deterministic injected failure seam can exercise the same rollback or cleanup path on Windows ```yaml diff --git a/runecontext/standards/testing/performance-contract-governance.md b/runecontext/standards/testing/performance-contract-governance.md new file mode 100644 index 00000000..f2947595 --- /dev/null +++ b/runecontext/standards/testing/performance-contract-governance.md @@ -0,0 +1,25 @@ +--- +schema_version: 1 +id: testing/performance-contract-governance +title: Performance Contract Governance +status: active +suggested_context_bundles: + - ci-tooling +--- + +# Performance Contract Governance + +Use `tools/perfcontracts/manifest.json` as the authoritative inventory for checked-in performance contracts and reviewed baselines. + +- Keep performance contracts separate from `runecontext/assurance/baseline.yaml` +- Keep CI check-only: verification must never auto-rewrite performance baselines +- Require explicit `threshold_origin` per threshold: `product_budget | investigation_baseline | first_calibration | temporary_guardrail` +- Require explicit timing boundaries (`start_event`, `end_event`, `clock_source`, `evidence_source`, `included_phases`) for every metric +- Treat the manifest baseline entry for each metric as authoritative: required `regression-budget` and `hybrid-budget` metrics must point `baseline_ref` at the exact path registered in `tools/perfcontracts/manifest.json` +- Reject duplicate `metric_id` entries in manifest baselines; provenance must never depend on last-write-wins manifest ordering +- Treat required enforcement as the intersection of reviewed `lane_authority` and `activation_state: required`; defined, informational, and `contract_pending_dependency` metrics stay outside required numeric enforcement +- Keep the shared-Linux required lane truthful: it enforces only the current checked-in `required_shared_linux` subset, while broader surfaces may remain informational or `contract_pending_dependency` +- Keep perf-tool diagnostics sanitized: do not leak sensitive local paths, tokens, or raw startup output in check failures +- Keep measurement boundaries honest: validate fixture or path preconditions before timing, measure fresh-process startup or attach when startup cost is in scope, and preserve the authoritative timing source when a script or tool emits the measurement directly +- Treat baseline refresh as explicit reviewed change; do not hide threshold loosening in silent baseline updates +- Keep broader fixture ladders and cross-platform expansion in `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0` diff --git a/runner/scripts/perf-runner-workflow.js b/runner/scripts/perf-runner-workflow.js new file mode 100644 index 00000000..f56ce23a --- /dev/null +++ b/runner/scripts/perf-runner-workflow.js @@ -0,0 +1,139 @@ +#!/usr/bin/env node + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { performance } = require("node:perf_hooks"); + +const repoRoot = path.resolve(__dirname, "..", ".."); + +async function loadRunner() { + return import("../src/index.ts"); +} + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + if (!token.startsWith("--")) { + continue; + } + const key = token.slice(2); + const value = argv[i + 1]; + if (value === undefined || value.startsWith("--")) { + out[key] = ""; + continue; + } + out[key] = value; + i += 1; + } + return out; +} + +async function loadPlan(runplanPath) { + const { ProtocolSchemaBundle, RunPlanLoader } = await loadRunner(); + const schemaBundle = await ProtocolSchemaBundle.fromProtocolSchemasRoot(path.join(repoRoot, "protocol", "schemas")); + const loader = new RunPlanLoader(schemaBundle); + const resolvedRunplanPath = fs.realpathSync(path.resolve(runplanPath)); + const tmpRoot = path.resolve(os.tmpdir()); + if (!resolvedRunplanPath.startsWith(`${tmpRoot}${path.sep}`) && !resolvedRunplanPath.startsWith(`${repoRoot}${path.sep}`)) { + throw new Error("--runplan must resolve under the repository root or system temp directory"); + } + const raw = fs.readFileSync(resolvedRunplanPath, "utf8"); + const parsed = JSON.parse(raw); + return loader.loadFromUnknown(parsed); +} + +async function runMode(mode, runplanPath, fixtureID) { + const { PlanScheduler } = await loadRunner(); + const plan = await loadPlan(runplanPath); + const scheduler = new PlanScheduler(); + const normalizedFixtureID = String(fixtureID || "").trim(); + + const expectFirstPartyMinimalFixture = () => { + if (normalizedFixtureID !== "workflow.first-party-minimal.v1") { + throw new Error(`mode ${mode} requires --fixture workflow.first-party-minimal.v1`); + } + if (String(plan.workflow_id || "").trim() !== "workflow_first_party_minimal") { + throw new Error(`mode ${mode} requires workflow_id workflow_first_party_minimal`); + } + if (String(plan.process_id || "").trim() !== "process_first_party_minimal") { + throw new Error(`mode ${mode} requires process_id process_first_party_minimal`); + } + }; + + switch (mode) { + case "cold-start": { + const start = performance.now(); + const work = scheduler.listPlannedWork(plan); + if (!Array.isArray(work) || work.length === 0) { + throw new Error("cold-start failed: no planned work"); + } + return Math.max(0, Math.round(performance.now() - start)); + } + case "workflow-path": { + expectFirstPartyMinimalFixture(); + const start = performance.now(); + const blocked = scheduler.listPlannedWork(plan, { + pending_approval_waits: [{ blocked_scope: { scope_kind: "run", run_id: plan.run_id } }], + }); + if (!Array.isArray(blocked) || blocked.length !== 0) { + throw new Error("workflow-path failed: expected wait-scoped blocking"); + } + const first = scheduler.listPlannedWork(plan, { pending_approval_waits: [], completed_entry_ids: [] }); + if (!Array.isArray(first) || first.length === 0) { + throw new Error("workflow-path failed: no schedulable work on supported path"); + } + const completed = new Set(first.map((w) => w.entry.entry_id)); + const second = scheduler.listPlannedWork(plan, { pending_approval_waits: [], completed_entry_ids: [...completed] }); + if (!Array.isArray(second)) { + throw new Error("workflow-path failed: invalid scheduler result"); + } + return Math.max(0, Math.round(performance.now() - start)); + } + case "first-party-beta": { + expectFirstPartyMinimalFixture(); + const start = performance.now(); + const work = scheduler.listPlannedWork(plan, { pending_approval_waits: [] }); + if (work.length < 1) { + throw new Error("first-party-beta failed: no schedulable entry"); + } + if (work[0]?.entry?.entry_id !== "quality_lint" || work[0]?.entry?.entry_kind !== "gate") { + throw new Error("first-party-beta failed: fixture does not match supported first-party beta slice"); + } + return Math.max(0, Math.round(performance.now() - start)); + } + case "immutable-startup": { + const start = performance.now(); + const serialized = JSON.stringify(plan); + const roundtrip = JSON.parse(serialized); + if (roundtrip.plan_id !== plan.plan_id) { + throw new Error("immutable-startup failed: plan roundtrip mismatch"); + } + const work = scheduler.listPlannedWork(roundtrip, { pending_approval_waits: [] }); + if (work.length === 0) { + throw new Error("immutable-startup failed: no planned work"); + } + return Math.max(0, Math.round(performance.now() - start)); + } + default: + throw new Error(`unsupported --mode ${mode}`); + } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const mode = String(args.mode || "").trim(); + const runplanPath = String(args.runplan || "").trim(); + const fixtureID = String(args.fixture || "").trim(); + if (!mode || !runplanPath) { + throw new Error("--mode and --runplan are required"); + } + const wallMs = await runMode(mode, runplanPath, fixtureID); + process.stdout.write(`${wallMs}\n`); +} + +main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exit(1); +}); diff --git a/runner/scripts/perf-runner-workflow.test.js b/runner/scripts/perf-runner-workflow.test.js new file mode 100644 index 00000000..3479ed2f --- /dev/null +++ b/runner/scripts/perf-runner-workflow.test.js @@ -0,0 +1,196 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const test = require("node:test"); +const { spawnSync } = require("node:child_process"); + +const scriptPath = path.join(__dirname, "perf-runner-workflow.js"); + +function writeRunPlan(root, overrides = {}) { + const runPlan = { + schema_id: "runecode.protocol.v0.RunPlan", + schema_version: "0.4.0", + plan_id: "plan_workflow_first_party_minimal", + run_id: "run_workflow_first_party_minimal", + workflow_id: "workflow_first_party_minimal", + workflow_version: "1.0.0", + process_id: "process_first_party_minimal", + approval_profile: "moderate", + autonomy_posture: "balanced", + workflow_definition_hash: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + process_definition_hash: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + policy_context_hash: "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + compiled_at: "2026-01-01T00:00:00Z", + role_instance_ids: ["role_alpha"], + executor_bindings: [{ + binding_id: "binding_alpha", + executor_id: "executor_alpha", + executor_class: "workspace_ordinary", + allowed_role_kinds: ["developer"], + }], + gate_definitions: [{ + schema_id: "runecode.protocol.v0.GateDefinition", + schema_version: "0.2.0", + gate: { + schema_id: "runecode.protocol.v0.GateContract", + schema_version: "0.1.0", + gate_id: "lint", + gate_kind: "lint", + gate_version: "0.1.0", + normalized_inputs: [], + plan_binding: { checkpoint_code: "quality", order_index: 0 }, + retry_semantics: { retry_mode: "new_attempt_required", max_attempts: 2 }, + override_semantics: { override_mode: "policy_action_required", action_kind: "action_gate_override", approval_trigger_code: "gate_override" }, + }, + checkpoint_code: "quality", + order_index: 0, + stage_id: "quality_stage", + step_id: "quality_lint", + role_instance_id: "role_alpha", + executor_binding_id: "binding_alpha", + dependency_cache_handoffs: [{ + request_digest: { hash_alg: "sha256", hash: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, + consumer_role: "workspace", + required: true, + }], + }], + dependency_edges: [], + entries: [{ + entry_id: "quality_lint", + entry_kind: "gate", + order_index: 0, + stage_id: "quality_stage", + step_id: "quality_lint", + role_instance_id: "role_alpha", + executor_binding_id: "binding_alpha", + checkpoint_code: "quality", + gate: { + schema_id: "runecode.protocol.v0.GateContract", + schema_version: "0.1.0", + gate_id: "lint", + gate_kind: "lint", + gate_version: "0.1.0", + normalized_inputs: [], + plan_binding: { checkpoint_code: "quality", order_index: 0 }, + retry_semantics: { retry_mode: "new_attempt_required", max_attempts: 2 }, + override_semantics: { override_mode: "policy_action_required", action_kind: "action_gate_override", approval_trigger_code: "gate_override" }, + }, + dependency_cache_handoffs: [{ + request_digest: { hash_alg: "sha256", hash: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, + consumer_role: "workspace", + required: true, + }], + depends_on_entry_ids: [], + blocks_entry_ids: [], + supported_wait_kinds: ["waiting_operator_input", "waiting_approval"], + }], + ...overrides, + }; + const runplanPath = path.join(root, "runplan.json"); + fs.writeFileSync(runplanPath, JSON.stringify(runPlan, null, 2)); + return runplanPath; +} + +function runPerf(mode, runplanPath, fixture) { + const args = ["--experimental-strip-types", scriptPath, "--mode", mode, "--runplan", runplanPath]; + if (fixture) { + args.push("--fixture", fixture); + } + return spawnSync(process.execPath, args, { encoding: "utf8" }); +} + +test("workflow-path requires supported first-party fixture argument", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "runecode-perf-workflow-")); + try { + const runplanPath = writeRunPlan(root); + const result = runPerf("workflow-path", runplanPath, ""); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires --fixture workflow\.first-party-minimal\.v1/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("workflow-path rejects supported fixture when no work is schedulable", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "runecode-perf-workflow-")); + try { + const runplanPath = writeRunPlan(root, { + dependency_edges: [{ + dependency_kind: "step_completed", + upstream_step_id: "quality_lint", + downstream_step_id: "quality_lint", + }], + entries: [{ + entry_id: "quality_lint", + entry_kind: "gate", + order_index: 0, + stage_id: "quality_stage", + step_id: "quality_lint", + role_instance_id: "role_alpha", + executor_binding_id: "binding_alpha", + checkpoint_code: "quality", + gate: { + schema_id: "runecode.protocol.v0.GateContract", + schema_version: "0.1.0", + gate_id: "lint", + gate_kind: "lint", + gate_version: "0.1.0", + normalized_inputs: [], + plan_binding: { checkpoint_code: "quality", order_index: 0 }, + retry_semantics: { retry_mode: "new_attempt_required", max_attempts: 2 }, + override_semantics: { override_mode: "policy_action_required", action_kind: "action_gate_override", approval_trigger_code: "gate_override" }, + }, + dependency_cache_handoffs: [{ + request_digest: { hash_alg: "sha256", hash: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, + consumer_role: "workspace", + required: true, + }], + depends_on_entry_ids: ["quality_lint"], + blocks_entry_ids: ["quality_lint"], + supported_wait_kinds: ["waiting_operator_input", "waiting_approval"], + }], + }); + const result = runPerf("workflow-path", runplanPath, "workflow.first-party-minimal.v1"); + assert.equal(result.status, 1); + assert.match(result.stderr, /no schedulable work on supported path/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("workflow-path accepts supported fixture and runplan", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "runecode-perf-workflow-")); + try { + const runplanPath = writeRunPlan(root); + const result = runPerf("workflow-path", runplanPath, "workflow.first-party-minimal.v1"); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout.trim(), /^\d+$/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("first-party-beta rejects non-supported runplan identity", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "runecode-perf-workflow-")); + try { + const runplanPath = writeRunPlan(root, { workflow_id: "workflow_other" }); + const result = runPerf("first-party-beta", runplanPath, "workflow.first-party-minimal.v1"); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires workflow_id workflow_first_party_minimal/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("first-party-beta accepts supported fixture and runplan", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "runecode-perf-workflow-")); + try { + const runplanPath = writeRunPlan(root); + const result = runPerf("first-party-beta", runplanPath, "workflow.first-party-minimal.v1"); + assert.equal(result.status, 0); + assert.match(result.stdout.trim(), /^\d+$/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/brokerperf/main.go b/tools/brokerperf/main.go new file mode 100644 index 00000000..92d6375f --- /dev/null +++ b/tools/brokerperf/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/runecode-ai/runecode/internal/brokerperf" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "brokerperf usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "brokerperf failed: %v\n", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("brokerperf", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + output := fs.String("output", "", "output check json path") + trials := fs.Int("trials", 30, "number of deterministic local trials") + repositoryRoot := fs.String("repository-root", "", "repository root for broker service") + if err := fs.Parse(args); err != nil { + return usageError{err: err} + } + if strings.TrimSpace(*output) == "" { + return usageError{err: fmt.Errorf("--output is required")} + } + out, err := brokerperf.Run(brokerperf.HarnessConfig{Trials: *trials, RepositoryRoot: strings.TrimSpace(*repositoryRoot)}) + if err != nil { + return err + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + return os.WriteFile(strings.TrimSpace(*output), raw, 0o644) +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/brokerperf/main_test.go b/tools/brokerperf/main_test.go new file mode 100644 index 00000000..1e2b615f --- /dev/null +++ b/tools/brokerperf/main_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "errors" + "testing" +) + +func TestRunReturnsUsageErrorWhenOutputMissing(t *testing.T) { + err := run([]string{}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} + +func TestRunReturnsUsageErrorForInvalidFlag(t *testing.T) { + err := run([]string{"--bad-flag"}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} diff --git a/tools/launcherperf/main.go b/tools/launcherperf/main.go new file mode 100644 index 00000000..90eb5d47 --- /dev/null +++ b/tools/launcherperf/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/runecode-ai/runecode/internal/launcherperf" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "launcherperf usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "launcherperf failed: %v\n", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("launcherperf", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + output := fs.String("output", "", "output check json path") + if err := fs.Parse(args); err != nil { + return usageError{err: err} + } + if strings.TrimSpace(*output) == "" { + return usageError{err: fmt.Errorf("--output is required")} + } + out, err := launcherperf.Run(launcherperf.HarnessConfig{}) + if err != nil { + return err + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + return os.WriteFile(strings.TrimSpace(*output), raw, 0o644) +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/launcherperf/main_test.go b/tools/launcherperf/main_test.go new file mode 100644 index 00000000..1e2b615f --- /dev/null +++ b/tools/launcherperf/main_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "errors" + "testing" +) + +func TestRunReturnsUsageErrorWhenOutputMissing(t *testing.T) { + err := run([]string{}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} + +func TestRunReturnsUsageErrorForInvalidFlag(t *testing.T) { + err := run([]string{"--bad-flag"}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} diff --git a/tools/perfcontracts/README.md b/tools/perfcontracts/README.md new file mode 100644 index 00000000..cbd8659d --- /dev/null +++ b/tools/perfcontracts/README.md @@ -0,0 +1,89 @@ +# Performance Contracts + +This directory under `tools/perfcontracts/` is the reviewed, machine-readable contract surface for performance verification. + +It is intentionally outside `runecontext/` and separate from `runecontext/assurance/baseline.yaml`. + +## Artifact Format + +- `manifest.json` is the authoritative inventory for checked-in performance contracts and optional repeated-sample baselines. +- `fixtures/*.json` contains reviewed fixture inventory and stable fixture IDs. +- `contracts/*.json` contains per-surface metric contracts. +- `baselines/*.json` contains optional repeated-sample baseline artifacts for regression and hybrid budgets. + +Each metric contract declares: + +- metric identity and fixture identity +- measurement kind and unit +- budget class (`exact`, `absolute-budget`, `regression-budget`, `hybrid-budget`) +- lane authority and activation state +- threshold origin +- timing boundary (`start_event`, `end_event`, `clock_source`, `evidence_source`, `included_phases`) + +## Baseline Governance + +### Threshold review process + +Threshold changes are contract changes, not harness-only edits. + +- Tightening a threshold requires explicit rationale, evidence, and expected operator or product impact. +- Deliberate threshold loosening (accepted regression) requires explicit justification in review notes and must explain why the regression is acceptable now. +- Every threshold keeps a reviewed `threshold_origin` (`product_budget`, `investigation_baseline`, `first_calibration`, `temporary_guardrail`) so provenance remains inspectable. +- `threshold_origin` values are validated by `internal/perfcontracts` and must not be free-form. + +### Baseline refresh policy for major architecture shifts + +When a major reviewed architectural shift lands, refresh baselines with an explicit review path: + +1. Keep metric identity stable (`metric_id`, fixture, timing boundary, budget class) unless semantics truly changed. +2. If semantics changed, add a new metric or fixture identity rather than silently reusing old identities. +3. Collect repeated samples using the reviewed defaults below in the authoritative environment. +4. Commit refreshed baseline artifacts and any threshold changes together with rationale. +5. Keep normal CI check-only; baseline refresh is intentional and reviewed, never auto-mutated. + +## Statistical defaults (reviewed v1) + +These defaults are the initial contract constants for CHG-053 and are tuned only through explicit follow-up review. + +- **Microbenchmarks** + - repeated samples: `10` for required PR comparisons + - repeated samples: `20` preferred for baseline refresh or recalibration + - comparison: robust repeated-sample regression check with practical noise-floor gate +- **Latency metrics** + - trials: `30` when `p95` is authoritative + - p95 eligibility: require fixed repeated trials sufficient for meaningful p95; otherwise use median+max while informational + - comparison: explicit reviewed ceilings (`p95` or median+max per metric contract) +- **CPU/process-behavior metrics** + - warmup window: `3000ms` + - observation window: `20000ms` + - repeated windows: `3` + - comparison: sustained average/median signal plus max guardrail +- **Exact metrics** + - comparison: exact value or hard bound only (no inferential statistics) + +### Practical noise-floor policy + +Regression checks that use repeated-sample comparisons must require both: + +- regression threshold exceeded (for example, `max_regression_percent`) +- practical noise floor exceeded (`practical_noise_floor`) + +This avoids gate churn from statistically detectable but operationally irrelevant movement. + +## CI Contract + +The trusted compare/enforce tool is `go run ./tools/perfcontracts`. + +Normal CI runs are check-only: + +- contract validation and compare are read-only +- no baseline rewrite behavior is allowed in verification flows + +## Scope + +This initial inventory intentionally covers one reviewed MVP fixture set per major surface. +Broader fixture ladders and cross-platform expansion are deferred to: + +- `CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0` + +Deferred broader performance surfaces include larger fixture ladders, wider workflow-pack and git-gateway coverage, and tuned macOS/Windows numeric-gate programs. diff --git a/tools/perfcontracts/baselines/metric.audit.finalize_verify.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.audit.finalize_verify.wall_ms.v1.json new file mode 100644 index 00000000..9310b59a --- /dev/null +++ b/tools/perfcontracts/baselines/metric.audit.finalize_verify.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.audit.finalize_verify.wall_ms", + "unit": "ms", + "samples": [894.885], + "summary": { + "median": 894.885 + } +} diff --git a/tools/perfcontracts/baselines/metric.audit.verify_current_segment.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.audit.verify_current_segment.wall_ms.v1.json new file mode 100644 index 00000000..8989589a --- /dev/null +++ b/tools/perfcontracts/baselines/metric.audit.verify_current_segment.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.audit.verify_current_segment.wall_ms", + "unit": "ms", + "samples": [234.0275], + "summary": { + "median": 234.0275 + } +} diff --git a/tools/perfcontracts/baselines/metric.broker.unary.session_list.p95_ms.v1.json b/tools/perfcontracts/baselines/metric.broker.unary.session_list.p95_ms.v1.json new file mode 100644 index 00000000..ba8638dd --- /dev/null +++ b/tools/perfcontracts/baselines/metric.broker.unary.session_list.p95_ms.v1.json @@ -0,0 +1,20 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.broker.unary.session_list.p95_ms", + "unit": "ms", + "samples": [ + 47, + 51, + 49, + 52, + 50, + 48, + 53, + 50, + 49, + 51 + ], + "summary": { + "median": 50 + } +} diff --git a/tools/perfcontracts/baselines/metric.deps.cache_hit.small.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.deps.cache_hit.small.wall_ms.v1.json new file mode 100644 index 00000000..4c241e62 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.deps.cache_hit.small.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.deps.cache_hit.small.wall_ms", + "unit": "ms", + "samples": [34, 36, 35, 35, 37], + "summary": { + "median": 35 + } +} diff --git a/tools/perfcontracts/baselines/metric.deps.cache_miss.small.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.deps.cache_miss.small.wall_ms.v1.json new file mode 100644 index 00000000..9953921b --- /dev/null +++ b/tools/perfcontracts/baselines/metric.deps.cache_miss.small.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.deps.cache_miss.small.wall_ms", + "unit": "ms", + "samples": [338, 345, 342, 340, 344], + "summary": { + "median": 342 + } +} diff --git a/tools/perfcontracts/baselines/metric.deps.materialization.workspace_handoff.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.deps.materialization.workspace_handoff.wall_ms.v1.json new file mode 100644 index 00000000..be370e6f --- /dev/null +++ b/tools/perfcontracts/baselines/metric.deps.materialization.workspace_handoff.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.deps.materialization.workspace_handoff.wall_ms", + "unit": "ms", + "samples": [58, 61, 60, 59, 62], + "summary": { + "median": 60 + } +} diff --git a/tools/perfcontracts/baselines/metric.runner.boundary_check.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.runner.boundary_check.wall_ms.v1.json new file mode 100644 index 00000000..4f1efbf8 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.runner.boundary_check.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.runner.boundary_check.wall_ms", + "unit": "ms", + "samples": [980, 1005, 995, 1010, 990], + "summary": { + "median": 995 + } +} diff --git a/tools/perfcontracts/baselines/metric.runner.protocol_fixtures.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.runner.protocol_fixtures.wall_ms.v1.json new file mode 100644 index 00000000..1ffd0e2e --- /dev/null +++ b/tools/perfcontracts/baselines/metric.runner.protocol_fixtures.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.runner.protocol_fixtures.wall_ms", + "unit": "ms", + "samples": [2650, 2725, 2685, 2710, 2690], + "summary": { + "median": 2690 + } +} diff --git a/tools/perfcontracts/baselines/metric.tui.render.shell_view_waiting.ns_op.v1.json b/tools/perfcontracts/baselines/metric.tui.render.shell_view_waiting.ns_op.v1.json new file mode 100644 index 00000000..26081eb8 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.tui.render.shell_view_waiting.ns_op.v1.json @@ -0,0 +1,11 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.tui.render.shell_view_waiting.ns_op", + "unit": "ns/op", + "samples": [ + 2503093.5 + ], + "summary": { + "median": 2503093.5 + } +} diff --git a/tools/perfcontracts/baselines/metric.workflow.chg049.first_party_beta_slice.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.workflow.chg049.first_party_beta_slice.wall_ms.v1.json new file mode 100644 index 00000000..27507897 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.workflow.chg049.first_party_beta_slice.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.workflow.chg049.first_party_beta_slice.wall_ms", + "unit": "ms", + "samples": [420, 432, 428, 425, 430], + "summary": { + "median": 428 + } +} diff --git a/tools/perfcontracts/baselines/metric.workflow.chg050.compile.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.workflow.chg050.compile.wall_ms.v1.json new file mode 100644 index 00000000..94dc77d5 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.workflow.chg050.compile.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.workflow.chg050.compile.wall_ms", + "unit": "ms", + "samples": [278, 284, 281, 286, 280], + "summary": { + "median": 281 + } +} diff --git a/tools/perfcontracts/baselines/metric.workflow.chg050.runplan_persist_load.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.workflow.chg050.runplan_persist_load.wall_ms.v1.json new file mode 100644 index 00000000..c294be4d --- /dev/null +++ b/tools/perfcontracts/baselines/metric.workflow.chg050.runplan_persist_load.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.workflow.chg050.runplan_persist_load.wall_ms", + "unit": "ms", + "samples": [68, 71, 69, 70, 72], + "summary": { + "median": 70 + } +} diff --git a/tools/perfcontracts/baselines/metric.workflow.chg050.validation_canonicalization.wall_ms.v1.json b/tools/perfcontracts/baselines/metric.workflow.chg050.validation_canonicalization.wall_ms.v1.json new file mode 100644 index 00000000..427fe0b4 --- /dev/null +++ b/tools/perfcontracts/baselines/metric.workflow.chg050.validation_canonicalization.wall_ms.v1.json @@ -0,0 +1,9 @@ +{ + "schema_version":"runecode.performance.baseline.v1", + "metric_id": "metric.workflow.chg050.validation_canonicalization.wall_ms", + "unit": "ms", + "samples": [92, 96, 94, 95, 93], + "summary": { + "median": 94 + } +} diff --git a/tools/perfcontracts/contracts/attestation.v1.json b/tools/perfcontracts/contracts/attestation.v1.json new file mode 100644 index 00000000..fa438342 --- /dev/null +++ b/tools/perfcontracts/contracts/attestation.v1.json @@ -0,0 +1,57 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.attestation.v1", + "surface": "attestation", + "metrics": [ + { + "metric_id": "metric.attestation.cold.verify.wall_ms", + "subsystem": "attestation", + "runtime_regime": "cold_path", + "fixture_id": "attestation.cold.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 15, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "runtime.launch.start", + "end_event": "attestation.verification.persisted", + "clock_source": "monotonic", + "evidence_source": "trusted_runtime_evidence_and_broker_projection", + "included_phases": ["launch", "secure_session_validation", "post_handshake_runtime_evidence", "attestation_verification", "replay_check", "freshness_check", "evidence_persistence", "broker_projection"] + }, + "notes": "Pending CHG-2026-054 truthful post-handshake attestation gate" + }, + { + "metric_id": "metric.attestation.warm.verify.wall_ms", + "subsystem": "attestation", + "runtime_regime": "warm_path", + "fixture_id": "attestation.warm.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "runtime.launch.start", + "end_event": "attestation.verification.persisted", + "clock_source": "monotonic", + "evidence_source": "trusted_runtime_evidence_and_broker_projection", + "included_phases": ["launch", "secure_session_validation", "post_handshake_runtime_evidence", "attestation_verification_cache_hit", "replay_check", "freshness_check", "evidence_persistence", "broker_projection"] + }, + "notes": "Pending CHG-2026-054 truthful post-handshake attestation gate" + } + ] +} diff --git a/tools/perfcontracts/contracts/broker.v1.json b/tools/perfcontracts/contracts/broker.v1.json new file mode 100644 index 00000000..5c045e09 --- /dev/null +++ b/tools/perfcontracts/contracts/broker.v1.json @@ -0,0 +1,608 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.broker.v1", + "surface": "broker", + "metrics": [ + { + "metric_id": "metric.broker.unary.session_list.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "hybrid-budget", + "threshold": {"max_value": 150, "max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.broker.unary.session_list.p95_ms.v1.json", + "comparison_method": "p95_ceiling_plus_regression", + "practical_noise_floor": 5, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "store_query", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.session_get.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "store_query", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.run_list.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "store_query", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.run_get.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "store_query", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.approval_list.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "store_query", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.readiness_get.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "readiness_projection", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.version_info_get.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "version_projection", "response_projection"] + } + }, + { + "metric_id": "metric.broker.unary.project_substrate_posture_get.p95_ms", + "subsystem": "broker", + "runtime_regime": "unary_api", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "broker.rpc.request.accepted", + "end_event": "broker.rpc.response.serialized", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "project_substrate_projection", "response_projection"] + } + }, + { + "metric_id": "metric.broker.watch.run.snapshot_follow.p95_ms", + "subsystem": "broker", + "runtime_regime": "watch_run", + "fixture_id": "broker.watch.run.snapshot-follow.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.subscribe", + "end_event": "watch.snapshot_follow.received", + "clock_source": "monotonic", + "evidence_source": "broker_watch_events", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.approval.snapshot_follow.p95_ms", + "subsystem": "broker", + "runtime_regime": "watch_approval", + "fixture_id": "broker.watch.approval.snapshot-follow.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.subscribe", + "end_event": "watch.snapshot_follow.received", + "clock_source": "monotonic", + "evidence_source": "broker_watch_events", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.session.snapshot_follow.p95_ms", + "subsystem": "broker", + "runtime_regime": "watch_session", + "fixture_id": "broker.watch.session.snapshot-follow.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.subscribe", + "end_event": "watch.snapshot_follow.received", + "clock_source": "monotonic", + "evidence_source": "broker_watch_events", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.turn_execution.snapshot_follow.p95_ms", + "subsystem": "broker", + "runtime_regime": "watch_turn_execution", + "fixture_id": "broker.watch.turn-execution.snapshot-follow.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.subscribe", + "end_event": "watch.snapshot_follow.received", + "clock_source": "monotonic", + "evidence_source": "broker_watch_events", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.run.snapshot_follow.payload_bytes", + "subsystem": "broker", + "runtime_regime": "watch_run", + "fixture_id": "broker.watch.run.snapshot-follow.v1", + "measurement_kind": "payload_growth", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 100000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot.serialized", + "end_event": "watch.terminal.serialized", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.approval.snapshot_follow.payload_bytes", + "subsystem": "broker", + "runtime_regime": "watch_approval", + "fixture_id": "broker.watch.approval.snapshot-follow.v1", + "measurement_kind": "payload_growth", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 100000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot.serialized", + "end_event": "watch.terminal.serialized", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.session.snapshot_follow.payload_bytes", + "subsystem": "broker", + "runtime_regime": "watch_session", + "fixture_id": "broker.watch.session.snapshot-follow.v1", + "measurement_kind": "payload_growth", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 100000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot.serialized", + "end_event": "watch.terminal.serialized", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.turn_execution.snapshot_follow.payload_bytes", + "subsystem": "broker", + "runtime_regime": "watch_turn_execution", + "fixture_id": "broker.watch.turn-execution.snapshot-follow.v1", + "measurement_kind": "payload_growth", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 100000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot.serialized", + "end_event": "watch.terminal.serialized", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.run.snapshot_follow.event_count", + "subsystem": "broker", + "runtime_regime": "watch_run", + "fixture_id": "broker.watch.run.snapshot-follow.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 3}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot", + "end_event": "watch.terminal", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.approval.snapshot_follow.event_count", + "subsystem": "broker", + "runtime_regime": "watch_approval", + "fixture_id": "broker.watch.approval.snapshot-follow.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 3}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot", + "end_event": "watch.terminal", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.session.snapshot_follow.event_count", + "subsystem": "broker", + "runtime_regime": "watch_session", + "fixture_id": "broker.watch.session.snapshot-follow.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 3}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot", + "end_event": "watch.terminal", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.watch.turn_execution.snapshot_follow.event_count", + "subsystem": "broker", + "runtime_regime": "watch_turn_execution", + "fixture_id": "broker.watch.turn-execution.snapshot-follow.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 3}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "watch.snapshot", + "end_event": "watch.terminal", + "clock_source": "event_counter", + "evidence_source": "watch_payload", + "included_phases": ["snapshot", "follow"] + } + }, + { + "metric_id": "metric.broker.mutation.session_execution_trigger.p95_ms", + "subsystem": "broker", + "runtime_regime": "mutation_session_execution_trigger", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "session.execution.trigger.request.accepted", + "end_event": "session.execution.trigger.ack.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "policy_evaluation", "durable_trigger_persist", "ack_projection"] + } + }, + { + "metric_id": "metric.broker.mutation.session_execution_continue.p95_ms", + "subsystem": "broker", + "runtime_regime": "mutation_session_execution_continue", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "session.execution.continue.request.accepted", + "end_event": "session.execution.continue.ack.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "target_resume_selection", "durable_turn_update", "ack_projection"] + } + }, + { + "metric_id": "metric.broker.mutation.approval_resolve.p95_ms", + "subsystem": "broker", + "runtime_regime": "mutation_approval_resolve", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "approval.resolve.request.accepted", + "end_event": "approval.resolve.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "signature_verification", "approval_state_update", "posture_apply_or_publish", "audit_persist"] + } + }, + { + "metric_id": "metric.broker.mutation.backend_posture_change.p95_ms", + "subsystem": "broker", + "runtime_regime": "mutation_backend_posture_change", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 200}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "backend.posture.change.request.accepted", + "end_event": "backend.posture.change.outcome.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_rpc_events", + "included_phases": ["request_validation", "policy_evaluation", "approval_gate_or_apply", "outcome_projection"] + } + }, + { + "metric_id": "metric.broker.attach.local_control_plane.p95_ms", + "subsystem": "broker", + "runtime_regime": "attach_local_control_plane", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "attach.request.accepted", + "end_event": "product.lifecycle.posture.attachable_projected", + "clock_source": "monotonic", + "evidence_source": "broker_product_lifecycle_posture", + "included_phases": ["request_validation", "project_substrate_discovery", "lifecycle_projection"] + } + }, + { + "metric_id": "metric.broker.resume.local_control_plane.p95_ms", + "subsystem": "broker", + "runtime_regime": "resume_local_control_plane", + "fixture_id": "broker.unary.beta-small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "resume.attach.request.accepted", + "end_event": "product.lifecycle.posture.attachable_projected", + "clock_source": "monotonic", + "evidence_source": "broker_product_lifecycle_posture", + "included_phases": ["request_validation", "lifecycle_generation_reconcile", "lifecycle_projection"] + } + } + ] +} diff --git a/tools/perfcontracts/contracts/dependency-audit.v1.json b/tools/perfcontracts/contracts/dependency-audit.v1.json new file mode 100644 index 00000000..c11d423e --- /dev/null +++ b/tools/perfcontracts/contracts/dependency-audit.v1.json @@ -0,0 +1,367 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.dependency-audit.v1", + "surface": "dependency-audit", + "metrics": [ + { + "metric_id": "metric.deps.cache_miss.small.wall_ms", + "subsystem": "dependency", + "runtime_regime": "cache_miss", + "fixture_id": "deps.cache-miss.small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.deps.cache_miss.small.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "dependency.cache.miss.start", + "end_event": "dependency.cache.miss.complete", + "clock_source": "monotonic", + "evidence_source": "broker_dependency_audit", + "included_phases": ["fetch", "cas_write", "manifest_persist"] + } + }, + { + "metric_id": "metric.deps.cache_hit.small.wall_ms", + "subsystem": "dependency", + "runtime_regime": "cache_hit", + "fixture_id": "deps.cache-hit.small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.deps.cache_hit.small.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "dependency.cache.hit.start", + "end_event": "dependency.cache.hit.complete", + "clock_source": "monotonic", + "evidence_source": "broker_dependency_audit", + "included_phases": ["lookup", "response_projection"] + } + }, + { + "metric_id": "metric.deps.cache_coalesced.upstream_fetch_count", + "subsystem": "dependency", + "runtime_regime": "coalesced_miss", + "fixture_id": "deps.coalesced-miss.small.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 1}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "dependency.cache.ensure.start", + "end_event": "dependency.cache.ensure.complete", + "clock_source": "event_counter", + "evidence_source": "broker_dependency_audit", + "included_phases": ["fetch", "cas_write"] + } + }, + { + "metric_id": "metric.deps.cache_coalesced.cas_write_count", + "subsystem": "dependency", + "runtime_regime": "coalesced_miss", + "fixture_id": "deps.coalesced-miss.small.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 1}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "dependency.cache.ensure.start", + "end_event": "dependency.cache.ensure.complete", + "clock_source": "event_counter", + "evidence_source": "broker_dependency_audit", + "included_phases": ["single_flight", "cas_write"] + } + }, + { + "metric_id": "metric.deps.materialization.workspace_handoff.wall_ms", + "subsystem": "dependency", + "runtime_regime": "materialization_workspace_handoff", + "fixture_id": "deps.cache-hit.small.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.deps.materialization.workspace_handoff.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "dependency.handoff.request.accepted", + "end_event": "dependency.handoff.metadata.projected", + "clock_source": "monotonic", + "evidence_source": "broker_dependency_audit", + "included_phases": ["request_validation", "cache_lookup", "handoff_projection"] + } + }, + { + "metric_id": "metric.deps.materialization.workspace_handoff.found_count", + "subsystem": "dependency", + "runtime_regime": "materialization_workspace_handoff", + "fixture_id": "deps.cache-hit.small.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 1}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "exact_match", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "dependency.handoff.request.accepted", + "end_event": "dependency.handoff.metadata.projected", + "clock_source": "event_counter", + "evidence_source": "broker_dependency_audit", + "included_phases": ["lookup", "projection"] + } + }, + { + "metric_id": "metric.deps.stream_to_cas.max_read_buffer_bytes", + "subsystem": "dependency", + "runtime_regime": "stream_to_cas", + "fixture_id": "deps.cache-miss.small.v1", + "measurement_kind": "count", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 131072}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "dependency.cache.fill.stream.start", + "end_event": "dependency.cache.fill.stream.complete", + "clock_source": "event_counter", + "evidence_source": "bounded_buffer_instrumentation", + "included_phases": ["read_chunk", "stream_to_cas"] + } + }, + { + "metric_id": "metric.deps.stream_to_cas.read_calls", + "subsystem": "dependency", + "runtime_regime": "stream_to_cas", + "fixture_id": "deps.cache-miss.small.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 2000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "dependency.cache.fill.stream.start", + "end_event": "dependency.cache.fill.stream.complete", + "clock_source": "event_counter", + "evidence_source": "bounded_buffer_instrumentation", + "included_phases": ["read_chunk", "stream_to_cas"] + } + }, + { + "metric_id": "metric.deps.stream_to_cas.fetched_bytes", + "subsystem": "dependency", + "runtime_regime": "stream_to_cas", + "fixture_id": "deps.cache-miss.small.v1", + "measurement_kind": "count", + "unit": "bytes", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 4194304}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "dependency.cache.fill.stream.start", + "end_event": "dependency.cache.fill.stream.complete", + "clock_source": "event_counter", + "evidence_source": "broker_dependency_audit", + "included_phases": ["stream_to_cas"] + } + }, + { + "metric_id": "metric.deps.cache_fill.peak_alloc_mb", + "subsystem": "dependency", + "runtime_regime": "stream_to_cas_memory_guardrail", + "fixture_id": "deps.cache-miss.small.v1", + "measurement_kind": "memory", + "unit": "mb", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 5, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 32}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "dependency.cache.fill.start", + "end_event": "dependency.cache.fill.complete", + "clock_source": "runtime_memstats", + "evidence_source": "process_memory_observation", + "included_phases": ["stream_to_cas", "manifest_persist"] + } + }, + { + "metric_id": "metric.deps.cache_ensure.registry_requests", + "subsystem": "dependency", + "runtime_regime": "cache_ensure", + "fixture_id": "deps.cache-hit.small.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 1}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "absolute_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "dependency.cache.ensure.start", + "end_event": "dependency.cache.ensure.complete", + "clock_source": "event_counter", + "evidence_source": "broker_dependency_audit", + "included_phases": ["lookup", "fetch_optional"] + } + }, + { + "metric_id": "metric.audit.verify_current_segment.wall_ms", + "subsystem": "audit", + "runtime_regime": "verify_current_segment", + "fixture_id": "audit.ledger.standard.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.audit.verify_current_segment.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "audit.verify_current_segment.start", + "end_event": "audit.verify_current_segment.complete", + "clock_source": "monotonic", + "evidence_source": "auditd_runtime", + "included_phases": ["seal_load", "verify", "report_persist"] + } + }, + { + "metric_id": "metric.audit.finalize_verify.wall_ms", + "subsystem": "audit", + "runtime_regime": "verify_finalize", + "fixture_id": "audit.ledger.standard.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.audit.finalize_verify.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "audit.finalize_verify.start", + "end_event": "audit.finalize_verify.complete", + "clock_source": "monotonic", + "evidence_source": "auditd_runtime", + "included_phases": ["finalize", "verify"] + } + }, + { + "metric_id": "metric.protocol.schema_validation.wall_ms", + "subsystem": "protocol", + "runtime_regime": "schema_validation", + "fixture_id": "protocol.schema.bundle.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 5, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 2000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "max_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "protocol.schema.validation.start", + "end_event": "protocol.schema.validation.complete", + "clock_source": "monotonic", + "evidence_source": "go_test_timing", + "included_phases": ["bundle_load", "schema_validation"] + } + }, + { + "metric_id": "metric.protocol.fixture_parity.wall_ms", + "subsystem": "protocol", + "runtime_regime": "fixture_parity", + "fixture_id": "protocol.fixture.parity.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 5, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 2000}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "max_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "protocol.fixture.parity.start", + "end_event": "protocol.fixture.parity.complete", + "clock_source": "monotonic", + "evidence_source": "runner_node_test_timing", + "included_phases": ["fixture_manifest_load", "schema_validation", "parity_checks"] + } + } + ] +} diff --git a/tools/perfcontracts/contracts/external-anchor.v1.json b/tools/perfcontracts/contracts/external-anchor.v1.json new file mode 100644 index 00000000..e1dc0aaa --- /dev/null +++ b/tools/perfcontracts/contracts/external-anchor.v1.json @@ -0,0 +1,178 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.external-anchor.v1", + "surface": "external-anchor", + "metrics": [ + { + "metric_id": "metric.anchor.prepare.latency.p95_ms", + "subsystem": "external-anchor", + "runtime_regime": "prepare", + "fixture_id": "anchor.fast-complete.stub.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "p95_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "external_anchor.prepare.start", + "end_event": "external_anchor.prepare.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["prepare", "persist"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.execute.deferred.handoff.p95_ms", + "subsystem": "external-anchor", + "runtime_regime": "execute_deferred", + "fixture_id": "anchor.deferred.stub.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "p95_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "external_anchor.execute.start", + "end_event": "external_anchor.execute.deferred_persisted", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["execute", "deferred_handoff"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.execute.completed.p95_ms", + "subsystem": "external-anchor", + "runtime_regime": "execute_completed", + "fixture_id": "anchor.fast-complete.stub.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "external_anchor.execute.start", + "end_event": "external_anchor.execute.completed_persisted", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["execute", "proof_admission", "persist"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.deferred.visibility.p95_ms", + "subsystem": "external-anchor", + "runtime_regime": "deferred_visibility", + "fixture_id": "anchor.deferred.stub.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "external_anchor.execute.deferred_persisted", + "end_event": "external_anchor.execute.completed_visible", + "clock_source": "monotonic", + "evidence_source": "broker_get_watch_surfaces", + "included_phases": ["deferred_background_execute", "visibility_projection"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.receipt_admission.unchanged_seal.p95_ms", + "subsystem": "external-anchor", + "runtime_regime": "receipt_admission_unchanged_seal", + "fixture_id": "anchor.fast-complete.stub.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 5, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "external_anchor.receipt_admission.start", + "end_event": "external_anchor.receipt_admission.persisted", + "clock_source": "monotonic", + "evidence_source": "auditd_verifier_runtime", + "included_phases": ["preverified_seal_lookup", "receipt_admission", "persistence"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.network_io_under_ledger_lock.count", + "subsystem": "external-anchor", + "runtime_regime": "lock_boundary", + "fixture_id": "anchor.deferred.stub.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 0}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "exact_match", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "external_anchor.runtime.execute.start", + "end_event": "external_anchor.runtime.execute.complete", + "clock_source": "event_counter", + "evidence_source": "lock_boundary_instrumentation", + "included_phases": ["snapshot_outside_lock", "network_execute"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + }, + { + "metric_id": "metric.anchor.verifier_bypass.count", + "subsystem": "external-anchor", + "runtime_regime": "authoritative_verifier_admission", + "fixture_id": "anchor.fast-complete.stub.v1", + "measurement_kind": "count", + "unit": "count", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {}, + "budget_class": "exact", + "threshold": {"exact_value": 0}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "exact_match", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "external_anchor.receipt_admission.start", + "end_event": "external_anchor.receipt_admission.persisted", + "clock_source": "event_counter", + "evidence_source": "verifier_admission_instrumentation", + "included_phases": ["preverified_seal_lookup", "receipt_admission"] + }, + "notes": "Pending CHG-2026-025 external anchoring authoritative path" + } + ] +} diff --git a/tools/perfcontracts/contracts/gateway-secrets.v1.json b/tools/perfcontracts/contracts/gateway-secrets.v1.json new file mode 100644 index 00000000..6e6c991d --- /dev/null +++ b/tools/perfcontracts/contracts/gateway-secrets.v1.json @@ -0,0 +1,76 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.gateway-secrets.v1", + "surface": "gateway-secrets", + "metrics": [ + { + "metric_id": "metric.gateway.model_invoke.overhead.p95_ms", + "subsystem": "model-gateway", + "runtime_regime": "stubbed_provider_invoke", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 100}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "gateway.invoke.request.accepted", + "end_event": "gateway.invoke.response.projected", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["admission", "translation", "stubbed_invoke", "projection"] + } + }, + { + "metric_id": "metric.secrets.lease_issue.p95_ms", + "subsystem": "secrets", + "runtime_regime": "stubbed_secret_ingress", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 150}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "secrets.lease.issue.start", + "end_event": "secrets.lease.issue.persisted", + "clock_source": "monotonic", + "evidence_source": "secretsd_events", + "included_phases": ["policy", "issue", "persist"] + } + }, + { + "metric_id": "metric.secrets.ingress.prepare_submit.p95_ms", + "subsystem": "secrets", + "runtime_regime": "stubbed_secret_ingress", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 300}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "secrets.ingress.prepare.start", + "end_event": "secrets.ingress.submit.persisted", + "clock_source": "monotonic", + "evidence_source": "secretsd_events", + "included_phases": ["prepare", "submit", "persist"] + } + } + ] +} diff --git a/tools/perfcontracts/contracts/launcher.v1.json b/tools/perfcontracts/contracts/launcher.v1.json new file mode 100644 index 00000000..8fded66b --- /dev/null +++ b/tools/perfcontracts/contracts/launcher.v1.json @@ -0,0 +1,99 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.launcher.v1", + "surface": "launcher", + "metrics": [ + { + "metric_id": "metric.launcher.microvm.cold_start.wall_ms", + "subsystem": "launcher", + "runtime_regime": "microvm_cold", + "fixture_id": "launcher.microvm.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 8000}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "launcher.runtime_admission.start", + "end_event": "broker.runtime_ready.projected", + "clock_source": "monotonic", + "evidence_source": "runtime_evidence_and_broker_projection", + "included_phases": ["signature_verification", "component_digest_verification", "admission", "launch", "broker_projection"] + } + }, + { + "metric_id": "metric.launcher.microvm.warm_start.wall_ms", + "subsystem": "launcher", + "runtime_regime": "microvm_warm", + "fixture_id": "launcher.microvm.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 3000}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "launcher.verified_cache_lookup.start", + "end_event": "broker.runtime_ready.projected", + "clock_source": "monotonic", + "evidence_source": "runtime_evidence_and_broker_projection", + "included_phases": ["verified_cache_hit", "launch", "broker_projection"] + } + }, + { + "metric_id": "metric.launcher.container.cold_start.wall_ms", + "subsystem": "launcher", + "runtime_regime": "container_cold", + "fixture_id": "launcher.container.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 4000}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "launcher.runtime_admission.start", + "end_event": "broker.runtime_ready.projected", + "clock_source": "monotonic", + "evidence_source": "runtime_evidence_and_broker_projection", + "included_phases": ["signature_verification", "component_digest_verification", "admission", "launch", "broker_projection"] + } + }, + { + "metric_id": "metric.launcher.container.warm_start.wall_ms", + "subsystem": "launcher", + "runtime_regime": "container_warm", + "fixture_id": "launcher.container.signed-runtime.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 2000}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "launcher.verified_cache_lookup.start", + "end_event": "broker.runtime_ready.projected", + "clock_source": "monotonic", + "evidence_source": "runtime_evidence_and_broker_projection", + "included_phases": ["verified_cache_hit", "launch", "broker_projection"] + } + } + ] +} diff --git a/tools/perfcontracts/contracts/runner-workflow.v1.json b/tools/perfcontracts/contracts/runner-workflow.v1.json new file mode 100644 index 00000000..c8e0a148 --- /dev/null +++ b/tools/perfcontracts/contracts/runner-workflow.v1.json @@ -0,0 +1,234 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.runner-workflow.v1", + "surface": "runner-workflow", + "metrics": [ + { + "metric_id": "metric.runner.boundary_check.wall_ms", + "subsystem": "runner", + "runtime_regime": "boundary_check", + "fixture_id": "runner.boundary.minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 5, "median_max_authoritative": true}, + "budget_class": "hybrid-budget", + "threshold": {"max_value": 5000, "max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.runner.boundary_check.wall_ms.v1.json", + "comparison_method": "median_plus_regression", + "practical_noise_floor": 30, + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "runner.boundary_check.start", + "end_event": "runner.boundary_check.finish", + "clock_source": "monotonic", + "evidence_source": "command_timing", + "included_phases": ["runner_checks"] + } + }, + { + "metric_id": "metric.runner.protocol_fixtures.wall_ms", + "subsystem": "runner", + "runtime_regime": "protocol_fixtures", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 5, "median_max_authoritative": true}, + "budget_class": "hybrid-budget", + "threshold": {"max_value": 10000, "max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.runner.protocol_fixtures.wall_ms.v1.json", + "comparison_method": "median_plus_regression", + "practical_noise_floor": 50, + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "runner.protocol_fixtures.start", + "end_event": "runner.protocol_fixtures.finish", + "clock_source": "monotonic", + "evidence_source": "command_timing", + "included_phases": ["fixture_load", "schema_validation", "parity_checks"] + } + }, + { + "metric_id": "metric.runner.cold_start.minimal_workflow.wall_ms", + "subsystem": "runner", + "runtime_regime": "cold_start", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 1000}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "max_ceiling", + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "runner.startup.start", + "end_event": "runner.plan.first_durable_checkpoint", + "clock_source": "monotonic", + "evidence_source": "runner_runtime_events", + "included_phases": ["plan_load", "scheduler_bootstrap", "checkpoint_projection"] + } + }, + { + "metric_id": "metric.workflow.mvp_execution.supported_path.wall_ms", + "subsystem": "workflow", + "runtime_regime": "supported_mvp_path", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "workflow.trigger.accepted", + "end_event": "workflow.completed.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["trigger", "execution", "completion_projection"] + } + }, + { + "metric_id": "metric.workflow.chg049.first_party_beta_slice.wall_ms", + "subsystem": "workflow", + "runtime_regime": "first_party_beta_slice", + "fixture_id": "workflow.first-party-minimal.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.workflow.chg049.first_party_beta_slice.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "workflow.first_party_beta.start", + "end_event": "workflow.first_party_beta.completed", + "clock_source": "monotonic", + "evidence_source": "runner_runtime_events", + "included_phases": ["plan_load", "scheduler_dispatch", "state_projection"] + } + }, + { + "metric_id": "metric.workflow.chg050.compile.wall_ms", + "subsystem": "workflow", + "runtime_regime": "chg050_compile", + "fixture_id": "workflow.chg050-compile.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.workflow.chg050.compile.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "workflow.compile.start", + "end_event": "workflow.compile.persisted", + "clock_source": "monotonic", + "evidence_source": "broker_events", + "included_phases": ["validation", "canonicalization", "compile", "persist"] + } + }, + { + "metric_id": "metric.workflow.chg050.validation_canonicalization.wall_ms", + "subsystem": "workflow", + "runtime_regime": "chg050_validate_canonicalize", + "fixture_id": "workflow.chg050-compile.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.workflow.chg050.validation_canonicalization.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "workflow.validate.start", + "end_event": "workflow.canonicalization.complete", + "clock_source": "monotonic", + "evidence_source": "compile_pipeline_events", + "included_phases": ["schema_validation", "canonicalization"] + } + }, + { + "metric_id": "metric.workflow.chg050.runplan_persist_load.wall_ms", + "subsystem": "workflow", + "runtime_regime": "chg050_persist_load", + "fixture_id": "workflow.chg050-compile.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.workflow.chg050.runplan_persist_load.wall_ms.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "runplan.persist.start", + "end_event": "runplan.load.complete", + "clock_source": "monotonic", + "evidence_source": "broker_store_records", + "included_phases": ["persist", "authority_record", "cache_key_lookup", "load"] + } + }, + { + "metric_id": "metric.workflow.chg050.runner_start_immutable_runplan.wall_ms", + "subsystem": "workflow", + "runtime_regime": "chg050_start_from_immutable_runplan", + "fixture_id": "workflow.chg050-compile.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 10, "median_max_authoritative": true}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "contract_pending_dependency", + "activation_state": "contract_pending_dependency", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 10, + "threshold_origin": "temporary_guardrail", + "timing_boundary": { + "start_event": "runner.immutable_runplan.start", + "end_event": "runner.immutable_runplan.ready", + "clock_source": "monotonic", + "evidence_source": "runner_runtime_events", + "included_phases": ["immutable_plan_load", "startup", "attach_ready_projection"] + } + } + ] +} diff --git a/tools/perfcontracts/contracts/tui.v1.json b/tools/perfcontracts/contracts/tui.v1.json new file mode 100644 index 00000000..bc368e72 --- /dev/null +++ b/tools/perfcontracts/contracts/tui.v1.json @@ -0,0 +1,289 @@ +{ + "schema_version":"runecode.performance.contract.v1", + "contract_id": "performance.tui.v1", + "surface": "tui", + "metrics": [ + { + "metric_id": "metric.tui.attach.quiet.p95_ms", + "subsystem": "tui", + "runtime_regime": "attach_quiet", + "fixture_id": "tui.empty.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "tui.process.spawn", + "end_event": "broker.attach.ready.projected_frame", + "clock_source": "monotonic", + "evidence_source": "pty_transcript", + "included_phases": ["launch", "broker_attach", "first_settled_frame"] + } + }, + { + "metric_id": "metric.tui.attach.waiting.p95_ms", + "subsystem": "tui", + "runtime_regime": "attach_waiting", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 500}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "tui.process.spawn", + "end_event": "broker.attach.ready.projected_frame", + "clock_source": "monotonic", + "evidence_source": "pty_transcript", + "included_phases": ["launch", "broker_attach", "first_settled_frame"] + } + }, + { + "metric_id": "metric.tui.key_response.quiet.p95_ms", + "subsystem": "tui", + "runtime_regime": "key_response_quiet", + "fixture_id": "tui.empty.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 50}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "tui.key.injected", + "end_event": "broker.attach.ready.frame_delta", + "clock_source": "monotonic", + "evidence_source": "pty_transcript", + "included_phases": ["input_dispatch", "update", "render", "frame_flush"] + } + }, + { + "metric_id": "metric.tui.key_response.waiting.p95_ms", + "subsystem": "tui", + "runtime_regime": "key_response_waiting", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "latency", + "unit": "ms", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"trials": 30, "p95_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 75}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "comparison_method": "p95_ceiling", + "threshold_origin": "product_budget", + "timing_boundary": { + "start_event": "tui.key.injected", + "end_event": "broker.attach.ready.frame_delta", + "clock_source": "monotonic", + "evidence_source": "pty_transcript", + "included_phases": ["input_dispatch", "update", "render", "frame_flush"] + } + }, + { + "metric_id": "metric.tui.idle_cpu.empty.avg_pct", + "subsystem": "tui", + "runtime_regime": "empty_idle", + "fixture_id": "tui.empty.v1", + "measurement_kind": "cpu", + "unit": "percent", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"warmup_millis": 3000, "observation_window_millis": 20000, "observation_windows": 3, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 2}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "window_average", + "threshold_origin": "investigation_baseline", + "timing_boundary": { + "start_event": "tui.warmup.complete", + "end_event": "cpu.observation.window.complete", + "clock_source": "monotonic", + "evidence_source": "proc_stat", + "included_phases": ["idle_observation"] + } + }, + { + "metric_id": "metric.tui.idle_cpu.empty.max_pct", + "subsystem": "tui", + "runtime_regime": "empty_idle", + "fixture_id": "tui.empty.v1", + "measurement_kind": "cpu", + "unit": "percent", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"warmup_millis": 3000, "observation_window_millis": 20000, "observation_windows": 3, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 4}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "window_max", + "threshold_origin": "investigation_baseline", + "timing_boundary": { + "start_event": "tui.warmup.complete", + "end_event": "cpu.observation.window.complete", + "clock_source": "monotonic", + "evidence_source": "proc_stat", + "included_phases": ["idle_observation"] + } + }, + { + "metric_id": "metric.tui.idle_cpu.waiting.avg_pct", + "subsystem": "tui", + "runtime_regime": "waiting_state", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "cpu", + "unit": "percent", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"warmup_millis": 3000, "observation_window_millis": 20000, "observation_windows": 3, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 8}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "window_average", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "tui.warmup.complete", + "end_event": "cpu.observation.window.complete", + "clock_source": "monotonic", + "evidence_source": "proc_stat", + "included_phases": ["waiting_observation"] + } + }, + { + "metric_id": "metric.tui.idle_cpu.waiting.max_pct", + "subsystem": "tui", + "runtime_regime": "waiting_state", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "cpu", + "unit": "percent", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"warmup_millis": 3000, "observation_window_millis": 20000, "observation_windows": 3, "median_max_authoritative": true}, + "budget_class": "absolute-budget", + "threshold": {"max_value": 12}, + "lane_authority": "informational_until_stable", + "activation_state": "informational", + "comparison_method": "window_max", + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "tui.warmup.complete", + "end_event": "cpu.observation.window.complete", + "clock_source": "monotonic", + "evidence_source": "proc_stat", + "included_phases": ["waiting_observation"] + } + }, + { + "metric_id": "metric.tui.render.shell_view_empty.ns_op", + "subsystem": "tui", + "runtime_regime": "render_empty", + "fixture_id": "tui.empty.v1", + "measurement_kind": "microbenchmark", + "unit": "ns/op", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"repeated_samples": 10}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "defined", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 1000, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "bench.iteration.start", + "end_event": "bench.iteration.end", + "clock_source": "go_benchmark_timer", + "evidence_source": "go_test_bench_output", + "included_phases": ["render"] + } + }, + { + "metric_id": "metric.tui.render.shell_view_waiting.ns_op", + "subsystem": "tui", + "runtime_regime": "waiting_state_render", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "microbenchmark", + "unit": "ns/op", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"repeated_samples": 10}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "required", + "baseline_source": "reviewed_repeated_samples", + "baseline_ref": "baselines/metric.tui.render.shell_view_waiting.ns_op.v1.json", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 1000, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "bench.iteration.start", + "end_event": "bench.iteration.end", + "clock_source": "go_benchmark_timer", + "evidence_source": "go_test_bench_output", + "included_phases": ["render"] + } + }, + { + "metric_id": "metric.tui.update.shell_watch_apply.ns_op", + "subsystem": "tui", + "runtime_regime": "watch_apply", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "microbenchmark", + "unit": "ns/op", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"repeated_samples": 10}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "defined", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 1000, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "bench.iteration.start", + "end_event": "bench.iteration.end", + "clock_source": "go_benchmark_timer", + "evidence_source": "go_test_bench_output", + "included_phases": ["update"] + } + }, + { + "metric_id": "metric.tui.update.build_palette_entries.ns_op", + "subsystem": "tui", + "runtime_regime": "palette_entries", + "fixture_id": "tui.waiting.v1", + "measurement_kind": "microbenchmark", + "unit": "ns/op", + "authoritative_environment": "linux_shared_ci", + "sampling_policy": {"repeated_samples": 10}, + "budget_class": "regression-budget", + "threshold": {"max_regression_percent": 15}, + "lane_authority": "required_shared_linux", + "activation_state": "defined", + "comparison_method": "median_regression_with_noise_floor", + "practical_noise_floor": 1000, + "threshold_origin": "first_calibration", + "timing_boundary": { + "start_event": "bench.iteration.start", + "end_event": "bench.iteration.end", + "clock_source": "go_benchmark_timer", + "evidence_source": "go_test_bench_output", + "included_phases": ["update"] + } + } + ] +} diff --git a/tools/perfcontracts/fixtures/inventory.v1.json b/tools/perfcontracts/fixtures/inventory.v1.json new file mode 100644 index 00000000..20392a91 --- /dev/null +++ b/tools/perfcontracts/fixtures/inventory.v1.json @@ -0,0 +1,27 @@ +{ + "schema_version":"runecode.performance.fixtures.v1", + "fixtures": [ + {"fixture_id": "tui.empty.v1", "surface": "tui", "runtime_regime": "empty_idle", "status": "mvp_reviewed"}, + {"fixture_id": "tui.waiting.v1", "surface": "tui", "runtime_regime": "waiting_state", "status": "mvp_reviewed"}, + {"fixture_id": "broker.unary.beta-small.v1", "surface": "broker", "runtime_regime": "unary_api", "status": "mvp_reviewed"}, + {"fixture_id": "broker.watch.run.snapshot-follow.v1", "surface": "broker", "runtime_regime": "watch_run", "status": "mvp_reviewed"}, + {"fixture_id": "broker.watch.approval.snapshot-follow.v1", "surface": "broker", "runtime_regime": "watch_approval", "status": "mvp_reviewed"}, + {"fixture_id": "broker.watch.session.snapshot-follow.v1", "surface": "broker", "runtime_regime": "watch_session", "status": "mvp_reviewed"}, + {"fixture_id": "broker.watch.turn-execution.snapshot-follow.v1", "surface": "broker", "runtime_regime": "watch_turn_execution", "status": "mvp_reviewed"}, + {"fixture_id": "workflow.first-party-minimal.v1", "surface": "runner-workflow", "runtime_regime": "minimal_workflow", "status": "mvp_reviewed"}, + {"fixture_id": "workflow.chg050-compile.v1", "surface": "runner-workflow", "runtime_regime": "chg050_compile", "status": "mvp_reviewed"}, + {"fixture_id": "runner.boundary.minimal.v1", "surface": "runner-workflow", "runtime_regime": "boundary_check", "status": "mvp_reviewed"}, + {"fixture_id": "launcher.microvm.signed-runtime.v1", "surface": "launcher", "runtime_regime": "microvm_startup", "status": "mvp_reviewed"}, + {"fixture_id": "launcher.container.signed-runtime.v1", "surface": "launcher", "runtime_regime": "container_startup", "status": "mvp_reviewed"}, + {"fixture_id": "deps.cache-miss.small.v1", "surface": "dependency", "runtime_regime": "cache_miss", "status": "mvp_reviewed"}, + {"fixture_id": "deps.cache-hit.small.v1", "surface": "dependency", "runtime_regime": "cache_hit", "status": "mvp_reviewed"}, + {"fixture_id": "deps.coalesced-miss.small.v1", "surface": "dependency", "runtime_regime": "coalesced_miss", "status": "mvp_reviewed"}, + {"fixture_id": "audit.ledger.standard.v1", "surface": "audit", "runtime_regime": "verify_finalize", "status": "mvp_reviewed"}, + {"fixture_id": "protocol.schema.bundle.v1", "surface": "protocol", "runtime_regime": "schema_validation", "status": "mvp_reviewed"}, + {"fixture_id": "protocol.fixture.parity.v1", "surface": "protocol", "runtime_regime": "fixture_parity", "status": "mvp_reviewed"}, + {"fixture_id": "anchor.fast-complete.stub.v1", "surface": "external-anchor", "runtime_regime": "execute_fast_complete", "status": "mvp_reviewed"}, + {"fixture_id": "anchor.deferred.stub.v1", "surface": "external-anchor", "runtime_regime": "execute_deferred", "status": "mvp_reviewed"}, + {"fixture_id": "attestation.cold.signed-runtime.v1", "surface": "attestation", "runtime_regime": "cold_path", "status": "mvp_reviewed"}, + {"fixture_id": "attestation.warm.signed-runtime.v1", "surface": "attestation", "runtime_regime": "warm_path", "status": "mvp_reviewed"} + ] +} diff --git a/tools/perfcontracts/main.go b/tools/perfcontracts/main.go new file mode 100644 index 00000000..52f27d66 --- /dev/null +++ b/tools/perfcontracts/main.go @@ -0,0 +1,181 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +type config struct { + contractsRoot string + checkOutput string + lane string + metricIDs []string +} + +func main() { + if err := run(os.Args[1:]); err != nil { + var usage usageError + if errors.As(err, &usage) { + fmt.Fprintf(os.Stderr, "perfcontracts usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "perfcontracts check failed: %v\n", err) + os.Exit(1) + } +} + +func run(args []string) error { + cfg, err := parseArgs(args) + if err != nil { + return err + } + manifest, inventory, contracts, baselinesByMetric, err := loadContractSet(cfg.contractsRoot) + if err != nil { + return err + } + if err := perfcontracts.ValidateWithBaselines(manifest, inventory, contracts, baselinesByMetric); err != nil { + return err + } + checkOutput, err := perfcontracts.LoadCheckOutput(cfg.checkOutput) + if err != nil { + return err + } + filtered := filterContractsForLane(contracts, cfg.lane, cfg.metricIDs) + if countMetrics(filtered) == 0 { + return fmt.Errorf("no required metrics selected for lane %q", cfg.lane) + } + violations := perfcontracts.Evaluate(checkOutput, filtered, baselinesByMetric) + if len(violations) > 0 { + for _, violation := range violations { + fmt.Fprintf(os.Stderr, "- %s: %s\n", violation.MetricID, violation.Reason) + } + return fmt.Errorf("%d performance contract violation(s)", len(violations)) + } + fmt.Printf("Performance contracts check passed (%d metrics evaluated).\n", countMetrics(filtered)) + return nil +} + +func parseArgs(args []string) (config, error) { + fs := flag.NewFlagSet("perfcontracts", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + contractsRoot := fs.String("contracts-root", filepath.FromSlash("tools/perfcontracts"), "path to performance contracts root") + checkOutput := fs.String("check-output", "", "path to performance check output json") + lane := fs.String("lane", "required_shared_linux", "lane authority to enforce") + metricIDFlags := multiStringFlag{} + fs.Var(&metricIDFlags, "metric-id", "optional metric_id to enforce; repeatable") + if err := fs.Parse(args); err != nil { + return config{}, usageError{err} + } + if strings.TrimSpace(*checkOutput) == "" { + return config{}, usageError{errors.New("--check-output is required")} + } + return config{contractsRoot: strings.TrimSpace(*contractsRoot), checkOutput: strings.TrimSpace(*checkOutput), lane: strings.TrimSpace(*lane), metricIDs: metricIDFlags.values()}, nil +} + +func loadContractSet(root string) (perfcontracts.Manifest, perfcontracts.FixtureInventory, []perfcontracts.ContractFile, map[string]perfcontracts.BaselineFile, error) { + manifest, err := perfcontracts.LoadManifest(root) + if err != nil { + return perfcontracts.Manifest{}, perfcontracts.FixtureInventory{}, nil, nil, err + } + inventory, err := perfcontracts.LoadFixtureInventory(root, manifest.FixtureInventoryRef) + if err != nil { + return perfcontracts.Manifest{}, perfcontracts.FixtureInventory{}, nil, nil, err + } + contracts := make([]perfcontracts.ContractFile, 0, len(manifest.Contracts)) + for _, entry := range manifest.Contracts { + contract, loadErr := perfcontracts.LoadContract(root, entry.Path) + if loadErr != nil { + return perfcontracts.Manifest{}, perfcontracts.FixtureInventory{}, nil, nil, loadErr + } + contracts = append(contracts, contract) + } + baselinesByMetric := map[string]perfcontracts.BaselineFile{} + for _, entry := range manifest.Baselines { + baseline, loadErr := perfcontracts.LoadBaseline(root, entry.Path) + if loadErr != nil { + return perfcontracts.Manifest{}, perfcontracts.FixtureInventory{}, nil, nil, loadErr + } + baselinesByMetric[entry.MetricID] = baseline + } + return manifest, inventory, contracts, baselinesByMetric, nil +} + +func filterContractsForLane(contracts []perfcontracts.ContractFile, lane string, metricIDs []string) []perfcontracts.ContractFile { + allowedMetrics := metricFilterSet(metricIDs) + + var filtered []perfcontracts.ContractFile + for _, contract := range contracts { + next := perfcontracts.ContractFile{SchemaVersion: contract.SchemaVersion, ContractID: contract.ContractID, Surface: contract.Surface} + for _, metric := range contract.Metrics { + if includeMetric(metric, lane, allowedMetrics) { + next.Metrics = append(next.Metrics, metric) + } + } + if len(next.Metrics) > 0 { + filtered = append(filtered, next) + } + } + return filtered +} + +func metricFilterSet(metricIDs []string) map[string]struct{} { + allowedMetrics := map[string]struct{}{} + for _, metricID := range metricIDs { + trimmed := strings.TrimSpace(metricID) + if trimmed != "" { + allowedMetrics[trimmed] = struct{}{} + } + } + return allowedMetrics +} + +func includeMetric(metric perfcontracts.MetricContract, lane string, allowedMetrics map[string]struct{}) bool { + if metric.LaneAuthority != lane || metric.ActivationState != "required" { + return false + } + if len(allowedMetrics) == 0 { + return true + } + _, ok := allowedMetrics[metric.MetricID] + return ok +} + +func countMetrics(contracts []perfcontracts.ContractFile) int { + total := 0 + for _, contract := range contracts { + total += len(contract.Metrics) + } + return total +} + +type usageError struct{ err error } + +func (u usageError) Error() string { return u.err.Error() } + +func (u usageError) Unwrap() error { return u.err } + +type multiStringFlag struct{ items []string } + +func (m *multiStringFlag) String() string { + if m == nil { + return "" + } + return strings.Join(m.items, ",") +} + +func (m *multiStringFlag) Set(value string) error { + m.items = append(m.items, value) + return nil +} + +func (m *multiStringFlag) values() []string { + out := make([]string, len(m.items)) + copy(out, m.items) + return out +} diff --git a/tools/perfcontracts/main_test.go b/tools/perfcontracts/main_test.go new file mode 100644 index 00000000..6261fd8c --- /dev/null +++ b/tools/perfcontracts/main_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunPassesForRequiredSharedLinuxMetrics(t *testing.T) { + root := t.TempDir() + writeContractsFixture(t, root) + checkOutput := filepath.Join(root, "check.json") + writeFile(t, checkOutput, `{"schema_version":"v1","measurements":[{"metric_id":"metric.tui.attach.latency.p95","value":420,"unit":"ms"}]}`) + if err := run([]string{"--contracts-root", root, "--check-output", checkOutput, "--lane", "required_shared_linux"}); err != nil { + t.Fatalf("run returned error: %v", err) + } +} + +func TestRunFailsOnThresholdViolation(t *testing.T) { + root := t.TempDir() + writeContractsFixture(t, root) + checkOutput := filepath.Join(root, "check.json") + writeFile(t, checkOutput, `{"schema_version":"v1","measurements":[{"metric_id":"metric.tui.attach.latency.p95","value":999,"unit":"ms"}]}`) + if err := run([]string{"--contracts-root", root, "--check-output", checkOutput, "--lane", "required_shared_linux"}); err == nil { + t.Fatal("run error = nil, want threshold violation") + } +} + +func TestRunIgnoresInformationalAndPendingMetrics(t *testing.T) { + root := t.TempDir() + writeContractsFixture(t, root) + checkOutput := filepath.Join(root, "check.json") + writeFile(t, checkOutput, `{"schema_version":"v1","measurements":[{"metric_id":"metric.tui.attach.latency.p95","value":420,"unit":"ms"},{"metric_id":"metric.broker.watch.latency.p95","value":999,"unit":"ms"},{"metric_id":"metric.anchor.prepare.latency.p95","value":999,"unit":"ms"}]}`) + if err := run([]string{"--contracts-root", root, "--check-output", checkOutput, "--lane", "required_shared_linux"}); err != nil { + t.Fatalf("run returned error for non-required metrics: %v", err) + } +} + +func TestRunFiltersRequiredLaneByMetricID(t *testing.T) { + root := t.TempDir() + writeContractsFixture(t, root) + checkOutput := filepath.Join(root, "check.json") + writeFile(t, checkOutput, `{"schema_version":"v1","measurements":[{"metric_id":"metric.tui.attach.latency.p95","value":420,"unit":"ms"},{"metric_id":"metric.tui.key_response.quiet.p95_ms","value":999,"unit":"ms"}]}`) + appendRequiredMetric(t, root) + if err := run([]string{"--contracts-root", root, "--check-output", checkOutput, "--lane", "required_shared_linux", "--metric-id", "metric.tui.attach.latency.p95"}); err != nil { + t.Fatalf("run returned error for filtered required metric: %v", err) + } +} + +func writeContractsFixture(t *testing.T, root string) { + t.Helper() + contractsDir, baselinesDir := createFixtureDirs(t, root) + writeFixtureManifest(t, root) + writeFixtureInventory(t, root) + writeFixtureContract(t, contractsDir) + writeFixtureBaseline(t, baselinesDir) +} + +func createFixtureDirs(t *testing.T, root string) (string, string) { + t.Helper() + contractsDir := filepath.Join(root, "contracts") + baselinesDir := filepath.Join(root, "baselines") + if err := os.MkdirAll(contractsDir, 0o755); err != nil { + t.Fatalf("MkdirAll contracts: %v", err) + } + if err := os.MkdirAll(baselinesDir, 0o755); err != nil { + t.Fatalf("MkdirAll baselines: %v", err) + } + return contractsDir, baselinesDir +} + +func writeFixtureManifest(t *testing.T, root string) { + t.Helper() + writeFile(t, filepath.Join(root, "manifest.json"), `{ + "schema_version":"runecode.performance.manifest.v1", + "manifest_version":"1", + "change_ref":"CHG-2026-053-9d2b-performance-baselines-verification-gates-v0", + "fixture_inventory_ref":"fixtures.json", + "contracts":[{"surface":"tui","path":"contracts/tui.json"}], + "baselines":[{"metric_id":"metric.tui.render.ns","path":"baselines/metric.tui.render.ns.json"}], + "metric_taxonomy":{"budget_classes":["exact","absolute-budget","regression-budget","hybrid-budget"]}, + "lane_authorities":["required_shared_linux","required_tight_linux","informational_until_stable","contract_pending_dependency","extended"], + "activation_states":["defined","informational","required","contract_pending_dependency"] + }`) +} + +func writeFixtureInventory(t *testing.T, root string) { + t.Helper() + writeFile(t, filepath.Join(root, "fixtures.json"), `{ + "schema_version":"runecode.performance.fixtures.v1", + "fixtures":[ + {"fixture_id":"tui.empty.v1","surface":"tui","runtime_regime":"empty","status":"mvp_reviewed"}, + {"fixture_id":"broker.watch.run.snapshot-follow.v1","surface":"broker","runtime_regime":"watch","status":"mvp_reviewed"}, + {"fixture_id":"anchor.fast-complete.stub.v1","surface":"external-anchor","runtime_regime":"prepare","status":"mvp_reviewed"} + ] + }`) +} + +func writeFixtureContract(t *testing.T, contractsDir string) { + t.Helper() + writeFile(t, filepath.Join(contractsDir, "tui.json"), fixtureContractJSON) +} + +const fixtureContractJSON = `{ + "schema_version":"runecode.performance.contract.v1", + "contract_id":"performance.tui.v1", + "surface":"tui", + "metrics":[ + { + "metric_id":"metric.tui.attach.latency.p95", + "subsystem":"tui", + "runtime_regime":"attach", + "fixture_id":"tui.empty.v1", + "measurement_kind":"latency", + "unit":"ms", + "authoritative_environment":"linux_shared_ci", + "sampling_policy":{"trials":30,"p95_authoritative":true}, + "budget_class":"absolute-budget", + "threshold":{"max_value":500}, + "lane_authority":"required_shared_linux", + "activation_state":"required", + "comparison_method":"p95_ceiling", + "threshold_origin":"product_budget", + "timing_boundary":{"start_event":"tui.process.spawn","end_event":"broker.attach.ready","clock_source":"monotonic","evidence_source":"pty_transcript","included_phases":["launch","attach"]} + }, + { + "metric_id":"metric.broker.watch.latency.p95", + "subsystem":"broker", + "runtime_regime":"watch", + "fixture_id":"broker.watch.run.snapshot-follow.v1", + "measurement_kind":"latency", + "unit":"ms", + "authoritative_environment":"linux_shared_ci", + "sampling_policy":{"trials":30,"p95_authoritative":true}, + "budget_class":"absolute-budget", + "threshold":{"max_value":200}, + "lane_authority":"informational_until_stable", + "activation_state":"informational", + "comparison_method":"p95_ceiling", + "threshold_origin":"first_calibration", + "timing_boundary":{"start_event":"rpc.request_sent","end_event":"watch.snapshot_follow_received","clock_source":"monotonic","evidence_source":"broker_events","included_phases":["watch"]} + }, + { + "metric_id":"metric.anchor.prepare.latency.p95", + "subsystem":"external-anchor", + "runtime_regime":"prepare", + "fixture_id":"anchor.fast-complete.stub.v1", + "measurement_kind":"latency", + "unit":"ms", + "authoritative_environment":"linux_shared_ci", + "sampling_policy":{"trials":30,"p95_authoritative":true}, + "budget_class":"absolute-budget", + "threshold":{"max_value":500}, + "lane_authority":"contract_pending_dependency", + "activation_state":"contract_pending_dependency", + "comparison_method":"p95_ceiling", + "threshold_origin":"temporary_guardrail", + "timing_boundary":{"start_event":"anchor.prepare.begin","end_event":"anchor.prepare.persisted","clock_source":"monotonic","evidence_source":"broker_events","included_phases":["prepare"]} + } + ] + }` + +func writeFixtureBaseline(t *testing.T, baselinesDir string) { + t.Helper() + writeFile(t, filepath.Join(baselinesDir, "metric.tui.render.ns.json"), `{"schema_version":"runecode.performance.baseline.v1","metric_id":"metric.tui.render.ns","unit":"ns/op","samples":[100,101,99],"summary":{"median":100}}`) +} + +func appendRequiredMetric(t *testing.T, root string) { + t.Helper() + contractPath := filepath.Join(root, "contracts", "tui.json") + raw, err := os.ReadFile(contractPath) + if err != nil { + t.Fatalf("ReadFile(%s): %v", contractPath, err) + } + content := strings.TrimSpace(string(raw)) + content = strings.TrimSuffix(content, "}") + content = strings.TrimSpace(content) + content = strings.TrimSuffix(content, "]") + `, + { + "metric_id":"metric.tui.key_response.quiet.p95_ms", + "subsystem":"tui", + "runtime_regime":"key_response_quiet", + "fixture_id":"tui.empty.v1", + "measurement_kind":"latency", + "unit":"ms", + "authoritative_environment":"linux_shared_ci", + "sampling_policy":{"trials":30,"p95_authoritative":true}, + "budget_class":"absolute-budget", + "threshold":{"max_value":50}, + "lane_authority":"required_shared_linux", + "activation_state":"required", + "comparison_method":"p95_ceiling", + "threshold_origin":"product_budget", + "timing_boundary":{"start_event":"tui.key.injected","end_event":"broker.attach.ready.frame_delta","clock_source":"monotonic","evidence_source":"pty_transcript","included_phases":["input_dispatch","render"]} + } + ] + }` + writeFile(t, contractPath, content) +} + +func writeFile(t *testing.T, path string, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile(%s) error: %v", path, err) + } +} diff --git a/tools/perfcontracts/manifest.json b/tools/perfcontracts/manifest.json new file mode 100644 index 00000000..46852366 --- /dev/null +++ b/tools/perfcontracts/manifest.json @@ -0,0 +1,58 @@ +{ + "schema_version":"runecode.performance.manifest.v1", + "manifest_version": "1", + "change_ref": "CHG-2026-053-9d2b-performance-baselines-verification-gates-v0", + "fixture_inventory_ref": "fixtures/inventory.v1.json", + "contracts": [ + {"surface": "tui", "path": "contracts/tui.v1.json"}, + {"surface": "broker", "path": "contracts/broker.v1.json"}, + {"surface": "runner-workflow", "path": "contracts/runner-workflow.v1.json"}, + {"surface": "launcher", "path": "contracts/launcher.v1.json"}, + {"surface": "dependency-audit", "path": "contracts/dependency-audit.v1.json"}, + {"surface": "gateway-secrets", "path": "contracts/gateway-secrets.v1.json"}, + {"surface": "external-anchor", "path": "contracts/external-anchor.v1.json"}, + {"surface": "attestation", "path": "contracts/attestation.v1.json"} + ], + "baselines": [ + {"metric_id": "metric.tui.render.shell_view_waiting.ns_op", "path": "baselines/metric.tui.render.shell_view_waiting.ns_op.v1.json"}, + {"metric_id": "metric.broker.unary.session_list.p95_ms", "path": "baselines/metric.broker.unary.session_list.p95_ms.v1.json"}, + {"metric_id": "metric.runner.boundary_check.wall_ms", "path": "baselines/metric.runner.boundary_check.wall_ms.v1.json"}, + {"metric_id": "metric.runner.protocol_fixtures.wall_ms", "path": "baselines/metric.runner.protocol_fixtures.wall_ms.v1.json"}, + {"metric_id": "metric.workflow.chg049.first_party_beta_slice.wall_ms", "path": "baselines/metric.workflow.chg049.first_party_beta_slice.wall_ms.v1.json"}, + {"metric_id": "metric.workflow.chg050.compile.wall_ms", "path": "baselines/metric.workflow.chg050.compile.wall_ms.v1.json"}, + {"metric_id": "metric.workflow.chg050.validation_canonicalization.wall_ms", "path": "baselines/metric.workflow.chg050.validation_canonicalization.wall_ms.v1.json"}, + {"metric_id": "metric.workflow.chg050.runplan_persist_load.wall_ms", "path": "baselines/metric.workflow.chg050.runplan_persist_load.wall_ms.v1.json"}, + {"metric_id": "metric.deps.cache_miss.small.wall_ms", "path": "baselines/metric.deps.cache_miss.small.wall_ms.v1.json"}, + {"metric_id": "metric.deps.cache_hit.small.wall_ms", "path": "baselines/metric.deps.cache_hit.small.wall_ms.v1.json"}, + {"metric_id": "metric.deps.materialization.workspace_handoff.wall_ms", "path": "baselines/metric.deps.materialization.workspace_handoff.wall_ms.v1.json"}, + {"metric_id": "metric.audit.verify_current_segment.wall_ms", "path": "baselines/metric.audit.verify_current_segment.wall_ms.v1.json"}, + {"metric_id": "metric.audit.finalize_verify.wall_ms", "path": "baselines/metric.audit.finalize_verify.wall_ms.v1.json"} + ], + "metric_taxonomy": { + "budget_classes": [ + "exact", + "absolute-budget", + "regression-budget", + "hybrid-budget" + ] + }, + "lane_authorities": [ + "required_shared_linux", + "required_tight_linux", + "informational_until_stable", + "contract_pending_dependency", + "extended" + ], + "activation_states": [ + "defined", + "informational", + "required", + "contract_pending_dependency" + ], + "deferrals": [ + { + "change_ref": "CHG-2026-061-45fe-performance-program-expansion-cross-platform-gates-v0", + "reason": "broader fixture ladders and cross-platform numeric-gate expansion" + } + ] +} diff --git a/tools/perfgatesharedlinux/main.go b/tools/perfgatesharedlinux/main.go new file mode 100644 index 00000000..07f8ebdb --- /dev/null +++ b/tools/perfgatesharedlinux/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/projectsubstrate" +) + +const checkSchemaVersion = "runecode.performance.check.v1" + +type config struct { + outputPath string + repository string + trials int + timeout time.Duration +} + +type deps struct { + runRunnerWorkflow func(repoRoot string, timeout time.Duration) (perfcontracts.CheckOutput, error) + runBrokerPerf func(repoRoot string, trials int) (perfcontracts.CheckOutput, error) + runPhase5Perf func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) + runTUIQuiet func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) + runTUIWaiting func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) + runTUIBench func(repoRoot string, timeout time.Duration) (perfcontracts.MeasurementRecord, error) + listRequiredIDs func(repoRoot string) ([]string, error) +} + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "perfgatesharedlinux usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "perfgatesharedlinux failed: %v\n", err) + os.Exit(1) + } +} + +func parseArgs(args []string) (config, error) { + fs := flag.NewFlagSet("perfgatesharedlinux", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + output := fs.String("output", "", "output check json path") + repository := fs.String("repository-root", "", "repository root path") + trials := fs.Int("trials", 30, "deterministic broker unary trials") + timeoutMs := fs.Int("timeout-ms", 120000, "runner command timeout milliseconds") + if err := fs.Parse(args); err != nil { + return config{}, usageError{err: err} + } + if strings.TrimSpace(*output) == "" { + return config{}, usageError{err: fmt.Errorf("--output is required")} + } + if *trials <= 0 { + return config{}, usageError{err: fmt.Errorf("--trials must be > 0")} + } + timeout := time.Duration(*timeoutMs) * time.Millisecond + if timeout <= 0 { + return config{}, usageError{err: fmt.Errorf("--timeout-ms must be > 0")} + } + return config{ + outputPath: strings.TrimSpace(*output), + repository: strings.TrimSpace(*repository), + trials: *trials, + timeout: timeout, + }, nil +} + +func resolveRepoRoot(explicit string) (string, error) { + if strings.TrimSpace(explicit) != "" { + return validateRepoRoot(strings.TrimSpace(explicit)) + } + cwd, err := os.Getwd() + if err != nil { + return "", err + } + return validateRepoRoot(cwd) +} + +func validateRepoRoot(root string) (string, error) { + clean := filepath.Clean(root) + if _, err := os.Stat(filepath.Join(clean, "runner", "package.json")); err != nil { + return "", fmt.Errorf("repository root missing runner/package.json: %w", err) + } + if _, err := os.Stat(filepath.Join(clean, "protocol", "schemas")); err != nil { + return "", fmt.Errorf("repository root missing protocol/schemas: %w", err) + } + if _, err := projectsubstrate.DiscoverAndValidate(projectsubstrate.DiscoveryInput{RepositoryRoot: clean, Authority: projectsubstrate.RepoRootAuthorityExplicitConfig}); err != nil { + return "", fmt.Errorf("repository root validation failed: %w", err) + } + return clean, nil +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/perfgatesharedlinux/main_test.go b/tools/perfgatesharedlinux/main_test.go new file mode 100644 index 00000000..a4eed4b9 --- /dev/null +++ b/tools/perfgatesharedlinux/main_test.go @@ -0,0 +1,290 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func repositoryRootForPerfGateTests(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) +} + +func TestParseArgsRequiresOutput(t *testing.T) { + _, err := parseArgs([]string{}) + if err == nil { + t.Fatal("parseArgs error = nil, want required output error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("parseArgs error = %T, want usageError", err) + } +} + +func TestRunWithDepsWritesMergedOutput(t *testing.T) { + root := repositoryRootForPerfGateTests(t) + output := filepath.Join(root, "check.json") + cfg := config{outputPath: output, repository: root, trials: 30, timeout: 2 * time.Second} + defer os.Remove(output) + err := runWithDeps(cfg, mergedOutputDeps(t, root)) + if err != nil { + t.Fatalf("runWithDeps error: %v", err) + } + + raw, err := os.ReadFile(output) + if err != nil { + t.Fatalf("ReadFile output: %v", err) + } + var parsed perfcontracts.CheckOutput + if err := json.Unmarshal(raw, &parsed); err != nil { + t.Fatalf("Unmarshal output: %v", err) + } + if parsed.SchemaVersion != checkSchemaVersion { + t.Fatalf("schema_version = %q, want %q", parsed.SchemaVersion, checkSchemaVersion) + } + if len(parsed.Measurements) != 7 { + t.Fatalf("measurements = %d, want 7", len(parsed.Measurements)) + } +} + +func mergedOutputDeps(t *testing.T, root string) deps { + t.Helper() + return deps{ + runRunnerWorkflow: expectedHarnessOutput(t, root, 2*time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.runner.boundary_check.wall_ms", Value: 50, Unit: "ms"}, + perfcontracts.MeasurementRecord{MetricID: "metric.runner.protocol_fixtures.wall_ms", Value: 75, Unit: "ms"}, + ), + runBrokerPerf: expectedBrokerOutput(t, root, 30, + perfcontracts.MeasurementRecord{MetricID: "metric.broker.unary.session_list.p95_ms", Value: 1, Unit: "ms"}, + ), + runPhase5Perf: expectedPhase5Output(t, root, 30, 2*time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.protocol.schema_validation.wall_ms", Value: 10, Unit: "ms"}, + ), + runTUIQuiet: expectedTUIOutput(t, root, 30, 2*time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.tui.attach.quiet.p95_ms", Value: 20, Unit: "ms"}, + ), + runTUIWaiting: expectedTUIOutput(t, root, 30, 2*time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.tui.attach.waiting.p95_ms", Value: 22, Unit: "ms"}, + ), + runTUIBench: expectedTUIBench(t, root, 2*time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.tui.render.shell_view_waiting.ns_op", Value: 1234, Unit: "ns/op"}, + ), + listRequiredIDs: expectedRequiredIDs(t, root, + "metric.runner.boundary_check.wall_ms", + "metric.runner.protocol_fixtures.wall_ms", + "metric.broker.unary.session_list.p95_ms", + "metric.protocol.schema_validation.wall_ms", + "metric.tui.attach.quiet.p95_ms", + "metric.tui.attach.waiting.p95_ms", + "metric.tui.render.shell_view_waiting.ns_op", + ), + } +} + +func expectedRequiredIDs(t *testing.T, root string, metricIDs ...string) func(string) ([]string, error) { + t.Helper() + return func(repoRoot string) ([]string, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + return metricIDs, nil + } +} + +func expectedHarnessOutput(t *testing.T, root string, wantTimeout time.Duration, items ...perfcontracts.MeasurementRecord) func(string, time.Duration) (perfcontracts.CheckOutput, error) { + t.Helper() + return func(repoRoot string, timeout time.Duration) (perfcontracts.CheckOutput, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + if timeout != wantTimeout { + t.Fatalf("timeout = %s, want %s", timeout, wantTimeout) + } + return perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: items}, nil + } +} + +func expectedBrokerOutput(t *testing.T, root string, wantTrials int, items ...perfcontracts.MeasurementRecord) func(string, int) (perfcontracts.CheckOutput, error) { + t.Helper() + return func(repoRoot string, trials int) (perfcontracts.CheckOutput, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + if trials != wantTrials { + t.Fatalf("trials = %d, want %d", trials, wantTrials) + } + return perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: items}, nil + } +} + +func expectedPhase5Output(t *testing.T, root string, wantTrials int, wantTimeout time.Duration, items ...perfcontracts.MeasurementRecord) func(string, int, time.Duration) (perfcontracts.CheckOutput, error) { + t.Helper() + return func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + if trials != wantTrials { + t.Fatalf("trials = %d, want %d", trials, wantTrials) + } + if timeout != wantTimeout { + t.Fatalf("timeout = %s, want %s", timeout, wantTimeout) + } + return perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: items}, nil + } +} + +func expectedTUIOutput(t *testing.T, root string, wantTrials int, wantTimeout time.Duration, items ...perfcontracts.MeasurementRecord) func(string, int, time.Duration) (perfcontracts.CheckOutput, error) { + t.Helper() + return func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + if trials != wantTrials { + t.Fatalf("trials = %d, want %d", trials, wantTrials) + } + if timeout != wantTimeout { + t.Fatalf("timeout = %s, want %s", timeout, wantTimeout) + } + return perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: items}, nil + } +} + +func expectedTUIBench(t *testing.T, root string, wantTimeout time.Duration, item perfcontracts.MeasurementRecord) func(string, time.Duration) (perfcontracts.MeasurementRecord, error) { + t.Helper() + return func(repoRoot string, timeout time.Duration) (perfcontracts.MeasurementRecord, error) { + if repoRoot != root { + t.Fatalf("repoRoot = %q, want %q", repoRoot, root) + } + if timeout != wantTimeout { + t.Fatalf("timeout = %s, want %s", timeout, wantTimeout) + } + return item, nil + } +} + +func TestRunWithDepsPropagatesHarnessError(t *testing.T) { + root := repositoryRootForPerfGateTests(t) + cfg := config{outputPath: filepath.Join(root, "check.json"), repository: root, trials: 30, timeout: time.Second} + err := runWithDeps(cfg, deps{ + runRunnerWorkflow: func(_ string, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, assertErr("runner workflow boom") + }, + runBrokerPerf: func(_ string, _ int) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runPhase5Perf: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIQuiet: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIWaiting: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIBench: func(_ string, _ time.Duration) (perfcontracts.MeasurementRecord, error) { + return perfcontracts.MeasurementRecord{}, nil + }, + listRequiredIDs: func(_ string) ([]string, error) { return nil, nil }, + }) + if err == nil || !strings.Contains(err.Error(), "run runner workflow perf") { + t.Fatalf("runWithDeps error = %v, want runner workflow context", err) + } +} + +func TestRunWithDepsPropagatesBrokerError(t *testing.T) { + root := repositoryRootForPerfGateTests(t) + cfg := config{outputPath: filepath.Join(root, "check.json"), repository: root, trials: 30, timeout: time.Second} + err := runWithDeps(cfg, deps{ + runRunnerWorkflow: func(_ string, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: []perfcontracts.MeasurementRecord{{MetricID: "metric.runner.boundary_check.wall_ms", Value: 1, Unit: "ms"}}}, nil + }, + runBrokerPerf: func(_ string, _ int) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, assertErr("broker boom") + }, + runPhase5Perf: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIQuiet: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIWaiting: func(_ string, _ int, _ time.Duration) (perfcontracts.CheckOutput, error) { + return perfcontracts.CheckOutput{}, nil + }, + runTUIBench: func(_ string, _ time.Duration) (perfcontracts.MeasurementRecord, error) { + return perfcontracts.MeasurementRecord{}, nil + }, + listRequiredIDs: func(_ string) ([]string, error) { return nil, nil }, + }) + if err == nil || !strings.Contains(err.Error(), "run broker perf") { + t.Fatalf("runWithDeps error = %v, want broker context", err) + } +} + +func TestRunWithDepsRejectsInvalidRepositoryRoot(t *testing.T) { + t.Parallel() + cfg := config{outputPath: filepath.Join(t.TempDir(), "check.json"), repository: t.TempDir(), trials: 30, timeout: time.Second} + err := runWithDeps(cfg, deps{}) + if err == nil || !strings.Contains(err.Error(), "repository root") { + t.Fatalf("runWithDeps error = %v, want repository root validation failure", err) + } +} + +func TestRunWithDepsFailsWhenRequiredMetricMissingFromAggregation(t *testing.T) { + root := repositoryRootForPerfGateTests(t) + cfg := config{outputPath: filepath.Join(root, "check.json"), repository: root, trials: 30, timeout: time.Second} + err := runWithDeps(cfg, deps{ + runRunnerWorkflow: expectedHarnessOutput(t, root, time.Second, + perfcontracts.MeasurementRecord{MetricID: "metric.runner.boundary_check.wall_ms", Value: 1, Unit: "ms"}, + ), + runBrokerPerf: expectedBrokerOutput(t, root, 30), + runPhase5Perf: expectedPhase5Output(t, root, 30, time.Second), + runTUIQuiet: expectedTUIOutput(t, root, 30, time.Second), + runTUIWaiting: expectedTUIOutput(t, root, 30, time.Second), + runTUIBench: expectedTUIBench(t, root, time.Second, perfcontracts.MeasurementRecord{MetricID: "metric.tui.render.shell_view_waiting.ns_op", Value: 1, Unit: "ns/op"}), + listRequiredIDs: func(_ string) ([]string, error) { + return []string{"metric.runner.boundary_check.wall_ms", "metric.missing"}, nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "required_shared_linux metrics missing") { + t.Fatalf("runWithDeps error = %v, want missing required metric error", err) + } +} + +func TestParseBenchmarkMedianNSOp(t *testing.T) { + out := strings.Join([]string{ + "BenchmarkShellViewWaitingSession-8 12493 8912 ns/op 1200 B/op 10 allocs/op", + "BenchmarkShellViewWaitingSession-8 12493 9012 ns/op 1200 B/op 10 allocs/op", + "BenchmarkShellViewWaitingSession-8 12493 8812 ns/op 1200 B/op 10 allocs/op", + }, "\n") + "\n" + value, err := parseBenchmarkMedianNSOp(out, "BenchmarkShellViewWaitingSession", 3) + if err != nil { + t.Fatalf("parseBenchmarkMedianNSOp returned error: %v", err) + } + if value != 8912 { + t.Fatalf("value = %v, want 8912", value) + } +} + +func TestParseBenchmarkMedianNSOpRejectsWrongSampleCount(t *testing.T) { + out := "BenchmarkShellViewWaitingSession-8 12493 8912 ns/op 1200 B/op 10 allocs/op\n" + _, err := parseBenchmarkMedianNSOp(out, "BenchmarkShellViewWaitingSession", 2) + if err == nil || !strings.Contains(err.Error(), "sample count") { + t.Fatalf("parseBenchmarkMedianNSOp error = %v, want sample count failure", err) + } +} + +type assertErr string + +func (e assertErr) Error() string { return string(e) } diff --git a/tools/perfgatesharedlinux/required.go b/tools/perfgatesharedlinux/required.go new file mode 100644 index 00000000..0d723fc5 --- /dev/null +++ b/tools/perfgatesharedlinux/required.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +func requiredSharedLinuxMetricIDs(repoRoot string) ([]string, error) { + root := filepath.Join(repoRoot, "tools", "perfcontracts") + manifest, err := perfcontracts.LoadManifest(root) + if err != nil { + return nil, err + } + required, err := requiredMetricSet(root, manifest.Contracts) + if err != nil { + return nil, err + } + return sortedMetricIDs(required), nil +} + +func requiredMetricSet(root string, entries []perfcontracts.ManifestContract) (map[string]struct{}, error) { + required := map[string]struct{}{} + for _, entry := range entries { + contract, err := perfcontracts.LoadContract(root, entry.Path) + if err != nil { + return nil, err + } + collectRequiredMetrics(required, contract.Metrics) + } + return required, nil +} + +func collectRequiredMetrics(required map[string]struct{}, metrics []perfcontracts.MetricContract) { + for _, metric := range metrics { + if metric.LaneAuthority == "required_shared_linux" && metric.ActivationState == "required" { + required[metric.MetricID] = struct{}{} + } + } +} + +func sortedMetricIDs(required map[string]struct{}) []string { + out := make([]string, 0, len(required)) + for metricID := range required { + out = append(out, metricID) + } + sort.Strings(out) + return out +} + +func selectRequiredMeasurements(measurements []perfcontracts.MeasurementRecord, requiredIDs []string) ([]perfcontracts.MeasurementRecord, error) { + byMetric := map[string]perfcontracts.MeasurementRecord{} + for _, measurement := range measurements { + byMetric[measurement.MetricID] = measurement + } + selected := make([]perfcontracts.MeasurementRecord, 0, len(requiredIDs)) + missing := make([]string, 0) + for _, metricID := range requiredIDs { + measurement, ok := byMetric[metricID] + if !ok { + missing = append(missing, metricID) + continue + } + selected = append(selected, measurement) + } + if len(missing) > 0 { + return nil, fmt.Errorf("required_shared_linux metrics missing from aggregated output: %s", strings.Join(missing, ", ")) + } + return selected, nil +} diff --git a/tools/perfgatesharedlinux/run.go b/tools/perfgatesharedlinux/run.go new file mode 100644 index 00000000..aaa41093 --- /dev/null +++ b/tools/perfgatesharedlinux/run.go @@ -0,0 +1,103 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/brokerperf" + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/runnerworkflowperf" +) + +func run(args []string) error { + cfg, err := parseArgs(args) + if err != nil { + return err + } + return runWithDeps(cfg, deps{ + runRunnerWorkflow: func(repoRoot string, timeout time.Duration) (perfcontracts.CheckOutput, error) { + return runnerworkflowperf.Run(runnerworkflowperf.HarnessConfig{RepositoryRoot: repoRoot, CommandTimeout: timeout}) + }, + runBrokerPerf: func(repoRoot string, trials int) (perfcontracts.CheckOutput, error) { + return brokerperf.Run(brokerperf.HarnessConfig{RepositoryRoot: repoRoot, Trials: trials}) + }, + runPhase5Perf: func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + return brokerapi.RunPhase5PerformanceHarness(brokerapi.Phase5PerformanceHarnessConfig{RepositoryRoot: repoRoot, Trials: trials, CommandTimeout: timeout}) + }, + runTUIQuiet: func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + return measureTUILatency(repoRoot, "tui.empty.v1", trials, timeout) + }, + runTUIWaiting: func(repoRoot string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + return measureTUILatency(repoRoot, "tui.waiting.v1", trials, timeout) + }, + runTUIBench: measureTUIRenderWaitingBenchmark, + listRequiredIDs: requiredSharedLinuxMetricIDs, + }) +} + +func runWithDeps(cfg config, d deps) error { + repoRoot, err := resolveRepoRoot(cfg.repository) + if err != nil { + return err + } + measurements, err := collectMeasurements(cfg, repoRoot, d) + if err != nil { + return err + } + requiredIDs, err := d.listRequiredIDs(repoRoot) + if err != nil { + return err + } + measurements, err = selectRequiredMeasurements(measurements, requiredIDs) + if err != nil { + return err + } + raw, err := json.MarshalIndent(perfcontracts.CheckOutput{SchemaVersion: checkSchemaVersion, Measurements: measurements}, "", " ") + if err != nil { + return err + } + return os.WriteFile(cfg.outputPath, raw, 0o644) +} + +func collectMeasurements(cfg config, repoRoot string, d deps) ([]perfcontracts.MeasurementRecord, error) { + runnerWorkflow, err := d.runRunnerWorkflow(repoRoot, cfg.timeout) + if err != nil { + return nil, fmt.Errorf("run runner workflow perf: %w", err) + } + brokerOut, err := d.runBrokerPerf(repoRoot, cfg.trials) + if err != nil { + return nil, fmt.Errorf("run broker perf: %w", err) + } + phase5Out, err := d.runPhase5Perf(repoRoot, cfg.trials, cfg.timeout) + if err != nil { + return nil, fmt.Errorf("run phase5 perf: %w", err) + } + tuiQuietOut, err := d.runTUIQuiet(repoRoot, cfg.trials, cfg.timeout) + if err != nil { + return nil, fmt.Errorf("run tui latency (quiet) perf: %w", err) + } + tuiWaitingOut, err := d.runTUIWaiting(repoRoot, cfg.trials, cfg.timeout) + if err != nil { + return nil, fmt.Errorf("run tui latency (waiting) perf: %w", err) + } + tuiBench, err := d.runTUIBench(repoRoot, cfg.timeout) + if err != nil { + return nil, fmt.Errorf("run tui benchmark perf: %w", err) + } + return mergedMeasurements(tuiBench, runnerWorkflow, brokerOut, phase5Out, tuiQuietOut, tuiWaitingOut), nil +} + +func mergedMeasurements(tuiBench perfcontracts.MeasurementRecord, outputs ...perfcontracts.CheckOutput) []perfcontracts.MeasurementRecord { + total := 1 + for _, output := range outputs { + total += len(output.Measurements) + } + measurements := make([]perfcontracts.MeasurementRecord, 0, total) + for _, output := range outputs { + measurements = append(measurements, output.Measurements...) + } + return append(measurements, tuiBench) +} diff --git a/tools/perfgatesharedlinux/tui.go b/tools/perfgatesharedlinux/tui.go new file mode 100644 index 00000000..39477169 --- /dev/null +++ b/tools/perfgatesharedlinux/tui.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +const tuiRenderBenchmarkSamples = 10 + +func measureTUILatency(repoRoot, fixtureID string, trials int, timeout time.Duration) (perfcontracts.CheckOutput, error) { + tmpDir, err := os.MkdirTemp("", "runecode-perfgate-tui-latency-") + if err != nil { + return perfcontracts.CheckOutput{}, err + } + defer os.RemoveAll(tmpDir) + outputPath := filepath.Join(tmpDir, "latency.json") + ctx, cancel := context.WithTimeout(context.Background(), latencyBatchTimeout(timeout, trials)) + defer cancel() + cmd := exec.CommandContext(ctx, + "go", "run", "./tools/tuiperf", + "--mode", "latency", + "--output", outputPath, + "--fixture-id", fixtureID, + "--runtime-dir", filepath.Join(tmpDir, "runtime"), + "--socket-name", "runecode.sock", + "--state-root", filepath.Join(tmpDir, "state"), + "--audit-ledger-root", filepath.Join(tmpDir, "audit-ledger"), + "--target-alias", "default", + "--trials", strconv.Itoa(trials), + "--timeout-ms", strconv.Itoa(int(timeout.Milliseconds())), + ) + cmd.Dir = repoRoot + cmd.Stdout = io.Discard + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return perfcontracts.CheckOutput{}, commandFailure(err, stderr.String(), fmt.Sprintf("tuiperf latency %s", fixtureID)) + } + return perfcontracts.LoadCheckOutput(outputPath) +} + +func latencyBatchTimeout(perTrialTimeout time.Duration, trials int) time.Duration { + if trials < 1 { + trials = 1 + } + estimated := 30*time.Second + time.Duration(trials)*7*time.Second + if estimated > perTrialTimeout { + return estimated + } + return perTrialTimeout +} + +func measureTUIRenderWaitingBenchmark(repoRoot string, timeout time.Duration) (perfcontracts.MeasurementRecord, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "test", "./cmd/runecode-tui", "-run", "^$", "-bench", "BenchmarkShellViewWaitingSession$", "-benchmem", "-count", strconv.Itoa(tuiRenderBenchmarkSamples)) + cmd.Dir = repoRoot + out, err := cmd.CombinedOutput() + if err != nil { + return perfcontracts.MeasurementRecord{}, commandFailure(err, string(out), "run bench BenchmarkShellViewWaitingSession") + } + value, err := parseBenchmarkMedianNSOp(string(out), "BenchmarkShellViewWaitingSession", tuiRenderBenchmarkSamples) + if err != nil { + return perfcontracts.MeasurementRecord{}, err + } + return perfcontracts.MeasurementRecord{MetricID: "metric.tui.render.shell_view_waiting.ns_op", Value: value, Unit: "ns/op"}, nil +} + +func commandFailure(err error, output, label string) error { + msg := strings.TrimSpace(output) + if msg == "" { + msg = err.Error() + } + return fmt.Errorf("%s failed: %s", label, msg) +} + +func parseBenchmarkMedianNSOp(output, benchmark string, wantSamples int) (float64, error) { + values, err := parseBenchmarkNSOps(output, benchmark) + if err != nil { + return 0, err + } + if wantSamples > 0 && len(values) != wantSamples { + return 0, fmt.Errorf("benchmark %s ns/op sample count = %d, want %d", benchmark, len(values), wantSamples) + } + return medianFloat64(values), nil +} + +func parseBenchmarkNSOps(output, benchmark string) ([]float64, error) { + pattern := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(benchmark) + `-\d+\s+\d+\s+([0-9]+(?:\.[0-9]+)?)\s+ns/op`) + matches := pattern.FindAllStringSubmatch(output, -1) + if len(matches) == 0 { + return nil, fmt.Errorf("benchmark %s ns/op missing from output", benchmark) + } + values := make([]float64, 0, len(matches)) + for _, match := range matches { + value, err := strconv.ParseFloat(match[1], 64) + if err != nil { + return nil, fmt.Errorf("parse benchmark %s ns/op: %w", benchmark, err) + } + values = append(values, value) + } + return values, nil +} + +func medianFloat64(values []float64) float64 { + cp := append([]float64(nil), values...) + sort.Float64s(cp) + mid := len(cp) / 2 + if len(cp)%2 == 0 { + return (cp[mid-1] + cp[mid]) / 2 + } + return cp[mid] +} diff --git a/tools/perfseedwait/main.go b/tools/perfseedwait/main.go index 37ab68d5..76f6fbd4 100644 --- a/tools/perfseedwait/main.go +++ b/tools/perfseedwait/main.go @@ -9,16 +9,23 @@ import ( "time" "github.com/runecode-ai/runecode/internal/brokerapi" + "github.com/runecode-ai/runecode/internal/perffixtures" ) type config struct { runtimeDir string socketName string sessionID string + storeRoot string + fixtureID string } func main() { cfg := parseConfig() + if cfg.fixtureID != "" { + seedDeterministicStoreFixture(cfg) + return + } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() client, err := brokerapi.DialLocalRPC(ctx, brokerapi.LocalIPCConfig{RuntimeDir: cfg.runtimeDir, SocketName: cfg.socketName}) @@ -36,7 +43,16 @@ func parseConfig() config { runtimeDir := flag.String("runtime-dir", "", "runtime directory") socketName := flag.String("socket-name", "perf.sock", "socket name") sessionID := flag.String("session-id", "", "session id") + storeRoot := flag.String("store-root", "", "store root for deterministic fixture mode") + fixtureID := flag.String("fixture-id", "", "deterministic fixture id (tui.empty.v1 or tui.waiting.v1)") flag.Parse() + if *fixtureID != "" { + if *storeRoot == "" { + fmt.Fprintln(os.Stderr, "--store-root is required when --fixture-id is provided") + os.Exit(2) + } + return config{storeRoot: *storeRoot, fixtureID: *fixtureID} + } if *runtimeDir == "" { fmt.Fprintln(os.Stderr, "--runtime-dir is required") os.Exit(2) @@ -48,6 +64,20 @@ func parseConfig() config { return config{runtimeDir: *runtimeDir, socketName: *socketName, sessionID: *sessionID} } +func seedDeterministicStoreFixture(cfg config) { + result, err := perffixtures.BuildBrokerStoreFixture(cfg.storeRoot, cfg.fixtureID) + if err != nil { + fmt.Fprintf(os.Stderr, "build deterministic fixture: %v\n", err) + os.Exit(1) + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(result); err != nil { + fmt.Fprintf(os.Stderr, "encode fixture result: %v\n", err) + os.Exit(1) + } +} + func seedWaitSession(ctx context.Context, client *brokerapi.LocalRPCClient, cfg config) { for i, msg := range []string{"first", "second"} { resp := brokerapi.SessionExecutionTriggerResponse{} @@ -82,9 +112,18 @@ func printSession(ctx context.Context, client *brokerapi.LocalRPCClient, session os.Exit(1) } + summary := struct { + SessionID string `json:"session_id"` + Status string `json:"status"` + WorkState string `json:"work_posture"` + }{ + SessionID: result.Session.Summary.Identity.SessionID, + Status: result.Session.Summary.Status, + WorkState: result.Session.Summary.WorkPosture, + } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") - if err := enc.Encode(result.Session); err != nil { + if err := enc.Encode(summary); err != nil { fmt.Fprintf(os.Stderr, "encode result: %v\n", err) os.Exit(1) } diff --git a/tools/phase5perf/main.go b/tools/phase5perf/main.go new file mode 100644 index 00000000..b5979fd9 --- /dev/null +++ b/tools/phase5perf/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/brokerapi" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "phase5perf usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "phase5perf failed: %v\n", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("phase5perf", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + output := fs.String("output", "", "output check json path") + repositoryRoot := fs.String("repository-root", "", "repository root") + trials := fs.Int("trials", 10, "deterministic trial count for p95 metrics") + timeoutMS := fs.Int("timeout-ms", 120000, "per-command timeout milliseconds") + if err := fs.Parse(args); err != nil { + return usageError{err: err} + } + if strings.TrimSpace(*output) == "" { + return usageError{err: fmt.Errorf("--output is required")} + } + out, err := brokerapi.RunPhase5PerformanceHarness(brokerapi.Phase5PerformanceHarnessConfig{ + RepositoryRoot: strings.TrimSpace(*repositoryRoot), + Trials: *trials, + CommandTimeout: time.Duration(*timeoutMS) * time.Millisecond, + }) + if err != nil { + return err + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + return os.WriteFile(strings.TrimSpace(*output), raw, 0o644) +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/phase5perf/main_test.go b/tools/phase5perf/main_test.go new file mode 100644 index 00000000..1e2b615f --- /dev/null +++ b/tools/phase5perf/main_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "errors" + "testing" +) + +func TestRunReturnsUsageErrorWhenOutputMissing(t *testing.T) { + err := run([]string{}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} + +func TestRunReturnsUsageErrorForInvalidFlag(t *testing.T) { + err := run([]string{"--bad-flag"}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} diff --git a/tools/runnerworkflowperf/main.go b/tools/runnerworkflowperf/main.go new file mode 100644 index 00000000..3158c94d --- /dev/null +++ b/tools/runnerworkflowperf/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/runnerworkflowperf" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "runnerworkflowperf usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "runnerworkflowperf failed: %v\n", err) + os.Exit(1) + } +} + +func run(args []string) error { + fs := flag.NewFlagSet("runnerworkflowperf", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + output := fs.String("output", "", "output check json path") + repositoryRoot := fs.String("repository-root", "", "repository root") + timeoutMs := fs.Int("timeout-ms", 120000, "per-command timeout milliseconds") + if err := fs.Parse(args); err != nil { + return usageError{err: err} + } + if strings.TrimSpace(*output) == "" { + return usageError{err: fmt.Errorf("--output is required")} + } + if strings.TrimSpace(*repositoryRoot) == "" { + return usageError{err: fmt.Errorf("--repository-root is required")} + } + out, err := runnerworkflowperf.Run(runnerworkflowperf.HarnessConfig{ + RepositoryRoot: strings.TrimSpace(*repositoryRoot), + CommandTimeout: time.Duration(*timeoutMs) * time.Millisecond, + }) + if err != nil { + return err + } + raw, err := json.MarshalIndent(out, "", " ") + if err != nil { + return err + } + return os.WriteFile(strings.TrimSpace(*output), raw, 0o644) +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/runnerworkflowperf/main_test.go b/tools/runnerworkflowperf/main_test.go new file mode 100644 index 00000000..1e2b615f --- /dev/null +++ b/tools/runnerworkflowperf/main_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "errors" + "testing" +) + +func TestRunReturnsUsageErrorWhenOutputMissing(t *testing.T) { + err := run([]string{}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} + +func TestRunReturnsUsageErrorForInvalidFlag(t *testing.T) { + err := run([]string{"--bad-flag"}) + if err == nil { + t.Fatal("run error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("run error = %T, want usageError", err) + } +} diff --git a/tools/tlccheck/fs.go b/tools/tlccheck/fs.go new file mode 100644 index 00000000..967ea606 --- /dev/null +++ b/tools/tlccheck/fs.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" +) + +func resolveRepoRoot() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("resolve current directory: %w", err) + } + root, ok := findRepoRoot(cwd) + if !ok { + return "", fmt.Errorf("resolve repo root from %s: required markers go.mod, justfile, and %s", cwd, specDirRelative) + } + return root, nil +} + +func findRepoRoot(start string) (string, bool) { + dir := start + for { + if looksLikeRepoRoot(dir) { + return dir, true + } + parent := filepath.Dir(dir) + if parent == dir { + return "", false + } + dir = parent + } +} + +func looksLikeRepoRoot(path string) bool { + if !fileExists(filepath.Join(path, "go.mod")) { + return false + } + if !fileExists(filepath.Join(path, "justfile")) { + return false + } + return dirExists(filepath.Join(path, specDirRelative)) +} + +func ensureDir(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat dir %s: %w", path, err) + } + if !info.IsDir() { + return fmt.Errorf("expected directory: %s", path) + } + return nil +} + +func ensureFile(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("stat file %s: %w", path, err) + } + if info.IsDir() { + return fmt.Errorf("expected file, got directory: %s", path) + } + return nil +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return !info.IsDir() +} + +func dirExists(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} diff --git a/tools/tlccheck/main.go b/tools/tlccheck/main.go index ce5ccad1..ea00b139 100644 --- a/tools/tlccheck/main.go +++ b/tools/tlccheck/main.go @@ -3,6 +3,7 @@ package main import ( "errors" + "flag" "fmt" "os" "os/exec" @@ -29,13 +30,28 @@ var ( ) func main() { - if err := run(); err != nil { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "tlccheck usage error: %v\n", err) + os.Exit(2) + } fmt.Fprintf(os.Stderr, "tlc model check failed: %v\n", err) os.Exit(1) } } -func run() error { +func run(args []string) error { + fs := flag.NewFlagSet("tlccheck", flag.ContinueOnError) + mode := fs.String("mode", "all", "model-check mode: all, core, or replay") + if err := fs.Parse(args); err != nil { + return usageError{err: err} + } + configs, err := selectedModelConfigs(*mode) + if err != nil { + return err + } + repoRoot, err := resolveRepoRoot() if err != nil { return err @@ -51,7 +67,7 @@ func run() error { return err } - for _, cfg := range modelConfigs { + for _, cfg := range configs { cfgPath := filepath.Join(specDir, cfg) if err := ensureFile(cfgPath); err != nil { return err @@ -199,82 +215,8 @@ func nixTLCRunner(repoRoot, nixPath string) tlcRunner { } } -func resolveRepoRoot() (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("resolve current directory: %w", err) - } - - root, ok := findRepoRoot(cwd) - if !ok { - return "", fmt.Errorf("resolve repo root from %s: required markers go.mod, justfile, and %s", cwd, specDirRelative) - } - - return root, nil -} - -func findRepoRoot(start string) (string, bool) { - dir := start - for { - if looksLikeRepoRoot(dir) { - return dir, true - } - parent := filepath.Dir(dir) - if parent == dir { - return "", false - } - dir = parent - } -} +type usageError struct{ err error } -func looksLikeRepoRoot(path string) bool { - if !fileExists(filepath.Join(path, "go.mod")) { - return false - } - if !fileExists(filepath.Join(path, "justfile")) { - return false - } - return dirExists(filepath.Join(path, specDirRelative)) -} +func (e usageError) Error() string { return e.err.Error() } -func ensureDir(path string) error { - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("stat dir %s: %w", path, err) - } - if !info.IsDir() { - return fmt.Errorf("expected directory: %s", path) - } - - return nil -} - -func ensureFile(path string) error { - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("stat file %s: %w", path, err) - } - if info.IsDir() { - return fmt.Errorf("expected file, got directory: %s", path) - } - - return nil -} - -func fileExists(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - - return !info.IsDir() -} - -func dirExists(path string) bool { - info, err := os.Stat(path) - if err != nil { - return false - } - - return info.IsDir() -} +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/tlccheck/main_test.go b/tools/tlccheck/main_test.go index 0a3424bf..2049da26 100644 --- a/tools/tlccheck/main_test.go +++ b/tools/tlccheck/main_test.go @@ -146,6 +146,37 @@ func TestFindRepoRootWalksUpToRepoMarkers(t *testing.T) { } } +func TestSelectedModelConfigs(t *testing.T) { + tests := []struct { + mode string + want []string + }{ + {mode: "all", want: []string{"SecurityKernelV0.core.cfg", "SecurityKernelV0.replay.cfg"}}, + {mode: "core", want: []string{"SecurityKernelV0.core.cfg"}}, + {mode: "replay", want: []string{"SecurityKernelV0.replay.cfg"}}, + } + for _, test := range tests { + got, err := selectedModelConfigs(test.mode) + if err != nil { + t.Fatalf("selectedModelConfigs(%q) error = %v, want nil", test.mode, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("selectedModelConfigs(%q) = %#v, want %#v", test.mode, got, test.want) + } + } +} + +func TestSelectedModelConfigsRejectsUnknownMode(t *testing.T) { + _, err := selectedModelConfigs("unknown") + if err == nil { + t.Fatal("selectedModelConfigs error = nil, want unsupported mode failure") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("selectedModelConfigs error = %T, want usageError", err) + } +} + func lookPathStub(entries map[string]string) func(string) (string, error) { return func(file string) (string, error) { if path, ok := entries[file]; ok { diff --git a/tools/tlccheck/modes.go b/tools/tlccheck/modes.go new file mode 100644 index 00000000..8982039c --- /dev/null +++ b/tools/tlccheck/modes.go @@ -0,0 +1,19 @@ +package main + +import ( + "fmt" + "strings" +) + +func selectedModelConfigs(mode string) ([]string, error) { + switch strings.TrimSpace(mode) { + case "", "all": + return modelConfigs, nil + case "core": + return []string{"SecurityKernelV0.core.cfg"}, nil + case "replay": + return []string{"SecurityKernelV0.replay.cfg"}, nil + default: + return nil, usageError{err: fmt.Errorf("unsupported mode %q", mode)} + } +} diff --git a/tools/tuiperf/args.go b/tools/tuiperf/args.go new file mode 100644 index 00000000..f5eccfe3 --- /dev/null +++ b/tools/tuiperf/args.go @@ -0,0 +1,111 @@ +//go:build linux + +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +func run(args []string) error { + cfg, err := parseArgs(args) + if err != nil { + return err + } + return runMode(cfg) +} + +func runMode(cfg config) error { + switch cfg.mode { + case "cpu": + return runCPUMode(cfg) + case "latency": + return runLatencyMode(cfg) + case "bench-parse": + return runBenchParseMode(cfg) + default: + return usageError{err: fmt.Errorf("unsupported mode %q", cfg.mode)} + } +} + +func parseArgs(args []string) (config, error) { + fs := flag.NewFlagSet("tuiperf", flag.ContinueOnError) + fs.SetOutput(io.Discard) + mode := fs.String("mode", "", "cpu|latency|bench-parse") + output := fs.String("output", "", "output check json path") + fixtureID := fs.String("fixture-id", "", "tui.empty.v1|tui.waiting.v1") + runtimeDir := fs.String("runtime-dir", "", "isolated runtime dir") + socketName := fs.String("socket-name", "", "isolated socket name") + stateRoot := fs.String("state-root", "", "isolated broker state root") + auditLedgerRoot := fs.String("audit-ledger-root", "", "isolated broker audit ledger root") + targetAlias := fs.String("target-alias", "", "RUNECODE_TUI_BROKER_TARGET alias") + trials := fs.Int("trials", 30, "latency trials") + warmupMs := fs.Int("warmup-ms", 3000, "cpu warmup millis") + windowMs := fs.Int("window-ms", 20000, "cpu observation window millis") + windows := fs.Int("windows", 3, "cpu observation windows") + timeoutMs := fs.Int("timeout-ms", 120000, "mode timeout millis") + benchOutput := fs.String("bench-output", "", "go test bench output path for bench-parse mode") + if err := fs.Parse(args); err != nil { + return config{}, usageError{err: err} + } + return buildConfig(mode, output, fixtureID, runtimeDir, socketName, stateRoot, auditLedgerRoot, targetAlias, trials, warmupMs, windowMs, windows, timeoutMs, benchOutput) +} + +func buildConfig( + mode *string, + output *string, + fixtureID *string, + runtimeDir *string, + socketName *string, + stateRoot *string, + auditLedgerRoot *string, + targetAlias *string, + trials *int, + warmupMs *int, + windowMs *int, + windows *int, + timeoutMs *int, + benchOutput *string, +) (config, error) { + if strings.TrimSpace(*mode) == "" || strings.TrimSpace(*output) == "" { + return config{}, usageError{err: errors.New("--mode and --output are required")} + } + timeout := time.Duration(*timeoutMs) * time.Millisecond + if timeout <= 0 { + timeout = 120 * time.Second + } + wd, err := os.Getwd() + if err != nil { + return config{}, err + } + repoRoot := filepath.Clean(wd) + return config{ + mode: strings.TrimSpace(*mode), + outputPath: strings.TrimSpace(*output), + fixtureID: strings.TrimSpace(*fixtureID), + runtimeDir: strings.TrimSpace(*runtimeDir), + socketName: strings.TrimSpace(*socketName), + stateRoot: strings.TrimSpace(*stateRoot), + auditLedgerRoot: strings.TrimSpace(*auditLedgerRoot), + targetAlias: strings.TrimSpace(*targetAlias), + repoRoot: repoRoot, + trials: *trials, + warmup: time.Duration(*warmupMs) * time.Millisecond, + window: time.Duration(*windowMs) * time.Millisecond, + windows: *windows, + timeout: timeout, + benchOutput: strings.TrimSpace(*benchOutput), + }, nil +} + +type usageError struct{ err error } + +func (e usageError) Error() string { return e.err.Error() } + +func (e usageError) Unwrap() error { return e.err } diff --git a/tools/tuiperf/args_test.go b/tools/tuiperf/args_test.go new file mode 100644 index 00000000..5b0ca7e6 --- /dev/null +++ b/tools/tuiperf/args_test.go @@ -0,0 +1,30 @@ +//go:build linux + +package main + +import ( + "errors" + "testing" +) + +func TestParseArgsReturnsUsageErrorWhenRequiredFlagsMissing(t *testing.T) { + _, err := parseArgs([]string{}) + if err == nil { + t.Fatal("parseArgs error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("parseArgs error = %T, want usageError", err) + } +} + +func TestParseArgsReturnsUsageErrorForInvalidFlag(t *testing.T) { + _, err := parseArgs([]string{"--bad-flag"}) + if err == nil { + t.Fatal("parseArgs error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("parseArgs error = %T, want usageError", err) + } +} diff --git a/tools/tuiperf/harness.go b/tools/tuiperf/harness.go new file mode 100644 index 00000000..974187de --- /dev/null +++ b/tools/tuiperf/harness.go @@ -0,0 +1,261 @@ +//go:build linux + +package main + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" + + "github.com/creack/pty" +) + +const terminateGracePeriod = 2 * time.Second + +var ( + absPathPattern = regexp.MustCompile(`(?:[A-Za-z]:\\|/)[^\s"']+`) + longTokenPattern = regexp.MustCompile(`\b[A-Za-z0-9_=-]{24,}\b`) + wsPattern = regexp.MustCompile(`\s+`) + nonPrintablePattern = regexp.MustCompile(`[^[:print:]\t\n\r]`) +) + +type runningHarness struct { + ctx context.Context + cancel context.CancelFunc + brokerCmd *exec.Cmd + tuiCmd *exec.Cmd + tuiOut io.ReadCloser + tuiIn io.WriteCloser +} + +func startTUIHarness(cfg config) (context.Context, context.CancelFunc, runningHarness, error) { + if err := requireTUIFixtureConfig(cfg); err != nil { + return nil, nil, runningHarness{}, err + } + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + preparedCfg, err := prepareHarnessBinaries(cfg) + if err != nil { + cancel() + return nil, nil, runningHarness{}, err + } + cancelWithCleanup := func() { + cancel() + cleanupHarnessBinaries(preparedCfg) + } + if err := prepareTUIIsolation(cfg); err != nil { + cancelWithCleanup() + return nil, nil, runningHarness{}, err + } + harness, err := startHarnessProcesses(ctx, preparedCfg) + if err != nil { + cancelWithCleanup() + return nil, nil, runningHarness{}, err + } + return ctx, cancelWithCleanup, harness, nil +} + +func prepareHarnessBinaries(cfg config) (config, error) { + if cfg.repoRoot == "" { + return cfg, fmt.Errorf("repository root required") + } + binDir, err := os.MkdirTemp("", "runecode-tuiperf-bin-") + if err != nil { + return cfg, err + } + brokerBin := filepath.Join(binDir, "runecode-broker") + tuiBin := filepath.Join(binDir, "runecode-tui") + cfg.harnessBinDir = binDir + if err := buildHarnessBinary(cfg.repoRoot, "./cmd/runecode-broker", brokerBin); err != nil { + _ = os.RemoveAll(binDir) + return cfg, err + } + if err := buildHarnessBinary(cfg.repoRoot, "./cmd/runecode-tui", tuiBin); err != nil { + _ = os.RemoveAll(binDir) + return cfg, err + } + cfg.brokerBin = brokerBin + cfg.tuiBin = tuiBin + return cfg, nil +} + +func cleanupHarnessBinaries(cfg config) { + if cfg.harnessBinDir != "" { + _ = os.RemoveAll(cfg.harnessBinDir) + } +} + +func buildHarnessBinary(repoRoot, pkg, output string) error { + cmd := exec.Command("go", "build", "-o", output, pkg) + cmd.Dir = repoRoot + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build %s: %s", pkg, strings.TrimSpace(string(out))) + } + return nil +} + +func startHarnessProcesses(ctx context.Context, cfg config) (runningHarness, error) { + brokerCmd, err := startBrokerProcess(ctx, cfg) + if err != nil { + return runningHarness{}, err + } + tuiCmd, tuiOut, tuiIn, err := startTUIProcess(ctx, cfg) + if err != nil { + terminateProcess(brokerCmd.Process) + return runningHarness{}, err + } + return runningHarness{ctx: ctx, brokerCmd: brokerCmd, tuiCmd: tuiCmd, tuiOut: tuiOut, tuiIn: tuiIn}, nil +} + +func requireTUIFixtureConfig(cfg config) error { + if err := requireIsolationInputs(cfg); err != nil { + return err + } + if cfg.fixtureID != "tui.empty.v1" && cfg.fixtureID != "tui.waiting.v1" { + return usageError{err: fmt.Errorf("mode requires --fixture-id tui.empty.v1|tui.waiting.v1")} + } + if cfg.trials <= 0 { + return usageError{err: fmt.Errorf("--trials must be > 0")} + } + return nil +} + +func prepareTUIIsolation(cfg config) error { + if err := seedFixture(cfg.stateRoot, cfg.fixtureID); err != nil { + return err + } + if err := os.MkdirAll(cfg.runtimeDir, 0o700); err != nil { + return err + } + if err := os.Chmod(cfg.runtimeDir, 0o700); err != nil { + return err + } + return os.MkdirAll(cfg.auditLedgerRoot, 0o700) +} + +func startBrokerProcess(ctx context.Context, cfg config) (*exec.Cmd, error) { + brokerBin := cfg.brokerBin + if brokerBin == "" { + brokerBin = "go" + } + brokerArgs := []string{"--state-root", cfg.stateRoot, "--audit-ledger-root", cfg.auditLedgerRoot, "serve-local", "--runtime-dir", cfg.runtimeDir, "--socket-name", cfg.socketName} + brokerCmd := exec.CommandContext(ctx, brokerBin, brokerArgs...) + if cfg.brokerBin == "" { + brokerCmd = exec.CommandContext(ctx, "go", append([]string{"run", "./cmd/runecode-broker"}, brokerArgs...)...) + } + brokerCmd.Env = os.Environ() + var stdout bytes.Buffer + var stderr bytes.Buffer + brokerCmd.Stdout = &stdout + brokerCmd.Stderr = &stderr + if err := brokerCmd.Start(); err != nil { + return nil, err + } + if err := waitForSocket(filepath.Join(cfg.runtimeDir, cfg.socketName), 5*time.Second); err != nil { + terminateProcess(brokerCmd.Process) + return nil, fmt.Errorf("%w; broker startup summary: %s", err, summarizeBrokerStartupOutput(stdout.String(), stderr.String())) + } + return brokerCmd, nil +} + +func startTUIProcess(ctx context.Context, cfg config) (*exec.Cmd, io.ReadCloser, io.WriteCloser, error) { + tuiBin := cfg.tuiBin + tuiArgs := []string{"--runtime-dir", cfg.runtimeDir, "--socket-name", cfg.socketName} + tuiCmd := exec.CommandContext(ctx, tuiBin, tuiArgs...) + if cfg.tuiBin == "" { + tuiCmd = exec.CommandContext(ctx, "go", append([]string{"run", "./cmd/runecode-tui"}, tuiArgs...)...) + } + tuiCmd.Env = stableTTYEnv(os.Environ()) + tuiCmd.Env = append(tuiCmd.Env, "RUNECODE_TUI_BROKER_TARGET="+cfg.targetAlias) + tty, err := pty.Start(tuiCmd) + if err != nil { + return nil, nil, nil, err + } + return tuiCmd, newTerminalQueryResponder(tty, tty), tty, nil +} + +func stopHarness(h runningHarness) { + terminateProcess(processOf(h.tuiCmd)) + terminateProcess(processOf(h.brokerCmd)) +} + +func processOf(cmd *exec.Cmd) *os.Process { + if cmd == nil { + return nil + } + return cmd.Process +} + +func requireIsolationInputs(cfg config) error { + if cfg.runtimeDir == "" || cfg.socketName == "" || cfg.stateRoot == "" || cfg.auditLedgerRoot == "" || cfg.targetAlias == "" { + return usageError{err: fmt.Errorf("isolation inputs required: --runtime-dir --socket-name --state-root --audit-ledger-root --target-alias")} + } + return nil +} + +func waitForSocket(path string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + info, err := os.Stat(path) + if err == nil && (info.Mode()&os.ModeSocket) != 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("socket not ready: %s", path) + } + time.Sleep(20 * time.Millisecond) + } +} + +func terminateProcess(p *os.Process) { + if p == nil { + return + } + done := make(chan struct{}) + go func() { + _, _ = p.Wait() + close(done) + }() + _ = p.Signal(syscall.SIGTERM) + select { + case <-done: + return + case <-time.After(terminateGracePeriod): + } + _ = p.Signal(syscall.SIGKILL) + select { + case <-done: + case <-time.After(terminateGracePeriod): + } +} + +func summarizeBrokerStartupOutput(stdoutRaw, stderrRaw string) string { + return fmt.Sprintf("stdout=%s stderr=%s", summarizeStartupOutput(stdoutRaw), summarizeStartupOutput(stderrRaw)) +} + +func summarizeStartupOutput(raw string) string { + text := strings.TrimSpace(raw) + if text == "" { + return "" + } + text = nonPrintablePattern.ReplaceAllString(text, "") + text = absPathPattern.ReplaceAllString(text, "") + text = longTokenPattern.ReplaceAllString(text, "") + text = wsPattern.ReplaceAllString(text, " ") + text = strings.TrimSpace(text) + if text == "" { + return "" + } + const maxLen = 200 + if len(text) <= maxLen { + return text + } + return "…" + text[len(text)-maxLen:] +} diff --git a/tools/tuiperf/harness_test.go b/tools/tuiperf/harness_test.go new file mode 100644 index 00000000..7c1b6f8a --- /dev/null +++ b/tools/tuiperf/harness_test.go @@ -0,0 +1,92 @@ +//go:build linux + +package main + +import ( + "bytes" + "io" + "reflect" + "strings" + "testing" +) + +func TestSummarizeStartupOutputSanitizesSensitiveData(t *testing.T) { + t.Parallel() + + raw := "listen failed for /tmp/private/runtime.sock token=abcdefghijklmnopqrstuvwxyz123456" + summary := summarizeStartupOutput(raw) + + if strings.Contains(summary, "/tmp/private/runtime.sock") { + t.Fatalf("summary leaked absolute path: %q", summary) + } + if strings.Contains(summary, "abcdefghijklmnopqrstuvwxyz123456") { + t.Fatalf("summary leaked long token: %q", summary) + } + if !strings.Contains(summary, "") { + t.Fatalf("summary missing redacted path marker: %q", summary) + } + if !strings.Contains(summary, "") { + t.Fatalf("summary missing redacted token marker: %q", summary) + } +} + +func TestSummarizeStartupOutputTruncatesLongOutput(t *testing.T) { + t.Parallel() + + long := strings.Repeat("segment ", 40) + summary := summarizeStartupOutput(long) + if !strings.HasPrefix(summary, "…") { + t.Fatalf("summary = %q, want ellipsis prefix", summary) + } +} + +func TestSummarizeBrokerStartupOutputIncludesBothStreams(t *testing.T) { + t.Parallel() + + summary := summarizeBrokerStartupOutput("stdout ok", "stderr boom") + if !strings.Contains(summary, "stdout=stdout ok") { + t.Fatalf("summary missing stdout segment: %q", summary) + } + if !strings.Contains(summary, "stderr=stderr boom") { + t.Fatalf("summary missing stderr segment: %q", summary) + } +} + +func TestStableTTYEnvOverridesOrAddsTERM(t *testing.T) { + t.Parallel() + + got := stableTTYEnv([]string{"FOO=bar", "TERM=dumb"}) + if !reflect.DeepEqual(got, []string{"FOO=bar", "TERM=" + stableTUITerm}) { + t.Fatalf("stableTTYEnv override = %v", got) + } + + got = stableTTYEnv([]string{"FOO=bar"}) + if !reflect.DeepEqual(got, []string{"FOO=bar", "TERM=" + stableTUITerm}) { + t.Fatalf("stableTTYEnv append = %v", got) + } +} + +func TestTerminalQueryResponderAnswersSplitQueries(t *testing.T) { + t.Parallel() + + reader := io.NopCloser(strings.NewReader("prefix \x1b]11;?\x1b\\ middle \x1b[6n suffix")) + var responses bytes.Buffer + responder := newTerminalQueryResponder(reader, &responses) + buf := make([]byte, 4) + for { + _, err := responder.Read(buf) + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Read error = %v", err) + } + } + got := responses.String() + if !strings.Contains(got, terminalBackgroundColorResponse) { + t.Fatalf("responses missing background color response: %q", got) + } + if !strings.Contains(got, terminalCPRResponse) { + t.Fatalf("responses missing CPR response: %q", got) + } +} diff --git a/tools/tuiperf/main.go b/tools/tuiperf/main.go new file mode 100644 index 00000000..74af16f0 --- /dev/null +++ b/tools/tuiperf/main.go @@ -0,0 +1,21 @@ +//go:build linux + +package main + +import ( + "errors" + "fmt" + "os" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + var usageErr usageError + if errors.As(err, &usageErr) { + fmt.Fprintf(os.Stderr, "tuiperf usage error: %v\n", err) + os.Exit(2) + } + fmt.Fprintf(os.Stderr, "tuiperf failed: %v\n", err) + os.Exit(1) + } +} diff --git a/tools/tuiperf/modes.go b/tools/tuiperf/modes.go new file mode 100644 index 00000000..5adbdaa6 --- /dev/null +++ b/tools/tuiperf/modes.go @@ -0,0 +1,189 @@ +//go:build linux + +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" + "github.com/runecode-ai/runecode/internal/tuiperf" +) + +const latencyMarkerTimeout = 20 * time.Second + +func runCPUMode(cfg config) error { + _, cancel, harness, err := startTUIHarness(cfg) + if err != nil { + return err + } + defer cancel() + defer stopHarness(harness) + targetPID, err := tuiperf.WaitForChildByComm("/proc", harness.tuiCmd.Process.Pid, "runecode-tui", 3*time.Second, 20*time.Millisecond) + if err != nil { + return err + } + result, err := tuiperf.SampleProcessCPU(targetPID, tuiperf.CPUSampleConfig{Warmup: cfg.warmup, Window: cfg.window, Windows: cfg.windows}) + if err != nil { + return err + } + measurements := cpuMeasurementsForFixture(cfg.fixtureID, result) + return writeEnvelope(cfg.outputPath, checkEnvelope{SchemaVersion: checkSchemaVersion, Metadata: map[string]any{"mode": "cpu", "fixture_id": cfg.fixtureID, "target_pid": result.TargetPID, "target_comm": result.TargetComm, "sampling": result}, Measurements: measurements}) +} + +func runLatencyMode(cfg config) error { + preparedCfg, cleanup, err := prepareLatencyMode(cfg) + if err != nil { + return err + } + defer cleanup() + attachDurations, keyDurations, err := collectLatencySamplesFromFreshSpawn( + preparedCfg.trials, + func() (runningHarness, error) { return startLatencyHarness(preparedCfg) }, + stopLatencyHarness, + collectLatencySampleFromHarness, + ) + if err != nil { + return err + } + attachP95, keyP95, err := latencyP95(attachDurations, keyDurations) + if err != nil { + return err + } + measurements := latencyMeasurementsForFixture(preparedCfg.fixtureID, attachP95, keyP95) + return writeEnvelope(preparedCfg.outputPath, checkEnvelope{SchemaVersion: checkSchemaVersion, Metadata: map[string]any{"mode": "latency", "fixture_id": preparedCfg.fixtureID, "trials": preparedCfg.trials, "attach_samples_ms": attachDurations, "key_samples_ms": keyDurations}, Measurements: measurements}) +} + +func prepareLatencyMode(cfg config) (config, func(), error) { + if err := requireTUIFixtureConfig(cfg); err != nil { + return config{}, nil, err + } + preparedCfg, err := prepareHarnessBinaries(cfg) + if err != nil { + return config{}, nil, err + } + cleanup := func() { cleanupHarnessBinaries(preparedCfg) } + if err := prepareTUIIsolation(cfg); err != nil { + cleanup() + return config{}, nil, err + } + return preparedCfg, cleanup, nil +} + +func startLatencyHarness(cfg config) (runningHarness, error) { + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + harness, err := startHarnessProcesses(ctx, cfg) + if err != nil { + cancel() + return runningHarness{}, err + } + harness.ctx = ctx + harness.cancel = cancel + return harness, nil +} + +func stopLatencyHarness(h runningHarness) { + stopHarness(h) + if h.cancel != nil { + h.cancel() + } +} + +func cpuMeasurementsForFixture(fixtureID string, result tuiperf.CPUSampleResult) []perfcontracts.MeasurementRecord { + if fixtureID == "tui.empty.v1" { + return []perfcontracts.MeasurementRecord{{MetricID: "metric.tui.idle_cpu.empty.avg_pct", Value: result.AverageCPUPercent, Unit: "percent"}, {MetricID: "metric.tui.idle_cpu.empty.max_pct", Value: result.MaxCPUPercent, Unit: "percent"}} + } + return []perfcontracts.MeasurementRecord{{MetricID: "metric.tui.idle_cpu.waiting.avg_pct", Value: result.AverageCPUPercent, Unit: "percent"}, {MetricID: "metric.tui.idle_cpu.waiting.max_pct", Value: result.MaxCPUPercent, Unit: "percent"}} +} + +func collectLatencySamplesFromFreshSpawn( + trials int, + startHarness func() (runningHarness, error), + stopHarnessFn func(runningHarness), + collectSample func(runningHarness, string, time.Time) (float64, float64, error), +) ([]float64, []float64, error) { + marker := "Runecode TUI α shell" + attachDurations := make([]float64, 0, trials) + keyDurations := make([]float64, 0, trials) + for i := 0; i < trials; i++ { + start := time.Now() + h, err := startHarness() + if err != nil { + return nil, nil, err + } + attachMS, keyMS, err := func() (float64, float64, error) { + defer stopHarnessFn(h) + return collectSample(h, marker, start) + }() + if err != nil { + return nil, nil, err + } + attachDurations = append(attachDurations, attachMS) + keyDurations = append(keyDurations, keyMS) + } + return attachDurations, keyDurations, nil +} + +func collectLatencySampleFromHarness(h runningHarness, marker string, start time.Time) (float64, float64, error) { + events := make(chan tuiperf.MarkerEvent, 64) + const keyResponseMarker = "focus=MAIN" + go tuiperf.WatchMarkers(h.ctx, h.tuiOut, []string{marker, keyResponseMarker}, events) + attachAt, err := waitForMarker(events, marker, latencyMarkerTimeout) + if err != nil { + return 0, 0, err + } + keyStart := time.Now() + if _, err := io.WriteString(h.tuiIn, "\t"); err != nil { + return 0, 0, err + } + keyAt, err := waitForMarkerAfter(events, keyResponseMarker, keyStart, latencyMarkerTimeout) + if err != nil { + return 0, 0, err + } + return float64(attachAt.Sub(start).Milliseconds()), float64(keyAt.Sub(keyStart).Milliseconds()), nil +} + +func latencyP95(attachDurations, keyDurations []float64) (float64, float64, error) { + attachP95, err := tuiperf.P95Millis(attachDurations) + if err != nil { + return 0, 0, err + } + keyP95, err := tuiperf.P95Millis(keyDurations) + if err != nil { + return 0, 0, err + } + return attachP95, keyP95, nil +} + +func latencyMeasurementsForFixture(fixtureID string, attachP95, keyP95 float64) []perfcontracts.MeasurementRecord { + if fixtureID == "tui.empty.v1" { + return []perfcontracts.MeasurementRecord{{MetricID: "metric.tui.attach.quiet.p95_ms", Value: attachP95, Unit: "ms"}, {MetricID: "metric.tui.key_response.quiet.p95_ms", Value: keyP95, Unit: "ms"}} + } + return []perfcontracts.MeasurementRecord{{MetricID: "metric.tui.attach.waiting.p95_ms", Value: attachP95, Unit: "ms"}, {MetricID: "metric.tui.key_response.waiting.p95_ms", Value: keyP95, Unit: "ms"}} +} + +func runBenchParseMode(cfg config) error { + if strings.TrimSpace(cfg.benchOutput) == "" { + return usageError{err: fmt.Errorf("bench-parse mode requires --bench-output")} + } + file, err := os.Open(cfg.benchOutput) + if err != nil { + return err + } + defer file.Close() + measurements, err := tuiperf.ParseGoTestBenchOutput(file, []tuiperf.BenchmarkMetricMap{ + {Benchmark: "BenchmarkShellViewEmpty", Field: "ns/op", MetricID: "metric.tui.render.shell_view_empty.ns_op", Unit: "ns/op"}, + {Benchmark: "BenchmarkShellViewWaitingSession", Field: "ns/op", MetricID: "metric.tui.render.shell_view_waiting.ns_op", Unit: "ns/op"}, + {Benchmark: "BenchmarkShellWatchApply", Field: "ns/op", MetricID: "metric.tui.update.shell_watch_apply.ns_op", Unit: "ns/op"}, + {Benchmark: "BenchmarkBuildPaletteEntries", Field: "ns/op", MetricID: "metric.tui.update.build_palette_entries.ns_op", Unit: "ns/op"}, + }) + if err != nil { + return err + } + return writeEnvelope(cfg.outputPath, checkEnvelope{SchemaVersion: checkSchemaVersion, Metadata: map[string]any{"mode": "bench-parse", "bench_output": filepath.Base(cfg.benchOutput)}, Measurements: measurements}) +} diff --git a/tools/tuiperf/modes_test.go b/tools/tuiperf/modes_test.go new file mode 100644 index 00000000..54ab2a9a --- /dev/null +++ b/tools/tuiperf/modes_test.go @@ -0,0 +1,221 @@ +//go:build linux + +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/runecode-ai/runecode/internal/tuiperf" +) + +func TestRunModeUnsupportedModeReturnsUsageError(t *testing.T) { + t.Parallel() + + err := runMode(config{mode: "unknown"}) + if err == nil { + t.Fatal("runMode error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("runMode error = %T, want usageError", err) + } +} + +func TestRunBenchParseModeRequiresBenchOutputAsUsageError(t *testing.T) { + t.Parallel() + + err := runBenchParseMode(config{}) + if err == nil { + t.Fatal("runBenchParseMode error = nil, want usage error") + } + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("runBenchParseMode error = %T, want usageError", err) + } +} + +func TestCollectLatencySamplesFromFreshSpawnStartsAndStopsPerTrial(t *testing.T) { + t.Parallel() + + started, stopped, attach, key, err := collectTrialLifecycleSamples(t) + if err != nil { + t.Fatalf("collectLatencySamplesFromFreshSpawn error = %v", err) + } + assertTrialLifecycle(t, started, stopped, attach, key) +} + +func collectTrialLifecycleSamples(t *testing.T) ([]string, []string, []float64, []float64, error) { + t.Helper() + var started []string + var stopped []string + sampleCalls := 0 + attach, key, err := collectLatencySamplesFromFreshSpawn( + 3, + func() (runningHarness, error) { + id := fmt.Sprintf("trial-%d", len(started)+1) + started = append(started, id) + return runningHarness{tuiCmd: &exec.Cmd{Path: id}}, nil + }, + func(h runningHarness) { stopped = append(stopped, h.tuiCmd.Path) }, + func(_ runningHarness, marker string, _ time.Time) (float64, float64, error) { + sampleCalls++ + if marker != "Runecode TUI α shell" { + t.Fatalf("marker = %q, want %q", marker, "Runecode TUI α shell") + } + return float64(sampleCalls), float64(sampleCalls + 10), nil + }, + ) + return started, stopped, attach, key, err +} + +func assertTrialLifecycle(t *testing.T, started, stopped []string, attach, key []float64) { + t.Helper() + assertLifecycleCounts(t, started, stopped, attach, key) + assertLifecycleOrder(t, started, stopped) + assertLifecycleSamples(t, attach, key) +} + +func assertLifecycleCounts(t *testing.T, started, stopped []string, attach, key []float64) { + t.Helper() + if got, want := len(started), 3; got != want { + t.Fatalf("start calls = %d, want %d", got, want) + } + if got, want := len(stopped), 3; got != want { + t.Fatalf("stop calls = %d, want %d", got, want) + } + if got, want := len(attach), 3; got != want { + t.Fatalf("attach sample count = %d, want %d", got, want) + } + if got, want := len(key), 3; got != want { + t.Fatalf("key sample count = %d, want %d", got, want) + } +} + +func assertLifecycleOrder(t *testing.T, started, stopped []string) { + t.Helper() + for i := range started { + if started[i] != stopped[i] { + t.Fatalf("stopped[%d] = %q, want %q", i, stopped[i], started[i]) + } + } +} + +func assertLifecycleSamples(t *testing.T, attach, key []float64) { + t.Helper() + if attach[0] != 1 || attach[1] != 2 || attach[2] != 3 { + t.Fatalf("attach samples = %v, want [1 2 3]", attach) + } + if key[0] != 11 || key[1] != 12 || key[2] != 13 { + t.Fatalf("key samples = %v, want [11 12 13]", key) + } +} + +func TestCollectLatencySamplesFromFreshSpawnStopsHarnessOnSampleError(t *testing.T) { + t.Parallel() + + var stopped []string + starts := 0 + + _, _, err := collectLatencySamplesFromFreshSpawn( + 3, + func() (runningHarness, error) { + starts++ + return runningHarness{tuiCmd: &exec.Cmd{Path: fmt.Sprintf("trial-%d", starts)}}, nil + }, + func(h runningHarness) { + stopped = append(stopped, h.tuiCmd.Path) + }, + func(h runningHarness, _ string, _ time.Time) (float64, float64, error) { + if h.tuiCmd.Path == "trial-2" { + return 0, 0, errors.New("sample failed") + } + return 1, 1, nil + }, + ) + if err == nil { + t.Fatal("collectLatencySamplesFromFreshSpawn error = nil, want error") + } + if got, want := starts, 2; got != want { + t.Fatalf("start calls = %d, want %d", got, want) + } + if got, want := len(stopped), 2; got != want { + t.Fatalf("stop calls = %d, want %d", got, want) + } + if stopped[1] != "trial-2" { + t.Fatalf("stopped harness on error = %q, want trial-2", stopped[1]) + } +} + +func TestCollectLatencySamplesFromFreshSpawnMeasuresFromPreSpawnStart(t *testing.T) { + t.Parallel() + + spawnDelay := 15 * time.Millisecond + _, _, err := collectLatencySamplesFromFreshSpawn( + 1, + func() (runningHarness, error) { + time.Sleep(spawnDelay) + return runningHarness{tuiCmd: &exec.Cmd{Path: "trial-1"}}, nil + }, + func(runningHarness) {}, + func(_ runningHarness, _ string, start time.Time) (float64, float64, error) { + if elapsed := time.Since(start); elapsed < spawnDelay { + t.Fatalf("elapsed since start = %s, want >= %s", elapsed, spawnDelay) + } + return 1, 1, nil + }, + ) + if err != nil { + t.Fatalf("collectLatencySamplesFromFreshSpawn error = %v", err) + } +} + +func TestWaitForMarkerAfterSkipsStaleEvents(t *testing.T) { + t.Parallel() + events := make(chan tuiperf.MarkerEvent, 3) + start := time.Now() + events <- tuiperf.MarkerEvent{Marker: "Runecode TUI α shell", At: start.Add(-time.Millisecond)} + events <- tuiperf.MarkerEvent{Marker: "Runecode TUI α shell", At: start.Add(time.Millisecond)} + got, err := waitForMarkerAfter(events, "Runecode TUI α shell", start, time.Second) + if err != nil { + t.Fatalf("waitForMarkerAfter error = %v", err) + } + if got.Before(start) { + t.Fatalf("got = %s, want >= %s", got, start) + } +} + +func TestRunBenchParseModeStoresBenchOutputBaseName(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + benchPath := filepath.Join(tmp, "bench.txt") + outputPath := filepath.Join(tmp, "out.json") + content := strings.Join([]string{ + "BenchmarkShellViewEmpty-8 1000 10 ns/op", + "BenchmarkShellViewWaitingSession-8 1000 11 ns/op", + "BenchmarkShellWatchApply-8 1000 12 ns/op", + "BenchmarkBuildPaletteEntries-8 1000 13 ns/op", + }, "\n") + "\n" + if err := os.WriteFile(benchPath, []byte(content), 0o644); err != nil { + t.Fatalf("WriteFile benchPath: %v", err) + } + if err := runBenchParseMode(config{benchOutput: benchPath, outputPath: outputPath}); err != nil { + t.Fatalf("runBenchParseMode error = %v", err) + } + raw, err := os.ReadFile(outputPath) + if err != nil { + t.Fatalf("ReadFile output: %v", err) + } + if strings.Contains(string(raw), benchPath) { + t.Fatalf("output leaked full bench path: %s", benchPath) + } + if !strings.Contains(string(raw), filepath.Base(benchPath)) { + t.Fatalf("output missing bench basename: %s", filepath.Base(benchPath)) + } +} diff --git a/tools/tuiperf/terminal_queries.go b/tools/tuiperf/terminal_queries.go new file mode 100644 index 00000000..70bbee92 --- /dev/null +++ b/tools/tuiperf/terminal_queries.go @@ -0,0 +1,74 @@ +//go:build linux + +package main + +import ( + "io" + "strings" +) + +const stableTUITerm = "xterm-256color" + +const ( + terminalCPRQuery = "\x1b[6n" + terminalCPRResponse = "\x1b[1;1R" + terminalBackgroundColorQuery = "\x1b]11;?\x1b\\" + terminalBackgroundColorResponse = "\x1b]11;rgb:0000/0000/0000\x1b\\" +) + +type terminalQueryResponder struct { + io.ReadCloser + w io.Writer + pending string +} + +func newTerminalQueryResponder(r io.ReadCloser, w io.Writer) io.ReadCloser { + return &terminalQueryResponder{ReadCloser: r, w: w} +} + +func (r *terminalQueryResponder) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if n > 0 { + r.respondToTerminalQueries(p[:n]) + } + return n, err +} + +func (r *terminalQueryResponder) respondToTerminalQueries(chunk []byte) { + text := r.pending + string(chunk) + if strings.Contains(text, terminalCPRQuery) { + _, _ = io.WriteString(r.w, terminalCPRResponse) + } + if strings.Contains(text, terminalBackgroundColorQuery) { + _, _ = io.WriteString(r.w, terminalBackgroundColorResponse) + } + r.pending = terminalQueryTail(text) +} + +func terminalQueryTail(text string) string { + keep := len(terminalBackgroundColorQuery) - 1 + if len(terminalCPRQuery) > len(terminalBackgroundColorQuery) { + keep = len(terminalCPRQuery) - 1 + } + if len(text) <= keep { + return text + } + return text[len(text)-keep:] +} + +func stableTTYEnv(base []string) []string { + filtered := make([]string, 0, len(base)+1) + hasTerm := false + for _, entry := range base { + if strings.HasPrefix(entry, "TERM=") { + filtered = append(filtered, "TERM="+stableTUITerm) + hasTerm = true + continue + } + filtered = append(filtered, entry) + } + if !hasTerm { + filtered = append(filtered, "TERM="+stableTUITerm) + } + return filtered +} diff --git a/tools/tuiperf/types.go b/tools/tuiperf/types.go new file mode 100644 index 00000000..af525047 --- /dev/null +++ b/tools/tuiperf/types.go @@ -0,0 +1,38 @@ +//go:build linux + +package main + +import ( + "time" + + "github.com/runecode-ai/runecode/internal/perfcontracts" +) + +const checkSchemaVersion = "runecode.performance.check.v1" + +type config struct { + mode string + outputPath string + fixtureID string + runtimeDir string + socketName string + stateRoot string + auditLedgerRoot string + targetAlias string + repoRoot string + harnessBinDir string + brokerBin string + tuiBin string + trials int + warmup time.Duration + window time.Duration + windows int + timeout time.Duration + benchOutput string +} + +type checkEnvelope struct { + SchemaVersion string `json:"schema_version"` + Metadata map[string]any `json:"metadata,omitempty"` + Measurements []perfcontracts.MeasurementRecord `json:"measurements"` +} diff --git a/tools/tuiperf/util.go b/tools/tuiperf/util.go new file mode 100644 index 00000000..94afba66 --- /dev/null +++ b/tools/tuiperf/util.go @@ -0,0 +1,82 @@ +//go:build linux + +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "strings" + "time" + + "github.com/runecode-ai/runecode/internal/tuiperf" +) + +func waitForMarker(events <-chan tuiperf.MarkerEvent, marker string, timeout time.Duration) (time.Time, error) { + deadline := time.After(timeout) + for { + select { + case ev, ok := <-events: + if !ok { + return time.Time{}, fmt.Errorf("marker stream closed before marker %q", marker) + } + if ev.Marker == marker { + return ev.At, nil + } + case <-deadline: + return time.Time{}, fmt.Errorf("timeout waiting for marker %q", marker) + } + } +} + +func waitForMarkerAfter(events <-chan tuiperf.MarkerEvent, marker string, earliest time.Time, timeout time.Duration) (time.Time, error) { + deadline := time.After(timeout) + for { + select { + case ev, ok := <-events: + if !ok { + return time.Time{}, fmt.Errorf("marker stream closed before marker %q", marker) + } + if ev.Marker == marker && !ev.At.Before(earliest) { + return ev.At, nil + } + case <-deadline: + return time.Time{}, fmt.Errorf("timeout waiting for marker %q", marker) + } + } +} + +func seedFixture(storeRoot, fixtureID string) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "run", "./tools/perfseedwait", "--fixture-id", fixtureID, "--store-root", storeRoot) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return fmt.Errorf("seed fixture %s timed out: %w", fixtureID, ctx.Err()) + } + return fmt.Errorf("seed fixture %s: %w: %s", fixtureID, err, strings.TrimSpace(stderr.String())) + } + return nil +} + +func writeEnvelope(path string, envelope checkEnvelope) error { + raw, err := json.MarshalIndent(envelope, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, raw, 0o644) +} + +func shellEscape(v string) string { + if v == "" { + return "''" + } + return "'" + strings.ReplaceAll(v, "'", "'\\''") + "'" +}