diff --git a/cmd/axis/main.go b/cmd/axis/main.go index 232f663..5847139 100644 --- a/cmd/axis/main.go +++ b/cmd/axis/main.go @@ -67,6 +67,7 @@ func newRootCmd() *cobra.Command { root.AddCommand(doctorCmd()) root.AddCommand(summaryCmd()) root.AddCommand(reservationsCmd()) + root.AddCommand(observationsCmd()) ui.ApplyHelpTemplate(root) diff --git a/cmd/axis/observations.go b/cmd/axis/observations.go new file mode 100644 index 0000000..3fbcf87 --- /dev/null +++ b/cmd/axis/observations.go @@ -0,0 +1,231 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/toasterbook88/axis/internal/models" + "github.com/toasterbook88/axis/internal/state" + "github.com/toasterbook88/axis/internal/ui" +) + +var loadObservationsState = state.Load + +func observationsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "observations", + Short: "Show execution observations tracked by the cluster", + RunE: func(cmd *cobra.Command, args []string) error { + return runObservationsLocal(cmd) + }, + } + cmd.AddCommand(observationsListCmd()) + cmd.AddCommand(observationsInspectCmd()) + return cmd +} + +func runObservationsLocal(cmd *cobra.Command) error { + st, err := loadObservationsState() + if err != nil { + return fmt.Errorf("loading state: %w", err) + } + if st == nil || len(st.Observations) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No observations tracked") + return nil + } + entries := make([]models.ExecutionObservation, 0, len(st.Observations)) + for _, obs := range st.Observations { + entries = append(entries, obs) + } + fmt.Fprint(cmd.OutOrStdout(), renderObservationTable(entries)) + return nil +} + +func renderObservationTable(entries []models.ExecutionObservation) string { + var b strings.Builder + sep := strings.Repeat("─", 90) + b.WriteString("\n") + ui.WhiteColor.Fprintf(&b, " EXECUTION OBSERVATIONS\n") + b.WriteString(" ") + b.WriteString(sep) + b.WriteString("\n") + + if len(entries) == 0 { + ui.DimColor.Fprintf(&b, " No observations tracked\n\n") + return b.String() + } + + ui.WhiteColor.Fprintf(&b, " %-15s %-12s %-12s %-12s %10s %10s %8s %8s\n", + "NODE", "WORKLOAD", "BACKEND", "TOOL", "WALL MS", "PEAK RAM", "PEAK VRAM", "SAMPLES") + b.WriteString(" ") + b.WriteString(sep) + b.WriteString("\n") + + display := entries + truncated := 0 + if len(entries) > 50 { + display = entries[:50] + truncated = len(entries) - 50 + } + + for _, obs := range display { + peakVRAM := "-" + if obs.PeakVRAMMB > 0 { + peakVRAM = fmt.Sprintf("%d MB", obs.PeakVRAMMB) + } + success := "" + if !obs.LastSuccess { + success = ui.RedColor.Sprintf(" (last failed)") + } + fmt.Fprintf(&b, " %-15s %-12s %-12s %-12s %10d %10d %8s %8d%s\n", + obs.Scope.Node, + obs.Scope.Workload, + obs.Scope.Backend, + obs.Scope.Tool, + obs.WallTimeMS, + obs.PeakRAMMB, + peakVRAM, + obs.SampleCount, + success, + ) + } + + if truncated > 0 { + ui.DimColor.Fprintf(&b, "\n ... and %d more observations.\n", truncated) + } + + b.WriteString("\n") + return b.String() +} + +func observationsListCmd() *cobra.Command { + var format string + + cmd := &cobra.Command{ + Use: "list", + Short: "List execution observations from the local state ledger", + RunE: func(cmd *cobra.Command, args []string) error { + st, err := loadObservationsState() + if err != nil { + return fmt.Errorf("loading state: %w", err) + } + + entries := make([]models.ExecutionObservation, 0, len(st.Observations)) + for _, obs := range st.Observations { + entries = append(entries, obs) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].ObservedAt.After(entries[j].ObservedAt) + }) + + switch format { + case "json": + return json.NewEncoder(cmd.OutOrStdout()).Encode(entries) + default: + if len(entries) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No observations tracked") + return nil + } + tbl := ui.NewTable("KEY", "NODE", "WORKLOAD", "BACKEND", "TOOL", "WALL MS", "PEAK RAM", "PEAK VRAM", "SAMPLES", "OBSERVED") + for _, obs := range entries { + key := state.ObservationKey(obs.Scope) + peakVRAM := "-" + if obs.PeakVRAMMB > 0 { + peakVRAM = fmt.Sprintf("%d MB", obs.PeakVRAMMB) + } + tbl.AddRow( + truncateID(key, 12), + obs.Scope.Node, + string(obs.Scope.Workload), + obs.Scope.Backend, + obs.Scope.Tool, + fmt.Sprintf("%d", obs.WallTimeMS), + fmt.Sprintf("%d MB", obs.PeakRAMMB), + peakVRAM, + fmt.Sprintf("%d", obs.SampleCount), + obs.ObservedAt.Format(time.RFC3339), + ) + } + tbl.Render(cmd.OutOrStdout()) + return nil + } + }, + } + cmd.Flags().StringVar(&format, "format", "text", "Output format: text or json") + return cmd +} + +func observationsInspectCmd() *cobra.Command { + var format string + + cmd := &cobra.Command{ + Use: "inspect ", + Short: "Show full details of an execution observation", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + key := args[0] + st, err := loadObservationsState() + if err != nil { + return fmt.Errorf("loading state: %w", err) + } + + var found *models.ExecutionObservation + for k, obs := range st.Observations { + if k == key { + obsCopy := obs + found = &obsCopy + break + } + } + if found == nil { + // Allow lookup by prefix for convenience. + for k, obs := range st.Observations { + if strings.HasPrefix(k, key) { + obsCopy := obs + found = &obsCopy + break + } + } + } + + if found == nil { + return ExitCodeError{Code: ExitErrGeneric, Message: fmt.Sprintf("observation %q not found", key)} + } + + switch format { + case "json": + return json.NewEncoder(cmd.OutOrStdout()).Encode(found) + default: + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Key: %s\n", state.ObservationKey(found.Scope)) + fmt.Fprintf(out, "Node: %s\n", found.Scope.Node) + fmt.Fprintf(out, "Workload: %s\n", found.Scope.Workload) + fmt.Fprintf(out, "Backend: %s\n", found.Scope.Backend) + fmt.Fprintf(out, "Tool: %s\n", found.Scope.Tool) + if found.Scope.ModelName != "" { + fmt.Fprintf(out, "Model: %s\n", found.Scope.ModelName) + } + fmt.Fprintf(out, "Wall Time: %d ms\n", found.WallTimeMS) + fmt.Fprintf(out, "Peak RAM: %d MB\n", found.PeakRAMMB) + if found.PeakVRAMMB > 0 { + fmt.Fprintf(out, "Peak VRAM: %d MB\n", found.PeakVRAMMB) + } + fmt.Fprintf(out, "Samples: %d\n", found.SampleCount) + fmt.Fprintf(out, "Last Success:%v\n", found.LastSuccess) + fmt.Fprintf(out, "Observed At: %s\n", found.ObservedAt.Format(time.RFC3339)) + isStale := "" + if !state.ObservationIsFresh(*found, time.Now().UTC()) { + isStale = " (stale)" + } + fmt.Fprintf(out, "Fresh: %v%s\n", state.ObservationIsFresh(*found, time.Now().UTC()), isStale) + return nil + } + }, + } + cmd.Flags().StringVar(&format, "format", "text", "Output format: text or json") + return cmd +} diff --git a/cmd/axis/observations_test.go b/cmd/axis/observations_test.go new file mode 100644 index 0000000..5c09a16 --- /dev/null +++ b/cmd/axis/observations_test.go @@ -0,0 +1,336 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/toasterbook88/axis/internal/models" + "github.com/toasterbook88/axis/internal/state" +) + +func stubObservationsState(t *testing.T, st *state.ClusterState, err error) func() { + t.Helper() + prev := loadObservationsState + loadObservationsState = func() (*state.ClusterState, error) { + return st, err + } + return func() { + loadObservationsState = prev + } +} + +func TestObservationsCmdNoObservations(t *testing.T) { + st := &state.ClusterState{Observations: map[string]models.ExecutionObservation{}} + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + if !strings.Contains(stdout, "No observations tracked") { + t.Fatalf("expected 'No observations tracked' in stdout, got %q", stdout) + } +} + +func TestObservationsListCmdTextOutput(t *testing.T) { + st := &state.ClusterState{ + Observations: map[string]models.ExecutionObservation{ + "abc123": { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + ObservedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + SampleCount: 3, + WallTimeMS: 1200, + PeakRAMMB: 512, + PeakVRAMMB: 256, + LastSuccess: true, + }, + }, + } + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsListCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{}) + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + if !strings.Contains(stdout, "alpha") { + t.Fatalf("expected 'alpha' in stdout, got %q", stdout) + } + if !strings.Contains(stdout, "512 MB") { + t.Fatalf("expected '512 MB' in stdout, got %q", stdout) + } + if !strings.Contains(stdout, "256 MB") { + t.Fatalf("expected '256 MB' in stdout, got %q", stdout) + } +} + +func TestObservationsListCmdJSONOutput(t *testing.T) { + st := &state.ClusterState{ + Observations: map[string]models.ExecutionObservation{ + "abc123": { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + ObservedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + SampleCount: 3, + WallTimeMS: 1200, + PeakRAMMB: 512, + LastSuccess: true, + }, + }, + } + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsListCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{"--format", "json"}) + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + var entries []models.ExecutionObservation + if err := json.Unmarshal([]byte(stdout), &entries); err != nil { + t.Fatalf("unmarshal JSON: %v\noutput: %q", err, stdout) + } + if len(entries) != 1 { + t.Fatalf("expected 1 entry, got %d", len(entries)) + } + if entries[0].Scope.Node != "alpha" { + t.Fatalf("expected node alpha, got %q", entries[0].Scope.Node) + } +} + +func TestObservationsInspectCmdTextOutput(t *testing.T) { + st := &state.ClusterState{ + Observations: map[string]models.ExecutionObservation{ + "abc123def456": { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + ObservedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + SampleCount: 3, + WallTimeMS: 1200, + PeakRAMMB: 512, + LastSuccess: true, + }, + }, + } + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsInspectCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{"abc123def456"}) + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + if !strings.Contains(stdout, "alpha") { + t.Fatalf("expected 'alpha' in stdout, got %q", stdout) + } + if !strings.Contains(stdout, "512 MB") { + t.Fatalf("expected '512 MB' in stdout, got %q", stdout) + } +} + +func TestObservationsInspectCmdJSONOutput(t *testing.T) { + st := &state.ClusterState{ + Observations: map[string]models.ExecutionObservation{ + "abc123def456": { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + ObservedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + SampleCount: 3, + WallTimeMS: 1200, + PeakRAMMB: 512, + LastSuccess: true, + }, + }, + } + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsInspectCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{"abc123def456", "--format", "json"}) + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + var obs models.ExecutionObservation + if err := json.Unmarshal([]byte(stdout), &obs); err != nil { + t.Fatalf("unmarshal JSON: %v\noutput: %q", err, stdout) + } + if obs.Scope.Node != "alpha" { + t.Fatalf("expected node alpha, got %q", obs.Scope.Node) + } +} + +func TestObservationsInspectCmdNotFound(t *testing.T) { + st := &state.ClusterState{Observations: map[string]models.ExecutionObservation{}} + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsInspectCmd() + _, _, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{"missing"}) + return cmd.Execute() + }) + if err == nil { + t.Fatal("expected error for missing observation") + } + code := ExitCode(err) + if code != ExitErrGeneric { + t.Fatalf("expected exit code %d, got %d", ExitErrGeneric, code) + } +} + +func TestObservationsInspectCmdPrefixMatch(t *testing.T) { + st := &state.ClusterState{ + Observations: map[string]models.ExecutionObservation{ + "abc123def456": { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + ObservedAt: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), + SampleCount: 3, + WallTimeMS: 1200, + PeakRAMMB: 512, + LastSuccess: true, + }, + }, + } + restore := stubObservationsState(t, st, nil) + defer restore() + + cmd := observationsInspectCmd() + stdout, stderr, err := captureProcessOutput(t, func() error { + cmd.SetArgs([]string{"abc123"}) + return cmd.Execute() + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if stderr != "" { + t.Fatalf("unexpected stderr: %q", stderr) + } + if !strings.Contains(stdout, "alpha") { + t.Fatalf("expected 'alpha' in stdout, got %q", stdout) + } +} + +func TestRenderObservationTable(t *testing.T) { + entries := []models.ExecutionObservation{ + { + Scope: models.ObservationScope{ + Node: "alpha", + Workload: "inference", + Backend: "ollama", + Tool: "llama3", + }, + WallTimeMS: 1200, + PeakRAMMB: 512, + PeakVRAMMB: 256, + SampleCount: 3, + LastSuccess: true, + }, + { + Scope: models.ObservationScope{ + Node: "beta", + Workload: "script", + Backend: "local", + Tool: "git", + }, + WallTimeMS: 500, + PeakRAMMB: 128, + SampleCount: 1, + LastSuccess: false, + }, + } + out := renderObservationTable(entries) + if !strings.Contains(out, "alpha") { + t.Fatalf("expected 'alpha' in output, got %q", out) + } + if !strings.Contains(out, "beta") { + t.Fatalf("expected 'beta' in output, got %q", out) + } + if !strings.Contains(out, "512") { + t.Fatalf("expected '512' in output, got %q", out) + } + if !strings.Contains(out, "256 MB") { + t.Fatalf("expected '256 MB' in output, got %q", out) + } + if !strings.Contains(out, "last failed") { + t.Fatalf("expected 'last failed' in output, got %q", out) + } +} + +func TestRenderObservationTableEmpty(t *testing.T) { + out := renderObservationTable([]models.ExecutionObservation{}) + if !strings.Contains(out, "No observations tracked") { + t.Fatalf("expected 'No observations tracked', got %q", out) + } +} + +func TestRenderObservationTableTruncation(t *testing.T) { + entries := make([]models.ExecutionObservation, 55) + for i := range entries { + entries[i] = models.ExecutionObservation{ + Scope: models.ObservationScope{ + Node: fmt.Sprintf("node-%02d", i), + }, + } + } + out := renderObservationTable(entries) + if !strings.Contains(out, "and 5 more observations") { + t.Fatalf("expected truncation message, got %q", out) + } +} diff --git a/cmd/axis/task.go b/cmd/axis/task.go index 6ac6d97..9b9c7bc 100644 --- a/cmd/axis/task.go +++ b/cmd/axis/task.go @@ -328,8 +328,14 @@ func taskRunCmd() *cobra.Command { return ExitCodeError{Code: ExitErrNoNodesFit, Message: "no suitable node found"} } if err != nil { + if s := formatObservationSummary(resp); s != "" { + fmt.Fprintln(w, s) + } return err } + if s := formatObservationSummary(resp); s != "" { + fmt.Fprintln(w, s) + } return nil }, } @@ -428,6 +434,23 @@ func printBlockedResult(w io.Writer, resp execution.GuardedExecutionResult) { fmt.Fprintln(w, "Nothing was executed. Fix your request.") } +func formatObservationSummary(resp execution.GuardedExecutionResult) string { + if resp.WallTimeMS <= 0 { + return "" + } + parts := []string{fmt.Sprintf("wall %dms", resp.WallTimeMS)} + if resp.PeakRAMMB > 0 { + parts = append(parts, fmt.Sprintf("peak RAM %dMB", resp.PeakRAMMB)) + } + if resp.PeakVRAMMB > 0 { + parts = append(parts, fmt.Sprintf("peak VRAM %dMB", resp.PeakVRAMMB)) + } + if !resp.OK { + parts = append(parts, "unsuccessful") + } + return fmt.Sprintf("Recorded observation: %s", strings.Join(parts, ", ")) +} + // === NEW: axis task context — zero-risk token saver === func taskContextCmd() *cobra.Command { var cached bool diff --git a/docs/current-state.md b/docs/current-state.md index 5f236e5..6819ff1 100644 --- a/docs/current-state.md +++ b/docs/current-state.md @@ -13,8 +13,8 @@ Refresh this section with `./hack/refresh-current-state.sh`. - Refreshed: 2026-05-22 EDT - Repo version: `0.10.7` -- Latest published GitHub release: `v0.10.6` (2026-05-22T23:10:31Z) -- Release truth: repo version is ahead of the latest published release +- Latest published GitHub release: `v0.10.7` (2026-05-22T23:49:15Z) +- Release truth: repo version matches the latest published release ## Executive Summary @@ -149,7 +149,7 @@ Refresh this section with `./hack/refresh-current-state.sh`. - `coverage gate passed: internal/api 80.2% >= 50.0%` - `coverage gate passed: internal/mcp 88.9% >= 35.0%` - `coverage gate passed: internal/ui 94.0% >= 80.0%` - - `coverage gate passed: total 72.3% >= 45.0%` + - `coverage gate passed: total 72.5% >= 45.0%` ## Degraded-State Matrix diff --git a/internal/api/server.go b/internal/api/server.go index 7595ab2..aa4c0bd 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -74,6 +74,9 @@ type RunResponse struct { ExitCode int `json:"exit_code,omitempty"` SnapshotStatus models.SnapshotStatus `json:"snapshot_status,omitempty"` Summary *models.ClusterSummary `json:"summary,omitempty"` + PeakRAMMB int64 `json:"peak_ram_mb,omitempty"` + PeakVRAMMB int64 `json:"peak_vram_mb,omitempty"` + WallTimeMS int64 `json:"wall_time_ms,omitempty"` } type runnerContext struct { diff --git a/internal/execution/guarded.go b/internal/execution/guarded.go index d1699cb..899d66c 100644 --- a/internal/execution/guarded.go +++ b/internal/execution/guarded.go @@ -98,6 +98,9 @@ type GuardedExecutionResult struct { ExitCode int `json:"exit_code,omitempty"` SnapshotStatus models.SnapshotStatus `json:"snapshot_status,omitempty"` Summary *models.ClusterSummary `json:"summary,omitempty"` + PeakRAMMB int64 `json:"peak_ram_mb,omitempty"` + PeakVRAMMB int64 `json:"peak_vram_mb,omitempty"` + WallTimeMS int64 `json:"wall_time_ms,omitempty"` } type PreparedExecution struct { @@ -727,6 +730,8 @@ func runLocal( recordFailure(skillStore, req.Description, resp.ExitCode) runtimeChanged = true recordExecutionOutcome(st, reqs, resp, runErr, elapsed, peakRAMMB) + resp.PeakRAMMB = peakRAMMB + resp.WallTimeMS = durationMilliseconds(elapsed) return resp, runErr } @@ -734,6 +739,8 @@ func runLocal( runtimeChanged = true recordExecutionOutcome(st, reqs, resp, nil, elapsed, peakRAMMB) resp.OK = true + resp.PeakRAMMB = peakRAMMB + resp.WallTimeMS = durationMilliseconds(elapsed) return resp, nil } @@ -822,6 +829,7 @@ func runRemote( recordFailure(skillStore, req.Description, resp.ExitCode) runtimeChanged = true recordExecutionOutcome(st, reqs, resp, runErr, elapsed, 0) + resp.WallTimeMS = durationMilliseconds(elapsed) return resp, runErr } @@ -829,6 +837,7 @@ func runRemote( runtimeChanged = true recordExecutionOutcome(st, reqs, resp, nil, elapsed, 0) resp.OK = true + resp.WallTimeMS = durationMilliseconds(elapsed) return resp, nil }